E2E Testing (Helicone#730)

kufeng76 · Aug 22, 2023 · 55923e9 · 55923e9
1 parent e1dc5e5
commit 55923e9
Show file tree

Hide file tree

Showing 9 changed files with 355 additions and 9 deletions.
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -0,0 +1,98 @@
+name: e2e tests
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  db-type-checks:
+    name: DB type checks
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+      - uses: supabase/setup-cli@v1
+        with:
+          version: 1.86.2
+      - run: supabase db start
+      - run: supabase db lint
+
+      - name: Verify generated types match Postgres schema for web (ignores date_count ordering)
+        run: |
+          supabase gen types typescript --local > web/supabase/database.types.ts
+          if ! git diff --ignore-space-at-eol web/supabase/database.types.ts | grep -v "date_count" | grep -q "."; then
+              echo "Detected uncommitted changes after build in web/supabase/database.types.ts. See status below:"
+              git diff
+              exit 1
+          fi
+
+      - name: Verify generated types match Postgres schema for worker (ignores date_count ordering)
+        run: |
+          supabase gen types typescript --local > worker/supabase/database.types.ts
+          if ! git diff --ignore-space-at-eol worker/supabase/database.types.ts | grep -v "date_count" | grep -q "."; then
+              echo "Detected uncommitted changes after build in worker/supabase/database.types.ts. See status below:"
+              git diff
+              exit 1
+          fi
+
+  e2e-test-python:
+    name: Python E2E tests
+    runs-on: ubuntu-latest
+    services:
+      clickhouse:
+        image: yandex/clickhouse-server
+        ports:
+          - 8123:8123
+    steps:
+      - uses: actions/checkout@v3
+      - uses: supabase/[email protected]
+      - name: Use Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version: "18"
+      - name: Cache npm packages
+        uses: actions/cache@v3
+        with:
+          path: |
+            ./worker/node_modules
+          key: ${{ runner.os }}-node-${{ hashFiles('./worker/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-node-
+      - name: Install node dependencies
+        run: |
+          cd worker
+          npm install -g wrangler
+          npm install
+      - name: Start proxy worker
+        run: |
+          cd worker
+          wrangler dev --port 8787 --var WORKER_TYPE:OPENAI_PROXY &
+        env:
+          CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
+      - name: Start async worker
+        run: |
+          cd worker
+          wrangler dev --port 8788 --var WORKER_TYPE:HELICONE_API &
+        env:
+          CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
+      - name: Start Supabase DB & API
+        run: supabase start db,api
+      - name: Install python dependencies
+        run: |
+          pip install requests pytest psycopg2 python-dotenv helicone
+      - name: Run integration tests
+        working-directory: tests
+        env:
+          HELICONE_PROXY_URL: "http://127.0.0.1:8787/v1"
+          HELICONE_ASYNC_URL: "http://127.0.0.1:8788"
+          SUPABASE_KEY: "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZS1kZW1vIiwicm9sZSI6InNlcnZpY2Vfcm9sZSIsImV4cCI6MTk4MzgxMjk5Nn0.EGIM96RAZx35lJzdJsyH-qQwv8Hdp7fsn3W0YpN81IU"
+          SUPABASE_URL: "http://localhost:54321"
+          HELICONE_API_KEY: "sk-helicone-aizk36y-5yue2my-qmy5tza-n7x3aqa"
+          OPENAI_API_KEY: ${{ secrets.CI_OPENAI_API_KEY }}
+          OPENAI_ORG: ${{ secrets.CI_OPENAI_ORG }}
+        run: |
+          pytest python_integration_tests.py -s
diff --git a/supabase/database.types.ts b/supabase/database.types.ts
@@ -37,6 +37,39 @@ export interface Database {
           }
         ]
       }
+      contact_submissions: {
+        Row: {
+          company_description: string | null
+          company_name: string | null
+          created_at: string | null
+          email_address: string | null
+          first_name: string | null
+          id: number
+          last_name: string | null
+          tag: string | null
+        }
+        Insert: {
+          company_description?: string | null
+          company_name?: string | null
+          created_at?: string | null
+          email_address?: string | null
+          first_name?: string | null
+          id?: number
+          last_name?: string | null
+          tag?: string | null
+        }
+        Update: {
+          company_description?: string | null
+          company_name?: string | null
+          created_at?: string | null
+          email_address?: string | null
+          first_name?: string | null
+          id?: number
+          last_name?: string | null
+          tag?: string | null
+        }
+        Relationships: []
+      }
       feature_flags: {
         Row: {
           created_at: string | null

diff --git a/supabase/seed.sql b/supabase/seed.sql
@@ -0,0 +1,24 @@
+INSERT INTO auth.users (instance_id,id,aud,"role",email,encrypted_password,email_confirmed_at,last_sign_in_at,raw_app_meta_data,raw_user_meta_data,is_super_admin,created_at,updated_at,phone,phone_confirmed_at,confirmation_token,email_change,email_change_token_new,recovery_token) VALUES
+	('00000000-0000-0000-0000-000000000000'::uuid,'f76629c5-a070-4bbc-9918-64beaea48848'::uuid,'authenticated','authenticated','[email protected]','$2a$10$PznXR5VSgzjnAp7T/X7PCu6vtlgzdFt1zIr41IqP0CmVHQtShiXxS','2022-02-11 21:02:04.547','2022-02-11 22:53:12.520','{"provider": "email", "providers": ["email"]}','{}',FALSE,'2022-02-11 21:02:04.542','2022-02-11 21:02:04.542',NULL,NULL,'','','',''),
+	('00000000-0000-0000-0000-000000000000'::uuid,'d9064bb5-1501-4ec9-bfee-21ab74d645b8'::uuid,'authenticated','authenticated','[email protected]','$2a$10$mOJUAphJbZR4CdM38.bgOeyySurPeFHoH/T1s7HuGdpRb7JgatF7K','2022-02-12 07:40:23.616','2022-02-12 07:40:23.621','{"provider": "email", "providers": ["email"]}','{}',FALSE,'2022-02-12 07:40:23.612','2022-02-12 07:40:23.613',NULL,NULL,'','','','')
+ON CONFLICT (id) DO NOTHING;
+
+INSERT INTO auth.identities (id,user_id,identity_data,provider,last_sign_in_at,created_at,updated_at) VALUES
+	('f76629c5-a070-4bbc-9918-64beaea48848','f76629c5-a070-4bbc-9918-64beaea48848'::uuid,'{"sub": "f76629c5-a070-4bbc-9918-64beaea48848"}','email','2022-02-11 21:02:04.545','2022-02-11 21:02:04.545','2022-02-11 21:02:04.545'),
+	('d9064bb5-1501-4ec9-bfee-21ab74d645b8','d9064bb5-1501-4ec9-bfee-21ab74d645b8'::uuid,'{"sub": "d9064bb5-1501-4ec9-bfee-21ab74d645b8"}','email','2022-02-12 07:40:23.615','2022-02-12 07:40:23.615','2022-02-12 07:40:23.615')
+ON CONFLICT (id, provider) DO NOTHING;
+
+INSERT INTO public.organization (id, name, owner)
+VALUES
+    ('83635a30-5ba6-41a8-8cc6-fb7df941b24a', 'Organization for Test', 'f76629c5-a070-4bbc-9918-64beaea48848'),
+    ('a75d76e3-02e7-4d02-8a2b-c65ed27c69b2', 'Organization for Demo', 'd9064bb5-1501-4ec9-bfee-21ab74d645b8');
+
+-- sk-helicone-aizk36y-5yue2my-qmy5tza-n7x3aqa
+INSERT INTO public.helicone_api_keys (api_key_hash, api_key_name, user_id, organization_id, soft_delete) 
+VALUES 
+('014c02f17208d4f521b0bad39a1d0174e015e86f925cbbcc25108ffea74fd7f2', 'Test', 'f76629c5-a070-4bbc-9918-64beaea48848', '83635a30-5ba6-41a8-8cc6-fb7df941b24a', FALSE);
+
+-- sk-helicone-zk6xu4a-kluegtq-sbljk7q-drnixzi
+INSERT INTO public.helicone_api_keys (api_key_hash, api_key_name, user_id, organization_id, soft_delete) 
+VALUES 
+('4a6ced8c61492670cfd56f31e6e4c40e7cf8c6b88b7ab0a70efc5f86c136b9c3', 'Demo', 'd9064bb5-1501-4ec9-bfee-21ab74d645b8', 'a75d76e3-02e7-4d02-8a2b-c65ed27c69b2', FALSE);
diff --git a/tests/python_integration_tests.py b/tests/python_integration_tests.py
@@ -0,0 +1,130 @@
+import requests
+import pytest
+import os
+from dotenv import load_dotenv
+import psycopg2
+from psycopg2.extras import DictCursor
+import uuid
+import time
+from helicone.openai_async import openai, Meta
+from helicone.globals import helicone_global
+
+load_dotenv()
+
+helicone_proxy_url = os.environ["HELICONE_PROXY_URL"]
+helicone_async_url = os.environ["HELICONE_ASYNC_URL"]
+openai_api_key = os.environ["OPENAI_API_KEY"]
+openai_org_id = os.environ["OPENAI_ORG"]
+helicone_api_key = os.environ["HELICONE_API_KEY"]
+supabase_key = os.environ["SUPABASE_KEY"]
+supabase_url = os.environ["SUPABASE_URL"]
+
+def connect_to_db():
+    return psycopg2.connect(
+        dbname="postgres",
+        user="postgres",
+        password="postgres",
+        host="localhost",
+        port="54322"
+    )
+
+def fetch_from_db(query, params=None):
+    conn = connect_to_db()
+    cur = conn.cursor(cursor_factory=DictCursor)
+    cur.execute(query, params)
+    results = cur.fetchall()
+    cur.close()
+    conn.close()
+    return results
+
+def fetch(base_url, endpoint, method="GET", json=None, headers=None):
+    url = f"{base_url}/{endpoint}"
+    response = requests.request(method, url, json=json, headers=headers)
+    response.raise_for_status()
+    return response.json()
+
+def test_openai_proxy():
+    print("\n---------Running test_proxy---------")
+    requestId = str(uuid.uuid4())
+    print("Request ID: " + requestId + "")
+    message_content = test_openai_proxy.__name__ + " - " + requestId
+    messages = [
+        {
+            "role": "user",
+            "content": message_content
+        }
+    ]
+    data = {
+        "model": "gpt-3.5-turbo",
+        "messages": messages,
+        "max_tokens": 10
+    }
+    headers = {
+        "Authorization": f"Bearer {openai_api_key}",
+        "Helicone-Auth": f"Bearer {helicone_api_key}",
+        "Helicone-Property-RequestId": requestId,
+        "OpenAI-Organization": openai_org_id
+    }
+
+    response = fetch(helicone_proxy_url, "chat/completions", method="POST", json=data, headers=headers)
+    assert response, "Response from OpenAI API is empty"
+
+    time.sleep(1) # Helicone needs time to insert request into the database
+
+    query = "SELECT * FROM properties INNER JOIN request ON properties.request_id = request.id WHERE key = 'requestid' AND value = %s LIMIT 1"
+    request_data = fetch_from_db(query, (requestId,))
+    assert request_data, "Request data not found in the database for the given property request id"
+
+    latest_request = request_data[0]
+    assert message_content in latest_request["body"]["messages"][0]["content"], "Request not found in the database"
+
+    query = "SELECT * FROM response WHERE request = %s LIMIT 1"
+    response_data = fetch_from_db(query, (latest_request["id"],))
+    assert response_data, "Response data not found in the database for the given request ID"
+    print("passed")
+
+def test_openai_async():
+    print("---------Running test_openai_async---------")
+
+    # Set the API key for Helicone
+    helicone_global.api_key = helicone_api_key
+    helicone_global.base_url = helicone_async_url
+    openai.api_key = openai_api_key
+    openai.organization = openai_org_id
+
+    requestId = str(uuid.uuid4())
+    print("Request ID: " + requestId)
+    message_content = test_openai_async.__name__ + " - " + requestId
+    messages = [
+        {
+            "role": "system",
+            "content": message_content
+        }
+    ]
+
+    # Using the helicone package for async logging
+    response = openai.ChatCompletion.create(
+        model="gpt-3.5-turbo",
+        messages=messages,
+        max_tokens=10,
+        helicone_meta=Meta(
+            custom_properties={
+                "requestId": requestId
+            }
+        )
+    )
+    assert response, "Response from OpenAI API is empty"
+
+    time.sleep(1)  # Give some time for the async logging to complete
+
+    query = "SELECT * FROM properties INNER JOIN request ON properties.request_id = request.id WHERE key = 'requestid' AND value = %s LIMIT 1"
+    request_data = fetch_from_db(query, (requestId,))
+    assert request_data, "Request data not found in the database for the given property request id"
+
+    latest_request = request_data[0]
+    assert message_content in latest_request["body"]["messages"][0]["content"], "Request not found in the database"
+
+    query = "SELECT * FROM response WHERE request = %s LIMIT 1"
+    response_data = fetch_from_db(query, (latest_request["id"],))
+    assert response_data, "Response data not found in the database for the given request ID"
+    print("passed")
diff --git a/web/supabase/database.types.ts b/web/supabase/database.types.ts
@@ -37,6 +37,39 @@ export interface Database {
           }
         ]
       }
+      contact_submissions: {
+        Row: {
+          company_description: string | null
+          company_name: string | null
+          created_at: string | null
+          email_address: string | null
+          first_name: string | null
+          id: number
+          last_name: string | null
+          tag: string | null
+        }
+        Insert: {
+          company_description?: string | null
+          company_name?: string | null
+          created_at?: string | null
+          email_address?: string | null
+          first_name?: string | null
+          id?: number
+          last_name?: string | null
+          tag?: string | null
+        }
+        Update: {
+          company_description?: string | null
+          company_name?: string | null
+          created_at?: string | null
+          email_address?: string | null
+          first_name?: string | null
+          id?: number
+          last_name?: string | null
+          tag?: string | null
+        }
+        Relationships: []
+      }
       feature_flags: {
         Row: {
           created_at: string | null

diff --git a/worker/package.json b/worker/package.json
@@ -18,7 +18,8 @@
   "scripts": {
     "start": "wrangler dev",
     "deploy": "wrangler publish",
-    "lint": "eslint --ext .js,.jsx,.ts,.tsx . --fix",
+    "lint": "eslint --ext .js,.jsx,.ts,.tsx ./src --max-warnings=0",
+    "lint-fix": "eslint --ext .js,.jsx,.ts,.tsx ./src --fix",
     "test": "jest"
   },
   "dependencies": {

diff --git a/worker/src/routers/apiRouter.ts b/worker/src/routers/apiRouter.ts
@@ -24,10 +24,7 @@ export const getAPIRouter = () => {
     ) => {
       const asyncLogModel = await requestWrapper.getJson<AsyncLogModel>();
       //TODO Check to make sure auth is correct
-      if (
-        !requestWrapper.getAuthorization() ||
-        !requestWrapper.heliconeHeaders.heliconeAuth
-      ) {
+      if (!requestWrapper.getAuthorization()) {
         return new Response("Unauthorized", { status: 401 });
       }
 
@@ -77,10 +74,7 @@ export const getAPIRouter = () => {
     ) => {
       const asyncLogModel = await requestWrapper.getJson<AsyncLogModel>();
 
-      if (
-        !requestWrapper.getAuthorization() ||
-        !requestWrapper.heliconeHeaders.heliconeAuth
-      ) {
+      if (!requestWrapper.getAuthorization()) {
         return new Response("Unauthorized", { status: 401 });
       }