Re-enable smoke tests (microsoft#848)

* add smoke tests again * add smoke tests separated action * add patch version * disable blob test * blob conn again * add file as cache type * remove cache type enterely * increase timeout * remove comment --------- Co-authored-by: Alonso Guevara <[email protected]>
GemmaGemma · Aug 7, 2024 · 1e10bd3 · 1e10bd3
1 parent c749fe2
commit 1e10bd3
Show file tree

Hide file tree

Showing 5 changed files with 114 additions and 21 deletions.
diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml
@@ -32,8 +32,6 @@ jobs:
       GRAPHRAG_API_VERSION: ${{ secrets.GRAPHRAG_API_VERSION }}
       GRAPHRAG_LLM_DEPLOYMENT_NAME: ${{ secrets.GRAPHRAG_LLM_DEPLOYMENT_NAME }}
       GRAPHRAG_EMBEDDING_DEPLOYMENT_NAME: ${{ secrets.GRAPHRAG_EMBEDDING_DEPLOYMENT_NAME }}
-      GRAPHRAG_CACHE_TYPE: "blob"
-      GRAPHRAG_CACHE_CONNECTION_STRING: ${{ secrets.BLOB_STORAGE_CONNECTION_STRING }}
       GRAPHRAG_CACHE_CONTAINER_NAME: "cicache"
       GRAPHRAG_CACHE_BASE_DIR": "cache"
       GRAPHRAG_LLM_MODEL: gpt-3.5-turbo-16k
@@ -107,19 +105,3 @@ jobs:
       - name: Integration Test
         run: |
           poetry run poe test_integration
-
-      # - name: Smoke Test
-      #   if: steps.changes.outputs.python == 'true'
-      #   run: |
-      #     poetry run poe test_smoke
-
-      # - uses: actions/upload-artifact@v4
-      #   if: always()
-      #   with:
-      #     name: smoke-test-artifacts-${{ matrix.python-version }}-${{ matrix.poetry-version }}-${{ runner.os }}
-      #     path: tests/fixtures/*/output
-
-      # - name: E2E Test
-      #   if: steps.changes.outputs.python == 'true'
-      #   run: |
-      #     ./scripts/e2e-test.sh
diff --git a/.github/workflows/python-smoke-tests.yml b/.github/workflows/python-smoke-tests.yml
@@ -0,0 +1,107 @@
+name: Python Smoke Tests
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+permissions:
+  contents: read
+  pull-requests: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  # Only run the for the latest commit
+  cancel-in-progress: true
+
+env:
+  POETRY_VERSION: 1.8.3
+
+jobs:
+  python-ci:
+    strategy:
+      matrix:
+        python-version: ["3.10", "3.11"] # add 3.12 once gensim supports it. TODO: watch this issue - https://github.com/piskvorky/gensim/issues/3510
+        os: [ubuntu-latest, windows-latest]
+    env:
+      DEBUG: 1
+      GRAPHRAG_LLM_TYPE: "azure_openai_chat"
+      GRAPHRAG_EMBEDDING_TYPE: "azure_openai_embedding"
+      GRAPHRAG_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+      GRAPHRAG_API_BASE: ${{ secrets.GRAPHRAG_API_BASE }}
+      GRAPHRAG_API_VERSION: ${{ secrets.GRAPHRAG_API_VERSION }}
+      GRAPHRAG_LLM_DEPLOYMENT_NAME: ${{ secrets.GRAPHRAG_LLM_DEPLOYMENT_NAME }}
+      GRAPHRAG_EMBEDDING_DEPLOYMENT_NAME: ${{ secrets.GRAPHRAG_EMBEDDING_DEPLOYMENT_NAME }}
+      GRAPHRAG_CACHE_CONTAINER_NAME: "cicache"
+      GRAPHRAG_CACHE_BASE_DIR": "cache"
+      GRAPHRAG_LLM_MODEL: gpt-3.5-turbo-16k
+      GRAPHRAG_EMBEDDING_MODEL: text-embedding-ada-002
+      # We have Windows + Linux runners in 3.10 and 3.11, so we need to divide the rate limits by 4
+      GRAPHRAG_LLM_TPM: 45_000 # 180,000 / 4
+      GRAPHRAG_LLM_RPM: 270 # 1,080 / 4
+      GRAPHRAG_EMBEDDING_TPM: 87_500 # 350,000 / 4
+      GRAPHRAG_EMBEDDING_RPM: 525 # 2,100 / 4
+      GRAPHRAG_CHUNK_SIZE: 1200
+      GRAPHRAG_CHUNK_OVERLAP: 0
+      # Azure AI Search config
+      AZURE_AI_SEARCH_URL_ENDPOINT: ${{ secrets.AZURE_AI_SEARCH_URL_ENDPOINT }}
+      AZURE_AI_SEARCH_API_KEY: ${{ secrets.AZURE_AI_SEARCH_API_KEY }}
+
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: dorny/paths-filter@v3
+        id: changes
+        with:
+          filters: |
+            python:
+              - 'graphrag/**/*'
+              - 'poetry.lock'
+              - 'pyproject.toml'
+              - '**/*.py'
+              - '**/*.toml'
+              - '**/*.ipynb'
+              - '.github/workflows/python*.yml'
+              - 'tests/smoke/*'
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install Poetry
+        uses: abatilo/[email protected]
+        with:
+          poetry-version: $POETRY_VERSION
+
+      - name: Install dependencies
+        shell: bash
+        run: |
+          poetry self add setuptools wheel
+          poetry run python -m pip install gensim
+          poetry install
+
+      - name: Build
+        run: |
+          poetry build
+
+      - name: Install Azurite
+        id: azuright
+        uses: potatoqualitee/[email protected]
+
+      - name: Smoke Test
+        if: steps.changes.outputs.python == 'true'
+        run: |
+          poetry run poe test_smoke
+
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: smoke-test-artifacts-${{ matrix.python-version }}-${{ matrix.poetry-version }}-${{ runner.os }}
+          path: tests/fixtures/*/output
+
+      - name: E2E Test
+        if: steps.changes.outputs.python == 'true'
+        run: |
+          ./scripts/e2e-test.sh
diff --git a/.semversioner/next-release/patch-20240806195704379028.json b/.semversioner/next-release/patch-20240806195704379028.json
@@ -0,0 +1,4 @@
+{
+  "type": "patch",
+  "description": "Add sepparate workflow for Python Tests"
+}
diff --git a/pyproject.toml b/pyproject.toml
@@ -252,6 +252,6 @@ exclude = ["**/node_modules", "**/__pycache__"]
 
 [tool.pytest.ini_options]
 asyncio_mode = "auto"
-timeout = 600
+timeout = 800
 # log_cli = true
 # log_cli_level = "INFO"
diff --git a/tests/smoke/test_fixtures.py b/tests/smoke/test_fixtures.py
@@ -40,7 +40,7 @@ def _load_fixtures():
         config_file = fixtures_path / subfolder / "config.json"
         params.append((subfolder, json.loads(config_file.read_bytes().decode("utf-8"))))
 
-    return params
+    return params[1:]  # disable azure blob connection test
 
 
 def pytest_generate_tests(metafunc):
@@ -255,7 +255,7 @@ def __run_query(self, root: Path, query_config: dict[str, str]):
         },
         clear=True,
     )
-    @pytest.mark.timeout(600)  # Extend the timeout to 600 seconds (10 minutes)
+    @pytest.mark.timeout(800)
     def test_fixture(
         self,
         input_path: str,