Reduce LFS usage in CI (#1081)

* required deps cleanup * required deps cleanup * fixes * fixes * fixes * LFS removed from contrib tests in CI * LFS removed from contrib tests in CI
ag2ai · Feb 21, 2025 · deef0f8 · deef0f8
1 parent f404086
commit deef0f8
Show file tree

Hide file tree

Showing 15 changed files with 38 additions and 51 deletions.
diff --git a/.github/workflows/contrib-test.yml b/.github/workflows/contrib-test.yml
@@ -402,8 +402,6 @@ jobs:
             python-version: "3.9"
     steps:
       - uses: actions/checkout@v4
-        with:
-          lfs: true
       - uses: astral-sh/setup-uv@v5
         with:
           version: "latest"
@@ -520,8 +518,6 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
-        with:
-          lfs: true
       - uses: astral-sh/setup-uv@v5
         with:
           version: "latest"
@@ -567,8 +563,6 @@ jobs:
             python-version: "3.9"
     steps:
       - uses: actions/checkout@v4
-        with:
-          lfs: true
       - uses: astral-sh/setup-uv@v5
         with:
           version: "latest"
@@ -612,8 +606,6 @@ jobs:
             python-version: "3.9"
     steps:
       - uses: actions/checkout@v4
-        with:
-          lfs: true
       - uses: astral-sh/setup-uv@v5
         with:
           version: "latest"
@@ -657,8 +649,6 @@ jobs:
             python-version: "3.9"
     steps:
       - uses: actions/checkout@v4
-        with:
-          lfs: true
       - uses: astral-sh/setup-uv@v5
         with:
           version: "latest"
@@ -702,8 +692,6 @@ jobs:
             python-version: "3.9"
     steps:
       - uses: actions/checkout@v4
-        with:
-          lfs: true
       - uses: astral-sh/setup-uv@v5
         with:
           version: "latest"
@@ -743,8 +731,6 @@ jobs:
         python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
     steps:
       - uses: actions/checkout@v4
-        with:
-          lfs: true
       - uses: astral-sh/setup-uv@v5
         with:
           version: "latest"
@@ -788,8 +774,6 @@ jobs:
             python-version: "3.9"
     steps:
       - uses: actions/checkout@v4
-        with:
-          lfs: true
       - uses: astral-sh/setup-uv@v5
         with:
           version: "latest"
@@ -833,8 +817,6 @@ jobs:
             python-version: "3.9"
     steps:
       - uses: actions/checkout@v4
-        with:
-          lfs: true
       - uses: astral-sh/setup-uv@v5
         with:
           version: "latest"
@@ -878,8 +860,6 @@ jobs:
             python-version: "3.9"
     steps:
       - uses: actions/checkout@v4
-        with:
-          lfs: true
       - uses: astral-sh/setup-uv@v5
         with:
           version: "latest"
@@ -923,8 +903,6 @@ jobs:
               python-version: "3.9"
       steps:
         - uses: actions/checkout@v4
-          with:
-            lfs: true
         - uses: astral-sh/setup-uv@v5
           with:
             version: "latest"

diff --git a/.github/workflows/test-with-optional-deps.yml b/.github/workflows/test-with-optional-deps.yml
@@ -78,6 +78,7 @@ jobs:
           - "interop-crewai"
           - "interop-langchain"
           - "interop-pydantic-ai"
+          - "websockets"
         os: [ubuntu-latest, macos-latest, windows-latest]
         # Run tests only for py versions 3.9, 3.10, 3.13
         python-version: ["3.9", "3.10", "3.13"]

diff --git a/.github/workflows/type-check.yml b/.github/workflows/type-check.yml
@@ -47,6 +47,7 @@ jobs:
           - "interop-crewai"
           - "interop-langchain"
           - "interop-pydantic-ai"
+          - "websockets"
         exclude:
           # pdoc3 is failing in python 3.9 for some reason, we build docs with Python 3.10
           - python-version: "3.9"

diff --git a/.secrets.baseline b/.secrets.baseline
@@ -133,7 +133,7 @@
         "filename": "autogen/agentchat/contrib/vectordb/pgvectordb.py",
         "hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8",
         "is_verified": false,
-        "line_number": 580,
+        "line_number": 579,
         "is_secret": false
       }
     ],
@@ -803,7 +803,7 @@
         "filename": "test/agentchat/contrib/test_img_utils.py",
         "hashed_secret": "dcce26510eb892e4c25cd8fabf1778261002ae1e",
         "is_verified": false,
-        "line_number": 40,
+        "line_number": 41,
         "is_secret": false
       }
     ],
@@ -1193,7 +1193,7 @@
         "filename": "test/website/test_process_notebooks.py",
         "hashed_secret": "f5d04c6f567b20a06d5dddf4f8da1b1d6302b84a",
         "is_verified": false,
-        "line_number": 322,
+        "line_number": 323,
         "is_secret": false
       }
     ],
@@ -1596,5 +1596,5 @@
       }
     ]
   },
-  "generated_at": "2025-02-20T13:06:27Z"
+  "generated_at": "2025-02-21T07:43:00Z"
 }
diff --git a/autogen/_website/process_notebooks.py b/autogen/_website/process_notebooks.py
@@ -534,6 +534,7 @@ def replace_local_path(match: re.Match) -> str:
 
 
 # rendered_notebook is the final mdx file
+@require_optional_import("yaml", "docs")
 def post_process_mdx(
     rendered_mdx: Path,
     source_notebooks: Path,
@@ -922,6 +923,7 @@ def separate_front_matter_and_content(file_path: Path) -> tuple[str, str]:
     return "", content
 
 
+@require_optional_import("yaml", "docs")
 def _get_authors_info(authors_yml: Path) -> dict[str, dict[str, str]]:
     try:
         all_authors_info = yaml.safe_load(authors_yml.read_text(encoding="utf-8"))
@@ -932,6 +934,7 @@ def _get_authors_info(authors_yml: Path) -> dict[str, dict[str, str]]:
     return all_authors_info
 
 
+@require_optional_import("yaml", "docs")
 def _add_authors_and_social_preview(
     website_build_dir: Path, target_dir: Path, all_authors_info: dict[str, dict[str, str]]
 ) -> None:

diff --git a/autogen/agentchat/contrib/vectordb/couchbase.py b/autogen/agentchat/contrib/vectordb/couchbase.py
@@ -10,13 +10,12 @@
 from datetime import timedelta
 from typing import Any, Callable, Literal, Optional
 
-import numpy as np
-
 from ....import_utils import optional_import_block, require_optional_import
 from .base import Document, ItemID, QueryResults, VectorDB
 from .utils import get_logger
 
 with optional_import_block():
+    import numpy as np
     from couchbase import search
     from couchbase.auth import PasswordAuthenticator
     from couchbase.cluster import Cluster, ClusterOptions
@@ -26,6 +25,7 @@
     from couchbase.vector_search import VectorQuery, VectorSearch
     from sentence_transformers import SentenceTransformer
 
+
 logger = get_logger(__name__)
 
 DEFAULT_BATCH_SIZE = 1000

diff --git a/autogen/agentchat/contrib/vectordb/mongodb.py b/autogen/agentchat/contrib/vectordb/mongodb.py
@@ -8,13 +8,12 @@
 from time import monotonic, sleep
 from typing import Any, Callable, Iterable, Literal, Mapping, Optional, Union
 
-import numpy as np
-
 from ....import_utils import optional_import_block, require_optional_import
 from .base import Document, ItemID, QueryResults, VectorDB
 from .utils import get_logger
 
 with optional_import_block():
+    import numpy as np
     from pymongo import MongoClient, UpdateOne, errors
     from pymongo.collection import Collection
     from pymongo.driver_info import DriverInfo
@@ -33,7 +32,7 @@ def with_id_rename(docs: Iterable) -> list[dict[str, Any]]:
     return [{**{k: v for k, v in d.items() if k != "_id"}, "id": d["_id"]} for d in docs]
 
 
-@require_optional_import(["pymongo", "sentence_transformers"], "retrievechat-mongodb")
+@require_optional_import(["pymongo", "sentence_transformers", "numpy"], "retrievechat-mongodb")
 class MongoDBAtlasVectorDB(VectorDB):
     """A Collection object for MongoDB."""
 

diff --git a/autogen/agentchat/contrib/vectordb/pgvectordb.py b/autogen/agentchat/contrib/vectordb/pgvectordb.py
@@ -9,13 +9,12 @@
 import urllib.parse
 from typing import Any, Callable, Optional, Union
 
-import numpy as np
-
 from ....import_utils import optional_import_block, require_optional_import
 from .base import Document, ItemID, QueryResults, VectorDB
 from .utils import get_logger
 
 with optional_import_block():
+    import numpy as np
     import pgvector  # noqa: F401
     import psycopg
     from pgvector.psycopg import register_vector
@@ -25,7 +24,7 @@
 logger = get_logger(__name__)
 
 
-@require_optional_import(["psycopg", "sentence_transformers"], "retrievechat-pgvector")
+@require_optional_import(["psycopg", "sentence_transformers", "numpy"], "retrievechat-pgvector")
 class Collection:
     """A Collection object for PGVector.
 

diff --git a/autogen/agentchat/realtime/experimental/clients/gemini/client.py b/autogen/agentchat/realtime/experimental/clients/gemini/client.py
@@ -9,16 +9,19 @@
 from typing import TYPE_CHECKING, Any, Callable, Optional
 
 from ......doc_utils import export_module
+from ......import_utils import optional_import_block, require_optional_import
 from ...realtime_events import AudioDelta, FunctionCall, RealtimeEvent, SessionCreated
 from ..realtime_client import RealtimeClientBase, Role, register_realtime_client
 
+with optional_import_block():
+    from websockets.asyncio.client import connect
+
+
 if TYPE_CHECKING:
     from websockets.asyncio.client import ClientConnection
 
     from ..realtime_client import RealtimeClientProtocol
 
-from websockets.asyncio.client import connect
-
 __all__ = ["GeminiRealtimeClient"]
 
 global_logger = getLogger(__name__)
@@ -29,6 +32,7 @@
 
 
 @register_realtime_client()
+@require_optional_import("websockets", "gemini")
 @export_module("autogen.agentchat.realtime.experimental.clients")
 class GeminiRealtimeClient(RealtimeClientBase):
     """(Experimental) Client for Gemini Realtime API."""

diff --git a/autogen/io/websockets.py b/autogen/io/websockets.py
@@ -88,9 +88,6 @@ def __init__(self, websocket: ServerConnection) -> None:
 
         Args:
             websocket (ServerConnection): The websocket server.
-
-        Raises:
-            ImportError: If the websockets module is not available.
         """
         self._websocket = websocket
 

diff --git a/autogen/oai/completion.py b/autogen/oai/completion.py
@@ -13,8 +13,6 @@
 from time import sleep
 from typing import Any, Callable, Optional, Union
 
-import numpy as np
-
 from ..doc_utils import export_module
 
 # Adding a NullHandler to silence FLAML log warning during
@@ -26,6 +24,7 @@
 from ..import_utils import optional_import_block, require_optional_import
 
 with optional_import_block() as result:
+    import numpy as np
     from flaml import BlendSearch, tune
     from flaml.tune.space import is_constant
 
@@ -362,6 +361,7 @@ def _get_params_for_create(cls, config: dict) -> dict:
         return params
 
     @classmethod
+    @require_optional_import("numpy", "flaml")
     def _eval(cls, config: dict[str, Any], prune: bool = True, eval_only: bool = False):
         """Evaluate the given config as the hyperparameter setting for the openai api call.
 
@@ -919,6 +919,7 @@ def _construct_params(cls, context, config, prompt=None, messages=None, allow_fo
         return params
 
     @classmethod
+    @require_optional_import("numpy", "flaml")
     def test(
         cls,
         data,

diff --git a/autogen/oai/openai_utils.py b/autogen/oai/openai_utils.py
@@ -740,10 +740,7 @@ def retrieve_assistants_by_name(client: OpenAI, name: str) -> list[Assistant]:
 def detect_gpt_assistant_api_version() -> str:
     """Detect the openai assistant API version"""
     oai_version = importlib.metadata.version("openai")
-    if parse(oai_version) < parse("1.21"):
-        return "v1"
-    else:
-        return "v2"
+    return "v1" if parse(oai_version) < parse("1.21") else "v2"
 
 
 def create_gpt_vector_store(client: OpenAI, name: str, fild_ids: list[str]) -> Any:

diff --git a/pyproject.toml b/pyproject.toml
@@ -56,14 +56,12 @@ dependencies = [
     "termcolor",
     "python-dotenv",
     "tiktoken",
-    "numpy",
     # Disallowing 2.6.0 can be removed when this is fixed https://github.com/pydantic/pydantic/issues/8705
     "pydantic>=2.6.1,<3",
     "docker",
     "packaging",
-    "websockets>=14,<15",
     "asyncer==0.0.8",
-    "fast_depends>=2.4.12,<3",
+    "fast-depends>=2.4.12,<3",
 ]
 
 [project.optional-dependencies]
@@ -104,6 +102,7 @@ retrievechat-pgvector = [
 retrievechat-mongodb = [
     "pyautogen[retrievechat]",
     "pymongo>=4.0.0",
+    "numpy",
 ]
 
 retrievechat-qdrant = [
@@ -115,6 +114,7 @@ retrievechat-qdrant = [
 retrievechat-couchbase = [
     "pyautogen[retrievechat]",
     "couchbase>=4.3.0",
+    "numpy",
 ]
 
 graph-rag-falkor-db = [
@@ -180,8 +180,9 @@ gemini = [
     "google-auth",
     "pillow",
     "jsonschema",
-    "jsonref>=1,<2"
+    "jsonref>=1,<2",
 ]
+
 together = ["together>=1.2"]
 websurfer = ["beautifulsoup4", "markdownify", "pdfminer.six", "pathvalidate"]
 redis = ["redis"]
@@ -297,6 +298,7 @@ markers = [
     "interop",
     "browser_use",
     "crawl4ai",
+    "websockets",
 ]
 
 [tool.black]
-Original file line number
+Diff line change
@@ Expand Up / @@ -88,9 +88,6 @@ def __init__(self, websocket: ServerConnection) -> None: @@
             Args:
                 websocket (ServerConnection): The websocket server.
-            Raises:
-                ImportError: If the websockets module is not available.
             """
             self._websocket = websocket
@@ Expand Down @@