Add file based as option, move providers/requirements to hub (Josh-XT…

…#826)

* Moving imports to hub

* Add sleep

* Optional file vs db

* Move providers to hub, add api key to agixt

* Update docs

* Update SDK to 0.0.8

* Enable tests

* Update requirements with hub split

* Add AGIXT_API_KEY

* Fix lint

.env.example

@@ -1,10 +1,12 @@
+DB_CONNECTED=false
+AGIXT_API_KEY=
+AGIXT_HUB=AGiXT/hub
+AGIXT_URI=http://agixt:7437
+GITHUB_USER=
+GITHUB_TOKEN=
+UVICORN_WORKERS=6
 POSTGRES_SERVER=db
 POSTGRES_DB=postgres
 POSTGRES_PORT=5432
 POSTGRES_USER=postgres
-POSTGRES_PASSWORD=postgres
-UVICORN_WORKERS=6
-AGIXT_HUB=AGiXT/hub
-AGIXT_URI=http://agixt:7437
-GITHUB_USER=
-GITHUB_TOKEN=
+POSTGRES_PASSWORD=postgres

.github/workflows/operation-test-with-jupyter.yml

@@ -43,15 +43,6 @@ jobs:
     outputs:
       digest: ${{ steps.dockerBuild.outputs.digest }}
     services:
-      db:
-        image: postgres:latest
-        env:
-          POSTGRES_USER: postgres
-          POSTGRES_PASSWORD: postgres
-          POSTGRES_DB: postgres
-        ports:
-          - 5432:5432
-        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
       service-under-test:
         image: ${{ inputs.image }}
         ports:

.github/workflows/publish-docker.yml

@@ -27,11 +27,11 @@ jobs:
       platforms: linux/amd64,linux/arm64/v8
     secrets:
       DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
-#  test-agixt:
-#    uses: josh-xt/AGiXT/.github/workflows/operation-test-with-jupyter.yml@main
-#    with:
-#      notebook: tests/tests.ipynb
-#      image: ${{ needs.build-agixt.outputs.primary-image }}
-#      port: "7437"
-#      additional-python-dependencies: agixtsdk
-#    needs: build-agixt
+  test-agixt:
+    uses: josh-xt/AGiXT/.github/workflows/operation-test-with-jupyter.yml@main
+    with:
+      notebook: tests/tests.ipynb
+      image: ${{ needs.build-agixt.outputs.primary-image }}
+      port: "7437"
+      additional-python-dependencies: agixtsdk
+    needs: build-agixt

agixt/Extensions.py

@@ -4,140 +4,6 @@
 from inspect import signature, Parameter
 import logging
 import inspect
-from DBConnection import (
-    session,
-    Extension,
-    Command,
-    Argument,
-    Setting,
-)
-
-
-def import_extensions():
-    extensions_data = Extensions().get_extensions()
-    extension_settings_data = Extensions().get_extension_settings()
-
-    # Get the existing extensions and commands from the database
-    existing_extensions = session.query(Extension).all()
-    existing_commands = session.query(Command).all()
-
-    # Delete commands that don't exist in the extensions data
-    for command in existing_commands:
-        command_exists = any(
-            extension_data["extension_name"] == command.extension.name
-            and any(
-                cmd["friendly_name"] == command.name
-                for cmd in extension_data["commands"]
-            )
-            for extension_data in extensions_data
-        )
-        if not command_exists:
-            session.delete(command)
-
-    # Add new extensions and commands, and update existing commands
-    for extension_data in extensions_data:
-        extension_name = extension_data["extension_name"]
-        description = extension_data.get(
-            "description", ""
-        )  # Assign an empty string if description is missing
-
-        # Find the existing extension or create a new one
-        extension = next(
-            (ext for ext in existing_extensions if ext.name == extension_name),
-            None,
-        )
-        if extension is None:
-            extension = Extension(name=extension_name, description=description)
-            session.add(extension)
-            session.flush()
-            existing_extensions.append(extension)
-
-        commands = extension_data["commands"]
-
-        for command_data in commands:
-            if "friendly_name" not in command_data:
-                continue
-
-            command_name = command_data["friendly_name"]
-
-            # Find the existing command or create a new one
-            command = next(
-                (
-                    cmd
-                    for cmd in existing_commands
-                    if cmd.extension_id == extension.id and cmd.name == command_name
-                ),
-                None,
-            )
-            if command is None:
-                command = Command(
-                    extension_id=extension.id,
-                    name=command_name,
-                )
-                session.add(command)
-                session.flush()
-                existing_commands.append(command)
-                print(f"Adding command: {command_name}")
-
-            # Add command arguments
-            if "command_args" in command_data:
-                command_args = command_data["command_args"]
-                for arg, arg_type in command_args.items():
-                    if (
-                        session.query(Argument)
-                        .filter_by(command_id=command.id, name=arg)
-                        .first()
-                    ):
-                        continue
-                    command_arg = Argument(
-                        command_id=command.id,
-                        name=arg,
-                    )
-                    session.add(command_arg)
-                    print(f"Adding argument: {arg} to command: {command_name}")
-
-    session.commit()
-
-    # Add extensions to the database if they don't exist
-    for extension_name in extension_settings_data.keys():
-        extension = session.query(Extension).filter_by(name=extension_name).first()
-        if not extension:
-            extension = Extension(name=extension_name)
-            session.add(extension)
-            session.flush()
-            existing_extensions.append(extension)
-            print(f"Adding extension: {extension_name}")
-
-    session.commit()
-
-    # Migrate extension settings
-    for extension_name, settings in extension_settings_data.items():
-        extension = session.query(Extension).filter_by(name=extension_name).first()
-        if not extension:
-            print(f"Extension '{extension_name}' not found.")
-            continue
-
-        for setting_name, setting_value in settings.items():
-            setting = (
-                session.query(Setting)
-                .filter_by(extension_id=extension.id, name=setting_name)
-                .first()
-            )
-            if setting:
-                setting.value = setting_value
-                print(
-                    f"Updating setting: {setting_name} for extension: {extension_name}"
-                )
-            else:
-                setting = Setting(
-                    extension_id=extension.id,
-                    name=setting_name,
-                    value=setting_value,
-                )
-                session.add(setting)
-                print(f"Adding setting: {setting_name} for extension: {extension_name}")
-
-    session.commit()
 
 
 class Extensions:

agixt/Hub.py

@@ -1,20 +1,33 @@
 import io
 import os
+import time
 import shutil
 import requests
 import zipfile
 import hashlib
 from dotenv import load_dotenv
-from Extensions import import_extensions
-from Chain import import_chains
-from Agent import import_agents
-from History import import_conversations
-from Prompts import import_prompts
-from provider import import_providers
-
 
 load_dotenv()
 
+db_connected = bool(os.getenv("DB_CONNECTED", False))
+
+if db_connected:
+    from db.imports import (
+        import_extensions,
+        import_prompts,
+        import_providers,
+        import_agents,
+        import_chains,
+        import_conversations,
+    )
+else:
+    import_extensions = lambda: None
+    import_prompts = lambda: None
+    import_providers = lambda: None
+    import_agents = lambda: None
+    import_chains = lambda: None
+    import_conversations = lambda: None
+
 
 def import_agixt_hub():
     github_user = os.getenv("GITHUB_USER")
@@ -64,12 +77,14 @@ def import_agixt_hub():
         print(f"Updated AGiXT Hub from {github_repo}")
     except Exception as e:
         print(f"AGiXT Hub Import Error: {e}")
-    import_extensions()
-    import_prompts()
-    import_providers()
-    import_agents()
-    import_chains()
-    import_conversations()
+    time.sleep(5)
+    if db_connected:
+        import_extensions()
+        import_prompts()
+        import_providers()
+        import_agents()
+        import_chains()
+        import_conversations()
 
 
 if __name__ == "__main__":

agixt/Interactions.py

@@ -1,20 +1,34 @@
 import re
+import os
 import regex
 import json
 import time
 import logging
 from datetime import datetime
-from Agent import Agent
-from Prompts import Prompts
+from dotenv import load_dotenv
+
+load_dotenv()
+
+db_connected = bool(os.getenv("DB_CONNECTED", False))
+if db_connected:
+    from db.Agent import Agent
+    from db.Prompts import Prompts
+    from db.Chain import Chain
+    from db.History import log_interaction
+else:
+    from fb.Agent import Agent
+    from fb.Prompts import Prompts
+    from fb.Chain import Chain
+    from fb.History import log_interaction
+
 from Embedding import get_tokens
-from Chain import Chain
 from concurrent.futures import Future
 from agixtsdk import AGiXTSDK
-from History import log_interaction
 from Websearch import Websearch
 
+agixt_api_key = os.getenv("AGIXT_API_KEY")
 base_uri = "http://localhost:7437"
-ApiClient = AGiXTSDK(base_uri=base_uri)
+ApiClient = AGiXTSDK(base_uri=base_uri, api_key=agixt_api_key)
 chain = Chain()
 cp = Prompts()
 

agixt/provider/__init__.py → agixt/Providers.py

@@ -4,60 +4,6 @@
 import glob
 import os
 import inspect
-from DBConnection import (
-    session,
-    Provider as ProviderModel,
-    ProviderSetting,
-)
-
-
-def import_providers():
-    providers = get_providers()
-    existing_providers = session.query(ProviderModel).all()
-    existing_provider_names = [provider.name for provider in existing_providers]
-
-    for provider in existing_providers:
-        if provider.name not in providers:
-            session.delete(provider)
-
-    for provider_name in providers:
-        provider_options = get_provider_options(provider_name)
-
-        provider = (
-            session.query(ProviderModel).filter_by(name=provider_name).one_or_none()
-        )
-
-        if provider:
-            print(f"Updating provider: {provider_name}")
-        else:
-            provider = ProviderModel(name=provider_name)
-            session.add(provider)
-            existing_provider_names.append(provider_name)
-            print(f"Adding provider: {provider_name}")
-
-        for option_name, option_value in provider_options.items():
-            provider_setting = (
-                session.query(ProviderSetting)
-                .filter_by(provider_id=provider.id, name=option_name)
-                .one_or_none()
-            )
-            if provider_setting:
-                provider_setting.value = option_value
-                print(
-                    f"Updating provider setting: {option_name} for provider: {provider_name}"
-                )
-            else:
-                provider_setting = ProviderSetting(
-                    provider_id=provider.id,
-                    name=option_name,
-                    value=option_value,
-                )
-                session.add(provider_setting)
-                print(
-                    f"Adding provider setting: {option_name} for provider: {provider_name}"
-                )
-
-    session.commit()
 
 
 def get_providers():
@@ -82,7 +28,7 @@ def get_provider_options(provider_name):
     return options
 
 
-class Provider:
+class Providers:
     def __init__(self, name, **kwargs):
         try:
             module = importlib.import_module(f"provider.{name}")
@@ -114,4 +60,4 @@ def install_requirements(self):
 
 
 def __getattr__(name):
-    return Provider(name)
+    return Providers(name)

agixt/Websearch.py

@@ -9,9 +9,13 @@
 from bs4 import BeautifulSoup
 from agixtsdk import AGiXTSDK
 from typing import List
+import os
+from dotenv import load_dotenv
 
+load_dotenv()
+agixt_api_key = os.getenv("AGIXT_API_KEY")
 base_uri = "http://localhost:7437"
-ApiClient = AGiXTSDK(base_uri=base_uri)
+ApiClient = AGiXTSDK(base_uri=base_uri, api_key=agixt_api_key)
 
 
 class Websearch: