SQLite Release PR (chroma-core#808)

## Description of changes
Base PR to release sqlite refactor, which spans many stacked PRs.

Remaining
- [x] Merge this to main
- [x] Layered Persistent Index chroma-core#761 
- [x] Remove old impls (In chroma-core#781 )
- [x] Remove persist() API (In chroma-core#787)
- [x] Add telemetry to SegmentAPI, it was not included. (chroma-core#788)
- [x] New clients chroma-core#805 
- [x] locking and soak tests for thread-safety 
- [x] Migration tool
- [x] Fix chroma-core#739 
- [x] Fix metadata None vs empty
- [x] Fix persist directory (addressed in chroma-core#761)
- [x] Leave files open in chroma-core#761 (merge stacked PR)

Post Release
- [ ] Un xfail cross version tests once we cut the release
- [x] Documentation updates for new silent ADD failure.
- [x] Update all documentation for new API instantiation
- [x] Update all documentation for settings changes
- [ ] Update terraform deployment
- [ ] Update cloudformation deployment

---------

Co-authored-by: Luke VanderHart <[email protected]>
Co-authored-by: Jeffrey Huber <[email protected]>
Co-authored-by: Anton Troynikov <[email protected]>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: Sebastian Sosa <[email protected]>
Co-authored-by: Russell Pollari <[email protected]>
Co-authored-by: russell-pollari <[email protected]>

.github/workflows/chroma-client-integration-test.yml

@@ -7,7 +7,7 @@ on:
   pull_request:
     branches:
       - main
-      - '*'
+      - '**'
 
 jobs:
   test:

.github/workflows/chroma-integration-test.yml

@@ -8,7 +8,7 @@ on:
   pull_request:
     branches:
       - main
-      - '*'
+      - '**'
 
 jobs:
   test:

.github/workflows/chroma-test.yml

@@ -8,7 +8,7 @@ on:
   pull_request:
     branches:
       - main
-      - '*'
+      - '**'
 
 jobs:
   test:

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.10-slim-bullseye as builder
+FROM python:3.10-slim-bookworm as builder
 
 #RUN apt-get update -qq
 #RUN apt-get install python3.10 python3-pip -y --no-install-recommends && rm -rf /var/lib/apt/lists_/*
@@ -11,7 +11,7 @@ COPY ./requirements.txt requirements.txt
 
 RUN pip install --no-cache-dir --upgrade --prefix="/install" -r requirements.txt
 
-FROM python:3.10-slim-bullseye as final
+FROM python:3.10-slim-bookworm as final
 
 RUN mkdir /chroma
 WORKDIR /chroma

bin/docker_entrypoint.sh

@@ -1,5 +1,6 @@
 #!/bin/bash
 
 echo "Rebuilding hnsw to ensure architecture compatibility"
-pip install --force-reinstall --no-cache-dir hnswlib
+pip install --force-reinstall --no-cache-dir chroma-hnswlib
+export IS_PERSISTENT=1
 uvicorn chromadb.app:app --workers 1 --host 0.0.0.0 --port 8000 --proxy-headers --log-config log_config.yml

bin/integration-test

@@ -13,7 +13,7 @@ trap cleanup EXIT
 docker compose -f docker-compose.test.yml up --build -d
 
 export CHROMA_INTEGRATION_TEST_ONLY=1
-export CHROMA_API_IMPL=rest
+export CHROMA_API_IMPL=chromadb.api.fastapi.FastAPI
 export CHROMA_SERVER_HOST=localhost
 export CHROMA_SERVER_HTTP_PORT=8000
 

bin/templates/docker-compose.yml

@@ -9,37 +9,12 @@ services:
     image: ghcr.io/chroma-core/chroma:${ChromaVersion}
     volumes:
       - index_data:/index_data
-    environment:
-      - CHROMA_DB_IMPL=clickhouse
-      - CLICKHOUSE_HOST=clickhouse
-      - CLICKHOUSE_PORT=8123
     ports:
       - 8000:8000
-    depends_on:
-      - clickhouse
-    networks:
-      - net
-
-  clickhouse:
-    image: clickhouse/clickhouse-server:22.9-alpine
-    environment:
-      - ALLOW_EMPTY_PASSWORD=yes
-      - CLICKHOUSE_TCP_PORT=9000
-      - CLICKHOUSE_HTTP_PORT=8123
-    ports:
-      - '8123:8123'
-      - '9000:9000'
-    volumes:
-      - clickhouse_data:/bitnami/clickhouse
-      - backups:/backups
-      - ./config/backup_disk.xml:/etc/clickhouse-server/config.d/backup_disk.xml
-      - ./config/chroma_users.xml:/etc/clickhouse-server/users.d/chroma.xml
     networks:
       - net
 
 volumes:
-  clickhouse_data:
-    driver: local
   index_data:
     driver: local
   backups:

bin/test-remote

@@ -10,7 +10,7 @@ fi
 
 export CHROMA_INTEGRATION_TEST_ONLY=1
 export CHROMA_SERVER_HOST=$1
-export CHROMA_API_IMPL=rest
+export CHROMA_API_IMPL=chromadb.api.fastapi.FastAPI
 export CHROMA_SERVER_HTTP_PORT=8000
 
 python -m pytest

chromadb/__init__.py

@@ -1,3 +1,4 @@
+from typing import Dict
 import chromadb.config
 import logging
 from chromadb.telemetry.events import ClientStartEvent
@@ -22,6 +23,58 @@ def get_settings() -> Settings:
     return __settings
 
 
+def EphemeralClient(settings: Settings = Settings()) -> API:
+    """
+    Creates an in-memory instance of Chroma. This is useful for testing and
+    development, but not recommended for production use.
+    """
+    settings.is_persistent = False
+
+    return Client(settings)
+
+
+def PersistentClient(path: str = "./chroma", settings: Settings = Settings()) -> API:
+    """
+    Creates a persistent instance of Chroma that saves to disk. This is useful for
+    testing and development, but not recommended for production use.
+
+    Args:
+        path: The directory to save Chroma's data to. Defaults to "./chroma".
+    """
+    settings.persist_directory = path
+    settings.is_persistent = True
+
+    return Client(settings)
+
+
+def HttpClient(
+    host: str = "localhost",
+    port: str = "8000",
+    ssl: bool = False,
+    headers: Dict[str, str] = {},
+    settings: Settings = Settings(),
+) -> API:
+    """
+    Creates a client that connects to a remote Chroma server. This supports
+    many clients connecting to the same server, and is the recommended way to
+    use Chroma in production.
+
+    Args:
+        host: The hostname of the Chroma server. Defaults to "localhost".
+        port: The port of the Chroma server. Defaults to "8000".
+        ssl: Whether to use SSL to connect to the Chroma server. Defaults to False.
+        headers: A dictionary of headers to send to the Chroma server. Defaults to {}.
+    """
+
+    settings.chroma_api_impl = "chromadb.api.fastapi.FastAPI"
+    settings.chroma_server_host = host
+    settings.chroma_server_http_port = port
+    settings.chroma_server_ssl_enabled = ssl
+    settings.chroma_server_headers = headers
+
+    return Client(settings)
+
+
 def Client(settings: Settings = __settings) -> API:
     """Return a running chroma.API instance"""
 

chromadb/api/__init__.py

@@ -16,7 +16,7 @@
     GetResult,
     WhereDocument,
 )
-from chromadb.config import Component
+from chromadb.config import Component, Settings
 import chromadb.utils.embedding_functions as ef
 
 
@@ -390,22 +390,21 @@ def create_index(self, collection_name: str) -> bool:
         pass
 
     @abstractmethod
-    def persist(self) -> bool:
-        """Persist the database to disk
+    def get_version(self) -> str:
+        """Get the version of Chroma.
 
         Returns:
-            bool: True if the database was persisted successfully
+            str: The version of Chroma
 
         """
-
         pass
 
     @abstractmethod
-    def get_version(self) -> str:
-        """Get the version of Chroma.
+    def get_settings(self) -> Settings:
+        """Get the settings used to initialize the client.
 
         Returns:
-            str: The version of Chroma
+            Settings: The settings used to initialize the client.
 
         """
         pass