feat: add Milvus as vectorStore provider (langchain-ai#600)

* feat: add Milvus as vectorStore provider * docs: add docs for Milvus * fix: add Milvus's missing method "fromExistingCollection" * feat: add example * metadata supports complex data. * Add entrypoint, lint, fix some issues * Update yarnlock * Fix CI --------- Co-authored-by: Nuno Campos <[email protected]>
zombieyang · Apr 11, 2023 · 883ddec · 883ddec
1 parent f37a880
commit 883ddec
Show file tree

Hide file tree

Showing 13 changed files with 883 additions and 2 deletions.
diff --git a/docs/docs/modules/indexes/vector_stores/integrations/milvus.md b/docs/docs/modules/indexes/vector_stores/integrations/milvus.md
@@ -0,0 +1,78 @@
+---
+sidebar_class_name: node-only
+---
+
+# Milvus
+
+[Milvus](https://milvus.io/) is an vector database built for embedding similarity search and AI applications.
+
+:::tip Compatibility
+Only available on Node.js.
+:::
+
+## Setup
+
+1. Run milvus instance inside of docker on your computer [docs](https://milvus.io/docs/v2.1.x/install_standalone-docker.md)
+2. Install the Milvus Node.js SDK.
+
+```bash npm2yarn
+npm install -S @zilliz/milvus2-sdk-node
+```
+
+3. Setup Env variables for Milvus before running the code
+
+```bash
+export OPENAI_API_KEY=YOUR_OPEN_API_HERE
+export MILVUS_URL=YOUR_MILVUS_URL_HERE # for example http://localhost:19530
+```
+
+## Index and query docs
+
+```typescript
+import { Milvus } from "langchain/vectorstores/milvus";
+import { OpenAIEmbeddings } from "langchain/embeddings/openai";
+
+// text sample from Godel, Escher, Bach
+const vectorStore = await Milvus.fromTexts(
+  [
+    "Tortoise: Labyrinth? Labyrinth? Could it Are we in the notorious Little\
+            Harmonic Labyrinth of the dreaded Majotaur?",
+    "Achilles: Yiikes! What is that?",
+    "Tortoise: They say-although I person never believed it myself-that an I\
+            Majotaur has created a tiny labyrinth sits in a pit in the middle of\
+            it, waiting innocent victims to get lost in its fears complexity.\
+            Then, when they wander and dazed into the center, he laughs and\
+            laughs at them-so hard, that he laughs them to death!",
+    "Achilles: Oh, no!",
+    "Tortoise: But it's only a myth. Courage, Achilles.",
+  ],
+  [{ id: 2 }, { id: 1 }, { id: 3 }, { id: 4 }, { id: 5 }],
+  new OpenAIEmbeddings(),
+  {
+    collectionName: "goldel-escher-bach",
+  }
+);
+
+// or alternatively from docs
+const vectorStore = await Milvus.fromDocuments(docs, new OpenAIEmbeddings(), {
+  collectionName: "goldel-escher-bach",
+});
+
+const response = await vectorStore.similaritySearch("scared", 2);
+```
+
+## Query docs from existing collection
+
+```typescript
+import { Milvus } from "langchain/vectorstores/milvus";
+import { OpenAIEmbeddings } from "langchain/embeddings/openai";
+
+const vectorStore = await Milvus.fromExistingCollection(
+  new OpenAIEmbeddings(),
+  {
+    collectionName: "goldel-escher-bach",
+  }
+);
+
+const response = await vectorStore.similaritySearch("scared", 2);
+```
diff --git a/examples/package.json b/examples/package.json
@@ -26,6 +26,7 @@
     "@pinecone-database/pinecone": "^0.0.12",
     "@prisma/client": "^4.11.0",
     "@supabase/supabase-js": "^2.10.0",
+    "@zilliz/milvus2-sdk-node": "^2.2.0",
     "chromadb": "^1.3.0",
     "js-yaml": "^4.1.0",
     "langchain": "workspace:*",

diff --git a/examples/src/indexes/vector_stores/milvus.ts b/examples/src/indexes/vector_stores/milvus.ts
@@ -0,0 +1,13 @@
+import { Milvus } from "langchain/vectorstores/milvus";
+import { OpenAIEmbeddings } from "langchain/embeddings/openai";
+
+export const run = async () => {
+  const vectorStore = await Milvus.fromTexts(
+    ["Hello world", "Bye bye", "hello nice world"],
+    [{ id: 2 }, { id: 1 }, { id: 3 }],
+    new OpenAIEmbeddings()
+  );
+
+  const resultOne = await vectorStore.similaritySearch("hello world", 1);
+  console.log(resultOne);
+};
diff --git a/langchain/.env.example b/langchain/.env.example
@@ -11,4 +11,5 @@ SUPABASE_PRIVATE_KEY=ADD_YOURS_HERE
 SUPABASE_URL=ADD_YOURS_HERE
 ZAPIER_NLA_API_KEY=ADD_YOURS_HERE
 ANTHROPIC_API_KEY=ADD_YOURS_HERE
-REPLICATE_API_KEY=ADD_YOURS_HERE
+REPLICATE_API_KEY=ADD_YOURS_HERE
+MILVUS_URL=ADD_YOURS_HERE
diff --git a/langchain/.eslintrc.cjs b/langchain/.eslintrc.cjs
@@ -35,6 +35,8 @@ module.exports = {
               "it",
               "beforeEach",
               "afterEach",
+              "beforeAll",
+              "afterAll",
               "skip",
               "each",
               "only",

diff --git a/langchain/.gitignore b/langchain/.gitignore
@@ -79,6 +79,9 @@ vectorstores/pinecone.d.ts
 vectorstores/supabase.cjs
 vectorstores/supabase.js
 vectorstores/supabase.d.ts
+vectorstores/milvus.cjs
+vectorstores/milvus.js
+vectorstores/milvus.d.ts
 vectorstores/prisma.cjs
 vectorstores/prisma.js
 vectorstores/prisma.d.ts

diff --git a/langchain/package.json b/langchain/package.json
@@ -91,6 +91,9 @@
     "vectorstores/supabase.cjs",
     "vectorstores/supabase.js",
     "vectorstores/supabase.d.ts",
+    "vectorstores/milvus.cjs",
+    "vectorstores/milvus.js",
+    "vectorstores/milvus.d.ts",
     "vectorstores/prisma.cjs",
     "vectorstores/prisma.js",
     "vectorstores/prisma.d.ts",
@@ -296,6 +299,7 @@
     "@huggingface/inference": "^1.5.1",
     "@pinecone-database/pinecone": "*",
     "@supabase/supabase-js": "^2.10.0",
+    "@zilliz/milvus2-sdk-node": "^2.2.0",
     "cheerio": "^1.0.0-rc.12",
     "chromadb": "^1.3.0",
     "cohere-ai": "^5.0.2",
@@ -328,6 +332,9 @@
     "@supabase/supabase-js": {
       "optional": true
     },
+    "@zilliz/milvus2-sdk-node": {
+      "optional": true
+    },
     "cheerio": {
       "optional": true
     },
@@ -554,6 +561,11 @@
       "import": "./vectorstores/supabase.js",
       "require": "./vectorstores/supabase.cjs"
     },
+    "./vectorstores/milvus": {
+      "types": "./vectorstores/milvus.d.ts",
+      "import": "./vectorstores/milvus.js",
+      "require": "./vectorstores/milvus.cjs"
+    },
     "./vectorstores/prisma": {
       "types": "./vectorstores/prisma.d.ts",
       "import": "./vectorstores/prisma.js",

diff --git a/langchain/scripts/check-tree-shaking.js b/langchain/scripts/check-tree-shaking.js
@@ -28,6 +28,8 @@ export function listExternals() {
     /node\:/,
     "axios", // axios is a dependency of openai
     "pdf-parse/lib/pdf-parse.js",
+    "@zilliz/milvus2-sdk-node/dist/milvus/const/Milvus.js",
+    "@zilliz/milvus2-sdk-node/dist/milvus/types.js",
   ];
 }
 

diff --git a/langchain/scripts/create-entrypoints.js b/langchain/scripts/create-entrypoints.js
@@ -42,6 +42,7 @@ const entrypoints = {
   "vectorstores/hnswlib": "vectorstores/hnswlib",
   "vectorstores/pinecone": "vectorstores/pinecone",
   "vectorstores/supabase": "vectorstores/supabase",
+  "vectorstores/milvus": "vectorstores/milvus",
   "vectorstores/prisma": "vectorstores/prisma",
   // text_splitter
   text_splitter: "text_splitter",
@@ -126,6 +127,7 @@ const requiresOptionalDependency = [
   "vectorstores/hnswlib",
   "vectorstores/pinecone",
   "vectorstores/supabase",
+  "vectorstores/milvus",
   "document_loaders/web/cheerio",
   "document_loaders/web/puppeteer",
   "document_loaders/web/playwright",