Harrison/pinecone (langchain-ai#142)

* Use official pinecone npm package (langchain-ai#112) * use official pinecone npm package * update doc * fix types for addVectors * sync up with 0.0.8 * UpsertResponse is not used * fix formatting for docs and examples * cr --------- Co-authored-by: Fraser Xu <[email protected]>
NJordan72 · Feb 26, 2023 · 6e1e274 · 6e1e274
1 parent 0dc1d54
commit 6e1e274
Show file tree

Hide file tree

Showing 6 changed files with 96 additions and 62 deletions.
diff --git a/docs/docs/modules/indexes/vectorstore.md b/docs/docs/modules/indexes/vectorstore.md
@@ -56,21 +56,47 @@ console.log(resultOne); // -> 'Achilles: Yiikes! What is that?'
 
 ## Pinecone vectorstore
 
-Langchain.js accepts [pinecone-client](https://github.com/rileytomasek/pinecone-client) as the client for Pinecone vectorstore. Install the library with `npm install -S pinecone-client`.
+Langchain.js accepts [@pinecone-database/pinecone](https://docs.pinecone.io/docs/node-client) as the client for Pinecone vectorstore. Install the library with `npm install -S @pinecone-database/pinecone`.
+
+Index docs
 
 ```typescript
 import { PineconeStore } from "langchain/vectorstores";
 import { OpenAIEmbeddings } from "langchain/embeddings";
-import { PineconeClient } from "pinecone-client";
+import { PineconeClient } from "@pinecone-database/pinecone";
+
+const pinecone = new PineconeClient();
+await pinecone.init({
+  environment: "us-west1-gcp",
+  apiKey: "apiKey",
+});
+const index = pinecone.Index("my-index");
+await PineconeStore.fromDocuments(index, docs, new OpenAIEmbeddings());
+```
 
-const client = new PineconeClient({});
+Query docs
 
-const vectorStore = await PineconeStore.fromTexts(
-  client,
-  ["Hello world", "Bye bye", "hello nice world"],
-  [{ id: 2 }, { id: 1 }, { id: 3 }],
+```typescript
+import { PineconeStore } from "langchain/vectorstores";
+import { OpenAIEmbeddings } from "langchain/embeddings";
+import { PineconeClient } from "@pinecone-database/pinecone";
+import { VectorDBQAChain } from "langchain/chains";
+import { OpenAI } from "langchain/llms";
+
+const pinecone = new PineconeClient();
+await pinecone.init({
+  environment: "us-west1-gcp",
+  apiKey: "apiKey",
+});
+const index = pinecone.Index("my-index");
+const vectorStore = await PineconeStore.fromExistingIndex(
+  index,
   new OpenAIEmbeddings()
 );
 
-const resultOne = await vectorStore.similaritySearch("Hello world", 2);
+const model = new OpenAI();
+const chain = VectorDBQAChain.fromLLM(model, vectorStore);
+const response = await chain.call({
+  query: "what does the doc say about pinecone",
+});
 ```
diff --git a/examples/package.json b/examples/package.json
@@ -20,10 +20,10 @@
   "license": "MIT",
   "dependencies": {
     "@dqbd/tiktoken": "^0.4.0",
+    "@pinecone-database/pinecone": "^0.0.8",
     "chromadb": "^1.3.0",
     "langchain": "workspace:*",
     "openai": "^3.1.0",
-    "pinecone-client": "^1.0.1",
     "serpapi": "^1.1.0",
     "sqlite3": "^5.1.4"
   },

diff --git a/examples/src/indexes/vectorstores_pinecone.ts b/examples/src/indexes/vectorstores_pinecone.ts
@@ -1,12 +1,17 @@
 import { PineconeStore } from "langchain/vectorstores";
 import { OpenAIEmbeddings } from "langchain/embeddings";
-import { PineconeClient } from "pinecone-client";
+import { PineconeClient } from "@pinecone-database/pinecone";
 
 export const run = async () => {
-  const client = new PineconeClient({});
+  const client = new PineconeClient();
+  await client.init({
+    environment: "us-west1-gcp",
+    apiKey: "apiKey",
+  });
 
+  const index = client.Index("my-index");
   const vectorStore = await PineconeStore.fromTexts(
-    client,
+    index,
     ["Hello world", "Bye bye", "hello nice world"],
     [{ id: 2 }, { id: 1 }, { id: 3 }],
     new OpenAIEmbeddings()

diff --git a/langchain/package.json b/langchain/package.json
@@ -72,6 +72,7 @@
     "@babel/preset-env": "^7.20.2",
     "@dqbd/tiktoken": "^0.4.0",
     "@jest/globals": "^29.4.2",
+    "@pinecone-database/pinecone": "^0.0.8",
     "@tsconfig/recommended": "^1.0.2",
     "@types/node-fetch": "2",
     "@types/uuid": "^9",
@@ -92,7 +93,6 @@
     "jest": "^29.4.2",
     "lint-staged": "^13.1.1",
     "openai": "^3.1.0",
-    "pinecone-client": "^1.0.1",
     "prettier": "^2.8.3",
     "serpapi": "^1.1.1",
     "srt-parser-2": "^1.2.2",
@@ -104,20 +104,23 @@
   },
   "peerDependencies": {
     "@dqbd/tiktoken": "^0.4.0",
+    "@pinecone-database/pinecone": "^0.0.8",
     "cheerio": "^1.0.0-rc.12",
     "chromadb": "^1.3.0",
     "cohere-ai": "^5.0.2",
     "hnswlib-node": "^1.3.0",
     "huggingface": "^1.4.0",
     "openai": "^3.1.0",
-    "pinecone-client": "^1.0.1",
     "serpapi": "^1.1.1",
     "srt-parser-2": "^1.2.2"
   },
   "peerDependenciesMeta": {
     "@dqbd/tiktoken": {
       "optional": true
     },
+    "@pinecone-database/pinecone": {
+      "optional": true
+    },
     "cheerio": {
       "optional": true
     },
@@ -133,9 +136,6 @@
     "openai": {
       "optional": true
     },
-    "pinecone-client": {
-      "optional": true
-    },
     "serpapi": {
       "optional": true
     },
@@ -232,4 +232,4 @@
       "default": "./document_loaders.js"
     }
   }
-}
+}
diff --git a/langchain/vectorstores/pinecone.ts b/langchain/vectorstores/pinecone.ts
@@ -1,4 +1,4 @@
-import type { PineconeClient } from "pinecone-client";
+import type { VectorOperationsApi } from "@pinecone-database/pinecone/dist/pinecone-generated-ts-fetch";
 import { v4 as uuidv4 } from "uuid";
 
 import { VectorStore } from "./base";
@@ -11,10 +11,10 @@ type PineconeMetadata = Record<string, any>;
 export class PineconeStore extends VectorStore {
   textKey: string;
 
-  pineconeClient: PineconeClient<PineconeMetadata>;
+  pineconeClient: VectorOperationsApi;
 
   constructor(
-    pineconeClient: PineconeClient<PineconeMetadata>,
+    pineconeClient: VectorOperationsApi,
     embeddings: Embeddings,
     textKey = "text"
   ) {
@@ -34,18 +34,24 @@ export class PineconeStore extends VectorStore {
     );
   }
 
-  async addVectors(vectors: number[][], documents: Document[], ids?: string[]) {
+  async addVectors(
+    vectors: number[][],
+    documents: Document[],
+    ids?: string[]
+  ): Promise<void> {
     const documentIds = ids == null ? documents.map(() => uuidv4()) : ids;
 
-    return this.pineconeClient.upsert({
-      vectors: vectors.map((values, idx) => ({
-        id: documentIds[idx],
-        metadata: {
-          ...documents[idx].metadata,
-          [this.textKey]: documents[idx].pageContent,
-        },
-        values,
-      })),
+    await this.pineconeClient.upsert({
+      upsertRequest: {
+        vectors: vectors.map((values, idx) => ({
+          id: documentIds[idx],
+          metadata: {
+            ...documents[idx].metadata,
+            [this.textKey]: documents[idx].pageContent,
+          },
+          values,
+        })),
+      },
     });
   }
 
@@ -54,24 +60,30 @@ export class PineconeStore extends VectorStore {
     k: number
   ): Promise<[Document, number][]> {
     const results = await this.pineconeClient.query({
-      topK: k,
-      includeMetadata: true,
-      vector: query,
+      queryRequest: {
+        topK: k,
+        includeMetadata: true,
+        vector: query,
+      },
     });
 
     const result: [Document, number][] = [];
 
-    for (const res of results.matches) {
-      const { [this.textKey]: pageContent, ...metadata } =
-        res.metadata as PineconeMetadata;
-      result.push([new Document({ metadata, pageContent }), res.score]);
+    if (results.matches) {
+      for (const res of results.matches) {
+        const { [this.textKey]: pageContent, ...metadata } =
+          res.metadata as PineconeMetadata;
+        if (res.score) {
+          result.push([new Document({ metadata, pageContent }), res.score]);
+        }
+      }
     }
 
     return result;
   }
 
   static async fromTexts(
-    pineconeClient: PineconeClient<PineconeMetadata>,
+    pineconeClient: VectorOperationsApi,
     texts: string[],
     metadatas: object[],
     embeddings: Embeddings,
@@ -95,7 +107,7 @@ export class PineconeStore extends VectorStore {
   }
 
   static async fromDocuments(
-    pineconeClient: PineconeClient<PineconeMetadata>,
+    pineconeClient: VectorOperationsApi,
     docs: Document[],
     embeddings: Embeddings,
     textKey = "text"
@@ -106,7 +118,7 @@ export class PineconeStore extends VectorStore {
   }
 
   static async fromExistingIndex(
-    pineconeClient: PineconeClient<PineconeMetadata>,
+    pineconeClient: VectorOperationsApi,
     embeddings: Embeddings,
     textKey = "text"
   ): Promise<PineconeStore> {

diff --git a/yarn.lock b/yarn.lock
@@ -2799,6 +2799,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@pinecone-database/pinecone@npm:^0.0.8":
+  version: 0.0.8
+  resolution: "@pinecone-database/pinecone@npm:0.0.8"
+  checksum: 8a7ea30b2bd9a6d0ee06e719e94b122753adc8dc468d4ab70da67a30964dfeb3539f4fac996b8ebadf6a0d739546e76ef463d00b30d68a9d567c779a6cbf8b4a
+  languageName: node
+  linkType: hard
+
 "@polka/url@npm:^1.0.0-next.20":
   version: 1.0.0-next.21
   resolution: "@polka/url@npm:1.0.0-next.21"
@@ -9535,18 +9542,12 @@ __metadata:
   languageName: node
   linkType: hard
 
-"ky@npm:^0.33.1":
-  version: 0.33.2
-  resolution: "ky@npm:0.33.2"
-  checksum: bb5b1685b7d639dad5fc0a41a218f76c2400d2a9189bcf747f541633af13b68a7ef12bb8aa971bc1667087789ec5965cc3728e24b693897a2d0cc539d7593655
-  languageName: node
-  linkType: hard
-
 "langchain-examples@workspace:examples":
   version: 0.0.0-use.local
   resolution: "langchain-examples@workspace:examples"
   dependencies:
     "@dqbd/tiktoken": ^0.4.0
+    "@pinecone-database/pinecone": ^0.0.8
     "@tsconfig/recommended": ^1.0.2
     "@typescript-eslint/eslint-plugin": ^5.51.0
     "@typescript-eslint/parser": ^5.51.0
@@ -9559,7 +9560,6 @@ __metadata:
     eslint-plugin-prettier: ^4.2.1
     langchain: "workspace:*"
     openai: ^3.1.0
-    pinecone-client: ^1.0.1
     prettier: ^2.8.3
     serpapi: ^1.1.0
     sqlite3: ^5.1.4
@@ -9575,6 +9575,7 @@ __metadata:
     "@babel/preset-env": ^7.20.2
     "@dqbd/tiktoken": ^0.4.0
     "@jest/globals": ^29.4.2
+    "@pinecone-database/pinecone": ^0.0.8
     "@tsconfig/recommended": ^1.0.2
     "@types/node-fetch": 2
     "@types/uuid": ^9
@@ -9602,7 +9603,6 @@ __metadata:
     node-fetch: 2
     openai: ^3.1.0
     p-queue: 6
-    pinecone-client: ^1.0.1
     prettier: ^2.8.3
     serpapi: ^1.1.1
     sqlite3: ^5.1.4
@@ -9616,18 +9616,20 @@ __metadata:
     yaml: ^2.2.1
   peerDependencies:
     "@dqbd/tiktoken": ^0.4.0
+    "@pinecone-database/pinecone": ^0.0.8
     cheerio: ^1.0.0-rc.12
     chromadb: ^1.3.0
     cohere-ai: ^5.0.2
     hnswlib-node: ^1.3.0
     huggingface: ^1.4.0
     openai: ^3.1.0
-    pinecone-client: ^1.0.1
     serpapi: ^1.1.1
     srt-parser-2: ^1.2.2
   peerDependenciesMeta:
     "@dqbd/tiktoken":
       optional: true
+    "@pinecone-database/pinecone":
+      optional: true
     cheerio:
       optional: true
     chromadb:
@@ -9638,8 +9640,6 @@ __metadata:
       optional: true
     openai:
       optional: true
-    pinecone-client:
-      optional: true
     serpapi:
       optional: true
     srt-parser-2:
@@ -11162,15 +11162,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"pinecone-client@npm:^1.0.1":
-  version: 1.0.1
-  resolution: "pinecone-client@npm:1.0.1"
-  dependencies:
-    ky: ^0.33.1
-  checksum: ec7cc0f3a2a9f580d1dcb5229b86ee9c120d322b48dd37311875f8a3872128a7999e0bc9af2969d457fd52098c9ec9ca231bd73a9bc08a4b40eefd1f94333fef
-  languageName: node
-  linkType: hard
-
 "pirates@npm:^4.0.4":
   version: 4.0.5
   resolution: "pirates@npm:4.0.5"