Fix weaviate object type metadata (langchain-ai#908, langchain-ai#926) (

langchain-ai#1001) - Flatten document metadata before inserting - Test to test out deep document metadata Co-authored-by: Nuno Campos <[email protected]>
philrwebb · Apr 27, 2023 · 5f131a6 · 5f131a6
1 parent bc2e14f
commit 5f131a6
Show file tree

Hide file tree

Showing 3 changed files with 80 additions and 9 deletions.
diff --git a/langchain/src/util/flatten.ts b/langchain/src/util/flatten.ts
@@ -0,0 +1,19 @@
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export const flattenObject = (obj: Record<string, any>, deep = true) => {
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  const flattenedObject: Record<string, any> = {};
+
+  for (const key in obj) {
+    if (typeof obj[key] === "object" && !Array.isArray(obj[key])) {
+      let recursiveResult = obj[key];
+      if (deep) recursiveResult = flattenObject(recursiveResult);
+
+      for (const deepKey in recursiveResult) {
+        if (Object.hasOwn(obj, key))
+          flattenedObject[`${key}.${deepKey}`] = recursiveResult[deepKey];
+      }
+    }
+  }
+
+  return flattenedObject;
+};
diff --git a/langchain/src/vectorstores/tests/weaviate.int.test.ts b/langchain/src/vectorstores/tests/weaviate.int.test.ts
@@ -43,4 +43,47 @@ test.skip("WeaviateStore", async () => {
   expect(results2).toEqual([
     new Document({ pageContent: "hi there", metadata: { foo: "baz" } }),
   ]);
+
+  const testDocumentWithObjectMetadata = new Document({
+    pageContent: "this is the deep document world!",
+    metadata: {
+      deep: {
+        string: "deep string",
+        deepdeep: {
+          string: "even a deeper string",
+        },
+      },
+    },
+  });
+  const documentStore = await WeaviateStore.fromDocuments(
+    [testDocumentWithObjectMetadata],
+    new OpenAIEmbeddings(),
+    {
+      client,
+      indexName: "DocumentTest",
+      textKey: "text",
+      metadataKeys: ["deep.string", "deep.deepdeep.string"],
+    }
+  );
+
+  const result3 = await documentStore.similaritySearch(
+    "this is the deep document world!",
+    1,
+    {
+      where: {
+        operator: "Equal",
+        path: ["deep.string"],
+        valueText: "deep string",
+      },
+    }
+  );
+  expect(result3).toEqual([
+    new Document({
+      pageContent: "this is the deep document world!",
+      metadata: {
+        "deep.string": "deep string",
+        "deep.deepdeep.string": "even a deeper string",
+      },
+    }),
+  ]);
 });
diff --git a/langchain/src/vectorstores/weaviate.ts b/langchain/src/vectorstores/weaviate.ts
@@ -7,6 +7,7 @@ import type {
 import { VectorStore } from "./base.js";
 import { Embeddings } from "../embeddings/base.js";
 import { Document } from "../document.js";
+import { flattenObject } from "../util/flatten.js";
 
 export interface WeaviateLibArgs {
   client: WeaviateClient;
@@ -53,15 +54,23 @@ export class WeaviateStore extends VectorStore {
   }
 
   async addVectors(vectors: number[][], documents: Document[]): Promise<void> {
-    const batch: WeaviateObject[] = documents.map((document, index) => ({
-      class: this.indexName,
-      id: v4(),
-      vector: vectors[index],
-      properties: {
-        [this.textKey]: document.pageContent,
-        ...document.metadata,
-      },
-    }));
+    const batch: WeaviateObject[] = documents.map((document, index) => {
+      if (Object.hasOwn(document.metadata, "id"))
+        throw new Error(
+          "Document inserted to Weaviate vectorstore should not have `id` in their metadata."
+        );
+
+      const flattenedMetadata = flattenObject(document.metadata);
+      return {
+        class: this.indexName,
+        id: v4(),
+        vector: vectors[index],
+        properties: {
+          [this.textKey]: document.pageContent,
+          ...flattenedMetadata,
+        },
+      };
+    });
 
     try {
       await this.client.batch