feat: DataberryRetriever (langchain-ai#643) (langchain-ai#732)

* feat: DataberryRetriever (langchain-ai#643)

* cr

* cr

* Small fixes for new paths

---------

Co-authored-by: Georges Petrov <[email protected]>
Co-authored-by: Nuno Campos <[email protected]>

docs/docs/ecosystem/databerry.md

@@ -0,0 +1,40 @@
+# Databerry
+
+This page covers how to use the [Databerry](https://databerry.ai) within LangChain.
+
+## What is Databerry?
+
+Databerry is an [open source](https://github.com/gmpetrov/databerry) document retrievial platform that helps to connect your personal data with Large Language Models.
+
+![Databerry](/img/DataberryDashboard.png)
+
+## Quick start
+
+Retrieving documents stored in Databerry from LangChain is very easy!
+
+```typescript
+import { DataberryRetriever } from "langchain/retrievers/databerry";
+
+const retriever = new DataberryRetriever({
+  datastoreUrl: "https://api.databerry.ai/query/clg1xg2h80000l708dymr0fxc",
+  apiKey: "DATABERRY_API_KEY", // optional: needed for private datastores
+  topK: 8, // optional: default value is 3
+});
+
+// Create a chain that uses the OpenAI LLM and Databerry retriever.
+const chain = RetrievalQAChain.fromLLM(model, retriever);
+
+// Call the chain with a query.
+const res = await chain.call({
+  query: "What's Databerry?",
+});
+
+console.log({ res });
+/*
+{
+  res: {
+    text: 'Databerry provides a user-friendly solution to quickly setup a semantic search system over your personal data without any technical knowledge.'
+  }
+}
+*/
+```

docs/docs/modules/indexes/retrievers/databerry-retriever.mdx

@@ -0,0 +1,10 @@
+# Remote Retriever
+
+This example shows how to use the Databerry Retriever in a `RetrievalQAChain` to retrieve documents from a Databerry.ai datastore.
+
+## Usage
+
+import CodeBlock from "@theme/CodeBlock";
+import Example from "@examples/retrievers/databerry.ts";
+
+<CodeBlock language="typescript">{Example}</CodeBlock>

examples/src/retrievers/databerry.ts

@@ -0,0 +1,13 @@
+import { DataberryRetriever } from "langchain/retrievers/databerry";
+
+export const run = async () => {
+  const retriever = new DataberryRetriever({
+    datastoreUrl: "https://api.databerry.ai/query/clg1xg2h80000l708dymr0fxc",
+    apiKey: "DATABERRY_API_KEY", // optional: needed for private datastores
+    topK: 8, // optional: default value is 3
+  });
+
+  const docs = await retriever.getRelevantDocuments("hello");
+
+  console.log(docs);
+};

langchain/.gitignore

@@ -193,6 +193,9 @@ retrievers/supabase.d.ts
 retrievers/metal.cjs
 retrievers/metal.js
 retrievers/metal.d.ts
+retrievers/databerry.cjs
+retrievers/databerry.js
+retrievers/databerry.d.ts
 cache.cjs
 cache.js
 cache.d.ts

langchain/package.json

@@ -205,6 +205,9 @@
     "retrievers/metal.cjs",
     "retrievers/metal.js",
     "retrievers/metal.d.ts",
+    "retrievers/databerry.cjs",
+    "retrievers/databerry.js",
+    "retrievers/databerry.d.ts",
     "cache.cjs",
     "cache.js",
     "cache.d.ts",
@@ -739,6 +742,11 @@
       "import": "./retrievers/metal.js",
       "require": "./retrievers/metal.cjs"
     },
+    "./retrievers/databerry": {
+      "types": "./retrievers/databerry.d.ts",
+      "import": "./retrievers/databerry.js",
+      "require": "./retrievers/databerry.cjs"
+    },
     "./cache": {
       "types": "./cache.d.ts",
       "import": "./cache.js",

langchain/scripts/create-entrypoints.js

@@ -92,6 +92,7 @@ const entrypoints = {
   "retrievers/remote": "retrievers/remote/index",
   "retrievers/supabase": "retrievers/supabase",
   "retrievers/metal": "retrievers/metal",
+  "retrievers/databerry": "retrievers/databerry",
   // cache
   cache: "cache",
 };

langchain/src/retrievers/databerry.ts

@@ -0,0 +1,63 @@
+import { BaseRetriever } from "../schema/index.js";
+import { Document } from "../document.js";
+import { AsyncCaller, AsyncCallerParams } from "../util/async_caller.js";
+
+interface DataberryRetrieverArgs extends AsyncCallerParams {
+  datastoreUrl: string;
+  topK?: number;
+  apiKey?: string;
+}
+
+interface Berry {
+  text: string;
+  score: number;
+  source?: string;
+  [key: string]: unknown;
+}
+
+export class DataberryRetriever extends BaseRetriever {
+  caller: AsyncCaller;
+
+  datastoreUrl: string;
+
+  topK?: number;
+
+  apiKey?: string;
+
+  constructor({ datastoreUrl, apiKey, topK, ...rest }: DataberryRetrieverArgs) {
+    super();
+
+    this.caller = new AsyncCaller(rest);
+    this.datastoreUrl = datastoreUrl;
+    this.apiKey = apiKey;
+    this.topK = topK;
+  }
+
+  async getRelevantDocuments(query: string): Promise<Document[]> {
+    const r = await this.caller.call(fetch, this.datastoreUrl, {
+      method: "POST",
+      body: JSON.stringify({
+        query,
+        ...(this.topK ? { topK: this.topK } : {}),
+      }),
+      headers: {
+        "Content-Type": "application/json",
+        ...(this.apiKey ? { Authorization: `Bearer ${this.apiKey}` } : {}),
+      },
+    });
+
+    const { results } = (await r.json()) as { results: Berry[] };
+
+    return results.map(
+      ({ text, score, source, ...rest }) =>
+        new Document({
+          pageContent: text,
+          metadata: {
+            score,
+            source,
+            ...rest,
+          },
+        })
+    );
+  }
+}

langchain/tsconfig.json

@@ -93,6 +93,7 @@
       "src/retrievers/remote/index.ts",
       "src/retrievers/supabase.ts",
       "src/retrievers/metal.ts",
+      "src/retrievers/databerry.ts",
       "src/cache.ts"
     ],
     "excludePrivate": true,

test-exports-cf/src/entrypoints.js

@@ -22,4 +22,5 @@ export * from "langchain/schema";
 export * from "langchain/callbacks";
 export * from "langchain/output_parsers";
 export * from "langchain/retrievers/remote";
+export * from "langchain/retrievers/databerry";
 export * from "langchain/cache";

test-exports-cjs/src/entrypoints.js

@@ -22,4 +22,5 @@ const schema = require("langchain/schema");
 const callbacks = require("langchain/callbacks");
 const output_parsers = require("langchain/output_parsers");
 const retrievers_remote = require("langchain/retrievers/remote");
+const retrievers_databerry = require("langchain/retrievers/databerry");
 const cache = require("langchain/cache");

test-exports-cra/src/entrypoints.js

@@ -22,4 +22,5 @@ export * from "langchain/schema";
 export * from "langchain/callbacks";
 export * from "langchain/output_parsers";
 export * from "langchain/retrievers/remote";
+export * from "langchain/retrievers/databerry";
 export * from "langchain/cache";

test-exports-esm/src/entrypoints.js

@@ -22,4 +22,5 @@ import * as schema from "langchain/schema";
 import * as callbacks from "langchain/callbacks";
 import * as output_parsers from "langchain/output_parsers";
 import * as retrievers_remote from "langchain/retrievers/remote";
+import * as retrievers_databerry from "langchain/retrievers/databerry";
 import * as cache from "langchain/cache";

test-exports-vercel/src/entrypoints.js

@@ -22,4 +22,5 @@ export * from "langchain/schema";
 export * from "langchain/callbacks";
 export * from "langchain/output_parsers";
 export * from "langchain/retrievers/remote";
+export * from "langchain/retrievers/databerry";
 export * from "langchain/cache";

test-exports-vite/src/entrypoints.js

@@ -22,4 +22,5 @@ export * from "langchain/schema";
 export * from "langchain/callbacks";
 export * from "langchain/output_parsers";
 export * from "langchain/retrievers/remote";
+export * from "langchain/retrievers/databerry";
 export * from "langchain/cache";