diff --git a/docs/docs/ecosystem/databerry.md b/docs/docs/ecosystem/databerry.md new file mode 100644 index 000000000000..e5fabd04a5aa --- /dev/null +++ b/docs/docs/ecosystem/databerry.md @@ -0,0 +1,40 @@ +# Databerry + +This page covers how to use the [Databerry](https://databerry.ai) within LangChain. + +## What is Databerry? + +Databerry is an [open source](https://github.com/gmpetrov/databerry) document retrievial platform that helps to connect your personal data with Large Language Models. + +![Databerry](/img/DataberryDashboard.png) + +## Quick start + +Retrieving documents stored in Databerry from LangChain is very easy! + +```typescript +import { DataberryRetriever } from "langchain/retrievers/databerry"; + +const retriever = new DataberryRetriever({ + datastoreUrl: "https://api.databerry.ai/query/clg1xg2h80000l708dymr0fxc", + apiKey: "DATABERRY_API_KEY", // optional: needed for private datastores + topK: 8, // optional: default value is 3 +}); + +// Create a chain that uses the OpenAI LLM and Databerry retriever. +const chain = RetrievalQAChain.fromLLM(model, retriever); + +// Call the chain with a query. +const res = await chain.call({ + query: "What's Databerry?", +}); + +console.log({ res }); +/* +{ + res: { + text: 'Databerry provides a user-friendly solution to quickly setup a semantic search system over your personal data without any technical knowledge.' + } +} +*/ +``` diff --git a/docs/docs/modules/indexes/retrievers/databerry-retriever.mdx b/docs/docs/modules/indexes/retrievers/databerry-retriever.mdx new file mode 100644 index 000000000000..6037c716ca79 --- /dev/null +++ b/docs/docs/modules/indexes/retrievers/databerry-retriever.mdx @@ -0,0 +1,10 @@ +# Remote Retriever + +This example shows how to use the Databerry Retriever in a `RetrievalQAChain` to retrieve documents from a Databerry.ai datastore. + +## Usage + +import CodeBlock from "@theme/CodeBlock"; +import Example from "@examples/retrievers/databerry.ts"; + +{Example} diff --git a/docs/static/img/DataberryDashboard.png b/docs/static/img/DataberryDashboard.png new file mode 100644 index 000000000000..090f7d095523 Binary files /dev/null and b/docs/static/img/DataberryDashboard.png differ diff --git a/examples/src/retrievers/databerry.ts b/examples/src/retrievers/databerry.ts new file mode 100644 index 000000000000..8ab4d3ff5f71 --- /dev/null +++ b/examples/src/retrievers/databerry.ts @@ -0,0 +1,13 @@ +import { DataberryRetriever } from "langchain/retrievers/databerry"; + +export const run = async () => { + const retriever = new DataberryRetriever({ + datastoreUrl: "https://api.databerry.ai/query/clg1xg2h80000l708dymr0fxc", + apiKey: "DATABERRY_API_KEY", // optional: needed for private datastores + topK: 8, // optional: default value is 3 + }); + + const docs = await retriever.getRelevantDocuments("hello"); + + console.log(docs); +}; diff --git a/langchain/.gitignore b/langchain/.gitignore index 5bba09e1b6ff..14549e16f39c 100644 --- a/langchain/.gitignore +++ b/langchain/.gitignore @@ -193,6 +193,9 @@ retrievers/supabase.d.ts retrievers/metal.cjs retrievers/metal.js retrievers/metal.d.ts +retrievers/databerry.cjs +retrievers/databerry.js +retrievers/databerry.d.ts cache.cjs cache.js cache.d.ts diff --git a/langchain/package.json b/langchain/package.json index a13519c3d0d9..e0ce89d6d029 100644 --- a/langchain/package.json +++ b/langchain/package.json @@ -205,6 +205,9 @@ "retrievers/metal.cjs", "retrievers/metal.js", "retrievers/metal.d.ts", + "retrievers/databerry.cjs", + "retrievers/databerry.js", + "retrievers/databerry.d.ts", "cache.cjs", "cache.js", "cache.d.ts", @@ -739,6 +742,11 @@ "import": "./retrievers/metal.js", "require": "./retrievers/metal.cjs" }, + "./retrievers/databerry": { + "types": "./retrievers/databerry.d.ts", + "import": "./retrievers/databerry.js", + "require": "./retrievers/databerry.cjs" + }, "./cache": { "types": "./cache.d.ts", "import": "./cache.js", diff --git a/langchain/scripts/create-entrypoints.js b/langchain/scripts/create-entrypoints.js index 3bc8ed1ba8a0..9b41604ebb04 100644 --- a/langchain/scripts/create-entrypoints.js +++ b/langchain/scripts/create-entrypoints.js @@ -92,6 +92,7 @@ const entrypoints = { "retrievers/remote": "retrievers/remote/index", "retrievers/supabase": "retrievers/supabase", "retrievers/metal": "retrievers/metal", + "retrievers/databerry": "retrievers/databerry", // cache cache: "cache", }; diff --git a/langchain/src/retrievers/databerry.ts b/langchain/src/retrievers/databerry.ts new file mode 100644 index 000000000000..815d15d77a8e --- /dev/null +++ b/langchain/src/retrievers/databerry.ts @@ -0,0 +1,63 @@ +import { BaseRetriever } from "../schema/index.js"; +import { Document } from "../document.js"; +import { AsyncCaller, AsyncCallerParams } from "../util/async_caller.js"; + +interface DataberryRetrieverArgs extends AsyncCallerParams { + datastoreUrl: string; + topK?: number; + apiKey?: string; +} + +interface Berry { + text: string; + score: number; + source?: string; + [key: string]: unknown; +} + +export class DataberryRetriever extends BaseRetriever { + caller: AsyncCaller; + + datastoreUrl: string; + + topK?: number; + + apiKey?: string; + + constructor({ datastoreUrl, apiKey, topK, ...rest }: DataberryRetrieverArgs) { + super(); + + this.caller = new AsyncCaller(rest); + this.datastoreUrl = datastoreUrl; + this.apiKey = apiKey; + this.topK = topK; + } + + async getRelevantDocuments(query: string): Promise { + const r = await this.caller.call(fetch, this.datastoreUrl, { + method: "POST", + body: JSON.stringify({ + query, + ...(this.topK ? { topK: this.topK } : {}), + }), + headers: { + "Content-Type": "application/json", + ...(this.apiKey ? { Authorization: `Bearer ${this.apiKey}` } : {}), + }, + }); + + const { results } = (await r.json()) as { results: Berry[] }; + + return results.map( + ({ text, score, source, ...rest }) => + new Document({ + pageContent: text, + metadata: { + score, + source, + ...rest, + }, + }) + ); + } +} diff --git a/langchain/tsconfig.json b/langchain/tsconfig.json index fb442e35f2a4..c4d611e9993e 100644 --- a/langchain/tsconfig.json +++ b/langchain/tsconfig.json @@ -93,6 +93,7 @@ "src/retrievers/remote/index.ts", "src/retrievers/supabase.ts", "src/retrievers/metal.ts", + "src/retrievers/databerry.ts", "src/cache.ts" ], "excludePrivate": true, diff --git a/test-exports-cf/src/entrypoints.js b/test-exports-cf/src/entrypoints.js index 7d2bc264059c..c9e8af349c16 100644 --- a/test-exports-cf/src/entrypoints.js +++ b/test-exports-cf/src/entrypoints.js @@ -22,4 +22,5 @@ export * from "langchain/schema"; export * from "langchain/callbacks"; export * from "langchain/output_parsers"; export * from "langchain/retrievers/remote"; +export * from "langchain/retrievers/databerry"; export * from "langchain/cache"; diff --git a/test-exports-cjs/src/entrypoints.js b/test-exports-cjs/src/entrypoints.js index 9d3551f1c2fc..e37dfa633c85 100644 --- a/test-exports-cjs/src/entrypoints.js +++ b/test-exports-cjs/src/entrypoints.js @@ -22,4 +22,5 @@ const schema = require("langchain/schema"); const callbacks = require("langchain/callbacks"); const output_parsers = require("langchain/output_parsers"); const retrievers_remote = require("langchain/retrievers/remote"); +const retrievers_databerry = require("langchain/retrievers/databerry"); const cache = require("langchain/cache"); diff --git a/test-exports-cra/src/entrypoints.js b/test-exports-cra/src/entrypoints.js index 7d2bc264059c..c9e8af349c16 100644 --- a/test-exports-cra/src/entrypoints.js +++ b/test-exports-cra/src/entrypoints.js @@ -22,4 +22,5 @@ export * from "langchain/schema"; export * from "langchain/callbacks"; export * from "langchain/output_parsers"; export * from "langchain/retrievers/remote"; +export * from "langchain/retrievers/databerry"; export * from "langchain/cache"; diff --git a/test-exports-esm/src/entrypoints.js b/test-exports-esm/src/entrypoints.js index e745d9dd640c..7c4521e1a689 100644 --- a/test-exports-esm/src/entrypoints.js +++ b/test-exports-esm/src/entrypoints.js @@ -22,4 +22,5 @@ import * as schema from "langchain/schema"; import * as callbacks from "langchain/callbacks"; import * as output_parsers from "langchain/output_parsers"; import * as retrievers_remote from "langchain/retrievers/remote"; +import * as retrievers_databerry from "langchain/retrievers/databerry"; import * as cache from "langchain/cache"; diff --git a/test-exports-vercel/src/entrypoints.js b/test-exports-vercel/src/entrypoints.js index 7d2bc264059c..c9e8af349c16 100644 --- a/test-exports-vercel/src/entrypoints.js +++ b/test-exports-vercel/src/entrypoints.js @@ -22,4 +22,5 @@ export * from "langchain/schema"; export * from "langchain/callbacks"; export * from "langchain/output_parsers"; export * from "langchain/retrievers/remote"; +export * from "langchain/retrievers/databerry"; export * from "langchain/cache"; diff --git a/test-exports-vite/src/entrypoints.js b/test-exports-vite/src/entrypoints.js index 7d2bc264059c..c9e8af349c16 100644 --- a/test-exports-vite/src/entrypoints.js +++ b/test-exports-vite/src/entrypoints.js @@ -22,4 +22,5 @@ export * from "langchain/schema"; export * from "langchain/callbacks"; export * from "langchain/output_parsers"; export * from "langchain/retrievers/remote"; +export * from "langchain/retrievers/databerry"; export * from "langchain/cache";