Make openai classes use fetch, remove use of node-fetch (langchain-ai…

…#118), Convert library to ESM codebase, ESM output (langchain-ai#124), Add GH (manual) action for running integration tests (langchain-ai#119) * Use fetch adapter for openai axios * Remove node-fetch, add instructions for node 16 * Add GH (manual) action for running integration tests (langchain-ai#119) * Add integration tests GH action (manual trigger for now), reduce cost of integration tests by using smallest possible models * Convert library to ESM codebase, ESM output (langchain-ai#124) * Use fetch adapter for openai axios * Update tsc build to output ESM only * Update all import paths to have extension per ESM requirements * Move all source files to src/ * Remove circular dependency * Fix jest config for ESM * Remove circular deps not possible with ESM * Throw error in index getter to dedupe code and have safer usage (langchain-ai#35) * Fix hnsw for esm * Fix usage of hnswlib with index passed in * Fix textsplitter for esm * Fix openai for esm * Fix hf for esm * Fix ESM for cohere * Fix ESM for serpapi * Fix esm in srt * Remove dependency on @vespaiach/axios-fetch-adapter which has an incorrect export * Fix examples for esm * Fix entrypoints * Fix test-exports for esm * Add fetch flag for node 16 ci job * Add a more thorough test for packaging * Fix docs build --------- Co-authored-by: micahriggan <[email protected]> --------- Co-authored-by: micahriggan <[email protected]> * Fix one more import * Fix example * Also build docs in ci * Fix sql test in ci --------- Co-authored-by: micahriggan <[email protected]>
lzhme · Feb 27, 2023 · 1f7045a · 1f7045a
1 parent fced19c
commit 1f7045a
Show file tree

Hide file tree

Showing 145 changed files with 1,244 additions and 759 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -29,3 +29,6 @@ jobs:
         run: yarn install --immutable
       - run: yarn run ci
       - run: yarn workspace langchain run test
+        if: matrix.node-version != '16.x'
+      - run: yarn workspace langchain run test:node16
+        if: matrix.node-version == '16.x'
diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml
@@ -0,0 +1,29 @@
+# This workflow will do a clean installation of node dependencies, cache/restore them, build the source code and run tests across different versions of node
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-nodejs
+
+name: Node.js Integration Tests
+
+on:
+  workflow_dispatch:
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        os: [macos-latest, windows-latest, ubuntu-latest]
+        node-version: [16.x, 18.x]
+        # See supported Node.js release schedule at https://nodejs.org/en/about/releases/
+
+    runs-on: ${{ matrix.os }}
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node-version }}
+          cache: "yarn"
+      - name: Install dependencies
+        run: yarn install --immutable
+      - run: yarn run ci
+      - run: yarn workspace langchain run test:integration
diff --git a/docs/docs/getting-started.md b/docs/docs/getting-started.md
@@ -10,6 +10,11 @@ To get started, install LangChain with the following command:
 npm i langchain
 ```
 
+If you are running this on Node.js 16, either:
+
+- run your application with `NODE_OPTIONS='--experimental-fetch' node ...`, or
+- install `node-fetch` and follow the instructions [here](https://github.com/node-fetch/node-fetch#providing-global-access)
+
 ## Picking up a LLM
 
 Using LangChain will usually require integrations with one or more model providers, data stores, apis, etc.

diff --git a/docs/package.json b/docs/package.json
@@ -12,7 +12,7 @@
     "serve": "docusaurus serve",
     "write-translations": "docusaurus write-translations",
     "write-heading-ids": "docusaurus write-heading-ids",
-    "ci": "yarn lint && yarn format:diff",
+    "ci": "yarn lint && yarn format:diff && yarn build",
     "lint": "eslint --cache \"**/*.js\"",
     "format": "prettier --write \"**/*.{js,jsx,ts,tsx,md,mdx}\"",
     "format:diff": "prettier --list-different \"**/*.{js,jsx,ts,tsx,md,mdx}\""

diff --git a/examples/.eslintrc.js → examples/.eslintrc.cjs b/examples/.eslintrc.js → examples/.eslintrc.cjs
diff --git a/examples/package.json b/examples/package.json
@@ -4,6 +4,7 @@
   "private": true,
   "description": "Langchain examples",
   "main": "./dist/index.js",
+  "type": "module",
   "files": [
     "dist/"
   ],

diff --git a/examples/src/agents/agents_vectorstore.ts b/examples/src/agents/agents_vectorstore.ts
@@ -13,7 +13,7 @@ export const run = async () => {
   const text = fs.readFileSync("state_of_the_union.txt", "utf8");
   /* Split the text into chunks */
   const textSplitter = new RecursiveCharacterTextSplitter({ chunkSize: 1000 });
-  const docs = textSplitter.createDocuments([text]);
+  const docs = await textSplitter.createDocuments([text]);
   /* Create the vectorstore */
   const vectorStore = await HNSWLib.fromDocuments(docs, new OpenAIEmbeddings());
   /* Create the chain */

diff --git a/examples/src/chains/chat_vector_db.ts b/examples/src/chains/chat_vector_db.ts
@@ -12,7 +12,7 @@ export const run = async () => {
   const text = fs.readFileSync("state_of_the_union.txt", "utf8");
   /* Split the text into chunks */
   const textSplitter = new RecursiveCharacterTextSplitter({ chunkSize: 1000 });
-  const docs = textSplitter.createDocuments([text]);
+  const docs = await textSplitter.createDocuments([text]);
   /* Create the vectorstore */
   const vectorStore = await HNSWLib.fromDocuments(docs, new OpenAIEmbeddings());
   /* Create the chain */

diff --git a/examples/src/chains/chat_vector_db_chroma.ts b/examples/src/chains/chat_vector_db_chroma.ts
@@ -14,7 +14,7 @@ export const run = async () => {
   const text = fs.readFileSync("state_of_the_union.txt", "utf8");
   /* Split the text into chunks */
   const textSplitter = new RecursiveCharacterTextSplitter({ chunkSize: 1000 });
-  const docs = textSplitter.createDocuments([text]);
+  const docs = await textSplitter.createDocuments([text]);
   /* Create the vectorstore */
   const vectorStore = await Chroma.fromDocuments(
     docs,

diff --git a/examples/src/chains/vector_db_qa.ts b/examples/src/chains/vector_db_qa.ts
@@ -12,7 +12,7 @@ export const run = async () => {
   const text = fs.readFileSync("state_of_the_union.txt", "utf8");
   /* Split the text into chunks */
   const textSplitter = new RecursiveCharacterTextSplitter({ chunkSize: 1000 });
-  const docs = textSplitter.createDocuments([text]);
+  const docs = await textSplitter.createDocuments([text]);
   /* Create the vectorstore */
   const vectorStore = await HNSWLib.fromDocuments(docs, new OpenAIEmbeddings());
   /* Create the chain */

diff --git a/examples/tsconfig.json b/examples/tsconfig.json
@@ -4,6 +4,7 @@
     "outDir": "dist",
     "lib": [
       "ESNext",
+      "DOM"
     ],
     "sourceMap": true,
     "allowSyntheticDefaultImports": true,

diff --git a/langchain/.eslintrc.js → langchain/.eslintrc.cjs b/langchain/.eslintrc.js → langchain/.eslintrc.cjs
@@ -12,7 +12,7 @@ module.exports = {
     sourceType: "module",
   },
   plugins: ["@typescript-eslint"],
-  ignorePatterns: ["dist", "docs", "node_modules", "*.d.ts", "*.js", "*.mjs"],
+  ignorePatterns: ["dist", "docs", "node_modules", "*.d.ts", "*.cjs", "*.js"],
   rules: {
     "@typescript-eslint/explicit-module-boundary-types": 0,
     "@typescript-eslint/no-empty-function": 0,

diff --git a/langchain/.gitignore b/langchain/.gitignore
@@ -1,39 +1,26 @@
 agents.js
-agents.mjs
 agents.d.ts
 tools.js
-tools.mjs
 tools.d.ts
 chains.js
-chains.mjs
 chains.d.ts
 embeddings.js
-embeddings.mjs
 embeddings.d.ts
 llms.js
-llms.mjs
 llms.d.ts
 prompts.js
-prompts.mjs
 prompts.d.ts
 vectorstores.js
-vectorstores.mjs
 vectorstores.d.ts
 text_splitter.js
-text_splitter.mjs
 text_splitter.d.ts
 memory.js
-memory.mjs
 memory.d.ts
 document.js
-document.mjs
 document.d.ts
 docstore.js
-docstore.mjs
 docstore.d.ts
 document_loaders.js
-document_loaders.mjs
 document_loaders.d.ts
 index.js
-index.mjs
 index.d.ts
diff --git a/langchain/.prettierignore b/langchain/.prettierignore
diff --git a/langchain/agents/agent_toolkits/index.ts b/langchain/agents/agent_toolkits/index.ts
diff --git a/langchain/agents/index.ts b/langchain/agents/index.ts
diff --git a/langchain/agents/tools/index.ts b/langchain/agents/tools/index.ts
diff --git a/langchain/babel.config.js → langchain/babel.config.cjs b/langchain/babel.config.js → langchain/babel.config.cjs
diff --git a/langchain/create-entrypoints.js b/langchain/create-entrypoints.js
@@ -1,6 +1,6 @@
-/* eslint-disable @typescript-eslint/no-var-requires */
-const path = require("path");
-const fs = require("fs");
+import path from "path";
+import url from "url";
+import fs from "fs";
 
 const entrypoints = {
   agents: "agents/index",
@@ -18,7 +18,10 @@ const entrypoints = {
 };
 
 const updateJsonFile = (relativePath, updateFunction) => {
-  const filePath = path.resolve(__dirname, relativePath);
+  const filePath = path.resolve(
+    path.dirname(url.fileURLToPath(import.meta.url)),
+    relativePath
+  );
   const contents = fs.readFileSync(filePath).toString();
   const res = updateFunction(JSON.parse(contents));
   fs.writeFileSync(filePath, JSON.stringify(res, null, 2));
@@ -27,12 +30,9 @@ const updateJsonFile = (relativePath, updateFunction) => {
 const generateFiles = () => {
   const files = [...Object.entries(entrypoints), ["index", "index"]].flatMap(
     ([key, value]) => {
-      const modulePath =
-        path.basename(value) === "index" ? path.dirname(value) : value;
-      const compiledPath = `./dist/${modulePath}`;
+      const compiledPath = `./dist/${value}.js`;
       return [
-        [`${key}.js`, `module.exports = require('${compiledPath}')`],
-        [`${key}.mjs`, `export * from './dist/${value}.js'`],
+        [`${key}.js`, `export * from '${compiledPath}'`],
         [`${key}.d.ts`, `export * from '${compiledPath}'`],
       ];
     }
@@ -47,7 +47,7 @@ const updateConfig = () => {
     typedocOptions: {
       ...json.typedocOptions,
       entryPoints: [...Object.values(entrypoints), "index"].map(
-        (value) => `./${value}.ts`
+        (value) => `src/${value}.ts`
       ),
     },
   }));
@@ -60,8 +60,8 @@ const updateConfig = () => {
     exports: Object.fromEntries(
       ["index", ...Object.keys(entrypoints)].map((key) => {
         const entryPoint = {
-          import: `./${key}.mjs`,
-          default: `./${key}.js`,
+          types: `./${key}.d.ts`,
+          import: `./${key}.js`,
         };
         return [key === "index" ? "." : `./${key}`, entryPoint];
       })

diff --git a/langchain/docstore/index.ts b/langchain/docstore/index.ts
diff --git a/langchain/document_loaders/index.ts b/langchain/document_loaders/index.ts
diff --git a/langchain/embeddings/index.ts b/langchain/embeddings/index.ts
diff --git a/langchain/index.ts b/langchain/index.ts
diff --git a/langchain/jest.config.js → langchain/jest.config.cjs b/langchain/jest.config.js → langchain/jest.config.cjs
@@ -1,12 +1,13 @@
 /** @type {import('ts-jest').JestConfigWithTsJest} */
 module.exports = {
-  preset: "ts-jest/presets/js-with-ts",
+  preset: "ts-jest/presets/default-esm",
   testEnvironment: "node",
   modulePathIgnorePatterns: ["dist/", "docs/"],
+  moduleNameMapper: {
+    "^(\\.{1,2}/.*)\\.js$": "$1",
+  },
   transform: {
-    "^.+\\.(ts|tsx)$": "ts-jest",
-    "^.+\\.(js)$": "babel-jest",
+    "^.+\\.m?[tj]sx?$": ["ts-jest", { useESM: true }],
   },
-  transformIgnorePatterns: [],
   setupFiles: ["dotenv/config"],
 };
diff --git a/langchain/memory/index.ts b/langchain/memory/index.ts