all[minor]: Replace all LC package scripts with @langchain/scripts (l…

…angchain-ai#4092) * scripts package * added dep omit from import map arg * test exports for lc * format * revert tsconfig.json change * prettierrc file * updated langchain scripts to use scripts package * format * fixes * updated other packages * update turbo build commands * format * updated core scripts * gitignore update' * update community * cr * cr * update pure * chore: lint files * prefix node modules w/ node: * one more node: prefix * move type * cr * try something * try something * try something * try something * try something * modify script * modify script * modify script * test something * cr * revert and use version for scripts pkg * cr * cr * cr * cr * chore: lint files * chore: lint files * use lc config.js file for scripts * chore: lint files * update' * bump * drop scripts from format * update exa * cr * pull in main entrypoints * chore: lint files * fix build * fix relative path issue * fix again * udate docs
AndiLeni · Jan 26, 2024 · 7a82e91 · 7a82e91
1 parent ec9a447
commit 7a82e91
Show file tree

Hide file tree

Showing 108 changed files with 1,714 additions and 5,857 deletions.
diff --git a/.gitignore b/.gitignore
@@ -41,4 +41,5 @@ tmp/
 langchain/api_refs_docs_build/dist/**/*
 
 .docusaurus/
-docs/build/
+docs/build/
+docs/api_refs/typedoc.json
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -235,24 +235,28 @@ import { OpenAI } from "langchain/llms/openai";
 We call these subpaths "entrypoints". In general, you should create a new entrypoint if you are adding a new integration with a 3rd party library. If you're adding self-contained functionality without any external dependencies, you can add it to an existing entrypoint.
 
 In order to declare a new entrypoint that users can import from, you
-should edit the `langchain/scripts/create-entrypoints.js` or `libs/langchain-community/scripts/create-entrypoints.js` script. To add an
+should edit the `langchain/langchain.config.js` or `libs/langchain-community/langchain.config.js` file. To add an
 entrypoint `tools` that imports from `tools/index.ts` you'd add
-the following to the `entrypoints` variable:
+the following to the `entrypoints` key inside the `config` variable:
 
 ```typescript
-const entrypoints = {
+// ...
+entrypoints: {
   // ...
   tools: "tools/index",
-};
+},
+// ...
 ```
 
-If you're adding a new integration which requires installing a third party depencency, you must add the entrypoint to the `requiresOptionalDependency` array, also located inside `langchain/scripts/create-entrypoints.js` or `libs/langchain-community/scripts/create-entrypoints.js`.
+If you're adding a new integration which requires installing a third party dependency, you must add the entrypoint to the `requiresOptionalDependency` array, also located inside `langchain/langchain.config.js` or `libs/langchain-community/langchain.config.js`.
 
 ```typescript
-const requiresOptionalDependency = [
+// ...
+requiresOptionalDependency: [
   // ...
   "tools/index",
-];
+],
+// ...
 ```
 
 This will make sure the entrypoint is included in the published package,

diff --git a/docs/api_refs/.gitignore b/docs/api_refs/.gitignore
@@ -39,4 +39,3 @@ next-env.d.ts
 /public/*
 /langchain
 /langchain-core
-typedoc.json
diff --git a/docs/api_refs/package.json b/docs/api_refs/package.json
@@ -30,7 +30,6 @@
     "postcss": "^8",
     "prettier": "^2.8.3",
     "tailwindcss": "^3.3.0",
-    "ts-morph": "^20.0.0",
     "typedoc": "^0.25.6",
     "typescript": "~5.1.6"
   }

diff --git a/docs/api_refs/scripts/create-entrypoints.js b/docs/api_refs/scripts/create-entrypoints.js
@@ -1,4 +1,3 @@
-const { Project, SyntaxKind } = require("ts-morph");
 const fs = require("fs");
 const path = require("path");
 
@@ -39,62 +38,49 @@ const updateJsonFile = (relativePath, updateFunction) => {
   fs.writeFileSync(relativePath, JSON.stringify(res, null, 2) + "\n");
 };
 
-function main() {
-  const project = new Project();
+async function main() {
   const workspaces = fs
     .readdirSync("../../libs/")
     .filter((dir) => dir.startsWith("langchain-"))
-    .map((dir) =>
-      path.join("../../libs/", dir, "/scripts/create-entrypoints.js")
-    );
-  const entrypointFiles = [
-    "../../langchain/scripts/create-entrypoints.js",
-    "../../langchain-core/scripts/create-entrypoints.js",
+    .map((dir) => path.join("../../libs/", dir, "/langchain.config.js"));
+  const configFiles = [
+    "../../langchain/langchain.config.js",
+    "../../langchain-core/langchain.config.js",
     ...workspaces,
-  ];
+  ]
+    .map((configFile) => path.resolve(configFile))
+    .filter((configFile) => !configFile.includes("/langchain-scripts/"));
+
   /** @type {Array<string>} */
   const blacklistedEntrypoints = JSON.parse(
     fs.readFileSync("./blacklisted-entrypoints.json")
   );
 
   const entrypoints = new Set([]);
-  entrypointFiles.forEach((entrypointFile) => {
-    // load file contents from ts-morph
-    const file = project.addSourceFileAtPath(entrypointFile);
-    // extract the variable named entrypoints
-    const entrypointVar = file.getVariableDeclarationOrThrow("entrypoints");
-    // extract the `deprecatedNodeOnly` if it exists
-    const deprecatedNodeOnlyVar =
-      file.getVariableDeclaration("deprecatedNodeOnly");
-    /**
-     * @type {string[]}
-     */
-    let deprecatedNodeOnly = [];
-    if (deprecatedNodeOnlyVar) {
-      const deprecatedNodeOnlyKeys = deprecatedNodeOnlyVar
-        .getInitializerIfKindOrThrow(SyntaxKind.ArrayLiteralExpression)
-        .getElements()
-        .map((element) => element.getText().replaceAll('"', ""));
-      deprecatedNodeOnly = deprecatedNodeOnlyKeys;
+
+  for await (const configFile of configFiles) {
+    const langChainConfig = await import(configFile);
+    if (!("entrypoints" in langChainConfig.config)) {
+      throw new Error(
+        `The config file "${configFile}" does not contain any entrypoints.`
+      );
+    } else if (
+      langChainConfig.config.entrypoints === null ||
+      langChainConfig.config.entrypoints === undefined
+    ) {
+      return;
     }
-    // get all keys from the entrypoints object
-    const entrypointKeys = entrypointVar
-      .getInitializerIfKindOrThrow(SyntaxKind.ObjectLiteralExpression)
-      .getProperties()
-      .map((property) => property.getText());
-    const entrypointKeysArray = entrypointKeys.map((kv) =>
-      kv.split(":").map((part) => part.trim().replace(/^"|"$/g, ""))
+    const { config } = langChainConfig;
+
+    const entrypointDir = path.relative(
+      process.cwd(),
+      configFile.split("/langchain.config.js")[0]
     );
 
-    /**
-     * @type {Record<string, string>}
-     */
-    const entrypointsObject = Object.fromEntries(entrypointKeysArray);
-    const entrypointDir = entrypointFile.split(
-      "/scripts/create-entrypoints.js"
-    )[0];
+    const deprecatedNodeOnly =
+      "deprecatedNodeOnly" in config ? config.deprecatedNodeOnly : [];
 
-    Object.values(entrypointsObject)
+    Object.values(config.entrypoints)
       .filter((key) => !deprecatedNodeOnly.includes(key))
       .filter(
         (key) =>
@@ -104,8 +90,7 @@ function main() {
           )
       )
       .map((key) => entrypoints.add(`${entrypointDir}/src/${key}.ts`));
-  });
-
+  }
   // Check if the `./typedoc.json` file exists, since it is gitignored by default
   if (!fs.existsSync("./typedoc.json")) {
     fs.writeFileSync("./typedoc.json", "{}\n");

diff --git a/docs/api_refs/typedoc.json b/docs/api_refs/typedoc.json
@@ -192,6 +192,7 @@
     "../../langchain/src/smith/index.ts",
     "../../langchain/src/runnables/index.ts",
     "../../langchain/src/runnables/remote.ts",
+    "../../langchain/src/indexes/index.ts",
     "../../langchain-core/src/agents.ts",
     "../../langchain-core/src/caches.ts",
     "../../langchain-core/src/callbacks/base.ts",
@@ -226,6 +227,7 @@
     "../../langchain-core/src/utils/async_caller.ts",
     "../../langchain-core/src/utils/chunk_array.ts",
     "../../langchain-core/src/utils/env.ts",
+    "../../langchain-core/src/utils/function_calling.ts",
     "../../langchain-core/src/utils/hash.ts",
     "../../langchain-core/src/utils/json_patch.ts",
     "../../langchain-core/src/utils/json_schema.ts",
@@ -330,6 +332,7 @@
     "../../libs/langchain-community/src/vectorstores/singlestore.ts",
     "../../libs/langchain-community/src/vectorstores/supabase.ts",
     "../../libs/langchain-community/src/vectorstores/tigris.ts",
+    "../../libs/langchain-community/src/vectorstores/turbopuffer.ts",
     "../../libs/langchain-community/src/vectorstores/typeorm.ts",
     "../../libs/langchain-community/src/vectorstores/typesense.ts",
     "../../libs/langchain-community/src/vectorstores/usearch.ts",
@@ -357,6 +360,7 @@
     "../../libs/langchain-community/src/callbacks/handlers/llmonitor.ts",
     "../../libs/langchain-community/src/callbacks/handlers/lunary.ts",
     "../../libs/langchain-community/src/retrievers/amazon_kendra.ts",
+    "../../libs/langchain-community/src/retrievers/amazon_knowledge_base.ts",
     "../../libs/langchain-community/src/retrievers/chaindesk.ts",
     "../../libs/langchain-community/src/retrievers/databerry.ts",
     "../../libs/langchain-community/src/retrievers/metal.ts",
@@ -371,6 +375,7 @@
     "../../libs/langchain-community/src/caches/momento.ts",
     "../../libs/langchain-community/src/caches/upstash_redis.ts",
     "../../libs/langchain-community/src/graphs/neo4j_graph.ts",
+    "../../libs/langchain-community/src/graphs/memgraph_graph.ts",
     "../../libs/langchain-community/src/utils/event_source_parse.ts",
     "../../libs/langchain-community/src/document_transformers/html_to_text.ts",
     "../../libs/langchain-community/src/document_transformers/mozilla_readability.ts",
@@ -397,9 +402,14 @@
     "../../libs/langchain-community/src/memory/motorhead_memory.ts",
     "../../libs/langchain-community/src/memory/zep.ts",
     "../../libs/langchain-community/src/utils/convex.ts",
+    "../../libs/langchain-community/src/indexes/base.ts",
+    "../../libs/langchain-community/src/indexes/postgres.ts",
+    "../../libs/langchain-community/src/indexes/memory.ts",
+    "../../libs/langchain-exa/src/index.ts",
     "../../libs/langchain-google-genai/src/index.ts",
     "../../libs/langchain-mistralai/src/index.ts",
     "../../libs/langchain-openai/src/index.ts",
+    "../../libs/langchain-pinecone/src/index.ts",
     "../../libs/langchain-yandex/src/chat_models.ts",
     "../../libs/langchain-yandex/src/embeddings.ts",
     "../../libs/langchain-yandex/src/index.ts",

diff --git a/langchain-core/.gitignore b/langchain-core/.gitignore
@@ -130,6 +130,6 @@ utils/types.d.ts
 vectorstores.cjs
 vectorstores.js
 vectorstores.d.ts
-index.cjs
-index.js
-index.d.ts
+node_modules
+dist
+.yarn
diff --git a/langchain-core/langchain.config.js b/langchain-core/langchain.config.js
@@ -0,0 +1,65 @@
+import { resolve, dirname } from "node:path";
+import { fileURLToPath } from "node:url";
+
+/**
+ * @param {string} relativePath
+ * @returns {string}
+ */
+function abs(relativePath) {
+  return resolve(dirname(fileURLToPath(import.meta.url)), relativePath);
+}
+
+export const config = {
+  internals: [/node\:/, /js-tiktoken/],
+  entrypoints: {
+    agents: "agents",
+    caches: "caches",
+    "callbacks/base": "callbacks/base",
+    "callbacks/manager": "callbacks/manager",
+    "callbacks/promises": "callbacks/promises",
+    chat_history: "chat_history",
+    documents: "documents/index",
+    embeddings: "embeddings",
+    example_selectors: "example_selectors/index",
+    "language_models/base": "language_models/base",
+    "language_models/chat_models": "language_models/chat_models",
+    "language_models/llms": "language_models/llms",
+    load: "load/index",
+    "load/serializable": "load/serializable",
+    memory: "memory",
+    messages: "messages/index",
+    output_parsers: "output_parsers/index",
+    outputs: "outputs",
+    prompts: "prompts/index",
+    prompt_values: "prompt_values",
+    runnables: "runnables/index",
+    retrievers: "retrievers",
+    stores: "stores",
+    tools: "tools",
+    "tracers/base": "tracers/base",
+    "tracers/console": "tracers/console",
+    "tracers/initialize": "tracers/initialize",
+    "tracers/log_stream": "tracers/log_stream",
+    "tracers/run_collector": "tracers/run_collector",
+    "tracers/tracer_langchain": "tracers/tracer_langchain",
+    "tracers/tracer_langchain_v1": "tracers/tracer_langchain_v1",
+    "utils/async_caller": "utils/async_caller",
+    "utils/chunk_array": "utils/chunk_array",
+    "utils/env": "utils/env",
+    "utils/function_calling": "utils/function_calling",
+    "utils/hash": "utils/hash",
+    "utils/json_patch": "utils/json_patch",
+    "utils/json_schema": "utils/json_schema",
+    "utils/math": "utils/math",
+    "utils/stream": "utils/stream",
+    "utils/testing": "utils/testing/index",
+    "utils/tiktoken": "utils/tiktoken",
+    "utils/types": "utils/types",
+    vectorstores: "vectorstores",
+  },
+  tsConfigPath: resolve("./tsconfig.json"),
+  packageSuffix: "core",
+  cjsSource: "./dist-cjs",
+  cjsDestination: "./dist",
+  abs,
+}
diff --git a/langchain-core/package.json b/langchain-core/package.json
@@ -15,21 +15,24 @@
   "scripts": {
     "build": "yarn clean && yarn build:esm && yarn build:cjs && yarn run build:scripts",
     "build:esm": "NODE_OPTIONS=--max-old-space-size=4096 tsc --outDir dist/ && rimraf dist/tests dist/**/tests",
-    "build:cjs": "NODE_OPTIONS=--max-old-space-size=4096 tsc --outDir dist-cjs/ -p tsconfig.cjs.json && node scripts/move-cjs-to-dist.js && rimraf dist-cjs",
-    "build:watch": "node scripts/create-entrypoints.js && tsc --outDir dist/ --watch",
-    "build:scripts": "node scripts/create-entrypoints.js && node scripts/check-tree-shaking.js",
+    "build:cjs": "NODE_OPTIONS=--max-old-space-size=4096 tsc --outDir dist-cjs/ -p tsconfig.cjs.json && yarn move-cjs-to-dist && rimraf dist-cjs",
+    "build:watch": "yarn create-entrypoints && tsc --outDir dist/ --watch",
+    "build:scripts": "yarn create-entrypoints && yarn check-tree-shaking",
     "lint:eslint": "NODE_OPTIONS=--max-old-space-size=4096 eslint --cache --ext .ts,.js src/",
     "lint:dpdm": "dpdm --exit-code circular:1 --no-warning --no-tree src/*.ts src/**/*.ts",
     "lint": "yarn lint:eslint && yarn lint:dpdm",
     "lint:fix": "yarn lint:eslint --fix && yarn lint:dpdm",
-    "clean": "rimraf .turbo/ dist/ && NODE_OPTIONS=--max-old-space-size=4096 node scripts/create-entrypoints.js pre",
+    "clean": "rimraf .turbo/ dist/ && NODE_OPTIONS=--max-old-space-size=4096 yarn create-entrypoints -- --pre",
     "prepack": "yarn build",
     "release": "release-it --only-version --config .release-it.json",
     "test": "NODE_OPTIONS=--experimental-vm-modules jest --testPathIgnorePatterns=\\.int\\.test.ts --testTimeout 30000 --maxWorkers=50%",
     "test:watch": "NODE_OPTIONS=--experimental-vm-modules jest --watch --testPathIgnorePatterns=\\.int\\.test.ts",
     "test:single": "NODE_OPTIONS=--experimental-vm-modules yarn run jest --config jest.config.cjs --testTimeout 100000",
-    "format": "prettier --config .prettierrc --write \"src\" \"scripts\"",
-    "format:check": "prettier --config .prettierrc --check \"src\" \"scripts\""
+    "format": "prettier --config .prettierrc --write \"src\"",
+    "format:check": "prettier --config .prettierrc --check \"src\"",
+    "move-cjs-to-dist": "yarn lc-build --config ./langchain.config.js --move-cjs-dist",
+    "create-entrypoints": "yarn lc-build --config ./langchain.config.js --create-entrypoints",
+    "check-tree-shaking": "yarn lc-build --config ./langchain.config.js --tree-shaking"
   },
   "author": "LangChain",
   "license": "MIT",
@@ -48,6 +51,7 @@
   },
   "devDependencies": {
     "@jest/globals": "^29.5.0",
+    "@langchain/scripts": "^0.0.2",
     "@swc/core": "^1.3.90",
     "@swc/jest": "^0.2.29",
     "dpdm": "^3.12.0",
@@ -440,9 +444,6 @@
     "utils/types.d.ts",
     "vectorstores.cjs",
     "vectorstores.js",
-    "vectorstores.d.ts",
-    "index.cjs",
-    "index.js",
-    "index.d.ts"
+    "vectorstores.d.ts"
   ]
 }