Nc/callbacks docs (langchain-ai#1012)

* Add docs on how/when to use callbacks

* Update "create custom handler" section

* Update hierarchy

* Update constructor for BaseChain to allow receiving an object with args, rather than positional args

Doing this in a backwards compat way, ie. still supporting old positional args

* Remove requirement to implement serialize method in subcalsses of BaseChain to make it easier to subclass (until we work more on serialization)

* Fix code block api docs section for renamed imports

* Add docs and examples on creating custom chains, small reorg to chain docs

* Split callbacks docs into multiple pages, add docs on using callbacks in subclasses

* Fix broken links

* Fix one more

* Fix tsc transpiling issue

TSC was defining this property before the `super` call

* Lint

* Add example of __run

* Update

* Typo

docs/code-block-loader.js

@@ -38,30 +38,31 @@ async function webpackLoader(content, map, meta) {
         node.specifiers.forEach((specifier) => {
           if (specifier.type === "ImportSpecifier") {
             const local = specifier.local.name;
-            imports.push({ local, source });
+            const imported = specifier.imported.name;
+            imports.push({ local, imported, source });
           } else {
             throw new Error("Unsupported import type");
           }
         });
       }
     });
 
-    imports.forEach((imported) => {
-      const { local, source } = imported;
+    imports.forEach((imp) => {
+      const { imported, source } = imp;
       const moduleName = source.split("/").slice(1).join("_");
       const docsPath = path.resolve(__dirname, "docs", "api", moduleName);
       const available = fs.readdirSync(docsPath, { withFileTypes: true });
       const found = available.find(
         (dirent) =>
           dirent.isDirectory() &&
-          fs.existsSync(path.resolve(docsPath, dirent.name, local + ".md"))
+          fs.existsSync(path.resolve(docsPath, dirent.name, imported + ".md"))
       );
       if (found) {
-        imported.docs =
-          "/" + path.join("docs", "api", moduleName, found.name, local);
+        imp.docs =
+          "/" + path.join("docs", "api", moduleName, found.name, imported);
       } else {
         throw new Error(
-          `Could not find docs for ${source}.${local} in docs/api/`
+          `Could not find docs for ${source}.${imported} in docs/api/`
         );
       }
     });

docs/docs/modules/chains/index.mdx

@@ -5,6 +5,7 @@ sidebar_position: 6
 ---
 
 import DocCardList from "@theme/DocCardList";
+import CodeBlock from "@theme/CodeBlock";
 
 # Getting Started: Chains
 
@@ -17,3 +18,23 @@ Using a language model in isolation is fine for some applications, but it is oft
 LangChain provides a standard interface for chains, as well as a number of built-in chains that can be used out of the box. You can also create your own chains.
 
 <DocCardList />
+
+## Advanced
+
+To implement your own custom chain you can subclass `BaseChain` and implement the following methods:
+
+import SubclassInterface from "@examples/chains/advanced_subclass.ts";
+
+<CodeBlock language="typescript">{SubclassInterface}</CodeBlock>
+
+### Subclassing `BaseChain`
+
+The `_call` method is the main method custom chains must implement. It takes a record of inputs and returns a record of outputs. The inputs received should conform the `inputKeys` array, and the outputs returned should conform to the `outputKeys` array.
+
+When implementing this method in a custom chain it's worth paying special attention to the `runManager` argument, which is what allows your custom chains to participate in the same [callbacks system](../../production/callbacks/) as the built-in chains.
+
+If you call into another chain/model/agent inside your custom chain then you should pass it the result of calling `runManager?.getChild()` which will produce a new callback manager scoped to that inner run. An example:
+
+import SubclassCall from "@examples/chains/advanced_subclass_call.ts";
+
+<CodeBlock language="typescript">{SubclassCall}</CodeBlock>

docs/docs/modules/chains/index_related_chains/conversational_retrieval.mdx

@@ -1,7 +1,12 @@
+---
+hide_table_of_contents: true
+sidebar_position: 3
+---
+
 import CodeBlock from "@theme/CodeBlock";
 import ConvoRetrievalQAExample from "@examples/chains/conversational_qa.ts";
 
-# `ConversationalRetrievalQAChain`
+# Conversational Retrieval QA
 
 The `ConversationalRetrievalQA` chain builds on `RetrievalQAChain` to provide a chat history component.
 

docs/docs/modules/chains/index_related_chains/document_qa.mdx

@@ -1,12 +1,13 @@
 ---
 hide_table_of_contents: true
+sidebar_position: 1
 ---
 
 import QAExample from "@examples/chains/question_answering.ts";
 import RefineExample from "@examples/chains/qa_refine.ts";
 import CodeBlock from "@theme/CodeBlock";
 
-# Document QA Chains
+# Document QA
 
 LangChain provides chains used for processing unstructured text data: `StuffDocumentsChain`, `MapReduceDocumentsChain` and `RefineDocumentsChain`.
 These chains are the building blocks more complex chains for processing unstructured text data and receive both documents and a question as input. They then utilize the language model to provide an answer to the question based on the given documents.

docs/docs/modules/chains/index_related_chains/index.mdx

@@ -1,6 +1,7 @@
 ---
 hide_table_of_contents: true
 sidebar_label: Index Related Chains
+sidebar_position: 2
 ---
 
 import DocCardList from "@theme/DocCardList";

docs/docs/modules/chains/index_related_chains/retrieval_qa.mdx

@@ -1,8 +1,13 @@
+---
+hide_table_of_contents: true
+sidebar_position: 2
+---
+
 import RetrievalQAExample from "@examples/chains/retrieval_qa.ts";
 import RetrievalQAExampleCustom from "@examples/chains/retrieval_qa_custom.ts";
 import CodeBlock from "@theme/CodeBlock";
 
-# `RetrievalQAChain`
+# Retrieval QA
 
 The `RetrievalQAChain` is a chain that combines a `Retriever` and a QA chain (described above). It is used to retrieve documents from a `Retriever` and then use a `QA` chain to answer a question based on the retrieved documents.
 

docs/docs/modules/chains/llmchain/index.mdx → docs/docs/modules/chains/llmchain.mdx

@@ -1,6 +1,7 @@
 ---
 hide_table_of_contents: true
 sidebar_label: LLM Chain
+sidebar_position: 1
 ---
 
 import CodeBlock from "@theme/CodeBlock";
@@ -14,7 +15,7 @@ import Example from "@examples/chains/llm_chain.ts";
 
 An `LLMChain` is a simple chain that adds some functionality around language models. It is used widely throughout LangChain, including in other chains and agents.
 
-An `LLMChain` consists of a `PromptTemplate` and a language model (either and LLM or chat model).
+An `LLMChain` consists of a `PromptTemplate` and a language model (either an [LLM](../models/llms/) or [chat model](../models/chat/)).
 
 We can construct an LLMChain which takes user input, formats it with a PromptTemplate, and then passes the formatted response to an LLM:
 

docs/docs/modules/chains/sequential_chain.mdx

@@ -1,6 +1,6 @@
 ---
 hide_table_of_contents: true
-sidebar_label: SequentialChain
+sidebar_position: 3
 ---
 
 import CodeBlock from "@theme/CodeBlock";

docs/docs/production/callbacks/create-handlers.mdx

@@ -0,0 +1,13 @@
+import CodeBlock from "@theme/CodeBlock";
+
+# Creating callback handlers
+
+## Creating a custom handler
+
+You can also create your own handler by implementing the `BaseCallbackHandler` interface. This is useful if you want to do something more complex than just logging to the console, eg. send the events to a logging service. As an example here is a simple implementation of a handler that logs to the console:
+
+import CustomHandlerExample from "@examples/callbacks/custom_handler.ts";
+
+<CodeBlock language="typescript">{CustomHandlerExample}</CodeBlock>
+
+You could then use it as described in the [section](#built-in-handlers) above.

docs/docs/production/callbacks/creating-subclasses.mdx

@@ -0,0 +1,11 @@
+---
+sidebar_label: Callbacks in custom Chains
+---
+
+# Callbacks in custom Chains/Agents
+
+LangChain is designed to be extensible. You can add your own custom Chains and Agents to the library. This page will show you how to add callbacks to your custom Chains and Agents.
+
+## Adding callbacks to custom Chains
+
+When you create a custom chain you can easily set it up to use the same callback system as all the built-in chains. See this guide for more information on how to [create custom chains and use callbacks inside them](../../modules/chains#subclassing-basechain).

docs/docs/production/callbacks/index.mdx

@@ -0,0 +1,55 @@
+import CodeBlock from "@theme/CodeBlock";
+
+# Events / Callbacks
+
+LangChain provides a callback system that allows you to hook into the various stages of your LLM application. This is useful for logging, [monitoring](../tracing), [streaming](../../modules/models/llms/additional_functionality#streaming-responses), and other tasks.
+
+You can subscribe to these events by using the `callbacks` argument available throughout the API. This method accepts a list of handler objects, which are expected to implement one or more of the methods described in the [API docs](../../api/callbacks/interfaces/CallbackHandlerMethods).
+
+## Dive deeper
+
+import DocCardList from "@theme/DocCardList";
+
+<DocCardList />
+
+## How to use callbacks
+
+The `callbacks` argument is available on most objects throughout the API (Chains, Models, Tools, Agents, etc.) in two different places:
+
+- **Constructor callbacks**: defined in the constructor, eg. `new LLMChain({ callbacks: [handler] })`, which will be used for all calls made on that object, and will be scoped to that object only, eg. if you pass a handler to the `LLMChain` constructor, it will not be used by the Model attached to that chain.
+- **Request callbacks**: defined in the `call()`/`run()`/`apply()` methods used for issuing a request, eg. `chain.call({ input: '...' }, [handler])`, which will be used for that specific request only, and all sub-requests that it contains (eg. a call to an LLMChain triggers a call to a Model, which uses the same handler passed in the `call()` method).
+
+The `verbose` argument is available on most objects throughout the API (Chains, Models, Tools, Agents, etc.) as a constructor argument, eg. `new LLMChain({ verbose: true })`, and it is equivalent to passing a `ConsoleCallbackHandler` to the `callbacks` argument of that object and all child objects. This is useful for debugging, as it will log all events to the console.
+
+### When do you want to use each of these?
+
+- Constructor callbacks are most useful for use cases such as logging, monitoring, etc., which are _not specific to a single request_, but rather to the entire chain. For example, if you want to log all the requests made to an LLMChain, you would pass a handler to the constructor.
+- Request callbacks are most useful for use cases such as streaming, where you want to stream the output of a single request to a specific websocket connection, or other similar use cases. For example, if you want to stream the output of a single request to a websocket, you would pass a handler to the `call()` method
+
+## Usage examples
+
+### Built-in handlers
+
+LangChain provides a few built-in handlers that you can use to get started. These are available in the `langchain/callbacks` module. The most basic handler is the `ConsoleCallbackHandler`, which simply logs all events to the console. In the future we will add more default handlers to the library. Note that when the `verbose` flag on the object is set to `true`, the `ConsoleCallbackHandler` will be invoked even without being explicitly passed in.
+
+import ConsoleExample from "@examples/callbacks/console_handler.ts";
+
+<CodeBlock language="typescript">{ConsoleExample}</CodeBlock>
+
+### One-off handlers
+
+You can create a one-off handler inline by passing a plain object to the `callbacks` argument. This object should implement the [`CallbackHandlerMethods`](../../api/callbacks/interfaces/CallbackHandlerMethods) interface. This is useful if eg. you need to create a handler that you will use only for a single request, eg to stream the output of an LLM/Agent/etc to a websocket.
+
+import StreamingExample from "@examples/models/llm/llm_streaming.ts";
+
+<CodeBlock language="typescript">{StreamingExample}</CodeBlock>
+
+### Multiple handlers
+
+We offer a method on the `CallbackManager` class that allows you to create a one-off handler. This is useful if eg. you need to create a handler that you will use only for a single request, eg to stream the output of an LLM/Agent/etc to a websocket.
+
+This is a more complete example that passes a `CallbackManager` to a ChatModel, and LLMChain, a Tool, and an Agent.
+
+import AgentExample from "@examples/agents/streaming.ts";
+
+<CodeBlock language="typescript">{AgentExample}</CodeBlock>

docs/src/theme/CodeBlock/index.js

@@ -20,10 +20,10 @@ function Imports({ imports }) {
         API Reference:
       </h4>
       <ul style={{ paddingBottom: "1rem" }}>
-        {imports.map(({ local, source, docs }) => (
+        {imports.map(({ imported, source, docs }) => (
           <li>
             <a href={docs}>
-              <span>{local}</span>
+              <span>{imported}</span>
             </a>{" "}
             from <code>{source}</code>
           </li>

examples/src/callbacks/console_handler.ts

@@ -9,9 +9,20 @@ export const run = async () => {
   const prompt = PromptTemplate.fromTemplate("1 + {number} =");
   const chain = new LLMChain({ prompt, llm, callbacks: [handler] });
 
-  await chain.call({ number: 2 });
+  const output = await chain.call({ number: 2 });
   /*
   Entering new llm_chain chain...
   Finished chain.
   */
+
+  console.log(output);
+  /*
+  { text: ' 3\n\n3 - 1 = 2' }
+   */
+
+  // The non-enumerable key `__run` contains the runId.
+  console.log(output.__run);
+  /*
+  { runId: '90e1f42c-7cb4-484c-bf7a-70b73ef8e64b' }
+  */
 };

examples/src/callbacks/custom_handler.ts

@@ -0,0 +1,30 @@
+import { BaseCallbackHandler } from "langchain/callbacks";
+import { AgentAction, AgentFinish, ChainValues } from "langchain/schema";
+
+export class MyCallbackHandler extends BaseCallbackHandler {
+  name = "MyCallbackHandler";
+
+  async handleChainStart(chain: { name: string }) {
+    console.log(`Entering new ${chain.name} chain...`);
+  }
+
+  async handleChainEnd(_output: ChainValues) {
+    console.log("Finished chain.");
+  }
+
+  async handleAgentAction(action: AgentAction) {
+    console.log(action.log);
+  }
+
+  async handleToolEnd(output: string) {
+    console.log(output);
+  }
+
+  async handleText(text: string) {
+    console.log(text);
+  }
+
+  async handleAgentEnd(action: AgentFinish) {
+    console.log(action.log);
+  }
+}

examples/src/chains/advanced_subclass.ts

@@ -0,0 +1,31 @@
+import { CallbackManagerForChainRun } from "langchain/callbacks";
+import { BaseChain as _ } from "langchain/chains";
+import { BaseMemory } from "langchain/memory";
+import { ChainValues } from "langchain/schema";
+
+abstract class BaseChain {
+  memory?: BaseMemory;
+
+  /**
+   * Run the core logic of this chain and return the output
+   */
+  abstract _call(
+    values: ChainValues,
+    runManager?: CallbackManagerForChainRun
+  ): Promise<ChainValues>;
+
+  /**
+   * Return the string type key uniquely identifying this class of chain.
+   */
+  abstract _chainType(): string;
+
+  /**
+   * Return the list of input keys this chain expects to receive when called.
+   */
+  abstract get inputKeys(): string[];
+
+  /**
+   * Return the list of output keys this chain will produce when called.
+   */
+  abstract get outputKeys(): string[];
+}