forked from langchain-ai/langchainjs
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: update StructuredOutputParser and CombiningOutputParser prompts…
… for increased reliability, add integration tests (langchain-ai#1109) * feat: integration tests for StructuredOutputParser * Update StructuredOutputParser prompt to improve reliability * Fix test * Fix combining output parser and add integration test for it, other tests --------- Co-authored-by: Jacob Lee <[email protected]>
- Loading branch information
1 parent
caa0203
commit ca8302f
Showing
6 changed files
with
270 additions
and
40 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,46 @@ | ||
| import { test } from "@jest/globals"; | ||
|
|
||
| import { OpenAI } from "../../llms/openai.js"; | ||
| import { PromptTemplate } from "../../prompts/index.js"; | ||
| import { | ||
| StructuredOutputParser, | ||
| RegexParser, | ||
| CombiningOutputParser, | ||
| } from "../index.js"; | ||
|
|
||
| test("CombiningOutputParser", async () => { | ||
| const answerParser = StructuredOutputParser.fromNamesAndDescriptions({ | ||
| answer: "answer to the user's question", | ||
| source: "source used to answer the user's question, should be a website.", | ||
| }); | ||
|
|
||
| const confidenceParser = new RegexParser( | ||
| /Confidence: (A|B|C), Explanation: (.*)/, | ||
| ["confidence", "explanation"], | ||
| "noConfidence" | ||
| ); | ||
|
|
||
| const parser = new CombiningOutputParser(answerParser, confidenceParser); | ||
| const formatInstructions = parser.getFormatInstructions(); | ||
|
|
||
| const prompt = new PromptTemplate({ | ||
| template: | ||
| "Answer the users question as best as possible.\n{format_instructions}\n{question}", | ||
| inputVariables: ["question"], | ||
| partialVariables: { format_instructions: formatInstructions }, | ||
| }); | ||
|
|
||
| const model = new OpenAI({ temperature: 0 }); | ||
|
|
||
| const input = await prompt.format({ | ||
| question: "What is the capital of France?", | ||
| }); | ||
|
|
||
| console.log(input); | ||
|
|
||
| const response = await model.call(input); | ||
|
|
||
| console.log(response); | ||
|
|
||
| console.log(await parser.parse(response)); | ||
| }); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
140 changes: 140 additions & 0 deletions
140
langchain/src/output_parsers/tests/structured.int.test.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,140 @@ | ||
| import { expect, test } from "@jest/globals"; | ||
|
|
||
| import { StructuredOutputParser } from "../structured.js"; | ||
| import { OpenAI } from "../../llms/openai.js"; | ||
| import { ChatOpenAI } from "../../chat_models/openai.js"; | ||
| import { LLMChain } from "../../chains/index.js"; | ||
| import { | ||
| ChatPromptTemplate, | ||
| PromptTemplate, | ||
| SystemMessagePromptTemplate, | ||
| } from "../../prompts/index.js"; | ||
|
|
||
| test("StructuredOutputParser deals special chars in prompt with llm model", async () => { | ||
| const model = new OpenAI({ | ||
| temperature: 0, | ||
| }); | ||
|
|
||
| const parser = StructuredOutputParser.fromNamesAndDescriptions({ | ||
| question1: "a very on-topic question", | ||
| question2: "a super weird question", | ||
| question3: "an on-topic, but slightly creative", | ||
| }); | ||
|
|
||
| const prompt = new PromptTemplate({ | ||
| template: "context:\n{context}\n---{format_instructions}", | ||
| inputVariables: ["context"], | ||
| partialVariables: { | ||
| format_instructions: parser.getFormatInstructions(), | ||
| }, | ||
| }); | ||
|
|
||
| const chain = new LLMChain({ | ||
| llm: model, | ||
| prompt, | ||
| outputParser: parser, | ||
| outputKey: "questions", | ||
| }); | ||
|
|
||
| const result = await chain.call({ | ||
| context: `The U2 ur-myth begins in 1976, when drummer Larry Mullen wanted to form a band. | ||
| He picked four school friends from Mount Temple Comprehensive School in Dublin. | ||
| “Larry formed U2,” says Paul McGuinness, U2’s manager from the beginning. “He | ||
| auditioned the other three and he chose them. The first name of U2 was the Larry | ||
| Mullen band,” McGuinness laughs. “And he never lets us forget it.” `, | ||
| }); | ||
|
|
||
| console.log("response", result); | ||
|
|
||
| expect(result.questions).toHaveProperty("question1"); | ||
| expect(result.questions).toHaveProperty("question2"); | ||
| expect(result.questions).toHaveProperty("question3"); | ||
| }); | ||
|
|
||
| test("StructuredOutputParser deals special chars in prompt with chat model", async () => { | ||
| const model = new ChatOpenAI({ | ||
| temperature: 0, | ||
| }); | ||
|
|
||
| const parser = StructuredOutputParser.fromNamesAndDescriptions({ | ||
| question1: "a very on-topic question", | ||
| question2: "a super weird question", | ||
| question3: "an on-topic, but slightly creative", | ||
| }); | ||
|
|
||
| const prompt = new ChatPromptTemplate({ | ||
| promptMessages: [ | ||
| SystemMessagePromptTemplate.fromTemplate("context:\n{context}\n---"), | ||
| SystemMessagePromptTemplate.fromTemplate(`{format_instructions}`), | ||
| ], | ||
| inputVariables: ["context"], | ||
| partialVariables: { | ||
| format_instructions: parser.getFormatInstructions(), | ||
| }, | ||
| }); | ||
|
|
||
| const chain = new LLMChain({ | ||
| llm: model, | ||
| prompt, | ||
| outputParser: parser, | ||
| outputKey: "questions", | ||
| }); | ||
|
|
||
| const result = await chain.call({ | ||
| context: `The U2 ur-myth begins in 1976, when drummer Larry Mullen wanted to form a band. | ||
| He picked four school friends from Mount Temple Comprehensive School in Dublin. | ||
| “Larry formed U2,” says Paul McGuinness, U2’s manager from the beginning. “He | ||
| auditioned the other three and he chose them. The first name of U2 was the Larry | ||
| Mullen band,” McGuinness laughs. “And he never lets us forget it.” `, | ||
| }); | ||
|
|
||
| console.log("response", result); | ||
|
|
||
| expect(result.questions).toHaveProperty("question1"); | ||
| expect(result.questions).toHaveProperty("question2"); | ||
| expect(result.questions).toHaveProperty("question3"); | ||
| }); | ||
|
|
||
| test("StructuredOutputParser deals special chars in prompt with chat model 2", async () => { | ||
| const model = new ChatOpenAI({ | ||
| temperature: 0, | ||
| }); | ||
|
|
||
| const parser = StructuredOutputParser.fromNamesAndDescriptions({ | ||
| question1: "a very on-topic question", | ||
| question2: "a super weird question", | ||
| question3: "an on-topic, but slightly creative", | ||
| }); | ||
|
|
||
| const prompt = new ChatPromptTemplate({ | ||
| promptMessages: [ | ||
| SystemMessagePromptTemplate.fromTemplate("context:\n{context}\n---"), | ||
| SystemMessagePromptTemplate.fromTemplate(`{format_instructions}`), | ||
| ], | ||
| inputVariables: ["context"], | ||
| partialVariables: { | ||
| format_instructions: parser.getFormatInstructions(), | ||
| }, | ||
| }); | ||
|
|
||
| const chain = new LLMChain({ | ||
| llm: model, | ||
| prompt, | ||
| outputKey: "questions", | ||
| }); | ||
|
|
||
| const result = await chain.call({ | ||
| context: `The U2 ur-myth begins in 1976, when drummer Larry Mullen wanted to form a band. | ||
| He picked four school friends from Mount Temple Comprehensive School in Dublin. | ||
| “Larry formed U2,” says Paul McGuinness, U2’s manager from the beginning. “He | ||
| auditioned the other three and he chose them. The first name of U2 was the Larry | ||
| Mullen band,” McGuinness laughs. “And he never lets us forget it.” `, | ||
| }); | ||
|
|
||
| console.log("response", result); | ||
| const parsed = await parser.parse(result.questions); | ||
|
|
||
| expect(parsed).toHaveProperty("question1"); | ||
| expect(parsed).toHaveProperty("question2"); | ||
| expect(parsed).toHaveProperty("question3"); | ||
| }); |
Oops, something went wrong.