Skip to content

Commit

Permalink
standard-tests[minor]: Add a test for more complex schemas (langchain…
Browse files Browse the repository at this point in the history
…-ai#6112)

* standard-tests[minor]: Add a test for more complex schemas

* chore: lint files

* fix some tests

* cr

* chore: lint files
  • Loading branch information
bracesproul authored Jul 25, 2024
1 parent 4f40e5c commit 88e2bca
Showing 1 changed file with 68 additions and 0 deletions.
68 changes: 68 additions & 0 deletions libs/langchain-standard-tests/src/integration_tests/chat_models.ts
Original file line number Diff line number Diff line change
Expand Up @@ -713,6 +713,67 @@ export abstract class ChatModelIntegrationTests<
expect(finalResult.content).not.toBe("");
}

/**
* Tests a more complex tool schema than standard tool tests. This schema
* contains the Zod field: `z.record(z.unknown())` which represents an object
* with unknown/any fields. Some APIs (e.g Google) do not accept JSON schemas
* where the object fields are unknown.
*/
async testInvokeMoreComplexTools() {
if (!this.chatModelHasToolCalling) {
console.log("Test requires tool calling. Skipping...");
return;
}

const model = new this.Cls(this.constructorArgs);
if (!model.bindTools) {
throw new Error(
"bindTools undefined. Cannot test OpenAI formatted tool calls."
);
}

const complexSchema = z.object({
decision: z.enum(["UseAPI", "UseFallback"]),
explanation: z.string(),
apiDetails: z.object({
serviceName: z.string(),
endpointName: z.string(),
parameters: z.record(z.unknown()),
extractionPath: z.string(),
}),
});
const toolName = "service_tool";

const prompt = ChatPromptTemplate.fromMessages([
["system", "You're a helpful assistant. Always use the {toolName} tool."],
[
"human",
`I want to use the UseAPI because it's faster. For the API details use the following:
Service name: {serviceName}
Endpoint name: {endpointName}
Parameters: {parameters}
Extraction path: {extractionPath}`,
],
]);

const modelWithTools = model.withStructuredOutput(complexSchema, {
name: toolName,
});

const result = await prompt.pipe(modelWithTools).invoke({
toolName,
serviceName: "MyService",
endpointName: "MyEndpoint",
parameters: JSON.stringify({ param1: "value1", param2: "value2" }),
extractionPath: "Users/johndoe/data",
});

expect(result.decision).toBeDefined();
expect(result.explanation).toBeDefined();
expect(result.apiDetails).toBeDefined();
expect(typeof result.apiDetails === "object").toBeTruthy();
}

/**
* Run all unit tests for the chat model.
* Each test is wrapped in a try/catch block to prevent the entire test suite from failing.
Expand Down Expand Up @@ -841,6 +902,13 @@ export abstract class ChatModelIntegrationTests<
console.error("testModelCanUseToolUseAIMessageWithStreaming failed", e);
}

try {
await this.testInvokeMoreComplexTools();
} catch (e: any) {
allTestsPassed = false;
console.error("testInvokeMoreComplexTools failed", e);
}

return allTestsPassed;
}
}

0 comments on commit 88e2bca

Please sign in to comment.