Skip to content

Commit

Permalink
standard-tests[minor]: Add test for streaming token counts with tools (
Browse files Browse the repository at this point in the history
  • Loading branch information
bracesproul authored Jul 25, 2024
1 parent e51ea3e commit 4f40e5c
Showing 1 changed file with 44 additions and 0 deletions.
44 changes: 44 additions & 0 deletions libs/langchain-standard-tests/src/integration_tests/chat_models.ts
Original file line number Diff line number Diff line change
Expand Up @@ -523,6 +523,43 @@ export abstract class ChatModelIntegrationTests<
expect(cacheValue2).toEqual(cacheValue);
}

async testStreamTokensWithToolCalls() {
const model = new this.Cls(this.constructorArgs);
if (!model.bindTools) {
throw new Error("bindTools is undefined");
}

const adderTool = new AdderTool();
const modelWithTools = model.bindTools([adderTool]);

const stream = await MATH_ADDITION_PROMPT.pipe(modelWithTools).stream({
toolName: "math_addition",
});
let result: AIMessageChunk | undefined;
for await (const chunk of stream) {
if (!result) {
result = chunk;
} else {
result = result.concat(chunk);
}
}

expect(result).toBeDefined();
if (!result) return;

// Verify a tool was actually called.
expect(result.tool_calls).toHaveLength(1);

// Verify usage metadata is present.
expect(result.usage_metadata).toBeDefined();

// Verify input and output tokens are present.
expect(result.usage_metadata?.input_tokens).toBeDefined();
expect(result.usage_metadata?.input_tokens).toBeGreaterThan(0);
expect(result.usage_metadata?.output_tokens).toBeDefined();
expect(result.usage_metadata?.output_tokens).toBeGreaterThan(0);
}

/**
* This test verifies models can invoke a tool, and use the AIMessage
* with the tool call in a followup request. This is useful when building
Expand Down Expand Up @@ -783,6 +820,13 @@ export abstract class ChatModelIntegrationTests<
console.error("testCacheComplexMessageTypes failed", e);
}

try {
await this.testStreamTokensWithToolCalls();
} catch (e: any) {
allTestsPassed = false;
console.error("testStreamTokensWithToolCalls failed", e);
}

try {
await this.testModelCanUseToolUseAIMessage();
} catch (e: any) {
Expand Down

0 comments on commit 4f40e5c

Please sign in to comment.