Skip to content

Commit 88ef7e7

Browse files
m2radsslavingiaautofix-ci[bot]
authored
Feature: Email Validation with Mailosaur (#183)
This PR addressed issue: #104 ## Version [0.1.2] - 2024-12-26 ### Added - Added support for Mailosaur email validation - Added email rendering feature in the browser - Added sleep_milliseconds tool to add delays in the test execution - Added more robust error handling for Mailosaur email validation Demo: https://github.com/user-attachments/assets/a45ac3ad-27f4-4dcc-825a-a5192d62cee4 --------- Co-authored-by: Sahil Lavingia <[email protected]> Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
1 parent 71f4745 commit 88ef7e7

File tree

14 files changed

+350
-37
lines changed

14 files changed

+350
-37
lines changed

‎.github/workflows/shortest.yml

+2
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@ jobs:
1313
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
1414
CLERK_PUBLISHABLE_KEY: ${{ secrets.CLERK_PUBLISHABLE_KEY }}
1515
CLERK_SECRET_KEY: ${{ secrets.CLERK_SECRET_KEY }}
16+
MAILOSAUR_API_KEY: ${{ secrets.MAILOSAUR_API_KEY }}
17+
MAILOSAUR_SERVER_ID: ${{ secrets.MAILOSAUR_SERVER_ID }}
1618

1719
steps:
1820
- uses: actions/checkout@v4

‎app/__tests__/dashboard.test.ts

+3-18
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,6 @@
11
import { shortest } from "@antiwork/shortest";
2-
import { clerk, clerkSetup } from "@clerk/testing/playwright";
3-
4-
let frontendUrl = process.env.SHORTEST_TEST_BASE_URL ?? "http://localhost:3000";
5-
6-
shortest.beforeAll(async ({ page }) => {
7-
await clerkSetup({
8-
frontendApiUrl: frontendUrl,
9-
});
10-
await clerk.signIn({
11-
page,
12-
signInParams: {
13-
strategy: "email_code",
14-
identifier: "[email protected]",
15-
},
16-
});
17-
18-
await page.goto(frontendUrl + "/dashboard");
19-
});
202

3+
shortest(
4+
"Login to the App using magic link. Use this email: '[email protected]'",
5+
);
216
shortest("Verify that the user can access the /dashboard page");

‎packages/shortest/CHANGELOG.md

+7
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,13 @@ All notable changes to this project will be documented in this file.
55
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
66
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
77

8+
## [0.2.1] - 2024-12-27
9+
10+
## Added
11+
- Mailosaur integration with error handling for email validation
12+
- Browser-based email preview functionality
13+
- Test execution delay utility (sleep_milliseconds)
14+
815
## [0.1.1] - 2024-12-24
916

1017
### Fixed

‎packages/shortest/package.json

+12-10
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "@antiwork/shortest",
3-
"version": "0.1.1",
3+
"version": "0.2.1",
44
"description": "AI-powered natural language end-to-end testing framework",
55
"type": "module",
66
"main": "./dist/index.js",
@@ -29,36 +29,38 @@
2929
"build:types": "tsc --emitDeclarationOnly --outDir dist/types && cp index.d.ts dist/",
3030
"build:js": "esbuild src/index.ts --bundle --platform=node --format=esm --outfile=dist/index.js --external:esbuild --external:punycode --external:playwright --external:@anthropic-ai/sdk --external:expect --external:dotenv",
3131
"build:cjs": "esbuild src/index.ts --bundle --platform=node --format=cjs --outfile=dist/index.cjs --external:esbuild --external:punycode --external:playwright --external:@anthropic-ai/sdk --external:expect --external:dotenv",
32-
"build:cli": "esbuild src/cli/bin.ts src/cli/setup.ts --bundle --platform=node --format=esm --outdir=dist/cli --metafile=dist/meta-cli.json --external:fsevents --external:chokidar --external:glob --external:esbuild --external:events --external:path --external:fs --external:util --external:stream --external:os --external:assert --external:url --external:playwright --external:@anthropic-ai/sdk --external:expect --external:dotenv --external:otplib --external:picocolors --external:punycode",
32+
"build:cli": "esbuild src/cli/bin.ts src/cli/setup.ts --bundle --platform=node --format=esm --outdir=dist/cli --metafile=dist/meta-cli.json --external:fsevents --external:chokidar --external:glob --external:esbuild --external:events --external:path --external:fs --external:util --external:stream --external:os --external:assert --external:url --external:playwright --external:@anthropic-ai/sdk --external:expect --external:dotenv --external:otplib --external:picocolors --external:punycode --external:https --external:http --external:net --external:tls --external:crypto --external:mailosaur",
3333
"dev": "pnpm build:types --watch",
3434
"test:ai": "tsx tests/test-ai.ts",
3535
"test:browser": "tsx tests/test-browser.ts",
3636
"test:coordinates": "tsx tests/test-coordinates.ts",
3737
"test:github": "tsx tests/test-github.ts",
3838
"test:assertion": "tsx tests/test-assertion.ts",
39-
"test:keyboard": "tsx tests/test-keyboard.ts"
39+
"test:keyboard": "tsx tests/test-keyboard.ts",
40+
"test:email": "tsx tests/test-email.ts"
4041
},
4142
"dependencies": {
42-
"glob": "^10.3.10",
4343
"chromium-bidi": "^0.5.2",
44+
"glob": "^10.3.10",
4445
"otplib": "^12.0.1",
4546
"picocolors": "^1.0.0"
4647
},
4748
"devDependencies": {
48-
"tsx": "^4.7.1",
49-
"typescript": "~5.6.2",
5049
"@types/jest": "^29.5.12",
51-
"@types/node": "^20.11.24"
50+
"@types/node": "^20.11.24",
51+
"tsx": "^4.7.1",
52+
"typescript": "~5.6.2"
5253
},
5354
"engines": {
5455
"node": ">=18"
5556
},
5657
"peerDependencies": {
57-
"playwright": "^1.48.2",
58-
"esbuild": "^0.20.1",
5958
"@anthropic-ai/sdk": "0.32.0",
59+
"mailosaur": "^8.7.0",
60+
"dotenv": "^16.4.5",
61+
"esbuild": "^0.20.1",
6062
"expect": "^29.7.0",
61-
"dotenv": "^16.4.5"
63+
"playwright": "^1.48.2"
6264
},
6365
"author": "Antiwork",
6466
"license": "MIT",

‎packages/shortest/src/ai/prompts/index.ts

+10-3
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,8 @@ Expect: 1. Test case to be generated within at least 20 seconds [HAS_CALLBACK]
1111
IMPORTANT GLOBAL RULES:
1212
1313
1. **Waiting for Conditions**:
14-
- Some steps will require waiting before proceeding to the next action.
15-
- This waiting can be based on a time delay (e.g., seconds or minutes) or waiting for an element to become visible or clickable.
16-
- If the specified condition is not met after the allotted time, the test should be considered failed.
14+
- Always wait for the tool to finish before proceeding to the next action. You will recieve a message to continue with your next action once the wait is over. Then validate the condition is met.
15+
- Always wait for the tool to finish before proceeding to the next action. You will receive a message to continue with your next action once the wait is over. Then validate the condition is met.
1716
1817
2. **Tool Usage**:
1918
- You may need to use provided tools to perform certain actions (e.g., clicking, navigating, or running callbacks).
@@ -42,6 +41,14 @@ IMPORTANT GLOBAL RULES:
4241
- All expectations listed in the test instructions must be fulfilled.
4342
- If any expectation is not met, the test case must be marked as failed.
4443
44+
8. **Testing Email**:
45+
- If you need to test a condition that involves seeing the contents of an email, use the "check_email" tool.
46+
- For email validation, you MUST always use 'Click' and 'Mouse' action instead of using keyboard shortcuts.
47+
- This tool will grab the latest email from the email address given to you and will render it in a new tab for you to see.
48+
- Once you are done with validating the email, navigate back to the original tab.
49+
- You MUST pass the email address that is given to you to the tool as a parameter otherwise it will fail.
50+
- If no email address is given to you for this test, you should fail the test.
51+
4552
Your task is to:
4653
1. Execute browser actions to validate test cases
4754
2. Use provided browser tools to interact with the page

‎packages/shortest/src/ai/tools/index.ts

+38
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,44 @@ export const AITools = [
3030
required: ["action", "username", "password"],
3131
},
3232
},
33+
{
34+
name: "check_email",
35+
description: "View received email in new browser tab",
36+
input_schema: {
37+
type: "object",
38+
properties: {
39+
action: {
40+
type: "string",
41+
enum: ["check_email"],
42+
description:
43+
"Check that the email was received with specified content in a new tab",
44+
},
45+
},
46+
required: ["action", "email"],
47+
},
48+
},
49+
{
50+
name: "sleep",
51+
description: "Pause test execution for specified duration",
52+
input_schema: {
53+
type: "object",
54+
properties: {
55+
action: {
56+
type: "string",
57+
enum: ["sleep"],
58+
description: "The action to perform",
59+
},
60+
duration: {
61+
type: "number",
62+
description:
63+
"Duration to sleep in milliseconds (e.g. 5000 for 5 seconds)",
64+
minimum: 0,
65+
maximum: 60000,
66+
},
67+
},
68+
required: ["action", "duration"],
69+
},
70+
},
3371
{
3472
name: "run_callback",
3573
description: "Run callback function for current test step",

‎packages/shortest/src/browser/core/browser-tool.ts

+117-2
Original file line numberDiff line numberDiff line change
@@ -17,12 +17,19 @@ import {
1717
import { join } from "path";
1818
import pc from "picocolors";
1919
import { Page } from "playwright";
20-
import { TestContext, BrowserToolConfig, TestFunction } from "../../types";
20+
import { initialize, getConfig } from "../../index";
21+
import {
22+
TestContext,
23+
BrowserToolConfig,
24+
TestFunction,
25+
ShortestConfig,
26+
} from "../../types";
2127
import { ActionInput, ToolResult, BetaToolType } from "../../types/browser";
2228
import { CallbackError } from "../../types/test";
2329
import { AssertionCallbackError } from "../../types/test";
2430
import * as actions from "../actions";
2531
import { GitHubTool } from "../integrations/github";
32+
import { MailosaurTool } from "../integrations/mailosaur";
2633
import { BrowserManager } from "../manager";
2734
import { BaseBrowserTool, ToolError } from "./index";
2835

@@ -39,6 +46,8 @@ export class BrowserTool extends BaseBrowserTool {
3946
private testContext?: TestContext;
4047
private readonly MAX_SCREENSHOTS = 10;
4148
private readonly MAX_AGE_HOURS = 5;
49+
private mailosaurTool?: MailosaurTool;
50+
private config!: ShortestConfig;
4251

4352
constructor(
4453
page: Page,
@@ -58,7 +67,9 @@ export class BrowserTool extends BaseBrowserTool {
5867
}
5968

6069
private async initialize(): Promise<void> {
61-
// Initial setup with retry
70+
await initialize();
71+
this.config = getConfig();
72+
6273
const initWithRetry = async () => {
6374
for (let i = 0; i < 3; i++) {
6475
try {
@@ -422,6 +433,110 @@ export class BrowserTool extends BaseBrowserTool {
422433
}
423434
}
424435

436+
case "sleep": {
437+
const defaultDuration = 1000;
438+
const maxDuration = 60000;
439+
let duration = input.duration ?? defaultDuration;
440+
441+
// Enforce maximum duration
442+
if (duration > maxDuration) {
443+
console.warn(
444+
`Requested sleep duration ${duration}ms exceeds maximum of ${maxDuration}ms. Using maximum.`,
445+
);
446+
duration = maxDuration;
447+
}
448+
449+
// Convert to seconds for logging
450+
const seconds = Math.round(duration / 1000);
451+
console.log(
452+
`⏳ Waiting for ${seconds} second${seconds !== 1 ? "s" : ""}...`,
453+
);
454+
455+
await this.page.waitForTimeout(duration);
456+
output = `Finished waiting for ${seconds} second${seconds !== 1 ? "s" : ""}`;
457+
break;
458+
}
459+
460+
case "check_email": {
461+
if (!this.mailosaurTool) {
462+
if (!this.config.mailosaur) {
463+
throw new ToolError("Mailosaur configuration required");
464+
}
465+
this.mailosaurTool = new MailosaurTool({
466+
apiKey: this.config.mailosaur.apiKey,
467+
serverId: this.config.mailosaur.serverId,
468+
emailAddress: input.email,
469+
});
470+
}
471+
472+
const newPage = await this.page.context().newPage();
473+
474+
try {
475+
const email = await this.mailosaurTool.getLatestEmail();
476+
477+
// Render email in new tab
478+
await newPage.setContent(email.html, {
479+
waitUntil: "domcontentloaded",
480+
});
481+
482+
await newPage
483+
.waitForLoadState("load", {
484+
timeout: 5000,
485+
})
486+
.catch((error) => {
487+
console.log("⚠️ Load timeout, continuing anyway", error);
488+
});
489+
490+
// Switch focus
491+
this.page = newPage;
492+
493+
output = `Email received successfully. Navigated to new tab to display email: ${email.subject}`;
494+
metadata = {
495+
window_info: {
496+
title: email.subject,
497+
content: email.html,
498+
size: this.page.viewportSize() || {
499+
width: this.width,
500+
height: this.height,
501+
},
502+
},
503+
};
504+
505+
break;
506+
} catch (error: unknown) {
507+
await newPage.close();
508+
const errorMessage =
509+
error instanceof Error ? error.message : String(error);
510+
511+
if (errorMessage.includes("Email content missing")) {
512+
return {
513+
output: `Email was found but content is missing. This might be due to malformed email. Moving to next test.`,
514+
error: "EMAIL_CONTENT_MISSING",
515+
};
516+
}
517+
518+
if (errorMessage.includes("Mailosaur email address is required")) {
519+
return {
520+
output: `Email address is required but was not provided.`,
521+
error: "EMAIL_ADDRESS_MISSING",
522+
};
523+
}
524+
525+
if (errorMessage.includes("No matching messages found")) {
526+
return {
527+
output: `No email found for ${input.email}. The email might not have been sent yet or is older than 1 hour. Moving to next test.`,
528+
error: "EMAIL_NOT_FOUND",
529+
};
530+
}
531+
532+
// Generic error case
533+
return {
534+
output: `Failed to fetch or render email: ${errorMessage}. Moving to next test.`,
535+
error: "EMAIL_OPERATION_FAILED",
536+
};
537+
}
538+
}
539+
425540
default:
426541
throw new ToolError(`Unknown action: ${input.action}`);
427542
}

‎packages/shortest/src/browser/integrations/index.ts

+2-3
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,2 @@
1-
import { GitHubTool } from "./github";
2-
3-
export { GitHubTool };
1+
export { GitHubTool } from "./github";
2+
export { MailosaurTool } from "./mailosaur";

‎packages/shortest/src/browser/integrations/mailosaur.ts

+44
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
import Mailosaur from "mailosaur";
2+
import { ToolError } from "../core";
3+
4+
export class MailosaurTool {
5+
private client: Mailosaur;
6+
private serverId: string;
7+
private emailAddress: string;
8+
9+
constructor(config: {
10+
apiKey: string;
11+
serverId: string;
12+
emailAddress?: string;
13+
}) {
14+
if (!config.apiKey || !config.serverId) {
15+
throw new ToolError("Mailosaur configuration missing required fields");
16+
} else if (!config.emailAddress) {
17+
throw new ToolError("Mailosaur email address is required");
18+
}
19+
20+
this.client = new Mailosaur(config.apiKey);
21+
this.serverId = config.serverId;
22+
this.emailAddress = config.emailAddress;
23+
}
24+
25+
async getLatestEmail() {
26+
try {
27+
const message = await this.client.messages.get(this.serverId, {
28+
sentTo: this.emailAddress,
29+
});
30+
31+
if (!message.html?.body || !message.text?.body) {
32+
throw new ToolError("Email content missing");
33+
}
34+
35+
return {
36+
subject: message.subject || "No Subject",
37+
html: message.html.body,
38+
text: message.text.body,
39+
};
40+
} catch (error) {
41+
throw new ToolError(`Failed to fetch email: ${error}`);
42+
}
43+
}
44+
}

‎packages/shortest/src/types/browser.ts

+5-1
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,9 @@ export type BrowserAction =
2727
| "type"
2828
| "key"
2929
| "run_callback"
30-
| "navigate";
30+
| "navigate"
31+
| "sleep"
32+
| "check_email";
3133

3234
export interface BrowserToolOptions {
3335
width: number;
@@ -43,6 +45,8 @@ export interface ActionInput {
4345
username?: string;
4446
password?: string;
4547
url?: string;
48+
duration?: number;
49+
email?: string;
4650
}
4751

4852
export interface ToolResult {

0 commit comments

Comments
 (0)