Feat: API refinement and API validation support (#114)

Addresses Issue: https://github.com/anti-work/iffy/issues/225 

This PR ships v0.0.8 and completes migrating Iffy from playwright test
to shortest.

### Added
- Added support for playwright's browser and playwright object model 
- Rename test namespace to shortest 
- Added new lifecycle method called .after() that will only run after
the specific test case

- Updated shortest.yml

Author: m2rads

.github/workflows/shortest.yml

@@ -22,6 +22,9 @@ jobs:
       - name: Install dependencies
         run: pnpm install
 
+      - name: Install Playwright browsers
+        run: pnpm exec playwright install
+
       - name: Get Vercel preview URL
         id: vercel_url
         run: |

README.md

@@ -11,11 +11,11 @@ AI-powered natural language end-to-end testing framework.
 
 ## Installation
 ```bash
-npm install -g @antiwork/shortest
+npm install @antiwork/shortest
 # or
-pnpm add -g @antiwork/shortest
+pnpm add @antiwork/shortest
 # or
-yarn add -g @antiwork/shortest
+yarn add @antiwork/shortest
 ```
 
 ### If you installed shortest without `-g` flag, you can run tests as follows:
@@ -43,25 +43,25 @@ export default {
 2. Write your test in your test directory: `app/__tests__/login.test.ts`
 
 ```typescript
-import { test } from '@antiwork/shortest'
+import { shortest } from '@antiwork/shortest'
 
-test('Login to the app using email and password', { username: process.env.GITHUB_USERNAME, password: process.env.GITHUB_PASSWORD })
+shortest('Login to the app using email and password', { username: process.env.GITHUB_USERNAME, password: process.env.GITHUB_PASSWORD })
 ```
 
 ## Using callback functions
 You can also use callback functions to add additoinal assertions and other logic. AI will execute the callback function after the test
 execution in browser is completed.
 
 ```typescript
-import { test } from '@antiwork/shortest';
+import { shortest } from '@antiwork/shortest';
 import { db } from '@/lib/db/drizzle';
 import { users } from '@/lib/db/schema';
 import { eq } from 'drizzle-orm';
 
-test('Login to the app using Github login', {
-  username: process.env.GITHUB_USERNAME,
-  password: process.env.GITHUB_PASSWORD
-}, async ({ page }) => {    
+shortest('Login to the app using username and password', {
+  username: process.env.USERNAME,
+  password: process.env.PASSWORD
+}).after(async ({ page }) => {    
   // Get current user's clerk ID from the page
   const clerkId = await page.evaluate(() => {
     return window.localStorage.getItem('clerk-user');
@@ -86,15 +86,15 @@ test('Login to the app using Github login', {
 You can use lifecycle hooks to run code before and after the test.
 
 ```typescript
-import { test } from '@antiwork/shortest';
+import { shortest } from '@antiwork/shortest';
 
-test.beforeAll(async ({ page }) => {
+shortest.beforeAll(async ({ page }) => {
   await clerkSetup({
     frontendApiUrl: process.env.PLAYWRIGHT_TEST_BASE_URL ?? "http://localhost:3000",
   });
 });
 
-test.beforeEach(async ({ page }) => {
+shortest.beforeEach(async ({ page }) => {
   await clerk.signIn({
     page,
     signInParams: { 
@@ -104,11 +104,11 @@ test.beforeEach(async ({ page }) => {
   });
 });
 
-test.afterEach(async ({ page }) => {
+shortest.afterEach(async ({ page }) => {
   await page.close();
 });
 
-test.afterAll(async ({ page }) => {
+shortest.afterAll(async ({ page }) => {
   await clerk.signOut({ page });
 });
 ```

package.json

@@ -18,6 +18,8 @@
     "prepare": "cd packages/shortest && pnpm build"
   },
   "dependencies": {
+    "@playwright/test": "^1.48.1",
+    "@faker-js/faker": "^8.4.1",
     "@ai-sdk/anthropic": "^0.0.50",
     "@ai-sdk/openai": "^0.0.61",
     "@clerk/nextjs": "^5.6.0",

packages/shortest/CHANGELOG.md

@@ -5,6 +5,15 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.0.8] - 2024-12-16
+
+### Added
+- Added support for playwright's browser and playwright object model 
+- Rename test namespace to shortest 
+- Added new lifecycle method called .after() that will only run after the specific test case
+- Improve system prompt to be more robust and structured
+- Added Windows support for playwright install command
+
 ## [0.0.7] - 2024-12-12
 
 ### Fixed

packages/shortest/README.md

@@ -10,11 +10,11 @@ AI-powered natural language end-to-end testing framework.
 
 ## Installation
 ```bash
-npm install -g @antiwork/shortest
+npm install @antiwork/shortest
 # or
-pnpm add -g @antiwork/shortest
+pnpm add @antiwork/shortest
 # or
-yarn add -g @antiwork/shortest
+yarn add @antiwork/shortest
 ```
 
 ## Quick Start
@@ -34,25 +34,25 @@ export default {
 
 2. Write your test in the test directory: `app/__tests__/login.test.ts`
 ```typescript
-import { test } from '@antiwork/shortest'
+import { shortest } from '@antiwork/shortest'
 
-test('Login to the app using email and password', { username: process.env.GITHUB_USERNAME, password: process.env.GITHUB_PASSWORD })
+shortest('Login to the app using email and password', { username: process.env.GITHUB_USERNAME, password: process.env.GITHUB_PASSWORD })
 ```
 
 ## Using callback functions
 You can also use callback functions to add additoinal assertions and other logic. AI will execute the callback function after the test
 execution in browser is completed.
 
 ```typescript
-import { test } from '@antiwork/shortest';
+import { shortest } from '@antiwork/shortest';
 import { db } from '@/lib/db/drizzle';
 import { users } from '@/lib/db/schema';
 import { eq } from 'drizzle-orm';
 
-test('Login to the app using Github login', {
-  username: process.env.GITHUB_USERNAME,
-  password: process.env.GITHUB_PASSWORD
-}, async ({ page }) => {    
+shortest('Login to the app using username and password', {
+  username: process.env.USERNAME,
+  password: process.env.PASSWORD
+}).after(async ({ page }) => {    
   // Get current user's clerk ID from the page
   const clerkId = await page.evaluate(() => {
     return window.localStorage.getItem('clerk-user');
@@ -77,15 +77,15 @@ test('Login to the app using Github login', {
 You can use lifecycle hooks to run code before and after the test.
 
 ```typescript
-import { test } from '@antiwork/shortest';
+import { shrotest } from '@antiwork/shortest';
 
-test.beforeAll(async ({ page }) => {
+shortest.beforeAll(async ({ page }) => {
   await clerkSetup({
     frontendApiUrl: process.env.PLAYWRIGHT_TEST_BASE_URL ?? "http://localhost:3000",
   });
 });
 
-test.beforeEach(async ({ page }) => {
+shortest.beforeEach(async ({ page }) => {
   await clerk.signIn({
     page,
     signInParams: { 
@@ -95,11 +95,11 @@ test.beforeEach(async ({ page }) => {
   });
 });
 
-test.afterEach(async ({ page }) => {
+shortest.afterEach(async ({ page }) => {
   await page.close();
 });
 
-test.afterAll(async ({ page }) => {
+shortest.afterAll(async ({ page }) => {
   await clerk.signOut({ page });
 });
 ```

packages/shortest/index.d.ts

@@ -1,5 +1,6 @@
 import type { Expect } from 'expect';
-import type { Page } from 'playwright';
+import type { Page, Browser, APIRequest, APIRequestContext } from 'playwright';
+import type * as playwright from 'playwright';
 import type { TestAPI, TestContext } from './dist/types/test';
 import type { ShortestConfig } from './dist/types/config';
 
@@ -10,15 +11,24 @@ declare global {
 declare module '@antiwork/shortest' {
   export type TestContextProps = {
     page: Page;
+    browser: Browser;
+    playwright: typeof playwright & {
+      request: APIRequest & {
+        newContext: (options?: { extraHTTPHeaders?: Record<string, string> }) => Promise<APIRequestContext>;
+      };
+    };
   };
 
   export type TestChain = {
+    expect(fn: (context: TestContextProps) => Promise<void>): TestChain;
     expect(description: string): TestChain;
     expect(description: string, fn?: (context: TestContextProps) => Promise<void>): TestChain;
     expect(description: string, payload?: any, fn?: (context: TestContextProps) => Promise<void>): TestChain;
+    after(fn: (context: TestContextProps) => void | Promise<void>): TestChain;
   };
 
   export type TestAPI = {
+    (fn: (context: TestContextProps) => Promise<void>): void;
     (name: string): TestChain;
     (name: string, fn?: (context: TestContextProps) => Promise<void>): TestChain;
     (name: string, payload?: any, fn?: (context: TestContextProps) => Promise<void>): TestChain;
@@ -36,6 +46,6 @@ declare module '@antiwork/shortest' {
     afterEach(name: string, fn: (context: TestContextProps) => Promise<void>): void;
   };
 
-  export const test: TestAPI;
+  export const shortest: TestAPI;
   export type { TestContext, ShortestConfig };
 } 

packages/shortest/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@antiwork/shortest",
-  "version": "0.0.7",
+  "version": "0.0.8",
   "description": "AI-powered natural language end-to-end testing framework",
   "type": "module",
   "main": "./dist/index.js",

packages/shortest/src/ai/prompts/index.ts

@@ -1,58 +1,50 @@
-export const SYSTEM_PROMPT = `
-You are a test automation expert with access to Chrome browser. When you are given a test case, 
-you will need to execute the browser actions to validate the test cases. 
-You are already in Chrome browser in the web page of the application in test case instructions, 
-so you don't need to load the browser yourself.
-
-This is an example of a test case that you might recieve: 
+export const SYSTEM_PROMPT = `You are a test automation expert working with a Chrome browser. You will be given test instructions, and your task is to execute specified browser actions to validate the provided test cases. You are already in the Chrome browser and on the relevant application page, so there is no need to open or initialize the browser yourself.
 
+EXAMPLE TEST CASE:
+------------------
 Test: "Login to the app using Github login"
 Context: {"username":"[email protected]","password":"password1234"}
-Callback function:  [NO_CALLBACK] 
-
-Expect:
- 1. Test case to be generated within at least 20 seconds [HAS_CALLBACK]
-
-IMPORTANT RULES THAT YOU MUST ALWAYS FOLLOW WHEN EXECUTING TEST CASES:
+Callback function: [NO_CALLBACK]
+Expect: 1. Test case to be generated within at least 20 seconds [HAS_CALLBACK]
+------------------
 
-1. Sometimes you may be instructed to wait for a certain condition to be met before you can continue with the next step.
-That condition might be time in seconds, or minutes. Or it can be for a certain element to be visible, 
-or a certain element to be clickable. Make sure you wait for the condition to be met before you continue with the next step. If the
-condition is not met after the specified time, you should fail the test case.
+IMPORTANT GLOBAL RULES:
 
-2. You might need to use tools api to do some actions. If that's the case, wait until the 
-tool has finished its execution before you continue with the next action. Once the tool 
-has finished its execution, you will recieve the result of the tool execution wether it failed or not. You can decide 
-to continue based on the result. Sometimes you might not understand the result of the tool based on screenshots, therefore you will
-always recieve metadata about the tool execution which will help you understand the result.
+1. **Waiting for Conditions**:
+   - Some steps will require waiting before proceeding to the next action.
+   - This waiting can be based on a time delay (e.g., seconds or minutes) or waiting for an element to become visible or clickable.
+   - If the specified condition is not met after the allotted time, the test should be considered failed.
 
-3. IMPORTANT! DO NOT ask for screenshot until the tool has finished its execution. Once the tool has finished its execution,
-you will recieve the result of the tool execution wether it failed or not.
-Then you can ask for a screenshot to determine for your next action if anything else is needed.
+2. **Tool Usage**:
+   - You may need to use provided tools to perform certain actions (e.g., clicking, navigating, or running callbacks).
+   - After invoking a tool, wait until the tool finishes its execution and you receive a success/failure result.
+   - You will also receive metadata about the tool's execution to help you interpret its outcome.
+   - Only after the tool finishes and you know the result should you request any screenshots or proceed to the next action.
 
-4. If you need to test a login flow with Github 2fa, you need to call the "github_login" tool only after you have 
-seen the github login page. If you call the tool before, it will not work as expected.
+3. **Screenshot Rule**:
+   - Do not request screenshots until after a tool has completely finished its execution.
+   - Once the tool execution result is received, you may then request a screenshot to determine subsequent actions if needed.
 
-5.IMPORTANT! There is a feature provided to you by tools api called "run_callback" that allows you to run callback functions for a test step.
-Whenever you see [HAS_CALLBACK] after the step description, you must call "run_callback" tool. Remember, only 
-call "run_callback" tool after you have completed the browser actions for that step otherwise the callback will not work as expected.
-When done, you can continue with the next step. If result of the callback is failed, you must fail the test case.
+4. **Github Login Flow with 2FA**:
+   - If you need to test a Github login flow that involves 2FA, only call the "github_login" tool after you have confirmed that the Github login page is displayed.
+   - Calling the "github_login" tool prematurely (before the Github login page is visible) will lead to incorrect test behavior.
 
-6. IMPORTANT! ONLY USE THIS TOOL IF YOU ARE SPECIFIED TO NAVIGATE TO A NEW PAGE IN THE TEST CASE INSTRUCTIONS. 
-DO NOT USE THIS TOOL BASED ON YOUR INTUITION! If you need to navigate to a new page, you must use the "navigate" tool. 
-Although you are already in a browser, you do not have access to the browser search bar, therefore, 
-you must use the "navigate" tool to navigate to the new page. After navigating to the new page is done, 
-you will recieve the result of the navigation and you can see if the the requested page is loaded or not from the 
-url field in the metadata.
+5. **Callbacks**:
+   - Steps may include a notation like [HAS_CALLBACK], which means after completing the browser actions for that step, you must call the "run_callback" tool.
 
-7. IMPORTANT! If there is a "Expect" present in the test intruction, you must make sure it is fulfilled. If not, you must fail the test case.
+6. **Navigation Rule**:
+   - Only use the "navigate" tool when explicitly specified in the test case instructions.
+   - Do not use navigation based on intuition - follow test instructions exactly.
+   - You must use the "navigate" tool as you don't have direct access to the browser search bar.
+   - After navigation, verify the requested page is loaded by checking the URL in the metadata.
 
-MUST FOLLOW THIS RULE: perform exactly as instructed in the test case instructions.
+7. **Test Expectations**:
+   - All expectations listed in the test instructions must be fulfilled.
+   - If any expectation is not met, the test case must be marked as failed.
 
 Your task is to:
 1. Execute browser actions to validate test cases
 2. Use provided browser tools to interact with the page
-3. You must return the result of test execution in strict JSON format: { result: "pass" | "fail", reason: string }. 
-for the failure reason, provide a maximum of 1 sentence.
-4. For any click actions, you will need to provide the x,y coordinates of the element to click.
-`;
+3. Return test execution results in strict JSON format: { result: "pass" | "fail", reason: string }
+   For failures, provide a maximum 1-sentence reason.
+4. For click actions, provide x,y coordinates of the element to click.`;

packages/shortest/src/browser/core/browser-tool.ts

@@ -6,7 +6,7 @@ declare global {
   }
 }
 
-import { Page } from 'playwright';
+import { Page, Browser } from 'playwright';
 import { BaseBrowserTool, ToolError } from './index';
 import { ActionInput, ToolResult, BetaToolType } from '../../types/browser';
 import { writeFileSync, mkdirSync, readdirSync, statSync, unlinkSync } from 'fs';
@@ -228,27 +228,21 @@ export class BrowserTool extends BaseBrowserTool {
 
           const testContext = this.testContext;
           const currentTest = testContext.currentTest as TestFunction;
+          
           const currentStepIndex = testContext.currentStepIndex ?? 0;
 
           try {
-            if (currentStepIndex === 0) {
-              if (currentTest.fn) {
-                await currentTest.fn({ page: this.page });
-                testContext.currentStepIndex = 1;
-                return {
-                  output: 'Test function executed successfully'
-                };
-              }
-              return {
-                output: 'Skipping callback execution: No callback function defined for this test'
-              };
+            if (currentStepIndex === 0 && currentTest.fn) {
+              await currentTest.fn(testContext);
+              testContext.currentStepIndex = 1;
+              return { output: 'Test function executed successfully' };
             } else {
               // Handle expectations
               const expectationIndex = currentStepIndex - 1;
               const expectation = currentTest.expectations?.[expectationIndex];
 
               if (expectation?.fn) {
-                await expectation.fn({ page: this.page });
+                await expectation.fn(this.testContext);
                 testContext.currentStepIndex = currentStepIndex + 1;
                 return {
                   output: `Callback function for "${expectation.description}" passed successfully`