feat: using named arguments to handle better arg parsing in API test …

…deno app

.github/workflows/deploy-stack.yml

@@ -100,3 +100,17 @@ jobs:
     with:
       environment: ${{ inputs.environment }}
     secrets: inherit
+
+  api-test:
+    # We want to ensure that in-progress cron runs against tools-prod
+    # are canceled when we do a deploy so they don't fail erroneously
+    concurrency:
+      group: e2e-${{ inputs.environment }}
+      cancel-in-progress: true
+    needs:
+      - upgrade-web
+      - upgrade-and-migrate-sdf
+    uses: ./.github/workflows/run-api-test.yml
+    with:
+      environment: ${{ inputs.environment }}
+    secrets: inherit

.github/workflows/run-api-test.yml

@@ -61,7 +61,11 @@ jobs:
         run: |
           cd bin/si-api-test
           echo ${{ github.ref }}
-          deno task run ${{ vars.API_TEST_WORKSPACE_ID }} ${{ secrets.API_TEST_EMAIL }} ${{ secrets.API_TEST_PASSWORD }} ${{ matrix.tests }}
+          deno task run \
+          --workspaceId ${{ vars.API_TEST_WORKSPACE_ID }} \
+          --userId ${{ secrets.API_TEST_EMAIL }} \
+          --password ${{ secrets.API_TEST_PASSWORD }} \
+          --tests ${{ matrix.tests }}
 
   on-failure:
     runs-on: ubuntu-latest

bin/si-api-test/README.md

@@ -5,12 +5,31 @@
 Run all tests (with auth api running locally):
 
 ```shell
-SDF_API_URL="http://localhost:8080" AUTH_API_URL="http://localhost:9001" deno task run $WORKSPACE_ID $EMAIL $PASSWORD
+export SDF_API_URL="http://localhost:8080" 
+export AUTH_API_URL="http://localhost:9001" 
+
+deno task run --workspace-id $WORKSPACE_ID \
+              --userId $EMAIL \
+              --password $PASSWORD \
+              --profile '{"Duration": "5", "Requests": 5}'
+              --tests create_and_use_variant,get_head_changeset 
+
+
+Usage: deno run main.ts [options]
+
+Options:
+  --workspaceId, -w   Workspace ID (required)
+  --userId, -u        User ID (required)
+  --password, -p      User password (optional)
+  --tests, -t         Test names to run (comma-separated, optional)
+  --profile, -l       Test profile in JSON format (optional)
+  --help              Show this help message
+
 ```
 
 Alternately, you can skip the password argument, pass in a userId in place of the email and set a jwt private key,
 such as [dev.jwt_signing_private_key.pem](../../config/keys/dev.jwt_signing_private_key.pem) in our config/keys folder,
 to the JWT_PRIVATE_KEY env variable. This is good for local development, but not how we'll do it in GitHub actions.
 
 ## Adding new tests
-
+Add a new file into ./tests/<something>.ts and then invoke it using the --tests param in the binary execution

bin/si-api-test/binary_execution_lib.ts

@@ -0,0 +1,56 @@
+import { parse } from "https://deno.land/[email protected]/flags/mod.ts";
+
+export function parseArgs(args: string[]) {
+  // Parse arguments using std/flags
+  const parsedArgs = parse(args, {
+    string: ["workspaceId", "userId", "password", "profile", "tests"],
+    alias: { w: "workspaceId", u: "userId", p: "password", t: "tests", l: "profile" },
+    default: { profile: undefined, tests: "" },
+    boolean: ["help"],
+  });
+
+  // Display help information if required arguments are missing or help flag is set
+  if (parsedArgs.help || !parsedArgs.workspaceId || !parsedArgs.userId) {
+    console.log(`
+Usage: deno run main.ts [options]
+
+Options:
+  --workspaceId, -w   Workspace ID (required)
+  --userId, -u        User ID (required)
+  --password, -p      User password (optional)
+  --tests, -t         Test names to run (comma-separated, optional)
+  --profile, -l       Test profile in JSON format (optional)
+  --help              Show this help message
+`);
+    Deno.exit(0);
+  }
+
+  // Extract parsed arguments
+  const workspaceId = parsedArgs.workspaceId;
+  const userId = parsedArgs.userId;
+  const password = parsedArgs.password || undefined;
+
+  // Handle optional tests argument
+  const testsToRun = parsedArgs.tests ? parsedArgs.tests.split(",").map(test => test.trim()).filter(test => test) : [];
+
+  // Parse profile JSON if provided, otherwise the profile is one shot [aka single execution]
+  let testProfile = "one-shot";
+  if (parsedArgs.profile) {
+    try {
+      testProfile = JSON.parse(parsedArgs.profile);
+    } catch (error) {
+      throw new Error(`Failed to parse profile JSON: ${error.message}`);
+    }
+  }
+
+  return { workspaceId, userId, password, testsToRun, testProfile };
+}
+
+export function checkEnvironmentVariables(env: Record<string, string | undefined>) {
+  const requiredVars = ["SDF_API_URL", "AUTH_API_URL"];
+  const missingVars = requiredVars.filter(varName => !env[varName] || env[varName]?.length === 0);
+
+  if (missingVars.length > 0) {
+    throw new Error(`Missing environment variables: ${missingVars.join(", ")}`);
+  }
+}

bin/si-api-test/main.ts

@@ -1,12 +1,10 @@
-import assert from "node:assert";
 import { SdfApiClient } from "./sdf_api_client.ts";
 import { createDefaultTestReportEntry, printTestReport, TestReportEntry } from "./test_execution_lib.ts";
-import { checkEnvironmentVariables, parseArgs } from "./binary_exeuction_lib.ts";
+import { checkEnvironmentVariables, parseArgs } from "./binary_execution_lib.ts";
 
 if (import.meta.main) {
-
-  // All environment variable and runtime parsing happens in this block
-  const { workspaceId, userId, password, testsToRun } = parseArgs(Deno.args);
+  // Parse args and check environment variables
+  const { workspaceId, userId, password, testsToRun, testProfile } = parseArgs(Deno.args);
   checkEnvironmentVariables(Deno.env.toObject());
 
   // Init the SDF Module
@@ -23,41 +21,116 @@ if (import.meta.main) {
 
   // If no tests are specified, run all tests by default
   const tests = testsToRun.length > 0 ? testsToRun : Object.keys(testFiles);
+
   const testReport: TestReportEntry[] = [];
+  let completedTests = 0;
+  let triggeredTests = 0;
+  let testExecutionSequence = 1;
+  const totalTests = (testProfile?.Requests ?? 1) * tests.length;
+  const startTime = Date.now();
+  const totalDurationMs = Number(testProfile?.Duration) * 1000;
 
-  for (const testName of tests) {
-    const testPath = testFiles[testName];
+  // Function to display the progress bars
+  function displayProgressBars(triggered: number, completed: number, total: number) {
+    const barLength = 15; // Length of each progress bar
+
+    // Calculate progress for triggered tests
+    const triggeredProgress = Math.min(triggered / total, 1);
+    const triggeredBars = Math.round(triggeredProgress * barLength);
+    const triggeredProgressBar = `[${"=".repeat(triggeredBars)}${" ".repeat(barLength - triggeredBars)}]`;
+    const triggeredPercent = (triggeredProgress * 100).toFixed(2);
+
+    // Calculate progress for completed tests
+    const completedProgress = Math.min(completed / total, 1);
+    const completedBars = Math.round(completedProgress * barLength);
+    const completedProgressBar = `[${"=".repeat(completedBars)}${" ".repeat(barLength - completedBars)}]`;
+    const completedPercent = (completedProgress * 100).toFixed(2);
+
+    // Calculate elapsed and remaining time
+    const elapsed = Date.now() - startTime;
+    const remaining = Math.max(totalDurationMs - elapsed, 0);
+    const remainingSeconds = Math.ceil(remaining / 1000);
+
+    // Log progress bars with total and remaining time
+    console.log(`Triggered: ${triggeredProgressBar} ${triggeredPercent}% (${triggered}/${total}) | Completed: ${completedProgressBar} ${completedPercent}% (${completed}/${total}) | Remaining Time: ${remainingSeconds}s`);
+  }
+
+  // Define the test execution function
+  const executeTest = async (testName: string, sdfApiClient: SdfApiClient, sequence: number, showProgressBar: boolean) => {
     const testEntry = createDefaultTestReportEntry(testName);
+    const testPath = testFiles[testName];
 
-    try {
-      if (testPath) {
-        try {
-          const { default: testFunc } = await import(testPath);
-          let testStart = new Date();
-          await testFunc(sdfApiClient);
-          testEntry.test_result = "success";
-          testEntry.finish_time = new Date().toISOString();
-          testEntry.test_duration = `${new Date().getTime() - testStart.getTime()}ms`;
-        } catch (importError) {
-          testEntry.message = `Failed to load test file "${testPath}": ${importError.message}`;
-        }
-      } else {
-        testEntry.message = `test "${testName}" not found.`;
+    // Display progress bar immediately when the test is triggered (only if showProgressBar is true)
+    if (showProgressBar) {
+      triggeredTests++;
+      displayProgressBars(triggeredTests, completedTests, totalTests);
+    }
+
+    if (!testPath) {
+      testEntry.message = `Test "${testName}" not found.`;
+      testReport.push(testEntry);
+      completedTests++;
+      if (showProgressBar) {
+        displayProgressBars(triggeredTests, completedTests, totalTests);
       }
+      return;
+    }
+
+    try {
+      const { default: testFunc } = await import(testPath);
+      const testStart = new Date();
+      await testFunc(sdfApiClient);
+      testEntry.test_result = "success";
+      testEntry.finish_time = new Date().toISOString();
+      testEntry.test_duration = `${new Date().getTime() - testStart.getTime()}ms`;
     } catch (error) {
-      testEntry.message = error.message;
+      testEntry.message = `Error in test "${testName}": ${error.message}`;
+      testEntry.test_result = "failure";
     } finally {
       testEntry.finish_time = new Date().toISOString();
       testEntry.test_duration = `${new Date().getTime() - new Date(testEntry.start_time).getTime()}ms`;
+      testEntry.test_execution_sequence = sequence;
       testReport.push(testEntry);
+
+      // Update the completed tests count and display progress bars (only if showProgressBar is true)
+      completedTests++;
+      if (showProgressBar) {
+        displayProgressBars(triggeredTests, completedTests, totalTests);
+      }
     }
-  }
+  };
+
+  if (testProfile?.Duration && testProfile?.Requests) {
+    const interval = Math.floor(Number(testProfile.Duration) * 1000 / testProfile.Requests);
+    let elapsed = 0;
+
+    // Create a list of promises for all test executions
+    const testPromises: Promise<void>[] = [];
 
-  // Generate and print the report
-  printTestReport(testReport);
+    // Execute tests based on the profile
+    const intervalId = setInterval(async () => {
+      if (elapsed >= Number(testProfile.Duration) * 1000) {
+        clearInterval(intervalId);
+        // Wait for all test executions to complete
+        await Promise.all(testPromises);
+        console.log("~~ FINAL REPORT GENERATED ~~");
+        printTestReport(testReport);
+        return;
+      }
 
-  // Assert that all tests passed
-  assert(testReport.every(entry => entry.test_result === "success"), "Not all tests passed. Please check the logs for details.");
+      for (const testName of tests) {
+        // Execute tests asynchronously and increment sequence, show progress bar
+        const testPromise = executeTest(testName, sdfApiClient, testExecutionSequence++, true);
+        testPromises.push(testPromise);
+      }
 
-  console.log("~~ ALL TESTS PASSED ~~");
+      elapsed += interval;
+    }, interval);
+  } else {
+    // Fallback to one-shot execution if no profile is set, do not show progress bar
+    const oneShotPromises: Promise<void>[] = tests.map(testName => executeTest(testName, sdfApiClient, testExecutionSequence++, false));
+    await Promise.all(oneShotPromises);
+    console.log("~~ FINAL REPORT GENERATED ~~");
+    printTestReport(testReport);
+  }
 }

bin/si-api-test/sdf_api_client.ts

@@ -156,7 +156,6 @@ export class SdfApiClient {
   }) {
     const url = `${this.baseUrl}${path}`;
     const method = options?.method || "GET";
-    console.log(`calling ${method} ${url}`);
 
     const headers = {
       "Content-Type": "application/json",

bin/si-api-test/test_helpers.ts

@@ -3,20 +3,17 @@ import assert from "node:assert";
 
 export async function sleep(seconds: number) {
   const natural_seconds = Math.max(0, Math.floor(seconds));
-  console.log(`Sleeping for ${natural_seconds} seconds`);
   return new Promise((resolve) => setTimeout(resolve, natural_seconds * 1000));
 }
 
 // Run fn n times, with increasing intervals between tries
-export async function retryWithBackoff(fn: () => Promise<void>, retries = 6, backoffFactor = 3, initialDelay = 2) {
+export async function retryWithBackoff(fn: () => Promise<void>, retries = 6, backoffFactor = 2, initialDelay = 2) {
   let latest_err;
   let try_count = 0;
   let delay = initialDelay;
 
-  console.log("Running retry_with_backoff block");
   do {
     try_count++;
-    console.log(`try number ${try_count}`);
     latest_err = undefined;
 
     try {
@@ -56,7 +53,6 @@ export async function runWithTemporaryChangeset(sdf: SdfApiClient, fn: (sdf: Sdf
   try {
     await fn(sdf, changeSetId);
   } catch (e) {
-    console.log("Function failed, deleting changeset");
     err = e;
   }