send playwright tests results (#4784)

.changeset/eleven-mirrors-guess.md

@@ -0,0 +1,5 @@
+---
+"saleor-dashboard": patch
+---
+
+Add results to testmo, add notification on slack after tests run

.github/actions/run-pw-tests/action.yml

@@ -0,0 +1,63 @@
+name: Run Playwright tests
+description: Run e2e tests using Playwright
+inputs:
+  BASE_URL:
+    description: "Dashboard base url"
+    required: true
+  API_URL:
+    description: "API url"
+    required: true
+  E2E_USER_NAME:
+    description: "Username for e2e tests"
+    required: true
+  E2E_USER_PASSWORD:
+    description: "Password for e2e tests"
+    required: true
+  E2E_PERMISSIONS_USERS_PASSWORD:
+    description: "Permissions user password for e2e tests"
+    required: true
+  SHARD:
+    description: "Shard number"
+    required: true
+  MAILPITURL:
+    description: "mailpit uri"
+    required: true
+  URL_TO_RUN: 
+    description: "Url which will be passed to testmo where can be found artifacts of the run"
+    required: false
+
+runs:
+  using: "composite"
+  steps:
+    - uses: actions/setup-node@v3
+      with:
+        node-version-file: ".nvmrc"
+
+    - name: Install dependencies
+      shell: bash
+      run: npm ci
+
+    - name: Install Playwright Browsers
+      shell: bash
+      run: npx playwright install --with-deps
+
+    - name: Run tests
+      shell: bash
+      env:
+        API_URI: ${{ inputs.API_URL }}
+        BASE_URL: ${{ inputs.BASE_URL }}
+        E2E_USER_NAME: ${{ inputs.E2E_USER_NAME }}
+        E2E_USER_PASSWORD: ${{ inputs.E2E_USER_PASSWORD }}
+        E2E_PERMISSIONS_USERS_PASSWORD: ${{ inputs.E2E_PERMISSIONS_USERS_PASSWORD }}
+        SHARD_NUMBER: ${{ inputs.SHARD }}
+        MAILPITURL: ${{ inputs.MAILPITURL }} 
+        URL_TO_RUN: ${{ inputs.URL_TO_RUN }} 
+      run: npm run qa:pw-e2e -- --shard "$SHARD_NUMBER"
+
+    - name: Upload blob report to GitHub Actions Artifacts
+      uses: actions/upload-artifact@v3
+      if: always()
+      with:
+        name: all-blob-reports
+        path: blob-report
+        retention-days: 1

.github/actions/testmo/testmo-init/action.yml

@@ -7,6 +7,10 @@ inputs:
   testmoToken:
     description: "Testmo token"
     required: true
+  testmoRunName:
+    description: "Displayed name in testmo"
+    required: true
+
 outputs:
   testmo-run-id:
     description: "Testmo run id"
@@ -29,10 +33,11 @@ runs:
         ID=$(npx testmo automation:run:create \
           --instance "$TESTMO_URL" \
           --project-id 1 \
-          --name "Deployment tests" \
+          --name "$RUN_NAME" \
           --source frontend-e2e-tests)
         echo "TESTMO_RUN_ID=$ID" >> $GITHUB_OUTPUT
       env:
         TESTMO_URL: ${{ inputs.testmoUrl }}
         TESTMO_TOKEN: ${{ inputs.testmoToken }}
+        RUN_NAME: ${{inputs.testmoRunName}}
       id: run-tests

.github/actions/testmo/testmo-threads-submit-playwright/action.yml

@@ -0,0 +1,57 @@
+name: submit-run-results-to-testmo
+description: "Sends tests results from run to testmo"
+inputs:
+  testmoUrl:
+    description: "Testmo project URL"
+    required: true
+  testmoToken:
+    description: "Testmo token"
+    required: true
+  testmoRunId:
+    description: "Parallelized job Testmo run id"
+    required: true
+runs:
+  using: composite
+  steps:
+    - uses: actions/setup-node@v3
+      with:
+        node-version-file: ".nvmrc"
+        cache: npm
+    - name: Cache node modules
+      uses: actions/cache@v3
+      env:
+        cache-name: cache-node-modules
+      with:
+        path: ~/.npm
+        key: ${{ runner.os }}-qa-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }}
+        restore-keys: |
+          ${{ runner.os }}-qa-${{ env.cache-name }}-
+          ${{ runner.os }}-qa-
+          ${{ runner.os }}-
+
+    - name: Install Dependencies
+      if: steps.cache-node-modules.outputs.cache-hit != 'true'
+      shell: bash
+      run: NODE_OPTIONS=--max_old_space_size=4096 npm install
+
+    - name: Install dependencies
+      if: ${{ ! cancelled() }}
+      working-directory: .github/workflows
+      shell: bash
+      run: npm ci
+
+    - name: Submit results
+      working-directory: .github/workflows
+      shell: bash
+      run: |
+          npx testmo automation:run:submit-thread \
+            --instance "$TESTMO_URL" \
+            --run-id "$TESTMO_RUN_ID" \
+            --results ../../testmo/testmo.xml 
+      env:
+        TESTMO_URL: ${{ inputs.testmoUrl }}
+        TESTMO_TOKEN: ${{ inputs.testmoToken }}
+        TESTMO_RUN_ID: ${{ inputs.testmoRunId}}
+
+
+

.github/workflows/postTestsResults.js

@@ -0,0 +1,118 @@
+const { Command } = require("commander");
+const program = new Command();
+const { Octokit } = require("@octokit/core");
+
+program
+  .name("Send tests results")
+  .description(
+    "Get tests results from testmo and post message to slack or on release PR",
+  )
+  .option("--run_id <run_id>", "Testmo run id")
+  .option(
+    "--testmo_token <testmo_token>",
+    "Bearer token for authorization in testmo",
+  )
+  .option(
+    "--slack_webhook_url <slack_webhook_url>",
+    "Should send notification on slack",
+  )
+  .option("--environment <environment>", "Environment")
+  .option("--url_to_action <url_to_action>", "Url to enter github action")
+  .option("--ref_name <ref_name>", "Ref to point where tests where run")
+  .action(async options => {
+    const runId = options.run_id;
+    const testmoAuthToken = options.testmo_token;
+    const testsResults = await getTestsStatus(runId, testmoAuthToken);
+    const testsStatus = convertResults(
+      testsResults,
+      options.environment,
+      options.ref_name,
+    );
+
+    await sendMessageOnSlack(
+      testsStatus,
+      options.slack_webhook_url,
+      options.url_to_action,
+    );
+  })
+  .parse();
+
+async function getTestsStatus(runId, testmoToken) {
+  const runResult = await fetch(
+    `https://saleor.testmo.net/api/v1/automation/runs/${runId}`,
+    {
+      headers: {
+        Authorization: `Bearer ${testmoToken}`,
+      },
+    },
+  );
+  return await runResult.json();
+}
+
+function convertResults(results, environment, refName) {
+  let status = results?.result?.status === 2 ? "SUCCESS" : "FAILURE";
+  let message = `Tests run on environment: \n${environment}\n`;
+  const linkToResults = `https:\/\/saleor.testmo.net\/automation\/runs\/view\/${results.result.id}`;
+  const threads = results.result.threads;
+
+  if (Array.isArray(threads)) {
+    const failureCount = threads
+      .map(thread => thread.failure_count)
+      .reduce((accumulator, currentValue) => accumulator + currentValue, 0);
+
+    const successCount = threads
+      .map(thread => thread.success_count)
+      .reduce((accumulator, currentValue) => accumulator + currentValue, 0);
+
+    const skippedCount = threads
+      .map(thread => thread.total_count - thread.completed_count)
+      .reduce((accumulator, currentValue) => accumulator + currentValue, 0);
+
+    if (failureCount > 0) {
+      message += `${failureCount} tests failed. `;
+    }
+    if (successCount > 0) {
+      message += `${successCount} tests passed. `;
+    }
+    if (skippedCount > 0) {
+      message += `${skippedCount} tests skipped. `;
+    }
+  } else {
+    status = "FAILURE";
+    message = "Empty test run. ";
+  }
+
+  message += `See results at ${linkToResults}`;
+
+  return {
+    status,
+    message,
+    title: `Playwright tests run on ${refName}`,
+    linkToResults,
+  };
+}
+
+async function sendMessageOnSlack(testsStatus, webhookUrl, urlToAction) {
+  const JOB_STATUS_COLOR_MAP = {
+    SUCCESS: "#5DC292",
+    FAILURE: "#FE6E76",
+  };
+
+  const messageData = {
+    attachments: [
+      {
+        fallback: testsStatus.message,
+        pretext: testsStatus.status,
+        title: testsStatus.title,
+        title_link: urlToAction,
+        text: testsStatus.message,
+        color: JOB_STATUS_COLOR_MAP[testsStatus.status],
+      },
+    ],
+  };
+  await fetch(webhookUrl, {
+    body: JSON.stringify(messageData),
+    method: "post",
+    headers: { "content-type": "application/json" },
+  });
+}

.github/workflows/pr-automation.yml

@@ -335,3 +335,5 @@ jobs:
           name: html-report--attempt-${{ github.run_attempt }}
           path: playwright-report
           retention-days: 14
+      - name: Merge playwright reports
+        uses: ./.github/actions/merge-pw-reports