Add traces to integration tests + suppress one flaky assertion. (#11595)

Fixes #11604

Most issues were caused by a problem with Project List flooding the network with its requests - this was fixed on develop.
But one assertion was flaky - it assumed we will see the "real" run result on `write` node, but sometimes it is immediately overwritten by dry run.

But the most important part of this PR is adding traces to Electron packages - it's should be much easier now to debug E2E test failures.

Also renamed the previously misnamed "E2E tests" to "[GUI] integration tests".

.github/CODEOWNERS

@@ -31,8 +31,8 @@ tsconfig.json
 
 # GUI/Dashboard
 /app @Frizi @farmaazon @vitvakatu @kazcw @AdRiley @PabloBuchu @indiv0 @somebody1234 @MrFlashAccount
-/app/gui/e2e/dashboard @PabloBuchu @indiv0 @somebody1234 @MrFlashAccount
-/app/gui/e2e/project-view @Frizi @farmaazon @vitvakatu @kazcw @AdRiley 
+/app/gui/integration-test/dashboard @PabloBuchu @indiv0 @somebody1234 @MrFlashAccount
+/app/gui/integration-test/project-view @Frizi @farmaazon @vitvakatu @kazcw @AdRiley 
 /app/gui/src/dashboard @PabloBuchu @indiv0 @somebody1234 @MrFlashAccount
 /app/gui/src/project-view @Frizi @farmaazon @vitvakatu @kazcw @AdRiley 
 /app/ide-desktop/ @PabloBuchu @indiv0 @somebody1234 @MrFlashAccount

.github/workflows/gui-checks.yml

@@ -70,7 +70,7 @@ jobs:
       - name: 🧪 Unit Tests
         id: unit-tests
         continue-on-error: true
-        run: pnpm run ci:test
+        run: pnpm run ci:unit-test
 
       - name: 📝 Annotate Code Linting Results
         if: always()
@@ -146,7 +146,7 @@ jobs:
 
       - name: 🎭 Playwright Tests
         working-directory: app/gui
-        run: pnpm run e2e --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
+        run: pnpm run test:integration --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
 
       - name: ⬆️ Upload blob report to GitHub Actions Artifacts
         if: ${{ !cancelled() }}

.github/workflows/gui.yml

@@ -372,6 +372,13 @@ jobs:
           ENSO_TEST_USER: ${{ secrets.ENSO_CLOUD_TEST_ACCOUNT_USERNAME }}
           ENSO_TEST_USER_PASSWORD: ${{ secrets.ENSO_CLOUD_TEST_ACCOUNT_PASSWORD }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - if: failure()
+        name: Upload Test Traces
+        uses: actions/upload-artifact@v4
+        with:
+          compression-level: 0
+          name: test-traces-linux-amd64
+          path: app/ide-desktop/client/test-traces
       - run: rm $HOME/.enso/credentials
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -444,6 +451,13 @@ jobs:
           ENSO_TEST_USER: ${{ secrets.ENSO_CLOUD_TEST_ACCOUNT_USERNAME }}
           ENSO_TEST_USER_PASSWORD: ${{ secrets.ENSO_CLOUD_TEST_ACCOUNT_PASSWORD }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - if: failure()
+        name: Upload Test Traces
+        uses: actions/upload-artifact@v4
+        with:
+          compression-level: 0
+          name: test-traces-macos-amd64
+          path: app/ide-desktop/client/test-traces
       - run: rm $HOME/.enso/credentials
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -512,7 +526,13 @@ jobs:
           ENSO_TEST_USER: ${{ secrets.ENSO_CLOUD_TEST_ACCOUNT_USERNAME }}
           ENSO_TEST_USER_PASSWORD: ${{ secrets.ENSO_CLOUD_TEST_ACCOUNT_PASSWORD }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        continue-on-error: true
+      - if: failure()
+        name: Upload Test Traces
+        uses: actions/upload-artifact@v4
+        with:
+          compression-level: 0
+          name: test-traces-windows-amd64
+          path: app/ide-desktop/client/test-traces
       - run: rm $HOME/.enso/credentials
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -182,5 +182,6 @@ test-results
 ##########################
 
 test-results/
+test-traces/
 playwright-report/
 playwright/.cache/

app/.vscode/launch.json

@@ -4,121 +4,73 @@
     {
       "type": "node",
       "request": "launch",
-      "name": "Dashboard",
+      "name": "GUI",
       "runtimeExecutable": "pnpm",
-      "runtimeArgs": ["run", "--filter", "enso-dashboard", "dev"],
+      "runtimeArgs": ["dev:gui"],
       "outputCapture": "std"
     },
     {
       "type": "node",
       "request": "launch",
-      "name": "Dashboard (Electron, Linux)",
+      "name": "GUI (Electron, Linux)",
       "runtimeExecutable": "pnpm",
       "runtimeArgs": ["run", "--filter", "enso", "watch:linux"],
       "outputCapture": "std"
     },
     {
       "type": "node",
       "request": "launch",
-      "name": "Dashboard (Electron, macOS)",
+      "name": "GUI (Electron, macOS)",
       "runtimeExecutable": "pnpm",
       "runtimeArgs": ["run", "--filter", "enso", "watch:macos"],
       "outputCapture": "std"
     },
     {
       "type": "node",
       "request": "launch",
-      "name": "Dashboard (Electron, Windows)",
+      "name": "GUI (Electron, Windows)",
       "runtimeExecutable": "pnpm",
       "runtimeArgs": ["run", "--filter", "enso", "watch:windows"],
       "outputCapture": "std"
     },
     {
       "type": "node",
       "request": "launch",
-      "name": "GUI",
-      "runtimeExecutable": "pnpm",
-      "runtimeArgs": ["dev:gui"],
-      "outputCapture": "std"
-    },
-    {
-      "type": "node",
-      "request": "launch",
-      "name": "GUI (Storybook)",
-      "runtimeExecutable": "pnpm",
-      "runtimeArgs": ["run", "--filter", "enso-gui", "story:dev"],
-      "outputCapture": "std"
-    },
-    {
-      "type": "node",
-      "request": "launch",
-      "name": "Dashboard (Build)",
+      "name": "GUI (Build)",
       "runtimeExecutable": "pnpm",
       "runtimeArgs": ["build:gui"],
       "outputCapture": "std"
     },
     {
       "type": "node",
       "request": "launch",
-      "name": "Dashboard (E2E UI)",
-      "runtimeExecutable": "pnpm",
-      "runtimeArgs": ["run", "--filter", "enso-dashboard", "test-dev:e2e"],
-      "outputCapture": "std"
-    },
-    {
-      "type": "node",
-      "request": "launch",
-      "name": "GUI (E2E UI)",
-      "runtimeExecutable": "pnpm",
-      "runtimeArgs": ["run", "--filter", "enso-gui", "test:e2e", "--", "--ui"],
-      "outputCapture": "std"
-    },
-    {
-      "type": "node",
-      "request": "launch",
-      "name": "Dashboard (All tests)",
+      "name": "GUI (All tests)",
       "runtimeExecutable": "pnpm",
-      "runtimeArgs": ["run", "--filter", "enso-dashboard", "test"],
-      "outputCapture": "std"
+      "runtimeArgs": ["run", "--filter", "enso-gui", "test"]
     },
     {
       "type": "node",
       "request": "launch",
-      "name": "Dashboard (E2E tests)",
+      "name": "GUI (Unit tests)",
       "runtimeExecutable": "pnpm",
-      "runtimeArgs": ["run", "--filter", "enso-dashboard", "test:e2e"],
+      "runtimeArgs": ["run", "--filter", "enso-gui", "test:unit"],
       "outputCapture": "std"
     },
     {
       "type": "node",
       "request": "launch",
-      "name": "Dashboard (Unit tests)",
+      "name": "GUI (Integration tests)",
       "runtimeExecutable": "pnpm",
-      "runtimeArgs": ["run", "--filter", "enso-dashboard", "test:unit"],
+      "runtimeArgs": ["run", "--filter", "enso-gui", "test:integration"],
       "outputCapture": "std"
     },
     {
       "type": "node",
       "request": "launch",
-      "name": "GUI (All tests)",
-      "runtimeExecutable": "pnpm",
-      "runtimeArgs": ["run", "--filter", "enso-gui", "test"]
-    },
-    {
-      "type": "node",
-      "request": "launch",
-      "name": "GUI (E2E tests)",
+      "name": "GUI (Integration tests with UI)",
       "runtimeExecutable": "pnpm",
-      "runtimeArgs": ["run", "--filter", "enso-gui", "test:e2e"],
+      "runtimeArgs": ["run", "--filter", "enso-gui", "test-dev:integration"],
       "outputCapture": "std"
     },
-    {
-      "type": "node",
-      "request": "launch",
-      "name": "GUI (Unit tests)",
-      "runtimeExecutable": "pnpm",
-      "runtimeArgs": ["run", "--filter", "enso-gui", "test:unit", "--", "run"],
-      "outputCapture": "std"
-    }
   ]
 }

app/gui/env.d.ts

@@ -234,7 +234,7 @@ declare global {
       // @ts-expect-error The index signature is intentional to disallow unknown env vars.
       readonly ENSO_CLOUD_ENSO_HOST?: string
 
-      // === E2E test variables ===
+      // === Integration test variables ===
       readonly PWDEBUG?: '1'
       readonly IS_IN_PLAYWRIGHT_TEST?: `${boolean}`
 

app/gui/e2e/dashboard/README.md → app/gui/integration-test/dashboard/README.md

@@ -1,25 +1,25 @@
-# End-to-end tests
+# Integration tests
 
 ## Running tests
 
 Execute all commands from the parent directory.
 
 ```sh
 # Run tests normally
-pnpm run test:e2e
+pnpm run test:integration
 # Open UI to run tests
-pnpm run test:e2e:debug
+pnpm run test:integration:debug
 # Run tests in a specific file only
-pnpm run test:e2e -- e2e/file-name-here.spec.ts
-pnpm run test:e2e:debug -- e2e/file-name-here.spec.ts
+pnpm run test:integration -- integration-test/file-name-here.spec.ts
+pnpm run test:integration:debug -- integration-test/file-name-here.spec.ts
 # Compile the entire app before running the tests.
 # DOES NOT hot reload the tests.
 # Prefer not using this when you are trying to fix a test;
 # prefer using this when you just want to know which tests are failing (if any).
-PROD=1 pnpm run test:e2e
-PROD=1 pnpm run test:e2e:debug
-PROD=1 pnpm run test:e2e -- e2e/file-name-here.spec.ts
-PROD=1 pnpm run test:e2e:debug -- e2e/file-name-here.spec.ts
+PROD=1 pnpm run test:integration
+PROD=1 pnpm run test:integration:debug
+PROD=1 pnpm run test:integration -- integration-test/file-name-here.spec.ts
+PROD=1 pnpm run test:integration:debug -- integration-test/file-name-here.spec.ts
 ```
 
 ## Getting started

app/gui/e2e/project-view/customExpect.ts → app/gui/integration-test/project-view/customExpect.ts

@@ -13,8 +13,7 @@ export const expect = baseExpect.extend({
     try {
       await expect(locator.first()).toBeVisible()
       pass = true
-    } catch (e) {
-      console.log(e)
+    } catch {
       pass = false
     }
 

app/gui/e2e/project-view/nodeClipboard.spec.ts → app/gui/integration-test/project-view/nodeClipboard.spec.ts

@@ -7,7 +7,7 @@ import { edgesFromNodeWithBinding, edgesToNodeWithBinding } from './locate'
 
 /**
  * Every edge consists of multiple parts.
- * See e2e/edgeRendering.spec.ts for explanation.
+ * See edgeRendering.spec.ts for explanation.
  */
 const EDGE_PARTS = 2
 

app/gui/e2e/project-view/setup.ts → app/gui/integration-test/project-view/setup.ts

@@ -10,7 +10,7 @@ import {
 } from './projectManager'
 
 /**
- * Setup for all E2E tests.
+ * Setup for all Project View's Integration tests.
  *
  * It runs mocked project manager server.
  */

app/gui/package.json

@@ -26,9 +26,10 @@
     "dev:vite": "vite",
     "test": "corepack pnpm run /^^^^test:.*/",
     "test:unit": "vitest run",
+    "test:integration": "cross-env NODE_ENV=production playwright test",
     "test-dev:unit": "vitest",
-    "test-dev:e2e": "cross-env NODE_ENV=production playwright test --ui",
-    "test-dev-dashboard:e2e": "cross-env NODE_ENV=production playwright test ./e2e/dashboard/ --ui",
+    "test-dev:integration": "cross-env NODE_ENV=production playwright test --ui",
+    "test-dev-dashboard:integration": "cross-env NODE_ENV=production playwright test ./integration-test/dashboard/ --ui",
     "preinstall": "corepack pnpm run generate-metadata",
     "generate-metadata": "node scripts/generateIconMetadata.js",
     "storybook:react": "cross-env FRAMEWORK=react storybook dev",
@@ -37,7 +38,6 @@
     "build-storybook:vue": "cross-env FRAMEWORK=vue storybook build",
     "chromatic:react": "cross-env FRAMEWORK=react chromatic deploy",
     "chromatic:vue": "cross-env FRAMEWORK=vue chromatic deploy",
-    "e2e": "cross-env NODE_ENV=production playwright test",
     "playwright:install": "playwright install chromium"
   },
   "dependencies": {

app/gui/playwright.config.ts

@@ -9,7 +9,7 @@
 import { defineConfig } from '@playwright/test'
 import net from 'net'
 
-const DEBUG = process.env.DEBUG_E2E === 'true'
+const DEBUG = process.env.DEBUG_TEST === 'true'
 const isCI = process.env.CI === 'true'
 const isProd = process.env.PROD === 'true'
 
@@ -101,7 +101,7 @@ export default defineConfig({
     // Setup project
     {
       name: 'Setup Dashboard',
-      testDir: './e2e/dashboard',
+      testDir: './integration-test/dashboard',
       testMatch: /.*\.setup\.ts/,
       timeout: TIMEOUT_MS,
       use: {
@@ -111,7 +111,7 @@ export default defineConfig({
     },
     {
       name: 'Dashboard',
-      testDir: './e2e/dashboard',
+      testDir: './integration-test/dashboard',
       testMatch: /.*\.spec\.ts/,
       dependencies: ['Setup Dashboard'],
       expect: {
@@ -127,7 +127,7 @@ export default defineConfig({
     },
     {
       name: 'Auth',
-      testDir: './e2e/dashboard/auth',
+      testDir: './integration-test/dashboard/auth',
       expect: {
         toHaveScreenshot: { threshold: 0 },
         timeout: TIMEOUT_MS,
@@ -140,13 +140,15 @@ export default defineConfig({
     },
     {
       name: 'Setup Tests for Project View',
-      testMatch: /e2e\/project-view\/setup\.ts/,
+      testMatch: /integration-test\/project-view\/setup\.ts/,
     },
     {
       name: 'Project View',
       dependencies: ['Setup Tests for Project View'],
-      testDir: './e2e/project-view',
+      testDir: './integration-test/project-view',
       timeout: 60000,
+      repeatEach: 3,
+      retries: 0,
       expect: {
         timeout: 5000,
         toHaveScreenshot: { threshold: 0 },
@@ -159,7 +161,7 @@ export default defineConfig({
   ],
   webServer: [
     {
-      env: { E2E: 'true' },
+      env: { INTEGRATION_TEST: 'true' },
       command:
         isCI || isProd ?
           `corepack pnpm build && corepack pnpm exec vite preview --port ${ports.projectView} --strictPort`

app/gui/src/dashboard/modules/payments/components/AddPaymentMethodForm.tsx

@@ -64,7 +64,7 @@ export function AddPaymentMethodForm<
   const cardElement =
     // FIXME[sb]: I do not understand why `useWatch` is not sufficient for Playwright.
     // (The value is always `undefined` with `useWatch` alone)
-    // It is worth noting that E2E tests previously worked without requiring this change - as of:
+    // It is worth noting that integration tests previously worked without requiring this change - as of:
     // 1500849c32f70f5f4d95240b7e31377c649dc25b
     Form.useWatch({ control: form.control, name: 'cardElement' }) ?? form.getValues().cardElement
 

app/gui/src/project-view/components/GraphEditor/GraphEdge.vue

@@ -105,7 +105,7 @@ const sourceRect = computed<Rect | undefined>(() => {
 
 /**
  * Edges which do not have `sourceRect` and `targetPos` initialized are marked by a special
- * `broken-edge` data-testid, for debugging and e2e test purposes.
+ * `broken-edge` data-testid, for debugging and integration test purposes.
  */
 const edgeIsBroken = computed(
   () =>

app/gui/src/project-view/e2e-entrypoint.ts → app/gui/src/project-view/test-entrypoint.ts

@@ -1,5 +1,5 @@
 /***
- * This is a web entrypoint file for the GUI application running in e2e tests (playwright). It is
+ * This is a web entrypoint file for the GUI application running in integration tests (playwright). It is
  * not included in normal application distribution. The goal of separate entrypoint is to allow
  * providing mocks for connections with engine and to avoid running dashboard.
  */