chore: stabilize all flaky playwright tests (evcc-io#15458)

assets/js/components/Savings.vue

@@ -23,6 +23,7 @@
 				tabindex="-1"
 				role="dialog"
 				aria-hidden="true"
+				data-testid="savings-modal"
 			>
 				<div class="modal-dialog modal-lg modal-dialog-centered" role="document">
 					<div class="modal-content">

assets/js/views/Config.vue

@@ -43,7 +43,9 @@
 							:editable="!gridMeter || !!gridMeter.id"
 							:error="deviceError('meter', gridMeter?.name)"
 							data-testid="grid"
-							@edit="gridMeter?.id ? editMeter(gridMeter.id, 'pv') : addMeter('grid')"
+							@edit="
+								gridMeter?.id ? editMeter(gridMeter.id, 'grid') : addMeter('grid')
+							"
 						>
 							<template #icon>
 								<shopicon-regular-powersupply></shopicon-regular-powersupply>

assets/js/views/Log.vue

@@ -42,6 +42,7 @@
 							class="form-control search"
 							:placeholder="$t('log.search')"
 							v-model="search"
+							data-testid="log-search"
 						/>
 					</div>
 					<div class="filterLevel col-6 col-lg-2">

package.json

@@ -18,7 +18,7 @@
     "@guolao/vue-monaco-editor": "^1.5.1",
     "@h2d2/shopicons": "^1.2.0",
     "@histoire/plugin-vue": "^0.17.1",
-    "@playwright/test": "^1.34.3",
+    "@playwright/test": "^1.46.1",
     "@popperjs/core": "^2.11.5",
     "@unhead/vue": "^1.8.9",
     "@vitejs/plugin-legacy": "^5.3.0",
@@ -56,7 +56,7 @@
     "vue-hot-reload-api": "^2.3.4",
     "vue-i18n": "^9.2.2",
     "vue-router": "^4.0.12",
-    "wait-on": "^7.0.1"
+    "wait-on": "^8.0.0"
   },
   "engines": {
     "npm": ">=8.0.0",

playwright.config.js

@@ -13,6 +13,8 @@ export default defineConfig({
   use: {
     baseURL: "http://127.0.0.1:7070",
     trace: "on-first-retry",
+    video: "on-first-retry",
+    screenshot: "only-on-failure",
   },
   projects: [
     {

tests/auth.spec.js

@@ -128,6 +128,15 @@ test("update password", async ({ page }) => {
   await expect(page.getByRole("heading", { name: "Configuration" })).toBeVisible();
   await expect(loginNew).not.toBeVisible();
 
-  // hard stop, since password is updated
-  await stop(instance);
+  // revert to old password
+  await page.getByTestId("generalconfig-password").getByRole("button", { name: "edit" }).click();
+  await modal.getByLabel("Current password").fill(newPassword);
+  await modal.getByLabel("New password").fill(oldPassword);
+  await modal.getByLabel("Repeat password").fill(oldPassword);
+  await modal.getByRole("button", { name: "Update Password" }).click();
+  await expect(
+    modal.getByRole("heading", { name: "Update Administrator Password" })
+  ).not.toBeVisible();
+
+  await stop();
 });

tests/config-mqtt.spec.js

@@ -4,6 +4,7 @@ import { start, stop, restart, baseUrl } from "./evcc";
 const CONFIG = "config-grid-only.evcc.yaml";
 
 test.use({ baseURL: baseUrl() });
+test.describe.configure({ mode: "parallel" });
 
 test.beforeEach(async ({ page }) => {
   await start(CONFIG, "password.sql");

tests/config-tariffs.spec.js

@@ -5,6 +5,7 @@ const CONFIG_GRID_ONLY = "config-grid-only.evcc.yaml";
 const CONFIG_WITH_TARIFFS = "config-with-tariffs.evcc.yaml";
 
 test.use({ baseURL: baseUrl() });
+test.describe.configure({ mode: "parallel" });
 
 test.afterEach(async () => {
   await stop();

tests/config-vehicles.spec.js

@@ -5,11 +5,9 @@ const CONFIG_GRID_ONLY = "config-grid-only.evcc.yaml";
 const CONFIG_WITH_VEHICLE = "config-with-vehicle.evcc.yaml";
 
 test.use({ baseURL: baseUrl() });
+test.describe.configure({ mode: "parallel" });
 
-test.beforeAll(async () => {
-  await start(CONFIG_GRID_ONLY, "password.sql");
-});
-test.afterAll(async () => {
+test.afterEach(async () => {
   await stop();
 });
 
@@ -30,6 +28,8 @@ async function enableExperimental(page) {
 
 test.describe("vehicles", async () => {
   test("create, edit and delete vehicles", async ({ page }) => {
+    await start(CONFIG_GRID_ONLY, "password.sql");
+
     await page.goto("/#/config");
     await login(page);
     await enableExperimental(page);
@@ -79,6 +79,8 @@ test.describe("vehicles", async () => {
   });
 
   test("config should survive restart", async ({ page }) => {
+    await start(CONFIG_GRID_ONLY, "password.sql");
+
     await page.goto("/#/config");
     await login(page);
     await enableExperimental(page);
@@ -111,7 +113,7 @@ test.describe("vehicles", async () => {
   });
 
   test("mixed config (yaml + db)", async ({ page }) => {
-    await cleanRestart(CONFIG_WITH_VEHICLE, "password.sql");
+    await start(CONFIG_WITH_VEHICLE, "password.sql");
 
     await page.goto("/#/config");
     await login(page);
@@ -120,7 +122,7 @@ test.describe("vehicles", async () => {
     await expect(page.getByTestId("vehicle")).toHaveCount(1);
     const vehicleModal = page.getByTestId("vehicle-modal");
 
-    // create #1
+    // create #2
     await page.getByTestId("add-vehicle").click();
     await vehicleModal.getByLabel("Manufacturer").selectOption("Generic vehicle");
     await vehicleModal.getByLabel("Title").fill("Green Car");
@@ -132,6 +134,8 @@ test.describe("vehicles", async () => {
   });
 
   test("advanced fields", async ({ page }) => {
+    await start(CONFIG_GRID_ONLY, "password.sql");
+
     await page.goto("/#/config");
     await login(page);
     await enableExperimental(page);
@@ -169,6 +173,8 @@ test.describe("vehicles", async () => {
   });
 
   test("save and restore rfid identifiers", async ({ page }) => {
+    await start(CONFIG_GRID_ONLY, "password.sql");
+
     await page.goto("/#/config");
     await login(page);
     await enableExperimental(page);
@@ -188,13 +194,11 @@ test.describe("vehicles", async () => {
     await restart(CONFIG_GRID_ONLY);
     await page.reload();
 
-    await page
-      .locator(`[data-testid="vehicle"]:has-text("RFID Car")`)
-      .getByRole("button", { name: "edit" })
-      .click();
-    await page.getByRole("button", { name: "Show advanced settings" }).click();
+    await expect(page.getByTestId("vehicle")).toHaveCount(1);
+    await page.getByTestId("vehicle").getByRole("button", { name: "edit" }).click();
+    await vehicleModal.getByRole("button", { name: "Show advanced settings" }).click();
     await expect(vehicleModal.getByLabel("RFID identifiers")).toHaveValue("aaa\nbbb\nccc\nddd");
-    await page.getByTestId("vehicle-modal").getByLabel("Close").click();
+    await vehicleModal.getByLabel("Close").click();
     await expect(page.getByTestId("fatal-error")).not.toBeVisible();
   });
 });

tests/evcc.js

@@ -1,9 +1,10 @@
 import fs from "fs";
 import waitOn from "wait-on";
 import axios from "axios";
-import { exec, execSync } from "child_process";
+import { spawn, execSync } from "child_process";
 import os from "os";
 import path from "path";
+import { Transform } from "stream";
 
 const BINARY = "./evcc";
 
@@ -12,8 +13,24 @@ function workerPort() {
   return 11000 + index;
 }
 
-function sleep(ms) {
-  return new Promise((resolve) => setTimeout(resolve, ms));
+function logPrefix() {
+  return `[worker:${process.env.TEST_WORKER_INDEX}]`;
+}
+
+function createSteamLog() {
+  return new Transform({
+    transform(chunk, encoding, callback) {
+      const lines = chunk.toString().split("\n");
+      lines.forEach((line) => {
+        if (line.trim()) log(line);
+      });
+      callback();
+    },
+  });
+}
+
+function log(...args) {
+  console.log(logPrefix(), ...args);
 }
 
 export function baseUrl() {
@@ -55,52 +72,56 @@ export async function cleanRestart(config, sqlDumps) {
 async function _restoreDatabase(sqlDumps) {
   const dumps = Array.isArray(sqlDumps) ? sqlDumps : [sqlDumps];
   for (const dump of dumps) {
-    console.log("loading database", dbPath(), dump);
+    log("loading database", dbPath(), dump);
     execSync(`sqlite3 ${dbPath()} < tests/${dump}`);
   }
 }
 
 async function _start(config) {
   const configFile = config.includes("/") ? config : `tests/${config}`;
   const port = workerPort();
-  console.log(`wait until port ${port} is available`);
-  await waitOn({ resources: [`tcp:localhost:${port}`], reverse: true });
-  console.log("starting evcc", { config, port });
-  const instance = exec(
-    `EVCC_NETWORK_PORT=${port} EVCC_DATABASE_DSN=${dbPath()} ${BINARY} --config ${configFile}`
-  );
-  instance.stdout.pipe(process.stdout);
-  instance.stderr.pipe(process.stderr);
+  log(`wait until port ${port} is available`);
+  // wait for port to be available
+  await waitOn({ resources: [`tcp:${port}`], reverse: true, log: true });
+  log("starting evcc", { config, port });
+  const instance = spawn(BINARY, ["--config", configFile], {
+    env: { EVCC_NETWORK_PORT: port.toString(), EVCC_DATABASE_DSN: dbPath() },
+    stdio: ["pipe", "pipe", "pipe"],
+  });
+  const steamLog = createSteamLog();
+  instance.stdout.pipe(steamLog);
+  instance.stderr.pipe(steamLog);
   instance.on("exit", (code) => {
-    console.log("evcc terminated", { code, port, config });
+    log("evcc terminated", { code, port, config });
+    steamLog.end();
   });
-  await waitOn({ resources: [baseUrl()] });
+  await waitOn({ resources: [baseUrl()], log: true });
   return instance;
 }
 
 async function _stop(instance) {
+  const port = workerPort();
   if (instance) {
-    console.log("shutting down evcc hard");
-    // hard kill, only use of normal shutdown doesn't work
+    log("shutting down evcc hard", { port });
     instance.kill("SIGKILL");
-    await sleep(300);
+    await waitOn({ resources: [`tcp:${port}`], reverse: true, log: true });
+    log("evcc is down", { port });
     return;
   }
-  const port = workerPort();
-  console.log("shutting down evcc", { port });
+  log("shutting down evcc", { port });
   const res = await axios.post(`${baseUrl()}/api/auth/login`, { password: "secret" });
-  console.log(res.status, res.statusText);
+  log(res.status, res.statusText);
   const cookie = res.headers["set-cookie"];
   await axios.post(`${baseUrl()}/api/system/shutdown`, {}, { headers: { cookie } });
-  console.log(`wait until port ${port} is closed`);
-  await waitOn({ resources: [`tcp:localhost:${port}`], reverse: true });
-  console.log("evcc is down", { port });
+  log(`wait until port ${port} is closed`);
+  await waitOn({ resources: [`tcp:${port}`], reverse: true, log: true });
+  log("evcc is down", { port });
 }
 
 async function _clean() {
   const db = dbPath();
   if (fs.existsSync(db)) {
-    console.log("delete database", db);
+    log("delete database", db);
     fs.unlinkSync(db);
   }
 }

tests/logs.spec.js

@@ -47,6 +47,7 @@ test.describe("features", async () => {
   test("content", async ({ page }) => {
     await page.goto("/#/log");
     await login(page);
+    await page.getByTestId("log-search").fill("listening at");
     await expect(page.getByTestId("log-content")).toContainText("listening at");
   });
 });

tests/plan.spec.js

@@ -2,6 +2,7 @@ import { test, expect } from "@playwright/test";
 import { start, stop, baseUrl } from "./evcc";
 
 test.use({ baseURL: baseUrl() });
+test.describe.configure({ mode: "parallel" });
 
 const CONFIG = "plan.evcc.yaml";