From cb7c6437f2939f1f31e72d49f3e3374423babe06 Mon Sep 17 00:00:00 2001 From: Philippe Serhal Date: Mon, 14 Apr 2025 09:10:16 -0400 Subject: [PATCH 1/2] test: fix flakey snapshot normalization logic This logic was normalizing integration test snapshots by replacing any sequence of five digits with `88888`. The intent is to handle some randomness in test output. It wasn't working some percentage of the time because randomly selected ports are sometimes _four_ digits. You can see an example in this failed run: https://github.com/netlify/cli/actions/runs/14434953424/job/40474516002?pr=7202#step:10:1143. While I was at it, I made it less much less generic and moved it into the only test suite that relies on it. --- tests/integration/framework-detection.test.js | 42 +++++++++++-------- tests/integration/utils/snapshots.js | 2 - 2 files changed, 24 insertions(+), 20 deletions(-) diff --git a/tests/integration/framework-detection.test.js b/tests/integration/framework-detection.test.js index ed8626fc34b..39128e77501 100644 --- a/tests/integration/framework-detection.test.js +++ b/tests/integration/framework-detection.test.js @@ -10,6 +10,12 @@ import { normalize } from './utils/snapshots.js' const content = 'Hello World!' +// Normalize random ports +const normalizeSnapshot = (output, opts) => + normalize(output, opts) + .replaceAll(/localhost:\d+/g, 'localhost:88888') + .replaceAll(/listening to \d+/g, 'listening to 88888') + describe.concurrent('frameworks/framework-detection', () => { test('should default to process.cwd() and static server', async (t) => { await withSiteBuilder(t, async (builder) => { @@ -25,7 +31,7 @@ describe.concurrent('frameworks/framework-detection', () => { const responseContent = await response.text() t.expect(responseContent).toEqual(content) - t.expect(normalize(output, { duration: true, filePath: true })).toMatchSnapshot() + t.expect(normalizeSnapshot(output, { duration: true, filePath: true })).toMatchSnapshot() }) }) }) @@ -44,7 +50,7 @@ describe.concurrent('frameworks/framework-detection', () => { const responseContent = await response.text() t.expect(responseContent).toEqual(content) - t.expect(normalize(output, { duration: true, filePath: true })).toMatchSnapshot() + t.expect(normalizeSnapshot(output, { duration: true, filePath: true })).toMatchSnapshot() }) }) }) @@ -64,7 +70,7 @@ describe.concurrent('frameworks/framework-detection', () => { const responseContent = await response.text() t.expect(responseContent).toEqual(content) - t.expect(normalize(output, { duration: true, filePath: true })).toMatchSnapshot() + t.expect(normalizeSnapshot(output, { duration: true, filePath: true })).toMatchSnapshot() }) }) }) @@ -85,7 +91,7 @@ describe.concurrent('frameworks/framework-detection', () => { const responseContent = await response.text() t.expect(responseContent).toEqual(content) - t.expect(normalize(output, { duration: true, filePath: true })).toMatchSnapshot() + t.expect(normalizeSnapshot(output, { duration: true, filePath: true })).toMatchSnapshot() }, ) }) @@ -102,7 +108,7 @@ describe.concurrent('frameworks/framework-detection', () => { true, ).catch((error_) => error_) - t.expect(normalize(error.stdout, { duration: true, filePath: true })).toMatchSnapshot() + t.expect(normalizeSnapshot(error.stdout, { duration: true, filePath: true })).toMatchSnapshot() }) }) @@ -112,7 +118,7 @@ describe.concurrent('frameworks/framework-detection', () => { // a failure is expected since this is not a true create-react-app project const error = await withDevServer({ cwd: builder.directory }, () => {}, true).catch((error_) => error_) - t.expect(normalize(error.stdout, { duration: true, filePath: true })).toMatchSnapshot() + t.expect(normalizeSnapshot(error.stdout, { duration: true, filePath: true })).toMatchSnapshot() }) }) @@ -121,7 +127,7 @@ describe.concurrent('frameworks/framework-detection', () => { await builder.withNetlifyToml({ config: { dev: { framework: 'to-infinity-and-beyond-js' } } }).build() const error = await withDevServer({ cwd: builder.directory }, () => {}, true).catch((error_) => error_) - t.expect(normalize(error.stdout, { duration: true, filePath: true })).toMatchSnapshot() + t.expect(normalizeSnapshot(error.stdout, { duration: true, filePath: true })).toMatchSnapshot() }) }) @@ -135,7 +141,7 @@ describe.concurrent('frameworks/framework-detection', () => { // a failure is expected since this is not a true create-react-app project const error = await withDevServer({ cwd: builder.directory }, () => {}, true).catch((error_) => error_) - t.expect(normalize(error.stdout, { duration: true, filePath: true })).toMatchSnapshot() + t.expect(normalizeSnapshot(error.stdout, { duration: true, filePath: true })).toMatchSnapshot() }) }) @@ -148,7 +154,7 @@ describe.concurrent('frameworks/framework-detection', () => { () => {}, true, ).catch((error_) => error_) - t.expect(normalize(error.stdout, { duration: true, filePath: true })).toMatchSnapshot() + t.expect(normalizeSnapshot(error.stdout, { duration: true, filePath: true })).toMatchSnapshot() }) }) @@ -161,7 +167,7 @@ describe.concurrent('frameworks/framework-detection', () => { () => {}, true, ).catch((error_) => error_) - t.expect(normalize(error.stdout, { duration: true, filePath: true })).toMatchSnapshot() + t.expect(normalizeSnapshot(error.stdout, { duration: true, filePath: true })).toMatchSnapshot() }) }) @@ -174,7 +180,7 @@ describe.concurrent('frameworks/framework-detection', () => { () => {}, true, ).catch((error_) => error_) - t.expect(normalize(error.stdout, { duration: true, filePath: true })).toMatchSnapshot() + t.expect(normalizeSnapshot(error.stdout, { duration: true, filePath: true })).toMatchSnapshot() }) }) @@ -198,7 +204,7 @@ describe.concurrent('frameworks/framework-detection', () => { true, ).catch((error_) => error_) - t.expect(normalize(error.stdout, { duration: true, filePath: true })).toMatchSnapshot() + t.expect(normalizeSnapshot(error.stdout, { duration: true, filePath: true })).toMatchSnapshot() }) }) @@ -232,7 +238,7 @@ describe.concurrent('frameworks/framework-detection', () => { await childProcess } const error = await asyncErrorBlock().catch((error_) => error_) - t.expect(normalize(error.stdout, { duration: true, filePath: true })).toMatchSnapshot() + t.expect(normalizeSnapshot(error.stdout, { duration: true, filePath: true })).toMatchSnapshot() }) }) @@ -259,7 +265,7 @@ describe.concurrent('frameworks/framework-detection', () => { } const error = await asyncErrorBlock().catch((error_) => error_) t.expect( - normalize(error.stdout, { duration: true, filePath: true }).includes( + normalizeSnapshot(error.stdout, { duration: true, filePath: true }).includes( 'Detected commands for: Gatsby, Create React App. Update your settings to specify which to use. Refer to https://ntl.fyi/dev-monorepo for more information.', ), ) @@ -278,7 +284,7 @@ describe.concurrent('frameworks/framework-detection', () => { true, ).catch((error_) => error_) - t.expect(normalize(error.stdout, { duration: true, filePath: true })).toMatchSnapshot() + t.expect(normalizeSnapshot(error.stdout, { duration: true, filePath: true })).toMatchSnapshot() }) }) @@ -299,7 +305,7 @@ describe.concurrent('frameworks/framework-detection', () => { const responseContent = await response.text() t.expect(responseContent).toEqual(content) - t.expect(normalize(output, { duration: true, filePath: true })).toMatchSnapshot() + t.expect(normalizeSnapshot(output, { duration: true, filePath: true })).toMatchSnapshot() }) }) }) @@ -317,7 +323,7 @@ describe.concurrent('frameworks/framework-detection', () => { // a failure is expected since this is not a true Gatsby project const error = await withDevServer({ cwd: builder.directory }, () => {}, true).catch((error_) => error_) - t.expect(normalize(error.stdout, { duration: true, filePath: true })).toMatchSnapshot() + t.expect(normalizeSnapshot(error.stdout, { duration: true, filePath: true })).toMatchSnapshot() }) }) @@ -395,7 +401,7 @@ describe.concurrent('frameworks/framework-detection', () => { const responseJson = await response.json() t.expect(responseJson).toStrictEqual({ CONTEXT_CHECK: 'PRODUCTION' }) - const normalizedText = normalize(output, { duration: true, filePath: true }) + const normalizedText = normalizeSnapshot(output, { duration: true, filePath: true }) t.expect( normalizedText.includes( `Changes will not be hot-reloaded, so if you need to rebuild your site you must exit and run 'netlify serve' again`, diff --git a/tests/integration/utils/snapshots.js b/tests/integration/utils/snapshots.js index f8eecb5c556..1d506134e1d 100644 --- a/tests/integration/utils/snapshots.js +++ b/tests/integration/utils/snapshots.js @@ -2,8 +2,6 @@ const baseNormalizers = [ // Information about the package and the OS { pattern: /netlify-cli\/.+node-.+/g, value: 'netlify-cli/test-version test-os test-node-version' }, { pattern: /@netlify\/build (\d+\.\d+\.\d+)/g, value: '@netlify/build 0.0.0' }, - // normalize random ports - { pattern: /\d{5}/g, value: '88888' }, // windows specific { pattern: /\\/gu, value: '/' }, { pattern: /\r\n/gu, value: '\n' }, From 10c10babe3fb4514850fe08a65e67f97220a0563 Mon Sep 17 00:00:00 2001 From: Philippe Serhal Date: Mon, 14 Apr 2025 12:45:30 -0400 Subject: [PATCH 2/2] test: make normalized snapshot port more explicit --- .../framework-detection.test.js.snap | 20 +++++++++---------- tests/integration/framework-detection.test.js | 4 ++-- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/integration/__snapshots__/framework-detection.test.js.snap b/tests/integration/__snapshots__/framework-detection.test.js.snap index 2edcb2bec97..365ccbc583f 100644 --- a/tests/integration/__snapshots__/framework-detection.test.js.snap +++ b/tests/integration/__snapshots__/framework-detection.test.js.snap @@ -9,11 +9,11 @@ exports[`frameworks/framework-detection > should default to process.cwd() and st ◈ Running static server from \\"should-default-to-process-cwd-and-static-server\\" ◈ Setting up local development server -◈ Static server listening to 88888 +◈ Static server listening to ┌──────────────────────────────────────────────────┐ │ │ - │ ◈ Server now ready on http://localhost:88888 │ + │ ◈ Server now ready on http://localhost: │ │ │ └──────────────────────────────────────────────────┘" `; @@ -38,11 +38,11 @@ exports[`frameworks/framework-detection > should filter frameworks with no dev c ◈ Running static server from \\"should-filter-frameworks-with-no-dev-command\\" ◈ Setting up local development server -◈ Static server listening to 88888 +◈ Static server listening to ┌──────────────────────────────────────────────────┐ │ │ - │ ◈ Server now ready on http://localhost:88888 │ + │ ◈ Server now ready on http://localhost: │ │ │ └──────────────────────────────────────────────────┘" `; @@ -150,11 +150,11 @@ exports[`frameworks/framework-detection > should use static server when --dir fl ◈ Running static server from \\"should-use-static-server-when-dir-flag-is-passed/public\\" ◈ Setting up local development server -◈ Static server listening to 88888 +◈ Static server listening to ┌──────────────────────────────────────────────────┐ │ │ - │ ◈ Server now ready on http://localhost:88888 │ + │ ◈ Server now ready on http://localhost: │ │ │ └──────────────────────────────────────────────────┘" `; @@ -168,11 +168,11 @@ exports[`frameworks/framework-detection > should use static server when framewor ◈ Running static server from \\"should-use-static-server-when-framework-is-set-to-static\\" ◈ Setting up local development server -◈ Static server listening to 88888 +◈ Static server listening to ┌──────────────────────────────────────────────────┐ │ │ - │ ◈ Server now ready on http://localhost:88888 │ + │ ◈ Server now ready on http://localhost: │ │ │ └──────────────────────────────────────────────────┘" `; @@ -185,11 +185,11 @@ exports[`frameworks/framework-detection > should warn if using static server and ◈ Running static server from \\"should-warn-if-using-static-server-and-target-port-is-configured/public\\" ◈ Setting up local development server -◈ Static server listening to 88888 +◈ Static server listening to ┌──────────────────────────────────────────────────┐ │ │ - │ ◈ Server now ready on http://localhost:88888 │ + │ ◈ Server now ready on http://localhost: │ │ │ └──────────────────────────────────────────────────┘" `; diff --git a/tests/integration/framework-detection.test.js b/tests/integration/framework-detection.test.js index 39128e77501..29753046555 100644 --- a/tests/integration/framework-detection.test.js +++ b/tests/integration/framework-detection.test.js @@ -13,8 +13,8 @@ const content = 'Hello World!' // Normalize random ports const normalizeSnapshot = (output, opts) => normalize(output, opts) - .replaceAll(/localhost:\d+/g, 'localhost:88888') - .replaceAll(/listening to \d+/g, 'listening to 88888') + .replaceAll(/localhost:\d+/g, 'localhost:') + .replaceAll(/listening to \d+/g, 'listening to ') describe.concurrent('frameworks/framework-detection', () => { test('should default to process.cwd() and static server', async (t) => {