From 2ea0e520f11db671486103c09612bffb906ab14e Mon Sep 17 00:00:00 2001 From: Mateusz Bocian Date: Mon, 31 Mar 2025 14:54:10 -0400 Subject: [PATCH 01/14] test: fixture for dynamic cms results --- tests/fixtures/dynamic-cms/README.md | 1 + .../dynamic-cms/netlify/functions/cms.ts | 24 ++++++++++++ tests/fixtures/dynamic-cms/next.config.ts | 10 +++++ tests/fixtures/dynamic-cms/package.json | 24 ++++++++++++ tests/fixtures/dynamic-cms/pages/404.js | 3 ++ .../dynamic-cms/pages/api/revalidate.js | 9 +++++ .../dynamic-cms/pages/content/[...slug].js | 37 +++++++++++++++++++ 7 files changed, 108 insertions(+) create mode 100644 tests/fixtures/dynamic-cms/README.md create mode 100644 tests/fixtures/dynamic-cms/netlify/functions/cms.ts create mode 100644 tests/fixtures/dynamic-cms/next.config.ts create mode 100644 tests/fixtures/dynamic-cms/package.json create mode 100644 tests/fixtures/dynamic-cms/pages/404.js create mode 100644 tests/fixtures/dynamic-cms/pages/api/revalidate.js create mode 100644 tests/fixtures/dynamic-cms/pages/content/[...slug].js diff --git a/tests/fixtures/dynamic-cms/README.md b/tests/fixtures/dynamic-cms/README.md new file mode 100644 index 0000000000..e9f8e0c8ce --- /dev/null +++ b/tests/fixtures/dynamic-cms/README.md @@ -0,0 +1 @@ +This fixture is meant to emulate dynamic content responses of a CMS-backed next site diff --git a/tests/fixtures/dynamic-cms/netlify/functions/cms.ts b/tests/fixtures/dynamic-cms/netlify/functions/cms.ts new file mode 100644 index 0000000000..243421e0be --- /dev/null +++ b/tests/fixtures/dynamic-cms/netlify/functions/cms.ts @@ -0,0 +1,24 @@ +import { getDeployStore } from '@netlify/blobs' +import { Context } from '@netlify/functions' + +// publish or unpublish "cms content" depending on the sent operation +export default async function handler(_request: Request, context: Context) { + const store = getDeployStore({ name: 'cms-content', consistency: 'strong' }) + const BLOB_KEY = 'key' + + const operation = context.params['operation'] + + if (operation === 'publish') { + await store.setJSON(BLOB_KEY, { content: true }) + } + + if (operation === 'unpublish') { + await store.delete(BLOB_KEY) + } + + return Response.json({ ok: true }) +} + +export const config = { + path: '/cms/:operation', +} diff --git a/tests/fixtures/dynamic-cms/next.config.ts b/tests/fixtures/dynamic-cms/next.config.ts new file mode 100644 index 0000000000..6346ab0742 --- /dev/null +++ b/tests/fixtures/dynamic-cms/next.config.ts @@ -0,0 +1,10 @@ +/** @type {import('next').NextConfig} */ +const nextConfig = { + output: 'standalone', + eslint: { + ignoreDuringBuilds: true, + }, + generateBuildId: () => 'build-id', +} + +module.exports = nextConfig diff --git a/tests/fixtures/dynamic-cms/package.json b/tests/fixtures/dynamic-cms/package.json new file mode 100644 index 0000000000..33ebc7f177 --- /dev/null +++ b/tests/fixtures/dynamic-cms/package.json @@ -0,0 +1,24 @@ +{ + "name": "dynamic-cms", + "version": "0.1.0", + "private": true, + "scripts": { + "postinstall": "next build", + "dev": "next dev", + "build": "next build" + }, + "dependencies": { + "@netlify/blobs": "^8.1.0", + "@netlify/functions": "^2.7.0", + "@netlify/plugin-nextjs": "^5.10.1", + "netlify-cli": "^19.0.3", + "next": "latest", + "react": "18.2.0", + "react-dom": "18.2.0" + }, + "devDependencies": { + "@types/node": "22.13.13", + "@types/react": "19.0.12", + "typescript": "5.8.2" + } +} diff --git a/tests/fixtures/dynamic-cms/pages/404.js b/tests/fixtures/dynamic-cms/pages/404.js new file mode 100644 index 0000000000..3c251e6665 --- /dev/null +++ b/tests/fixtures/dynamic-cms/pages/404.js @@ -0,0 +1,3 @@ +export default function NotFound() { + return

Custom 404 page

+} diff --git a/tests/fixtures/dynamic-cms/pages/api/revalidate.js b/tests/fixtures/dynamic-cms/pages/api/revalidate.js new file mode 100644 index 0000000000..e134a56577 --- /dev/null +++ b/tests/fixtures/dynamic-cms/pages/api/revalidate.js @@ -0,0 +1,9 @@ +export default async function handler(req, res) { + try { + const pathToPurge = req.query.path ?? '/static/revalidate-manual' + await res.revalidate(pathToPurge) + return res.json({ code: 200, message: 'success' }) + } catch (err) { + return res.status(500).send({ code: 500, message: err.message }) + } +} diff --git a/tests/fixtures/dynamic-cms/pages/content/[...slug].js b/tests/fixtures/dynamic-cms/pages/content/[...slug].js new file mode 100644 index 0000000000..54abf74b16 --- /dev/null +++ b/tests/fixtures/dynamic-cms/pages/content/[...slug].js @@ -0,0 +1,37 @@ +import { getDeployStore } from '@netlify/blobs' + +const Content = ({ value }) => ( +
+

+ {JSON.stringify(value)} +

+
+) + +export async function getStaticProps() { + const store = getDeployStore({ name: 'cms-content', consistency: 'strong' }) + const BLOB_KEY = 'key' + + const value = await store.get(BLOB_KEY, { type: 'json' }) + + if (!value) { + return { + notFound: true, + } + } + + return { + props: { + value: value, + }, + } +} + +export const getStaticPaths = () => { + return { + paths: [], + fallback: 'blocking', // false or "blocking" + } +} + +export default Content From d278f66b9b9591dab474a0a335994d8b006257c2 Mon Sep 17 00:00:00 2001 From: Mateusz Bocian Date: Mon, 31 Mar 2025 14:54:49 -0400 Subject: [PATCH 02/14] test: case for invalidating dynamic 404 pages --- tests/e2e/dynamic-cms.test.ts | 52 +++++++++++++++++++++++++++++++ tests/utils/create-e2e-fixture.ts | 1 + 2 files changed, 53 insertions(+) create mode 100644 tests/e2e/dynamic-cms.test.ts diff --git a/tests/e2e/dynamic-cms.test.ts b/tests/e2e/dynamic-cms.test.ts new file mode 100644 index 0000000000..cb0ac01828 --- /dev/null +++ b/tests/e2e/dynamic-cms.test.ts @@ -0,0 +1,52 @@ +import { expect } from '@playwright/test' +import { test } from '../utils/playwright-helpers.js' + +test.describe('Dynamic CMS', () => { + test('Invalidates 404 pages from durable cache', async ({ page, dynamicCms }) => { + // 1. Verify the status and headers of the dynamic page + const response1 = await page.goto(new URL('/content/blog', dynamicCms.url).href) + const headers1 = response1?.headers() || {} + + expect(response1?.status()).toEqual(404) + expect(headers1['cache-control']).toEqual('public,max-age=0,must-revalidate') + expect(headers1['cache-status']).toEqual( + '"Next.js"; fwd=miss, "Netlify Durable"; fwd=uri-miss; stored, "Netlify Edge"; fwd=miss', + ) + expect(headers1['netlify-cache-tag']).toEqual('_n_t_/content/blog') + expect(headers1['netlify-cdn-cache-control']).toEqual('s-maxage=31536000, durable') + + // 2. Publish the blob, revalidate the dynamic page, and wait to regenerate + await page.goto(new URL('/cms/publish', dynamicCms.url).href) + await page.goto(new URL('/api/revalidate?path=/content/blog', dynamicCms.url).href) + await page.waitForTimeout(1000) + + // 3. Verify the status and headers of the dynamic page + const response2 = await page.goto(new URL('/content/blog', dynamicCms.url).href) + const headers2 = response2?.headers() || {} + + expect(response2?.status()).toEqual(200) + expect(headers2['cache-control']).toEqual('public,max-age=0,must-revalidate') + expect(headers2['cache-status']).toMatch( + /"Next.js"; hit, "Netlify Durable"; fwd=stale; ttl=[0-9]+; stored, "Netlify Edge"; fwd=stale/, + ) + expect(headers2['netlify-cache-tag']).toEqual('_n_t_/content/blog') + expect(headers2['netlify-cdn-cache-control']).toEqual('s-maxage=31536000, durable') + + // 4. Unpublish the blob, revalidate the dynamic page, and wait to regenerate + await page.goto(new URL('/cms/unpublish', dynamicCms.url).href) + await page.goto(new URL('/api/revalidate?path=/content/blog', dynamicCms.url).href) + await page.waitForTimeout(1000) + + // 5. Verify the status and headers of the dynamic page + const response3 = await page.goto(new URL('/content/blog', dynamicCms.url).href) + const headers3 = response3?.headers() || {} + + expect(response3?.status()).toEqual(404) + expect(headers3['cache-control']).toEqual('public,max-age=0,must-revalidate') + expect(headers3['cache-status']).toMatch( + /"Next.js"; fwd=miss, "Netlify Durable"; fwd=stale; ttl=[0-9]+; stored, "Netlify Edge"; fwd=stale/, + ) + expect(headers3['netlify-cache-tag']).toEqual('_n_t_/content/blog') + expect(headers3['netlify-cdn-cache-control']).toEqual('s-maxage=31536000, durable') + }) +}) diff --git a/tests/utils/create-e2e-fixture.ts b/tests/utils/create-e2e-fixture.ts index b133e6bc25..a2b5c48f00 100644 --- a/tests/utils/create-e2e-fixture.ts +++ b/tests/utils/create-e2e-fixture.ts @@ -440,5 +440,6 @@ export const fixtureFactories = { publishDirectory: 'apps/site/.next', smoke: true, }), + dynamicCms: () => createE2EFixture('dynamic-cms'), after: () => createE2EFixture('after'), } From beec5f5b015f3fe41c477b4a37258fc8650f00ee Mon Sep 17 00:00:00 2001 From: Mateusz Bocian Date: Mon, 31 Mar 2025 14:54:59 -0400 Subject: [PATCH 03/14] fix: wip - fix for caching issues on catch all routes --- src/run/handlers/cache.cts | 5 +++-- src/run/handlers/server.ts | 2 +- src/run/headers.ts | 25 +++++++++++++++++++------ 3 files changed, 23 insertions(+), 9 deletions(-) diff --git a/src/run/handlers/cache.cts b/src/run/handlers/cache.cts index 294700901c..fbc546560b 100644 --- a/src/run/handlers/cache.cts +++ b/src/run/handlers/cache.cts @@ -121,11 +121,12 @@ export class NetlifyCacheHandler implements CacheHandlerForMultipleVersions { } private captureCacheTags(cacheValue: NetlifyIncrementalCacheValue | null, key: string) { + const requestContext = getRequestContext() + if (!cacheValue) { return } - const requestContext = getRequestContext() // Bail if we can't get request context if (!requestContext) { return @@ -393,7 +394,7 @@ export class NetlifyCacheHandler implements CacheHandlerForMultipleVersions { await this.cacheStore.set(key, { lastModified, value }, 'blobStore.set') - if (data?.kind === 'PAGE' || data?.kind === 'PAGES') { + if (!data || data?.kind === 'PAGE' || data?.kind === 'PAGES') { const requestContext = getRequestContext() if (requestContext?.didPagesRouterOnDemandRevalidate) { // encode here to deal with non ASCII characters in the key diff --git a/src/run/handlers/server.ts b/src/run/handlers/server.ts index 72666dd5b3..b0b23392d4 100644 --- a/src/run/handlers/server.ts +++ b/src/run/handlers/server.ts @@ -132,7 +132,7 @@ export default async ( } setCacheControlHeaders(response, request, requestContext, nextConfig) - setCacheTagsHeaders(response.headers, requestContext) + setCacheTagsHeaders(response.headers, request, requestContext) setVaryHeaders(response.headers, request, nextConfig) setCacheStatusHeader(response.headers, nextCache) diff --git a/src/run/headers.ts b/src/run/headers.ts index bfc386506c..9f1d156437 100644 --- a/src/run/headers.ts +++ b/src/run/headers.ts @@ -220,7 +220,6 @@ export const setCacheControlHeaders = ( .log('NetlifyHeadersHandler.trailingSlashRedirect') } - const cacheControl = headers.get('cache-control') if (status === 404) { if (request.url.endsWith('.php')) { // temporary CDN Cache Control handling for bot probes on PHP files @@ -241,6 +240,8 @@ export const setCacheControlHeaders = ( } } + const cacheControl = headers.get('cache-control') + if ( cacheControl !== null && ['GET', 'HEAD'].includes(request.method) && @@ -273,6 +274,7 @@ export const setCacheControlHeaders = ( ['GET', 'HEAD'].includes(request.method) && !headers.has('cdn-cache-control') && !headers.has('netlify-cdn-cache-control') && + !new URL(request.url).pathname.startsWith('/api/') && requestContext.usedFsReadForNonFallback ) { // handle CDN Cache Control on static files @@ -281,13 +283,24 @@ export const setCacheControlHeaders = ( } } -export const setCacheTagsHeaders = (headers: Headers, requestContext: RequestContext) => { - if ( - requestContext.responseCacheTags && - (headers.has('cache-control') || headers.has('netlify-cdn-cache-control')) - ) { +export const setCacheTagsHeaders = ( + headers: Headers, + request: Request, + requestContext: RequestContext, +) => { + if (!headers.has('cache-control') && !headers.has('netlify-cdn-cache-control')) { + return + } + + if (requestContext.responseCacheTags) { headers.set('netlify-cache-tag', requestContext.responseCacheTags.join(',')) + return } + + const key = new URL(request.url).pathname + const cacheTag = `_N_T_${key === '/index' ? '/' : encodeURI(key)}` + console.log('setCacheTagsHeaders', 'netlify-cache-tag', key) + headers.set('netlify-cache-tag', cacheTag) } /** From e95639ac609d6de931bb4b06b6f148884738e3fc Mon Sep 17 00:00:00 2001 From: Mateusz Bocian Date: Tue, 1 Apr 2025 11:55:57 -0400 Subject: [PATCH 04/14] test: not all versions of next support next.config.ts --- tests/fixtures/dynamic-cms/{next.config.ts => next.config.js} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/fixtures/dynamic-cms/{next.config.ts => next.config.js} (100%) diff --git a/tests/fixtures/dynamic-cms/next.config.ts b/tests/fixtures/dynamic-cms/next.config.js similarity index 100% rename from tests/fixtures/dynamic-cms/next.config.ts rename to tests/fixtures/dynamic-cms/next.config.js From 1aee505b57f93c94104b0fe6e5de9d9c993d652d Mon Sep 17 00:00:00 2001 From: Mateusz Bocian Date: Tue, 1 Apr 2025 11:57:29 -0400 Subject: [PATCH 05/14] test: older versions of next have different cache headers --- tests/e2e/dynamic-cms.test.ts | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/e2e/dynamic-cms.test.ts b/tests/e2e/dynamic-cms.test.ts index cb0ac01828..d40f045257 100644 --- a/tests/e2e/dynamic-cms.test.ts +++ b/tests/e2e/dynamic-cms.test.ts @@ -13,7 +13,9 @@ test.describe('Dynamic CMS', () => { '"Next.js"; fwd=miss, "Netlify Durable"; fwd=uri-miss; stored, "Netlify Edge"; fwd=miss', ) expect(headers1['netlify-cache-tag']).toEqual('_n_t_/content/blog') - expect(headers1['netlify-cdn-cache-control']).toEqual('s-maxage=31536000, durable') + expect(headers1['netlify-cdn-cache-control']).toMatch( + /s-maxage=31536000,( stale-while-revalidate=31536000,)? durable/ + ) // 2. Publish the blob, revalidate the dynamic page, and wait to regenerate await page.goto(new URL('/cms/publish', dynamicCms.url).href) @@ -30,7 +32,9 @@ test.describe('Dynamic CMS', () => { /"Next.js"; hit, "Netlify Durable"; fwd=stale; ttl=[0-9]+; stored, "Netlify Edge"; fwd=stale/, ) expect(headers2['netlify-cache-tag']).toEqual('_n_t_/content/blog') - expect(headers2['netlify-cdn-cache-control']).toEqual('s-maxage=31536000, durable') + expect(headers2['netlify-cdn-cache-control']).toMatch( + /s-maxage=31536000,( stale-while-revalidate=31536000,)? durable/ + ) // 4. Unpublish the blob, revalidate the dynamic page, and wait to regenerate await page.goto(new URL('/cms/unpublish', dynamicCms.url).href) @@ -47,6 +51,8 @@ test.describe('Dynamic CMS', () => { /"Next.js"; fwd=miss, "Netlify Durable"; fwd=stale; ttl=[0-9]+; stored, "Netlify Edge"; fwd=stale/, ) expect(headers3['netlify-cache-tag']).toEqual('_n_t_/content/blog') - expect(headers3['netlify-cdn-cache-control']).toEqual('s-maxage=31536000, durable') + expect(headers3['netlify-cdn-cache-control']).toMatch( + /s-maxage=31536000,( stale-while-revalidate=31536000,)? durable/ + ) }) }) From 2b0280bb4c18ce1bd79a7289e847a88d6ad30073 Mon Sep 17 00:00:00 2001 From: Mateusz Bocian Date: Tue, 1 Apr 2025 11:57:55 -0400 Subject: [PATCH 06/14] test: edge cache response depends on the node we hit --- tests/e2e/dynamic-cms.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/e2e/dynamic-cms.test.ts b/tests/e2e/dynamic-cms.test.ts index d40f045257..92b9b2c020 100644 --- a/tests/e2e/dynamic-cms.test.ts +++ b/tests/e2e/dynamic-cms.test.ts @@ -29,7 +29,7 @@ test.describe('Dynamic CMS', () => { expect(response2?.status()).toEqual(200) expect(headers2['cache-control']).toEqual('public,max-age=0,must-revalidate') expect(headers2['cache-status']).toMatch( - /"Next.js"; hit, "Netlify Durable"; fwd=stale; ttl=[0-9]+; stored, "Netlify Edge"; fwd=stale/, + /"Next.js"; hit, "Netlify Durable"; fwd=stale; ttl=[0-9]+; stored, "Netlify Edge"; fwd=(stale|miss)/, ) expect(headers2['netlify-cache-tag']).toEqual('_n_t_/content/blog') expect(headers2['netlify-cdn-cache-control']).toMatch( @@ -48,7 +48,7 @@ test.describe('Dynamic CMS', () => { expect(response3?.status()).toEqual(404) expect(headers3['cache-control']).toEqual('public,max-age=0,must-revalidate') expect(headers3['cache-status']).toMatch( - /"Next.js"; fwd=miss, "Netlify Durable"; fwd=stale; ttl=[0-9]+; stored, "Netlify Edge"; fwd=stale/, + /"Next.js"; fwd=miss, "Netlify Durable"; fwd=stale; ttl=[0-9]+; stored, "Netlify Edge"; fwd=(stale|miss)/, ) expect(headers3['netlify-cache-tag']).toEqual('_n_t_/content/blog') expect(headers3['netlify-cdn-cache-control']).toMatch( From 0fd916d782f61688aa148de359b21e95533c4580 Mon Sep 17 00:00:00 2001 From: Mateusz Bocian Date: Tue, 1 Apr 2025 12:02:05 -0400 Subject: [PATCH 07/14] fix: set cache tags on 404 pages --- src/run/handlers/cache.cts | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/run/handlers/cache.cts b/src/run/handlers/cache.cts index fbc546560b..e2ffbcbd22 100644 --- a/src/run/handlers/cache.cts +++ b/src/run/handlers/cache.cts @@ -123,10 +123,6 @@ export class NetlifyCacheHandler implements CacheHandlerForMultipleVersions { private captureCacheTags(cacheValue: NetlifyIncrementalCacheValue | null, key: string) { const requestContext = getRequestContext() - if (!cacheValue) { - return - } - // Bail if we can't get request context if (!requestContext) { return @@ -142,6 +138,13 @@ export class NetlifyCacheHandler implements CacheHandlerForMultipleVersions { return } + // Set cache tags for 404 pages as well so that the content can later be purged + if (!cacheValue) { + const cacheTags = [`_N_T_${key === '/index' ? '/' : encodeURI(key)}`] + requestContext.responseCacheTags = cacheTags + return + } + if ( cacheValue.kind === 'PAGE' || cacheValue.kind === 'PAGES' || From d96b20c69519746fc10ddaaf8679120ba3775887 Mon Sep 17 00:00:00 2001 From: Mateusz Bocian Date: Tue, 1 Apr 2025 12:03:28 -0400 Subject: [PATCH 08/14] chore: clean up variable names --- src/run/handlers/cache.cts | 6 +++--- src/run/revalidate.ts | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/run/handlers/cache.cts b/src/run/handlers/cache.cts index e2ffbcbd22..ffa35b795c 100644 --- a/src/run/handlers/cache.cts +++ b/src/run/handlers/cache.cts @@ -230,7 +230,7 @@ export class NetlifyCacheHandler implements CacheHandlerForMultipleVersions { ...args: Parameters ): ReturnType { return this.tracer.withActiveSpan('get cache key', async (span) => { - const [key, ctx = {}] = args + const [key, context = {}] = args getLogger().debug(`[NetlifyCacheHandler.get]: ${key}`) span.setAttributes({ key }) @@ -263,7 +263,7 @@ export class NetlifyCacheHandler implements CacheHandlerForMultipleVersions { return null } - const staleByTags = await this.checkCacheEntryStaleByTags(blob, ctx.tags, ctx.softTags) + const staleByTags = await this.checkCacheEntryStaleByTags(blob, context.tags, context.softTags) if (staleByTags) { span.addEvent('Stale', { staleByTags, key, ttl }) @@ -277,7 +277,7 @@ export class NetlifyCacheHandler implements CacheHandlerForMultipleVersions { case 'FETCH': span.addEvent('FETCH', { lastModified: blob.lastModified, - revalidate: ctx.revalidate, + revalidate: context.revalidate, ttl, }) return { diff --git a/src/run/revalidate.ts b/src/run/revalidate.ts index 25a5b86660..6789072209 100644 --- a/src/run/revalidate.ts +++ b/src/run/revalidate.ts @@ -15,8 +15,8 @@ function isRevalidateMethod( } // Needing to proxy the response object to intercept the revalidate call for on-demand revalidation on page routes -export const nextResponseProxy = (res: ServerResponse, requestContext: RequestContext) => { - return new Proxy(res, { +export const nextResponseProxy = (response: ServerResponse, requestContext: RequestContext) => { + return new Proxy(response, { get(target: ServerResponse, key: string) { const originalValue = Reflect.get(target, key) if (isRevalidateMethod(key, originalValue)) { From de960f4c76e83e0cfcfb028516034f900747cdab Mon Sep 17 00:00:00 2001 From: Mateusz Bocian Date: Tue, 1 Apr 2025 12:04:07 -0400 Subject: [PATCH 09/14] chore: remove previous attempt to set cache tags on 404s --- src/run/handlers/server.ts | 2 +- src/run/headers.ts | 7 ------- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/src/run/handlers/server.ts b/src/run/handlers/server.ts index b0b23392d4..72666dd5b3 100644 --- a/src/run/handlers/server.ts +++ b/src/run/handlers/server.ts @@ -132,7 +132,7 @@ export default async ( } setCacheControlHeaders(response, request, requestContext, nextConfig) - setCacheTagsHeaders(response.headers, request, requestContext) + setCacheTagsHeaders(response.headers, requestContext) setVaryHeaders(response.headers, request, nextConfig) setCacheStatusHeader(response.headers, nextCache) diff --git a/src/run/headers.ts b/src/run/headers.ts index 9f1d156437..2495844bea 100644 --- a/src/run/headers.ts +++ b/src/run/headers.ts @@ -285,7 +285,6 @@ export const setCacheControlHeaders = ( export const setCacheTagsHeaders = ( headers: Headers, - request: Request, requestContext: RequestContext, ) => { if (!headers.has('cache-control') && !headers.has('netlify-cdn-cache-control')) { @@ -294,13 +293,7 @@ export const setCacheTagsHeaders = ( if (requestContext.responseCacheTags) { headers.set('netlify-cache-tag', requestContext.responseCacheTags.join(',')) - return } - - const key = new URL(request.url).pathname - const cacheTag = `_N_T_${key === '/index' ? '/' : encodeURI(key)}` - console.log('setCacheTagsHeaders', 'netlify-cache-tag', key) - headers.set('netlify-cache-tag', cacheTag) } /** From 4cf7b2f400a31d1a067ca8e129fae04106cf3070 Mon Sep 17 00:00:00 2001 From: Mateusz Bocian Date: Tue, 1 Apr 2025 12:09:58 -0400 Subject: [PATCH 10/14] fix: also target 404 pages when purging cache --- src/run/handlers/cache.cts | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/src/run/handlers/cache.cts b/src/run/handlers/cache.cts index ffa35b795c..6e51eb0129 100644 --- a/src/run/handlers/cache.cts +++ b/src/run/handlers/cache.cts @@ -271,7 +271,12 @@ export class NetlifyCacheHandler implements CacheHandlerForMultipleVersions { } this.captureResponseCacheLastModified(blob, key, span) - this.captureCacheTags(blob.value, key) + + // Next sets a kind/kindHint and fetchUrl for data requests, however fetchUrl was found to be most reliable across versions + const isDataRequest = Boolean(context.fetchUrl) + if (!isDataRequest) { + this.captureCacheTags(blob.value, key) + } switch (blob.value?.kind) { case 'FETCH': @@ -391,13 +396,17 @@ export class NetlifyCacheHandler implements CacheHandlerForMultipleVersions { const value = this.transformToStorableObject(data, context) - // if previous CacheHandler.get call returned null (page was either never rendered or was on-demand revalidated) - // and we didn't yet capture cache tags, we try to get cache tags from freshly produced cache value - this.captureCacheTags(value, key) + // Next sets a fetchCache and fetchUrl for data requests, however fetchUrl was found to be most reliable across versions + const isDataReq = Boolean(context.fetchUrl) + if (!isDataReq) { + // if previous CacheHandler.get call returned null (page was either never rendered or was on-demand revalidated) + // and we didn't yet capture cache tags, we try to get cache tags from freshly produced cache value + this.captureCacheTags(value, key) + } await this.cacheStore.set(key, { lastModified, value }, 'blobStore.set') - if (!data || data?.kind === 'PAGE' || data?.kind === 'PAGES') { + if ((!data && !isDataReq) || data?.kind === 'PAGE' || data?.kind === 'PAGES') { const requestContext = getRequestContext() if (requestContext?.didPagesRouterOnDemandRevalidate) { // encode here to deal with non ASCII characters in the key From ca7ca4e0fc556b2bf5919a42e42abb4a5d16d104 Mon Sep 17 00:00:00 2001 From: Mateusz Bocian Date: Tue, 1 Apr 2025 12:16:32 -0400 Subject: [PATCH 11/14] chore: remove wip check for cached api calls --- src/run/headers.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/run/headers.ts b/src/run/headers.ts index 9776dd9058..471464c0c6 100644 --- a/src/run/headers.ts +++ b/src/run/headers.ts @@ -266,7 +266,6 @@ export const setCacheControlHeaders = ( ['GET', 'HEAD'].includes(request.method) && !headers.has('cdn-cache-control') && !headers.has('netlify-cdn-cache-control') && - !new URL(request.url).pathname.startsWith('/api/') && requestContext.usedFsReadForNonFallback ) { // handle CDN Cache Control on static files From cf47c741d27f84c64542ab0829a9b481c6b21a0a Mon Sep 17 00:00:00 2001 From: Mateusz Bocian Date: Tue, 1 Apr 2025 12:20:01 -0400 Subject: [PATCH 12/14] chore: run format:fix --- src/run/handlers/cache.cts | 6 +++++- src/run/headers.ts | 5 +---- tests/e2e/dynamic-cms.test.ts | 6 +++--- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/run/handlers/cache.cts b/src/run/handlers/cache.cts index 6e51eb0129..4e9b61ce4a 100644 --- a/src/run/handlers/cache.cts +++ b/src/run/handlers/cache.cts @@ -263,7 +263,11 @@ export class NetlifyCacheHandler implements CacheHandlerForMultipleVersions { return null } - const staleByTags = await this.checkCacheEntryStaleByTags(blob, context.tags, context.softTags) + const staleByTags = await this.checkCacheEntryStaleByTags( + blob, + context.tags, + context.softTags, + ) if (staleByTags) { span.addEvent('Stale', { staleByTags, key, ttl }) diff --git a/src/run/headers.ts b/src/run/headers.ts index 471464c0c6..1931f61330 100644 --- a/src/run/headers.ts +++ b/src/run/headers.ts @@ -274,10 +274,7 @@ export const setCacheControlHeaders = ( } } -export const setCacheTagsHeaders = ( - headers: Headers, - requestContext: RequestContext, -) => { +export const setCacheTagsHeaders = (headers: Headers, requestContext: RequestContext) => { if (!headers.has('cache-control') && !headers.has('netlify-cdn-cache-control')) { return } diff --git a/tests/e2e/dynamic-cms.test.ts b/tests/e2e/dynamic-cms.test.ts index 92b9b2c020..a157b47727 100644 --- a/tests/e2e/dynamic-cms.test.ts +++ b/tests/e2e/dynamic-cms.test.ts @@ -14,7 +14,7 @@ test.describe('Dynamic CMS', () => { ) expect(headers1['netlify-cache-tag']).toEqual('_n_t_/content/blog') expect(headers1['netlify-cdn-cache-control']).toMatch( - /s-maxage=31536000,( stale-while-revalidate=31536000,)? durable/ + /s-maxage=31536000,( stale-while-revalidate=31536000,)? durable/, ) // 2. Publish the blob, revalidate the dynamic page, and wait to regenerate @@ -33,7 +33,7 @@ test.describe('Dynamic CMS', () => { ) expect(headers2['netlify-cache-tag']).toEqual('_n_t_/content/blog') expect(headers2['netlify-cdn-cache-control']).toMatch( - /s-maxage=31536000,( stale-while-revalidate=31536000,)? durable/ + /s-maxage=31536000,( stale-while-revalidate=31536000,)? durable/, ) // 4. Unpublish the blob, revalidate the dynamic page, and wait to regenerate @@ -52,7 +52,7 @@ test.describe('Dynamic CMS', () => { ) expect(headers3['netlify-cache-tag']).toEqual('_n_t_/content/blog') expect(headers3['netlify-cdn-cache-control']).toMatch( - /s-maxage=31536000,( stale-while-revalidate=31536000,)? durable/ + /s-maxage=31536000,( stale-while-revalidate=31536000,)? durable/, ) }) }) From caeef63d9e4f742daa5b3b5b34322ed5b1b84101 Mon Sep 17 00:00:00 2001 From: Michal Piechowiak Date: Wed, 2 Apr 2025 11:45:20 +0200 Subject: [PATCH 13/14] fix: don't set permanent caching header when res.revalidate() was used --- src/run/headers.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/run/headers.ts b/src/run/headers.ts index 1931f61330..35a1a4b37a 100644 --- a/src/run/headers.ts +++ b/src/run/headers.ts @@ -266,7 +266,8 @@ export const setCacheControlHeaders = ( ['GET', 'HEAD'].includes(request.method) && !headers.has('cdn-cache-control') && !headers.has('netlify-cdn-cache-control') && - requestContext.usedFsReadForNonFallback + requestContext.usedFsReadForNonFallback && + !requestContext.didPagesRouterOnDemandRevalidate ) { // handle CDN Cache Control on static files headers.set('cache-control', 'public, max-age=0, must-revalidate') From c3a0a48e4cea0bff2024d995fa87db14d807fc2e Mon Sep 17 00:00:00 2001 From: Michal Piechowiak Date: Wed, 2 Apr 2025 12:49:23 +0200 Subject: [PATCH 14/14] test: expand test fixture and cases for i18n and json requests --- tests/e2e/dynamic-cms.test.ts | 154 ++++++++++++------ .../netlify/functions/{cms.ts => cms.mts} | 12 +- tests/fixtures/dynamic-cms/next.config.js | 4 + tests/fixtures/dynamic-cms/package.json | 1 - .../dynamic-cms/pages/content/[...slug].js | 9 +- 5 files changed, 119 insertions(+), 61 deletions(-) rename tests/fixtures/dynamic-cms/netlify/functions/{cms.ts => cms.mts} (57%) diff --git a/tests/e2e/dynamic-cms.test.ts b/tests/e2e/dynamic-cms.test.ts index a157b47727..ae179c68fc 100644 --- a/tests/e2e/dynamic-cms.test.ts +++ b/tests/e2e/dynamic-cms.test.ts @@ -2,57 +2,107 @@ import { expect } from '@playwright/test' import { test } from '../utils/playwright-helpers.js' test.describe('Dynamic CMS', () => { - test('Invalidates 404 pages from durable cache', async ({ page, dynamicCms }) => { - // 1. Verify the status and headers of the dynamic page - const response1 = await page.goto(new URL('/content/blog', dynamicCms.url).href) - const headers1 = response1?.headers() || {} - - expect(response1?.status()).toEqual(404) - expect(headers1['cache-control']).toEqual('public,max-age=0,must-revalidate') - expect(headers1['cache-status']).toEqual( - '"Next.js"; fwd=miss, "Netlify Durable"; fwd=uri-miss; stored, "Netlify Edge"; fwd=miss', - ) - expect(headers1['netlify-cache-tag']).toEqual('_n_t_/content/blog') - expect(headers1['netlify-cdn-cache-control']).toMatch( - /s-maxage=31536000,( stale-while-revalidate=31536000,)? durable/, - ) - - // 2. Publish the blob, revalidate the dynamic page, and wait to regenerate - await page.goto(new URL('/cms/publish', dynamicCms.url).href) - await page.goto(new URL('/api/revalidate?path=/content/blog', dynamicCms.url).href) - await page.waitForTimeout(1000) - - // 3. Verify the status and headers of the dynamic page - const response2 = await page.goto(new URL('/content/blog', dynamicCms.url).href) - const headers2 = response2?.headers() || {} - - expect(response2?.status()).toEqual(200) - expect(headers2['cache-control']).toEqual('public,max-age=0,must-revalidate') - expect(headers2['cache-status']).toMatch( - /"Next.js"; hit, "Netlify Durable"; fwd=stale; ttl=[0-9]+; stored, "Netlify Edge"; fwd=(stale|miss)/, - ) - expect(headers2['netlify-cache-tag']).toEqual('_n_t_/content/blog') - expect(headers2['netlify-cdn-cache-control']).toMatch( - /s-maxage=31536000,( stale-while-revalidate=31536000,)? durable/, - ) - - // 4. Unpublish the blob, revalidate the dynamic page, and wait to regenerate - await page.goto(new URL('/cms/unpublish', dynamicCms.url).href) - await page.goto(new URL('/api/revalidate?path=/content/blog', dynamicCms.url).href) - await page.waitForTimeout(1000) - - // 5. Verify the status and headers of the dynamic page - const response3 = await page.goto(new URL('/content/blog', dynamicCms.url).href) - const headers3 = response3?.headers() || {} - - expect(response3?.status()).toEqual(404) - expect(headers3['cache-control']).toEqual('public,max-age=0,must-revalidate') - expect(headers3['cache-status']).toMatch( - /"Next.js"; fwd=miss, "Netlify Durable"; fwd=stale; ttl=[0-9]+; stored, "Netlify Edge"; fwd=(stale|miss)/, - ) - expect(headers3['netlify-cache-tag']).toEqual('_n_t_/content/blog') - expect(headers3['netlify-cdn-cache-control']).toMatch( - /s-maxage=31536000,( stale-while-revalidate=31536000,)? durable/, - ) + test.describe('Invalidates 404 pages from durable cache', () => { + // using postFix allows to rerun tests without having to redeploy the app because paths/keys will be unique for each test run + const postFix = Date.now() + for (const { label, contentKey, expectedCacheTag, urlPath, pathToRevalidate } of [ + { + label: 'Invalidates 404 html from durable cache (implicit default locale)', + urlPath: `/content/html-implicit-default-locale-${postFix}`, + contentKey: `html-implicit-default-locale-${postFix}`, + expectedCacheTag: `_n_t_/en/content/html-implicit-default-locale-${postFix}`, + }, + { + label: 'Invalidates 404 html from durable cache (explicit default locale)', + urlPath: `/en/content/html-explicit-default-locale-${postFix}`, + contentKey: `html-explicit-default-locale-${postFix}`, + expectedCacheTag: `_n_t_/en/content/html-explicit-default-locale-${postFix}`, + }, + // json paths don't have implicit locale routing + { + label: 'Invalidates 404 json from durable cache (default locale)', + urlPath: `/_next/data/build-id/en/content/json-default-locale-${postFix}.json`, + // for html, we can use html path as param for revalidate, + // for json we can't use json path and instead use one of html paths + // let's use implicit default locale here, as we will have another case for + // non-default locale which will have to use explicit one + pathToRevalidate: `/content/json-default-locale-${postFix}`, + contentKey: `json-default-locale-${postFix}`, + expectedCacheTag: `_n_t_/en/content/json-default-locale-${postFix}`, + }, + { + label: 'Invalidates 404 html from durable cache (non-default locale)', + urlPath: `/fr/content/html-non-default-locale-${postFix}`, + contentKey: `html-non-default-locale-${postFix}`, + expectedCacheTag: `_n_t_/fr/content/html-non-default-locale-${postFix}`, + }, + { + label: 'Invalidates 404 json from durable cache (non-default locale)', + urlPath: `/_next/data/build-id/fr/content/json-non-default-locale-${postFix}.json`, + pathToRevalidate: `/fr/content/json-non-default-locale-${postFix}`, + contentKey: `json-non-default-locale-${postFix}`, + expectedCacheTag: `_n_t_/fr/content/json-non-default-locale-${postFix}`, + }, + ]) { + test(label, async ({ page, dynamicCms }) => { + const routeUrl = new URL(urlPath, dynamicCms.url).href + const revalidateAPiUrl = new URL( + `/api/revalidate?path=${pathToRevalidate ?? urlPath}`, + dynamicCms.url, + ).href + + // 1. Verify the status and headers of the dynamic page + const response1 = await page.goto(routeUrl) + const headers1 = response1?.headers() || {} + + expect(response1?.status()).toEqual(404) + expect(headers1['cache-control']).toEqual('public,max-age=0,must-revalidate') + expect(headers1['cache-status']).toEqual( + '"Next.js"; fwd=miss, "Netlify Durable"; fwd=uri-miss; stored, "Netlify Edge"; fwd=miss', + ) + expect(headers1['netlify-cache-tag']).toEqual(expectedCacheTag) + expect(headers1['netlify-cdn-cache-control']).toMatch( + /s-maxage=31536000,( stale-while-revalidate=31536000,)? durable/, + ) + + // 2. Publish the blob, revalidate the dynamic page, and wait to regenerate + await page.goto(new URL(`/cms/publish/${contentKey}`, dynamicCms.url).href) + await page.goto(revalidateAPiUrl) + await page.waitForTimeout(1000) + + // 3. Verify the status and headers of the dynamic page + const response2 = await page.goto(routeUrl) + const headers2 = response2?.headers() || {} + + expect(response2?.status()).toEqual(200) + expect(headers2['cache-control']).toEqual('public,max-age=0,must-revalidate') + expect(headers2['cache-status']).toMatch( + /"Next.js"; hit, "Netlify Durable"; fwd=stale; ttl=[0-9]+; stored, "Netlify Edge"; fwd=(stale|miss)/, + ) + expect(headers2['netlify-cache-tag']).toEqual(expectedCacheTag) + expect(headers2['netlify-cdn-cache-control']).toMatch( + /s-maxage=31536000,( stale-while-revalidate=31536000,)? durable/, + ) + + // 4. Unpublish the blob, revalidate the dynamic page, and wait to regenerate + await page.goto(new URL(`/cms/unpublish/${contentKey}`, dynamicCms.url).href) + await page.goto(revalidateAPiUrl) + await page.waitForTimeout(1000) + + // 5. Verify the status and headers of the dynamic page + const response3 = await page.goto(routeUrl) + const headers3 = response3?.headers() || {} + + expect(response3?.status()).toEqual(404) + expect(headers3['cache-control']).toEqual('public,max-age=0,must-revalidate') + expect(headers3['cache-status']).toMatch( + /"Next.js"; fwd=miss, "Netlify Durable"; fwd=stale; ttl=[0-9]+; stored, "Netlify Edge"; fwd=(stale|miss)/, + ) + expect(headers3['netlify-cache-tag']).toEqual(expectedCacheTag) + expect(headers3['netlify-cdn-cache-control']).toMatch( + /s-maxage=31536000,( stale-while-revalidate=31536000,)? durable/, + ) + }) + } }) }) diff --git a/tests/fixtures/dynamic-cms/netlify/functions/cms.ts b/tests/fixtures/dynamic-cms/netlify/functions/cms.mts similarity index 57% rename from tests/fixtures/dynamic-cms/netlify/functions/cms.ts rename to tests/fixtures/dynamic-cms/netlify/functions/cms.mts index 243421e0be..fb3a9ad2d5 100644 --- a/tests/fixtures/dynamic-cms/netlify/functions/cms.ts +++ b/tests/fixtures/dynamic-cms/netlify/functions/cms.mts @@ -4,21 +4,25 @@ import { Context } from '@netlify/functions' // publish or unpublish "cms content" depending on the sent operation export default async function handler(_request: Request, context: Context) { const store = getDeployStore({ name: 'cms-content', consistency: 'strong' }) - const BLOB_KEY = 'key' const operation = context.params['operation'] + // root of optional catch-all route in Next.js sets 'index.html' as param + // while it's undefined in the Netlify function, because we need to declare + // path without wildcard + const contentKey = context.params['0'] ?? 'index.html' + if (operation === 'publish') { - await store.setJSON(BLOB_KEY, { content: true }) + await store.setJSON(contentKey, { content: true }) } if (operation === 'unpublish') { - await store.delete(BLOB_KEY) + await store.delete(contentKey) } return Response.json({ ok: true }) } export const config = { - path: '/cms/:operation', + path: ['/cms/:operation/*', '/cms/:operation'], } diff --git a/tests/fixtures/dynamic-cms/next.config.js b/tests/fixtures/dynamic-cms/next.config.js index 6346ab0742..fb5bd7c039 100644 --- a/tests/fixtures/dynamic-cms/next.config.js +++ b/tests/fixtures/dynamic-cms/next.config.js @@ -4,6 +4,10 @@ const nextConfig = { eslint: { ignoreDuringBuilds: true, }, + i18n: { + locales: ['en', 'fr'], + defaultLocale: 'en', + }, generateBuildId: () => 'build-id', } diff --git a/tests/fixtures/dynamic-cms/package.json b/tests/fixtures/dynamic-cms/package.json index 33ebc7f177..ba2eca67e8 100644 --- a/tests/fixtures/dynamic-cms/package.json +++ b/tests/fixtures/dynamic-cms/package.json @@ -10,7 +10,6 @@ "dependencies": { "@netlify/blobs": "^8.1.0", "@netlify/functions": "^2.7.0", - "@netlify/plugin-nextjs": "^5.10.1", "netlify-cli": "^19.0.3", "next": "latest", "react": "18.2.0", diff --git a/tests/fixtures/dynamic-cms/pages/content/[...slug].js b/tests/fixtures/dynamic-cms/pages/content/[...slug].js index 54abf74b16..070d222e99 100644 --- a/tests/fixtures/dynamic-cms/pages/content/[...slug].js +++ b/tests/fixtures/dynamic-cms/pages/content/[...slug].js @@ -8,11 +8,12 @@ const Content = ({ value }) => ( ) -export async function getStaticProps() { +export async function getStaticProps({ params }) { + const contentKey = params.slug.join('/') + const store = getDeployStore({ name: 'cms-content', consistency: 'strong' }) - const BLOB_KEY = 'key' - const value = await store.get(BLOB_KEY, { type: 'json' }) + const value = await store.get(contentKey, { type: 'json' }) if (!value) { return { @@ -22,7 +23,7 @@ export async function getStaticProps() { return { props: { - value: value, + value, }, } }