Skip to content

Commit

Permalink
feat(insights): Use insights to prefetch bundles in priority order
Browse files Browse the repository at this point in the history
Use the insights data to pre-fetch bundles in priority order based
on usage statistics.
  • Loading branch information
mhevery committed Oct 11, 2023
1 parent b3cdc4e commit 53a8f3b
Show file tree
Hide file tree
Showing 27 changed files with 358 additions and 214 deletions.
5 changes: 4 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,10 @@ temp

# Application
qwik-app/
**/server/
/server/
/starters/**/server/
/packages/*/server/
/packages/*/src/styled-system/
todo-express/
target
!/packages/docs/src/routes/demo/events/target
Expand Down
16 changes: 15 additions & 1 deletion packages/docs/src/routes/api/qwik-optimizer/api.json
Original file line number Diff line number Diff line change
Expand Up @@ -196,6 +196,20 @@
"editUrl": "https://github.com/BuilderIO/qwik/tree/main/packages/qwik/src/optimizer/src/types.ts",
"mdFile": "qwik.inlineentrystrategy.md"
},
{
"name": "InsightManifest",
"id": "insightmanifest",
"hierarchy": [
{
"name": "InsightManifest",
"id": "insightmanifest"
}
],
"kind": "Interface",
"content": "```typescript\nexport interface InsightManifest \n```\n\n\n| Property | Modifiers | Type | Description |\n| --- | --- | --- | --- |\n| [manual](#) | | Record<string, string> | |\n| [prefetch](#) | | { route: string; symbols: string\\[\\]; }\\[\\] | |\n| [type](#) | | 'smart' | |",
"editUrl": "https://github.com/BuilderIO/qwik/tree/main/packages/qwik/src/optimizer/src/types.ts",
"mdFile": "qwik.insightmanifest.md"
},
{
"name": "isAbsolute",
"id": "path-isabsolute",
Expand Down Expand Up @@ -484,7 +498,7 @@
}
],
"kind": "Interface",
"content": "```typescript\nexport interface QwikVitePluginApi \n```\n\n\n| Property | Modifiers | Type | Description |\n| --- | --- | --- | --- |\n| [getClientOutDir](#) | | () => string \\| null | |\n| [getClientPublicOutDir](#) | | () => string \\| null | |\n| [getManifest](#) | | () => [QwikManifest](#qwikmanifest) \\| null | |\n| [getOptimizer](#) | | () => [Optimizer](#optimizer) \\| null | |\n| [getOptions](#) | | () => NormalizedQwikPluginOptions | |\n| [getRootDir](#) | | () => string \\| null | |",
"content": "```typescript\nexport interface QwikVitePluginApi \n```\n\n\n| Property | Modifiers | Type | Description |\n| --- | --- | --- | --- |\n| [getClientOutDir](#) | | () => string \\| null | |\n| [getClientPublicOutDir](#) | | () => string \\| null | |\n| [getInsightsManifest](#) | | () => Promise<[InsightManifest](#insightmanifest) \\| null> | |\n| [getManifest](#) | | () => [QwikManifest](#qwikmanifest) \\| null | |\n| [getOptimizer](#) | | () => [Optimizer](#optimizer) \\| null | |\n| [getOptions](#) | | () => NormalizedQwikPluginOptions | |\n| [getRootDir](#) | | () => string \\| null | |",
"editUrl": "https://github.com/BuilderIO/qwik/tree/main/packages/qwik/src/optimizer/src/plugins/vite.ts",
"mdFile": "qwik.qwikvitepluginapi.md"
},
Expand Down
31 changes: 23 additions & 8 deletions packages/docs/src/routes/api/qwik-optimizer/index.md
Original file line number Diff line number Diff line change
Expand Up @@ -192,6 +192,20 @@ export interface InlineEntryStrategy

[Edit this section](https://github.com/BuilderIO/qwik/tree/main/packages/qwik/src/optimizer/src/types.ts)

## InsightManifest

```typescript
export interface InsightManifest
```

| Property | Modifiers | Type | Description |
| ------------- | --------- | --------------------------------------- | ----------- |
| [manual](#) | | Record<string, string> | |
| [prefetch](#) | | { route: string; symbols: string[]; }[] | |
| [type](#) | | 'smart' | |

[Edit this section](https://github.com/BuilderIO/qwik/tree/main/packages/qwik/src/optimizer/src/types.ts)

## isAbsolute

```typescript
Expand Down Expand Up @@ -501,14 +515,15 @@ export interface QwikVitePlugin
export interface QwikVitePluginApi
```

| Property | Modifiers | Type | Description |
| -------------------------- | --------- | ---------------------------------------------- | ----------- |
| [getClientOutDir](#) | | () => string \| null | |
| [getClientPublicOutDir](#) | | () => string \| null | |
| [getManifest](#) | | () => [QwikManifest](#qwikmanifest) \| null | |
| [getOptimizer](#) | | () => [Optimizer](#optimizer) \| null | |
| [getOptions](#) | | () => NormalizedQwikPluginOptions | |
| [getRootDir](#) | | () => string \| null | |
| Property | Modifiers | Type | Description |
| -------------------------- | --------- | ------------------------------------------------------------------- | ----------- |
| [getClientOutDir](#) | | () => string \| null | |
| [getClientPublicOutDir](#) | | () => string \| null | |
| [getInsightsManifest](#) | | () => Promise<[InsightManifest](#insightmanifest) \| null> | |
| [getManifest](#) | | () => [QwikManifest](#qwikmanifest) \| null | |
| [getOptimizer](#) | | () => [Optimizer](#optimizer) \| null | |
| [getOptions](#) | | () => NormalizedQwikPluginOptions | |
| [getRootDir](#) | | () => string \| null | |

[Edit this section](https://github.com/BuilderIO/qwik/tree/main/packages/qwik/src/optimizer/src/plugins/vite.ts)

Expand Down
4 changes: 2 additions & 2 deletions packages/docs/src/routes/api/qwik-server/api.json
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,7 @@
}
],
"kind": "Interface",
"content": "```typescript\nexport interface RenderOptions extends SerializeDocumentOptions \n```\n**Extends:** [SerializeDocumentOptions](#serializedocumentoptions)\n\n\n| Property | Modifiers | Type | Description |\n| --- | --- | --- | --- |\n| [base?](#) | | string \\| ((options: [RenderOptions](#renderoptions)<!-- -->) =&gt; string) | _(Optional)_ Specifies the root of the JS files of the client build. Setting a base, will cause the render of the <code>q:base</code> attribute in the <code>q:container</code> element. |\n| [containerAttributes?](#) | | Record&lt;string, string&gt; | _(Optional)_ |\n| [containerTagName?](#) | | string | _(Optional)_ When set, the app is serialized into a fragment. And the returned html is not a complete document. Defaults to <code>html</code> |\n| [locale?](#) | | string \\| ((options: [RenderOptions](#renderoptions)<!-- -->) =&gt; string) | _(Optional)_ Language to use when rendering the document. |\n| [prefetchStrategy?](#) | | [PrefetchStrategy](#prefetchstrategy) \\| null | _(Optional)_ |\n| [qwikLoader?](#) | | [QwikLoaderOptions](#qwikloaderoptions) | _(Optional)_ Specifies if the Qwik Loader script is added to the document or not. Defaults to <code>{ include: true }</code>. |\n| [serverData?](#) | | Record&lt;string, any&gt; | _(Optional)_ |\n| [snapshot?](#) | | boolean | _(Optional)_ Defaults to <code>true</code> |",
"content": "```typescript\nexport interface RenderOptions extends SerializeDocumentOptions \n```\n**Extends:** [SerializeDocumentOptions](#serializedocumentoptions)\n\n\n| Property | Modifiers | Type | Description |\n| --- | --- | --- | --- |\n| [base?](#) | | string \\| ((options: [RenderOptions](#renderoptions)<!-- -->) =&gt; string) | _(Optional)_ Specifies the root of the JS files of the client build. Setting a base, will cause the render of the <code>q:base</code> attribute in the <code>q:container</code> element. |\n| [containerAttributes?](#) | | Record&lt;string, string&gt; | _(Optional)_ |\n| [containerTagName?](#) | | string | _(Optional)_ When set, the app is serialized into a fragment. And the returned html is not a complete document. Defaults to <code>html</code> |\n| [locale?](#) | | string \\| ((options: [RenderOptions](#renderoptions)<!-- -->) =&gt; string) | _(Optional)_ Language to use when rendering the document. |\n| [prefetchStrategy?](#) | | [PrefetchStrategy](#prefetchstrategy) \\| null | _(Optional)_ |\n| [qwikLoader?](#) | | [QwikLoaderOptions](#qwikloaderoptions) | <p>_(Optional)_ Specifies if the Qwik Loader script is added to the document or not.</p><p>Defaults to <code>{ include: true }</code>.</p> |\n| [serverData?](#) | | Record&lt;string, any&gt; | _(Optional)_ |\n| [snapshot?](#) | | boolean | _(Optional)_ Defaults to <code>true</code> |",
"editUrl": "https://github.com/BuilderIO/qwik/tree/main/packages/qwik/src/server/types.ts",
"mdFile": "qwik.renderoptions.md"
},
Expand Down Expand Up @@ -334,7 +334,7 @@
}
],
"kind": "TypeAlias",
"content": "auto: Prefetch all possible QRLs used by the document. Default\n\n\n```typescript\nexport type SymbolsToPrefetch = 'auto' | ((opts: {\n manifest: QwikManifest;\n}) => PrefetchResource[]);\n```\n**References:** [PrefetchResource](#prefetchresource)",
"content": "Auto: Prefetch all possible QRLs used by the document. Default\n\n\n```typescript\nexport type SymbolsToPrefetch = 'auto' | ((opts: {\n manifest: QwikManifest;\n}) => PrefetchResource[]);\n```\n**References:** [PrefetchResource](#prefetchresource)",
"editUrl": "https://github.com/BuilderIO/qwik/tree/main/packages/qwik/src/server/types.ts",
"mdFile": "qwik.symbolstoprefetch.md"
},
Expand Down
4 changes: 2 additions & 2 deletions packages/docs/src/routes/api/qwik-server/index.md
Original file line number Diff line number Diff line change
Expand Up @@ -149,7 +149,7 @@ export interface RenderOptions extends SerializeDocumentOptions
| [containerTagName?](#) | | string | _(Optional)_ When set, the app is serialized into a fragment. And the returned html is not a complete document. Defaults to <code>html</code> |
| [locale?](#) | | string \| ((options: [RenderOptions](#renderoptions)) =&gt; string) | _(Optional)_ Language to use when rendering the document. |
| [prefetchStrategy?](#) | | [PrefetchStrategy](#prefetchstrategy) \| null | _(Optional)_ |
| [qwikLoader?](#) | | [QwikLoaderOptions](#qwikloaderoptions) | _(Optional)_ Specifies if the Qwik Loader script is added to the document or not. Defaults to <code>{ include: true }</code>. |
| [qwikLoader?](#) | | [QwikLoaderOptions](#qwikloaderoptions) | <p>_(Optional)_ Specifies if the Qwik Loader script is added to the document or not.</p><p>Defaults to <code>{ include: true }</code>.</p> |
| [serverData?](#) | | Record&lt;string, any&gt; | _(Optional)_ |
| [snapshot?](#) | | boolean | _(Optional)_ Defaults to <code>true</code> |
Expand Down Expand Up @@ -338,7 +338,7 @@ export interface StreamingOptions
## SymbolsToPrefetch
auto: Prefetch all possible QRLs used by the document. Default
Auto: Prefetch all possible QRLs used by the document. Default
```typescript
export type SymbolsToPrefetch =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import { computeSymbolGraph, computeSymbolVectors, computeBundles } from '~/stat
import { getRoutes } from '~/db/sql-routes';

interface Strategy {
entryStrategy: { type: 'smart' };
type: 'smart';
manual: Record<string, string>;
prefetch: Prefetch[];
}
Expand All @@ -19,7 +19,7 @@ export const onGet: RequestHandler = async ({ json, params }) => {
const publicApiKey = params.publicApiKey;
const db = getDB();
const strategy: Strategy = {
entryStrategy: { type: 'smart' },
type: 'smart',
manual: {},
prefetch: [],
};
Expand Down
Original file line number Diff line number Diff line change
@@ -1,10 +1,9 @@
import type { BuildContext } from '../types';
import swRegister from '@qwik-city-sw-register-build';
import type { QwikManifest } from '@builder.io/qwik/optimizer';
import type { QwikManifest, InsightManifest } from '@builder.io/qwik/optimizer';
import type { AppBundle } from '../../runtime/src/service-worker/types';
import { removeExtension } from '../../utils/fs';

export function generateServiceWorkerRegister(ctx: BuildContext) {
export function generateServiceWorkerRegister(ctx: BuildContext, swRegister: string) {
let swReg: string;

if (ctx.isDevServer) {
Expand All @@ -29,6 +28,7 @@ export function generateServiceWorkerRegister(ctx: BuildContext) {
export function prependManifestToServiceWorker(
ctx: BuildContext,
manifest: QwikManifest,
prefetch: InsightManifest['prefetch'] | null,
swCode: string
) {
const key = `/* Qwik Service Worker */`;
Expand All @@ -41,7 +41,7 @@ export function prependManifestToServiceWorker(
const appBundles: AppBundle[] = [];
const appBundlesCode = generateAppBundles(appBundles, manifest);
const libraryBundlesCode = generateLibraryBundles(appBundles, manifest);
const linkBundlesCode = generateLinkBundles(ctx, appBundles, manifest);
const [linkBundlesCode] = generateLinkBundles(ctx, appBundles, manifest, prefetch);

return [key, appBundlesCode, libraryBundlesCode, linkBundlesCode, swCode].join('\n');
}
Expand Down Expand Up @@ -92,8 +92,21 @@ function generateLibraryBundles(appBundles: AppBundle[], manifest: QwikManifest)
return `const libraryBundleIds=${JSON.stringify(libraryBundleIds)};`;
}

function generateLinkBundles(ctx: BuildContext, appBundles: AppBundle[], manifest: QwikManifest) {
export function generateLinkBundles(
ctx: BuildContext,
appBundles: AppBundle[],
manifest: QwikManifest,
prefetch: InsightManifest['prefetch'] | null
) {
const linkBundles: string[] = [];
const symbolToBundle = new Map<string, string>();
const routeToBundles: Record<string, string[]> = {};
for (const bundleName in manifest.bundles || []) {
manifest.bundles[bundleName].symbols?.forEach((symbol) => {
const idx = symbol.lastIndexOf('_');
symbolToBundle.set(idx === -1 ? symbol : symbol.substring(idx + 1), bundleName);
});
}

for (const r of ctx.routes) {
const linkBundleNames: string[] = [];
Expand Down Expand Up @@ -128,14 +141,33 @@ function generateLinkBundles(ctx: BuildContext, appBundles: AppBundle[], manifes
}
addFileBundles(r.filePath);

if (prefetch) {
// process the symbols from insights prefetch
const symbolsForRoute = prefetch.find((p) => p.route === r.routeName);
symbolsForRoute?.symbols?.reverse().forEach((symbol) => {
const bundle = symbolToBundle.get(symbol);
if (bundle) {
const idx = linkBundleNames.indexOf(bundle);
if (idx !== -1) {
linkBundleNames.splice(idx, 1);
}
linkBundleNames.unshift(bundle);
}
});
}

linkBundles.push(
`[${r.pattern.toString()},${JSON.stringify(
linkBundleNames.map((bundleName) => getAppBundleId(appBundles, bundleName))
)}]`
);
routeToBundles[r.routeName] = linkBundleNames;
}

return `const linkBundles=[${linkBundles.join(',')}];`;
return [`const linkBundles=[${linkBundles.join(',')}];`, routeToBundles] as [
string,
typeof routeToBundles,
];
}

function getAppBundleId(appBundles: AppBundle[], bundleName: string) {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
import { suite } from 'uvu';
import { equal } from 'uvu/assert';
import type { BuildContext, BuildRoute } from '../types';
import type { QwikManifest, InsightManifest } from '@builder.io/qwik/optimizer';
import type { AppBundle } from '../../runtime/src/service-worker/types';
import { generateLinkBundles } from './generate-service-worker';

const swSuite = suite('lint');

swSuite('incorporate qwik-insights', () => {
const routes: BuildRoute[] = [
{
routeName: '/',
pattern: /\//,
filePath: './src/routes/index.tsx',
layouts: [],
} /* satisfies Partial<BuildRoute> */ as any,
{
routeName: '/routeA',
pattern: /\/routeA/,
filePath: './src/routes/routeA/index.tsx',
layouts: [],
} /* satisfies Partial<BuildRoute> */ as any,
];
const ctx: BuildContext = { routes } /* satisfies Partial<BuildContext> */ as any;
const appBundles: AppBundle[] = [
['q-bundle-a.js', [12]],
['q-bundle-b.js', [34]],
['q-bundle-123.js', [123]],
['q-bundle-234.js', [234]],
['q-bundle-345.js', [345]],
];
const manifest: QwikManifest = {
bundles: {
'q-bundle-a.js': {
origins: ['./src/routes/index.tsx'],
} /* satisfies Partial<QwikManifest['bundles'][0]> */ as any,
'q-bundle-b.js': {
origins: ['./src/routes/routeA/index.tsx'],
} /* satisfies Partial<QwikManifest['bundles'][0]> */ as any,
'q-bundle-123.js': {
symbols: ['s_123'],
} /* satisfies Partial<QwikManifest['bundles'][0]> */ as any,
'q-bundle-234.js': {
symbols: ['s_234'],
} /* satisfies Partial<QwikManifest['bundles'][0]> */ as any,
'q-bundle-345.js': {
symbols: ['s_345'],
} /* satisfies Partial<QwikManifest['bundles'][0]> */ as any,
},
} /* satisfies Partial<QwikManifest> */ as any;
const prefetch: InsightManifest['prefetch'] = [
{ route: '/', symbols: ['123', '234'] },
{ route: '/routeA', symbols: ['345'] },
];
const [_, routeToBundles] = generateLinkBundles(ctx, appBundles, manifest, prefetch);
equal(routeToBundles['/'], ['q-bundle-123.js', 'q-bundle-234.js', 'q-bundle-a.js']);
equal(routeToBundles['/routeA'], ['q-bundle-345.js', 'q-bundle-b.js']);
});

swSuite.run();
11 changes: 9 additions & 2 deletions packages/qwik-city/buildtime/vite/plugin.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import swRegister from '@qwik-city-sw-register-build';
import { createMdxTransformer, type MdxTransform } from '../markdown/mdx';
import { basename, join, resolve, extname } from 'node:path';
import type { Plugin, PluginOption, UserConfig, Rollup } from 'vite';
Expand Down Expand Up @@ -177,7 +178,7 @@ function qwikCityPlugin(userOpts?: QwikCityVitePluginOptions): any {

if (isSwRegister) {
// @qwik-city-sw-register
return generateServiceWorkerRegister(ctx);
return generateServiceWorkerRegister(ctx, swRegister);
}
}
}
Expand Down Expand Up @@ -257,6 +258,7 @@ function qwikCityPlugin(userOpts?: QwikCityVitePluginOptions): any {
if (ctx?.target === 'ssr') {
// ssr build
const manifest = qwikPlugin!.api.getManifest();
const insightsManifest = await qwikPlugin!.api.getInsightsManifest();
const clientOutDir = qwikPlugin!.api.getClientOutDir();

if (manifest && clientOutDir) {
Expand All @@ -268,7 +270,12 @@ function qwikCityPlugin(userOpts?: QwikCityVitePluginOptions): any {
const swClientDistPath = join(clientOutBaseDir, swEntry.chunkFileName);
const swCode = await fs.promises.readFile(swClientDistPath, 'utf-8');
try {
const swCodeUpdate = prependManifestToServiceWorker(ctx, manifest, swCode);
const swCodeUpdate = prependManifestToServiceWorker(
ctx,
manifest,
insightsManifest?.prefetch || null,
swCode
);
if (swCodeUpdate) {
await fs.promises.mkdir(clientOutDir, { recursive: true });
await fs.promises.writeFile(swClientDistPath, swCodeUpdate);
Expand Down
Loading

0 comments on commit 53a8f3b

Please sign in to comment.