refactor: merge worker into fresh (denoland#2051)

.github/workflows/ci.yml

@@ -33,4 +33,4 @@ jobs:
 
       - name: Run Tests
         # TODO(kt3k): Enable type checking
-        run: deno test --allow-net --allow-env --allow-hrtime --no-check
+        run: deno test --allow-net --allow-read --allow-env --allow-hrtime --no-check

completions_v2.ts

@@ -0,0 +1,198 @@
+// Copyright 2022 the Deno authors. All rights reserved. MIT license.
+
+import {
+  checkVersionsFreshness,
+  fetchMeta,
+  fetchPackageData,
+  fetchVersions,
+  getFuse,
+  getInitialPackageList,
+  getLatestVersion,
+  getMeta,
+  getPreselectPath,
+  IMMUTABLE,
+  MAX_AGE_1_DAY,
+  MAX_AGE_1_HOUR,
+  packages,
+  toFileItems,
+  toPathDocs,
+  toStdPathDocs,
+  toStdVersionDocs,
+  toVersionDocs,
+  versions,
+} from "./util/completions_utils.ts";
+
+async function detailsStdVer(_req: Request, match: Record<string, string>) {
+  const { ver } = match;
+  const meta = getMeta("std", ver) ?? await fetchMeta("std", ver);
+  const body = {
+    kind: "markdown",
+    value: toStdVersionDocs(ver, meta),
+  };
+  return new Response(JSON.stringify(body), {
+    headers: {
+      "cache-control": IMMUTABLE,
+      "content-type": "application/json",
+    },
+  });
+}
+
+async function detailsStdVerPath(_req: Request, match: Record<string, string>) {
+  let { ver, path } = match;
+  if (ver === "_latest") {
+    ver = await getLatestVersion("std");
+  }
+  const meta = getMeta("std", ver) ?? await fetchMeta("std", ver);
+  const body = {
+    kind: "markdown",
+    value: toStdPathDocs(ver, path, meta),
+  };
+  return new Response(JSON.stringify(body), {
+    headers: {
+      "cache-control": IMMUTABLE,
+      "content-type": "application/json",
+    },
+  });
+}
+
+async function detailsXPkg(_req: Request, match: Record<string, string>) {
+  const { pkg } = match;
+  const pkgData = packages.get(pkg) ?? await fetchPackageData(pkg);
+  const body = {
+    kind: "markdown",
+    value:
+      `**${pkgData.name}**\n\n${pkgData.description}\n\n[code](https://deno.land/x/${pkg})${
+        pkgData.stars ? ` | stars: _${pkgData.stars}_` : ""
+      }\n\n`,
+  };
+  return new Response(JSON.stringify(body), {
+    headers: {
+      "cache-control": MAX_AGE_1_DAY,
+      "content-type": "application/json",
+    },
+  });
+}
+
+async function detailsXPkgVer(_req: Request, match: Record<string, string>) {
+  const { pkg, ver } = match;
+  const pkgData = packages.get(pkg) ?? await fetchPackageData(pkg);
+  const meta = getMeta(pkg, ver) ?? await fetchMeta(pkg, ver);
+  const body = {
+    kind: "markdown",
+    value: toVersionDocs(pkgData, ver, meta),
+  };
+  return new Response(JSON.stringify(body), {
+    headers: {
+      "cache-control": IMMUTABLE,
+      "content-type": "application/json",
+    },
+  });
+}
+
+async function detailsXPkgVerPath(
+  _req: Request,
+  match: Record<string, string>,
+) {
+  let { pkg, ver, path } = match;
+  if (ver === "_latest") {
+    ver = await getLatestVersion(pkg);
+  }
+  const meta = getMeta(pkg, ver) ?? await fetchMeta(pkg, ver);
+  const body = {
+    kind: "markdown",
+    value: toPathDocs(pkg, ver, path, meta),
+  };
+  return new Response(JSON.stringify(body), {
+    headers: {
+      "cache-control": IMMUTABLE,
+      "content-type": "application/json",
+    },
+  });
+}
+
+async function xPkg(_req: Request, match: Record<string, string>) {
+  const { pkg } = match;
+  if (!pkg) {
+    return new Response(
+      JSON.stringify({
+        items: await getInitialPackageList(),
+        isIncomplete: true,
+      }),
+      {
+        headers: {
+          "cache-control": MAX_AGE_1_HOUR,
+          "content-type": "application/json",
+        },
+      },
+    );
+  } else {
+    const fuse = await getFuse();
+    const foundItems = fuse.search(pkg);
+    const found: string[] = foundItems.map(({ item }: { item: string }) =>
+      item
+    );
+    const items = found.slice(0, 100);
+    const body = {
+      items,
+      isIncomplete: found.length > items.length,
+      preselect: (foundItems[0].score === 0) ? foundItems[0].item : undefined,
+    };
+    return new Response(JSON.stringify(body), {
+      headers: {
+        "cache-control": MAX_AGE_1_HOUR,
+        "content-type": "application/json",
+      },
+    });
+  }
+}
+
+async function xPkgVer(_req: Request, match: Record<string, string>) {
+  const { pkg, ver } = match;
+  checkVersionsFreshness();
+  const versionInfo = versions.get(pkg) ?? await fetchVersions(pkg);
+  const items = ver
+    ? versionInfo.versions.filter((v) => v.startsWith(ver))
+    : versionInfo.versions;
+  const body = {
+    items,
+    isIncomplete: false,
+    preselect: items.find((v) => v === versionInfo.latest),
+  };
+  return new Response(JSON.stringify(body), {
+    headers: {
+      "cache-control": MAX_AGE_1_HOUR,
+      "content-type": "application/json",
+    },
+  });
+}
+
+async function xPkgVerPath(_req: Request, match: Record<string, string>) {
+  let { pkg, ver, path } = match;
+  if (ver === "_latest") {
+    ver = await getLatestVersion(pkg);
+  }
+  const meta = getMeta(pkg, ver) ?? await fetchMeta(pkg, ver);
+  const items = toFileItems(meta, path);
+  const body = {
+    items,
+    isIncomplete: true,
+    preselect: getPreselectPath(items),
+  };
+  return new Response(JSON.stringify(body), {
+    headers: {
+      "cache-control": IMMUTABLE,
+      "content-type": "application/json",
+    },
+  });
+}
+
+export const routes = {
+  "/_api/details/std/:ver": detailsStdVer,
+  "/_api/details/std/:ver/:path*{/}?": detailsStdVerPath,
+  "/_api/details/x/:pkg": detailsXPkg,
+  "/_api/details/x/:pkg/:ver": detailsXPkgVer,
+  "/_api/details/x/:pkg/:ver/:path*{/}?": detailsXPkgVerPath,
+  "/_api/x/{:pkg}?": xPkg,
+  "/_api/x/:pkg/{:ver}?": xPkgVer,
+  "/_api/x/:pkg/:ver/:path*{/}?": xPkgVerPath,
+};

components/CodeBlock.tsx

@@ -5,27 +5,27 @@
 import { Fragment, h, htmlEscape, Prism } from "../deps.ts";
 import { normalizeTokens } from "../util/prism_utils.ts";
 
-import "https://esm.sh/[email protected]/components/prism-bash?no-check";
-//import "https://esm.sh/[email protected]/components/prism-batch?no-check";
-import "https://esm.sh/[email protected]/components/prism-css?no-check";
-//import "https://esm.sh/[email protected]/components/prism-css-extras?no-check";
-//import "https://esm.sh/[email protected]/components/prism-editorconfig?no-check";
-import "https://esm.sh/[email protected]/components/prism-diff?no-check";
-//import "https://esm.sh/[email protected]/components/prism-docker?no-check";
-//import "https://esm.sh/[email protected]/components/prism-git?no-check";
-//import "https://esm.sh/[email protected]/components/prism-ignore?no-check";
-import "https://esm.sh/[email protected]/components/prism-javascript?no-check";
-//import "https://esm.sh/[email protected]/components/prism-js-extras?no-check";
-//import "https://esm.sh/[email protected]/components/prism-js-templates?no-check";
+import "https://esm.sh/[email protected]/components/prism-bash?pin=v66&no-check";
+import "https://esm.sh/[email protected]/components/prism-batch?pin=v66&no-check";
+import "https://esm.sh/[email protected]/components/prism-css?pin=v66&no-check";
+import "https://esm.sh/[email protected]/components/prism-css-extras?pin=v66&no-check";
+import "https://esm.sh/[email protected]/components/prism-editorconfig?pin=v66&no-check";
+import "https://esm.sh/[email protected]/components/prism-diff?pin=v66&no-check";
+import "https://esm.sh/[email protected]/components/prism-docker?pin=v66&no-check";
+import "https://esm.sh/[email protected]/components/prism-git?pin=v66&no-check";
+import "https://esm.sh/[email protected]/components/prism-ignore?pin=v66&no-check";
+import "https://esm.sh/[email protected]/components/prism-javascript?pin=v66&no-check";
+import "https://esm.sh/[email protected]/components/prism-js-extras?pin=v66&no-check";
+import "https://esm.sh/[email protected]/components/prism-js-templates?pin=v66&no-check";
 //import "https://esm.sh/[email protected]/components/prism-jsdoc?no-check";
-import "https://esm.sh/[email protected]/components/prism-json?no-check";
-import "https://esm.sh/[email protected]/components/prism-jsx?no-check";
-//import "https://esm.sh/[email protected]/components/prism-markdown?no-check";
-import "https://esm.sh/[email protected]/components/prism-rust?no-check";
-//import "https://esm.sh/[email protected]/components/prism-toml?no-check";
-import "https://esm.sh/[email protected]/components/prism-tsx?no-check";
-import "https://esm.sh/[email protected]/components/prism-typescript?no-check";
-import "https://esm.sh/[email protected]/components/prism-yaml?no-check";
+import "https://esm.sh/[email protected]/components/prism-json?pin=v66&no-check";
+import "https://esm.sh/[email protected]/components/prism-jsx?pin=v66&no-check";
+import "https://esm.sh/[email protected]/components/prism-markdown?pin=v66&no-check";
+import "https://esm.sh/[email protected]/components/prism-rust?pin=v66&no-check";
+import "https://esm.sh/[email protected]/components/prism-toml?pin=v66&no-check";
+import "https://esm.sh/[email protected]/components/prism-tsx?pin=v66&no-check";
+import "https://esm.sh/[email protected]/components/prism-typescript?pin=v66&no-check";
+import "https://esm.sh/[email protected]/components/prism-yaml?pin=v66&no-check";
 
 // Modifies the color of 'variable' token
 // to avoid poor contrast

deps.ts

@@ -1,15 +1,17 @@
-export * from "https://raw.githubusercontent.com/lucacasonato/fresh/04a32f34987fbd59ce646967f216b2cca072a58f/runtime.ts";
-export { default as twas } from "https://esm.sh/[email protected]";
-export { create } from "https://esm.sh/[email protected]";
-export { css } from "https://esm.sh/[email protected]/css";
+export * from "https://raw.githubusercontent.com/lucacasonato/fresh/d441bf2489a51f67522ae2f932ae52e70f9ad131/runtime.ts";
+export { default as twas } from "https://esm.sh/[email protected]?pin=v66";
+export { create } from "https://esm.sh/[email protected]?pin=v66";
+export { css } from "https://esm.sh/[email protected]/css?pin=v66";
 export {
   apply,
   shim,
   theme,
   virtualSheet,
-} from "https://esm.sh/[email protected]/shim/server";
+} from "https://esm.sh/[email protected]/shim/server?pin=v66";
 
-export { default as Prism } from "https://esm.sh/[email protected]";
-export { escape as htmlEscape } from "https://esm.sh/[email protected]";
+export { default as Prism } from "https://esm.sh/[email protected]?pin=v66";
+export { escape as htmlEscape } from "https://esm.sh/[email protected]?pin=v66";
 export { render as gfm } from "https://raw.githubusercontent.com/crowlKats/deno-gfm/e1003cd3bb92f04bce8228eadb4a974ceb6fe260/mod.ts";
+
 export { emojify } from "https://deno.land/x/[email protected]/mod.ts";
+export { default as compareVersions } from "https://esm.sh/[email protected]?pin=v66";

main.ts

@@ -1,10 +1,94 @@
+// Copyright 2022 the Deno authors. All rights reserved. MIT license.
+
 /// <reference no-default-lib="true" />
 /// <reference lib="dom" />
 /// <reference lib="dom.asynciterable" />
 /// <reference lib="deno.ns" />
 /// <reference lib="deno.unstable" />
 
-import { start } from "https://raw.githubusercontent.com/lucacasonato/fresh/04a32f34987fbd59ce646967f216b2cca072a58f/server.ts";
+import {
+  accepts,
+  ConnInfo,
+  createReporter,
+  router,
+  serve,
+  ServerContext,
+} from "./server_deps.ts";
 import routes from "./routes.gen.ts";
 
-await start(routes);
+import { routes as completionsV2Routes } from "./completions_v2.ts";
+
+const ga = createReporter({
+  filter(req, res) {
+    const { pathname } = new URL(req.url);
+    const isHtml = accepts(req, "application/*", "text/html") === "text/html";
+    return pathname === "/" || pathname.startsWith("/std") ||
+      pathname.startsWith("/x") || isHtml || res.status >= 400;
+  },
+  metaData(req, res) {
+    const accept = req.headers.get("accept");
+    const referer = req.headers.get("referer");
+    const userAgent = req.headers.get("user-agent");
+
+    const { ok, statusText } = res;
+    const isHtml = accepts(req, "application/*", "text/html") === "text/html";
+
+    // Set the page title to "website" or "javascript" or "typescript" or "wasm"
+    const contentType = res.headers.get("content-type");
+    let documentTitle;
+    if (!ok) {
+      documentTitle = statusText.toLowerCase();
+    } else if (isHtml) {
+      documentTitle = "website";
+    } else if (contentType != null) {
+      documentTitle = /^application\/(.*?)(?:;|$)/i.exec(contentType)?.[1];
+    }
+
+    // Files downloaded by a bot (deno, curl) get a special medium/source tag.
+    let campaignMedium;
+    let campaignSource;
+    if (
+      referer == null &&
+      (userAgent == null || !userAgent.startsWith("Mozilla/")) &&
+      (accept == null || accept === "*/*")
+    ) {
+      campaignMedium = "Bot";
+      campaignSource = userAgent?.replace(/[^\w\-].*$/, "");
+    }
+
+    return { campaignMedium, campaignSource, documentTitle };
+  },
+});
+
+export function withLog(
+  handler: (request: Request) => Response | Promise<Response>,
+): (request: Request, connInfo: ConnInfo) => Promise<Response> {
+  return async (req, con) => {
+    let err: unknown;
+    let res!: Response;
+    const start = performance.now();
+    try {
+      res = await handler(req);
+    } catch (e) {
+      err = e;
+      console.error(err);
+      res = new Response(
+        "500 Internal Server Error\nPlease try again later.",
+        { status: 500 },
+      );
+    } finally {
+      await ga(req, con, res, start, err);
+    }
+    return res;
+  };
+}
+
+const ctx = await ServerContext.fromRoutes(routes);
+console.log("Server listening on http://localhost:8000");
+serve((req, conn) => {
+  return router(
+    completionsV2Routes,
+    (req) => withLog(ctx.handler())(req, conn),
+    // TODO: add 500 handler
+  )(req);
+});