Merge pull request github#18476 from github/refactor-site-tree

Refactor site tree

lib/create-tree.js

@@ -0,0 +1,62 @@
+const fs = require('fs').promises
+const path = require('path')
+const Page = require('./page')
+const { sortBy } = require('lodash')
+const basePath = path.posix.join(__dirname, '..', 'content')
+
+module.exports = async function createTree (originalPath, langObj) {
+  // On recursive runs, this is processing page.children items in `/<link>` format.
+  // If the path exists as is, assume this is a directory with a child index.md.
+  // Otherwise, assume it's a child .md file and add `.md` to the path.
+  let filepath
+  try {
+    await fs.access(originalPath)
+    filepath = `${originalPath}/index.md`
+  } catch {
+    filepath = `${originalPath}.md`
+  }
+
+  const relativePath = filepath.replace(`${basePath}/`, '')
+  const localizedBasePath = path.posix.join(__dirname, '..', langObj.dir, 'content')
+
+  // Initialize the Page! This is where the file reads happen.
+  let page = await Page.init({
+    basePath: localizedBasePath,
+    relativePath,
+    languageCode: langObj.code
+  })
+
+  if (!page) {
+    // Do not throw an error if Early Access is not available.
+    if (relativePath.startsWith('early-access')) return
+    // If a translated path doesn't exist, fall back to the English so there is parity between
+    // the English tree and the translated trees.
+    if (langObj.code !== 'en') {
+      page = await Page.init({
+        basePath: basePath,
+        relativePath,
+        languageCode: langObj.code
+      })
+    }
+
+    if (!page) throw Error(`Cannot initialize page for ${filepath}`)
+  }
+
+  // Create the root tree object on the first run, and create children recursively.
+  const item = {
+    page
+  }
+
+  // Process frontmatter children recursively.
+  if (item.page.children) {
+    item.childPages = sortBy(
+      (await Promise.all(item.page.children
+        .map(async (child) => await createTree(path.posix.join(originalPath, child), langObj))))
+        .filter(Boolean),
+      // Sort by the ordered array of `children` in the frontmatter.
+      item.page.children
+    )
+  }
+
+  return item
+}

lib/page-data.js

@@ -0,0 +1,121 @@
+const path = require('path')
+const languages = require('./languages')
+const versions = Object.keys(require('./all-versions'))
+const createTree = require('./create-tree')
+const nonEnterpriseDefaultVersion = require('./non-enterprise-default-version')
+const englishPath = path.posix.join(__dirname, '..', 'content')
+
+/**
+ * We only need to initialize pages _once per language_ since pages don't change per version. So we do that
+ * first since it's the most expensive work. This gets us a nested object with pages attached that we can use
+ * as the basis for the siteTree after we do some versioning. We can also use it to derive the pageList.
+*/
+async function loadUnversionedTree () {
+  const unversionedTree = {}
+
+  await Promise.all(Object.values(languages)
+    .map(async (langObj) => {
+      unversionedTree[langObj.code] = await createTree(englishPath, langObj)
+    }))
+
+  return unversionedTree
+}
+
+/**
+ * The siteTree is a nested object with pages for every language and version, useful for nav because it
+ * contains parent, child, and sibling relationships:
+ *
+ * siteTree[languageCode][version].childPages[<array of pages>].childPages[<array of pages>] (etc...)
+
+* Given an unversioned tree of all pages per language, we can walk it for each version and do a couple operations:
+ * 1. Add a versioned href to every item, where the href is the relevant permalink for the current version.
+ * 2. Drop any child pages that are not available in the current version.
+ *
+ * Order of languages and versions doesn't matter, but order of child page arrays DOES matter (for navigation).
+*/
+async function loadSiteTree (unversionedTree) {
+  const rawTree = Object.assign({}, (unversionedTree || await loadUnversionedTree()))
+  const siteTree = {}
+
+  // For every language...
+  await Promise.all(Object.keys(languages).map(async (langCode) => {
+    const treePerVersion = {}
+    // in every version...
+    await Promise.all(versions.map(async (version) => {
+      // "version" the pages.
+      treePerVersion[version] = versionPages(Object.assign({}, rawTree[langCode]), version)
+    }))
+
+    siteTree[langCode] = treePerVersion
+  }))
+
+  return siteTree
+}
+
+// This step can't be asynchronous because the order of child pages matters.
+function versionPages (obj, version) {
+  // Add a versioned href as a convenience for use in layouts.
+  obj.href = obj.page.permalinks
+    .find(pl => pl.pageVersion === version || (pl.pageVersion === 'homepage' && version === nonEnterpriseDefaultVersion))
+    .href
+
+  if (!obj.childPages) return obj
+
+  const versionedChildPages = obj.childPages
+    // Drop child pages that do not apply to the current version.
+    .filter(childPage => childPage.page.applicableVersions.includes(version))
+    // Version the child pages recursively.
+    .map(childPage => versionPages(Object.assign({}, childPage), version))
+
+  obj.childPages = [...versionedChildPages]
+
+  return obj
+}
+
+// Derive a flat array of Page objects in all languages.
+async function loadPageList (unversionedTree) {
+  const rawTree = unversionedTree || await loadUnversionedTree()
+  const pageList = []
+
+  await Promise.all(Object.keys(languages).map(async (langCode) => {
+    await addToCollection(rawTree[langCode], pageList)
+  }))
+
+  async function addToCollection (item, collection) {
+    if (!item.page) return
+    collection.push(item.page)
+
+    if (!item.childPages) return
+    await Promise.all(item.childPages.map(async (childPage) => await addToCollection(childPage, collection)))
+  }
+
+  return pageList
+}
+
+// Create an object from the list of all pages with permalinks as keys for fast lookup.
+function createMapFromArray (pageList) {
+  const pageMap =
+    pageList.reduce(
+      (pageMap, page) => {
+        for (const permalink of page.permalinks) {
+          pageMap[permalink.href] = page
+        }
+        return pageMap
+      },
+      {}
+    )
+
+  return pageMap
+}
+
+async function loadPageMap (pageList) {
+  const pages = pageList || await loadPageList()
+  return createMapFromArray(pages)
+}
+
+module.exports = {
+  loadUnversionedTree,
+  loadSiteTree,
+  loadPages: loadPageList,
+  loadPageMap
+}

lib/warm-server2.js

@@ -0,0 +1,87 @@
+const statsd = require('./statsd')
+const { loadUnversionedTree, loadSiteTree, loadPages, loadPageMap } = require('./page-data')
+const loadRedirects = require('./redirects/precompile')
+const loadSiteData = require('./site-data')
+
+// Instrument these functions so that
+// it's wrapped in a timer that reports to Datadog
+const dog = {
+  loadUnversionedTree: statsd.asyncTimer(loadUnversionedTree, 'load_unversioned_tree'),
+  loadSiteTree: statsd.asyncTimer(loadSiteTree, 'load_site_tree'),
+  loadPages: statsd.asyncTimer(loadPages, 'load_pages'),
+  loadPageMap: statsd.asyncTimer(loadPageMap, 'load_page_map'),
+  loadRedirects: statsd.asyncTimer(loadRedirects, 'load_redirects'),
+  loadSiteData: statsd.timer(loadSiteData, 'load_site_data')
+}
+
+// For local caching
+let pageList, pageMap, site, redirects, unversionedTree, siteTree
+
+function isFullyWarmed () {
+  // NOTE: Yes, `pageList` is specifically excluded here as it is transient data
+  const fullyWarmed = !!(pageMap && site && redirects && unversionedTree && siteTree)
+  return fullyWarmed
+}
+
+function getWarmedCache () {
+  return {
+    pages: pageMap,
+    site,
+    redirects,
+    unversionedTree,
+    siteTree
+  }
+}
+
+async function warmServer () {
+  const startTime = Date.now()
+
+  if (process.env.NODE_ENV !== 'test') {
+    console.log('Priming context information...')
+  }
+
+  if (!unversionedTree) {
+    unversionedTree = await dog.loadUnversionedTree()
+  }
+
+  if (!siteTree) {
+    siteTree = await dog.loadSiteTree(unversionedTree)
+  }
+
+  if (!pageList) {
+    pageList = await dog.loadPages(unversionedTree)
+  }
+
+  if (!pageMap) {
+    pageMap = await dog.loadPageMap(pageList)
+  }
+
+  if (!site) {
+    site = dog.loadSiteData()
+  }
+
+  if (!redirects) {
+    redirects = await dog.loadRedirects(pageList)
+  }
+
+  if (process.env.NODE_ENV !== 'test') {
+    console.log(`Context primed in ${Date.now() - startTime} ms`)
+  }
+
+  return getWarmedCache()
+}
+
+// Instrument the `warmServer` function so that
+// it's wrapped in a timer that reports to Datadog
+dog.warmServer = statsd.asyncTimer(warmServer, 'warm_server')
+
+// We only want statistics if the priming needs to occur, so let's wrap the
+// real method and return early [without statistics] whenever possible
+module.exports = async function warmServerWrapper () {
+  // Bail out early if everything is properly ready to use
+  if (isFullyWarmed()) {
+    return getWarmedCache()
+  }
+
+  return dog.warmServer()
+}

middleware/breadcrumbs.js

@@ -19,6 +19,11 @@ module.exports = async function breadcrumbs (req, res, next) {
   pathParts.shift()
 
   const productPath = path.posix.join('/', req.context.currentProduct)
+
+  if (!req.context.siteTree[req.language][req.context.currentVersion].products) {
+    return next()
+  }
+
   const product = req.context.siteTree[req.language][req.context.currentVersion].products[req.context.currentProduct]
 
   if (!product) {