gitbook-algolia-lambda.js

const https = require("https");
const algoliasearch = require("algoliasearch");
const aws = require("aws-sdk");

const SSM = new aws.SSM();

const SPACE_ID = "-Lzuzdhj8LjrQPaeyCxr-3757176148";

const orderMap = {
    "sample-apps": 0,
    "core-concepts/connecting-to-data-sources": 1,
    "core-concepts/displaying-data-read": 2,
    "core-concepts/capturing-data-write": 3,
    "core-concepts/writing-code": 4,
    "troubleshooting-guide": 5,
    "how-to-guides": 6,
    "framework-reference/cheat-sheet": 7
};

var options = {
  headers: {
    Authorization:
      "Bearer -M69pqpT9qMltpU6jHx0--M69pqpUahn0ZEcsN8tX",
    Cookie: "__cfduid=d8bac210f6e11cb26a8cd921f77727e3f1588323072",
  },
};

console.log("Loading function");


function getPage(pageId) {
  return new Promise((resolve, reject) => {
    const url = `https://api.gitbook.com/v1/spaces/${SPACE_ID}/content/page/${pageId}?format=markdown`;
    console.log("Getting URL", url);
    https.get(url, options, (res) => {
      res.setEncoding('utf8');
      let rawData = '';
      res.on('data', (chunk) => { rawData += chunk; });
      res.on('end', () => {
        try {
          const parsedData = JSON.parse(rawData);
          resolve(parsedData);
          console.log("fetched page " + pageId);
        } catch (e) {
          console.error(e.message);
          console.error("Full response body:", rawData);
          reject(e);
        }
      });
    });
  });
}

async function getPageRetry(pageId, retries) {
  let page = null;

  while (retries-- > 0) {
    try {
      console.log("Retries left", pageId, retries);
      page = await getPage(pageId);
      break;
    } catch (error) {
      await new Promise(resolve => setTimeout(resolve, 1000));
      continue;
    }
  }

  if (page == null) {
    throw new Error("Tried getting page " + retries + " times, but failed.");
  }

  return page;
}

const pages = {};


function pushChildPages(masterPage) {
  if (masterPage.pages) {
    masterPage.pages.forEach(page => {
      if (masterPage.path)
        page.path = (masterPage.path || masterPage.ref) + "/" + page.path;
      pushChildPages(page);
      page.pages = undefined;
      pages[page.uid] = page;
    });
  }
}

async function getAllPages(pageIds) {
  const allPages = [];
  for (const pageId of pageIds) {
    allPages.push(getPageRetry(pageId, 3));
  }
  return Promise.all(allPages);
}

function extractLinkMatches(markdown) {
  const regex =  /\[(.*?)\]\((.*?)\)/g;
  const matches = markdown.matchAll(regex);
  return [...matches];
}

function getAbsoluteLink(link, currentPageFullPath) {
  console.log("page path ", currentPageFullPath);
  console.log("relative link ", link);
  const linkPaths = link.split("/");
  const pagePaths = currentPageFullPath.split("/");
  let pathIndex = pagePaths.length > 1 ? (pagePaths.length - 1) : 0;
  linkPaths.map((path, index) => {
    if (path === "..") {
      pagePaths[pathIndex] = undefined;
      pathIndex -= 1;
    }
  });
  let absoluteLink = pagePaths.filter((path) => path !== undefined).join("/");
  if (absoluteLink.length > 0) {
    absoluteLink += "/";
  }
  absoluteLink += linkPaths.filter((path) => path !== "..").join("/");
  console.log("absolute link ", absoluteLink);
  return absoluteLink;
}

exports.handler = async (event, context, callback) => {
  console.log('Received event:', JSON.stringify(event, null, 2));

  const parameters = await loadParametersFromStore("/" + process.env.ENV + "/algolia");
  const client = algoliasearch(parameters.application_id, parameters.api_key);
  const algoliaIndex = client.initIndex("test_appsmith");

  return await new Promise((resolve, reject) => {
    https.get(`https://api.gitbook.com/v1/spaces/${SPACE_ID}/content`, options, (res) => {
      console.log("Setting up response handlers for GitBook API request");
      res.setEncoding('utf8');
      let rawData = '';
      res.on('data', (chunk) => { rawData += chunk; });
      res.on('end', () => {
        try {
          const parsedData = JSON.parse(rawData);
          let masterPage = parsedData;

          pushChildPages(masterPage);
          delete masterPage.pages;

          getAllPages(Object.keys(pages)).then(updatedPages => {
            // console.log("fetched all pages ", Object.keys(pages));
            updatedPages.forEach((page, index) => {
              // console.log("update page fullpath", page.uid);
              page.path = pages[page.uid].path;
              delete page.pages;
              page.objectID = page.uid;
              delete page.uid;
            });
            updatedPages.sort((pageA, pageB) => {
              const orderA = orderMap[pageA.path] || 999;
              const orderB = orderMap[pageB.path] || 999;
              return orderB - orderA;
            });
            console.log("sorted pages")
            updatedPages = updatedPages.map((item, index) => {return {...item, defaultOrder: index}});

            // Truncate large docs, marking few docs as default and changing the markdown key for backward compatibility
            updatedPages.filter(page => page.markdown)
              .forEach(page => {
                if (page.markdown) {
                  const size = JSON.stringify(page).length;
                  if (size > 10000) { 
                    page.markdown = page.markdown.substr(0, page.markdown.length - (JSON.stringify(page).length - 9900));
                  }
                  if (orderMap[page.path]) {
                    page.isDefault = true;
                  }
                  const matches = extractLinkMatches(page.markdown);
                  matches.map((match) => {
                    const link = match[2];
                    if (!link.startsWith('https:')) {
                      const absoluteLink = getAbsoluteLink(link, page.path);
                      console.log('replace', match[0], match[0].replace("(" + link + ")", "(" + absoluteLink + ")"));
                      page.markdown = page.markdown.replace(match[0], match[0].replace("(" + link + ")", "(" + absoluteLink + ")"));
                    }
                  });
                  page.document = page.markdown;
                  page.markdown = undefined;
                  page.kind = "document";
                }
            });//*/

            console.log("Pages:", updatedPages.map(page => ({objectID: page.objectID, size: JSON.stringify(page).length, title: page.title, path: page.path})));

            // resolve({
            //   statusCode: 200,
            //   body: JSON.stringify(updatedPages)
            // })
            if (updatedPages.length < 5) {
              reject({
                statusCode: 304,
                body: 'No content to update'
              });
            } else {
              algoliaIndex.replaceAllObjects(updatedPages, {
                autoGenerateObjectIDIfNotExist: true
              }).then(({ objectIDs }) => {
                console.log("Algolia upload finished", objectIDs);
                callback(null, 'Finished');
                resolve({
                  statusCode: 200,
                  body: JSON.stringify(updatedPages)
                });
              }).catch(e => {
                console.error("Algolia upload failed", e);
                reject({
                  statusCode: 500,
                  body: 'Algolia upload failed.'
                });
              });
            }
          });
        } catch (e) {
          reject({
              statusCode: 500,
              body: 'Most probably gitbook getPage apis failed'
          });
        }
      });
    }).on("error", (e) => {
      console.error(e);
      reject({
        statusCode: 500,
        body: "Error executing GitBook API request",
        error: e
      });
    });
  });

};

async function loadParametersFromStore(prefix) {
  const parametersResponse = await SSM.getParametersByPath({Path: prefix, WithDecryption: true}).promise();
  console.log("parametersResponse", parametersResponse);

  const parameters = {};
  for (const paramObject of parametersResponse.Parameters) {
    parameters[paramObject.Name.replace(prefix + "/", "")] = paramObject.Value;
  }

  console.log("Parameters", parameters);
  return parameters;
}