Crawled the documentation to make a snapshot available in structured form in the repo

.env-template

@@ -0,0 +1,5 @@
+TERMINUSDB_API_TOKEN=1234
+TERMINUSDB_API_ENDPOINT=https://cloud.terminusdb.com/TerminatorsX
+TERMINUSDB_TEAM=TerminatorsX
+TERMINUSDB_DB=terminusCMS_docs             
+[email protected]

.github/workflows/build.yml

@@ -2,15 +2,14 @@
 name: Build
 
 on:
-  push:
   pull_request:
 
 jobs:
   build:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - run: npm ci
-      - run: npm run build
+      - run: npx bun install
+      - run: npx bun run build
         env:
           TERMINUSDB_API_TOKEN: ${{ secrets.TERMINUSDB_API_TOKEN }}

.github/workflows/publish.yml

@@ -13,8 +13,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - run: npm ci
-      - run: npm run build
+      - run: npx bun install
+      - run: npx bun run build
         env:
           TERMINUSDB_API_TOKEN: ${{ secrets.TERMINUSDB_API_TOKEN }}
       - run: curl -X POST ${{ secrets.CLOUDFLARE_DEPLOY_HOOK }} -o /dev/null

.gitignore

@@ -1,4 +1,10 @@
 # See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
+#
+#
+.next
+.env
+bun.lock
+src/menu.json
 
 # dependencies
 /node_modules

README.md

@@ -4,3 +4,26 @@ The static TerminusCMS and TerminusDB documentation that can be found on
 https://terminusdb.com/docs . It is written in NextJS and can be build
 by running `npm run build`. Note that the API key is needed for access
 to the data product.
+
+## Schema and documentation snapshot
+
+A documentation snapshot can be generated by running the crawler:
+
+```sh
+./run-crawler.sh --assets-domain assets.terminusdb.com --max-pages 200 --download-media
+```
+
+This will generate a `all_documents.json` file in the `output` directory.
+
+## Schema
+
+The schema for the documentation snapshot is located in the `schema` directory, together with an example of the output and a copy of the menu-structure based on the CMS schema published in the Terminusdb blogposts and based on the generated json files on terminusdb.com/docs in the NEXT hydration section with minor edits.
+
+## Menu structure
+
+The menu structure is located in the `menu-structure.json` file in the `schema` directory and can be loaded directly into TerminusDB using the schema.
+
+## Media
+
+Media files can be downloaded using the command above.
+

crawl-docs/.gitignore

@@ -0,0 +1 @@
+*.js