Add lite environment to pixi (#7134)

.github/workflows/jupyterlite.yaml

@@ -3,59 +3,67 @@ name: jupyterlite
 on:
   push:
     tags:
-      - 'v[0-9]+.[0-9]+.[0-9]+'
-      - 'v[0-9]+.[0-9]+.[0-9]+a[0-9]+'
-      - 'v[0-9]+.[0-9]+.[0-9]+b[0-9]+'
-      - 'v[0-9]+.[0-9]+.[0-9]+rc[0-9]+'
+      - "v[0-9]+.[0-9]+.[0-9]+"
+      - "v[0-9]+.[0-9]+.[0-9]+a[0-9]+"
+      - "v[0-9]+.[0-9]+.[0-9]+b[0-9]+"
+      - "v[0-9]+.[0-9]+.[0-9]+rc[0-9]+"
   workflow_dispatch:
     inputs:
       target:
-        description: 'Site to build and deploy'
+        description: "Site to build and deploy"
         type: choice
         options:
-        - dev
-        - main
-        - dryrun
+          - dev
+          - main
+          - dryrun
         required: true
         default: dryrun
   schedule:
-    - cron: '0 19 * * SUN'
+    - cron: "0 19 * * SUN"
 
 jobs:
-  deploy_jupyterlite:
-    name: JupyterLite
+  pixi_lock:
+    name: Pixi lock
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout
-        uses: actions/checkout@v3
+      - uses: holoviz-dev/holoviz_tasks/pixi_lock@pixi
+
+  lite_build:
+    name: Build Jupyterlite
+    needs: [pixi_lock]
+    runs-on: "ubuntu-latest"
+    steps:
+      - uses: holoviz-dev/holoviz_tasks/pixi_install@pixi
+        with:
+          environments: "lite"
+          install: false
+          download-data: false
+      - name: lite build
+        run: pixi run lite-build
+      - uses: actions/upload-artifact@v4
+        if: always()
         with:
-          fetch-depth: 0
-      - name: Setup Python
-        uses: actions/setup-python@v4
+          name: jupyterlite
+          path: lite/dist/
+          if-no-files-found: error
+
+  lite_publish:
+    name: Publish Jupyterlite
+    runs-on: ubuntu-latest
+    needs: [lite_build]
+    steps:
+      - uses: actions/download-artifact@v4
         with:
-          python-version: '3.10'
-      - name: Set and echo git ref
+          name: jupyterlite
+          path: lite/dist/
+      - name: Set output
         id: vars
-        run: |
-          echo 'Deploying from ref ${GITHUB_REF#refs/*/}'
-          echo 'tag=${GITHUB_REF#refs/*/}' >> $GITHUB_OUTPUT
-      - name: Install the dependencies
-        run: |
-          python -m pip install -r ./lite/requirements.txt
-      - name: Build pyodide wheels for JupyterLite
-        run: |
-          python ./scripts/build_pyodide_wheels.py lite/pypi
-      - name: Convert content
-        run: |
-          python ./scripts/panelite/generate_panelite_content.py
-      - name: Build the JupyterLite site
-        run: |
-          jupyter lite build --lite-dir lite --output-dir lite/dist
+        run: echo "tag=${{ needs.docs_build.outputs.tag }}" >> $GITHUB_OUTPUT
       - name: Upload dev
         if: |
           (github.event_name == 'workflow_dispatch' && github.event.inputs.target == 'dev') ||
           (github.event_name == 'workflow_run' && (contains(steps.vars.outputs.tag, 'a') || contains(steps.vars.outputs.tag, 'b') || contains(steps.vars.outputs.tag, 'rc')))
-        uses: peaceiris/actions-gh-pages@v3
+        uses: peaceiris/actions-gh-pages@v4
         with:
           personal_token: ${{ secrets.ACCESS_TOKEN }}
           external_repository: holoviz-dev/panelite-dev
@@ -65,7 +73,7 @@ jobs:
         if: |
           (github.event_name == 'workflow_dispatch' && github.event.inputs.target == 'main') ||
           (github.event_name == 'push' && !(contains(steps.vars.outputs.tag, 'a') || contains(steps.vars.outputs.tag, 'b') || contains(steps.vars.outputs.tag, 'rc')))
-        uses: peaceiris/actions-gh-pages@v3
+        uses: peaceiris/actions-gh-pages@v4
         with:
           personal_token: ${{ secrets.ACCESS_TOKEN }}
           external_repository: holoviz-dev/panelite

.github/workflows/test.yaml

@@ -206,12 +206,7 @@ jobs:
           (jupyter lab --config panel/tests/ui/jupyter_server_test_config.py --port 8887 > /tmp/jupyterlab_server.log 2>&1) &
       - name: Build JupyterLite
         shell: pixi run -e test-ui bash -el {0}
-        run: |
-          # TODO: Make this a pixi feature/environment
-          python -m pip install -r ./lite/requirements.txt
-          python ./scripts/build_pyodide_wheels.py lite/pypi
-          python ./scripts/panelite/generate_panelite_content.py
-          jupyter lite build --lite-dir lite --output-dir lite/dist
+        run: pixi run -e lite lite-build
       - name: Wait for JupyterLab
         uses: ifaxity/[email protected]
         with:

panel/.eslintrc.js

@@ -9,7 +9,7 @@ module.exports = {
   },
   "plugins": ["@typescript-eslint", "@stylistic/eslint-plugin"],
   "extends": [],
-  "ignorePatterns": ["*/dist", "*/theme/**/*.js", ".eslintrc.js", "*/_templates/*.js", "*/template/**/*.js", "examples/*"],
+  "ignorePatterns": ["*/dist", "*/theme/**/*.js", ".eslintrc.js", "*/_templates/*.js", "*/template/**/*.js", "examples/*", "scripts/*"],
   "rules": {
     "@typescript-eslint/ban-types": ["error", {
       "types": {

pixi.toml

@@ -20,6 +20,7 @@ test-type = ["py311", "type"]
 docs = ["py311", "example", "doc"]
 build = ["py311", "build"]
 lint = ["py311", "lint"]
+lite = ["py311", "lite"]
 
 [dependencies]
 bleach = "*"
@@ -213,3 +214,18 @@ pre-commit = "*"
 [feature.lint.tasks]
 lint = 'pre-commit run --all-files'
 lint-install = 'pre-commit install'
+
+# =============================================
+# =================== LITE ====================
+# =============================================
+[feature.lite.dependencies]
+jupyterlab-myst = "*"
+jupyterlite-core = "*"
+jupyterlite-pyodide-kernel = "*"
+python-build = "*"
+
+[feature.lite.tasks]
+lite-build = "bash scripts/jupyterlite/build.sh"
+# Service worker only work on 127.0.0.1
+# https://jupyterlite.readthedocs.io/en/latest/howto/configure/advanced/service-worker.html#limitations
+lite-server = "python -m http.server --directory ./lite/dist --bind 127.0.0.1"

pyproject.toml

@@ -116,6 +116,7 @@ include = ["panel"]
 
 [tool.hatch.build.targets.sdist]
 include = ["panel", "scripts", "examples"]
+exclude = ["scripts/jupyterlite"]
 
 [tool.hatch.build.targets.sdist.force-include]
 "panel/dist" = "panel/dist"

scripts/jupyterlite/build.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+set -euxo pipefail
+
+python ./scripts/build_pyodide_wheels.py dist
+python ./scripts/panelite/generate_panelite_content.py
+
+# Update lockfiles
+cd "$(dirname "${BASH_SOURCE[0]}")"
+rm -rf node_modules
+npm install .
+node update_lock.js
+python patch_lock.py
+rm node_modules/pyodide/*.whl
+
+jupyter lite build
+
+cp -r node_modules/pyodide/ ../../lite/dist/pyodide
+mv pyodide-lock.json ../../lite/dist/pyodide/pyodide-lock.json
+mv ../../dist/* ../../lite/dist/pyodide

scripts/jupyterlite/extra_packages.json

@@ -0,0 +1,5 @@
+[
+  "panel",
+  "bokeh",
+  "pyodide-http"
+]

lite/files/Getting_Started.ipynb → scripts/jupyterlite/files/Getting_Started.ipynb

@@ -5,7 +5,7 @@
    "id": "8c91012d-3445-4052-ab2f-129ca785a666",
    "metadata": {},
    "source": [
-    "Panel is not installed by default in the Pyodide distribution that JupyterLite is built on, therefore we must install it manually:"
+    "Panel is installed by default in the Pyodide distribution that JupyterLite is built on. Though, not all dependencies are therefore we must install it manually:"
    ]
   },
   {
@@ -16,7 +16,7 @@
    "outputs": [],
    "source": [
     "import piplite\n",
-    "await piplite.install(['panel', 'pyodide-http', 'altair'])"
+    "await piplite.install(['altair'])"
    ]
   },
   {

lite/files/Reset_Jupyterlite.ipynb → scripts/jupyterlite/files/Reset_Jupyterlite.ipynb

@@ -57,5 +57,5 @@
   }
  },
  "nbformat": 4,
- "nbformat_minor": 2
+ "nbformat_minor": 4
 }

scripts/jupyterlite/jupyter-lite.json

@@ -0,0 +1,10 @@
+{
+    "jupyter-lite-schema-version": 0,
+    "jupyter-config-data": {
+      "litePluginSettings": {
+        "@jupyterlite/pyodide-kernel-extension:kernel": {
+          "pyodideUrl": "./pyodide/pyodide.js"
+        }
+      }
+    }
+  }

scripts/jupyterlite/jupyter_lite_config.json

@@ -0,0 +1,6 @@
+{
+  "LiteBuildConfig": {
+    "contents": ["../../lite/files", "files"],
+    "output_dir": "../../lite/dist"
+  }
+}

scripts/jupyterlite/package.json

@@ -0,0 +1,5 @@
+{
+  "dependencies": {
+    "pyodide": "^0.26.2"
+  }
+}

scripts/jupyterlite/patch_lock.py

@@ -0,0 +1,47 @@
+import hashlib
+import json
+import os.path
+
+from glob import glob
+
+from packaging.utils import parse_wheel_filename
+
+
+def calculate_sha256(file_path):
+    sha256_hash = hashlib.sha256()
+
+    with open(file_path, "rb") as file:
+        for byte_block in iter(lambda: file.read(4096), b""):
+            sha256_hash.update(byte_block)
+
+    return sha256_hash.hexdigest()
+
+
+with open("package.json") as f:
+    package = json.load(f)
+pyodide_version = package["dependencies"]["pyodide"].removeprefix("^")
+
+path = "pyodide-lock.json"
+url = f"https://cdn.jsdelivr.net/pyodide/v{pyodide_version}/full"
+
+with open(path) as f:
+    data = json.load(f)
+
+for p in data["packages"].values():
+    if not p["file_name"].startswith("http"):
+        p["file_name"] = f'{url}/{p["file_name"]}'
+
+
+whl_files = glob("../../dist/*.whl")
+for whl_file in whl_files:
+    name, version, *_ = parse_wheel_filename(os.path.basename(whl_file))
+
+    package = data["packages"][name]
+    package["version"] = str(version)
+    package["file_name"] = os.path.basename(whl_file)
+    package["sha256"] = calculate_sha256(whl_file)
+    package["imports"] = [name]
+
+
+with open(path, "w") as f:
+    data = json.dump(data, f)

scripts/jupyterlite/update_lock.js

@@ -0,0 +1,20 @@
+const { loadPyodide } = require("pyodide");
+const fs = require("fs");
+
+async function main() {
+  const extra = fs.readFileSync("extra_packages.json", "utf8");
+
+  let pyodide = await loadPyodide();
+  await pyodide.loadPackage(["micropip"]);
+
+  output = await pyodide.runPythonAsync(`
+import json
+import micropip
+extra = json.loads("""${extra}""")
+await micropip.install(extra)
+micropip.freeze()
+`);
+  fs.writeFileSync("pyodide-lock.json", output);
+}
+
+main();

scripts/panelite/generate_panelite_content.py

@@ -17,7 +17,7 @@
 EXAMPLES_DIR = PANEL_BASE / 'examples'
 LITE_FILES = PANEL_BASE / 'lite' / 'files'
 DOC_DIR = PANEL_BASE / 'doc'
-BASE_DEPENDENCIES = ['panel', 'pyodide-http']
+BASE_DEPENDENCIES = []
 MINIMUM_VERSIONS = {}
 
 INLINE_DIRECTIVE = re.compile('\{.*\}`.*`\s*')