automating persistence coverage

.github/workflows/update-persistence-coverage.yml

@@ -0,0 +1,65 @@
+name: Update Persistence Docs
+on:
+  schedule:
+    - cron: 0 5 * * MON
+  workflow_dispatch:
+    inputs:
+        targetBranch:
+          required: false
+          type: string
+          default: 'master'
+
+jobs:
+  update-persistence-docs:
+    name: Update Parity Docs
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout docs
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          path: docs
+          ref: ${{ github.event.inputs.targetBranch || 'master' }}
+
+      - name: Set up Python 3.11
+        id: setup-python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Update Coverage Docs with Persistence Coverage
+        working-directory: docs
+        run: |
+          cd scripts/persistence
+          python3 -m venv .venv
+          source .venv/bin/activate
+          pip3 install -r requirements.txt
+          python3 create_persistence_docs.py
+        env:
+          NOTION_TOKEN: ${{ secrets.NOTION_TOKEN }}
+
+      - name: Check for changes
+        id: check-for-changes
+        working-directory: docs
+        run: |
+          # Check if there are changed files and store the result in resources/diff-check.log
+          # Check against the PR branch if it exists, otherwise against the main
+          # Store the result in resources/diff-check.log and store the diff count in the GitHub Action output "diff-count"
+          mkdir -p resources
+          (git diff --name-only origin/persistence-auto-updates src/data/persistence/ 2>/dev/null || git diff --name-only origin/${{ github.event.inputs.targetBranch || 'master' }} src/data/persistence/ 2>/dev/null) | tee -a resources/diff-check.log
+          echo "diff-count=$(cat resources/diff-check.log | wc -l)" >> $GITHUB_OUTPUT
+          cat cat resources/diff-check.log
+
+      - name: Create PR
+        uses: peter-evans/create-pull-request@v7
+        if: ${{ success() && steps.check-for-changes.outputs.diff-count != '0' && steps.check-for-changes.outputs.diff-count != '' }}
+        with:
+          path: docs
+          title: "Update Persistence Docs"
+          body: "Updating Persistence Coverage Documentation based on the [Persistence Catalog](https://www.notion.so/localstack/Persistence-Catalog-a9e0e5cb89df4784adb4a1ed377b3c23) on Notion."
+          branch: "persistence-auto-updates"
+          author: "LocalStack Bot <[email protected]>"
+          committer: "LocalStack Bot <[email protected]>"
+          commit-message: "update generated persistence docs"
+          token: ${{ secrets.PRO_ACCESS_TOKEN }}
+          reviewers: giograno

.gitignore

@@ -20,4 +20,13 @@ pnpm-debug.log*
 # macOS-specific files
 .DS_Store
 .kiro
-.vscode
+.vscode
+
+# Python
+.venv/
+__pycache__/
+*.pyc
+*.pyo
+*.pyd
+*.pyw
+*.pyz

scripts/persistence/create_persistence_docs.py

@@ -0,0 +1,138 @@
+import os
+from io import BytesIO
+import json
+from pathlib import Path
+import notion_client as n_client
+import frontmatter
+from ruamel.yaml import YAML
+from frontmatter.default_handlers import YAMLHandler, DEFAULT_POST_TEMPLATE
+from notion.catalog import PersistenceCatalog
+
+token = os.getenv("NOTION_TOKEN")
+markdown_path = "../../src/content/docs/aws/services"
+persistence_path = "../../src/data/persistence"
+persistence_data = os.path.join(persistence_path, "coverage.json")
+
+
+class CustomYAMLHandler(YAMLHandler):
+
+    def load(self, fm: str, **kwargs: object):
+        yaml = YAML()
+        yaml.default_flow_style = False
+        yaml.preserve_quotes = True
+        return yaml.load(fm, **kwargs)  # type: ignore[arg-type]
+
+    def export(self, metadata: dict[str, object], **kwargs: object) -> str:
+        yaml = YAML()
+        yaml.default_flow_style = False
+        from io import StringIO
+        stream = StringIO()
+        yaml.dump(metadata, stream)
+        return stream.getvalue().rstrip()
+
+    def format(self, post, **kwargs):
+        """
+        Simple customization to avoid removing the last line.
+        """
+        start_delimiter = kwargs.pop("start_delimiter", self.START_DELIMITER)
+        end_delimiter = kwargs.pop("end_delimiter", self.END_DELIMITER)
+
+        metadata = self.export(post.metadata, **kwargs)
+
+        return DEFAULT_POST_TEMPLATE.format(
+            metadata=metadata,
+            content=post.content,
+            start_delimiter=start_delimiter,
+            end_delimiter=end_delimiter,
+        ).lstrip()
+
+
+def lookup_full_name(shortname: str) -> str:
+    """Given the short default name of a service, looks up for the full name"""
+    service_lookup = Path("../../src/data/coverage/service_display_name.json")
+    service_info = {}
+    if service_lookup.exists() and service_lookup.is_file():
+        with open(service_lookup, "r") as f:
+            service_info = json.load(f)
+
+    service_name_title = shortname
+
+    if service_name_details := service_info.get(shortname, {}):
+        service_name_title = service_name_details.get("long_name", shortname)
+        if service_name_title and (short_name := service_name_details.get("short_name")):
+            service_name_title = f"{short_name} ({service_name_title})"
+    return service_name_title
+
+
+def collect_status() -> dict:
+    """Reads the catalog on Notion and returns the status of persistence for each service"""
+    if not token:
+        print("Aborting, please provide a NOTION_TOKEN in the env") 
+    notion_client = n_client.Client(auth=token)
+
+    catalog_db = PersistenceCatalog(notion_client=notion_client)
+    statuses = {}
+    for item in catalog_db:
+        # we do not want some services to be mentioned in the docs (for instance, not yet released)
+        if item.exclude:
+            continue
+
+        # Skip entries with empty or placeholder names
+        if not item.name or not item.name.strip():
+            continue
+
+        # Skip template/placeholder entries
+        if item.name.strip().lower() in ['new service page', 'template', 'placeholder']:
+            continue
+
+        service = item.name.replace('_', '-')
+        status = item.status.lower()
+        statuses[service] = {
+            "service": service,
+            "full_name": lookup_full_name(service),
+            "support": status,
+            "test_suite": item.has_test or False,
+            # we collect limitations notes only for the services explicitly marked with limitations
+            "limitations": item.limitations if "limit" in status else "" 
+        }
+    statuses = dict(sorted(statuses.items()))
+
+    # save the data
+    if not os.path.exists(persistence_path):
+        os.mkdir(persistence_path)
+    with open(persistence_data, 'w') as f:
+        json.dump(statuses, f, indent=2)
+    return statuses
+
+
+def update_frontmatter(statuses: dict):
+    """Updates the frontmatter of the service page in the user guide Markdown file"""
+    for service, values in statuses.items():
+
+        # a bunch of special cases
+        if "cognito" in service:
+            service = "cognito"
+        if service == "kafka":
+            service = "msk"
+
+        _path = os.path.join(markdown_path, f"{service}.mdx")
+        if not os.path.exists(_path):
+            continue
+
+        support_value = values.get("support")
+        is_supported = support_value == "supported" or support_value == "supported with limitations"
+        if not is_supported:
+            # we don't want to modify the frontmatter for the services not supporting persistence
+            continue
+
+        # open the markdown file and read the content
+        content = frontmatter.load(_path, handler=CustomYAMLHandler())
+        desc = content.metadata["description"]
+        content.metadata["description"] = desc.strip()
+        content.metadata["persistence"] = values.get("support", "unknown")
+        frontmatter.dump(content, _path)
+
+
+if __name__ == "__main__":
+    data = collect_status()
+    update_frontmatter(statuses=data)

scripts/persistence/notion/catalog.py

@@ -0,0 +1,30 @@
+"""Models for the notion service catalog https://www.notion.so/localstack/3c0f615e7ffc4ae2a034f1ed9c444bd2"""
+
+from notion_client import Client as NotionClient
+
+from notion_objects import (
+    Page,
+    TitlePlainText,
+    Status,
+    Database,
+    Checkbox,
+    PeopleProperty,
+    Text
+)
+
+DEFAULT_CATALOG_DATABASE_ID = "3c0f615e7ffc4ae2a034f1ed9c444bd2"
+
+
+class PersistenceServiceItem(Page):
+    name = TitlePlainText("Name")
+    status = Status("Persistence")
+    has_test = Checkbox("Persistence Tests")
+    primary_owner = PeopleProperty("Primary Owner")
+    secondary_owner = PeopleProperty("Secondary Owner(s)")
+    limitations = Text("Limitations (synced with docs)")
+    exclude = Checkbox("Exclude from docs")
+
+
+class PersistenceCatalog(Database[PersistenceServiceItem]):
+    def __init__(self, notion_client: NotionClient, database_id: str | None = None):
+        super().__init__(PersistenceServiceItem, database_id or DEFAULT_CATALOG_DATABASE_ID, notion_client)

scripts/persistence/requirements.txt

@@ -0,0 +1,4 @@
+notion-client==2.2.1
+notion-objects==0.6.2
+python-frontmatter==1.1.0
+ruamel.yaml==0.18.6

src/content/docs/aws/services/pinpoint.mdx

@@ -3,7 +3,6 @@ title: "Pinpoint"
 description: Get started with Pinpoint on LocalStack
 tags: ["Ultimate"]
 persistence: supported
-
 ---
 
 import FeatureCoverage from "../../../../components/feature-coverage/FeatureCoverage";

src/data/persistence/coverage.json

@@ -2827,4 +2827,4 @@
     "test_suite": true,
     "limitations": ""
   }
-}
+}