From 4c695a92491505662ba3a6f5dc95e689cf045d96 Mon Sep 17 00:00:00 2001 From: HarshCasper Date: Thu, 26 Jun 2025 13:10:42 +0530 Subject: [PATCH 1/8] get script running --- .../persistence/create_persistence_docs.py | 140 ++++++++++++++++++ scripts/persistence/notion/__init__.py | 0 scripts/persistence/notion/catalog.py | 30 ++++ scripts/persistence/requirements.txt | 4 + 4 files changed, 174 insertions(+) create mode 100644 scripts/persistence/create_persistence_docs.py create mode 100644 scripts/persistence/notion/__init__.py create mode 100644 scripts/persistence/notion/catalog.py create mode 100644 scripts/persistence/requirements.txt diff --git a/scripts/persistence/create_persistence_docs.py b/scripts/persistence/create_persistence_docs.py new file mode 100644 index 00000000..13811a62 --- /dev/null +++ b/scripts/persistence/create_persistence_docs.py @@ -0,0 +1,140 @@ +import os +from io import BytesIO +import json +from pathlib import Path +import notion_client as n_client +import frontmatter +from ruamel.yaml import YAML +from frontmatter.default_handlers import YAMLHandler, DEFAULT_POST_TEMPLATE +from notion.catalog import PersistenceCatalog + +token = os.getenv("NOTION_TOKEN") +markdown_path = "../../src/content/docs/aws/services" +# markdown_path = "../../content/en/user-guide/aws" +persistence_path = "../../src/data/persistence" +persistence_data = os.path.join(persistence_path, "coverage.json") + + +class CustomYAMLHandler(YAMLHandler): + + def load(self, fm: str, **kwargs: object): + yaml = YAML() + yaml.default_flow_style = False + yaml.preserve_quotes = True + return yaml.load(fm, **kwargs) # type: ignore[arg-type] + + def export(self, metadata: dict[str, object], **kwargs: object) -> str: + yaml = YAML() + yaml.default_flow_style = False + from io import StringIO + stream = StringIO() + yaml.dump(metadata, stream) + return stream.getvalue() + + def format(self, post, **kwargs): + """ + Simple customization to avoid removing the last line. + """ + start_delimiter = kwargs.pop("start_delimiter", self.START_DELIMITER) + end_delimiter = kwargs.pop("end_delimiter", self.END_DELIMITER) + + metadata = self.export(post.metadata, **kwargs) + + return DEFAULT_POST_TEMPLATE.format( + metadata=metadata, + content=post.content, + start_delimiter=start_delimiter, + end_delimiter=end_delimiter, + ).lstrip() + + +def lookup_full_name(shortname: str) -> str: + """Given the short default name of a service, looks up for the full name""" + service_lookup = Path("../../src/data/coverage/service_display_name.json") + service_info = {} + if service_lookup.exists() and service_lookup.is_file(): + with open(service_lookup, "r") as f: + service_info = json.load(f) + + service_name_title = shortname + + if service_name_details := service_info.get(shortname, {}): + service_name_title = service_name_details.get("long_name", shortname) + if service_name_title and (short_name := service_name_details.get("short_name")): + service_name_title = f"{short_name} ({service_name_title})" + return service_name_title + + +def collect_status() -> dict: + """Reads the catalog on Notion and returns the status of persistence for each service""" + if not token: + print("Aborting, please provide a NOTION_TOKEN in the env") + notion_client = n_client.Client(auth=token) + + catalog_db = PersistenceCatalog(notion_client=notion_client) + statuses = {} + for item in catalog_db: + # we do not want some services to be mentioned in the docs (for instance, not yet released) + if item.exclude: + continue + + # Skip entries with empty or placeholder names + if not item.name or not item.name.strip(): + continue + + # Skip template/placeholder entries + if item.name.strip().lower() in ['new service page', 'template', 'placeholder']: + continue + + service = item.name.replace('_', '-') + status = item.status.lower() + statuses[service] = { + "service": service, + "full_name": lookup_full_name(service), + "support": status, + "test_suite": item.has_test or False, + # we collect limitations notes only for the services explicitly marked with limitations + "limitations": item.limitations if "limit" in status else "" + } + statuses = dict(sorted(statuses.items())) + + # save the data + if not os.path.exists(persistence_path): + os.mkdir(persistence_path) + with open(persistence_data, 'w') as f: + json.dump(statuses, f, indent=2) + return statuses + + +def update_frontmatter(statuses: dict): + """Updates the frontmatter of the service page in the user guide Markdown file""" + for service, values in statuses.items(): + + # a bunch of special cases + if "cognito" in service: + service = "cognito" + if service == "kafka": + service = "msk" + + _path = os.path.join(markdown_path, service, "index.md") + if not os.path.exists(_path): + print(f" Can't find index.md file for {service}") + continue + + support_value = values.get("support") + is_supported = support_value == "supported" or support_value == "supported with limitations" + if not is_supported: + # we don't want to modify the frontmatter for the services not supporting persistence + continue + + # open the markdown file and read the content + content = frontmatter.load(_path, handler=CustomYAMLHandler()) + desc = content.metadata["description"] + content.metadata["description"] = desc.strip() + content.metadata["persistence"] = values.get("support", "unknown") + frontmatter.dump(content, _path) + + +if __name__ == "__main__": + data = collect_status() + # update_frontmatter(statuses=data) diff --git a/scripts/persistence/notion/__init__.py b/scripts/persistence/notion/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/scripts/persistence/notion/catalog.py b/scripts/persistence/notion/catalog.py new file mode 100644 index 00000000..aa34a79f --- /dev/null +++ b/scripts/persistence/notion/catalog.py @@ -0,0 +1,30 @@ +"""Models for the notion service catalog https://www.notion.so/localstack/3c0f615e7ffc4ae2a034f1ed9c444bd2""" + +from notion_client import Client as NotionClient + +from notion_objects import ( + Page, + TitlePlainText, + Status, + Database, + Checkbox, + PeopleProperty, + Text +) + +DEFAULT_CATALOG_DATABASE_ID = "3c0f615e7ffc4ae2a034f1ed9c444bd2" + + +class PersistenceServiceItem(Page): + name = TitlePlainText("Name") + status = Status("Persistence") + has_test = Checkbox("Persistence Tests") + primary_owner = PeopleProperty("Primary Owner") + secondary_owner = PeopleProperty("Secondary Owner(s)") + limitations = Text("Limitations (synced with docs)") + exclude = Checkbox("Exclude from docs") + + +class PersistenceCatalog(Database[PersistenceServiceItem]): + def __init__(self, notion_client: NotionClient, database_id: str | None = None): + super().__init__(PersistenceServiceItem, database_id or DEFAULT_CATALOG_DATABASE_ID, notion_client) diff --git a/scripts/persistence/requirements.txt b/scripts/persistence/requirements.txt new file mode 100644 index 00000000..96912a1d --- /dev/null +++ b/scripts/persistence/requirements.txt @@ -0,0 +1,4 @@ +notion-client==2.2.1 +notion-objects==0.6.2 +python-frontmatter==1.1.0 +ruamel.yaml==0.18.6 From 5c9cb51b3851a3aa12c1750afebcfb4c895f4263 Mon Sep 17 00:00:00 2001 From: HarshCasper Date: Thu, 26 Jun 2025 15:10:22 +0530 Subject: [PATCH 2/8] update frontmatter --- scripts/persistence/create_persistence_docs.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/scripts/persistence/create_persistence_docs.py b/scripts/persistence/create_persistence_docs.py index 13811a62..2a9826d9 100644 --- a/scripts/persistence/create_persistence_docs.py +++ b/scripts/persistence/create_persistence_docs.py @@ -10,7 +10,6 @@ token = os.getenv("NOTION_TOKEN") markdown_path = "../../src/content/docs/aws/services" -# markdown_path = "../../content/en/user-guide/aws" persistence_path = "../../src/data/persistence" persistence_data = os.path.join(persistence_path, "coverage.json") @@ -29,7 +28,7 @@ def export(self, metadata: dict[str, object], **kwargs: object) -> str: from io import StringIO stream = StringIO() yaml.dump(metadata, stream) - return stream.getvalue() + return stream.getvalue().rstrip() def format(self, post, **kwargs): """ @@ -116,9 +115,8 @@ def update_frontmatter(statuses: dict): if service == "kafka": service = "msk" - _path = os.path.join(markdown_path, service, "index.md") + _path = os.path.join(markdown_path, f"{service}.mdx") if not os.path.exists(_path): - print(f" Can't find index.md file for {service}") continue support_value = values.get("support") @@ -137,4 +135,4 @@ def update_frontmatter(statuses: dict): if __name__ == "__main__": data = collect_status() - # update_frontmatter(statuses=data) + update_frontmatter(statuses=data) From 746e343ea96f3fe2ca5f334a70be3654a5957e31 Mon Sep 17 00:00:00 2001 From: HarshCasper Date: Thu, 26 Jun 2025 15:15:03 +0530 Subject: [PATCH 3/8] add an action --- .../workflows/update-persistence-coverage.yml | 65 +++++++++++++++++++ .gitignore | 11 +++- package-lock.json | 30 ++++----- src/content/docs/aws/services/pinpoint.mdx | 1 - src/data/persistence/coverage.json | 42 ++++++++++++ 5 files changed, 132 insertions(+), 17 deletions(-) create mode 100644 .github/workflows/update-persistence-coverage.yml diff --git a/.github/workflows/update-persistence-coverage.yml b/.github/workflows/update-persistence-coverage.yml new file mode 100644 index 00000000..50f9b48a --- /dev/null +++ b/.github/workflows/update-persistence-coverage.yml @@ -0,0 +1,65 @@ +name: Update Persistence Docs +on: + schedule: + - cron: 0 5 * * MON + workflow_dispatch: + inputs: + targetBranch: + required: false + type: string + default: 'master' + +jobs: + update-persistence-docs: + name: Update Parity Docs + runs-on: ubuntu-latest + steps: + - name: Checkout docs + uses: actions/checkout@v4 + with: + fetch-depth: 0 + path: docs + ref: ${{ github.event.inputs.targetBranch || 'master' }} + + - name: Set up Python 3.11 + id: setup-python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Update Coverage Docs with Persistence Coverage + working-directory: docs + run: | + cd scripts/persistence + python3 -m venv .venv + source .venv/bin/activate + pip3 install -r requirements.txt + python3 create_persistence_docs.py + env: + NOTION_TOKEN: ${{ secrets.NOTION_TOKEN }} + + - name: Check for changes + id: check-for-changes + working-directory: docs + run: | + # Check if there are changed files and store the result in resources/diff-check.log + # Check against the PR branch if it exists, otherwise against the main + # Store the result in resources/diff-check.log and store the diff count in the GitHub Action output "diff-count" + mkdir -p resources + (git diff --name-only origin/persistence-auto-updates src/data/persistence/ 2>/dev/null || git diff --name-only origin/${{ github.event.inputs.targetBranch || 'master' }} src/data/persistence/ 2>/dev/null) | tee -a resources/diff-check.log + echo "diff-count=$(cat resources/diff-check.log | wc -l)" >> $GITHUB_OUTPUT + cat cat resources/diff-check.log + + - name: Create PR + uses: peter-evans/create-pull-request@v7 + if: ${{ success() && steps.check-for-changes.outputs.diff-count != '0' && steps.check-for-changes.outputs.diff-count != '' }} + with: + path: docs + title: "Update Persistence Docs" + body: "Updating Persistence Coverage Documentation based on the [Persistence Catalog](https://www.notion.so/localstack/Persistence-Catalog-a9e0e5cb89df4784adb4a1ed377b3c23) on Notion." + branch: "persistence-auto-updates" + author: "LocalStack Bot " + committer: "LocalStack Bot " + commit-message: "update generated persistence docs" + token: ${{ secrets.PRO_ACCESS_TOKEN }} + reviewers: giograno diff --git a/.gitignore b/.gitignore index f3c36888..c9bb324f 100644 --- a/.gitignore +++ b/.gitignore @@ -20,4 +20,13 @@ pnpm-debug.log* # macOS-specific files .DS_Store .kiro -.vscode \ No newline at end of file +.vscode + +# Python +.venv/ +__pycache__/ +*.pyc +*.pyo +*.pyd +*.pyw +*.pyz \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 5f7520e9..5756047f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -28,8 +28,8 @@ "react": "^19.1.0", "react-dom": "^19.1.0", "sharp": "^0.32.5", - "starlight-image-zoom": "^0.12.0", "starlight-fullview-mode": "^0.2.3", + "starlight-image-zoom": "^0.12.0", "starlight-links-validator": "^0.17.0", "starlight-toc-overview-customizer": "^0.1.0", "tailwind-merge": "^3.2.0", @@ -7708,6 +7708,20 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/starlight-fullview-mode": { + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/starlight-fullview-mode/-/starlight-fullview-mode-0.2.3.tgz", + "integrity": "sha512-8sw/VKhWAtcD5eOUBz4T8U/rEtteOXXyKTp0QwNXlwAW4zgIb6qKsnRXOQearHMp3GTSUH/b+tISHrtz9+/ePQ==", + "dependencies": { + "@iconify-json/mdi": "^1.1.68" + }, + "engines": { + "node": "^18.17.1 || ^20.3.0 || >=21.0.0" + }, + "peerDependencies": { + "@astrojs/starlight": ">=0.32" + } + }, "node_modules/starlight-image-zoom": { "version": "0.12.0", "resolved": "https://registry.npmjs.org/starlight-image-zoom/-/starlight-image-zoom-0.12.0.tgz", @@ -7725,20 +7739,6 @@ "@astrojs/starlight": ">=0.32.0" } }, - "node_modules/starlight-fullview-mode": { - "version": "0.2.3", - "resolved": "https://registry.npmjs.org/starlight-fullview-mode/-/starlight-fullview-mode-0.2.3.tgz", - "integrity": "sha512-8sw/VKhWAtcD5eOUBz4T8U/rEtteOXXyKTp0QwNXlwAW4zgIb6qKsnRXOQearHMp3GTSUH/b+tISHrtz9+/ePQ==", - "dependencies": { - "@iconify-json/mdi": "^1.1.68" - }, - "engines": { - "node": "^18.17.1 || ^20.3.0 || >=21.0.0" - }, - "peerDependencies": { - "@astrojs/starlight": ">=0.32" - } - }, "node_modules/starlight-links-validator": { "version": "0.17.0", "resolved": "https://registry.npmjs.org/starlight-links-validator/-/starlight-links-validator-0.17.0.tgz", diff --git a/src/content/docs/aws/services/pinpoint.mdx b/src/content/docs/aws/services/pinpoint.mdx index 7960bf4e..f851ec3f 100644 --- a/src/content/docs/aws/services/pinpoint.mdx +++ b/src/content/docs/aws/services/pinpoint.mdx @@ -3,7 +3,6 @@ title: "Pinpoint" description: Get started with Pinpoint on LocalStack tags: ["Ultimate"] persistence: supported - --- import FeatureCoverage from "../../../../components/feature-coverage/FeatureCoverage"; diff --git a/src/data/persistence/coverage.json b/src/data/persistence/coverage.json index 957d82df..5fc2c131 100644 --- a/src/data/persistence/coverage.json +++ b/src/data/persistence/coverage.json @@ -27,6 +27,13 @@ "test_suite": true, "limitations": "" }, + "aiops": { + "service": "aiops", + "full_name": "aiops", + "support": "unknown", + "test_suite": false, + "limitations": "" + }, "amp": { "service": "amp", "full_name": "amp", @@ -1014,6 +1021,13 @@ "test_suite": false, "limitations": "" }, + "evs": { + "service": "evs", + "full_name": "evs", + "support": "unknown", + "test_suite": false, + "limitations": "" + }, "finspace": { "service": "finspace", "full_name": "finspace", @@ -1812,6 +1826,13 @@ "test_suite": false, "limitations": "" }, + "mpa": { + "service": "mpa", + "full_name": "mpa", + "support": "unknown", + "test_suite": false, + "limitations": "" + }, "mq": { "service": "mq", "full_name": "Amazon MQ", @@ -2575,6 +2596,13 @@ "test_suite": false, "limitations": "" }, + "ssm-guiconnect": { + "service": "ssm-guiconnect", + "full_name": "ssm-guiconnect", + "support": "unknown", + "test_suite": false, + "limitations": "" + }, "ssm-incidents": { "service": "ssm-incidents", "full_name": "ssm-incidents", @@ -2694,6 +2722,20 @@ "test_suite": true, "limitations": "" }, + "timestream-query": { + "service": "timestream-query", + "full_name": "Timestream Query", + "support": "unknown", + "test_suite": false, + "limitations": "" + }, + "timestream-write": { + "service": "timestream-write", + "full_name": "Timestream Write", + "support": "unknown", + "test_suite": false, + "limitations": "" + }, "tnb": { "service": "tnb", "full_name": "tnb", From b02a82215c39f869879bd0d330956259c95480ab Mon Sep 17 00:00:00 2001 From: HarshCasper Date: Thu, 26 Jun 2025 15:16:41 +0530 Subject: [PATCH 4/8] testing --- .github/workflows/update-persistence-coverage.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/update-persistence-coverage.yml b/.github/workflows/update-persistence-coverage.yml index 50f9b48a..31401969 100644 --- a/.github/workflows/update-persistence-coverage.yml +++ b/.github/workflows/update-persistence-coverage.yml @@ -2,6 +2,9 @@ name: Update Persistence Docs on: schedule: - cron: 0 5 * * MON + pull_request: + branches: + - master workflow_dispatch: inputs: targetBranch: From cb4e0c8d99a97495966b47bc168c459c9a120c5d Mon Sep 17 00:00:00 2001 From: HarshCasper Date: Thu, 26 Jun 2025 15:17:51 +0530 Subject: [PATCH 5/8] testing --- src/data/persistence/coverage.json | 44 +----------------------------- 1 file changed, 1 insertion(+), 43 deletions(-) diff --git a/src/data/persistence/coverage.json b/src/data/persistence/coverage.json index 5fc2c131..31f4c03b 100644 --- a/src/data/persistence/coverage.json +++ b/src/data/persistence/coverage.json @@ -27,13 +27,6 @@ "test_suite": true, "limitations": "" }, - "aiops": { - "service": "aiops", - "full_name": "aiops", - "support": "unknown", - "test_suite": false, - "limitations": "" - }, "amp": { "service": "amp", "full_name": "amp", @@ -1021,13 +1014,6 @@ "test_suite": false, "limitations": "" }, - "evs": { - "service": "evs", - "full_name": "evs", - "support": "unknown", - "test_suite": false, - "limitations": "" - }, "finspace": { "service": "finspace", "full_name": "finspace", @@ -1826,13 +1812,6 @@ "test_suite": false, "limitations": "" }, - "mpa": { - "service": "mpa", - "full_name": "mpa", - "support": "unknown", - "test_suite": false, - "limitations": "" - }, "mq": { "service": "mq", "full_name": "Amazon MQ", @@ -2596,13 +2575,6 @@ "test_suite": false, "limitations": "" }, - "ssm-guiconnect": { - "service": "ssm-guiconnect", - "full_name": "ssm-guiconnect", - "support": "unknown", - "test_suite": false, - "limitations": "" - }, "ssm-incidents": { "service": "ssm-incidents", "full_name": "ssm-incidents", @@ -2722,20 +2694,6 @@ "test_suite": true, "limitations": "" }, - "timestream-query": { - "service": "timestream-query", - "full_name": "Timestream Query", - "support": "unknown", - "test_suite": false, - "limitations": "" - }, - "timestream-write": { - "service": "timestream-write", - "full_name": "Timestream Write", - "support": "unknown", - "test_suite": false, - "limitations": "" - }, "tnb": { "service": "tnb", "full_name": "tnb", @@ -2869,4 +2827,4 @@ "test_suite": true, "limitations": "" } -} \ No newline at end of file +} From f4a5a7f8a990905c0c8a250d1d20673ef86e889d Mon Sep 17 00:00:00 2001 From: HarshCasper Date: Thu, 26 Jun 2025 15:20:32 +0530 Subject: [PATCH 6/8] testing --- .github/workflows/update-persistence-coverage.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/update-persistence-coverage.yml b/.github/workflows/update-persistence-coverage.yml index 31401969..6b0531ad 100644 --- a/.github/workflows/update-persistence-coverage.yml +++ b/.github/workflows/update-persistence-coverage.yml @@ -33,6 +33,7 @@ jobs: - name: Update Coverage Docs with Persistence Coverage working-directory: docs run: | + ls -la cd scripts/persistence python3 -m venv .venv source .venv/bin/activate From 1d1e6bcde13b843ef64fb3b4a010537ad6458ec8 Mon Sep 17 00:00:00 2001 From: HarshCasper Date: Thu, 26 Jun 2025 15:23:15 +0530 Subject: [PATCH 7/8] testing --- .github/workflows/update-persistence-coverage.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/update-persistence-coverage.yml b/.github/workflows/update-persistence-coverage.yml index 6b0531ad..344a99da 100644 --- a/.github/workflows/update-persistence-coverage.yml +++ b/.github/workflows/update-persistence-coverage.yml @@ -33,6 +33,8 @@ jobs: - name: Update Coverage Docs with Persistence Coverage working-directory: docs run: | + git branch + git log -1 ls -la cd scripts/persistence python3 -m venv .venv From 9663a9379f391db5cacf3ad913ec539b8bf36140 Mon Sep 17 00:00:00 2001 From: HarshCasper Date: Thu, 26 Jun 2025 15:35:21 +0530 Subject: [PATCH 8/8] testing --- .github/workflows/update-persistence-coverage.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/update-persistence-coverage.yml b/.github/workflows/update-persistence-coverage.yml index 344a99da..50f9b48a 100644 --- a/.github/workflows/update-persistence-coverage.yml +++ b/.github/workflows/update-persistence-coverage.yml @@ -2,9 +2,6 @@ name: Update Persistence Docs on: schedule: - cron: 0 5 * * MON - pull_request: - branches: - - master workflow_dispatch: inputs: targetBranch: @@ -33,9 +30,6 @@ jobs: - name: Update Coverage Docs with Persistence Coverage working-directory: docs run: | - git branch - git log -1 - ls -la cd scripts/persistence python3 -m venv .venv source .venv/bin/activate