From 9abab0286fa1b726747718767f1e39b3624f91db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6nke=20Liebau?= Date: Sun, 21 Jul 2024 22:12:33 +0200 Subject: [PATCH 1/2] First attempt at removing the duplication of repository data. --- .github/workflows/generate_prs.yml | 14 +---- config/repositories.yaml | 84 --------------------------- playbook/playbook.yaml | 13 ++--- playbook/vars_plugins/repositories.py | 15 +++++ test.sh | 2 +- 5 files changed, 23 insertions(+), 105 deletions(-) delete mode 100644 config/repositories.yaml create mode 100644 playbook/vars_plugins/repositories.py diff --git a/.github/workflows/generate_prs.yml b/.github/workflows/generate_prs.yml index b531ca90..733b3c97 100644 --- a/.github/workflows/generate_prs.yml +++ b/.github/workflows/generate_prs.yml @@ -134,26 +134,16 @@ jobs: run: | echo "REASON=Daily run triggered" >> "$GITHUB_ENV" - - name: Overwrite repositories.yaml for ${{ matrix.repository.name }} - run: | - cat >config/repositories.yaml < vars.json - ansible-playbook playbook/playbook.yaml --extra-vars "@vars.json" + ansible-playbook playbook/playbook.yaml --extra-vars "@vars.json" --extra-vars "only_operator=${{ matrix.repository.name }} env: GH_ACCESS_TOKEN: ${{ secrets.gh_access_token }} + ANSIBLE_VARS_ENABLED: repositories # Do Not Generate PRs - name: Run playbook (dry-run) diff --git a/config/repositories.yaml b/config/repositories.yaml deleted file mode 100644 index 4345c5e7..00000000 --- a/config/repositories.yaml +++ /dev/null @@ -1,84 +0,0 @@ -# This file exists just for the purpose of running test.sh locally. -# The GH workflow generate_prs.yml will overwrite it with the contents if the -# matrix run. -# If you add a new repository here, you also need to add it to that workflow file. ---- -repositories: - - name: airflow-operator - pretty_string: Apache Airflow - product_string: airflow - url: stackabletech/airflow-operator.git - - name: commons-operator - include_productconfig: false - has_product: false - pretty_string: Stackable Commons - product_string: commons - url: stackabletech/commons-operator.git - - name: druid-operator - pretty_string: Apache Druid - product_string: druid - url: stackabletech/druid-operator.git - - name: hbase-operator - pretty_string: Apache HBase - product_string: hbase - url: stackabletech/hbase-operator.git - - name: edc-operator - pretty_string: EDC - product_string: edc - url: stackabletech/edc-operator.git - - name: hdfs-operator - pretty_string: Apache HDFS - product_string: hdfs - url: stackabletech/hdfs-operator.git - - name: hello-world-operator - pretty_string: Hello World - product_string: hello - url: stackabletech/hello-world-operator.git - - name: hive-operator - pretty_string: Apache Hive - product_string: hive - url: stackabletech/hive-operator.git - - name: kafka-operator - pretty_string: Apache Kafka - product_string: kafka - url: stackabletech/kafka-operator.git - - name: nifi-operator - pretty_string: Apache NiFi - product_string: nifi - url: stackabletech/nifi-operator.git - - name: listener-operator - include_productconfig: false - has_product: false - pretty_string: Stackable Listener Operator - product_string: listener-operator - run_as: custom - url: stackabletech/listener-operator.git - - name: opa-operator - extra_crates: - - stackable-opa-bundle-builder - pretty_string: OpenPolicyAgent - product_string: opa - url: stackabletech/opa-operator.git - - name: secret-operator - include_productconfig: false - has_product: false - pretty_string: Stackable Secret Operator - product_string: secret-operator - run_as: custom - url: stackabletech/secret-operator.git - - name: spark-k8s-operator - pretty_string: Apache Spark-on-Kubernetes - product_string: spark-k8s - url: stackabletech/spark-k8s-operator.git - - name: superset-operator - pretty_string: Apache Superset - product_string: superset - url: stackabletech/superset-operator.git - - name: trino-operator - pretty_string: Trino - product_string: trino - url: stackabletech/trino-operator.git - - name: zookeeper-operator - pretty_string: Apache ZooKeeper - product_string: zookeeper - url: stackabletech/zookeeper-operator.git diff --git a/playbook/playbook.yaml b/playbook/playbook.yaml index 1203a294..98b9d2d2 100644 --- a/playbook/playbook.yaml +++ b/playbook/playbook.yaml @@ -4,13 +4,8 @@ connection: local tasks: - - name: Include data which repositories to check - include_vars: "{{ item }}" - tags: local - with_fileglob: - - "../config/*.yaml" - - name: Ensure work directory exists + tags: local file: path: "{{ work_dir }}" state: directory @@ -32,8 +27,10 @@ - name: Update repositories from templates include_tasks: "update_repo.yaml" - with_items: "{{ repositories }}" + loop: "{{ repository }}" loop_control: + loop_var: operator - index_var: operator_index + index_var: operator_index1 + when: operator.name == only_operator or only_operator is undefined tags: local diff --git a/playbook/vars_plugins/repositories.py b/playbook/vars_plugins/repositories.py new file mode 100644 index 00000000..8246d86d --- /dev/null +++ b/playbook/vars_plugins/repositories.py @@ -0,0 +1,15 @@ +DOCUMENTATION = ''' + name: repositories + version_added: "2.10" # for collections, use the collection version, not the Ansible version + short_description: Load Stackable operator repositories + description: Load Stackable operator repositories +''' +from ansible.plugins.vars import BaseVarsPlugin +from ansible.errors import AnsibleError +import yaml + +class VarsModule(BaseVarsPlugin): + def get_vars(self, loader, path, entities): + with open('/home/sliebau/IdeaProjects/stackable/operator-templating/.github/workflows/generate_prs.yml', 'r') as file: + data = yaml.safe_load(file) + return data['jobs']['create-prs']['strategy']['matrix'] diff --git a/test.sh b/test.sh index 53c28167..7113e69a 100755 --- a/test.sh +++ b/test.sh @@ -1,3 +1,3 @@ #!/usr/bin/env bash - +export ANSIBLE_VARS_ENABLED=repositories ansible-playbook playbook/playbook.yaml --tags "local" --extra-vars "gh_access_token=unneeded base_dir=$(pwd) commit_hash=12345 reason='original message'" From 1e3c2ed5e46d36f8b0a4bb317e07e12bc15c3208 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6nke=20Liebau?= Date: Sun, 21 Jul 2024 22:16:38 +0200 Subject: [PATCH 2/2] Swapped condititons because if only_operator is undefined the prior check for a value failed. Added test limit to test.sh for debugging. --- playbook/playbook.yaml | 3 +-- test.sh | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/playbook/playbook.yaml b/playbook/playbook.yaml index 98b9d2d2..5336b759 100644 --- a/playbook/playbook.yaml +++ b/playbook/playbook.yaml @@ -5,7 +5,6 @@ tasks: - name: Ensure work directory exists - tags: local file: path: "{{ work_dir }}" state: directory @@ -32,5 +31,5 @@ loop_var: operator index_var: operator_index1 - when: operator.name == only_operator or only_operator is undefined + when: only_operator is undefined or operator.name == only_operator tags: local diff --git a/test.sh b/test.sh index 7113e69a..6d407b34 100755 --- a/test.sh +++ b/test.sh @@ -1,3 +1,3 @@ #!/usr/bin/env bash export ANSIBLE_VARS_ENABLED=repositories -ansible-playbook playbook/playbook.yaml --tags "local" --extra-vars "gh_access_token=unneeded base_dir=$(pwd) commit_hash=12345 reason='original message'" +ansible-playbook playbook/playbook.yaml --tags "local" --extra-vars "gh_access_token=unneeded base_dir=$(pwd) commit_hash=12345 reason='original message' only_operator=hbase-operator"