diff --git a/demos/airflow-scheduled-job/03-enable-and-run-spark-dag.yaml b/demos/airflow-scheduled-job/03-enable-and-run-spark-dag.yaml
index f61a9aad..88dbc834 100644
--- a/demos/airflow-scheduled-job/03-enable-and-run-spark-dag.yaml
+++ b/demos/airflow-scheduled-job/03-enable-and-run-spark-dag.yaml
@@ -8,7 +8,7 @@ spec:
spec:
containers:
- name: start-pyspark-job
- image: oci.stackable.tech/sdp/tools:1.0.0-stackable0.0.0-dev
+ image: oci.stackable.tech/sdp/tools:1.0.0-stackable25.3.0
# N.B. it is possible for the scheduler to report that a DAG exists, only for the worker task to fail if a pod is unexpectedly
# restarted. Additionally, the db-init job takes a few minutes to complete before the cluster is deployed. The wait/watch steps
# below are not "water-tight" but add a layer of stability by at least ensuring that the db is initialized and ready and that
diff --git a/demos/airflow-scheduled-job/04-enable-and-run-date-dag.yaml b/demos/airflow-scheduled-job/04-enable-and-run-date-dag.yaml
index d72fe62c..17329793 100644
--- a/demos/airflow-scheduled-job/04-enable-and-run-date-dag.yaml
+++ b/demos/airflow-scheduled-job/04-enable-and-run-date-dag.yaml
@@ -8,7 +8,7 @@ spec:
spec:
containers:
- name: start-date-job
- image: oci.stackable.tech/sdp/tools:1.0.0-stackable0.0.0-dev
+ image: oci.stackable.tech/sdp/tools:1.0.0-stackable25.3.0
# N.B. it is possible for the scheduler to report that a DAG exists, only for the worker task to fail if a pod is unexpectedly
# restarted. Additionally, the db-init job takes a few minutes to complete before the cluster is deployed. The wait/watch steps
# below are not "water-tight" but add a layer of stability by at least ensuring that the db is initialized and ready and that
diff --git a/demos/data-lakehouse-iceberg-trino-spark/create-nifi-ingestion-job.yaml b/demos/data-lakehouse-iceberg-trino-spark/create-nifi-ingestion-job.yaml
index baea7957..4a5d79b2 100644
--- a/demos/data-lakehouse-iceberg-trino-spark/create-nifi-ingestion-job.yaml
+++ b/demos/data-lakehouse-iceberg-trino-spark/create-nifi-ingestion-job.yaml
@@ -9,11 +9,11 @@ spec:
serviceAccountName: demo-serviceaccount
initContainers:
- name: wait-for-kafka
- image: oci.stackable.tech/sdp/tools:1.0.0-stackable0.0.0-dev
+ image: oci.stackable.tech/sdp/tools:1.0.0-stackable25.3.0
command: ["bash", "-c", "echo 'Waiting for all kafka brokers to be ready' && kubectl wait --for=condition=ready --timeout=30m pod -l app.kubernetes.io/instance=kafka -l app.kubernetes.io/name=kafka"]
containers:
- name: create-nifi-ingestion-job
- image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev
+ image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable25.3.0
command:
- bash
- -euo
@@ -50,6 +50,7 @@ data:
from nipyapi.security import service_login
import nipyapi
import os
+ import requests
import urllib3
# As of 2022-08-29 we cant use "https://nifi:8443" here because The request contained an invalid host header [nifi:8443] in the request [/nifi-api]. Check for request manipulation or third-party intercept.
@@ -66,70 +67,36 @@ data:
service_login(username=USERNAME, password=PASSWORD)
print("Logged in")
- organization = "stackabletech"
- repository = "demos"
- branch = "main"
- version = "main"
- directory = "demos/data-lakehouse-iceberg-trino-spark"
- flow_name = "LakehouseKafkaIngest"
+ response = requests.get("https://raw.githubusercontent.com/stackabletech/demos/refs/heads/main/demos/data-lakehouse-iceberg-trino-spark/LakehouseKafkaIngest.json")
- # Check if the GitHub flow registry client already exists
- flow_registry_clients = nipyapi.nifi.ControllerApi().get_flow_registry_clients().registries
+ filename = "/tmp/LakehouseKafkaIngest.json"
+ with open(filename, "wb") as f:
+ f.write(response.content)
- github_client = None
- for client in flow_registry_clients:
- if client.component.name == "GitHubFlowRegistryClient":
- github_client = client
- print("Found existing GitHub flow registry client")
- break
+ pg_id = get_root_pg_id()
- if not github_client:
- print("Creating new GitHub flow registry client")
- github_client = nipyapi.nifi.ControllerApi().create_flow_registry_client(
- body={
- "revision": {"version": 0},
- "component": {
- "name": "GitHubFlowRegistryClient",
- "type": "org.apache.nifi.github.GitHubFlowRegistryClient",
- "properties": {
- "Repository Owner": organization,
- "Repository Name": repository,
- },
- "bundle": {
- "group": "org.apache.nifi",
- "artifact": "nifi-github-nar",
- "version": "2.2.0",
- },
- },
- }
- )
+ if not nipyapi.config.nifi_config.api_client:
+ nipyapi.config.nifi_config.api_client = ApiClient()
- pg_id = get_root_pg_id()
+ header_params = {}
+ header_params['Accept'] = nipyapi.config.nifi_config.api_client.select_header_accept(['application/json'])
+ header_params['Content-Type'] = nipyapi.config.nifi_config.api_client.select_header_content_type(['multipart/form-data'])
- try:
- # Create process group from the file in the Git repo
- nipyapi.nifi.ProcessGroupsApi().create_process_group(
- id=pg_id,
- body={
- "revision": {"version": 0},
- "component": {
- "position": {"x": 300, "y": 10},
- "versionControlInformation": {
- "registryId": github_client.component.id,
- "flowId": flow_name,
- "bucketId": directory,
- "branch": branch,
- "version": version,
- },
- },
- },
- )
- except ValueError as e:
- # Ignore, because nipyapi can't handle non-int versions yet
- if "invalid literal for int() with base 10" in str(e):
- print("Ignoring ValueError")
- else:
- raise e
+ nipyapi.config.nifi_config.api_client.call_api('/process-groups/{pg_id}/process-groups/upload', 'POST',
+ path_params={'pg_id': pg_id},
+ header_params=header_params,
+ _return_http_data_only=True,
+ post_params=[
+ ('id', pg_id),
+ ('groupName', 'LakehouseKafkaIngest'),
+ ('positionX', 100),
+ ('positionY', 10),
+ ('clientId', nipyapi.nifi.FlowApi().generate_client_id()),
+ ],
+ files={
+ 'file': filename
+ },
+ auth_settings=['tokenAuth'])
# Scheduling the `Kafka3ConnectionService` fails, if it is started before `StandardRestrictedSSLContextService`, since it depends on it
# To work around this, we try to schedule the controllers multiple times
diff --git a/demos/data-lakehouse-iceberg-trino-spark/create-spark-ingestion-job.yaml b/demos/data-lakehouse-iceberg-trino-spark/create-spark-ingestion-job.yaml
index 50e1d820..6838d7ba 100644
--- a/demos/data-lakehouse-iceberg-trino-spark/create-spark-ingestion-job.yaml
+++ b/demos/data-lakehouse-iceberg-trino-spark/create-spark-ingestion-job.yaml
@@ -11,12 +11,88 @@ spec:
spec:
serviceAccountName: demo-serviceaccount
initContainers:
- - name: wait-for-kafka
- image: oci.stackable.tech/sdp/tools:1.0.0-stackable0.0.0-dev
- command: ["bash", "-c", "echo 'Waiting for all kafka brokers to be ready' && kubectl wait --for=condition=ready --timeout=30m pod -l app.kubernetes.io/name=kafka -l app.kubernetes.io/instance=kafka"]
+ - name: wait-for-resources
+ image: oci.stackable.tech/sdp/tools:1.0.0-stackable25.3.0
+ command:
+ - bash
+ - -euo
+ - pipefail
+ - -c
+ - |
+ echo 'Waiting for all minio instances to be ready'
+ kubectl wait --for=condition=ready --timeout=30m pod -l app=minio,release=minio,stackable.tech/vendor=Stackable
+ echo 'Waiting for all kafka brokers to be ready'
+ kubectl wait --for=condition=ready --timeout=30m pod -l app.kubernetes.io/name=kafka,app.kubernetes.io/instance=kafka
+ echo 'Waiting for all nifi instances to be ready'
+ kubectl wait --for=condition=ready --timeout=30m pod -l app.kubernetes.io/name=nifi,app.kubernetes.io/instance=nifi
+ - name: wait-for-kafka-topics
+ image: oci.stackable.tech/sdp/kafka:3.9.0-stackable25.3.0
+ command:
+ - bash
+ - -euo
+ - pipefail
+ - -c
+ - |
+ #!/usr/bin/bash
+ #
+ # Wait for Kafka topics created by the Nifi workflows to be ready.
+ # Also wait for all topic partitions to have a leader.
+ # This is required for the Spark streaming job to be able to read from the topics.
+ # Without this check, the Spark job might hang indefinitely without processing any events.
+
+ BROKER="${BROKER:-kafka-broker-default:9093}"
+
+ log() {
+ level="$1"
+ shift
+ echo "[$level] $*"
+ }
+
+ check_leaders() {
+ local topic=$1
+ local failed=0
+
+ log INFO "Starting leader check on Kafka broker: $BROKER for topic: $topic"
+ metadata=$(kcat -b "$BROKER" -X security.protocol=SSL -X ssl.key.location=/stackable/tls-kcat/tls.key -X ssl.certificate.location=/stackable/tls-kcat/tls.crt -X ssl.ca.location=/stackable/tls-kcat/ca.crt -L -t "$topic" 2>/dev/null)
+
+ if [[ -z "$metadata" ]]; then
+ log ERROR "Failed to retrieve metadata for topic: $topic"
+ return 1
+ fi
+
+ log DEBUG "Metadata for $topic:"
+ echo "$metadata"
+
+ if echo "$metadata" | grep -q 'leader: -1'; then
+ log ERROR "Found 'leader: -1' in topic '$topic'; topic not ready yet!"
+ return 1
+ fi
+
+ if echo "$metadata" | grep -q 'Broker: Leader not available'; then
+ log ERROR "Topic '$topic' not available yet"
+ return 1
+ fi
+
+ log INFO "Check topic '$topic' was successful"
+ return 0
+ }
+
+ for topic in "shared_bikes_bike_status" "shared_bikes_station_status" "shared_bikes_station_information" "water_levels_measurements" "water_levels_stations"
+ do
+ result=$(check_leaders "$topic")
+ echo "$result"
+ if [ "$result" == "1" ]
+ then
+ exit 1
+ fi
+ done
+ exit 0
+ volumeMounts:
+ - name: tls-kcat
+ mountPath: /stackable/tls-kcat
containers:
- name: create-spark-ingestion-job
- image: oci.stackable.tech/sdp/tools:1.0.0-stackable0.0.0-dev
+ image: oci.stackable.tech/sdp/tools:1.0.0-stackable25.3.0
command: ["bash", "-c", "echo 'Submitting Spark job' && kubectl apply -f /tmp/manifest/spark-ingestion-job.yaml"]
volumeMounts:
- name: manifest
@@ -25,7 +101,26 @@ spec:
- name: manifest
configMap:
name: create-spark-ingestion-job-manifest
+ - name: tls-kcat
+ ephemeral:
+ volumeClaimTemplate:
+ metadata:
+ annotations:
+ secrets.stackable.tech/backend.autotls.cert.lifetime: "1d"
+ secrets.stackable.tech/class: "tls"
+ secrets.stackable.tech/format: "tls-pem"
+ secrets.stackable.tech/scope: "pod"
+ spec:
+ accessModes:
+ - ReadWriteOnce
+ resources:
+ requests:
+ storage: "1"
+ storageClassName: secrets.stackable.tech
+ volumeMode: Filesystem
restartPolicy: OnFailure
+ securityContext:
+ fsGroup: 1000
backoffLimit: 50
---
apiVersion: v1
diff --git a/demos/data-lakehouse-iceberg-trino-spark/create-trino-tables.yaml b/demos/data-lakehouse-iceberg-trino-spark/create-trino-tables.yaml
index 5a0f5ae9..a5dbec69 100644
--- a/demos/data-lakehouse-iceberg-trino-spark/create-trino-tables.yaml
+++ b/demos/data-lakehouse-iceberg-trino-spark/create-trino-tables.yaml
@@ -9,11 +9,11 @@ spec:
serviceAccountName: demo-serviceaccount
initContainers:
- name: wait-for-testdata
- image: oci.stackable.tech/sdp/tools:1.0.0-stackable0.0.0-dev
+ image: oci.stackable.tech/sdp/tools:1.0.0-stackable25.3.0
command: ["bash", "-c", "echo 'Waiting for job load-test-data to finish' && kubectl wait --for=condition=complete --timeout=30m job/load-test-data"]
containers:
- name: create-tables-in-trino
- image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev
+ image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable25.3.0
command: ["bash", "-c", "python -u /tmp/script/script.py"]
volumeMounts:
- name: script
diff --git a/demos/data-lakehouse-iceberg-trino-spark/setup-superset.yaml b/demos/data-lakehouse-iceberg-trino-spark/setup-superset.yaml
index 3275c927..6483db6e 100644
--- a/demos/data-lakehouse-iceberg-trino-spark/setup-superset.yaml
+++ b/demos/data-lakehouse-iceberg-trino-spark/setup-superset.yaml
@@ -8,8 +8,8 @@ spec:
spec:
containers:
- name: setup-superset
- image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev
- command: ["bash", "-c", "curl -o superset-assets.zip https://raw.githubusercontent.com/stackabletech/demos/main/demos/data-lakehouse-iceberg-trino-spark/superset-assets.zip && python -u /tmp/script/script.py"]
+ image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable25.3.0
+ command: ["bash", "-c", "curl -o superset-assets.zip https://raw.githubusercontent.com/stackabletech/demos/release-25.3/demos/data-lakehouse-iceberg-trino-spark/superset-assets.zip && python -u /tmp/script/script.py"]
volumeMounts:
- name: script
mountPath: /tmp/script
diff --git a/demos/demos-v2.yaml b/demos/demos-v2.yaml
index 44aa520a..da5076c7 100644
--- a/demos/demos-v2.yaml
+++ b/demos/demos-v2.yaml
@@ -7,10 +7,10 @@ demos:
- airflow
- job-scheduling
manifests:
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/airflow-scheduled-job/01-airflow-spark-clusterrole.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/airflow-scheduled-job/02-airflow-spark-clusterrolebinding.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/airflow-scheduled-job/03-enable-and-run-spark-dag.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/airflow-scheduled-job/04-enable-and-run-date-dag.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/demos/airflow-scheduled-job/01-airflow-spark-clusterrole.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/demos/airflow-scheduled-job/02-airflow-spark-clusterrolebinding.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/demos/airflow-scheduled-job/03-enable-and-run-spark-dag.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/demos/airflow-scheduled-job/04-enable-and-run-date-dag.yaml
supportedNamespaces: []
resourceRequests:
cpu: 2401m
@@ -24,8 +24,8 @@ demos:
- hdfs
- cycling-tripdata
manifests:
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/hbase-hdfs-load-cycling-data/distcp-cycling-data.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/hbase-hdfs-load-cycling-data/create-hfile-and-import-to-hbase.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/demos/hbase-hdfs-load-cycling-data/distcp-cycling-data.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/demos/hbase-hdfs-load-cycling-data/create-hfile-and-import-to-hbase.yaml
supportedNamespaces: []
resourceRequests:
cpu: "3"
@@ -43,9 +43,9 @@ demos:
- opa
- keycloak
manifests:
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/end-to-end-security/create-trino-tables.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/end-to-end-security/serviceaccount.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/end-to-end-security/create-spark-report.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/demos/end-to-end-security/create-trino-tables.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/demos/end-to-end-security/serviceaccount.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/demos/end-to-end-security/create-spark-report.yaml
supportedNamespaces: []
resourceRequests:
cpu: 9000m
@@ -64,10 +64,10 @@ demos:
- s3
- earthquakes
manifests:
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-earthquake-data/serviceaccount.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-earthquake-data/create-nifi-ingestion-job.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-earthquake-data/create-druid-ingestion-job.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-earthquake-data/setup-superset.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/demos/nifi-kafka-druid-earthquake-data/serviceaccount.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/demos/nifi-kafka-druid-earthquake-data/create-nifi-ingestion-job.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/demos/nifi-kafka-druid-earthquake-data/create-druid-ingestion-job.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/demos/nifi-kafka-druid-earthquake-data/setup-superset.yaml
supportedNamespaces: ["default"]
resourceRequests:
cpu: 8700m
@@ -86,10 +86,10 @@ demos:
- s3
- water-levels
manifests:
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-water-level-data/serviceaccount.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-water-level-data/create-nifi-ingestion-job.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-water-level-data/create-druid-ingestion-job.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-water-level-data/setup-superset.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/demos/nifi-kafka-druid-water-level-data/serviceaccount.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/demos/nifi-kafka-druid-water-level-data/create-nifi-ingestion-job.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/demos/nifi-kafka-druid-water-level-data/create-druid-ingestion-job.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/demos/nifi-kafka-druid-water-level-data/setup-superset.yaml
supportedNamespaces: ["default"]
resourceRequests:
cpu: 8900m
@@ -106,10 +106,10 @@ demos:
- s3
- ny-taxi-data
manifests:
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/spark-k8s-anomaly-detection-taxi-data/serviceaccount.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/spark-k8s-anomaly-detection-taxi-data/load-test-data.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/spark-k8s-anomaly-detection-taxi-data/create-spark-anomaly-detection-job.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/spark-k8s-anomaly-detection-taxi-data/setup-superset.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/demos/spark-k8s-anomaly-detection-taxi-data/serviceaccount.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/demos/spark-k8s-anomaly-detection-taxi-data/load-test-data.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/demos/spark-k8s-anomaly-detection-taxi-data/create-spark-anomaly-detection-job.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/demos/spark-k8s-anomaly-detection-taxi-data/setup-superset.yaml
supportedNamespaces: []
resourceRequests:
cpu: 6400m
@@ -141,9 +141,9 @@ demos:
- s3
- ny-taxi-data
manifests:
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/trino-taxi-data/load-test-data.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/trino-taxi-data/create-table-in-trino.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/trino-taxi-data/setup-superset.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/demos/trino-taxi-data/load-test-data.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/demos/trino-taxi-data/create-table-in-trino.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/demos/trino-taxi-data/setup-superset.yaml
supportedNamespaces: []
resourceRequests:
cpu: 6800m
@@ -166,12 +166,12 @@ demos:
- water-levels
- earthquakes
manifests:
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/data-lakehouse-iceberg-trino-spark/serviceaccount.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/data-lakehouse-iceberg-trino-spark/load-test-data.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/data-lakehouse-iceberg-trino-spark/create-trino-tables.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/data-lakehouse-iceberg-trino-spark/create-nifi-ingestion-job.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/data-lakehouse-iceberg-trino-spark/create-spark-ingestion-job.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/data-lakehouse-iceberg-trino-spark/setup-superset.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/demos/data-lakehouse-iceberg-trino-spark/serviceaccount.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/demos/data-lakehouse-iceberg-trino-spark/load-test-data.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/demos/data-lakehouse-iceberg-trino-spark/create-trino-tables.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/demos/data-lakehouse-iceberg-trino-spark/create-nifi-ingestion-job.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/demos/data-lakehouse-iceberg-trino-spark/create-spark-ingestion-job.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/demos/data-lakehouse-iceberg-trino-spark/setup-superset.yaml
supportedNamespaces: ["default"]
resourceRequests:
cpu: "80"
@@ -187,7 +187,7 @@ demos:
- pyspark
- ny-taxi-data
manifests:
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data/load-test-data.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/demos/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data/load-test-data.yaml
supportedNamespaces: []
resourceRequests:
cpu: 3350m
@@ -204,7 +204,7 @@ demos:
- vector
- zookeeper
manifests:
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/logging/zookeeper.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/demos/logging/zookeeper.yaml
supportedNamespaces: []
resourceRequests:
cpu: 6500m
@@ -220,9 +220,9 @@ demos:
- grafana-dashboards
- zookeeper
manifests:
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/signal-processing/serviceaccount.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/signal-processing/create-timescale-tables.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/signal-processing/create-nifi-ingestion-job.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/demos/signal-processing/serviceaccount.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/demos/signal-processing/create-timescale-tables.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/demos/signal-processing/create-nifi-ingestion-job.yaml
supportedNamespaces: []
resourceRequests:
cpu: "3"
@@ -238,7 +238,7 @@ demos:
- spark
- S3
manifests:
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/jupyterhub-keycloak/load-gas-data.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/demos/jupyterhub-keycloak/load-gas-data.yaml
supportedNamespaces: []
resourceRequests:
cpu: 6400m
diff --git a/demos/end-to-end-security/create-spark-report.yaml b/demos/end-to-end-security/create-spark-report.yaml
index c3b1245f..bfccb3aa 100644
--- a/demos/end-to-end-security/create-spark-report.yaml
+++ b/demos/end-to-end-security/create-spark-report.yaml
@@ -12,7 +12,7 @@ spec:
serviceAccountName: demo-serviceaccount
initContainers:
- name: wait-for-trino-tables
- image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev
+ image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable25.3.0
command:
- bash
- -euo
@@ -23,7 +23,7 @@ spec:
kubectl wait --timeout=30m --for=condition=complete job/create-tables-in-trino
containers:
- name: create-spark-report
- image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev
+ image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable25.3.0
command:
- bash
- -euo
diff --git a/demos/end-to-end-security/create-trino-tables.yaml b/demos/end-to-end-security/create-trino-tables.yaml
index 8778acb5..0b10a66d 100644
--- a/demos/end-to-end-security/create-trino-tables.yaml
+++ b/demos/end-to-end-security/create-trino-tables.yaml
@@ -8,7 +8,7 @@ spec:
spec:
containers:
- name: create-tables-in-trino
- image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev
+ image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable25.3.0
command: ["bash", "-c", "python -u /tmp/script/script.py"]
volumeMounts:
- name: script
diff --git a/demos/hbase-hdfs-load-cycling-data/create-hfile-and-import-to-hbase.yaml b/demos/hbase-hdfs-load-cycling-data/create-hfile-and-import-to-hbase.yaml
index ddc7aa86..8757d7d4 100644
--- a/demos/hbase-hdfs-load-cycling-data/create-hfile-and-import-to-hbase.yaml
+++ b/demos/hbase-hdfs-load-cycling-data/create-hfile-and-import-to-hbase.yaml
@@ -9,7 +9,7 @@ spec:
spec:
containers:
- name: create-hfile-and-import-to-hbase
- image: oci.stackable.tech/sdp/hbase:2.6.1-stackable0.0.0-dev
+ image: oci.stackable.tech/sdp/hbase:2.6.1-stackable25.3.0
env:
- name: HADOOP_USER_NAME
value: stackable
diff --git a/demos/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data/load-test-data.yaml b/demos/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data/load-test-data.yaml
index 800dd9c5..c7e3887e 100644
--- a/demos/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data/load-test-data.yaml
+++ b/demos/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data/load-test-data.yaml
@@ -8,7 +8,7 @@ spec:
spec:
containers:
- name: load-ny-taxi-data
- image: oci.stackable.tech/sdp/hadoop:3.4.1-stackable0.0.0-dev
+ image: oci.stackable.tech/sdp/hadoop:3.4.1-stackable25.3.0
# yamllint disable rule:line-length
command: ["bash", "-c", "/stackable/hadoop/bin/hdfs dfs -mkdir -p /ny-taxi-data/raw \
&& cd /tmp \
diff --git a/demos/nifi-kafka-druid-earthquake-data/create-druid-ingestion-job.yaml b/demos/nifi-kafka-druid-earthquake-data/create-druid-ingestion-job.yaml
index 2c25c32c..1c91aca1 100644
--- a/demos/nifi-kafka-druid-earthquake-data/create-druid-ingestion-job.yaml
+++ b/demos/nifi-kafka-druid-earthquake-data/create-druid-ingestion-job.yaml
@@ -9,7 +9,7 @@ spec:
serviceAccountName: demo-serviceaccount
initContainers:
- name: wait-for-druid-coordinator
- image: oci.stackable.tech/sdp/tools:1.0.0-stackable0.0.0-dev
+ image: oci.stackable.tech/sdp/tools:1.0.0-stackable25.3.0
command:
- bash
- -euo
@@ -22,7 +22,7 @@ spec:
kubectl wait --for=condition=Ready pod/druid-coordinator-default-0 --timeout=30m
containers:
- name: create-druid-ingestion-job
- image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev
+ image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable25.3.0
command:
- bash
- -euo
diff --git a/demos/nifi-kafka-druid-earthquake-data/create-nifi-ingestion-job.yaml b/demos/nifi-kafka-druid-earthquake-data/create-nifi-ingestion-job.yaml
index 82005839..f80e7a00 100644
--- a/demos/nifi-kafka-druid-earthquake-data/create-nifi-ingestion-job.yaml
+++ b/demos/nifi-kafka-druid-earthquake-data/create-nifi-ingestion-job.yaml
@@ -9,7 +9,7 @@ spec:
serviceAccountName: demo-serviceaccount
initContainers:
- name: wait-for-nifi
- image: oci.stackable.tech/sdp/tools:1.0.0-stackable0.0.0-dev
+ image: oci.stackable.tech/sdp/tools:1.0.0-stackable25.3.0
command:
- bash
- -euo
@@ -22,7 +22,7 @@ spec:
kubectl wait --for=condition=Ready pod/nifi-node-default-0 --timeout=30m
containers:
- name: create-nifi-ingestion-job
- image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev
+ image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable25.3.0
command:
- bash
- -euo
@@ -59,6 +59,7 @@ data:
from nipyapi.security import service_login
import nipyapi
import os
+ import requests
import urllib3
# As of 2022-08-29 we cant use "https://nifi:8443" here because The request contained an invalid host header [nifi:8443] in the request [/nifi-api]. Check for request manipulation or third-party intercept.
@@ -75,70 +76,35 @@ data:
service_login(username=USERNAME, password=PASSWORD)
print("Logged in")
- organization = "stackabletech"
- repository = "demos"
- branch = "main"
- version = "main"
- directory = "demos/nifi-kafka-druid-earthquake-data"
- flow_name = "IngestEarthquakesToKafka"
+ response = requests.get("https://raw.githubusercontent.com/stackabletech/demos/refs/heads/main/demos/nifi-kafka-druid-earthquake-data/IngestEarthquakesToKafka.json")
+ filename = "/tmp/IngestEarthquakesToKafka.json"
+ with open(filename, "wb") as f:
+ f.write(response.content)
- # Check if the GitHub flow registry client already exists
- flow_registry_clients = nipyapi.nifi.ControllerApi().get_flow_registry_clients().registries
-
- github_client = None
- for client in flow_registry_clients:
- if client.component.name == "GitHubFlowRegistryClient":
- github_client = client
- print("Found existing GitHub flow registry client")
- break
+ pg_id = get_root_pg_id()
- if not github_client:
- print("Creating new GitHub flow registry client")
- github_client = nipyapi.nifi.ControllerApi().create_flow_registry_client(
- body={
- "revision": {"version": 0},
- "component": {
- "name": "GitHubFlowRegistryClient",
- "type": "org.apache.nifi.github.GitHubFlowRegistryClient",
- "properties": {
- "Repository Owner": organization,
- "Repository Name": repository,
- },
- "bundle": {
- "group": "org.apache.nifi",
- "artifact": "nifi-github-nar",
- "version": "2.2.0",
- },
- },
- }
- )
+ if not nipyapi.config.nifi_config.api_client:
+ nipyapi.config.nifi_config.api_client = ApiClient()
- pg_id = get_root_pg_id()
+ header_params = {}
+ header_params['Accept'] = nipyapi.config.nifi_config.api_client.select_header_accept(['application/json'])
+ header_params['Content-Type'] = nipyapi.config.nifi_config.api_client.select_header_content_type(['multipart/form-data'])
- try:
- # Create process group from the file in the Git repo
- nipyapi.nifi.ProcessGroupsApi().create_process_group(
- id=pg_id,
- body={
- "revision": {"version": 0},
- "component": {
- "position": {"x": 300, "y": 10},
- "versionControlInformation": {
- "registryId": github_client.component.id,
- "flowId": flow_name,
- "bucketId": directory,
- "branch": branch,
- "version": version,
- },
- },
- },
- )
- except ValueError as e:
- # Ignore, because nipyapi can't handle non-int versions yet
- if "invalid literal for int() with base 10" in str(e):
- print("Ignoring ValueError")
- else:
- raise e
+ nipyapi.config.nifi_config.api_client.call_api('/process-groups/{pg_id}/process-groups/upload', 'POST',
+ path_params={'pg_id': pg_id},
+ header_params=header_params,
+ _return_http_data_only=True,
+ post_params=[
+ ('id', pg_id),
+ ('groupName', 'IngestEarthquakesToKafka'),
+ ('positionX', 100),
+ ('positionY', 10),
+ ('clientId', nipyapi.nifi.FlowApi().generate_client_id()),
+ ],
+ files={
+ 'file': filename
+ },
+ auth_settings=['tokenAuth'])
# Scheduling the `Kafka3ConnectionService` fails, if it is started before `StandardRestrictedSSLContextService`, since it depends on it
# To work around this, we try to schedule the controllers multiple times
diff --git a/demos/nifi-kafka-druid-earthquake-data/setup-superset.yaml b/demos/nifi-kafka-druid-earthquake-data/setup-superset.yaml
index 9be84027..02f7e37c 100644
--- a/demos/nifi-kafka-druid-earthquake-data/setup-superset.yaml
+++ b/demos/nifi-kafka-druid-earthquake-data/setup-superset.yaml
@@ -9,7 +9,7 @@ spec:
serviceAccountName: demo-serviceaccount
initContainers:
- name: wait-for-superset
- image: oci.stackable.tech/sdp/tools:1.0.0-stackable0.0.0-dev
+ image: oci.stackable.tech/sdp/tools:1.0.0-stackable25.3.0
command:
- bash
- -euo
@@ -22,14 +22,14 @@ spec:
kubectl wait --for=condition=Ready pod/superset-node-default-0 --timeout=30m
containers:
- name: setup-superset
- image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev
+ image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable25.3.0
command:
- bash
- -euo
- pipefail
- -c
- |
- curl -o superset-assets.zip https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-earthquake-data/superset-assets.zip
+ curl -o superset-assets.zip https://raw.githubusercontent.com/stackabletech/demos/release-25.3/demos/nifi-kafka-druid-earthquake-data/superset-assets.zip
python -u /tmp/script/script.py
volumeMounts:
- name: script
diff --git a/demos/nifi-kafka-druid-water-level-data/create-druid-ingestion-job.yaml b/demos/nifi-kafka-druid-water-level-data/create-druid-ingestion-job.yaml
index 3ce637dc..692ad2ed 100644
--- a/demos/nifi-kafka-druid-water-level-data/create-druid-ingestion-job.yaml
+++ b/demos/nifi-kafka-druid-water-level-data/create-druid-ingestion-job.yaml
@@ -9,7 +9,7 @@ spec:
serviceAccountName: demo-serviceaccount
initContainers:
- name: wait-for-druid-coordinator
- image: oci.stackable.tech/sdp/tools:1.0.0-stackable0.0.0-dev
+ image: oci.stackable.tech/sdp/tools:1.0.0-stackable25.3.0
command:
- bash
- -euo
@@ -22,7 +22,7 @@ spec:
kubectl wait --for=condition=Ready pod/druid-coordinator-default-0 --timeout=30m
containers:
- name: create-druid-ingestion-job
- image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev
+ image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable25.3.0
command:
- bash
- -euo
diff --git a/demos/nifi-kafka-druid-water-level-data/create-nifi-ingestion-job.yaml b/demos/nifi-kafka-druid-water-level-data/create-nifi-ingestion-job.yaml
index ac5961f1..ba0cea4a 100644
--- a/demos/nifi-kafka-druid-water-level-data/create-nifi-ingestion-job.yaml
+++ b/demos/nifi-kafka-druid-water-level-data/create-nifi-ingestion-job.yaml
@@ -9,7 +9,7 @@ spec:
serviceAccountName: demo-serviceaccount
initContainers:
- name: wait-for-nifi
- image: oci.stackable.tech/sdp/tools:1.0.0-stackable0.0.0-dev
+ image: oci.stackable.tech/sdp/tools:1.0.0-stackable25.3.0
command:
- bash
- -euo
@@ -22,7 +22,7 @@ spec:
kubectl wait --for=condition=Ready pod/nifi-node-default-0 --timeout=30m
containers:
- name: create-nifi-ingestion-job
- image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev
+ image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable25.3.0
command:
- bash
- -euo
@@ -59,6 +59,7 @@ data:
from nipyapi.security import service_login
import nipyapi
import os
+ import requests
import urllib3
# As of 2022-08-29 we cant use "https://nifi:8443" here because The request contained an invalid host header [nifi:8443] in the request [/nifi-api]. Check for request manipulation or third-party intercept.
@@ -75,70 +76,36 @@ data:
service_login(username=USERNAME, password=PASSWORD)
print("Logged in")
- organization = "stackabletech"
- repository = "demos"
- branch = "main"
- version = "main"
- directory = "demos/nifi-kafka-druid-water-level-data"
- flow_name = "IngestWaterLevelsToKafka"
+ response = requests.get("https://raw.githubusercontent.com/stackabletech/demos/refs/heads/main/demos/nifi-kafka-druid-water-level-data/IngestWaterLevelsToKafka.json")
- # Check if the GitHub flow registry client already exists
- flow_registry_clients = nipyapi.nifi.ControllerApi().get_flow_registry_clients().registries
+ filename = "/tmp/IngestWaterLevelsToKafka.json"
+ with open(filename, "wb") as f:
+ f.write(response.content)
- github_client = None
- for client in flow_registry_clients:
- if client.component.name == "GitHubFlowRegistryClient":
- github_client = client
- print("Found existing GitHub flow registry client")
- break
+ pg_id = get_root_pg_id()
- if not github_client:
- print("Creating new GitHub flow registry client")
- github_client = nipyapi.nifi.ControllerApi().create_flow_registry_client(
- body={
- "revision": {"version": 0},
- "component": {
- "name": "GitHubFlowRegistryClient",
- "type": "org.apache.nifi.github.GitHubFlowRegistryClient",
- "properties": {
- "Repository Owner": organization,
- "Repository Name": repository,
- },
- "bundle": {
- "group": "org.apache.nifi",
- "artifact": "nifi-github-nar",
- "version": "2.2.0",
- },
- },
- }
- )
+ if not nipyapi.config.nifi_config.api_client:
+ nipyapi.config.nifi_config.api_client = ApiClient()
- pg_id = get_root_pg_id()
+ header_params = {}
+ header_params['Accept'] = nipyapi.config.nifi_config.api_client.select_header_accept(['application/json'])
+ header_params['Content-Type'] = nipyapi.config.nifi_config.api_client.select_header_content_type(['multipart/form-data'])
- try:
- # Create process group from the file in the Git repo
- nipyapi.nifi.ProcessGroupsApi().create_process_group(
- id=pg_id,
- body={
- "revision": {"version": 0},
- "component": {
- "position": {"x": 300, "y": 10},
- "versionControlInformation": {
- "registryId": github_client.component.id,
- "flowId": flow_name,
- "bucketId": directory,
- "branch": branch,
- "version": version,
- },
- },
- },
- )
- except ValueError as e:
- # Ignore, because nipyapi can't handle non-int versions yet
- if "invalid literal for int() with base 10" in str(e):
- print("Ignoring ValueError")
- else:
- raise e
+ nipyapi.config.nifi_config.api_client.call_api('/process-groups/{pg_id}/process-groups/upload', 'POST',
+ path_params={'pg_id': pg_id},
+ header_params=header_params,
+ _return_http_data_only=True,
+ post_params=[
+ ('id', pg_id),
+ ('groupName', 'IngestWaterLevelsToKafka'),
+ ('positionX', 100),
+ ('positionY', 10),
+ ('clientId', nipyapi.nifi.FlowApi().generate_client_id()),
+ ],
+ files={
+ 'file': filename
+ },
+ auth_settings=['tokenAuth'])
# Scheduling the `Kafka3ConnectionService` fails, if it is started before `StandardRestrictedSSLContextService`, since it depends on it
# To work around this, we try to schedule the controllers multiple times
diff --git a/demos/nifi-kafka-druid-water-level-data/setup-superset.yaml b/demos/nifi-kafka-druid-water-level-data/setup-superset.yaml
index 0d85b0b0..baab0639 100644
--- a/demos/nifi-kafka-druid-water-level-data/setup-superset.yaml
+++ b/demos/nifi-kafka-druid-water-level-data/setup-superset.yaml
@@ -9,7 +9,7 @@ spec:
serviceAccountName: demo-serviceaccount
initContainers:
- name: wait-for-superset
- image: oci.stackable.tech/sdp/tools:1.0.0-stackable0.0.0-dev
+ image: oci.stackable.tech/sdp/tools:1.0.0-stackable25.3.0
command:
- bash
- -euo
@@ -22,14 +22,14 @@ spec:
kubectl wait --for=condition=Ready pod/superset-node-default-0 --timeout=30m
containers:
- name: setup-superset
- image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev
+ image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable25.3.0
command:
- bash
- -euo
- pipefail
- -c
- |
- curl -o superset-assets.zip https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-water-level-data/superset-assets.zip
+ curl -o superset-assets.zip https://raw.githubusercontent.com/stackabletech/demos/release-25.3/demos/nifi-kafka-druid-water-level-data/superset-assets.zip
python -u /tmp/script/script.py
volumeMounts:
- name: script
diff --git a/demos/signal-processing/Dockerfile-nifi b/demos/signal-processing/Dockerfile-nifi
index f39a639a..e220d3a0 100644
--- a/demos/signal-processing/Dockerfile-nifi
+++ b/demos/signal-processing/Dockerfile-nifi
@@ -1,4 +1,4 @@
-FROM oci.stackable.tech/sdp/nifi:2.2.0-stackable0.0.0-dev
+FROM oci.stackable.tech/sdp/nifi:2.2.0-stackable25.3.0
# This is the postgresql JDBC driver from https://jdbc.postgresql.org/download/
# There appear to be no signatures to validate against 😬
diff --git a/demos/signal-processing/create-nifi-ingestion-job.yaml b/demos/signal-processing/create-nifi-ingestion-job.yaml
index 1a858678..1cbaa6c9 100644
--- a/demos/signal-processing/create-nifi-ingestion-job.yaml
+++ b/demos/signal-processing/create-nifi-ingestion-job.yaml
@@ -9,7 +9,7 @@ spec:
serviceAccountName: demo-serviceaccount
initContainers:
- name: wait-for-timescale-job-and-nifi
- image: oci.stackable.tech/sdp/tools:1.0.0-stackable0.0.0-dev
+ image: oci.stackable.tech/sdp/tools:1.0.0-stackable25.3.0
command:
- bash
- -euo
@@ -28,7 +28,7 @@ spec:
kubectl wait --for=condition=Ready pod/nifi-node-default-0 --timeout=30m
containers:
- name: create-nifi-ingestion-job
- image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev
+ image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable25.3.0
command:
- bash
- -euo
@@ -70,6 +70,7 @@ data:
from nipyapi.security import service_login
import nipyapi
import os
+ import requests
import urllib3
# As of 2022-08-29 we cant use "https://nifi:8443" here because The request contained an invalid host header [nifi:8443] in the request [/nifi-api]. Check for request manipulation or third-party intercept.
@@ -86,60 +87,36 @@ data:
service_login(username=USERNAME, password=PASSWORD)
print("Logged in")
- organization = "stackabletech"
- repository = "demos"
- branch = "main"
- version = "main"
- directory = "demos/signal-processing"
- flow_name = "DownloadAndWriteToDB"
+ response = requests.get("https://raw.githubusercontent.com/stackabletech/demos/refs/heads/main/demos/signal-processing/DownloadAndWriteToDB.json")
- # Register the flow registry client
- response = nipyapi.nifi.ControllerApi().create_flow_registry_client(
- body={
- "revision": {"version": 0},
- "component": {
- "name": "GitHubFlowRegistryClient",
- "type": "org.apache.nifi.github.GitHubFlowRegistryClient",
- "properties": {
- "Repository Owner": organization,
- "Repository Name": repository,
- },
- "bundle": {
- "group": "org.apache.nifi",
- "artifact": "nifi-github-nar",
- "version": "2.2.0",
- },
- },
- }
- )
+ filename = "/tmp/DownloadAndWriteToDB.json"
+ with open(filename, "wb") as f:
+ f.write(response.content)
pg_id = get_root_pg_id()
- print(f"pgid={pg_id}")
- try:
- # Create process group from the file in the Git repo
- nipyapi.nifi.ProcessGroupsApi().create_process_group(
- id=pg_id,
- body={
- "revision": {"version": 0},
- "component": {
- "position": {"x": 300, "y": 10},
- "versionControlInformation": {
- "registryId": response.component.id,
- "flowId": flow_name,
- "bucketId": directory,
- "branch": branch,
- "version": version,
- },
- },
- },
- )
- except ValueError as e:
- # Ignore, because nipyapi can't handle non-int versions yet
- if "invalid literal for int() with base 10" in str(e):
- print("Ignoring ValueError")
- else:
- raise e
+ if not nipyapi.config.nifi_config.api_client:
+ nipyapi.config.nifi_config.api_client = ApiClient()
+
+ header_params = {}
+ header_params['Accept'] = nipyapi.config.nifi_config.api_client.select_header_accept(['application/json'])
+ header_params['Content-Type'] = nipyapi.config.nifi_config.api_client.select_header_content_type(['multipart/form-data'])
+
+ nipyapi.config.nifi_config.api_client.call_api('/process-groups/{pg_id}/process-groups/upload', 'POST',
+ path_params={'pg_id': pg_id},
+ header_params=header_params,
+ _return_http_data_only=True,
+ post_params=[
+ ('id', pg_id),
+ ('groupName', 'DownloadAndWriteToDB'),
+ ('positionX', 100),
+ ('positionY', 10),
+ ('clientId', nipyapi.nifi.FlowApi().generate_client_id()),
+ ],
+ files={
+ 'file': filename
+ },
+ auth_settings=['tokenAuth'])
# Update the controller services with the correct password
for controller in list_all_controllers(pg_id):
diff --git a/demos/signal-processing/create-timescale-tables.yaml b/demos/signal-processing/create-timescale-tables.yaml
index efa8c7d6..a0a8c0bc 100644
--- a/demos/signal-processing/create-timescale-tables.yaml
+++ b/demos/signal-processing/create-timescale-tables.yaml
@@ -9,7 +9,7 @@ spec:
serviceAccountName: demo-serviceaccount
initContainers:
- name: wait-for-timescale
- image: oci.stackable.tech/sdp/tools:1.0.0-stackable0.0.0-dev
+ image: oci.stackable.tech/sdp/tools:1.0.0-stackable25.3.0
command:
- bash
- -euo
diff --git a/demos/spark-k8s-anomaly-detection-taxi-data/create-spark-anomaly-detection-job.yaml b/demos/spark-k8s-anomaly-detection-taxi-data/create-spark-anomaly-detection-job.yaml
index 3623b9e7..17f47500 100644
--- a/demos/spark-k8s-anomaly-detection-taxi-data/create-spark-anomaly-detection-job.yaml
+++ b/demos/spark-k8s-anomaly-detection-taxi-data/create-spark-anomaly-detection-job.yaml
@@ -8,11 +8,11 @@ spec:
spec:
initContainers:
- name: wait-for-testdata
- image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev
+ image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable25.3.0
command: ["bash", "-c", "echo 'Waiting for job load-ny-taxi-data to finish' && kubectl wait --for=condition=complete --timeout=30m job/load-ny-taxi-data"]
containers:
- name: create-spark-anomaly-detection-job
- image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev
+ image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable25.3.0
command: ["bash", "-c", "echo 'Submitting Spark job' && kubectl apply -f /tmp/manifest/spark-ad-job.yaml"]
volumeMounts:
- name: manifest
diff --git a/demos/spark-k8s-anomaly-detection-taxi-data/setup-superset.yaml b/demos/spark-k8s-anomaly-detection-taxi-data/setup-superset.yaml
index 56f8d5a3..c7d81cc0 100644
--- a/demos/spark-k8s-anomaly-detection-taxi-data/setup-superset.yaml
+++ b/demos/spark-k8s-anomaly-detection-taxi-data/setup-superset.yaml
@@ -8,8 +8,8 @@ spec:
spec:
containers:
- name: setup-superset
- image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev
- command: ["bash", "-c", "curl -o superset-assets.zip https://raw.githubusercontent.com/stackabletech/demos/main/demos/spark-k8s-anomaly-detection-taxi-data/superset-assets.zip && python -u /tmp/script/script.py"]
+ image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable25.3.0
+ command: ["bash", "-c", "curl -o superset-assets.zip https://raw.githubusercontent.com/stackabletech/demos/release-25.3/demos/spark-k8s-anomaly-detection-taxi-data/superset-assets.zip && python -u /tmp/script/script.py"]
volumeMounts:
- name: script
mountPath: /tmp/script
diff --git a/demos/trino-taxi-data/create-table-in-trino.yaml b/demos/trino-taxi-data/create-table-in-trino.yaml
index de3fc97e..0e79f7ba 100644
--- a/demos/trino-taxi-data/create-table-in-trino.yaml
+++ b/demos/trino-taxi-data/create-table-in-trino.yaml
@@ -8,7 +8,7 @@ spec:
spec:
containers:
- name: create-ny-taxi-data-table-in-trino
- image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev
+ image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable25.3.0
command: ["bash", "-c", "python -u /tmp/script/script.py"]
volumeMounts:
- name: script
diff --git a/demos/trino-taxi-data/setup-superset.yaml b/demos/trino-taxi-data/setup-superset.yaml
index cb613aec..1de648ff 100644
--- a/demos/trino-taxi-data/setup-superset.yaml
+++ b/demos/trino-taxi-data/setup-superset.yaml
@@ -8,8 +8,8 @@ spec:
spec:
containers:
- name: setup-superset
- image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev
- command: ["bash", "-c", "curl -o superset-assets.zip https://raw.githubusercontent.com/stackabletech/demos/main/demos/trino-taxi-data/superset-assets.zip && python -u /tmp/script/script.py"]
+ image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable25.3.0
+ command: ["bash", "-c", "curl -o superset-assets.zip https://raw.githubusercontent.com/stackabletech/demos/release-25.3/demos/trino-taxi-data/superset-assets.zip && python -u /tmp/script/script.py"]
volumeMounts:
- name: script
mountPath: /tmp/script
diff --git a/docs/antora.yml b/docs/antora.yml
index 189f7d75..16df3633 100644
--- a/docs/antora.yml
+++ b/docs/antora.yml
@@ -2,4 +2,4 @@
# Use 'home' here so that the versioning is picked up automatically based
# on SDP releases
name: home
-version: "nightly"
+version: "25.3"
diff --git a/docs/modules/demos/images/nifi-kafka-druid-earthquake-data/druid_3.png b/docs/modules/demos/images/nifi-kafka-druid-earthquake-data/druid_3.png
index f4589843..92a38b95 100644
Binary files a/docs/modules/demos/images/nifi-kafka-druid-earthquake-data/druid_3.png and b/docs/modules/demos/images/nifi-kafka-druid-earthquake-data/druid_3.png differ
diff --git a/docs/modules/demos/images/nifi-kafka-druid-earthquake-data/nifi_1.png b/docs/modules/demos/images/nifi-kafka-druid-earthquake-data/nifi_1.png
index 9882ed17..e2691d94 100644
Binary files a/docs/modules/demos/images/nifi-kafka-druid-earthquake-data/nifi_1.png and b/docs/modules/demos/images/nifi-kafka-druid-earthquake-data/nifi_1.png differ
diff --git a/docs/modules/demos/images/nifi-kafka-druid-earthquake-data/nifi_2.png b/docs/modules/demos/images/nifi-kafka-druid-earthquake-data/nifi_2.png
index 36a8ac12..9d61afa5 100644
Binary files a/docs/modules/demos/images/nifi-kafka-druid-earthquake-data/nifi_2.png and b/docs/modules/demos/images/nifi-kafka-druid-earthquake-data/nifi_2.png differ
diff --git a/docs/modules/demos/images/nifi-kafka-druid-earthquake-data/nifi_3.png b/docs/modules/demos/images/nifi-kafka-druid-earthquake-data/nifi_3.png
index eaaf98ef..f4036d9b 100644
Binary files a/docs/modules/demos/images/nifi-kafka-druid-earthquake-data/nifi_3.png and b/docs/modules/demos/images/nifi-kafka-druid-earthquake-data/nifi_3.png differ
diff --git a/docs/modules/demos/images/nifi-kafka-druid-earthquake-data/nifi_4.png b/docs/modules/demos/images/nifi-kafka-druid-earthquake-data/nifi_4.png
index 967e7303..d427b42a 100644
Binary files a/docs/modules/demos/images/nifi-kafka-druid-earthquake-data/nifi_4.png and b/docs/modules/demos/images/nifi-kafka-druid-earthquake-data/nifi_4.png differ
diff --git a/docs/modules/demos/images/nifi-kafka-druid-earthquake-data/nifi_5.png b/docs/modules/demos/images/nifi-kafka-druid-earthquake-data/nifi_5.png
index b4c4bdea..5de84725 100644
Binary files a/docs/modules/demos/images/nifi-kafka-druid-earthquake-data/nifi_5.png and b/docs/modules/demos/images/nifi-kafka-druid-earthquake-data/nifi_5.png differ
diff --git a/docs/modules/demos/images/nifi-kafka-druid-water-level-data/druid_3.png b/docs/modules/demos/images/nifi-kafka-druid-water-level-data/druid_3.png
index 67518895..bd47ae9d 100644
Binary files a/docs/modules/demos/images/nifi-kafka-druid-water-level-data/druid_3.png and b/docs/modules/demos/images/nifi-kafka-druid-water-level-data/druid_3.png differ
diff --git a/docs/modules/demos/images/nifi-kafka-druid-water-level-data/nifi_10.png b/docs/modules/demos/images/nifi-kafka-druid-water-level-data/nifi_10.png
index d4bf96f1..7cb69d7a 100644
Binary files a/docs/modules/demos/images/nifi-kafka-druid-water-level-data/nifi_10.png and b/docs/modules/demos/images/nifi-kafka-druid-water-level-data/nifi_10.png differ
diff --git a/docs/modules/demos/images/nifi-kafka-druid-water-level-data/nifi_11.png b/docs/modules/demos/images/nifi-kafka-druid-water-level-data/nifi_11.png
index ae53f980..10d5ea7c 100644
Binary files a/docs/modules/demos/images/nifi-kafka-druid-water-level-data/nifi_11.png and b/docs/modules/demos/images/nifi-kafka-druid-water-level-data/nifi_11.png differ
diff --git a/docs/modules/demos/images/nifi-kafka-druid-water-level-data/nifi_12.png b/docs/modules/demos/images/nifi-kafka-druid-water-level-data/nifi_12.png
index 6ebabb21..af3e0090 100644
Binary files a/docs/modules/demos/images/nifi-kafka-druid-water-level-data/nifi_12.png and b/docs/modules/demos/images/nifi-kafka-druid-water-level-data/nifi_12.png differ
diff --git a/docs/modules/demos/images/nifi-kafka-druid-water-level-data/nifi_13.png b/docs/modules/demos/images/nifi-kafka-druid-water-level-data/nifi_13.png
new file mode 100644
index 00000000..507fca20
Binary files /dev/null and b/docs/modules/demos/images/nifi-kafka-druid-water-level-data/nifi_13.png differ
diff --git a/docs/modules/demos/images/nifi-kafka-druid-water-level-data/nifi_2.png b/docs/modules/demos/images/nifi-kafka-druid-water-level-data/nifi_2.png
index 4e3b86f1..a15c6bfc 100644
Binary files a/docs/modules/demos/images/nifi-kafka-druid-water-level-data/nifi_2.png and b/docs/modules/demos/images/nifi-kafka-druid-water-level-data/nifi_2.png differ
diff --git a/docs/modules/demos/images/nifi-kafka-druid-water-level-data/nifi_3.png b/docs/modules/demos/images/nifi-kafka-druid-water-level-data/nifi_3.png
index b41bf6bb..a133deb7 100644
Binary files a/docs/modules/demos/images/nifi-kafka-druid-water-level-data/nifi_3.png and b/docs/modules/demos/images/nifi-kafka-druid-water-level-data/nifi_3.png differ
diff --git a/docs/modules/demos/images/nifi-kafka-druid-water-level-data/nifi_4.png b/docs/modules/demos/images/nifi-kafka-druid-water-level-data/nifi_4.png
index 7e7c41c1..016f2dc9 100644
Binary files a/docs/modules/demos/images/nifi-kafka-druid-water-level-data/nifi_4.png and b/docs/modules/demos/images/nifi-kafka-druid-water-level-data/nifi_4.png differ
diff --git a/docs/modules/demos/images/nifi-kafka-druid-water-level-data/nifi_5.png b/docs/modules/demos/images/nifi-kafka-druid-water-level-data/nifi_5.png
index 2943b603..85457447 100644
Binary files a/docs/modules/demos/images/nifi-kafka-druid-water-level-data/nifi_5.png and b/docs/modules/demos/images/nifi-kafka-druid-water-level-data/nifi_5.png differ
diff --git a/docs/modules/demos/images/nifi-kafka-druid-water-level-data/nifi_6.png b/docs/modules/demos/images/nifi-kafka-druid-water-level-data/nifi_6.png
index baa00472..2b35368a 100644
Binary files a/docs/modules/demos/images/nifi-kafka-druid-water-level-data/nifi_6.png and b/docs/modules/demos/images/nifi-kafka-druid-water-level-data/nifi_6.png differ
diff --git a/docs/modules/demos/images/nifi-kafka-druid-water-level-data/nifi_7.png b/docs/modules/demos/images/nifi-kafka-druid-water-level-data/nifi_7.png
index c33b8bd5..b06b25ae 100644
Binary files a/docs/modules/demos/images/nifi-kafka-druid-water-level-data/nifi_7.png and b/docs/modules/demos/images/nifi-kafka-druid-water-level-data/nifi_7.png differ
diff --git a/docs/modules/demos/images/nifi-kafka-druid-water-level-data/nifi_8.png b/docs/modules/demos/images/nifi-kafka-druid-water-level-data/nifi_8.png
index cedf8721..97961e95 100644
Binary files a/docs/modules/demos/images/nifi-kafka-druid-water-level-data/nifi_8.png and b/docs/modules/demos/images/nifi-kafka-druid-water-level-data/nifi_8.png differ
diff --git a/docs/modules/demos/images/nifi-kafka-druid-water-level-data/nifi_9.png b/docs/modules/demos/images/nifi-kafka-druid-water-level-data/nifi_9.png
index e2d3d418..042a90b1 100644
Binary files a/docs/modules/demos/images/nifi-kafka-druid-water-level-data/nifi_9.png and b/docs/modules/demos/images/nifi-kafka-druid-water-level-data/nifi_9.png differ
diff --git a/docs/modules/demos/pages/nifi-kafka-druid-earthquake-data.adoc b/docs/modules/demos/pages/nifi-kafka-druid-earthquake-data.adoc
index f49a40c8..9fd42157 100644
--- a/docs/modules/demos/pages/nifi-kafka-druid-earthquake-data.adoc
+++ b/docs/modules/demos/pages/nifi-kafka-druid-earthquake-data.adoc
@@ -209,25 +209,26 @@ Log in with the username `admin` and password `adminadmin`.
image::nifi-kafka-druid-earthquake-data/nifi_2.png[]
-You can see the started ProcessGroup consisting of two processors.
+You can see the started ProcessGroup consisting of three processors.
The first one - `InvokeHTTP`, fetches the CSV file from the Internet and puts it into the queue of the next processor.
-The second processor - `PublishKafkaRecord_2_6`, parses the CSV file, converts it to JSON records and writes them out into Kafka.
+The second processor - `SplitRecords`, takes the single FlowFile (NiFi Record) which contains all CSV records and splits it into chunks of 2000 records, which are then separately put into the queue of the next processor.
+The third one - `PublishKafkaRecord`, parses the CSV chunk, converts it to JSON records and writes them out into Kafka.
Double-click on the `InvokeHTTP` processor to show the processor details.
image::nifi-kafka-druid-earthquake-data/nifi_3.png[]
-Head over to the tab `PROPERTIES`.
+Head over to the `Properties` tab.
image::nifi-kafka-druid-earthquake-data/nifi_4.png[]
Here, you can see the setting `HTTP URL`, which specifies the download URL from where the CSV file is retrieved.
-Close the processor details popup by clicking `OK`.
-Afterwards, double-click on the processor `PublishKafkaRecord_2_6`.
+Close the processor details popup by clicking `Close`.
+Afterwards, double-click on the processor `PublishKafkaRecord`.
image::nifi-kafka-druid-earthquake-data/nifi_5.png[]
-The Kafka connection details within this processor - like broker addresses and topic names - are specified.
+The Kafka connection service, which contains the connection details, and the topic name is specified.
It uses the `CSVReader` to parse the downloaded CSV and the `JsonRecordSetWriter` to split it into individual JSON records before writing it out.
== Druid
@@ -247,12 +248,13 @@ By clicking on `Supervisors` at the top, you can see the running ingestion jobs.
image::nifi-kafka-druid-earthquake-data/druid_2.png[]
You can see additional information after clicking on the magnification glass to the right side of the `RUNNING` supervisor.
-On the tab `Statistics` on the left, you can see the number of processed records as well as the number of errors.
+On the tab `Task stats` on the left, you can see the number of processed records as well as the number of errors.
image::nifi-kafka-druid-earthquake-data/druid_3.png[]
-The statistics show that Druid ingested `5074` records during the last minute and has ingested 3 million records already.
-All entries have been consumed successfully, indicated by having no `processWithError`, `thrownAway` or `unparseable` records.
+The statistics show that Druid ingested `13279` records per second within the last minute and has ingested around 600,000 records already.
+All entries have been consumed successfully, indicated by having no `processWithError`, `thrownAway` or `unparseable` records in the output of the `View raw`
+button at the top right.
=== Query the Data Source
diff --git a/docs/modules/demos/pages/nifi-kafka-druid-water-level-data.adoc b/docs/modules/demos/pages/nifi-kafka-druid-water-level-data.adoc
index 090ea66c..b7267171 100644
--- a/docs/modules/demos/pages/nifi-kafka-druid-water-level-data.adoc
+++ b/docs/modules/demos/pages/nifi-kafka-druid-water-level-data.adoc
@@ -292,9 +292,14 @@ image::nifi-kafka-druid-water-level-data/nifi_1.png[]
Log in with the username `admin` and password `adminadmin`.
+image::nifi-kafka-druid-water-level-data/nifi_13.png[]
+
+The NiFi workflow contains the `IngestWaterLevelsToKafka` Process Group. Enter the Process Group by double-clicking on it or
+right-clicking and selecting `Enter Group`.
+
image::nifi-kafka-druid-water-level-data/nifi_2.png[]
-As you can see, the NiFi workflow consists of lots of components. It is split into three main components:
+As you can see, the Process Group consists of lots of components. It is split into three main components:
. On the left is the first part bulk-loading all the known stations
. In the middle is the second part bulk-loading the historical data for the last 30 days
@@ -318,19 +323,19 @@ The left workflows works as follows:
measurements of the last 30 days.
. `Add station_uuid` will add the attribute `station_uuid` to the JSON list of measurements returned from the
{pegelonline}[PEGELONLINE web service], which is missing.
-. `PublishKafkaRecord_2_6` finally emits every measurement as a Kafka record to the topic `measurements`.
+. `PublishKafka` finally emits every measurement as a Kafka record to the topic `measurements`.
The right side works similarly but is executed in an endless loop to stream the data in near-realtime. Double-click on
the `Get station list` processor to show the processor details.
image::nifi-kafka-druid-water-level-data/nifi_5.png[]
-Head over to the tab `PROPERTIES`.
+Head over to the `Properties` tab.
image::nifi-kafka-druid-water-level-data/nifi_6.png[]
Here, you can see the setting `HTTP URL`, which specifies the download URL from where the JSON file containing the
-stations is retrieved. Close the processor details popup by clicking `OK`. You can also have a detailed view of the
+stations is retrieved. Close the processor details popup by clicking `Close`. You can also have a detailed view of the
`Produce station records` processor by double-clicking it.
image::nifi-kafka-druid-water-level-data/nifi_7.png[]
@@ -347,11 +352,11 @@ image::nifi-kafka-druid-water-level-data/nifi_9.png[]
The `HTTP URL` does contain the `$\{station_uuid\}` placeholder, which gets replaced for every station.
-Double-click the `PublishKafkaRecord_2_6` processor.
+Double-click the `PublishKafka` processor.
image::nifi-kafka-druid-water-level-data/nifi_10.png[]
-You can also see the number of produced records by right-clicking on `PublishKafkaRecord_2_6` and selecting
+You can also see the number of produced records by right-clicking on `PublishKafka` and selecting
`View status history`.
image::nifi-kafka-druid-water-level-data/nifi_11.png[]
@@ -385,13 +390,14 @@ By clicking on `Supervisors` at the top you can see the running ingestion jobs.
image::nifi-kafka-druid-water-level-data/druid_2.png[]
After clicking on the magnification glass to the right side of the `RUNNING` supervisor, you can see additional
-information (here, the supervisor `measurements` was chosen). On the tab `Statistics` on the left, you can see the
+information (here, the supervisor `measurements` was chosen). On the tab `Task stats` on the left, you can see the
number of processed records as well as the number of errors.
image::nifi-kafka-druid-water-level-data/druid_3.png[]
-The statistics show that Druid ingested `2435` records during the last minute and has already ingested ~30 million records in total. All records
-have been ingested successfully, indicated by having no `processWithError`, `thrownAway` or `unparseable` records.
+The statistics show that Druid ingested `594` records during the last minute and has already ingested ~1,3 million records in total. All records
+have been ingested successfully, indicated by having no `processWithError`, `thrownAway` or `unparseable` records in the output of the `View raw`
+button at the top right.
=== Query the Data Source
diff --git a/docs/modules/demos/pages/spark-k8s-anomaly-detection-taxi-data.adoc b/docs/modules/demos/pages/spark-k8s-anomaly-detection-taxi-data.adoc
index fcd12656..4406aaa1 100644
--- a/docs/modules/demos/pages/spark-k8s-anomaly-detection-taxi-data.adoc
+++ b/docs/modules/demos/pages/spark-k8s-anomaly-detection-taxi-data.adoc
@@ -3,7 +3,6 @@
:scikit-lib: https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.IsolationForest.html
:k8s-cpu: https://kubernetes.io/docs/tasks/debug/debug-cluster/resource-metrics-pipeline/#cpu
-:forest-article: https://towardsdatascience.com/isolation-forest-and-spark-b88ade6c63ff
:forest-algo: https://cs.nju.edu.cn/zhouzh/zhouzh.files/publication/icdm08b.pdf
Install this demo on an existing Kubernetes cluster:
@@ -63,22 +62,22 @@ To list the installed Stackable services run the following command:
----
$ stackablectl stacklet list
-┌──────────┬───────────────┬───────────┬───────────────────────────────────────────────┬─────────────────────────────────┐
-│ PRODUCT ┆ NAME ┆ NAMESPACE ┆ ENDPOINTS ┆ CONDITIONS │
-╞══════════╪═══════════════╪═══════════╪═══════════════════════════════════════════════╪═════════════════════════════════╡
-│ hive ┆ hive ┆ default ┆ ┆ Available, Reconciling, Running │
-├╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤
-│ hive ┆ hive-iceberg ┆ default ┆ ┆ Available, Reconciling, Running │
-├╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤
-│ opa ┆ opa ┆ default ┆ ┆ Available, Reconciling, Running │
-├╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤
-│ superset ┆ superset ┆ default ┆ external-http http://172.18.0.2:30562 ┆ Available, Reconciling, Running │
-├╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤
-│ trino ┆ trino ┆ default ┆ coordinator-metrics 172.18.0.2:31980 ┆ Available, Reconciling, Running │
-│ ┆ ┆ ┆ coordinator-https https://172.18.0.2:32186 ┆ │
-├╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤
-│ minio ┆ minio-console ┆ default ┆ http http://172.18.0.2:32276 ┆ │
-└──────────┴───────────────┴───────────┴───────────────────────────────────────────────┴─────────────────────────────────┘
+┌──────────┬───────────────┬───────────┬──────────────────────────────────────────────┬─────────────────────────────────┐
+│ PRODUCT ┆ NAME ┆ NAMESPACE ┆ ENDPOINTS ┆ CONDITIONS │
+╞══════════╪═══════════════╪═══════════╪══════════════════════════════════════════════╪═════════════════════════════════╡
+│ hive ┆ hive ┆ default ┆ ┆ Available, Reconciling, Running │
+├╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤
+│ hive ┆ hive-iceberg ┆ default ┆ ┆ Available, Reconciling, Running │
+├╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤
+│ opa ┆ opa ┆ default ┆ ┆ Available, Reconciling, Running │
+├╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤
+│ superset ┆ superset ┆ default ┆ external-http http://10.0.0.12:30171 ┆ Available, Reconciling, Running │
+├╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤
+│ trino ┆ trino ┆ default ┆ coordinator-metrics 10.0.0.12:30334 ┆ Available, Reconciling, Running │
+│ ┆ ┆ ┆ coordinator-https https://10.0.0.12:32663 ┆ │
+├╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤
+│ minio ┆ minio-console ┆ default ┆ https https://10.0.0.11:31142 ┆ │
+└──────────┴───────────────┴───────────┴──────────────────────────────────────────────┴─────────────────────────────────┘
----
include::partial$instance-hint.adoc[]
@@ -88,7 +87,14 @@ include::partial$instance-hint.adoc[]
=== List Buckets
The S3 provided by MinIO is used as persistent storage to store all the data used.
-Open the endpoint `http` retrieved by `stackablectl stacklet list` in your browser (http://172.18.0.2:32276 in this case).
+Open the endpoint `https` retrieved by `stackablectl stacklet list` in your browser (https://10.0.0.11:31142 in this case).
+You need to accept the self-signed certificate in your browser before you can access the MinIO console.
+If the console is not reachable, you can forward the port used by the MinIO console and use https://localhost:9001 instead.
+
+[source,console]
+----
+$ kubectl port-forward service/minio-console 9001:https
+----
image::spark-k8s-anomaly-detection-taxi-data/minio_0.png[]
@@ -124,7 +130,7 @@ This is a much smaller file, as it only contains scores for each aggregated peri
The Spark job ingests the raw data and performs straightforward data wrangling and feature engineering.
Any windowing features designed to capture the time-series nature of the data - such as lags or rolling averages - need to use evenly distributed partitions so that Spark can execute these tasks in parallel.
The job uses an implementation of the Isolation Forest {forest-algo}[algorithm] provided by the scikit-learn {scikit-lib}[library]:
-the model is trained in a single task but is then distributed to each executor from where a user-defined function invokes it (see {forest-article}[this article] for how to call the sklearn library with a pyspark UDF).
+the model is trained in a single task and distributed among executors with the help of a PySpark user defined function.
The Isolation Forest algorithm is used for unsupervised model training, meaning that a labelled set of data - against which the model is trained - is unnecessary.
This makes model preparation easier as we do not have to divide the data set into training and validation datasets.
@@ -142,6 +148,13 @@ image::spark-k8s-anomaly-detection-taxi-data/spark_job.png[]
== Dashboard
Open the `external-http` Superset endpoint found in the output of the `stackablectl stacklet list` command.
+Alternatively, create a port-forward to the Superset service and point your browser to http://localhost:8088:
+
+[source,console]
+----
+$ kubectl port-forward service/superset-external 8088:http
+----
+
The anomaly detection dashboard is pre-defined and accessible under the `Dashboards` tab when logged in to Superset using the username `admin` password `adminadmin`:
image::spark-k8s-anomaly-detection-taxi-data/superset_anomaly_scores.png[]
diff --git a/stacks/_templates/jupyterhub.yaml b/stacks/_templates/jupyterhub.yaml
index d0ef489d..53e8d198 100644
--- a/stacks/_templates/jupyterhub.yaml
+++ b/stacks/_templates/jupyterhub.yaml
@@ -50,8 +50,8 @@ options:
HADOOP_CONF_DIR: "/home/jovyan/hdfs"
initContainers:
- name: download-notebook
- image: oci.stackable.tech/sdp/tools:1.0.0-stackable0.0.0-dev
- command: ['sh', '-c', 'curl https://raw.githubusercontent.com/stackabletech/demos/main/stacks/jupyterhub-pyspark-hdfs/notebook.ipynb -o /notebook/notebook.ipynb']
+ image: oci.stackable.tech/sdp/tools:1.0.0-stackable25.3.0
+ command: ['sh', '-c', 'curl https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/jupyterhub-pyspark-hdfs/notebook.ipynb -o /notebook/notebook.ipynb']
volumeMounts:
- mountPath: /notebook
name: notebook
diff --git a/stacks/_templates/keycloak.yaml b/stacks/_templates/keycloak.yaml
index dee91706..58177993 100644
--- a/stacks/_templates/keycloak.yaml
+++ b/stacks/_templates/keycloak.yaml
@@ -48,7 +48,7 @@ spec:
- name: tls
mountPath: /tls/
- name: create-auth-class
- image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev
+ image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable25.3.0
command: ["/bin/bash", "-c"]
args:
- |
diff --git a/stacks/end-to-end-security/hive-metastore.yaml b/stacks/end-to-end-security/hive-metastore.yaml
index c64a5786..e1c08899 100644
--- a/stacks/end-to-end-security/hive-metastore.yaml
+++ b/stacks/end-to-end-security/hive-metastore.yaml
@@ -5,7 +5,7 @@ metadata:
name: hive-iceberg
spec:
image:
- productVersion: 4.0.1
+ productVersion: 4.0.0
clusterConfig:
database:
connString: jdbc:postgresql://postgresql-hive-iceberg:5432/hive
diff --git a/stacks/end-to-end-security/krb5.yaml b/stacks/end-to-end-security/krb5.yaml
index e92aa69f..353b8fe9 100644
--- a/stacks/end-to-end-security/krb5.yaml
+++ b/stacks/end-to-end-security/krb5.yaml
@@ -14,7 +14,7 @@ spec:
spec:
initContainers:
- name: init
- image: oci.stackable.tech/sdp/krb5:1.21.1-stackable0.0.0-dev
+ image: oci.stackable.tech/sdp/krb5:1.21.1-stackable25.3.0
args:
- sh
- -euo
@@ -35,7 +35,7 @@ spec:
name: data
containers:
- name: kdc
- image: oci.stackable.tech/sdp/krb5:1.21.1-stackable0.0.0-dev
+ image: oci.stackable.tech/sdp/krb5:1.21.1-stackable25.3.0
args:
- krb5kdc
- -n
@@ -48,7 +48,7 @@ spec:
- mountPath: /var/kerberos/krb5kdc
name: data
- name: kadmind
- image: oci.stackable.tech/sdp/krb5:1.21.1-stackable0.0.0-dev
+ image: oci.stackable.tech/sdp/krb5:1.21.1-stackable25.3.0
args:
- kadmind
- -nofork
@@ -61,7 +61,7 @@ spec:
- mountPath: /var/kerberos/krb5kdc
name: data
- name: client
- image: oci.stackable.tech/sdp/krb5:1.21.1-stackable0.0.0-dev
+ image: oci.stackable.tech/sdp/krb5:1.21.1-stackable25.3.0
tty: true
stdin: true
env:
diff --git a/stacks/end-to-end-security/setup-postgresql.yaml b/stacks/end-to-end-security/setup-postgresql.yaml
index 64e45ffb..9cc31af7 100644
--- a/stacks/end-to-end-security/setup-postgresql.yaml
+++ b/stacks/end-to-end-security/setup-postgresql.yaml
@@ -9,14 +9,14 @@ spec:
initContainers:
# The postgres image does not contain curl or wget...
- name: download-dump
- image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev
+ image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable25.3.0
command:
- bash
- -c
- |
kubectl rollout status --watch statefulset/postgresql-superset
cd /tmp
- curl --fail -O https://raw.githubusercontent.com/stackabletech/demos/main/stacks/end-to-end-security/postgres_superset_dump.sql.gz
+ curl --fail -O https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/end-to-end-security/postgres_superset_dump.sql.gz
gunzip postgres_superset_dump.sql.gz
# We need to omit changing the users password, as otherwise the content in the Secrets does not match
diff --git a/stacks/end-to-end-security/superset.yaml b/stacks/end-to-end-security/superset.yaml
index a35afa19..4a7c9635 100644
--- a/stacks/end-to-end-security/superset.yaml
+++ b/stacks/end-to-end-security/superset.yaml
@@ -23,7 +23,7 @@ spec:
spec:
initContainers:
- name: wait-for-setup-db-job
- image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev
+ image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable25.3.0
command:
- bash
- -c
diff --git a/stacks/jupyterhub-keycloak/jupyterhub.yaml b/stacks/jupyterhub-keycloak/jupyterhub.yaml
index 375763e1..5489c1aa 100644
--- a/stacks/jupyterhub-keycloak/jupyterhub.yaml
+++ b/stacks/jupyterhub-keycloak/jupyterhub.yaml
@@ -148,8 +148,8 @@ options:
stackable.tech/vendor: Stackable
initContainers:
- name: download-notebook
- image: oci.stackable.tech/sdp/tools:1.0.0-stackable0.0.0-dev
- command: ['sh', '-c', 'curl https://raw.githubusercontent.com/stackabletech/demos/main/stacks/jupyterhub-keycloak/process-s3.ipynb -o /notebook/process-s3.ipynb']
+ image: oci.stackable.tech/sdp/tools:1.0.0-stackable25.3.0
+ command: ['sh', '-c', 'curl https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/jupyterhub-keycloak/process-s3.ipynb -o /notebook/process-s3.ipynb']
volumeMounts:
- mountPath: /notebook
name: notebook
diff --git a/stacks/jupyterhub-keycloak/keycloak.yaml b/stacks/jupyterhub-keycloak/keycloak.yaml
index eb625208..b18a4893 100644
--- a/stacks/jupyterhub-keycloak/keycloak.yaml
+++ b/stacks/jupyterhub-keycloak/keycloak.yaml
@@ -50,7 +50,7 @@ spec:
mountPath: /tls/
- name: create-configmap
resources: {}
- image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev
+ image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable25.3.0
command: ["/bin/bash", "-c"]
args:
- |
diff --git a/stacks/keycloak-opa-poc/keycloak.yaml b/stacks/keycloak-opa-poc/keycloak.yaml
index a5d64637..4ad1b099 100644
--- a/stacks/keycloak-opa-poc/keycloak.yaml
+++ b/stacks/keycloak-opa-poc/keycloak.yaml
@@ -70,7 +70,7 @@ spec:
spec:
containers:
- name: propagate-keycloak-address
- image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev
+ image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable25.3.0
command:
- bash
- -x
diff --git a/stacks/keycloak-opa-poc/setup-keycloak.yaml b/stacks/keycloak-opa-poc/setup-keycloak.yaml
index fca8cdd5..b81934aa 100644
--- a/stacks/keycloak-opa-poc/setup-keycloak.yaml
+++ b/stacks/keycloak-opa-poc/setup-keycloak.yaml
@@ -29,7 +29,7 @@ spec:
spec:
containers:
- name: setup-keycloak
- image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev
+ image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable25.3.0
env:
- name: KEYCLOAK_ADMIN_PASSWORD
valueFrom:
diff --git a/stacks/logging/setup-opensearch-dashboards.yaml b/stacks/logging/setup-opensearch-dashboards.yaml
index bf707449..43653fba 100644
--- a/stacks/logging/setup-opensearch-dashboards.yaml
+++ b/stacks/logging/setup-opensearch-dashboards.yaml
@@ -8,7 +8,7 @@ spec:
spec:
containers:
- name: setup-opensearch-dashboards
- image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev
+ image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable25.3.0
env:
- name: OPEN_SEARCH_ADMIN_PASSWORD
valueFrom:
diff --git a/stacks/signal-processing/jupyterhub.yaml b/stacks/signal-processing/jupyterhub.yaml
index 7bec554e..6e28af01 100644
--- a/stacks/signal-processing/jupyterhub.yaml
+++ b/stacks/signal-processing/jupyterhub.yaml
@@ -30,7 +30,7 @@ options:
singleuser:
cmd: null
image:
- # TODO (@NickLarsenNZ): Use a versioned image with stackable0.0.0-dev or stackableXX.X.X so that
+ # TODO (@NickLarsenNZ): Use a versioned image with stackable25.3.0 or stackableXX.X.X so that
# the demo is reproducable for the release and it will be automatically replaced for the release branch.
name: oci.stackable.tech/stackable/jupyter-pyspark-with-alibi-detect
tag: python-3.9
@@ -41,8 +41,8 @@ options:
stackable.tech/vendor: Stackable
initContainers:
- name: download-notebook
- image: oci.stackable.tech/sdp/tools:1.0.0-stackable0.0.0-dev
- command: ['sh', '-c', 'curl https://raw.githubusercontent.com/stackabletech/demos/main/stacks/signal-processing/tsdb.ipynb -o /notebook/tsdb.ipynb']
+ image: oci.stackable.tech/sdp/tools:1.0.0-stackable25.3.0
+ command: ['sh', '-c', 'curl https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/signal-processing/tsdb.ipynb -o /notebook/tsdb.ipynb']
volumeMounts:
- mountPath: /notebook
name: notebook
diff --git a/stacks/signal-processing/nifi.yaml b/stacks/signal-processing/nifi.yaml
index 62231374..22f4415f 100644
--- a/stacks/signal-processing/nifi.yaml
+++ b/stacks/signal-processing/nifi.yaml
@@ -6,7 +6,7 @@ metadata:
spec:
image:
productVersion: 2.2.0
- # TODO (@NickLarsenNZ): Use a versioned image with stackable0.0.0-dev or stackableXX.X.X so that
+ # TODO (@NickLarsenNZ): Use a versioned image with stackable25.3.0 or stackableXX.X.X so that
# the demo is reproducable for the release and it will be automatically replaced for the release branch.
custom: oci.stackable.tech/stackable/nifi:2.2.0-postgresql
# pullPolicy: IfNotPresent
diff --git a/stacks/spark-trino-superset-s3/s3-connection.yaml b/stacks/spark-trino-superset-s3/s3-connection.yaml
index 55ae8400..980a3626 100644
--- a/stacks/spark-trino-superset-s3/s3-connection.yaml
+++ b/stacks/spark-trino-superset-s3/s3-connection.yaml
@@ -10,6 +10,11 @@ spec:
accessStyle: Path
credentials:
secretClass: minio-s3-credentials
+ tls:
+ verification:
+ server:
+ caCert:
+ secretClass: tls
---
apiVersion: secrets.stackable.tech/v1alpha1
kind: SecretClass
diff --git a/stacks/stacks-v2.yaml b/stacks/stacks-v2.yaml
index d013c95f..17c058ee 100644
--- a/stacks/stacks-v2.yaml
+++ b/stacks/stacks-v2.yaml
@@ -2,7 +2,7 @@
stacks:
monitoring:
description: Stack containing Prometheus and Grafana
- stackableRelease: dev
+ stackableRelease: 25.3
stackableOperators:
- commons
- listener
@@ -11,9 +11,9 @@ stacks:
- prometheus
- grafana
manifests:
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/monitoring/grafana-dashboards.yaml
- - helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/prometheus.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/prometheus-service-monitor.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/monitoring/grafana-dashboards.yaml
+ - helmChart: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/_templates/prometheus.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/_templates/prometheus-service-monitor.yaml
supportedNamespaces: []
resourceRequests:
cpu: 1750m
@@ -25,7 +25,7 @@ stacks:
default: adminadmin
logging:
description: Stack containing OpenSearch, OpenSearch Dashboards (Kibana) and Vector aggregator
- stackableRelease: dev
+ stackableRelease: 25.3
stackableOperators:
- commons
- listener
@@ -37,11 +37,11 @@ stacks:
- opensearch-dashboards
- vector
manifests:
- - helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/opensearch.yaml
- - helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/opensearch-dashboards.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/logging/setup-opensearch-dashboards.yaml
- - helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/vector-aggregator.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/vector-aggregator-discovery.yaml
+ - helmChart: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/_templates/opensearch.yaml
+ - helmChart: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/_templates/opensearch-dashboards.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/logging/setup-opensearch-dashboards.yaml
+ - helmChart: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/_templates/vector-aggregator.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/_templates/vector-aggregator-discovery.yaml
supportedNamespaces: []
resourceRequests:
cpu: 5150m
@@ -60,7 +60,7 @@ stacks:
observability:
description: >-
An observability stack with auto-injection of the opentelemetry-collector sidecar to receive traces/logs/metrics via OTLP, and send them to Jaeger/Tempo/Loki.
- stackableRelease: dev
+ stackableRelease: 25.3
stackableOperators:
- commons
- listener
@@ -71,21 +71,21 @@ stacks:
- observability
- tracing
manifests:
- - helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/observability/jaeger.yaml
- - helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/observability/opentelemetry-operator.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/observability/grafana-admin-credentials.yaml
- - helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/observability/grafana.yaml
- - helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/observability/grafana-tempo.yaml
- - helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/observability/grafana-loki.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/observability/opentelemetry-collector-sidecar.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/observability/opentelemetry-collector-deployment.yaml
+ - helmChart: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/observability/jaeger.yaml
+ - helmChart: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/observability/opentelemetry-operator.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/observability/grafana-admin-credentials.yaml
+ - helmChart: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/observability/grafana.yaml
+ - helmChart: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/observability/grafana-tempo.yaml
+ - helmChart: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/observability/grafana-loki.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/observability/opentelemetry-collector-sidecar.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/observability/opentelemetry-collector-deployment.yaml
parameters:
- name: grafanaAdminPassword
description: Password of the Grafana admin user
default: adminadmin
airflow:
description: Stack containing Airflow scheduling platform
- stackableRelease: dev
+ stackableRelease: 25.3
stackableOperators:
- commons
- listener
@@ -95,9 +95,9 @@ stacks:
labels:
- airflow
manifests:
- - helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/postgresql-airflow.yaml
- - helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/redis-airflow.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/airflow/airflow.yaml
+ - helmChart: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/_templates/postgresql-airflow.yaml
+ - helmChart: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/_templates/redis-airflow.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/airflow/airflow.yaml
supportedNamespaces: []
resourceRequests:
cpu: 3400m
@@ -112,7 +112,7 @@ stacks:
default: airflowSecretKey
data-lakehouse-iceberg-trino-spark:
description: Data lakehouse using Iceberg lakehouse on S3, Trino as query engine, Spark for streaming ingest and Superset for data visualization
- stackableRelease: dev
+ stackableRelease: 25.3
stackableOperators:
- commons
- listener
@@ -135,17 +135,17 @@ stacks:
- minio
- s3
manifests:
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/minio-distributed-tls/rendered-chart.yaml
- - helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/postgresql-hive.yaml
- - helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/postgresql-hive-iceberg.yaml
- - helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/postgresql-superset.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/data-lakehouse-iceberg-trino-spark/s3-connection.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/data-lakehouse-iceberg-trino-spark/hive-metastores.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/data-lakehouse-iceberg-trino-spark/trino.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/data-lakehouse-iceberg-trino-spark/zookeeper.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/data-lakehouse-iceberg-trino-spark/kafka.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/data-lakehouse-iceberg-trino-spark/nifi.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/nifi-kafka-druid-superset-s3/superset.yaml # Reuse
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/_templates/minio-distributed-tls/rendered-chart.yaml
+ - helmChart: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/_templates/postgresql-hive.yaml
+ - helmChart: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/_templates/postgresql-hive-iceberg.yaml
+ - helmChart: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/_templates/postgresql-superset.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/data-lakehouse-iceberg-trino-spark/s3-connection.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/data-lakehouse-iceberg-trino-spark/hive-metastores.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/data-lakehouse-iceberg-trino-spark/trino.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/data-lakehouse-iceberg-trino-spark/zookeeper.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/data-lakehouse-iceberg-trino-spark/kafka.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/data-lakehouse-iceberg-trino-spark/nifi.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/nifi-kafka-druid-superset-s3/superset.yaml # Reuse
supportedNamespaces: ["default"] # until namespace can be templates, the minio FQDN hard-codes the namespace
resourceRequests:
cpu: "71"
@@ -169,7 +169,7 @@ stacks:
default: supersetSecretKey
hdfs-hbase:
description: HBase cluster using HDFS as underlying storage
- stackableRelease: dev
+ stackableRelease: 25.3
stackableOperators:
- commons
- listener
@@ -181,9 +181,9 @@ stacks:
- hbase
- hdfs
manifests:
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/hdfs-hbase/zookeeper.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/hdfs-hbase/hdfs.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/hdfs-hbase/hbase.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/hdfs-hbase/zookeeper.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/hdfs-hbase/hdfs.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/hdfs-hbase/hbase.yaml
supportedNamespaces: []
resourceRequests:
cpu: 4200m
@@ -192,7 +192,7 @@ stacks:
parameters: []
nifi-kafka-druid-superset-s3:
description: Stack containing NiFi, Kafka, Druid, MinIO and Superset for data visualization
- stackableRelease: dev
+ stackableRelease: 25.3
stackableOperators:
- commons
- listener
@@ -210,14 +210,14 @@ stacks:
- minio
- s3
manifests:
- - helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/minio.yaml
- - helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/postgresql-druid.yaml
- - helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/postgresql-superset.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/nifi-kafka-druid-superset-s3/zookeeper.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/nifi-kafka-druid-superset-s3/kafka.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/nifi-kafka-druid-superset-s3/druid.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/nifi-kafka-druid-superset-s3/superset.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/nifi-kafka-druid-superset-s3/nifi.yaml
+ - helmChart: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/_templates/minio.yaml
+ - helmChart: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/_templates/postgresql-druid.yaml
+ - helmChart: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/_templates/postgresql-superset.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/nifi-kafka-druid-superset-s3/zookeeper.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/nifi-kafka-druid-superset-s3/kafka.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/nifi-kafka-druid-superset-s3/druid.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/nifi-kafka-druid-superset-s3/superset.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/nifi-kafka-druid-superset-s3/nifi.yaml
supportedNamespaces: []
resourceRequests:
cpu: 8900m
@@ -238,7 +238,7 @@ stacks:
default: adminadmin
spark-trino-superset-s3:
description: Stack containing MinIO, Trino and Superset for data visualization
- stackableRelease: dev
+ stackableRelease: 25.3
stackableOperators:
- commons
- listener
@@ -254,15 +254,15 @@ stacks:
- minio
- s3
manifests:
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/minio-tls/rendered-chart.yaml
- - helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/postgresql-hive.yaml
- - helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/postgresql-hive-iceberg.yaml
- - helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/postgresql-superset.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/spark-trino-superset-s3/s3-connection.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/spark-trino-superset-s3/hive-metastore.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/trino-superset-s3/trino.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/spark-trino-superset-s3/trino-prediction-catalog.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/trino-superset-s3/superset.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/_templates/minio-tls/rendered-chart.yaml
+ - helmChart: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/_templates/postgresql-hive.yaml
+ - helmChart: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/_templates/postgresql-hive-iceberg.yaml
+ - helmChart: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/_templates/postgresql-superset.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/spark-trino-superset-s3/s3-connection.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/spark-trino-superset-s3/hive-metastore.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/trino-superset-s3/trino.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/spark-trino-superset-s3/trino-prediction-catalog.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/trino-superset-s3/superset.yaml
supportedNamespaces: []
resourceRequests:
cpu: 7100m
@@ -283,7 +283,7 @@ stacks:
default: supersetSecretKey
trino-superset-s3:
description: Stack containing MinIO, Trino and Superset for data visualization
- stackableRelease: dev
+ stackableRelease: 25.3
stackableOperators:
- commons
- listener
@@ -298,13 +298,13 @@ stacks:
- minio
- s3
manifests:
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/minio-tls/rendered-chart.yaml
- - helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/postgresql-hive.yaml
- - helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/postgresql-superset.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/trino-superset-s3/s3-connection.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/trino-superset-s3/hive-metastore.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/trino-superset-s3/trino.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/trino-superset-s3/superset.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/_templates/minio-tls/rendered-chart.yaml
+ - helmChart: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/_templates/postgresql-hive.yaml
+ - helmChart: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/_templates/postgresql-superset.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/trino-superset-s3/s3-connection.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/trino-superset-s3/hive-metastore.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/trino-superset-s3/trino.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/trino-superset-s3/superset.yaml
supportedNamespaces: [default] # until namespace can be templates, the minio FQDN hard-codes the namespace
resourceRequests:
cpu: 6800m
@@ -325,7 +325,7 @@ stacks:
default: supersetSecretKey
trino-iceberg:
description: Stack containing Trino using Apache Iceberg as a S3 data lakehouse
- stackableRelease: dev
+ stackableRelease: 25.3
stackableOperators:
- commons
- listener
@@ -340,11 +340,11 @@ stacks:
- minio
- s3
manifests:
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/minio-distributed-small-tls/rendered-chart.yaml
- - helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/postgresql-hive-iceberg.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/trino-iceberg/s3-connection.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/trino-iceberg/hive-metastores.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/trino-iceberg/trino.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/_templates/minio-distributed-small-tls/rendered-chart.yaml
+ - helmChart: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/_templates/postgresql-hive-iceberg.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/trino-iceberg/s3-connection.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/trino-iceberg/hive-metastores.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/trino-iceberg/trino.yaml
supportedNamespaces: ["default"] # until namespace can be templates, the minio FQDN hard-codes the namespace
resourceRequests:
cpu: 6000m # Measured 5600m
@@ -359,7 +359,7 @@ stacks:
default: adminadmin
jupyterhub-pyspark-hdfs:
description: Jupyterhub with PySpark and HDFS integration
- stackableRelease: dev
+ stackableRelease: 25.3
stackableOperators:
- commons
- listener
@@ -372,12 +372,12 @@ stacks:
- hdfs
- pyspark
manifests:
- - helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/jupyterhub.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/jupyterhub-pyspark-hdfs/zookeeper.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/jupyterhub-pyspark-hdfs/hdfs.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/jupyterhub-pyspark-hdfs/serviceaccount.yaml
+ - helmChart: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/_templates/jupyterhub.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/jupyterhub-pyspark-hdfs/zookeeper.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/jupyterhub-pyspark-hdfs/hdfs.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/jupyterhub-pyspark-hdfs/serviceaccount.yaml
# TODO Use patched JHub that created service for us from customer setup (ask Sebastian)
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/jupyterhub-pyspark-hdfs/spark_driver_service.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/jupyterhub-pyspark-hdfs/spark_driver_service.yaml
supportedNamespaces: []
resourceRequests:
cpu: 3350m
@@ -389,7 +389,7 @@ stacks:
default: adminadmin
dual-hive-hdfs-s3:
description: Dual stack Hive on HDFS and S3 for Hadoop/Hive to Trino migration
- stackableRelease: dev
+ stackableRelease: 25.3
stackableOperators:
- commons
- listener
@@ -405,12 +405,12 @@ stacks:
- hdfs
- s3
manifests:
- - helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/postgresql-hivehdfs.yaml
- - helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/postgresql-hives3.yaml
- - helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/minio.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/dual-hive-hdfs-s3/hdfs.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/dual-hive-hdfs-s3/hive.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/dual-hive-hdfs-s3/trino.yaml
+ - helmChart: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/_templates/postgresql-hivehdfs.yaml
+ - helmChart: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/_templates/postgresql-hives3.yaml
+ - helmChart: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/_templates/minio.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/dual-hive-hdfs-s3/hdfs.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/dual-hive-hdfs-s3/hive.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/dual-hive-hdfs-s3/trino.yaml
supportedNamespaces: []
resourceRequests:
cpu: 7750m
@@ -426,7 +426,7 @@ stacks:
The bind user credentials are: ldapadmin:ldapadminpassword.
No AuthenticationClass is configured, The AuthenticationClass is created manually in the tutorial.
Use the 'openldap' Stack for an OpenLDAD with an AuthenticationClass already installed.
- stackableRelease: dev
+ stackableRelease: 25.3
stackableOperators:
- commons
- listener
@@ -435,7 +435,7 @@ stacks:
- authentication
- ldap
manifests:
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/authentication/openldap-tls.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/authentication/openldap-tls.yaml
supportedNamespaces: ["default"]
resourceRequests:
cpu: 1950m
@@ -449,7 +449,7 @@ stacks:
The bind user credentials are: ldapadmin:ldapadminpassword.
The LDAP AuthenticationClass is called 'ldap' and the SecretClass for the bind credentials is called 'ldap-bind-credentials'.
The stack already creates an appropriate Secret, so referring to the 'ldap' AuthenticationClass in your ProductCluster should be enough.
- stackableRelease: dev
+ stackableRelease: 25.3
stackableOperators:
- commons
- listener
@@ -458,8 +458,8 @@ stacks:
- authentication
- ldap
manifests:
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/authentication/openldap-tls.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/authentication/openldap-tls-authenticationclass.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/authentication/openldap-tls.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/authentication/openldap-tls-authenticationclass.yaml
supportedNamespaces: []
resourceRequests:
cpu: 1950m
@@ -475,7 +475,7 @@ stacks:
3 users are created in Keycloak: admin:adminadmin, alice:alicealice, bob:bobbob. admin and alice are admins with
full authorization in Druid and Trino, bob is not authorized.
This is a proof-of-concept and the mechanisms used here are subject to change.
- stackableRelease: dev
+ stackableRelease: 25.3
stackableOperators:
- commons
- listener
@@ -490,18 +490,18 @@ stacks:
- authentication
- sso
manifests:
- - helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/postgresql-superset.yaml
- - helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/postgresql-druid.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/keycloak-opa-poc/serviceaccount.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/keycloak-opa-poc/keycloak.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/keycloak-opa-poc/setup-keycloak.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/keycloak-opa-poc/opa.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/keycloak-opa-poc/policies.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/keycloak-opa-poc/zookeeper.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/keycloak-opa-poc/hdfs.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/keycloak-opa-poc/druid.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/keycloak-opa-poc/trino.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/keycloak-opa-poc/superset.yaml
+ - helmChart: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/_templates/postgresql-superset.yaml
+ - helmChart: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/_templates/postgresql-druid.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/keycloak-opa-poc/serviceaccount.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/keycloak-opa-poc/keycloak.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/keycloak-opa-poc/setup-keycloak.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/keycloak-opa-poc/opa.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/keycloak-opa-poc/policies.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/keycloak-opa-poc/zookeeper.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/keycloak-opa-poc/hdfs.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/keycloak-opa-poc/druid.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/keycloak-opa-poc/trino.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/keycloak-opa-poc/superset.yaml
supportedNamespaces: ["default"] # ClusterRoleBinding needs explicit namespace
resourceRequests:
cpu: 7850m
@@ -541,7 +541,7 @@ stacks:
Note that this stack is tightly coupled with the demo.
So if you install the stack you will get demo-specific parts (such as Keycloak users or regorules).
- stackableRelease: dev
+ stackableRelease: 25.3
stackableOperators:
- commons
- listener
@@ -567,24 +567,24 @@ stacks:
memory: 19586Mi
pvc: 40Gi
manifests:
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/keycloak-serviceaccount.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/end-to-end-security/keycloak-realm-config.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/keycloak.yaml
- - helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/postgresql-hive-iceberg.yaml
- - helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/postgresql-superset.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/end-to-end-security/krb5.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/end-to-end-security/kerberos-secretclass.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/end-to-end-security/opa.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/end-to-end-security/zookeeper.yaml # TODO: Add authentication
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/end-to-end-security/hdfs.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/end-to-end-security/hdfs-regorules.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/end-to-end-security/hive-metastore.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/end-to-end-security/trino.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/end-to-end-security/trino-regorules.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/end-to-end-security/trino-policies.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/end-to-end-security/rbac.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/end-to-end-security/setup-postgresql.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/end-to-end-security/superset.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/_templates/keycloak-serviceaccount.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/end-to-end-security/keycloak-realm-config.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/_templates/keycloak.yaml
+ - helmChart: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/_templates/postgresql-hive-iceberg.yaml
+ - helmChart: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/_templates/postgresql-superset.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/end-to-end-security/krb5.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/end-to-end-security/kerberos-secretclass.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/end-to-end-security/opa.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/end-to-end-security/zookeeper.yaml # TODO: Add authentication
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/end-to-end-security/hdfs.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/end-to-end-security/hdfs-regorules.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/end-to-end-security/hive-metastore.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/end-to-end-security/trino.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/end-to-end-security/trino-regorules.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/end-to-end-security/trino-policies.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/end-to-end-security/rbac.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/end-to-end-security/setup-postgresql.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/end-to-end-security/superset.yaml
parameters:
- name: keycloakAdminPassword
description: Password of the Keycloak admin user
@@ -613,7 +613,7 @@ stacks:
signal-processing:
description: >-
A stack used for creating, streaming and processing in-flight data and persisting it to TimescaleDB before it is displayed in Grafana
- stackableRelease: dev
+ stackableRelease: 25.3
stackableOperators:
- commons
- listener
@@ -626,15 +626,15 @@ stacks:
- jupyterhub
- grafana
manifests:
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/signal-processing/secrets.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/signal-processing/grafana-dashboards.yaml
- - helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/signal-processing/grafana.yaml
- - helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/postgresql-timescaledb.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/nifi-kafka-druid-superset-s3/zookeeper.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/signal-processing/nifi.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/jupyterhub-pyspark-hdfs/serviceaccount.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/jupyterhub-pyspark-hdfs/spark_driver_service.yaml
- - helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/signal-processing/jupyterhub.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/signal-processing/secrets.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/signal-processing/grafana-dashboards.yaml
+ - helmChart: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/signal-processing/grafana.yaml
+ - helmChart: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/_templates/postgresql-timescaledb.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/nifi-kafka-druid-superset-s3/zookeeper.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/signal-processing/nifi.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/jupyterhub-pyspark-hdfs/serviceaccount.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/jupyterhub-pyspark-hdfs/spark_driver_service.yaml
+ - helmChart: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/signal-processing/jupyterhub.yaml
parameters:
- name: nifiAdminPassword
description: Password of the NiFI admin user
@@ -651,7 +651,7 @@ stacks:
jupyterhub-keycloak:
description: >-
A stack used to demonstrate JupyterHub and Keycloak, using parts of the end-to-end security demo.
- stackableRelease: dev
+ stackableRelease: 25.3
stackableOperators:
- commons
- listener
@@ -666,13 +666,13 @@ stacks:
memory: 9010Mi
pvc: 24Gi
manifests:
- - helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/minio.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/jupyterhub-keycloak/keycloak-serviceaccount.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/jupyterhub-keycloak/keycloak-realm-config.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/jupyterhub-keycloak/keycloak.yaml
- - helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/jupyterhub-keycloak/jupyterhub.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/jupyterhub-keycloak/serviceaccount.yaml
- - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/jupyterhub-keycloak/s3-connection.yaml
+ - helmChart: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/_templates/minio.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/jupyterhub-keycloak/keycloak-serviceaccount.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/jupyterhub-keycloak/keycloak-realm-config.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/jupyterhub-keycloak/keycloak.yaml
+ - helmChart: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/jupyterhub-keycloak/jupyterhub.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/jupyterhub-keycloak/serviceaccount.yaml
+ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-25.3/stacks/jupyterhub-keycloak/s3-connection.yaml
parameters:
- name: keycloakAdminPassword
description: Password of the Keycloak admin user