full scheduler v2 CI testing (airbytehq#10199)

* fix normalization output processing in container orchestrator

* add full scheduler v2 acceptance tests

* speed up tests

* fixes

* clean up

.github/workflows/gradle.yml

@@ -564,7 +564,7 @@ jobs:
           HOME: /home/runner
 
       - name: Build Platform Docker Images
-        run: SUB_BUILD=PLATFORM ./gradlew assemble --scan
+        run: SUB_BUILD=PLATFORM ./gradlew assemble -x test --scan
 
       - name: Run Logging Tests
         run: ./tools/bin/cloud_storage_logging_test.sh
@@ -626,3 +626,135 @@ jobs:
           github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }}
           label: ${{ needs.start-kube-acceptance-test-runner.outputs.label }}
           ec2-instance-id: ${{ needs.start-kube-acceptance-test-runner.outputs.ec2-instance-id }}
+
+  ## Kube Acceptance Tests (with scheduler v2 - both temporal changes and container orchestrator)
+  # In case of self-hosted EC2 errors, remove this block.
+  # Docker acceptance tests run as part of the build job.
+  start-kube-acceptance-test-runner-v2:
+    name: Start Kube Acceptance Test EC2 Runner
+    timeout-minutes: 10
+    runs-on: ubuntu-latest
+    outputs:
+      label: ${{ steps.start-ec2-runner.outputs.label }}
+      ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
+    steps:
+      - name: Checkout Airbyte
+        uses: actions/checkout@v2
+      - name: Start AWS Runner
+        id: start-ec2-runner
+        uses: ./.github/actions/start-aws-runner
+        with:
+          # github-self-hosted-runner-ubuntu-20-with-120gdisk-docker-20.10.7
+          ec2-image-id: ami-0d4083c04fde515c4
+          aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
+          github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }}
+  kube-acceptance-test-v2:
+    # In case of self-hosted EC2 errors, removed the `needs` line and switch back to running on ubuntu-latest.
+    needs: start-kube-acceptance-test-runner-v2 # required to start the main job when the runner is ready
+    runs-on: ${{ needs.start-kube-acceptance-test-runner-v2.outputs.label }} # run the job on the newly created runner
+    environment: more-secrets
+    name: Acceptance Tests (Kube v2)
+    timeout-minutes: 90
+    steps:
+      - name: Checkout Airbyte
+        uses: actions/checkout@v2
+
+      - uses: actions/setup-java@v1
+        with:
+          java-version: "17"
+
+      - uses: actions/setup-node@v1
+        with:
+          node-version: "16.13.0"
+
+      - name: Fix EC-2 Runner
+        run: |
+          mkdir -p /home/runner
+
+      - name: Set up CI Gradle Properties
+        run: |
+          mkdir -p ~/.gradle/
+          cat > ~/.gradle/gradle.properties <<EOF
+          org.gradle.jvmargs=-Xmx8g -Xss4m --add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \
+            --add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED \
+            --add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED \
+            --add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED \
+            --add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
+          org.gradle.workers.max=8
+          org.gradle.vfs.watch=false
+          EOF
+
+      - name: Install socat (required for port forwarding)
+        run: |
+          # wait to receive lock (see https://askubuntu.com/questions/132059/how-to-make-a-package-manager-wait-if-another-instance-of-apt-is-running)
+          while sudo fuser /var/{lib/{dpkg,apt/lists},cache/apt/archives}/lock >/dev/null 2>&1; do
+             sleep 1
+          done
+
+          sudo apt-get update
+          sudo apt-get install socat
+
+      - name: Create cluster config file
+        run: |
+          cat > /tmp/kind-config.yaml <<EOF
+          kind: Cluster
+          apiVersion: kind.x-k8s.io/v1alpha4
+          nodes:
+          - role: control-plane
+          - role: worker
+          EOF
+
+      - name: KIND Kubernetes Cluster Setup
+        uses: helm/[email protected]
+        with:
+          node_image: kindest/node:v1.21.2
+          config: /tmp/kind-config.yaml
+        # In case of self-hosted EC2 errors, remove this env block.
+        env:
+          USER: root
+          HOME: /home/runner
+          CHANGE_MINIKUBE_NONE_USER: true
+
+      - name: Build Platform Docker Images
+        run: SUB_BUILD=PLATFORM ./gradlew assemble -x test --scan
+
+      - name: Run Kubernetes End-to-End Acceptance Tests
+        env:
+          USER: root
+          HOME: /home/runner
+          AWS_S3_INTEGRATION_TEST_CREDS: ${{ secrets.AWS_S3_INTEGRATION_TEST_CREDS }}
+          SECRET_STORE_GCP_CREDENTIALS: ${{ secrets.SECRET_STORE_GCP_CREDENTIALS }}
+          SECRET_STORE_GCP_PROJECT_ID: ${{ secrets.SECRET_STORE_GCP_PROJECT_ID }}
+        run: |
+          CI=true IS_MINIKUBE=true ./tools/bin/acceptance_test_kube.sh
+
+      - uses: actions/upload-artifact@v2
+        if: failure()
+        with:
+          name: Kubernetes Logs
+          path: /tmp/kubernetes_logs/*
+
+  # In case of self-hosted EC2 errors, remove this block.
+  stop-kube-acceptance-test-runner-v2:
+    name: Stop Kube Acceptance Test EC2 Runner
+    timeout-minutes: 10
+    needs:
+      - start-kube-acceptance-test-runner-v2 # required to get output from the start-runner job
+      - kube-acceptance-test-v2 # required to wait when the main job is done
+    runs-on: ubuntu-latest
+    if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs
+    steps:
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-east-2
+      - name: Stop EC2 runner
+        uses: machulav/[email protected]
+        with:
+          mode: stop
+          github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }}
+          label: ${{ needs.start-kube-acceptance-test-runner-v2.outputs.label }}
+          ec2-instance-id: ${{ needs.start-kube-acceptance-test-runner-v2.outputs.ec2-instance-id }}

airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/AcceptanceTests.java

@@ -721,8 +721,6 @@ public void testIncrementalDedupeSync() throws Exception {
 
   @Test
   @Order(14)
-  @DisabledIfEnvironmentVariable(named = "KUBE",
-                                 matches = "true")
   public void testCheckpointing() throws Exception {
     final SourceDefinitionRead sourceDefinition = apiClient.getSourceDefinitionApi().createSourceDefinition(new SourceDefinitionCreate()
         .name("E2E Test Source")

kube/overlays/dev-integration-test-schedulerv2/.env

@@ -0,0 +1,70 @@
+AIRBYTE_VERSION=dev
+
+# Airbyte Internal Database, see https://docs.airbyte.io/operator-guides/configuring-airbyte-db
+DATABASE_HOST=airbyte-db-svc
+DATABASE_PORT=5432
+DATABASE_DB=airbyte
+# translate manually DATABASE_URL=jdbc:postgresql://${DATABASE_HOST}:${DATABASE_PORT/${DATABASE_DB}
+DATABASE_URL=jdbc:postgresql://airbyte-db-svc:5432/airbyte
+JOBS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION=0.29.15.001
+CONFIGS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION=0.35.15.001
+
+# When using the airbyte-db via default docker image:
+CONFIG_ROOT=/configs
+DATA_DOCKER_MOUNT=airbyte_data
+DB_DOCKER_MOUNT=airbyte_db
+
+# Temporal.io worker configuration
+TEMPORAL_HOST=airbyte-temporal-svc:7233
+TEMPORAL_WORKER_PORTS=9001,9002,9003,9004,9005,9006,9007,9008,9009,9010,9011,9012,9013,9014,9015,9016,9017,9018,9019,9020,9021,9022,9023,9024,9025,9026,9027,9028,9029,9030,9031,9032,9033,9034,9035,9036,9037,9038,9039,9040
+
+# Workspace storage for running jobs (logs, etc)
+WORKSPACE_ROOT=/workspace
+WORKSPACE_DOCKER_MOUNT=airbyte_workspace
+
+LOCAL_ROOT=/tmp/airbyte_local
+
+# Maximum total simultaneous jobs across all worker nodes
+SUBMITTER_NUM_THREADS=10
+
+# Miscellaneous
+TRACKING_STRATEGY=logging
+WEBAPP_URL=airbyte-webapp-svc:80
+API_URL=/api/v1/
+INTERNAL_API_HOST=airbyte-server-svc:8001
+
+WORKER_ENVIRONMENT=kubernetes
+FULLSTORY=disabled
+IS_DEMO=false
+LOG_LEVEL=INFO
+
+# S3/Minio Log Configuration
+S3_LOG_BUCKET=airbyte-dev-logs
+S3_LOG_BUCKET_REGION=
+S3_MINIO_ENDPOINT=http://airbyte-minio-svc:9000
+S3_PATH_STYLE_ACCESS=true
+
+# GCS Log Configuration
+GCS_LOG_BUCKET=
+
+# State Storage Configuration
+STATE_STORAGE_MINIO_BUCKET_NAME=airbyte-dev-logs
+STATE_STORAGE_MINIO_ENDPOINT=http://airbyte-minio-svc:9000
+
+# Docker Resource Limits
+JOB_MAIN_CONTAINER_CPU_REQUEST=
+JOB_MAIN_CONTAINER_CPU_LIMIT=
+JOB_MAIN_CONTAINER_MEMORY_REQUEST=
+JOB_MAIN_CONTAINER_MEMORY_LIMIT=
+
+# Worker pod tolerations and node selectors
+JOB_KUBE_TOLERATIONS=
+JOB_KUBE_NODE_SELECTORS=
+
+# Job image pull policy
+JOB_KUBE_MAIN_CONTAINER_IMAGE_PULL_POLICY=
+
+# Launch a separate pod to orchestrate sync steps
+NEW_SCHEDULER=true
+CONTAINER_ORCHESTRATOR_ENABLED=true
+

kube/overlays/dev-integration-test-schedulerv2/.secrets

@@ -0,0 +1,7 @@
+DATABASE_USER=docker
+DATABASE_PASSWORD=docker
+AWS_ACCESS_KEY_ID=minio
+AWS_SECRET_ACCESS_KEY=minio123
+GOOGLE_APPLICATION_CREDENTIALS=
+STATE_STORAGE_MINIO_ACCESS_KEY=minio
+STATE_STORAGE_MINIO_SECRET_ACCESS_KEY=minio123

kube/overlays/dev-integration-test-schedulerv2/bootloader-patch.yaml

@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: airbyte-bootloader
+spec:
+  containers:
+    - name: airbyte-bootloader-container
+      env:
+        - name: NEW_SCHEDULER
+          valueFrom:
+            configMapKeyRef:
+              name: airbyte-env
+              key: NEW_SCHEDULER

kube/overlays/dev-integration-test-schedulerv2/kustomization.yaml

@@ -0,0 +1,37 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namespace: default
+
+bases:
+  - ../../resources
+
+images:
+  - name: airbyte/db
+    newTag: dev
+  - name: airbyte/bootloader
+    newTag: dev
+  - name: airbyte/scheduler
+    newTag: dev
+  - name: airbyte/server
+    newTag: dev
+  - name: airbyte/webapp
+    newTag: dev
+  - name: airbyte/worker
+    newTag: dev
+  - name: temporalio/auto-setup
+    newTag: 1.7.0
+
+configMapGenerator:
+  - name: airbyte-env
+    env: .env
+
+secretGenerator:
+  - name: airbyte-secrets
+    env: .secrets
+
+patchesStrategicMerge:
+  - bootloader-patch.yaml
+  - server-patch.yaml
+  - scheduler-patch.yaml
+  - worker-patch.yaml

kube/overlays/dev-integration-test-schedulerv2/scheduler-patch.yaml

@@ -0,0 +1,15 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: airbyte-scheduler
+spec:
+  template:
+    spec:
+      containers:
+        - name: airbyte-scheduler-container
+          env:
+            - name: NEW_SCHEDULER
+              valueFrom:
+                configMapKeyRef:
+                  name: airbyte-env
+                  key: NEW_SCHEDULER

kube/overlays/dev-integration-test-schedulerv2/server-patch.yaml

@@ -0,0 +1,15 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: airbyte-server
+spec:
+  template:
+    spec:
+      containers:
+        - name: airbyte-server-container
+          env:
+            - name: NEW_SCHEDULER
+              valueFrom:
+                configMapKeyRef:
+                  name: airbyte-env
+                  key: NEW_SCHEDULER

kube/overlays/dev-integration-test-schedulerv2/worker-patch.yaml

@@ -0,0 +1,15 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: airbyte-worker
+spec:
+  template:
+    spec:
+      containers:
+        - name: airbyte-worker-container
+          env:
+            - name: CONTAINER_ORCHESTRATOR_ENABLED
+              valueFrom:
+                configMapKeyRef:
+                  name: airbyte-env
+                  key: CONTAINER_ORCHESTRATOR_ENABLED

tools/bin/acceptance_test_kube.sh

@@ -8,15 +8,17 @@ assert_root
 
 # Since KIND does not have access to the local docker agent, manually load the minimum images required for the Kubernetes Acceptance Tests.
 # See https://kind.sigs.k8s.io/docs/user/quick-start/#loading-an-image-into-your-cluster.
-echo "Loading images into KIND..."
-kind load docker-image airbyte/server:dev --name chart-testing &
-kind load docker-image airbyte/scheduler:dev --name chart-testing &
-kind load docker-image airbyte/webapp:dev --name chart-testing &
-kind load docker-image airbyte/worker:dev --name chart-testing &
-kind load docker-image airbyte/db:dev --name chart-testing &
-kind load docker-image airbyte/container-orchestrator:dev --name chart-testing &
-kind load docker-image airbyte/bootloader:dev --name chart-testing &
-wait
+if [ -n "$CI" ]; then
+  echo "Loading images into KIND..."
+  kind load docker-image airbyte/server:dev --name chart-testing &
+  kind load docker-image airbyte/scheduler:dev --name chart-testing &
+  kind load docker-image airbyte/webapp:dev --name chart-testing &
+  kind load docker-image airbyte/worker:dev --name chart-testing &
+  kind load docker-image airbyte/db:dev --name chart-testing &
+  kind load docker-image airbyte/container-orchestrator:dev --name chart-testing &
+  kind load docker-image airbyte/bootloader:dev --name chart-testing &
+  wait
+fi
 
 echo "Starting app..."