implementation to run kube acceptance tests on GKE (airbytehq#4961)

* change acceptance tests to use GKE resources * make CI changes + introduce script * fix path * fix indentation * fix indentation * set home directory * use another action * fix sed command * setup kustomize * install kubectl * clean up * address review comments * remove comments * add master and schedule check * use correct if condition * fix if condition * update script and re-test * format * fix compilation error * add always clause * move gke kube test to a separate file * remove un-necessary line
beakalyk · Jul 27, 2021 · 13bc08c · 13bc08c
1 parent 9de5368
commit 13bc08c
Show file tree

Hide file tree

Showing 6 changed files with 474 additions and 64 deletions.
diff --git a/.github/workflows/gke-kube-test-command.yml b/.github/workflows/gke-kube-test-command.yml
@@ -0,0 +1,112 @@
+name: GKE Kube Acceptance Test
+on:
+  schedule:
+    - cron: '0 */6 * * *'
+
+jobs:
+  start-gke-kube-acceptance-test-runner:
+    name: Start GKE Kube Acceptance Test EC2 Runner
+    runs-on: ubuntu-latest
+    outputs:
+      label: ${{ steps.start-ec2-runner.outputs.label }}
+      ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
+    steps:
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-east-2
+      - name: Start EC2 runner
+        id: start-ec2-runner
+        uses: machulav/[email protected]
+        with:
+          mode: start
+          github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }}
+          ec2-image-id: ami-04bd6e81239f4f3fb
+          ec2-instance-type: c5.2xlarge
+          subnet-id: subnet-0469a9e68a379c1d3
+          security-group-id: sg-0793f3c9413f21970
+  gke-kube-acceptance-test:
+    # In case of self-hosted EC2 errors, removed the `needs` line and switch back to running on ubuntu-latest.
+    needs: start-gke-kube-acceptance-test-runner # required to start the main job when the runner is ready
+    runs-on: ${{ needs.start-gke-kube-acceptance-test-runner.outputs.label }} # run the job on the newly created runner
+    name: GKE Acceptance Tests (Kube)
+    steps:
+      - name: Checkout Airbyte
+        uses: actions/checkout@v2
+
+      - uses: actions/setup-java@v1
+        with:
+          java-version: '14'
+
+      - uses: actions/setup-node@v1
+        with:
+          node-version: '14.7'
+
+      - uses: actions/setup-python@v2
+        with:
+          python-version: '3.7'
+
+      - name: Install Pyenv
+        run: python3 -m pip install virtualenv==16.7.9 --user
+
+      - name: Fix EC-2 Runner
+        run: |
+          mkdir -p /home/runner
+
+      - name: Install socat (required for port forwarding)
+        run: |
+          sudo apt-get update
+          sudo apt-get install socat
+
+      - name: Set up Cloud SDK
+        uses: google-github-actions/setup-gcloud@master
+        with:
+          project_id: ${{ secrets.GKE_TEST_PROJECT_ID }}
+          service_account_key: ${{ secrets.GKE_TEST_SA_KEY }}
+          export_default_credentials: true
+
+      - name: Get GKE credentials
+        uses: google-github-actions/get-gke-credentials@main
+        with:
+          project_id: ${{ secrets.GKE_TEST_PROJECT_ID }}
+          credentials: ${{ secrets.GKE_TEST_SA_KEY }}
+          cluster_name: kube-acceptance-test-cluster
+          location: us-central1-c
+
+      - name: Install kubectl
+        run: |-
+          curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
+          sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
+
+      - name: Run End-to-End Acceptance Tests on GKE
+        env:
+          USER: root
+          HOME: /home/runner
+          DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+        run: |
+          ./tools/bin/gke-kube-acceptance-test/acceptance_test_kube_gke.sh
+  # In case of self-hosted EC2 errors, remove this block.
+  stop-gke-kube-acceptance-test-runner:
+    name: Stop GKE Kube Acceptance Test EC2 Runner
+    needs:
+      - start-gke-kube-acceptance-test-runner # required to get output from the start-runner job
+      - gke-kube-acceptance-test # required to wait when the main job is done
+    runs-on: ubuntu-latest
+    if: ${{ always() }}
+    steps:
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-east-2
+      - name: Stop EC2 runner
+        uses: machulav/[email protected]
+        with:
+          mode: stop
+          github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }}
+          label: ${{ needs.start-gke-kube-acceptance-test-runner.outputs.label }}
+          ec2-instance-id: ${{ needs.start-gke-kube-acceptance-test-runner.outputs.ec2-instance-id }}
+