A honcharenko/multiarch build 801 (airbytehq#12570)

* added multiarch image publishing/modified dockerfiles

* added new ami

* changed version to test

* rollback version

* check version test

* env vars temp fix

* apt-utils error fix

* disabled failed test

* remove excluded tests

* Excluded :airbyte-db:lib:test

* Excluded :airbyte-db:lib:test

* static jdk version for test

* ok test

* ok test

* ok test

* test vars

* quemu issue fix

* Returned version vars

* Scripts update

* Version vars change

* Comment fix

* Added comments, minor changes and comments in dockerfiles

* Uncomment line to push images

.github/actions/start-aws-runner/action.yml

@@ -9,7 +9,7 @@ inputs:
     required: true
   ec2-image-id:
     # github-self-hosted-runner-ubuntu-20-100g-disk-with-cypress-deps
-    default: "ami-08927c058921b27f4"
+    default: "ami-0f23be2f917510c26"
     required: true
   ec2-instance-type:
     default: "c5.2xlarge"

airbyte-container-orchestrator/Dockerfile

@@ -12,18 +12,19 @@ RUN apt-get update && apt-get install -y \
                           gnupg-agent \
                           software-properties-common
 RUN curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add -
-RUN add-apt-repository \
-       "deb [arch=${DOCKER_BUILD_ARCH}] https://download.docker.com/linux/debian \
-       $(lsb_release -cs) \
-       stable"
+# arch var used to detect architecture of container. Architecture should be spcified to get proper binaries from repo.
+RUN arch=$(dpkg --print-architecture) && \
+       add-apt-repository \
+       "deb [arch=${arch}] https://download.docker.com/linux/debian \
+       $(lsb_release -cs) stable"
 RUN apt-get update && apt-get install -y docker-ce-cli jq
 
 # Install kubectl for copying files to kube pods. Eventually should be replaced with a kube java client.
 # See https://github.com/airbytehq/airbyte/issues/8643 for more information on why we are using kubectl for copying.
 # The following commands were taken from https://kubernetes.io/docs/tasks/tools/install-kubectl-linux/#install-using-native-package-management
 RUN curl -fsSLo /usr/share/keyrings/kubernetes-archive-keyring.gpg https://packages.cloud.google.com/apt/doc/apt-key.gpg
 RUN echo "deb [signed-by=/usr/share/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" | tee /etc/apt/sources.list.d/kubernetes.list
-RUN apt-get update && apt-get install -y kubectl
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y kubectl
 
 # Don't change this manually.  Bump version expects to make moves based on this string
 ARG VERSION=0.38.1-alpha

airbyte-workers/Dockerfile

@@ -12,10 +12,11 @@ RUN apt-get update && apt-get install -y \
                           gnupg-agent \
                           software-properties-common
 RUN curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add -
-RUN add-apt-repository \
-       "deb [arch=${DOCKER_BUILD_ARCH}] https://download.docker.com/linux/debian \
-       $(lsb_release -cs) \
-       stable"
+# arch var used to detect architecture of container. Architecture should be spcified to get proper binaries from repo.
+RUN arch=$(dpkg --print-architecture) && \
+       add-apt-repository \
+       "deb [arch=${arch}] https://download.docker.com/linux/debian \
+       $(lsb_release -cs) stable"
 RUN apt-get update && apt-get install -y docker-ce-cli jq
 
 # Install kubectl for copying files to kube pods. Eventually should be replaced with a kube java client.

tools/bin/publish_docker.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+set -e
+
+# List of directories without "airbyte-" prefix.
+projectDir=( 
+  "workers"
+  "cli"
+  "webapp"
+  "server"
+  "temporal"
+  "container-orchestrator"
+  "config/init"
+  "bootloader"
+  "metrics/reporter"
+  "db/lib"
+  "scheduler/app"
+)
+
+# Set default values to required vars. If set in env, values will be taken from there.
+JDK_VERSION=${JDK_VERSION:-17.0.1}
+ALPINE_IMAGE=${ALPINE_IMAGE:-alpine:3.14}
+POSTGRES_IMAGE=${POSTGRES_IMAGE:-postgres:13-alpine}
+
+# Iterate over all directories in list to build one by one.
+# metrics-reporter are exception due to wrong artifact naming 
+for workdir in "${projectDir[@]}"
+  do
+    if [ $workdir = "metrics/reporter" ]; then
+      artifactName="metrics-reporter" 
+    else
+      artifactName=${workdir%/*}
+    fi
+    docker buildx create --use --name $artifactName &&      \
+    docker buildx build -t "airbyte/$artifactName:$VERSION" \
+      --platform linux/amd64,linux/arm64                    \
+      --build-arg VERSION=$VERSION                          \
+      --build-arg ALPINE_IMAGE=$ALPINE_IMAGE                \
+      --build-arg POSTGRES_IMAGE=$POSTGRES_IMAGE            \
+      --build-arg JDK_VERSION=$JDK_VERSION                  \
+      --push                                                \
+      airbyte-$workdir/build/docker
+    docker buildx rm $artifactName
+done

tools/bin/release_version.sh

@@ -26,5 +26,9 @@ source ./tools/bin/bump_version.sh
 echo "Building and publishing PLATFORM version $NEW_VERSION for git revision $GIT_REVISION..."
 VERSION=$NEW_VERSION SUB_BUILD=PLATFORM ./gradlew clean build
 VERSION=$NEW_VERSION SUB_BUILD=PLATFORM ./gradlew publish
-VERSION=$NEW_VERSION GIT_REVISION=$GIT_REVISION docker-compose -f docker-compose.build.yaml push
+
+# Container should be running before build starts
+# It generates binaries to build images for different CPU architecture
+docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+VERSION=$NEW_VERSION ./tools/bin/publish_docker.sh
 echo "Completed building and publishing PLATFORM..."