[ci] Support docker build args in buildImage steps (#13612)

Curious for your thoughts on this. I think this will make it easier to build `hailgenetics/hail` images for different python versions without duplicating Dockerfiles with the nice side-effect of not needing to jinja our Dockerfiles.
hail-is · Sep 16, 2023 · 96e5ae4 · 96e5ae4
1 parent d0c8ef5
commit 96e5ae4
Show file tree

Hide file tree

Showing 28 changed files with 141 additions and 60 deletions.
diff --git a/Makefile b/Makefile
@@ -142,21 +142,21 @@ $(HAILTOP_VERSION):
 
 hail-ubuntu-image: $(shell git ls-files docker/hail-ubuntu)
 	$(eval HAIL_UBUNTU_IMAGE := $(DOCKER_PREFIX)/hail-ubuntu:$(TOKEN))
-	python3 ci/jinja2_render.py '{"global":{"docker_prefix":"$(DOCKER_PREFIX)"}}' docker/hail-ubuntu/Dockerfile docker/hail-ubuntu/Dockerfile.out
-	./docker-build.sh docker/hail-ubuntu Dockerfile.out $(HAIL_UBUNTU_IMAGE)
+	DOCKER_BUILD_ARGS='--build-arg DOCKER_PREFIX=$(DOCKER_PREFIX)' \
+		./docker-build.sh docker/hail-ubuntu Dockerfile $(HAIL_UBUNTU_IMAGE)
 	echo $(HAIL_UBUNTU_IMAGE) > $@
 
 base-image: hail-ubuntu-image docker/Dockerfile.base
 	$(eval BASE_IMAGE := $(DOCKER_PREFIX)/base:$(TOKEN))
-	python3 ci/jinja2_render.py '{"hail_ubuntu_image":{"image":"'$$(cat hail-ubuntu-image)'"}}' docker/Dockerfile.base docker/Dockerfile.base.out
-	./docker-build.sh . docker/Dockerfile.base.out $(BASE_IMAGE)
+	DOCKER_BUILD_ARGS='--build-arg BASE_IMAGE='$$(cat hail-ubuntu-image) \
+		./docker-build.sh . docker/Dockerfile.base $(BASE_IMAGE)
 	echo $(BASE_IMAGE) > $@
 
 hail-run-image: base-image hail/Dockerfile.hail-run hail/python/pinned-requirements.txt hail/python/dev/pinned-requirements.txt docker/core-site.xml
 	$(eval BASE_IMAGE := $(DOCKER_PREFIX)/hail-run:$(TOKEN))
 	$(MAKE) -C hail wheel
-	python3 ci/jinja2_render.py '{"base_image":{"image":"'$$(cat base-image)'"}}' hail/Dockerfile.hail-run hail/Dockerfile.hail-run.out
-	./docker-build.sh . hail/Dockerfile.hail-run.out $(BASE_IMAGE)
+	DOCKER_BUILD_ARGS='--build-arg BASE_IMAGE='$$(cat base-image) \
+		./docker-build.sh . hail/Dockerfile.hail-run $(BASE_IMAGE)
 	echo $(BASE_IMAGE) > $@
 
 private-repo-hailgenetics-hail-image: hail-ubuntu-image docker/hailgenetics/hail/Dockerfile $(shell git ls-files hail/src/main hail/python)
@@ -165,8 +165,8 @@ private-repo-hailgenetics-hail-image: hail-ubuntu-image docker/hailgenetics/hail
 	tar -cvf wheel-container.tar \
 		-C hail/build/deploy/dist \
 		hail-$$(cat hail/python/hail/hail_pip_version)-py3-none-any.whl
-	python3 ci/jinja2_render.py '{"hail_ubuntu_image":{"image":"'$$(cat hail-ubuntu-image)'"}}' docker/hailgenetics/hail/Dockerfile docker/hailgenetics/hail/Dockerfile.out
-	./docker-build.sh . docker/hailgenetics/hail/Dockerfile.out $(PRIVATE_REPO_HAILGENETICS_HAIL_IMAGE)
+	DOCKER_BUILD_ARGS='--build-arg BASE_IMAGE='$$(cat hail-ubuntu-image) \
+		./docker-build.sh . docker/hailgenetics/hail/Dockerfile $(PRIVATE_REPO_HAILGENETICS_HAIL_IMAGE)
 	rm wheel-container.tar
 	echo $(PRIVATE_REPO_HAILGENETICS_HAIL_IMAGE) > $@
 
@@ -183,20 +183,20 @@ website-image: docs
 
 $(SERVICES_IMAGES): %-image: $(SERVICES_IMAGE_DEPS) $(shell git ls-files $$*)
 	$(eval IMAGE := $(DOCKER_PREFIX)/$*:$(TOKEN))
-	python3 ci/jinja2_render.py '{"hail_ubuntu_image":{"image":"'$$(cat hail-ubuntu-image)'"}}' $*/Dockerfile $*/Dockerfile.out
-	./docker-build.sh . $*/Dockerfile.out $(IMAGE)
+	DOCKER_BUILD_ARGS='--build-arg BASE_IMAGE='$$(cat hail-ubuntu-image) \
+		./docker-build.sh . $*/Dockerfile $(IMAGE)
 	echo $(IMAGE) > $@
 
 ci-utils-image: base-image $(SERVICES_IMAGE_DEPS) $(shell git ls-files ci)
 	$(eval CI_UTILS_IMAGE := $(DOCKER_PREFIX)/ci-utils:$(TOKEN))
-	python3 ci/jinja2_render.py '{"base_image":{"image":"'$$(cat base-image)'"}}' ci/Dockerfile.ci-utils ci/Dockerfile.ci-utils.out
-	./docker-build.sh . ci/Dockerfile.ci-utils.out $(CI_UTILS_IMAGE)
+	DOCKER_BUILD_ARGS='--build-arg BASE_IMAGE='$$(cat base-image) \
+		./docker-build.sh . ci/Dockerfile.ci-utils $(CI_UTILS_IMAGE)
 	echo $(CI_UTILS_IMAGE) > $@
 
 hail-buildkit-image: ci/buildkit/Dockerfile
 	$(eval HAIL_BUILDKIT_IMAGE := $(DOCKER_PREFIX)/hail-buildkit:$(TOKEN))
-	python3 ci/jinja2_render.py '{"global":{"docker_prefix":"$(DOCKER_PREFIX)"}}' ci/buildkit/Dockerfile ci/buildkit/Dockerfile.out
-	./docker-build.sh ci buildkit/Dockerfile.out $(HAIL_BUILDKIT_IMAGE)
+	DOCKER_BUILD_ARGS='--build-arg DOCKER_PREFIX=$(DOCKER_PREFIX)' \
+		./docker-build.sh ci buildkit/Dockerfile $(HAIL_BUILDKIT_IMAGE)
 	echo $(HAIL_BUILDKIT_IMAGE) > $@
 
 batch/jvm-entryway/build/libs/jvm-entryway.jar: $(shell git ls-files batch/jvm-entryway)
@@ -210,12 +210,12 @@ batch-worker-image: batch/jvm-entryway/build/libs/jvm-entryway.jar $(SERVICES_IM
 
 vep-grch37-image: hail-ubuntu-image
 	$(eval VEP_GRCH37_IMAGE := $(DOCKER_PREFIX)/hailgenetics/vep-grch37-85:$(TOKEN))
-	python3 ci/jinja2_render.py '{"hail_ubuntu_image":{"image":"'$$(cat hail-ubuntu-image)'"}}' vep/grch37/85/Dockerfile vep/grch37/85/Dockerfile.out
-	./docker-build.sh docker/vep/grch37/85/Dockerfile.out $(VEP_GRCH37_IMAGE)
+	DOCKER_BUILD_ARGS='--build-arg BASE_IMAGE='$$(cat hail-ubuntu-image) \
+		./docker-build.sh docker/vep docker/vep/grch37/85/Dockerfile $(VEP_GRCH37_IMAGE)
 	echo $(VEP_GRCH37_IMAGE) > $@
 
 vep-grch38-image: hail-ubuntu-image
 	$(eval VEP_GRCH38_IMAGE := $(DOCKER_PREFIX)/hailgenetics/vep-grch38-95:$(TOKEN))
-	python3 ci/jinja2_render.py '{"hail_ubuntu_image":{"image":"'$$(cat hail-ubuntu-image)'"}}' vep/grch38/95/Dockerfile vep/grch38/95/Dockerfile.out
-	./docker-build.sh docker/vep/grch38/95/Dockerfile.out $(VEP_GRCH38_IMAGE)
+	DOCKER_BUILD_ARGS='--build-arg BASE_IMAGE='$$(cat hail-ubuntu-image) \
+		./docker-build.sh docker/vep docker/vep/grch38/95/Dockerfile $(VEP_GRCH38_IMAGE)
 	echo $(VEP_GRCH38_IMAGE) > $@
diff --git a/admin-pod/Dockerfile b/admin-pod/Dockerfile
@@ -1,4 +1,5 @@
-FROM {{ hail_ubuntu_image.image }}
+ARG BASE_IMAGE={{ hail_ubuntu_image.image }}
+FROM $BASE_IMAGE
 
 # https://bugs.mysql.com/bug.php?id=105288&thanks=sub
 RUN hail-apt-get-install xz-utils libncurses5 && \

diff --git a/auth/Dockerfile b/auth/Dockerfile
@@ -1,4 +1,5 @@
-FROM {{ hail_ubuntu_image.image }}
+ARG BASE_IMAGE={{ hail_ubuntu_image.image }}
+FROM $BASE_IMAGE
 
 COPY hail/python/hailtop/pinned-requirements.txt hailtop-requirements.txt
 COPY gear/pinned-requirements.txt gear-requirements.txt

diff --git a/batch/Dockerfile b/batch/Dockerfile
@@ -1,4 +1,5 @@
-FROM {{ hail_ubuntu_image.image }}
+ARG BASE_IMAGE={{ hail_ubuntu_image.image }}
+FROM $BASE_IMAGE
 
 COPY hail/python/hailtop/pinned-requirements.txt hailtop-requirements.txt
 COPY gear/pinned-requirements.txt gear-requirements.txt

diff --git a/batch/Dockerfile.worker b/batch/Dockerfile.worker
@@ -1,4 +1,5 @@
-FROM {{ hail_ubuntu_image.image }} AS base
+ARG BASE_IMAGE={{ hail_ubuntu_image.image }}
+FROM $BASE_IMAGE AS base
 
 # libyajl-dev: crun runtime dependency
 RUN hail-apt-get-install \

diff --git a/blog/Dockerfile.nginx b/blog/Dockerfile.nginx
@@ -1,4 +1,5 @@
-FROM {{ hail_ubuntu_image.image }}
+ARG BASE_IMAGE={{ hail_ubuntu_image.image }}
+FROM $BASE_IMAGE
 
 RUN hail-apt-get-install nginx
 

diff --git a/bootstrap-gateway/Dockerfile b/bootstrap-gateway/Dockerfile
@@ -1,4 +1,5 @@
-FROM {{ hail_ubuntu_image.image }}
+ARG BASE_IMAGE={{ hail_ubuntu_image.image }}
+FROM $BASE_IMAGE
 
 RUN hail-apt-get-install nginx
 

diff --git a/build.yaml b/build.yaml
@@ -8,7 +8,8 @@ steps:
       memory: standard
     dockerFile:
       inline: |
-        FROM {{ global.docker_root_image }}
+        ARG BASE_IMAGE={{ global.docker_root_image }}
+        FROM $BASE_IMAGE
         RUN apt-get update && apt-get install -y git make bash
   - kind: runImage
     name: merge_code
@@ -1357,7 +1358,8 @@ steps:
     contextPath: /io/repo
     dockerFile:
       inline: |
-        FROM {{ hail_ubuntu_image.image }}
+        ARG BASE_IMAGE={{ hail_ubuntu_image.image }}
+        FROM $BASE_IMAGE
         RUN hail-apt-get-install make
         COPY hail/python/pinned-requirements.txt hail-requirements.txt
         COPY hail/python/dev/pinned-requirements.txt dev-requirements.txt
@@ -2482,7 +2484,8 @@ steps:
     publishAs: netcat
     dockerFile:
       inline: |
-        FROM {{ hail_ubuntu_image.image }}
+        ARG BASE_IMAGE={{ hail_ubuntu_image.image }}
+        FROM $BASE_IMAGE
         RUN hail-apt-get-install netcat
     dependsOn:
       - hail_ubuntu_image
@@ -2491,7 +2494,8 @@ steps:
     publishAs: volume
     dockerFile:
       inline: |
-        FROM {{ hail_ubuntu_image.image }}
+        ARG BASE_IMAGE={{ hail_ubuntu_image.image }}
+        FROM $BASE_IMAGE
         VOLUME ["/data"]
         WORKDIR "/tmp"
         VOLUME ["relative_volume"]
@@ -2502,7 +2506,8 @@ steps:
     publishAs: workdir
     dockerFile:
       inline: |
-        FROM {{ hail_ubuntu_image.image }}
+        ARG BASE_IMAGE={{ hail_ubuntu_image.image }}
+        FROM $BASE_IMAGE
         WORKDIR ["/work"]
     dependsOn:
       - hail_ubuntu_image
@@ -2511,7 +2516,8 @@ steps:
     publishAs: curl
     dockerFile:
       inline: |
-        FROM {{ hail_ubuntu_image.image }}
+        ARG BASE_IMAGE={{ hail_ubuntu_image.image }}
+        FROM $BASE_IMAGE
         RUN hail-apt-get-install curl
     dependsOn:
       - hail_ubuntu_image

diff --git a/ci/Dockerfile b/ci/Dockerfile
@@ -1,4 +1,5 @@
-FROM {{ hail_ubuntu_image.image }}
+ARG BASE_IMAGE={{ hail_ubuntu_image.image }}
+FROM $BASE_IMAGE
 
 RUN hail-apt-get-install git
 

diff --git a/ci/Dockerfile.ci-utils b/ci/Dockerfile.ci-utils
@@ -1,4 +1,5 @@
-FROM {{ base_image.image }} AS base
+ARG BASE_IMAGE={{ base_image.image }}
+FROM $BASE_IMAGE AS base
 
 # source: https://cloud.google.com/storage/docs/gsutil_install#linux
 RUN curl --remote-name https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-421.0.0-linux-x86_64.tar.gz && \

diff --git a/ci/buildkit/Dockerfile b/ci/buildkit/Dockerfile
@@ -1,4 +1,5 @@
-FROM {{ global.docker_prefix }}/moby/buildkit:v0.8.3-rootless
+ARG DOCKER_PREFIX={{ global.docker_prefix }}
+FROM $DOCKER_PREFIX/moby/buildkit:v0.8.3-rootless
 USER root
 RUN apk add python3 py-pip jq && pip3 install jinja2
 USER user

diff --git a/ci/ci/build.py b/ci/ci/build.py
@@ -38,7 +38,7 @@ class ServiceAccount(TypedDict):
     namespace: str
 
 
-def expand_value_from(value, config):
+def expand_value_from(value, config) -> str:
     if isinstance(value, str):
         return value
 
@@ -278,14 +278,26 @@ def cleanup(self, batch, scope, parents):
 
 class BuildImage2Step(Step):
     def __init__(
-        self, params: StepParameters, dockerfile, context_path, publish_as, inputs, resources
+        self,
+        params: StepParameters,
+        dockerfile,
+        context_path,
+        publish_as,
+        inputs,
+        resources,
+        build_args,
     ):  # pylint: disable=unused-argument
         super().__init__(params)
         self.dockerfile = dockerfile
         self.context_path = context_path
         self.inputs = inputs
         self.resources = resources
 
+        self.build_args = {
+            arg['name']: expand_value_from(arg['value'], self.input_config(params.code, params.scope))
+            for arg in build_args
+        }
+
         image_name = publish_as
         self.base_image = f'{DOCKER_PREFIX}/{image_name}'
         self.main_branch_cache_repository = f'{self.base_image}:cache'
@@ -326,6 +338,7 @@ def from_json(params: StepParameters):
             json['publishAs'],
             json.get('inputs'),
             json.get('resources'),
+            json.get('buildArgs', []),
         )
 
     def config(self, scope):  # pylint: disable=unused-argument
@@ -345,6 +358,10 @@ def build(self, batch, code, scope):
         if not context:
             context = '/io'
 
+        build_args_str = ' \\\n'.join(
+            f'--opt build-arg:{shq(name)}={shq(value)}' for name, value in self.build_args.items()
+        )
+
         if isinstance(self.dockerfile, dict):
             assert ['inline'] == list(self.dockerfile.keys())
             unrendered_dockerfile = f'/home/user/Dockerfile.in.{self.token}'
@@ -373,15 +390,16 @@ def build(self, batch, code, scope):
 
 export BUILDKITD_FLAGS='--oci-worker-no-process-sandbox --oci-worker-snapshotter=overlayfs'
 export BUILDCTL_CONNECT_RETRIES_MAX=100 # https://github.com/moby/buildkit/issues/1423
-retry buildctl-daemonless.sh \
-     build \
-     --frontend dockerfile.v0 \
-     --local context={shq(context)} \
-     --local dockerfile=/home/user \
-     --output 'type=image,"name={shq(self.image)},{shq(self.cache_repository)}",push=true' \
-     --export-cache type=inline \
-     --import-cache type=registry,ref={shq(self.cache_repository)} \
-     --import-cache type=registry,ref={shq(self.main_branch_cache_repository)} \
+retry buildctl-daemonless.sh \\
+     build \\
+     --frontend dockerfile.v0 \\
+     --local context={shq(context)} \\
+     --local dockerfile=/home/user \\
+     --output 'type=image,"name={shq(self.image)},{shq(self.cache_repository)}",push=true' \\
+     --export-cache type=inline \\
+     --import-cache type=registry,ref={shq(self.cache_repository)} \\
+     --import-cache type=registry,ref={shq(self.main_branch_cache_repository)} \\
+     {build_args_str} \\
      --trace=/home/user/trace
 cat /home/user/trace
 '''

diff --git a/ci/test/resources/Dockerfile b/ci/test/resources/Dockerfile
@@ -1,4 +1,5 @@
-FROM {{ ci_utils_image.image }}
+ARG BASE_IMAGE={{ ci_utils_image.image }}
+FROM $BASE_IMAGE
 
 COPY ci/test/resources/hello.py /hello/
 

diff --git a/ci/test/resources/build.yaml b/ci/test/resources/build.yaml
@@ -8,10 +8,15 @@ steps:
         optional: true
   - kind: buildImage2
     name: git_make_bash_image
+    buildArgs:
+      - name: BASE_IMAGE
+        value:
+          valueFrom: global.docker_root_image
     publishAs: git-make-bash
     dockerFile:
       inline: |
-        FROM {{ global.docker_root_image }}
+        ARG BASE_IMAGE
+        FROM $BASE_IMAGE
         RUN apt-get update && apt-get install -y git make bash
   - kind: runImage
     name: merge_code
@@ -46,6 +51,30 @@ steps:
         to: /io/hail-ubuntu
     dependsOn:
       - merge_code
+  - kind: buildImage2
+    name: hail_ubuntu_image_python_3_10
+    dockerFile: /io/hail-ubuntu/Dockerfile
+    contextPath: /io/hail-ubuntu
+    buildArgs:
+      - name: PYTHON_VERSION
+        value: "3.10"
+      - name: DOCKER_PREFIX
+        value:
+          valueFrom: global.docker_prefix
+    publishAs: hail-ubuntu-py-3-10
+    inputs:
+      - from: /repo/docker/hail-ubuntu
+        to: /io/hail-ubuntu
+    dependsOn:
+      - merge_code
+  - kind: runImage
+    name: test_build_image_build_args_override_default
+    image:
+      valueFrom: hail_ubuntu_image_python_3_10.image
+    script: |
+      python3 --version | grep '3.10'
+    dependsOn:
+      - hail_ubuntu_image_python_3_10
   - kind: buildImage2
     name: base_image
     dockerFile: /io/repo/docker/Dockerfile.base

diff --git a/docker-build.sh b/docker-build.sh
@@ -29,6 +29,7 @@ DOCKER_BUILDKIT=1 docker build \
        ${DEV_CACHE:+--cache-from ${DEV_CACHE}} \
        ${EXTRA_CACHE:+--cache-from ${EXTRA_CACHE}} \
        --build-arg BUILDKIT_INLINE_CACHE=1 \
+       ${DOCKER_BUILD_ARGS} \
        --tag ${REMOTE_IMAGE_NAME} \
        --tag ${CACHE_IMAGE_NAME} \
        ${CONTEXT}

diff --git a/docker/Dockerfile.base b/docker/Dockerfile.base
@@ -1,4 +1,5 @@
-FROM {{ hail_ubuntu_image.image }}
+ARG BASE_IMAGE={{ hail_ubuntu_image.image }}
+FROM $BASE_IMAGE
 
 RUN hail-apt-get-install \
     git \

diff --git a/docker/Dockerfile.hail-dev b/docker/Dockerfile.hail-dev
@@ -1,4 +1,5 @@
-FROM {{ hailgenetics_hail_image.image }}
+ARG HAIL_IMAGE={{ hailgenetics_hail_image.image }}
+FROM $HAIL_IMAGE
 
 COPY hail/python/dev/pinned-requirements.txt dev-requirements.txt
 RUN hail-pip-install -r dev-requirements.txt
diff --git a/docker/hail-ubuntu/Dockerfile b/docker/hail-ubuntu/Dockerfile
@@ -1,12 +1,17 @@
-FROM {{ global.docker_prefix }}/ubuntu:jammy-20230624
+ARG DOCKER_PREFIX={{ global.docker_prefix }}
+FROM $DOCKER_PREFIX/ubuntu:jammy-20230624
+
 ENV LANG C.UTF-8
 ENV DEBIAN_FRONTEND noninteractive
+
 COPY retry /bin/retry
 COPY hail-apt-get-install /bin/hail-apt-get-install
 COPY pip.conf /root/.config/pip/pip.conf
 COPY hail-pip-install /bin/hail-pip-install
 COPY controller.sh /
 COPY curlrc /root/.curlrc
+
+ARG PYTHON_VERSION=3.9
 RUN chmod 755 /bin/retry && \
     chmod 755 /bin/hail-apt-get-install && \
     chmod 755 /bin/hail-pip-install && \
@@ -20,8 +25,8 @@ RUN chmod 755 /bin/retry && \
          >> /etc/apt/sources.list && \
     echo 'deb-src [signed-by=/usr/share/keyrings/deadsnakes-ppa-archive-keyring.gpg] http://ppa.launchpad.net/deadsnakes/ppa/ubuntu jammy main' \
          >> /etc/apt/sources.list && \
-    hail-apt-get-install python3.9-minimal python3.9-dev python3.9-distutils gcc g++ && \
-    update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.9 1 && \
+    hail-apt-get-install python$PYTHON_VERSION-minimal python$PYTHON_VERSION-dev python$PYTHON_VERSION-distutils gcc g++ && \
+    update-alternatives --install /usr/bin/python3 python3 /usr/bin/python$PYTHON_VERSION 1 && \
     curl https://bootstrap.pypa.io/get-pip.py | python3 && \
     python3 -m pip install 'pip>=21<22' && \
     python3 -m pip check && \