From 81cc66f1fb040c4d2d7b6dadd39a77ad2ea0b9ac Mon Sep 17 00:00:00 2001 From: Jordan Sanders Date: Tue, 8 Aug 2023 09:50:13 -0500 Subject: [PATCH] Rebuild buildkite-test images (#15710) It's been a while since we built and pushed these and we need the node 20 upgrade. I've played whack-a-mole to get all of these building and pushed them. I'll run a test build on buildkite to make sure everything works (particularly because I changed how we install java). It might be worth switching this to build automatically at some cadence so it doesn't atrophy as much. --- .../docker/images/buildkite-test/Dockerfile | 19 +++++++++---------- .../images/buildkite-test/last_updated.yaml | 9 ++++----- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/python_modules/automation/automation/docker/images/buildkite-test/Dockerfile b/python_modules/automation/automation/docker/images/buildkite-test/Dockerfile index 2a77baf525ff1..66b2fdec60b71 100644 --- a/python_modules/automation/automation/docker/images/buildkite-test/Dockerfile +++ b/python_modules/automation/automation/docker/images/buildkite-test/Dockerfile @@ -55,9 +55,13 @@ RUN apt-get update -yqq \ libxmlsec1-dev \ pkg-config +# Install Java (required by pyspark) +COPY --from=openjdk:8-jre-slim /usr/local/openjdk-8 /usr/local/openjdk-8 +ENV JAVA_HOME /usr/local/openjdk-8 +RUN update-alternatives --install /usr/bin/java java /usr/local/openjdk-8/bin/java 1 + # Install various packages used in running/installing/testing Dagster: # - git/make (cloning dagster, running checks defined in Makefile) -# - openjdk-11-{jdk,jre}-headless 8 (Java required by pyspark) see: http://bit.ly/2ZIuHRh # - nodejs/yarn (dagster UI) # - assorted others-- some may no longer be required, but leaving in place for now # @@ -81,8 +85,6 @@ RUN curl -sL https://deb.nodesource.com/setup_20.x | bash - \ make \ mariadb-client \ nodejs \ - openjdk-11-jdk-headless \ - openjdk-11-jre-headless \ pandoc \ postgresql \ rabbitmq-server \ @@ -92,9 +94,7 @@ RUN curl -sL https://deb.nodesource.com/setup_20.x | bash - \ unzip \ wget \ xz-utils \ - yarn \ - # Confirm Java works - && java -version + yarn # Install FOSSA cli for analyzing our open-source dependencies # https://github.com/fossas/fossa-cli @@ -108,13 +108,12 @@ RUN wget https://github.com/fossas/fossa-cli/releases/download/v${FOSSA_VERSION} # Install Google Cloud SDK # https://cloud.google.com/sdk/docs/install#deb # Add Google Cloud GPG key and apt source -RUN curl https://packages.cloud.google.com/apt/doc/apt-key.gpg > /usr/share/keyrings/cloud.google.gpg \ - && echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] http://packages.cloud.google.com/apt cloud-sdk main" | \ +RUN curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key --keyring /usr/share/keyrings/cloud.google.gpg add - \ + && echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | \ tee -a /etc/apt/sources.list.d/google-cloud-sdk.list > /dev/null \ - # Install google cloud && apt-get -yqq update \ && apt-get -yqq install \ - google-cloud-sdk + google-cloud-cli # Install Docker -- note that we have some older code using `docker-compose` instead of `docker # compose`, so we have to install both the `docker-compose` executable and the docker compose CLI diff --git a/python_modules/automation/automation/docker/images/buildkite-test/last_updated.yaml b/python_modules/automation/automation/docker/images/buildkite-test/last_updated.yaml index 6f84170ad545a..94421f3f7d748 100644 --- a/python_modules/automation/automation/docker/images/buildkite-test/last_updated.yaml +++ b/python_modules/automation/automation/docker/images/buildkite-test/last_updated.yaml @@ -1,5 +1,4 @@ -'3.10': 2023-03-13T170520 -'3.11': 2023-03-13T170520 -'3.7': 2023-03-13T170520 -'3.8': 2023-03-13T170520 -'3.9': 2023-03-13T170520 +'3.10': 2023-08-07T143538 +'3.11': 2023-08-07T143538 +'3.8': 2023-08-07T143538 +'3.9': 2023-08-07T143538