Update Docker image builds to use newer tooling (#6878)

Docker recently introduced [provenance attestation](https://docs.docker.com/build/attestations/slsa-provenance/) in buildx 0.10.0, which broke our multi-arch image builds. To build our multi-arch manifests, we use a really old tool that isn't compatible with the buildx changes. To work around the problem, we disabled provenance attestation in our builds (see f82a7d9). With this change, we re-enable provenance attestation and upgrade our multi-arch build tooling to support it. I also took this opportunity to refactor our Dockerfiles into a single Dockerfile per module except API Proxy (which structures its x64 and arm images very differently). I initially did this refactoring because I planned to upgrade our pipelines/scripts to the canonical method for building Docker images: passing a comma-separated list of platforms to `docker buildx build` and letting it build the images _and_ the multi-arch manifest at once. This method would have required a single Dockerfile for all architectures. However, since we provide per-architecture tags for our images, it ended up being easier to keep our current pattern (build each single-arch image, then create the multi-arch manifest separately) and just upgrade the tools. Other changes: - In cases where I was already updating a pipeline script task (e.g., to add/change a parameter passed to a script), I converted the task syntax to use the newer 'script' alias for consistency with other pipelines. - In our API Proxy pipelines, there were three different places where we installed qemu and binfmt to prepare for cross-compiling the code, but the agent already has those installed so I removed it. - Also in our API Proxy pipelines, there was a place where we did some docker buildx setup even though the job doesn't use docker buildx. Maybe it used to? Anyway, I removed the setup task. - Removed the explicit bin_dir argument from all image-linux.yaml calls, since it is redundant with that parameter's default value. - Deleted all manifest.yaml.template files, since they were required for the old manifest tool and are no longer used. - In some of our release pipelines we have a default value for the 'tags' parameter that is given in not-quite-JSON format (an array with a single string value in _single_ quotes). I changed it to valid JSON so I can use jq to merge it with other tags when building the multi-arch image. - Removed the unused '--postfix' parameter from buildRocksDb.sh. - Removed the redundant and misnamed '--image-name' parameter from buildApiProxy.sh. - Cleaned up script variables and args parsing for consistency, and in a few cases to fix minor bugs. To test, I ran the CI build, which exercises the key YAML templates and all the scripts. I also ran the end-to-end tests and nested end-to-end tests to verify that the images still work as expected. ## Azure IoT Edge PR checklist:
Azure · Feb 3, 2023 · e6fc2ee · e6fc2ee
1 parent e70936f
commit e6fc2ee
Show file tree

Hide file tree

Showing 95 changed files with 383 additions and 1,347 deletions.
diff --git a/builds/misc/addons-publish.yaml b/builds/misc/addons-publish.yaml
@@ -54,9 +54,11 @@ jobs:
               filePath: '$(System.DefaultWorkingDirectory)/scripts/linux/moveImage.sh'
               arguments: '--from $(registry.address)/$(from.registry.namespace)/azureiotedge-api-proxy:$(version)-linux-arm64v8 --to $(registry.address)/$(to.registry.namespace)/azureiotedge-api-proxy:$(version)-linux-arm64v8'
 
-          - task: Bash@3
-            displayName: 'Publish Api Proxy Manifest'
-            inputs:
-              targetType: filePath
-              filePath: '$(System.DefaultWorkingDirectory)/scripts/linux/buildManifest.sh'
-              arguments: '-r $(registry.address) -v $(version) -t $(System.DefaultWorkingDirectory)/edge-modules/api-proxy-module/docker/manifest.yaml.template -n $(to.registry.namespace) --tags "$(tags)"'
+          - script: |
+              scripts/linux/buildManifest.sh \
+                -r $(registry.address) \
+                -i 'azureiotedge-api-proxy' \
+                -v $(version) \
+                -n $(to.registry.namespace) \
+                --tags '$(tags)'
+            displayName: 'Publish Api Proxy Manifest'
diff --git a/builds/misc/addons-release.yaml b/builds/misc/addons-release.yaml
@@ -91,16 +91,13 @@ stages:
         else
           echo '##vso[task.setvariable variable=buildVersion]$(version)'
         fi
-        sudo apt-get update && sudo apt-get -y install qemu binfmt-support qemu-user-static && \
-        docker -v
       displayName: 'Set build version'
     - template: templates/image-linux.yaml
       parameters:
         name: API Proxy
         imageName: azureiotedge-api-proxy
         project: api-proxy-module
         version: $(buildVersion)
-        bin_dir: '$(Build.BinariesDirectory)'      
 
 ################################################################################
 - stage: PublishManifests
@@ -114,10 +111,6 @@ stages:
   jobs:
   - job: PublishManifest
     displayName: Publish Manifest
-    strategy:
-      matrix:
-        ApiProxy:
-           manifestFilePath: '$(System.DefaultWorkingDirectory)/edge-modules/api-proxy-module/docker/manifest.yaml.template'
     steps:
     # Both docker logins needed for if we need to test this job. In this case images should go to edgebuilds.
     - task: Docker@2
@@ -137,7 +130,14 @@ stages:
           echo '##vso[task.setvariable variable=buildVersion]$(version)'
         fi
       displayName: 'Set build version'     
-    - script: scripts/linux/buildManifest.sh -r $(registry.address) -v $(buildVersion) -t $(manifestFilePath) -n microsoft --tags "$(tags)"
+    - script: |
+        scripts/linux/buildManifest.sh \
+          -r $(registry.address) \
+          -i 'azureiotedge-api-proxy' \
+          -v $(buildVersion) \
+          -n 'microsoft' \
+          --tags '$(tags)'
+      displayName: 'Create Api Proxy manifest'
 
   - job: ComponentGovernance
     steps:

diff --git a/builds/misc/images-publish.yaml b/builds/misc/images-publish.yaml
@@ -204,38 +204,38 @@ jobs:
                 echo "##vso[task.setvariable variable=tempSensor.tags;]$TEMP_SENSOR_TAGS_STR"
               displayName: Set Version and Tags
 
-            - task: Bash@3
+            - script: |
+                scripts/linux/buildManifest.sh \
+                  -r "$(to.registry.address)" \
+                  -i "azureiotedge-agent" \
+                  -v "$(resources.pipeline.images.runName)" \
+                  -n "$(to.registry.namespace)" \
+                  --tags '$(tags)'
               displayName: 'Publish Edge Agent Manifest'
-              inputs:
-                targetType: filePath
-                filePath: '$(System.DefaultWorkingDirectory)/scripts/linux/buildManifest.sh'
-                arguments: '-r "$(to.registry.address)" -v "$(resources.pipeline.images.runName)" -t $(System.DefaultWorkingDirectory)/edge-agent/docker/manifest.yaml.template -n "$(to.registry.namespace)" --tags "$(tags)"'
-              env:
-                BUILD_REPOSITORY_LOCALPATH: $(System.DefaultWorkingDirectory)
 
-            - task: Bash@3
+            - script: |
+                scripts/linux/buildManifest.sh \
+                  -r "$(to.registry.address)" \
+                  -i "azureiotedge-hub" \
+                  -v "$(resources.pipeline.images.runName)" \
+                  -n "$(to.registry.namespace)" \
+                  --tags '$(tags)'
               displayName: 'Publish Edge Hub Manifest'
-              inputs:
-                targetType: filePath
-                filePath: '$(System.DefaultWorkingDirectory)/scripts/linux/buildManifest.sh'
-                arguments: '-r "$(to.registry.address)" -v "$(resources.pipeline.images.runName)" -t $(System.DefaultWorkingDirectory)/edge-hub/docker/manifest.yaml.template -n "$(to.registry.namespace)" --tags "$(tags)"'
-              env:
-                BUILD_REPOSITORY_LOCALPATH: $(System.DefaultWorkingDirectory)
 
-            - task: Bash@3
+            - script: |
+                scripts/linux/buildManifest.sh \
+                  -r "$(to.registry.address)" \
+                  -i "azureiotedge-simulated-temperature-sensor" \
+                  -v "$(resources.pipeline.images.runName)" \
+                  -n "$(to.registry.namespace)" \
+                  --tags '$(tempSensor.tags)'
               displayName: 'Publish Temperature Sensor Manifest'
-              inputs:
-                targetType: filePath
-                filePath: '$(System.DefaultWorkingDirectory)/scripts/linux/buildManifest.sh'
-                arguments: '-r "$(to.registry.address)" -v "$(resources.pipeline.images.runName)" -t $(System.DefaultWorkingDirectory)/edge-modules/SimulatedTemperatureSensor/docker/manifest.yaml.template -n "$(to.registry.namespace)" --tags "$(tempSensor.tags)"'
-              env:
-                BUILD_REPOSITORY_LOCALPATH: $(System.DefaultWorkingDirectory)
 
-            - task: Bash@3
+            - script: |
+                scripts/linux/buildManifest.sh \
+                  -r "$(to.registry.address)" \
+                  -i "azureiotedge-diagnostics" \
+                  -v "$(resources.pipeline.images.runName)" \
+                  -n "$(to.unlisted.registry.namespace)" \
+                  --tags '$(tags)'
               displayName: 'Publish Diagnostic Module Manifest'
-              inputs:
-                targetType: filePath
-                filePath: '$(System.DefaultWorkingDirectory)/scripts/linux/buildManifest.sh'
-                arguments: '-r "$(to.registry.address)" -v "$(resources.pipeline.images.runName)" -t $(System.DefaultWorkingDirectory)/edge-modules/iotedge-diagnostics-dotnet/docker/manifest.yaml.template -n "$(to.unlisted.registry.namespace)" --tags "$(tags)"'
-              env:
-                BUILD_REPOSITORY_LOCALPATH: $(System.DefaultWorkingDirectory)
diff --git a/builds/misc/images-release.yaml b/builds/misc/images-release.yaml
@@ -376,7 +376,6 @@ stages:
           name: "Edge Hub"
           project: Microsoft.Azure.Devices.Edge.Hub.Service
           version: $(version)
-          bin_dir: '$(Build.BinariesDirectory)'
           use_rocksdb: true
     - job: BuildImageEdgeAgent
       steps:
@@ -386,7 +385,6 @@ stages:
           name: "Edge Agent"
           project: Microsoft.Azure.Devices.Edge.Agent.Service
           version: $(version)
-          bin_dir: '$(Build.BinariesDirectory)'
           use_rocksdb: true
     - job: BuildImageTemperatureSensor
       steps:
@@ -396,7 +394,6 @@ stages:
           name: "Temperature Sensor"
           project: SimulatedTemperatureSensor
           version: $(version)
-          bin_dir: '$(Build.BinariesDirectory)'
     - job: BuildImageDiagnostics
       steps:
       - template: templates/image-linux.yaml
@@ -405,7 +402,6 @@ stages:
           name: "Diagnostics Module"
           project: IotedgeDiagnosticsDotnet
           version: $(version)
-          bin_dir: '$(Build.BinariesDirectory)'
 
 ################################################################################
   - stage: PublishManifests
@@ -422,13 +418,13 @@ stages:
       strategy:
         matrix:
           EdgeAgent:
-            manifestFilePath: '$(System.DefaultWorkingDirectory)/edge-agent/docker/manifest.yaml.template'
+            imageName: 'azureiotedge-agent'
           EdgeHub:
-            manifestFilePath: '$(System.DefaultWorkingDirectory)/edge-hub/docker/manifest.yaml.template'
+            imageName: 'azureiotedge-hub'
           SimulatedTemperatureSensor:
-            manifestFilePath: '$(System.DefaultWorkingDirectory)/edge-modules/SimulatedTemperatureSensor/docker/manifest.yaml.template'
+            imageName: 'azureiotedge-simulated-temperature-sensor'
           Diagnostics:
-            manifestFilePath: '$(System.DefaultWorkingDirectory)/edge-modules/iotedge-diagnostics-dotnet/docker/manifest.yaml.template'
+            imageName: 'azureiotedge-diagnostics'
       steps:
       # Both docker logins needed for if we need to test this job. In this case images should go to edgebuilds.
       - task: Docker@2
@@ -441,4 +437,11 @@ stages:
         inputs:
           command: login
           containerRegistry: iotedge-release-acr
-      - script: scripts/linux/buildManifest.sh -r $(registry.address) -v $(version) -t $(manifestFilePath) -n '$(namespace)' --tags "$(tags)"
+      - script: |
+          scripts/linux/buildManifest.sh \
+            -r $(registry.address) \
+            -i $(imageName) \
+            -v $(version) \
+            -n '$(namespace)' \
+            --tags '$(tags)'
+        displayName: Create manifest for '$(imageName)'
diff --git a/builds/misc/templates/build-api-proxy.yaml b/builds/misc/templates/build-api-proxy.yaml
@@ -1,9 +1,7 @@
 parameters:
   name: 'API proxy'
-  imageName: 'api-proxy-module'
   project: 'api-proxy-module'
   configuration: 'release'
-  filePath: 'scripts/linux/buildAPIProxy.sh'
   binDir: '$(Build.BinariesDirectory)'
 
 jobs:
@@ -19,18 +17,13 @@ jobs:
         arm64:
           arch: "aarch64"
     steps:
-    - bash: |
-        sudo apt-get update && sudo apt-get -y install qemu binfmt-support qemu-user-static && \
-        docker run --rm --privileged multiarch/qemu-user-static --reset -p yes && \
-        docker buildx rm  mbuilder || true  && \
-        docker buildx create --name mbuilder  || true  && \
-        docker buildx use mbuilder
-        docker -v
-    - task: Bash@3
+    - script: |
+        scripts/linux/buildAPIProxy.sh \
+          -P ${{ parameters.project }} \
+          -c ${{ parameters.configuration }} \
+          --target-arch $(arch) \
+          --bin-dir ${{ parameters.binDir }}
       displayName: Build API-Proxy - $(arch)
-      inputs:
-        filePath: ${{ parameters.filePath }}
-        arguments: -i ${{ parameters.imageName }} -P ${{ parameters.project }} -c ${{ parameters.configuration }} --target-arch $(arch) --bin-dir ${{ parameters.binDir }}
     - task: PublishBuildArtifacts@1
       inputs:
         pathtoPublish: '${{ parameters.binDir }}/publish'