build with variants

Aaavm · Oct 27, 2022 · 3c892b2 · 3c892b2
1 parent e598611
commit 3c892b2
Show file tree

Hide file tree

Showing 5 changed files with 244 additions and 124 deletions.
diff --git a/.github/workflows/push-dev.yml b/.github/workflows/push-dev.yml
@@ -11,8 +11,8 @@ jobs:
  if: "github.ref == 'refs/heads/main'"
  strategy:
  matrix:
- page: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]
- page-total: [13]
+ page: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57]
+ page-total: [57]
  fail-fast: true
  runs-on: devcontainer-image-builder-ubuntu
  steps:

diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml
@@ -11,8 +11,8 @@ jobs:
  if: ${{ github.event.base_ref == 'refs/heads/main' }}
  strategy:
  matrix:
- page: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]
- page-total: [13]
+ page: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57]
+ page-total: [57]
  fail-fast: false
  runs-on: devcontainer-image-builder-ubuntu
  steps:

diff --git a/build/src/image-info.js b/build/src/image-info.js
@@ -27,7 +27,7 @@ async function generateImageInformationFiles(repo, release, registry, registryPa
  const manifestExists = await asyncUtils.exists(manifestPath);
 
  console.log('(*) Generating image information files...');
- const definitions = definitionId ? [definitionId] : configUtils.getSortedDefinitionBuildList();
+ const definitions = definitionId ? [definitionId] : configUtils.getDefinitionList();
  await asyncUtils.forEach(definitions, async (currentDefinitionId) => {
  // Target file paths and whether they exist
  const definitionRelativePath = configUtils.getDefinitionPath(currentDefinitionId, true);

diff --git a/build/src/push.js b/build/src/push.js
@@ -41,17 +41,26 @@ async function push(repo, release, updateLatest, registry, registryPath, stubReg
  await asyncUtils.spawn('docker', ['run', '--privileged', '--rm', 'tonistiigi/binfmt', '--install', 'all']);
 
  // Build and push subset of images
- const definitionsToPush = definitionId ? [definitionId] : configUtils.getSortedDefinitionBuildList(page, pageTotal, definitionsToSkip);
- await asyncUtils.forEach(definitionsToPush, async (currentDefinitionId) => {
- console.log(`**** Pushing ${currentDefinitionId} ${release} ****`);
- await pushImage(
- currentDefinitionId, repo, release, updateLatest, registry, registryPath, stubRegistry, stubRegistryPath, prepOnly, pushImages, replaceImages, secondaryRegistryPath);
- });
+ if (definitionId) {
+ const variants = configUtils.getVariants(definitionId) || [null];
+ await asyncUtils.forEach(variants, async (variant) => {
+ console.log(`**** Pushing ${definitionId}: ${variant} ${release} ****`);
+ await pushImage(
+ definitionId, variant, repo, release, updateLatest, registry, registryPath, stubRegistry, stubRegistryPath, prepOnly, pushImages, replaceImages, secondaryRegistryPath);
+ });
+ } else {
+ const definitionsToPush = configUtils.getSortedDefinitionBuildList(page, pageTotal, definitionsToSkip);
+ await asyncUtils.forEach(definitionsToPush, async (currentJob) => {
+ console.log(`**** Pushing ${currentJob['id']}: ${currentJob['variant']} ${release} ****`);
+ await pushImage(
+ currentJob['id'], currentJob['variant'] || null, repo, release, updateLatest, registry, registryPath, stubRegistry, stubRegistryPath, prepOnly, pushImages, replaceImages, secondaryRegistryPath);
+ });
+ }
 
  return stagingFolder;
 }
 
-async function pushImage(definitionId, repo, release, updateLatest,
+async function pushImage(definitionId, variant, repo, release, updateLatest,
  registry, registryPath, stubRegistry, stubRegistryPath, prepOnly, pushImages, replaceImage, secondaryRegistryPath) {
  const definitionPath = configUtils.getDefinitionPath(definitionId);
  const dotDevContainerPath = path.join(definitionPath, '.devcontainer');
@@ -69,98 +78,92 @@ async function pushImage(definitionId, repo, release, updateLatest,
  const devContainerJsonRaw = await asyncUtils.readFile(devContainerJsonPath);
  const devContainerJson = jsonc.parse(devContainerJsonRaw);
 
- // Process variants in reverse order to be sure the first one is tagged as "latest" if appropriate
- const variants = configUtils.getVariants(definitionId) || [null];
- for (let i = variants.length - 1; i > -1; i--) {
- const variant = variants[i];
+ // Update common setup script download URL, SHA, parent tag if applicable
+ console.log(`(*) Prep Dockerfile for ${definitionId} ${variant ? 'variant "' + variant + '"' : ''}...`);
+ const prepResult = await prep.prepDockerFile(dockerFilePath,
+ definitionId, repo, release, registry, registryPath, stubRegistry, stubRegistryPath, true, variant);
 
- // Update common setup script download URL, SHA, parent tag if applicable
- console.log(`(*) Prep Dockerfile for ${definitionId} ${variant ? 'variant "' + variant + '"' : ''}...`);
- const prepResult = await prep.prepDockerFile(dockerFilePath,
- definitionId, repo, release, registry, registryPath, stubRegistry, stubRegistryPath, true, variant);
+ if (prepOnly) {
+ console.log(`(*) Skipping build and push to registry.`);
+ } else {
+ if (prepResult.shouldFlattenBaseImage) {
+ console.log(`(*) Flattening base image...`);
+ await flattenBaseImage(prepResult.baseImageTag, prepResult.flattenedBaseImageTag, pushImages);
+ }
 
- if (prepOnly) {
- console.log(`(*) Skipping build and push to registry.`);
- } else {
- if (prepResult.shouldFlattenBaseImage) {
- console.log(`(*) Flattening base image...`);
- await flattenBaseImage(prepResult.baseImageTag, prepResult.flattenedBaseImageTag, pushImages);
- }
+ // Build image
+ console.log(`(*) Building image...`);
+ // Determine tags to use
+ const imageNamesWithVersionTags = configUtils.getTagList(definitionId, release, updateLatest, registry, registryPath, variant);
+ const imageName = imageNamesWithVersionTags[0].split(':')[0];
 
- // Build image
- console.log(`(*) Building image...`);
- // Determine tags to use
- const imageNamesWithVersionTags = configUtils.getTagList(definitionId, release, updateLatest, registry, registryPath, variant);
- const imageName = imageNamesWithVersionTags[0].split(':')[0];
+ // Dual publish image to devcontainers and vscode/devcontainers
+ const secondaryImageNamesWithVersionTags = configUtils.getTagList(definitionId, release, updateLatest, registry, secondaryRegistryPath, variant);
 
-  // Dual publish image to devcontainers and vscode/devcontainers
-  const secondaryImageNamesWithVersionTags = configUtils.getTagList(definitionId, release, updateLatest, registry, secondaryRegistryPath, variant);
+ console.log(`(*) Tags:${imageNamesWithVersionTags.reduce((prev, current) => prev += `\n ${current}`, '')}`);
+ console.log(`(*) Secondary Tags:${secondaryImageNamesWithVersionTags.reduce((prev, current) => prev += `\n ${current}`, '')}`);
 
- console.log(`(*) Tags:${imageNamesWithVersionTags.reduce((prev, current) => prev += `\n ${current}`, '')}`);
- console.log(`(*) Secondary Tags:${secondaryImageNamesWithVersionTags.reduce((prev, current) => prev += `\n ${current}`, '')}`);
+ const buildSettings = configUtils.getBuildSettings(definitionId);
 
- const buildSettings = configUtils.getBuildSettings(definitionId);
+ let architectures = buildSettings.architectures;
+ switch (typeof architectures) {
+ case 'string': architectures = [architectures]; break;
+ case 'object': if (!Array.isArray(architectures)) { architectures = architectures[variant]; } break;
+ case 'undefined': architectures = ['linux/amd64']; break;
+ }
 
- let architectures = buildSettings.architectures;
- switch (typeof architectures) {
- case 'string': architectures = [architectures]; break;
- case 'object': if (!Array.isArray(architectures)) { architectures = architectures[variant]; } break;
- case 'undefined': architectures = ['linux/amd64']; break;
- }
+ console.log(`(*) Target image architectures: ${architectures.reduce((prev, current) => prev += `\n ${current}`, '')}`);
+ let localArchitecture = process.arch;
+ switch (localArchitecture) {
+ case 'arm': localArchitecture = 'linux/arm/v7'; break;
+ case 'aarch32': localArchitecture = 'linux/arm/v7'; break;
+ case 'aarch64': localArchitecture = 'linux/arm64'; break;
+ case 'x64': localArchitecture = 'linux/amd64'; break;
+ case 'x32': localArchitecture = 'linux/386'; break;
+ default: localArchitecture = `linux/${localArchitecture}`; break;
+ }
 
- console.log(`(*) Target image architectures: ${architectures.reduce((prev, current) => prev += `\n ${current}`, '')}`);
- let localArchitecture = process.arch;
- switch (localArchitecture) {
- case 'arm': localArchitecture = 'linux/arm/v7'; break;
- case 'aarch32': localArchitecture = 'linux/arm/v7'; break;
- case 'aarch64': localArchitecture = 'linux/arm64'; break;
- case 'x64': localArchitecture = 'linux/amd64'; break;
- case 'x32': localArchitecture = 'linux/386'; break;
- default: localArchitecture = `linux/${localArchitecture}`; break;
+ console.log(`(*) Local architecture: ${localArchitecture}`);
+ if (!pushImages) {
+ console.log(`(*) Push disabled: Only building local architecture (${localArchitecture}).`);
+ }
+
+ if (replaceImage || !await isDefinitionVersionAlreadyPublished(definitionId, release, registry, registryPath, variant)) {
+
+ let platformParams = "";
+ // Universal image does not need to be multi-arch
+ // ubuntu:focal image supports multiarch but Universal does not. Hence, the build fails similar to https://github.com/docker/buildx/issues/235
+ if (definitionId != "universal") {
+ platformParams = "--platform " + (pushImages ? architectures.reduce((prev, current) => prev + ',' + current, '').substring(1) : localArchitecture)
  }
 
- console.log(`(*) Local architecture: ${localArchitecture}`);
+ const context = devContainerJson.build ? devContainerJson.build.context || '.' : devContainerJson.context || '.';
+ const workingDir = path.resolve(dotDevContainerPath, context);
+ let imageNameParams = imageNamesWithVersionTags.reduce((prev, current) => prev.concat(['--image-name', current]), []);
+
+ const secondaryImageNameParams = secondaryImageNamesWithVersionTags.reduce((prev, current) => prev.concat(['--image-name', current]), []);
+ imageNameParams = imageNameParams.concat(secondaryImageNameParams);
+
+ const spawnOpts = { stdio: 'inherit', cwd: workingDir, shell: true };
+ await asyncUtils.spawn('devcontainer', [
+ 'build',
+ '--workspace-folder', definitionPath,
+ '--log-level ', 'info',
+ ...imageNameParams,
+ '--no-cache', 'true',
+ platformParams,
+ pushImages ? '--push' : '',
+ ], spawnOpts);
+
  if (!pushImages) {
- console.log(`(*) Push disabled: Only building local architecture (${localArchitecture}).`);
+ console.log(`(*) Skipping push to registry.`);
  }
 
- if (replaceImage || !await isDefinitionVersionAlreadyPublished(definitionId, release, registry, registryPath, variant)) {
-
- let platformParams = "";
- // Universal image does not need to be multi-arch
- // ubuntu:focal image supports multiarch but Universal does not. Hence, the build fails similar to https://github.com/docker/buildx/issues/235
- if (definitionId != "universal") {
- platformParams = "--platform " + (pushImages ? architectures.reduce((prev, current) => prev + ',' + current, '').substring(1) : localArchitecture)
- }
-
- const context = devContainerJson.build ? devContainerJson.build.context || '.' : devContainerJson.context || '.';
- const workingDir = path.resolve(dotDevContainerPath, context);
- let imageNameParams = imageNamesWithVersionTags.reduce((prev, current) => prev.concat(['--image-name', current]), []);
-
- const secondaryImageNameParams = secondaryImageNamesWithVersionTags.reduce((prev, current) => prev.concat(['--image-name', current]), []);
- imageNameParams = imageNameParams.concat(secondaryImageNameParams);
-
- const spawnOpts = { stdio: 'inherit', cwd: workingDir, shell: true };
- await asyncUtils.spawn('devcontainer', [
- 'build',
- '--workspace-folder', definitionPath,
- '--log-level ', 'info',
- ...imageNameParams,
- '--no-cache', 'true',
- platformParams,
- pushImages ? '--push' : '',
- ], spawnOpts);
-
- if (!pushImages) {
- console.log(`(*) Skipping push to registry.`);
- }
-
- console.log("(*) Docker images", imageName);
- await asyncUtils.spawn('docker', [`images`], spawnOpts);
-
- } else {
- console.log(`(*) Version already published. Skipping.`);
- }
+ console.log("(*) Docker images", imageName);
+ await asyncUtils.spawn('docker', [`images`], spawnOpts);
+
+ } else {
+ console.log(`(*) Version already published. Skipping.`);
  }
  }