internal/relui: migrate Go installer construction off old builders

Move installer construction to happen in the same place that installer
signing already happens, under the ConstructInstallerOnly build types
that CL 550320 added.

Reuse the minimal xar parser that Russ implemented in cmd/gorebuild for
extracting signed binaries (that need to be inserted into the .tar.gz)
from a signed .pkg installer. This lets us drop the Xcode dependency
for that task and have it run as part of relui on secured machines
used only for release orchestration.

Delete the unused previous code and supporting files for building
installers, since what's used now is in the internal/installer
packages that CL 550321 added.

For golang/go#63147.

Change-Id: If8b207b7e3739052bc6d5f8ac13bbe5a05b50e0c
Cq-Include-Trybots: luci.golang.try:x_build-gotip-linux-amd64-longtest-race
Reviewed-on: https://go-review.googlesource.com/c/build/+/552016
Auto-Submit: Dmitri Shuralyov <[email protected]>
Reviewed-by: Carlos Amedee <[email protected]>
Reviewed-by: Dmitri Shuralyov <[email protected]>
LUCI-TryBot-Result: Go LUCI <[email protected]>

cmd/gorebuild/darwin.go

@@ -175,9 +175,12 @@ func indexPkg(log *Log, data []byte, fix Fixer) map[string]*CpioFile {
 	return ix
 }
 
-// xar parser, enough to read macOS pkg files.
-// https://en.wikipedia.org/wiki/Xar_(archiver)
-// https://github.com/mackyle/xar/wiki/xarformat
+// A minimal xar parser, enough to read macOS .pkg files.
+// Package golang.org/x/build/internal/task also has one
+// for its internal needs.
+//
+// See https://en.wikipedia.org/wiki/Xar_(archiver)
+// and https://github.com/mackyle/xar/wiki/xarformat.
 
 // xarHeader is the main XML data structure for the xar header.
 type xarHeader struct {
@@ -215,8 +218,8 @@ type xarEncoding struct {
 // pkgPayload parses data as a macOS pkg file for the Go installer
 // and returns the content of the file org.golang.go.pkg/Payload.
 func pkgPayload(log *Log, data []byte) []byte {
-	if string(data[0:4]) != "xar!" || len(data) < 28 {
-		log.Printf("not an xar! file")
+	if len(data) < 28 || string(data[0:4]) != "xar!" {
+		log.Printf("not an XAR file format (missing a 28+ byte header with 'xar!' magic number)")
 		return nil
 	}
 	be := binary.BigEndian

internal/relui/buildrelease_test.go

@@ -68,20 +68,6 @@ func TestSecurity(t *testing.T) {
 const fakeGo = `#!/bin/bash -eu
 
 case "$1" in
-"run")
-  case "$2" in
-  "releaselet.go")
-    # We're building an MSI. The command should be run in the gomote work dir.
-    ls go/src/make.bash >/dev/null
-    mkdir msi
-    echo "I'm an MSI!" > msi/thisisanmsi.msi
-    ;;
-  *)
-    echo "unknown main file $2"
-    exit 1
-    ;;
-  esac
-  ;;
 "get")
   ls go.mod go.sum >/dev/null
   for i in "${@:2}"; do
@@ -128,48 +114,7 @@ func newReleaseTestDeps(t *testing.T, previousTag string, major int, wantVersion
 	// Set up a server that will be used to serve inputs to the build.
 	bootstrapServer := httptest.NewServer(http.HandlerFunc(serveBootstrap))
 	t.Cleanup(bootstrapServer.Close)
-	fakeBuildlets := task.NewFakeBuildlets(t, bootstrapServer.URL, map[string]string{
-		"pkgbuild": `#!/bin/bash -eu
-case "$@" in
-"--identifier=org.golang.go --version ` + wantVersion + ` --scripts=pkg-scripts --root=pkg-root pkg-intermediate/org.golang.go.pkg")
-	# We're doing an intermediate step in building a PKG.
-	echo "I'm an intermediate PKG!" > "$6"
-	tar -cz -C pkg-root . >> "$6"
-	;;
-*)
-	echo "unexpected command $@"
-	exit 1
-	;;
-esac
-`,
-		"productbuild": `#!/bin/bash -eu
-case "$@" in
-"--distribution=pkg-distribution --resources=pkg-resources --package-path=pkg-intermediate pkg-out/` + wantVersion + `.pkg")
-	# We're building a PKG.
-	ls pkg-distribution pkg-resources/bg-light.png pkg-resources/bg-dark.png >/dev/null
-	cat pkg-intermediate/* | head -n 1 | sed "s/an intermediate PKG/a PKG/" > "$4"
-	cat pkg-intermediate/* | tail -n +2 >> "$4"
-	;;
-*)
-	echo "unexpected command $@"
-	exit 1
-	;;
-esac
-`,
-		"pkgutil": `#!/bin/bash -eu
-case "$@" in
-"--expand-full go.pkg pkg-expanded")
-	# We're expanding a PKG.
-	mkdir -p "$3/org.golang.go.pkg/Payload/usr/local/go"
-	tail -n +2 "$2" | tar -xz -C "$3/org.golang.go.pkg/Payload"
-	;;
-*)
-	echo "unexpected command $@"
-	exit 1
-	;;
-esac
-`,
-	})
+	fakeBuildlets := task.NewFakeBuildlets(t, bootstrapServer.URL, nil)
 
 	// Set up the fake CDN publishing process.
 	servingDir := t.TempDir()

internal/relui/workflows.go

@@ -12,6 +12,7 @@ import (
 	"compress/gzip"
 	"context"
 	"crypto/sha256"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
@@ -31,12 +32,15 @@ import (
 	"golang.org/x/build/buildlet"
 	"golang.org/x/build/dashboard"
 	"golang.org/x/build/internal/gcsfs"
+	"golang.org/x/build/internal/installer/darwinpkg"
+	"golang.org/x/build/internal/installer/windowsmsi"
 	"golang.org/x/build/internal/releasetargets"
 	"golang.org/x/build/internal/relui/db"
 	"golang.org/x/build/internal/relui/sign"
 	"golang.org/x/build/internal/task"
 	"golang.org/x/build/internal/workflow"
 	wf "golang.org/x/build/internal/workflow"
+	"golang.org/x/exp/maps"
 	"golang.org/x/net/context/ctxhttp"
 	"google.golang.org/protobuf/types/known/structpb"
 )
@@ -548,12 +552,11 @@ func (tasks *BuildReleaseTasks) addBuildTasks(wd *wf.Definition, major int, kind
 		// include the signed binaries.
 		switch target.GOOS {
 		case "darwin":
-			pkg := wf.Task2(wd, "Build PKG installer", tasks.buildDarwinPKG, version, tar)
+			pkg := wf.Task1(wd, "Build PKG installer", tasks.buildDarwinPKG, tar)
 			signedPKG := wf.Task2(wd, "Sign PKG installer", tasks.signArtifact, pkg, wf.Const(sign.BuildMacOS))
-			signedTGZ := wf.Task1(wd, "Convert PKG to .tgz", tasks.convertPKGToTGZ, signedPKG)
-			mergedTGZ := wf.Task2(wd, "Merge signed files into .tgz", tasks.mergeSignedToTGZ, tar, signedTGZ)
-			mod = wf.Task4(wd, "Merge signed files into module zip", tasks.mergeSignedToModule, version, timestamp, mod, signedTGZ)
-			artifacts = append(artifacts, signedPKG, mergedTGZ)
+			signedTGZ := wf.Task2(wd, "Merge signed files into .tgz", tasks.mergeSignedToTGZ, tar, signedPKG)
+			mod = wf.Task4(wd, "Merge signed files into module zip", tasks.mergeSignedToModule, version, timestamp, mod, signedPKG)
+			artifacts = append(artifacts, signedPKG, signedTGZ)
 		case "windows":
 			msi := wf.Task1(wd, "Build MSI installer", tasks.buildWindowsMSI, tar)
 			signedMSI := wf.Task2(wd, "Sign MSI installer", tasks.signArtifact, msi, wf.Const(sign.BuildWindows))
@@ -994,9 +997,11 @@ func (b *BuildReleaseTasks) modFilesFromBinary(ctx *wf.TaskContext, version stri
 
 func (b *BuildReleaseTasks) mergeSignedToTGZ(ctx *wf.TaskContext, unsigned, signed artifact) (artifact, error) {
 	return b.runBuildStep(ctx, unsigned.Target, nil, signed, "tar.gz", func(_ *task.BuildletStep, signed io.Reader, w io.Writer) error {
-		signedBinaries, err := loadBinaries(ctx, signed)
+		signedBinaries, err := task.ReadBinariesFromPKG(signed)
 		if err != nil {
 			return err
+		} else if _, ok := signedBinaries["go/bin/go"]; !ok {
+			return fmt.Errorf("didn't find go/bin/go among %d signed binaries %+q", len(signedBinaries), maps.Keys(signedBinaries))
 		}
 
 		// Copy files from the tgz, overwriting with binaries from the signed tar.
@@ -1051,9 +1056,11 @@ func (b *BuildReleaseTasks) mergeSignedToTGZ(ctx *wf.TaskContext, unsigned, sign
 
 func (b *BuildReleaseTasks) mergeSignedToModule(ctx *wf.TaskContext, version string, timestamp time.Time, mod moduleArtifact, signed artifact) (moduleArtifact, error) {
 	a, err := b.runBuildStep(ctx, nil, nil, signed, "signedmod.zip", func(_ *task.BuildletStep, signed io.Reader, w io.Writer) error {
-		signedBinaries, err := loadBinaries(ctx, signed)
+		signedBinaries, err := task.ReadBinariesFromPKG(signed)
 		if err != nil {
 			return err
+		} else if _, ok := signedBinaries["go/bin/go"]; !ok {
+			return fmt.Errorf("didn't find go/bin/go among %d signed binaries %+q", len(signedBinaries), maps.Keys(signedBinaries))
 		}
 
 		// Copy files from the module zip, overwriting with binaries from the signed tar.
@@ -1109,63 +1116,79 @@ func (b *BuildReleaseTasks) mergeSignedToModule(ctx *wf.TaskContext, version str
 	return mod, nil
 }
 
-// loadBinaries reads binaries that we expect to have been signed by the
-// macOS signing process from tgz.
-func loadBinaries(ctx *wf.TaskContext, tgz io.Reader) (map[string][]byte, error) {
-	zr, err := gzip.NewReader(tgz)
-	if err != nil {
-		return nil, err
-	}
-	defer zr.Close()
-	tr := tar.NewReader(zr)
-
-	binaries := map[string][]byte{}
-	for {
-		th, err := tr.Next()
-		if err == io.EOF {
-			break
-		} else if err != nil {
-			return nil, err
-		}
-		if !strings.HasPrefix(th.Name, "go/bin/") && !strings.HasPrefix(th.Name, "go/pkg/tool/") {
-			continue
-		}
-		if th.Typeflag != tar.TypeReg || th.Mode&0100 == 0 {
-			continue
-		}
-		contents, err := io.ReadAll(tr)
-		if err != nil {
-			return nil, err
-		}
-		binaries[th.Name] = contents
-	}
-	return binaries, nil
-}
-
 func (b *BuildReleaseTasks) buildBinary(ctx *wf.TaskContext, target *releasetargets.Target, source artifact) (artifact, error) {
 	bc := dashboard.Builders[target.Builder]
 	return b.runBuildStep(ctx, target, bc, source, "tar.gz", func(bs *task.BuildletStep, r io.Reader, w io.Writer) error {
 		return bs.BuildBinary(ctx, r, w)
 	})
 }
 
-func (b *BuildReleaseTasks) buildDarwinPKG(ctx *wf.TaskContext, version string, binary artifact) (artifact, error) {
-	bc := dashboard.Builders[binary.Target.Builder]
-	return b.runBuildStep(ctx, binary.Target, bc, binary, "pkg", func(bs *task.BuildletStep, r io.Reader, w io.Writer) error {
-		return bs.BuildDarwinPKG(ctx, r, version, w)
-	})
-}
-func (b *BuildReleaseTasks) convertPKGToTGZ(ctx *wf.TaskContext, pkg artifact) (tgz artifact, _ error) {
-	bc := dashboard.Builders[pkg.Target.Builder]
-	return b.runBuildStep(ctx, pkg.Target, bc, pkg, "tar.gz", func(bs *task.BuildletStep, r io.Reader, w io.Writer) error {
-		return bs.ConvertPKGToTGZ(ctx, r, w)
+// buildDarwinPKG constructs an installer for the given binary artifact, to be signed.
+func (b *BuildReleaseTasks) buildDarwinPKG(ctx *wf.TaskContext, binary artifact) (artifact, error) {
+	return b.runBuildStep(ctx, binary.Target, nil, artifact{}, "pkg", func(_ *task.BuildletStep, _ io.Reader, w io.Writer) error {
+		metadataFile, err := jsonEncodeScratchFile(ctx, b.ScratchFS, darwinpkg.InstallerOptions{
+			GOARCH:          binary.Target.GOARCH,
+			MinMacOSVersion: binary.Target.MinMacOSVersion,
+		})
+		if err != nil {
+			return err
+		}
+		installerPaths, err := b.signArtifacts(ctx, sign.BuildMacOSConstructInstallerOnly, []string{
+			b.ScratchFS.URL(ctx, binary.Scratch),
+			b.ScratchFS.URL(ctx, metadataFile),
+		})
+		if err != nil {
+			return err
+		} else if len(installerPaths) != 1 {
+			return fmt.Errorf("got %d outputs, want 1 macOS .pkg installer", len(installerPaths))
+		} else if ext := path.Ext(installerPaths[0]); ext != ".pkg" {
+			return fmt.Errorf("got output extension %q, want .pkg", ext)
+		}
+		resultFS, err := gcsfs.FromURL(ctx, b.GCSClient, b.SignedURL)
+		if err != nil {
+			return err
+		}
+		r, err := resultFS.Open(installerPaths[0])
+		if err != nil {
+			return err
+		}
+		defer r.Close()
+		_, err = io.Copy(w, r)
+		return err
 	})
 }
 
+// buildWindowsMSI constructs an installer for the given binary artifact, to be signed.
 func (b *BuildReleaseTasks) buildWindowsMSI(ctx *wf.TaskContext, binary artifact) (artifact, error) {
-	bc := dashboard.Builders[binary.Target.Builder]
-	return b.runBuildStep(ctx, binary.Target, bc, binary, "msi", func(bs *task.BuildletStep, r io.Reader, w io.Writer) error {
-		return bs.BuildWindowsMSI(ctx, r, w)
+	return b.runBuildStep(ctx, binary.Target, nil, artifact{}, "msi", func(_ *task.BuildletStep, _ io.Reader, w io.Writer) error {
+		metadataFile, err := jsonEncodeScratchFile(ctx, b.ScratchFS, windowsmsi.InstallerOptions{
+			GOARCH: binary.Target.GOARCH,
+		})
+		if err != nil {
+			return err
+		}
+		installerPaths, err := b.signArtifacts(ctx, sign.BuildWindowsConstructInstallerOnly, []string{
+			b.ScratchFS.URL(ctx, binary.Scratch),
+			b.ScratchFS.URL(ctx, metadataFile),
+		})
+		if err != nil {
+			return err
+		} else if len(installerPaths) != 1 {
+			return fmt.Errorf("got %d outputs, want 1 Windows .msi installer", len(installerPaths))
+		} else if ext := path.Ext(installerPaths[0]); ext != ".msi" {
+			return fmt.Errorf("got output extension %q, want .msi", ext)
+		}
+		resultFS, err := gcsfs.FromURL(ctx, b.GCSClient, b.SignedURL)
+		if err != nil {
+			return err
+		}
+		r, err := resultFS.Open(installerPaths[0])
+		if err != nil {
+			return err
+		}
+		defer r.Close()
+		_, err = io.Copy(w, r)
+		return err
 	})
 }
 
@@ -1706,3 +1729,22 @@ func (b *BuildReleaseTasks) awaitCloudBuild(ctx *wf.TaskContext, build task.Clou
 	})
 	return detail, err
 }
+
+// jsonEncodeScratchFile JSON encodes v into a new scratch file and returns its name.
+func jsonEncodeScratchFile(ctx *wf.TaskContext, fs *task.ScratchFS, v any) (name string, _ error) {
+	name, f, err := fs.OpenWrite(ctx, "f.json")
+	if err != nil {
+		return "", err
+	}
+	e := json.NewEncoder(f)
+	e.SetIndent("", "\t")
+	e.SetEscapeHTML(false)
+	if err := e.Encode(v); err != nil {
+		f.Close()
+		return "", err
+	}
+	if err := f.Close(); err != nil {
+		return "", err
+	}
+	return name, nil
+}