Merge pull request moby#2444 from tonistiigi/v0.9-cherry-picks-20211101

[v0.9] cherry-picks

README.md

@@ -194,7 +194,7 @@ buildctl build \
 buildctl build \
     --frontend gateway.v0 \
     --opt source=docker/dockerfile \
-    --opt context=git://github.com/moby/moby \
+    --opt context=https://github.com/moby/moby.git \
     --opt build-arg:APT_MIRROR=cdn-fastly.deb.debian.org
 ```
 

cache/remotecache/export.go

@@ -68,7 +68,7 @@ func NewExporter(ingester content.Ingester, ref string, oci bool) Exporter {
 
 func (ce *contentCacheExporter) Finalize(ctx context.Context) (map[string]string, error) {
 	res := make(map[string]string)
-	config, descs, err := ce.chains.Marshal()
+	config, descs, err := ce.chains.Marshal(ctx)
 	if err != nil {
 		return nil, err
 	}

cache/remotecache/gha/gha.go

@@ -106,7 +106,7 @@ func (ce *exporter) indexKey() string {
 
 func (ce *exporter) Finalize(ctx context.Context) (map[string]string, error) {
 	// res := make(map[string]string)
-	config, descs, err := ce.chains.Marshal()
+	config, descs, err := ce.chains.Marshal(ctx)
 	if err != nil {
 		return nil, err
 	}

cache/remotecache/inline/inline.go

@@ -38,8 +38,8 @@ func (ce *exporter) reset() {
 	ce.chains = cc
 }
 
-func (ce *exporter) ExportForLayers(layers []digest.Digest) ([]byte, error) {
-	config, descs, err := ce.chains.Marshal()
+func (ce *exporter) ExportForLayers(ctx context.Context, layers []digest.Digest) ([]byte, error) {
+	config, descs, err := ce.chains.Marshal(ctx)
 	if err != nil {
 		return nil, err
 	}
@@ -63,7 +63,7 @@ func (ce *exporter) ExportForLayers(layers []digest.Digest) ([]byte, error) {
 		return nil, err
 	}
 
-	cfg, _, err := cc.Marshal()
+	cfg, _, err := cc.Marshal(ctx)
 	if err != nil {
 		return nil, err
 	}

cache/remotecache/v1/chains.go

@@ -1,6 +1,7 @@
 package cacheimport
 
 import (
+	"context"
 	"strings"
 	"sync"
 	"time"
@@ -75,7 +76,7 @@ func (c *CacheChains) normalize() error {
 	return nil
 }
 
-func (c *CacheChains) Marshal() (*CacheConfig, DescriptorProvider, error) {
+func (c *CacheChains) Marshal(ctx context.Context) (*CacheConfig, DescriptorProvider, error) {
 	if err := c.normalize(); err != nil {
 		return nil, nil, err
 	}
@@ -87,7 +88,7 @@ func (c *CacheChains) Marshal() (*CacheConfig, DescriptorProvider, error) {
 	}
 
 	for _, it := range c.items {
-		if err := marshalItem(it, st); err != nil {
+		if err := marshalItem(ctx, it, st); err != nil {
 			return nil, nil, err
 		}
 	}

cache/remotecache/v1/chains_test.go

@@ -1,6 +1,7 @@
 package cacheimport
 
 import (
+	"context"
 	"encoding/json"
 	"testing"
 	"time"
@@ -33,7 +34,7 @@ func TestSimpleMarshal(t *testing.T) {
 
 	addRecords()
 
-	cfg, _, err := cc.Marshal()
+	cfg, _, err := cc.Marshal(context.TODO())
 	require.NoError(t, err)
 
 	require.Equal(t, len(cfg.Layers), 2)
@@ -65,7 +66,7 @@ func TestSimpleMarshal(t *testing.T) {
 	// adding same info again doesn't produce anything extra
 	addRecords()
 
-	cfg2, descPairs, err := cc.Marshal()
+	cfg2, descPairs, err := cc.Marshal(context.TODO())
 	require.NoError(t, err)
 
 	require.EqualValues(t, cfg, cfg2)
@@ -78,13 +79,13 @@ func TestSimpleMarshal(t *testing.T) {
 	err = Parse(dt, descPairs, newChains)
 	require.NoError(t, err)
 
-	cfg3, _, err := cc.Marshal()
+	cfg3, _, err := cc.Marshal(context.TODO())
 	require.NoError(t, err)
 	require.EqualValues(t, cfg, cfg3)
 
 	// add extra item
 	cc.Add(outputKey(dgst("bay"), 0))
-	cfg, _, err = cc.Marshal()
+	cfg, _, err = cc.Marshal(context.TODO())
 	require.NoError(t, err)
 
 	require.Equal(t, len(cfg.Layers), 2)

cache/remotecache/v1/utils.go

@@ -1,12 +1,14 @@
 package cacheimport
 
 import (
+	"context"
 	"fmt"
 	"sort"
 
 	"github.com/moby/buildkit/exporter/containerimage/exptypes"
 	"github.com/moby/buildkit/solver"
 	digest "github.com/opencontainers/go-digest"
+	ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 )
@@ -277,17 +279,27 @@ type marshalState struct {
 	recordsByItem map[*item]int
 }
 
-func marshalRemote(r *solver.Remote, state *marshalState) string {
+func marshalRemote(ctx context.Context, r *solver.Remote, state *marshalState) string {
 	if len(r.Descriptors) == 0 {
 		return ""
 	}
+
+	if cd, ok := r.Provider.(interface {
+		CheckDescriptor(context.Context, ocispecs.Descriptor) error
+	}); ok && len(r.Descriptors) > 0 {
+		for _, d := range r.Descriptors {
+			if cd.CheckDescriptor(ctx, d) != nil {
+				return ""
+			}
+		}
+	}
 	var parentID string
 	if len(r.Descriptors) > 1 {
 		r2 := &solver.Remote{
 			Descriptors: r.Descriptors[:len(r.Descriptors)-1],
 			Provider:    r.Provider,
 		}
-		parentID = marshalRemote(r2, state)
+		parentID = marshalRemote(ctx, r2, state)
 	}
 	desc := r.Descriptors[len(r.Descriptors)-1]
 
@@ -318,7 +330,7 @@ func marshalRemote(r *solver.Remote, state *marshalState) string {
 	return id
 }
 
-func marshalItem(it *item, state *marshalState) error {
+func marshalItem(ctx context.Context, it *item, state *marshalState) error {
 	if _, ok := state.recordsByItem[it]; ok {
 		return nil
 	}
@@ -330,7 +342,7 @@ func marshalItem(it *item, state *marshalState) error {
 
 	for i, m := range it.links {
 		for l := range m {
-			if err := marshalItem(l.src, state); err != nil {
+			if err := marshalItem(ctx, l.src, state); err != nil {
 				return err
 			}
 			idx, ok := state.recordsByItem[l.src]
@@ -345,7 +357,7 @@ func marshalItem(it *item, state *marshalState) error {
 	}
 
 	if it.result != nil {
-		id := marshalRemote(it.result, state)
+		id := marshalRemote(ctx, it.result, state)
 		if id != "" {
 			idx, ok := state.chainsByID[id]
 			if !ok {

hack/util

@@ -46,7 +46,7 @@ cacheRefFrom=""
 cacheRefTo=""
 currentref=""
 if [ "$GITHUB_ACTIONS" = "true" ]; then
-  currentref="git://github.com/$GITHUB_REPOSITORY#$GITHUB_REF"
+  currentref="https://github.com/$GITHUB_REPOSITORY.git#$GITHUB_REF"
   cacheType="local"
   cacheRefFrom="$CACHEDIR_FROM"
   cacheRefTo="$CACHEDIR_TO"

solver/exporter.go

@@ -11,7 +11,6 @@ type exporter struct {
 	records []*CacheRecord
 	record  *CacheRecord
 
-	res      []CacheExporterRecord
 	edge     *edge // for secondaryExporters
 	override *bool
 }
@@ -52,9 +51,10 @@ func addBacklinks(t CacheExporterTarget, rec CacheExporterRecord, cm *cacheManag
 	return rec, nil
 }
 
-type backlinkT struct{}
+type contextT string
 
-var backlinkKey = backlinkT{}
+var backlinkKey = contextT("solver/exporter/backlinks")
+var resKey = contextT("solver/exporter/res")
 
 func (e *exporter) ExportTo(ctx context.Context, t CacheExporterTarget, opt CacheExportOpt) ([]CacheExporterRecord, error) {
 	var bkm map[string]CacheExporterRecord
@@ -66,8 +66,16 @@ func (e *exporter) ExportTo(ctx context.Context, t CacheExporterTarget, opt Cach
 		bkm = bk.(map[string]CacheExporterRecord)
 	}
 
+	var res map[*exporter][]CacheExporterRecord
+	if r := ctx.Value(resKey); r == nil {
+		res = map[*exporter][]CacheExporterRecord{}
+		ctx = context.WithValue(ctx, resKey, res)
+	} else {
+		res = r.(map[*exporter][]CacheExporterRecord)
+	}
+
 	if t.Visited(e) {
-		return e.res, nil
+		return res[e], nil
 	}
 	t.Visit(e)
 
@@ -180,9 +188,9 @@ func (e *exporter) ExportTo(ctx context.Context, t CacheExporterTarget, opt Cach
 		}
 	}
 
-	e.res = allRec
+	res[e] = allRec
 
-	return e.res, nil
+	return allRec, nil
 }
 
 func getBestResult(records []*CacheRecord) *CacheRecord {

solver/llbsolver/solver.go

@@ -267,7 +267,7 @@ func (s *Solver) Solve(ctx context.Context, id string, sessionID string, req fro
 
 func inlineCache(ctx context.Context, e remotecache.Exporter, res solver.CachedResult, g session.Group) ([]byte, error) {
 	if efl, ok := e.(interface {
-		ExportForLayers([]digest.Digest) ([]byte, error)
+		ExportForLayers(context.Context, []digest.Digest) ([]byte, error)
 	}); ok {
 		workerRef, ok := res.Sys().(*worker.WorkerRef)
 		if !ok {
@@ -292,7 +292,7 @@ func inlineCache(ctx context.Context, e remotecache.Exporter, res solver.CachedR
 			return nil, err
 		}
 
-		return efl.ExportForLayers(digests)
+		return efl.ExportForLayers(ctx, digests)
 	}
 	return nil, nil
 }

util/resolver/authorizer.go

@@ -434,25 +434,28 @@ type scopes map[string]map[string]struct{}
 func parseScopes(s []string) scopes {
 	// https://docs.docker.com/registry/spec/auth/scope/
 	m := map[string]map[string]struct{}{}
-	for _, scope := range s {
-		parts := strings.SplitN(scope, ":", 3)
-		names := []string{parts[0]}
-		if len(parts) > 1 {
-			names = append(names, parts[1])
-		}
-		var actions []string
-		if len(parts) == 3 {
-			actions = append(actions, strings.Split(parts[2], ",")...)
-		}
-		name := strings.Join(names, ":")
-		ma, ok := m[name]
-		if !ok {
-			ma = map[string]struct{}{}
-			m[name] = ma
-		}
+	for _, scopeStr := range s {
+		// The scopeStr may have strings that contain multiple scopes separated by a space.
+		for _, scope := range strings.Split(scopeStr, " ") {
+			parts := strings.SplitN(scope, ":", 3)
+			names := []string{parts[0]}
+			if len(parts) > 1 {
+				names = append(names, parts[1])
+			}
+			var actions []string
+			if len(parts) == 3 {
+				actions = append(actions, strings.Split(parts[2], ",")...)
+			}
+			name := strings.Join(names, ":")
+			ma, ok := m[name]
+			if !ok {
+				ma = map[string]struct{}{}
+				m[name] = ma
+			}
 
-		for _, a := range actions {
-			ma[a] = struct{}{}
+			for _, a := range actions {
+				ma[a] = struct{}{}
+			}
 		}
 	}
 	return m

util/resolver/authorizer_test.go

@@ -0,0 +1,51 @@
+package resolver
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestParseScopes(t *testing.T) {
+	for _, tc := range []struct {
+		name     string
+		input    []string
+		expected scopes
+	}{
+		{
+			name: "SeparateStrings",
+			input: []string{
+				"repository:foo/bar:pull",
+				"repository:foo/baz:pull,push",
+			},
+			expected: map[string]map[string]struct{}{
+				"repository:foo/bar": {
+					"pull": struct{}{},
+				},
+				"repository:foo/baz": {
+					"pull": struct{}{},
+					"push": struct{}{},
+				},
+			},
+		},
+		{
+			name:  "CombinedStrings",
+			input: []string{"repository:foo/bar:pull repository:foo/baz:pull,push"},
+			expected: map[string]map[string]struct{}{
+				"repository:foo/bar": {
+					"pull": struct{}{},
+				},
+				"repository:foo/baz": {
+					"pull": struct{}{},
+					"push": struct{}{},
+				},
+			},
+		},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			parsed := parseScopes(tc.input)
+			if !reflect.DeepEqual(parsed, tc.expected) {
+				t.Fatalf("expected %v, got %v", tc.expected, parsed)
+			}
+		})
+	}
+}