Skip to content

Commit

Permalink
build: add docker output for non-moby drivers
Browse files Browse the repository at this point in the history
Signed-off-by: Tonis Tiigi <[email protected]>
  • Loading branch information
tonistiigi committed Apr 17, 2019
1 parent 6b0928d commit cac3743
Show file tree
Hide file tree
Showing 8 changed files with 214 additions and 93 deletions.
78 changes: 69 additions & 9 deletions build/build.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (

"github.com/containerd/containerd/platforms"
"github.com/docker/distribution/reference"
dockerclient "github.com/docker/docker/client"
"github.com/docker/docker/pkg/urlutil"
"github.com/moby/buildkit/client"
"github.com/moby/buildkit/session"
Expand Down Expand Up @@ -61,6 +62,10 @@ type DriverInfo struct {
Err error
}

type DockerAPI interface {
DockerAPI(name string) (dockerclient.APIClient, error)
}

func getFirstDriver(drivers []DriverInfo) (driver.Driver, error) {
err := errors.Errorf("no drivers found")
for _, di := range drivers {
Expand All @@ -74,7 +79,7 @@ func getFirstDriver(drivers []DriverInfo) (driver.Driver, error) {
return nil, err
}

func Build(ctx context.Context, drivers []DriverInfo, opt map[string]Options, pw progress.Writer) (map[string]*client.SolveResponse, error) {
func Build(ctx context.Context, drivers []DriverInfo, opt map[string]Options, docker DockerAPI, pw progress.Writer) (map[string]*client.SolveResponse, error) {
if len(drivers) == 0 {
return nil, errors.Errorf("driver required for build")
}
Expand All @@ -83,18 +88,17 @@ func Build(ctx context.Context, drivers []DriverInfo, opt map[string]Options, pw
return nil, errors.Errorf("multiple drivers currently not supported")
}

pwOld := pw
d, err := getFirstDriver(drivers)
if err != nil {
return nil, err
}
_, isDefaultMobyDriver := d.(interface {
IsDefaultMobyDriver()
})
c, pw, err := driver.Boot(ctx, d, pw)
c, err := driver.Boot(ctx, d, pw)
if err != nil {
close(pwOld.Status())
<-pwOld.Done()
close(pw.Status())
<-pw.Done()
return nil, err
}

Expand Down Expand Up @@ -173,10 +177,16 @@ func Build(ctx context.Context, drivers []DriverInfo, opt map[string]Options, pw
}
if e.Type == "docker" {
if e.Output == nil {
if !isDefaultMobyDriver {
return nil, errors.Errorf("loading to docker currently not implemented, specify dest file or -")
if isDefaultMobyDriver {
e.Type = "image"
} else {
w, cancel, err := newDockerLoader(ctx, docker, e.Attrs["context"], mw)
if err != nil {
return nil, err
}
defer cancel()
opt.Exports[i].Output = w
}
e.Type = "image"
} else if !d.Features()[driver.DockerExporter] {
return nil, notSupported(d, driver.DockerExporter)
}
Expand Down Expand Up @@ -245,6 +255,7 @@ func Build(ctx context.Context, drivers []DriverInfo, opt map[string]Options, pw

var statusCh chan *client.SolveStatus
if pw != nil {
pw = progress.ResetTime(pw)
statusCh = pw.Status()
eg.Go(func() error {
<-pw.Done()
Expand Down Expand Up @@ -380,5 +391,54 @@ func LoadInputs(inp Inputs, target *client.SolveOpt) (func(), error) {
}

func notSupported(d driver.Driver, f driver.Feature) error {
return errors.Errorf("%s feature is currently not supported for %s driver. Please switch to a different driver (eg. \"docker buildx new\")", f, d.Factory().Name())
return errors.Errorf("%s feature is currently not supported for %s driver. Please switch to a different driver (eg. \"docker buildx create\")", f, d.Factory().Name())
}

func newDockerLoader(ctx context.Context, d DockerAPI, name string, mw *progress.MultiWriter) (io.WriteCloser, func(), error) {
c, err := d.DockerAPI(name)
if err != nil {
return nil, nil, err
}

pr, pw := io.Pipe()
started := make(chan struct{})
w := &waitingWriter{
PipeWriter: pw,
f: func() {
resp, err := c.ImageLoad(ctx, pr, false)
if err != nil {
pr.CloseWithError(err)
return
}
prog := mw.WithPrefix("", false)
close(started)
progress.FromReader(prog, "importing to docker", resp.Body)
},
started: started,
}
return w, func() {
pr.Close()
}, nil
}

type waitingWriter struct {
*io.PipeWriter
f func()
once sync.Once
mu sync.Mutex
err error
started chan struct{}
}

func (w *waitingWriter) Write(dt []byte) (int, error) {
w.once.Do(func() {
go w.f()
})
return w.PipeWriter.Write(dt)
}

func (w *waitingWriter) Close() error {
err := w.PipeWriter.Close()
<-w.started
return err
}
2 changes: 1 addition & 1 deletion commands/build.go
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,7 @@ func buildTargets(ctx context.Context, dockerCli command.Cli, opts map[string]bu
defer cancel()
pw := progress.NewPrinter(ctx2, os.Stderr, progressMode)

_, err = build.Build(ctx, dis, opts, pw)
_, err = build.Build(ctx, dis, opts, dockerAPI(dockerCli), pw)
return err
}

Expand Down
2 changes: 1 addition & 1 deletion commands/inspect.go
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,7 @@ func boot(ctx context.Context, ngi *nginfo) (bool, error) {
func(idx int) {
eg.Go(func() error {
pw := mw.WithPrefix(ngi.ng.Nodes[idx].Name, len(toBoot) > 1)
_, _, err := driver.Boot(ctx, ngi.drivers[idx].di.Driver, pw)
_, err := driver.Boot(ctx, ngi.drivers[idx].di.Driver, pw)
if err != nil {
ngi.drivers[idx].err = err
}
Expand Down
15 changes: 15 additions & 0 deletions commands/util.go
Original file line number Diff line number Diff line change
Expand Up @@ -299,3 +299,18 @@ func loadNodeGroupData(ctx context.Context, dockerCli command.Cli, ngi *nginfo)

return eg.Wait()
}

func dockerAPI(dockerCli command.Cli) *api {
return &api{dockerCli: dockerCli}
}

type api struct {
dockerCli command.Cli
}

func (a *api) DockerAPI(name string) (dockerclient.APIClient, error) {
if name == "" {
name = a.dockerCli.CurrentContext()
}
return clientForEndpoint(a.dockerCli, name)
}
77 changes: 6 additions & 71 deletions driver/driver.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ package driver

import (
"context"
"time"

"github.com/moby/buildkit/client"
"github.com/pkg/errors"
Expand Down Expand Up @@ -52,24 +51,24 @@ type Driver interface {
Features() map[Feature]bool
}

func Boot(ctx context.Context, d Driver, pw progress.Writer) (*client.Client, progress.Writer, error) {
func Boot(ctx context.Context, d Driver, pw progress.Writer) (*client.Client, error) {
try := 0
for {
info, err := d.Info(ctx)
if err != nil {
return nil, nil, err
return nil, err
}
try++
if info.Status != Running {
if try > 2 {
return nil, nil, errors.Errorf("failed to bootstrap %T driver in attempts", d)
return nil, errors.Errorf("failed to bootstrap %T driver in attempts", d)
}
if err := d.Bootstrap(ctx, func(s *client.SolveStatus) {
if pw != nil {
pw.Status() <- s
}
}); err != nil {
return nil, nil, err
return nil, err
}
}

Expand All @@ -78,72 +77,8 @@ func Boot(ctx context.Context, d Driver, pw progress.Writer) (*client.Client, pr
if errors.Cause(err) == ErrNotRunning && try <= 2 {
continue
}
return nil, nil, err
return nil, err
}
return c, newResetWriter(pw), nil
return c, nil
}
}

func newResetWriter(in progress.Writer) progress.Writer {
w := &pw{Writer: in, status: make(chan *client.SolveStatus), tm: time.Now()}
go func() {
for {
select {
case <-in.Done():
return
case st, ok := <-w.status:
if !ok {
close(in.Status())
return
}
if w.diff == nil {
for _, v := range st.Vertexes {
if v.Started != nil {
d := v.Started.Sub(w.tm)
w.diff = &d
}
}
}
if w.diff != nil {
for _, v := range st.Vertexes {
if v.Started != nil {
d := v.Started.Add(-*w.diff)
v.Started = &d
}
if v.Completed != nil {
d := v.Completed.Add(-*w.diff)
v.Completed = &d
}
}
for _, v := range st.Statuses {
if v.Started != nil {
d := v.Started.Add(-*w.diff)
v.Started = &d
}
if v.Completed != nil {
d := v.Completed.Add(-*w.diff)
v.Completed = &d
}
v.Timestamp = v.Timestamp.Add(-*w.diff)
}
for _, v := range st.Logs {
v.Timestamp = v.Timestamp.Add(-*w.diff)
}
}
in.Status() <- st
}
}
}()
return w
}

type pw struct {
progress.Writer
tm time.Time
diff *time.Duration
status chan *client.SolveStatus
}

func (p *pw) Status() chan *client.SolveStatus {
return p.status
}
40 changes: 40 additions & 0 deletions util/progress/fromreader.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
package progress

import (
"io"
"io/ioutil"
"time"

"github.com/moby/buildkit/client"
"github.com/moby/buildkit/identity"
"github.com/opencontainers/go-digest"
)

func FromReader(w Writer, name string, rc io.ReadCloser) {
status := w.Status()
dgst := digest.FromBytes([]byte(identity.NewID()))
tm := time.Now()

vtx := client.Vertex{
Digest: dgst,
Name: name,
Started: &tm,
}

status <- &client.SolveStatus{
Vertexes: []*client.Vertex{&vtx},
}

_, err := io.Copy(ioutil.Discard, rc)

tm2 := time.Now()
vtx2 := vtx
vtx2.Completed = &tm2
if err != nil {
vtx2.Error = err.Error()
}
status <- &client.SolveStatus{
Vertexes: []*client.Vertex{&vtx2},
}
close(status)
}
71 changes: 71 additions & 0 deletions util/progress/reset.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
package progress

import (
"time"

"github.com/moby/buildkit/client"
)

func ResetTime(in Writer) Writer {
w := &pw{Writer: in, status: make(chan *client.SolveStatus), tm: time.Now()}
go func() {
for {
select {
case <-in.Done():
return
case st, ok := <-w.status:
if !ok {
close(in.Status())
return
}
if w.diff == nil {
for _, v := range st.Vertexes {
if v.Started != nil {
d := v.Started.Sub(w.tm)
w.diff = &d
}
}
}
if w.diff != nil {
for _, v := range st.Vertexes {
if v.Started != nil {
d := v.Started.Add(-*w.diff)
v.Started = &d
}
if v.Completed != nil {
d := v.Completed.Add(-*w.diff)
v.Completed = &d
}
}
for _, v := range st.Statuses {
if v.Started != nil {
d := v.Started.Add(-*w.diff)
v.Started = &d
}
if v.Completed != nil {
d := v.Completed.Add(-*w.diff)
v.Completed = &d
}
v.Timestamp = v.Timestamp.Add(-*w.diff)
}
for _, v := range st.Logs {
v.Timestamp = v.Timestamp.Add(-*w.diff)
}
}
in.Status() <- st
}
}
}()
return w
}

type pw struct {
Writer
tm time.Time
diff *time.Duration
status chan *client.SolveStatus
}

func (p *pw) Status() chan *client.SolveStatus {
return p.status
}
Loading

0 comments on commit cac3743

Please sign in to comment.