build: add experimental --no-console flag to support non-tty human-readable output with buildkit

Unfortunately, this is for now the only way to see the output of RUN commands when using buildkit.
It is equivalent to `DOCKER_BUILDKIT=1 docker build . 2>&1 | cat`

Signed-off-by: Tibor Vass <tibor@docker.com>

cli/command/image/build.go

@@ -57,6 +57,7 @@ type buildOptions struct {
 	isolation      string
 	quiet          bool
 	noCache        bool
+	noConsole      bool
 	rm             bool
 	forceRm        bool
 	pull           bool
@@ -151,6 +152,9 @@ func NewBuildCommand(dockerCli command.Cli) *cobra.Command {
 	flags.SetAnnotation("stream", "experimental", nil)
 	flags.SetAnnotation("stream", "version", []string{"1.31"})
 
+	flags.BoolVar(&options.noConsole, "no-console", false, "Show non-console output (with buildkit only)")
+	flags.SetAnnotation("no-console", "experimental", nil)
+	flags.SetAnnotation("no-console", "version", []string{"1.38"})
 	return cmd
 }
 

cli/command/image/build_buildkit.go

@@ -212,23 +212,15 @@ func runBuildBuildKit(dockerCli command.Cli, options buildOptions) error {
 		ssArr := []*client.SolveStatus{}
 
 		displayStatus := func(displayCh chan *client.SolveStatus) {
-			if c, err := console.ConsoleFromFile(os.Stderr); err == nil {
+			var c console.Console
-				// not using shared context to not disrupt display but let is finish reporting errors
+			out := os.Stderr
-				eg.Go(func() error {
+			if cons, err := console.ConsoleFromFile(out); err == nil && !options.noConsole {
-					return progressui.DisplaySolveStatus(context.TODO(), c, displayCh)
+				c = cons
-				})
-			} else {
-				// read from t.displayCh and send json to Stderr
-				eg.Go(func() error {
-					enc := json.NewEncoder(os.Stderr)
-					for ss := range displayCh {
-						if err := enc.Encode(ss); err != nil {
-							return err
-						}
-					}
-					return nil
-				})
 			}
+			// not using shared context to not disrupt display but let is finish reporting errors
+			eg.Go(func() error {
+				return progressui.DisplaySolveStatus(context.TODO(), c, out, displayCh)
+			})
 		}
 
 		if options.quiet {

vendor.conf

@@ -49,7 +49,7 @@ github.com/matttproud/golang_protobuf_extensions v1.0.0
 github.com/Microsoft/go-winio v0.4.6
 github.com/miekg/pkcs11 5f6e0d0dad6f472df908c8e968a98ef00c9224bb
 github.com/mitchellh/mapstructure f3009df150dadf309fdee4a54ed65c124afad715
-github.com/moby/buildkit 43e758232a0ac7d50c6a11413186e16684fc1e4f
+github.com/moby/buildkit 8ebd8cbe691a7047a1206e87564f363567d2f77b
 github.com/morikuni/aec 39771216ff4c63d11f5e604076f9c45e8be1067b
 github.com/Nvveen/Gotty a8b993ba6abdb0e0c12b0125c603323a71c7790c https://github.com/ijc25/Gotty
 github.com/opencontainers/go-digest v1.0.0-rc1
@@ -71,7 +71,7 @@ github.com/sirupsen/logrus v1.0.3
 github.com/spf13/cobra v0.0.3
 github.com/spf13/pflag v1.0.1
 github.com/theupdateframework/notary v0.6.1
-github.com/tonistiigi/fsutil dc68c74458923f357474a9178bd198aa3ed11a5f
+github.com/tonistiigi/fsutil 8839685ae8c3c8bd67d0ce28e9b3157b23c1c7a5
 github.com/xeipuuv/gojsonpointer e0fe6f68307607d540ed8eac07a342c33fa1b54a
 github.com/xeipuuv/gojsonreference e02fc20de94c78484cd5ffb007f8af96be030a45
 github.com/xeipuuv/gojsonschema 93e72a773fade158921402d6a24c819b48aba29d

vendor/github.com/moby/buildkit/client/client.go

@@ -1,6 +1,7 @@
 package client
 
 import (
+	"context"
 	"crypto/tls"
 	"crypto/x509"
 	"io/ioutil"
@@ -24,7 +25,6 @@ type ClientOpt interface{}
 // New returns a new buildkit client. Address can be empty for the system-default address.
 func New(address string, opts ...ClientOpt) (*Client, error) {
 	gopts := []grpc.DialOption{
-		grpc.WithTimeout(30 * time.Second),
 		grpc.WithDialer(dialer),
 		grpc.FailOnNonTempDialError(true),
 	}
@@ -53,7 +53,11 @@ func New(address string, opts ...ClientOpt) (*Client, error) {
 	if address == "" {
 		address = appdefaults.Address
 	}
-	conn, err := grpc.Dial(address, gopts...)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	conn, err := grpc.DialContext(ctx, address, gopts...)
 	if err != nil {
 		return nil, errors.Wrapf(err, "failed to dial %q . make sure buildkitd is running", address)
 	}

vendor/github.com/moby/buildkit/client/llb/source.go

@@ -210,6 +210,9 @@ func Local(name string, opts ...LocalOption) State {
 	if gi.IncludePatterns != "" {
 		attrs[pb.AttrIncludePatterns] = gi.IncludePatterns
 	}
+	if gi.FollowPaths != "" {
+		attrs[pb.AttrFollowPaths] = gi.FollowPaths
+	}
 	if gi.ExcludePatterns != "" {
 		attrs[pb.AttrExcludePatterns] = gi.ExcludePatterns
 	}
@@ -248,6 +251,17 @@ func IncludePatterns(p []string) LocalOption {
 	})
 }
 
+func FollowPaths(p []string) LocalOption {
+	return localOptionFunc(func(li *LocalInfo) {
+		if len(p) == 0 {
+			li.FollowPaths = ""
+			return
+		}
+		dt, _ := json.Marshal(p) // empty on error
+		li.FollowPaths = string(dt)
+	})
+}
+
 func ExcludePatterns(p []string) LocalOption {
 	return localOptionFunc(func(li *LocalInfo) {
 		if len(p) == 0 {
@@ -270,6 +284,7 @@ type LocalInfo struct {
 	SessionID       string
 	IncludePatterns string
 	ExcludePatterns string
+	FollowPaths     string
 	SharedKeyHint   string
 }
 

vendor/github.com/moby/buildkit/session/filesync/diffcopy.go

@@ -12,10 +12,11 @@ import (
 	"google.golang.org/grpc"
 )
 
-func sendDiffCopy(stream grpc.Stream, dir string, includes, excludes []string, progress progressCb, _map func(*fsutil.Stat) bool) error {
+func sendDiffCopy(stream grpc.Stream, dir string, includes, excludes, followPaths []string, progress progressCb, _map func(*fsutil.Stat) bool) error {
 	return fsutil.Send(stream.Context(), stream, dir, &fsutil.WalkOpt{
 		ExcludePatterns: excludes,
 		IncludePatterns: includes,
+		FollowPaths:     followPaths,
 		Map:             _map,
 	}, progress)
 }

vendor/github.com/moby/buildkit/session/filesync/filesync.go

@@ -18,6 +18,7 @@ const (
 	keyOverrideExcludes = "override-excludes"
 	keyIncludePatterns  = "include-patterns"
 	keyExcludePatterns  = "exclude-patterns"
+	keyFollowPaths      = "followpaths"
 	keyDirName          = "dir-name"
 )
 
@@ -87,6 +88,8 @@ func (sp *fsSyncProvider) handle(method string, stream grpc.ServerStream) (retEr
 	}
 	includes := opts[keyIncludePatterns]
 
+	followPaths := opts[keyFollowPaths]
+
 	var progress progressCb
 	if sp.p != nil {
 		progress = sp.p
@@ -98,7 +101,7 @@ func (sp *fsSyncProvider) handle(method string, stream grpc.ServerStream) (retEr
 		doneCh = sp.doneCh
 		sp.doneCh = nil
 	}
-	err := pr.sendFn(stream, dir.Dir, includes, excludes, progress, dir.Map)
+	err := pr.sendFn(stream, dir.Dir, includes, excludes, followPaths, progress, dir.Map)
 	if doneCh != nil {
 		if err != nil {
 			doneCh <- err
@@ -117,7 +120,7 @@ type progressCb func(int, bool)
 
 type protocol struct {
 	name   string
-	sendFn func(stream grpc.Stream, srcDir string, includes, excludes []string, progress progressCb, _map func(*fsutil.Stat) bool) error
+	sendFn func(stream grpc.Stream, srcDir string, includes, excludes, followPaths []string, progress progressCb, _map func(*fsutil.Stat) bool) error
 	recvFn func(stream grpc.Stream, destDir string, cu CacheUpdater, progress progressCb) error
 }
 
@@ -142,6 +145,7 @@ type FSSendRequestOpt struct {
 	Name             string
 	IncludePatterns  []string
 	ExcludePatterns  []string
+	FollowPaths      []string
 	OverrideExcludes bool // deprecated: this is used by docker/cli for automatically loading .dockerignore from the directory
 	DestDir          string
 	CacheUpdater     CacheUpdater
@@ -181,6 +185,10 @@ func FSSync(ctx context.Context, c session.Caller, opt FSSendRequestOpt) error {
 		opts[keyExcludePatterns] = opt.ExcludePatterns
 	}
 
+	if opt.FollowPaths != nil {
+		opts[keyFollowPaths] = opt.FollowPaths
+	}
+
 	opts[keyDirName] = []string{opt.Name}
 
 	ctx, cancel := context.WithCancel(ctx)
@@ -261,7 +269,7 @@ func CopyToCaller(ctx context.Context, srcPath string, c session.Caller, progres
 		return err
 	}
 
-	return sendDiffCopy(cc, srcPath, nil, nil, progress, nil)
+	return sendDiffCopy(cc, srcPath, nil, nil, nil, progress, nil)
 }
 
 func CopyFileWriter(ctx context.Context, c session.Caller) (io.WriteCloser, error) {

vendor/github.com/moby/buildkit/solver/pb/attr.go

@@ -4,6 +4,7 @@ const AttrKeepGitDir = "git.keepgitdir"
 const AttrFullRemoteURL = "git.fullurl"
 const AttrLocalSessionID = "local.session"
 const AttrIncludePatterns = "local.includepattern"
+const AttrFollowPaths = "local.followpaths"
 const AttrExcludePatterns = "local.excludepatterns"
 const AttrSharedKeyHint = "local.sharedkeyhint"
 const AttrLLBDefinitionFilename = "llbbuild.filename"

vendor/github.com/moby/buildkit/util/progress/progressui/display.go

@@ -1,6 +1,7 @@
 package progressui
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"io"
@@ -15,17 +16,21 @@ import (
 	"golang.org/x/time/rate"
 )
 
-func DisplaySolveStatus(ctx context.Context, c console.Console, ch chan *client.SolveStatus) error {
+func DisplaySolveStatus(ctx context.Context, c console.Console, w io.Writer, ch chan *client.SolveStatus) error {
-	disp := &display{c: c}
 
-	t := newTrace()
+	modeConsole := c != nil
+
+	disp := &display{c: c}
+	printer := &textMux{w: w}
+
+	t := newTrace(w)
+
+	var done bool
 	ticker := time.NewTicker(100 * time.Millisecond)
 	defer ticker.Stop()
 
 	displayLimiter := rate.NewLimiter(rate.Every(70*time.Millisecond), 1)
 
-	var done bool
-
 	for {
 		select {
 		case <-ctx.Done():
@@ -39,12 +44,21 @@ func DisplaySolveStatus(ctx context.Context, c console.Console, ch chan *client.
 			}
 		}
 
-		if done {
+		if modeConsole {
-			disp.print(t.displayInfo(), true)
+			if done {
-			t.printErrorLogs(c)
+				disp.print(t.displayInfo(), true)
-			return nil
+				t.printErrorLogs(c)
-		} else if displayLimiter.Allow() {
+				return nil
-			disp.print(t.displayInfo(), false)
+			} else if displayLimiter.Allow() {
+				disp.print(t.displayInfo(), false)
+			}
+		} else {
+			if done || displayLimiter.Allow() {
+				printer.print(t)
+				if done {
+					return nil
+				}
+			}
 		}
 	}
 }
@@ -66,37 +80,108 @@ type job struct {
 }
 
 type trace struct {
+	w             io.Writer
 	localTimeDiff time.Duration
 	vertexes      []*vertex
 	byDigest      map[digest.Digest]*vertex
+	nextIndex     int
+	updates       map[digest.Digest]struct{}
 }
 
 type vertex struct {
 	*client.Vertex
 	statuses []*status
 	byID     map[string]*status
-	logs     []*client.VertexLog
 	indent   string
+	index    int
+
+	logs          [][]byte
+	logsPartial   bool
+	logsOffset    int
+	prev          *client.Vertex
+	events        []string
+	lastBlockTime *time.Time
+	count         int
+	statusUpdates map[string]struct{}
+}
+
+func (v *vertex) update(c int) {
+	if v.count == 0 {
+		now := time.Now()
+		v.lastBlockTime = &now
+	}
+	v.count += c
 }
 
 type status struct {
 	*client.VertexStatus
 }
 
-func newTrace() *trace {
+func newTrace(w io.Writer) *trace {
 	return &trace{
 		byDigest: make(map[digest.Digest]*vertex),
+		updates:  make(map[digest.Digest]struct{}),
+		w:        w,
 	}
 }
 
+func (t *trace) triggerVertexEvent(v *client.Vertex) {
+	if v.Started == nil {
+		return
+	}
+
+	var old client.Vertex
+	vtx := t.byDigest[v.Digest]
+	if v := vtx.prev; v != nil {
+		old = *v
+	}
+
+	var ev []string
+	if v.Digest != old.Digest {
+		ev = append(ev, fmt.Sprintf("%13s %s", "digest:", v.Digest))
+	}
+	if v.Name != old.Name {
+		ev = append(ev, fmt.Sprintf("%13s %q", "name:", v.Name))
+	}
+	if v.Started != old.Started {
+		if v.Started != nil && old.Started == nil || !v.Started.Equal(*old.Started) {
+			ev = append(ev, fmt.Sprintf("%13s %v", "started:", v.Started))
+		}
+	}
+	if v.Completed != old.Completed && v.Completed != nil {
+		ev = append(ev, fmt.Sprintf("%13s %v", "completed:", v.Completed))
+		if v.Started != nil {
+			ev = append(ev, fmt.Sprintf("%13s %v", "duration:", v.Completed.Sub(*v.Started)))
+		}
+	}
+	if v.Cached != old.Cached {
+		ev = append(ev, fmt.Sprintf("%13s %v", "cached:", v.Cached))
+	}
+	if v.Error != old.Error {
+		ev = append(ev, fmt.Sprintf("%13s %q", "error:", v.Error))
+	}
+
+	if len(ev) > 0 {
+		vtx.events = append(vtx.events, ev...)
+		vtx.update(len(ev))
+		t.updates[v.Digest] = struct{}{}
+	}
+
+	t.byDigest[v.Digest].prev = v
+}
+
 func (t *trace) update(s *client.SolveStatus) {
 	for _, v := range s.Vertexes {
 		prev, ok := t.byDigest[v.Digest]
 		if !ok {
+			t.nextIndex++
 			t.byDigest[v.Digest] = &vertex{
-				byID: make(map[string]*status),
+				byID:          make(map[string]*status),
+				statusUpdates: make(map[string]struct{}),
+				index:         t.nextIndex,
 			}
 		}
+		t.triggerVertexEvent(v)
 		if v.Started != nil && (prev == nil || prev.Started == nil) {
 			if t.localTimeDiff == 0 {
 				t.localTimeDiff = time.Since(*v.Started)
@@ -118,13 +203,29 @@ func (t *trace) update(s *client.SolveStatus) {
 			v.statuses = append(v.statuses, v.byID[s.ID])
 		}
 		v.byID[s.ID].VertexStatus = s
+		v.statusUpdates[s.ID] = struct{}{}
+		t.updates[v.Digest] = struct{}{}
+		v.update(1)
 	}
 	for _, l := range s.Logs {
 		v, ok := t.byDigest[l.Vertex]
 		if !ok {
 			continue // shouldn't happen
 		}
-		v.logs = append(v.logs, l)
+		complete := split(l.Data, byte('\n'), func(dt []byte) {
+			if v.logsPartial && len(v.logs) != 0 {
+				v.logs[len(v.logs)-1] = append(v.logs[len(v.logs)-1], dt...)
+			} else {
+				ts := time.Duration(0)
+				if v.Started != nil {
+					ts = l.Timestamp.Sub(*v.Started)
+				}
+				v.logs = append(v.logs, []byte(fmt.Sprintf("#%d %s %s", v.index, fmt.Sprintf("%#.4g", ts.Seconds())[:5], dt)))
+			}
+		})
+		v.logsPartial = !complete
+		t.updates[v.Digest] = struct{}{}
+		v.update(1)
 	}
 }
 
@@ -134,12 +235,8 @@ func (t *trace) printErrorLogs(f io.Writer) {
 			fmt.Fprintln(f, "------")
 			fmt.Fprintf(f, " > %s:\n", v.Name)
 			for _, l := range v.logs {
-				switch l.Stream {
+				f.Write(l)
-				case 1:
+				fmt.Fprintln(f)
-					f.Write(l.Data)
-				case 2:
-					f.Write(l.Data)
-				}
 			}
 			fmt.Fprintln(f, "------")
 		}
@@ -196,6 +293,24 @@ func (t *trace) displayInfo() (d displayInfo) {
 	return d
 }
 
+func split(dt []byte, sep byte, fn func([]byte)) bool {
+	if len(dt) == 0 {
+		return false
+	}
+	for {
+		if len(dt) == 0 {
+			return true
+		}
+		idx := bytes.IndexByte(dt, sep)
+		if idx == -1 {
+			fn(dt)
+			return false
+		}
+		fn(dt[:idx])
+		dt = dt[idx+1:]
+	}
+}
+
 func addTime(tm *time.Time, d time.Duration) *time.Time {
 	if tm == nil {
 		return nil

vendor/github.com/moby/buildkit/util/progress/progressui/printer.go

@@ -0,0 +1,214 @@
+package progressui
+
+import (
+	"fmt"
+	"io"
+	"time"
+
+	digest "github.com/opencontainers/go-digest"
+	"github.com/tonistiigi/units"
+)
+
+const antiFlicker = 5 * time.Second
+const maxDelay = 10 * time.Second
+
+type textMux struct {
+	w       io.Writer
+	current digest.Digest
+}
+
+func (p *textMux) printVtx(t *trace, dgst digest.Digest) {
+	v, ok := t.byDigest[dgst]
+	if !ok {
+		return
+	}
+
+	if dgst != p.current {
+		if p.current != "" {
+			old := t.byDigest[p.current]
+			if old.logsPartial {
+				fmt.Fprintln(p.w, "")
+			}
+			old.logsOffset = 0
+			old.count = 0
+			fmt.Fprintf(p.w, "#%d ...\n", v.index)
+		}
+
+		fmt.Fprintf(p.w, "\n#%d %s\n", v.index, limitString(v.Name, 72))
+	}
+
+	if len(v.events) != 0 {
+		v.logsOffset = 0
+	}
+	for _, ev := range v.events {
+		fmt.Fprintf(p.w, "#%d %s\n", v.index, ev)
+	}
+	v.events = v.events[:0]
+
+	for _, s := range v.statuses {
+		if _, ok := v.statusUpdates[s.ID]; ok {
+			var bytes string
+			if s.Total != 0 {
+				bytes = fmt.Sprintf(" %.2f / %.2f", units.Bytes(s.Current), units.Bytes(s.Total))
+			} else if s.Current != 0 {
+				bytes = fmt.Sprintf(" %.2f", units.Bytes(s.Current))
+			}
+			var tm string
+			endTime := s.Timestamp
+			if s.Completed != nil {
+				endTime = *s.Completed
+			}
+			if s.Started != nil {
+				diff := endTime.Sub(*s.Started).Seconds()
+				if diff > 0.01 {
+					tm = fmt.Sprintf(" %.1fs", diff)
+				}
+			}
+			if s.Completed != nil {
+				tm += " done"
+			}
+			fmt.Fprintf(p.w, "#%d %s%s%s\n", v.index, s.ID, bytes, tm)
+		}
+	}
+	v.statusUpdates = map[string]struct{}{}
+
+	for i, l := range v.logs {
+		if i == 0 {
+			l = l[v.logsOffset:]
+		}
+		fmt.Fprintf(p.w, "%s", []byte(l))
+		if i != len(v.logs)-1 || !v.logsPartial {
+			fmt.Fprintln(p.w, "")
+		}
+	}
+
+	if len(v.logs) > 0 {
+		if v.logsPartial {
+			v.logs = v.logs[len(v.logs)-1:]
+			v.logsOffset = len(v.logs[0])
+		} else {
+			v.logs = nil
+			v.logsOffset = 0
+		}
+	}
+
+	p.current = dgst
+
+	if v.Completed != nil {
+		p.current = ""
+		v.count = 0
+		fmt.Fprintf(p.w, "\n")
+	}
+
+	delete(t.updates, dgst)
+}
+
+func (p *textMux) print(t *trace) {
+
+	completed := map[digest.Digest]struct{}{}
+	rest := map[digest.Digest]struct{}{}
+
+	for dgst := range t.updates {
+		v, ok := t.byDigest[dgst]
+		if !ok {
+			continue
+		}
+		if v.Vertex.Completed != nil {
+			completed[dgst] = struct{}{}
+		} else {
+			rest[dgst] = struct{}{}
+		}
+	}
+
+	current := p.current
+
+	// items that have completed need to be printed first
+	if _, ok := completed[current]; ok {
+		p.printVtx(t, current)
+	}
+
+	for dgst := range completed {
+		if dgst != current {
+			p.printVtx(t, dgst)
+		}
+	}
+
+	if len(rest) == 0 {
+		if current != "" {
+			if v := t.byDigest[current]; v.Started != nil && v.Completed == nil {
+				return
+			}
+		}
+		// make any open vertex active
+		for dgst, v := range t.byDigest {
+			if v.Started != nil && v.Completed == nil {
+				p.printVtx(t, dgst)
+				return
+			}
+		}
+		return
+	}
+
+	// now print the active one
+	if _, ok := rest[current]; ok {
+		p.printVtx(t, current)
+	}
+
+	stats := map[digest.Digest]*vtxStat{}
+	now := time.Now()
+	sum := 0.0
+	var max digest.Digest
+	if current != "" {
+		rest[current] = struct{}{}
+	}
+	for dgst := range rest {
+		v, ok := t.byDigest[dgst]
+		if !ok {
+			continue
+		}
+		tm := now.Sub(*v.lastBlockTime)
+		speed := float64(v.count) / tm.Seconds()
+		overLimit := tm > maxDelay && dgst != current
+		stats[dgst] = &vtxStat{blockTime: tm, speed: speed, overLimit: overLimit}
+		sum += speed
+		if overLimit || max == "" || stats[max].speed < speed {
+			max = dgst
+		}
+	}
+	for dgst := range stats {
+		stats[dgst].share = stats[dgst].speed / sum
+	}
+
+	if _, ok := completed[current]; ok || current == "" {
+		p.printVtx(t, max)
+		return
+	}
+
+	// show items that were hidden
+	for dgst := range rest {
+		if stats[dgst].overLimit {
+			p.printVtx(t, dgst)
+			return
+		}
+	}
+
+	// fair split between vertexes
+	if 1.0/(1.0-stats[current].share)*antiFlicker.Seconds() < stats[current].blockTime.Seconds() {
+		p.printVtx(t, max)
+		return
+	}
+}
+
+type vtxStat struct {
+	blockTime time.Duration
+	speed     float64
+	share     float64
+	overLimit bool
+}
+
+func limitString(s string, l int) string {
+	if len(s) > l {
+		return s[:l] + "..."
+	}
+	return s
+}

vendor/github.com/moby/buildkit/util/system/seccomp_linux.go

@@ -1,4 +1,4 @@
-// +build linux
+// +build linux,seccomp
 
 package system
 

vendor/github.com/moby/buildkit/util/system/seccomp_nolinux.go

@@ -1,4 +1,4 @@
-// +build !linux
+// +build !linux,seccomp
 
 package system
 

vendor/github.com/moby/buildkit/util/system/seccomp_noseccomp.go

@@ -0,0 +1,7 @@
+// +build !seccomp
+
+package system
+
+func SeccompSupported() bool {
+	return false
+}

vendor/github.com/moby/buildkit/vendor.conf

@@ -6,7 +6,7 @@ github.com/davecgh/go-spew v1.1.0
 github.com/pmezard/go-difflib v1.0.0
 golang.org/x/sys 314a259e304ff91bd6985da2a7149bbf91237993
 
-github.com/containerd/containerd e1428ef05460da40720d622c803262e6fc8d3477
+github.com/containerd/containerd 63522d9eaa5a0443d225642c4b6f4f5fdedf932b
 github.com/containerd/typeurl f6943554a7e7e88b3c14aad190bf05932da84788
 golang.org/x/sync 450f422ab23cf9881c94e2db30cac0eb1b7cf80c
 github.com/sirupsen/logrus v1.0.0
@@ -23,7 +23,7 @@ github.com/Microsoft/go-winio v0.4.7
 github.com/containerd/fifo 3d5202aec260678c48179c56f40e6f38a095738c
 github.com/opencontainers/runtime-spec v1.0.1
 github.com/containerd/go-runc f271fa2021de855d4d918dbef83c5fe19db1bdd5
-github.com/containerd/console cb7008ab3d8359b78c5f464cb7cf160107ad5925
+github.com/containerd/console 9290d21dc56074581f619579c43d970b4514bc08
 google.golang.org/genproto d80a6e20e776b0b17a324d0ba1ab50a39c8e8944
 golang.org/x/text 19e51611da83d6be54ddafce4a4af510cb3e9ea4
 github.com/docker/go-events 9461782956ad83b30282bf90e31fa6a70c255ba9
@@ -36,11 +36,10 @@ github.com/docker/go-units v0.3.1
 github.com/google/shlex 6f45313302b9c56850fc17f99e40caebce98c716
 golang.org/x/time f51c12702a4d776e4c1fa9b0fabab841babae631
 
-github.com/BurntSushi/locker a6e239ea1c69bff1cfdb20c4b73dadf52f784b6a
 github.com/docker/docker 71cd53e4a197b303c6ba086bd584ffd67a884281
 github.com/pkg/profile 5b67d428864e92711fcbd2f8629456121a56d91f
 
-github.com/tonistiigi/fsutil dc68c74458923f357474a9178bd198aa3ed11a5f
+github.com/tonistiigi/fsutil 8839685ae8c3c8bd67d0ce28e9b3157b23c1c7a5
 github.com/hashicorp/go-immutable-radix 826af9ccf0feeee615d546d69b11f8e98da8c8f1 git://github.com/tonistiigi/go-immutable-radix.git
 github.com/hashicorp/golang-lru a0d98a5f288019575c6d1f4bb1573fef2d1fcdc4
 github.com/mitchellh/hashstructure 2bca23e0e452137f789efbc8610126fd8b94f73b

vendor/github.com/tonistiigi/fsutil/chtimes_nolinux.go

@@ -9,5 +9,12 @@ import (
 
 func chtimes(path string, un int64) error {
 	mtime := time.Unix(0, un)
+	fi, err := os.Lstat(path)
+	if err != nil {
+		return err
+	}
+	if fi.Mode()&os.ModeSymlink != 0 {
+		return nil
+	}
 	return os.Chtimes(path, mtime, mtime)
 }

vendor/github.com/tonistiigi/fsutil/diff.go

@@ -1,10 +1,9 @@
 package fsutil
 
 import (
+	"context"
 	"hash"
 	"os"
-
-	"golang.org/x/net/context"
 )
 
 type walkerFn func(ctx context.Context, pathC chan<- *currentPath) error

vendor/github.com/tonistiigi/fsutil/diff_containerd.go

@@ -1,10 +1,10 @@
 package fsutil
 
 import (
+	"context"
 	"os"
 	"strings"
 
-	"golang.org/x/net/context"
 	"golang.org/x/sync/errgroup"
 )
 

vendor/github.com/tonistiigi/fsutil/diskwriter.go

@@ -1,6 +1,7 @@
 package fsutil
 
 import (
+	"context"
 	"hash"
 	"io"
 	"os"
@@ -9,9 +10,8 @@ import (
 	"sync"
 	"time"
 
-	digest "github.com/opencontainers/go-digest"
+	"github.com/opencontainers/go-digest"
 	"github.com/pkg/errors"
-	"golang.org/x/net/context"
 	"golang.org/x/sync/errgroup"
 )
 
@@ -80,9 +80,7 @@ func (dw *DiskWriter) HandleChange(kind ChangeKind, p string, fi os.FileInfo, er
 		}
 	}()
 
-	p = filepath.FromSlash(p)
+	destPath := filepath.Join(dw.dest, filepath.FromSlash(p))
-
-	destPath := filepath.Join(dw.dest, p)
 
 	if kind == ChangeKindDelete {
 		// todo: no need to validate if diff is trusted but is it always?
@@ -102,8 +100,10 @@ func (dw *DiskWriter) HandleChange(kind ChangeKind, p string, fi os.FileInfo, er
 		return errors.Errorf("%s invalid change without stat information", p)
 	}
 
+	statCopy := *stat
+
 	if dw.filter != nil {
-		if ok := dw.filter(stat); !ok {
+		if ok := dw.filter(&statCopy); !ok {
 			return nil
 		}
 	}
@@ -122,7 +122,7 @@ func (dw *DiskWriter) HandleChange(kind ChangeKind, p string, fi os.FileInfo, er
 	}
 
 	if oldFi != nil && fi.IsDir() && oldFi.IsDir() {
-		if err := rewriteMetadata(destPath, stat); err != nil {
+		if err := rewriteMetadata(destPath, &statCopy); err != nil {
 			return errors.Wrapf(err, "error setting dir metadata for %s", destPath)
 		}
 		return nil
@@ -141,16 +141,16 @@ func (dw *DiskWriter) HandleChange(kind ChangeKind, p string, fi os.FileInfo, er
 			return errors.Wrapf(err, "failed to create dir %s", newPath)
 		}
 	case fi.Mode()&os.ModeDevice != 0 || fi.Mode()&os.ModeNamedPipe != 0:
-		if err := handleTarTypeBlockCharFifo(newPath, stat); err != nil {
+		if err := handleTarTypeBlockCharFifo(newPath, &statCopy); err != nil {
 			return errors.Wrapf(err, "failed to create device %s", newPath)
 		}
 	case fi.Mode()&os.ModeSymlink != 0:
-		if err := os.Symlink(stat.Linkname, newPath); err != nil {
+		if err := os.Symlink(statCopy.Linkname, newPath); err != nil {
 			return errors.Wrapf(err, "failed to symlink %s", newPath)
 		}
-	case stat.Linkname != "":
+	case statCopy.Linkname != "":
-		if err := os.Link(filepath.Join(dw.dest, stat.Linkname), newPath); err != nil {
+		if err := os.Link(filepath.Join(dw.dest, statCopy.Linkname), newPath); err != nil {
-			return errors.Wrapf(err, "failed to link %s to %s", newPath, stat.Linkname)
+			return errors.Wrapf(err, "failed to link %s to %s", newPath, statCopy.Linkname)
 		}
 	default:
 		isRegularFile = true
@@ -170,7 +170,7 @@ func (dw *DiskWriter) HandleChange(kind ChangeKind, p string, fi os.FileInfo, er
 		}
 	}
 
-	if err := rewriteMetadata(newPath, stat); err != nil {
+	if err := rewriteMetadata(newPath, &statCopy); err != nil {
 		return errors.Wrapf(err, "error setting metadata for %s", newPath)
 	}
 
@@ -272,14 +272,27 @@ func (hw *hashedWriter) Digest() digest.Digest {
 }
 
 type lazyFileWriter struct {
-	dest string
+	dest     string
-	ctx  context.Context
+	ctx      context.Context
-	f    *os.File
+	f        *os.File
+	fileMode *os.FileMode
 }
 
 func (lfw *lazyFileWriter) Write(dt []byte) (int, error) {
 	if lfw.f == nil {
 		file, err := os.OpenFile(lfw.dest, os.O_WRONLY, 0) //todo: windows
+		if os.IsPermission(err) {
+			// retry after chmod
+			fi, er := os.Stat(lfw.dest)
+			if er == nil {
+				mode := fi.Mode()
+				lfw.fileMode = &mode
+				er = os.Chmod(lfw.dest, mode|0222)
+				if er == nil {
+					file, err = os.OpenFile(lfw.dest, os.O_WRONLY, 0)
+				}
+			}
+		}
 		if err != nil {
 			return 0, errors.Wrapf(err, "failed to open %s", lfw.dest)
 		}
@@ -289,10 +302,14 @@ func (lfw *lazyFileWriter) Write(dt []byte) (int, error) {
 }
 
 func (lfw *lazyFileWriter) Close() error {
+	var err error
 	if lfw.f != nil {
-		return lfw.f.Close()
+		err = lfw.f.Close()
 	}
-	return nil
+	if err == nil && lfw.fileMode != nil {
+		err = os.Chmod(lfw.dest, *lfw.fileMode)
+	}
+	return err
 }
 
 func mkdev(major int64, minor int64) uint32 {

vendor/github.com/tonistiigi/fsutil/followlinks.go

@@ -0,0 +1,150 @@
+package fsutil
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"runtime"
+	"sort"
+	strings "strings"
+
+	"github.com/pkg/errors"
+)
+
+func FollowLinks(root string, paths []string) ([]string, error) {
+	r := &symlinkResolver{root: root, resolved: map[string]struct{}{}}
+	for _, p := range paths {
+		if err := r.append(p); err != nil {
+			return nil, err
+		}
+	}
+	res := make([]string, 0, len(r.resolved))
+	for r := range r.resolved {
+		res = append(res, r)
+	}
+	sort.Strings(res)
+	return dedupePaths(res), nil
+}
+
+type symlinkResolver struct {
+	root     string
+	resolved map[string]struct{}
+}
+
+func (r *symlinkResolver) append(p string) error {
+	p = filepath.Join(".", p)
+	current := "."
+	for {
+		parts := strings.SplitN(p, string(filepath.Separator), 2)
+		current = filepath.Join(current, parts[0])
+
+		targets, err := r.readSymlink(current, true)
+		if err != nil {
+			return err
+		}
+
+		p = ""
+		if len(parts) == 2 {
+			p = parts[1]
+		}
+
+		if p == "" || targets != nil {
+			if _, ok := r.resolved[current]; ok {
+				return nil
+			}
+		}
+
+		if targets != nil {
+			r.resolved[current] = struct{}{}
+			for _, target := range targets {
+				if err := r.append(filepath.Join(target, p)); err != nil {
+					return err
+				}
+			}
+			return nil
+		}
+
+		if p == "" {
+			r.resolved[current] = struct{}{}
+			return nil
+		}
+	}
+}
+
+func (r *symlinkResolver) readSymlink(p string, allowWildcard bool) ([]string, error) {
+	realPath := filepath.Join(r.root, p)
+	base := filepath.Base(p)
+	if allowWildcard && containsWildcards(base) {
+		fis, err := ioutil.ReadDir(filepath.Dir(realPath))
+		if err != nil {
+			if os.IsNotExist(err) {
+				return nil, nil
+			}
+			return nil, errors.Wrapf(err, "failed to read dir %s", filepath.Dir(realPath))
+		}
+		var out []string
+		for _, f := range fis {
+			if ok, _ := filepath.Match(base, f.Name()); ok {
+				res, err := r.readSymlink(filepath.Join(filepath.Dir(p), f.Name()), false)
+				if err != nil {
+					return nil, err
+				}
+				out = append(out, res...)
+			}
+		}
+		return out, nil
+	}
+
+	fi, err := os.Lstat(realPath)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, nil
+		}
+		return nil, errors.Wrapf(err, "failed to lstat %s", realPath)
+	}
+	if fi.Mode()&os.ModeSymlink == 0 {
+		return nil, nil
+	}
+	link, err := os.Readlink(realPath)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to readlink %s", realPath)
+	}
+	link = filepath.Clean(link)
+	if filepath.IsAbs(link) {
+		return []string{link}, nil
+	}
+	return []string{
+		filepath.Join(string(filepath.Separator), filepath.Join(filepath.Dir(p), link)),
+	}, nil
+}
+
+func containsWildcards(name string) bool {
+	isWindows := runtime.GOOS == "windows"
+	for i := 0; i < len(name); i++ {
+		ch := name[i]
+		if ch == '\\' && !isWindows {
+			i++
+		} else if ch == '*' || ch == '?' || ch == '[' {
+			return true
+		}
+	}
+	return false
+}
+
+// dedupePaths expects input as a sorted list
+func dedupePaths(in []string) []string {
+	out := make([]string, 0, len(in))
+	var last string
+	for _, s := range in {
+		// if one of the paths is root there is no filter
+		if s == "." {
+			return nil
+		}
+		if strings.HasPrefix(s, last+string(filepath.Separator)) {
+			continue
+		}
+		out = append(out, s)
+		last = s
+	}
+	return out
+}

vendor/github.com/tonistiigi/fsutil/receive.go

@@ -1,12 +1,12 @@
 package fsutil
 
 import (
+	"context"
 	"io"
 	"os"
 	"sync"
 
 	"github.com/pkg/errors"
-	"golang.org/x/net/context"
 	"golang.org/x/sync/errgroup"
 )
 

vendor/github.com/tonistiigi/fsutil/send.go

@@ -1,13 +1,13 @@
 package fsutil
 
 import (
+	"context"
 	"io"
 	"os"
 	"path/filepath"
 	"sync"
 
 	"github.com/pkg/errors"
-	"golang.org/x/net/context"
 	"golang.org/x/sync/errgroup"
 )
 

vendor/github.com/tonistiigi/fsutil/walker.go

@@ -1,6 +1,7 @@
 package fsutil
 
 import (
+	"context"
 	"os"
 	"path/filepath"
 	"runtime"
@@ -9,13 +10,15 @@ import (
 
 	"github.com/docker/docker/pkg/fileutils"
 	"github.com/pkg/errors"
-	"golang.org/x/net/context"
 )
 
 type WalkOpt struct {
 	IncludePatterns []string
 	ExcludePatterns []string
-	Map             func(*Stat) bool
+	// FollowPaths contains symlinks that are resolved into include patterns
+	// before performing the fs walk
+	FollowPaths []string
+	Map         func(*Stat) bool
 }
 
 func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) error {
@@ -39,8 +42,25 @@ func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) err
 		}
 	}
 
+	var includePatterns []string
+	if opt != nil && opt.IncludePatterns != nil {
+		includePatterns = make([]string, len(opt.IncludePatterns))
+		for k := range opt.IncludePatterns {
+			includePatterns[k] = filepath.Clean(opt.IncludePatterns[k])
+		}
+	}
+	if opt != nil && opt.FollowPaths != nil {
+		targets, err := FollowLinks(p, opt.FollowPaths)
+		if err != nil {
+			return err
+		}
+		if targets != nil {
+			includePatterns = append(includePatterns, targets...)
+			includePatterns = dedupePaths(includePatterns)
+		}
+	}
+
 	var lastIncludedDir string
-	var includePatternPrefixes []string
 
 	seenFiles := make(map[uint64]string)
 	return filepath.Walk(root, func(path string, fi os.FileInfo, err error) (retErr error) {
@@ -66,34 +86,31 @@ func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) err
 		}
 
 		if opt != nil {
-			if opt.IncludePatterns != nil {
+			if includePatterns != nil {
-				if includePatternPrefixes == nil {
+				skip := false
-					includePatternPrefixes = patternPrefixes(opt.IncludePatterns)
-				}
-				matched := false
 				if lastIncludedDir != "" {
 					if strings.HasPrefix(path, lastIncludedDir+string(filepath.Separator)) {
-						matched = true
+						skip = true
 					}
 				}
-				if !matched {
+
-					for _, p := range opt.IncludePatterns {
+				if !skip {
-						if m, _ := filepath.Match(p, path); m {
+					matched := false
+					partial := true
+					for _, p := range includePatterns {
+						if ok, p := matchPrefix(p, path); ok {
 							matched = true
-							break
+							if !p {
+								partial = false
+								break
+							}
 						}
 					}
-					if matched && fi.IsDir() {
+					if !matched {
-						lastIncludedDir = path
-					}
-				}
-				if !matched {
-					if !fi.IsDir() {
 						return nil
-					} else {
+					}
-						if noPossiblePrefixMatch(path, includePatternPrefixes) {
+					if !partial && fi.IsDir() {
-							return filepath.SkipDir
+						lastIncludedDir = path
-						}
 					}
 				}
 			}
@@ -131,13 +148,13 @@ func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) err
 		stat := &Stat{
 			Path:    path,
 			Mode:    uint32(fi.Mode()),
-			Size_:   fi.Size(),
 			ModTime: fi.ModTime().UnixNano(),
 		}
 
 		setUnixOpt(fi, stat, path, seenFiles)
 
 		if !fi.IsDir() {
+			stat.Size_ = fi.Size()
 			if fi.Mode()&os.ModeSymlink != 0 {
 				link, err := os.Readlink(origpath)
 				if err != nil {
@@ -199,29 +216,28 @@ func (s *StatInfo) Sys() interface{} {
 	return s.Stat
 }
 
-func patternPrefixes(patterns []string) []string {
+func matchPrefix(pattern, name string) (bool, bool) {
-	pfxs := make([]string, 0, len(patterns))
+	count := strings.Count(name, string(filepath.Separator))
-	for _, ptrn := range patterns {
+	partial := false
-		idx := strings.IndexFunc(ptrn, func(ch rune) bool {
+	if strings.Count(pattern, string(filepath.Separator)) > count {
-			return ch == '*' || ch == '?' || ch == '[' || ch == '\\'
+		pattern = trimUntilIndex(pattern, string(filepath.Separator), count)
-		})
+		partial = true
-		if idx == -1 {
-			idx = len(ptrn)
-		}
-		pfxs = append(pfxs, ptrn[:idx])
 	}
-	return pfxs
+	m, _ := filepath.Match(pattern, name)
+	return m, partial
 }
 
-func noPossiblePrefixMatch(p string, pfxs []string) bool {
+func trimUntilIndex(str, sep string, count int) string {
-	for _, pfx := range pfxs {
+	s := str
-		chk := p
+	i := 0
-		if len(pfx) < len(p) {
+	c := 0
-			chk = p[:len(pfx)]
+	for {
-		}
+		idx := strings.Index(s, sep)
-		if strings.HasPrefix(pfx, chk) {
+		s = s[idx+len(sep):]
-			return false
+		i += idx + len(sep)
+		c++
+		if c >= count {
+			return str[:i-len(sep)]
 		}
 	}
-	return true
 }