From ed75f6202b163cb3e42c83a33787aabbe24dcbc0 Mon Sep 17 00:00:00 2001 From: Tibor Vass Date: Sat, 9 Jun 2018 01:07:42 +0000 Subject: [PATCH] build: add experimental --no-console flag to support non-tty human-readable output with buildkit Unfortunately, this is for now the only way to see the output of RUN commands when using buildkit. It is equivalent to `DOCKER_BUILDKIT=1 docker build . 2>&1 | cat` Signed-off-by: Tibor Vass --- cli/command/image/build.go | 4 + cli/command/image/build_buildkit.go | 24 +- vendor.conf | 4 +- .../github.com/moby/buildkit/client/client.go | 8 +- .../moby/buildkit/client/llb/source.go | 15 ++ .../buildkit/session/filesync/diffcopy.go | 3 +- .../buildkit/session/filesync/filesync.go | 14 +- .../moby/buildkit/solver/pb/attr.go | 1 + .../util/progress/progressui/display.go | 157 +++++++++++-- .../util/progress/progressui/printer.go | 214 ++++++++++++++++++ .../buildkit/util/system/seccomp_linux.go | 2 +- .../buildkit/util/system/seccomp_nolinux.go | 2 +- .../buildkit/util/system/seccomp_noseccomp.go | 7 + vendor/github.com/moby/buildkit/vendor.conf | 7 +- .../tonistiigi/fsutil/chtimes_nolinux.go | 7 + vendor/github.com/tonistiigi/fsutil/diff.go | 3 +- .../tonistiigi/fsutil/diff_containerd.go | 2 +- .../tonistiigi/fsutil/diskwriter.go | 53 +++-- .../tonistiigi/fsutil/followlinks.go | 150 ++++++++++++ .../github.com/tonistiigi/fsutil/receive.go | 2 +- vendor/github.com/tonistiigi/fsutil/send.go | 2 +- vendor/github.com/tonistiigi/fsutil/walker.go | 104 +++++---- 22 files changed, 667 insertions(+), 118 deletions(-) create mode 100644 vendor/github.com/moby/buildkit/util/progress/progressui/printer.go create mode 100644 vendor/github.com/moby/buildkit/util/system/seccomp_noseccomp.go create mode 100644 vendor/github.com/tonistiigi/fsutil/followlinks.go diff --git a/cli/command/image/build.go b/cli/command/image/build.go index 193a6a2799..e6d108895f 100644 --- a/cli/command/image/build.go +++ b/cli/command/image/build.go @@ -57,6 +57,7 @@ type buildOptions struct { isolation string quiet bool noCache bool + noConsole bool rm bool forceRm bool pull bool @@ -151,6 +152,9 @@ func NewBuildCommand(dockerCli command.Cli) *cobra.Command { flags.SetAnnotation("stream", "experimental", nil) flags.SetAnnotation("stream", "version", []string{"1.31"}) + flags.BoolVar(&options.noConsole, "no-console", false, "Show non-console output (with buildkit only)") + flags.SetAnnotation("no-console", "experimental", nil) + flags.SetAnnotation("no-console", "version", []string{"1.38"}) return cmd } diff --git a/cli/command/image/build_buildkit.go b/cli/command/image/build_buildkit.go index f5d2f35be6..49f55d0bec 100644 --- a/cli/command/image/build_buildkit.go +++ b/cli/command/image/build_buildkit.go @@ -212,23 +212,15 @@ func runBuildBuildKit(dockerCli command.Cli, options buildOptions) error { ssArr := []*client.SolveStatus{} displayStatus := func(displayCh chan *client.SolveStatus) { - if c, err := console.ConsoleFromFile(os.Stderr); err == nil { - // not using shared context to not disrupt display but let is finish reporting errors - eg.Go(func() error { - return progressui.DisplaySolveStatus(context.TODO(), c, displayCh) - }) - } else { - // read from t.displayCh and send json to Stderr - eg.Go(func() error { - enc := json.NewEncoder(os.Stderr) - for ss := range displayCh { - if err := enc.Encode(ss); err != nil { - return err - } - } - return nil - }) + var c console.Console + out := os.Stderr + if cons, err := console.ConsoleFromFile(out); err == nil && !options.noConsole { + c = cons } + // not using shared context to not disrupt display but let is finish reporting errors + eg.Go(func() error { + return progressui.DisplaySolveStatus(context.TODO(), c, out, displayCh) + }) } if options.quiet { diff --git a/vendor.conf b/vendor.conf index a925ba3b54..b3b51dafb6 100755 --- a/vendor.conf +++ b/vendor.conf @@ -49,7 +49,7 @@ github.com/matttproud/golang_protobuf_extensions v1.0.0 github.com/Microsoft/go-winio v0.4.6 github.com/miekg/pkcs11 5f6e0d0dad6f472df908c8e968a98ef00c9224bb github.com/mitchellh/mapstructure f3009df150dadf309fdee4a54ed65c124afad715 -github.com/moby/buildkit 43e758232a0ac7d50c6a11413186e16684fc1e4f +github.com/moby/buildkit 8ebd8cbe691a7047a1206e87564f363567d2f77b github.com/morikuni/aec 39771216ff4c63d11f5e604076f9c45e8be1067b github.com/Nvveen/Gotty a8b993ba6abdb0e0c12b0125c603323a71c7790c https://github.com/ijc25/Gotty github.com/opencontainers/go-digest v1.0.0-rc1 @@ -71,7 +71,7 @@ github.com/sirupsen/logrus v1.0.3 github.com/spf13/cobra v0.0.3 github.com/spf13/pflag v1.0.1 github.com/theupdateframework/notary v0.6.1 -github.com/tonistiigi/fsutil dc68c74458923f357474a9178bd198aa3ed11a5f +github.com/tonistiigi/fsutil 8839685ae8c3c8bd67d0ce28e9b3157b23c1c7a5 github.com/xeipuuv/gojsonpointer e0fe6f68307607d540ed8eac07a342c33fa1b54a github.com/xeipuuv/gojsonreference e02fc20de94c78484cd5ffb007f8af96be030a45 github.com/xeipuuv/gojsonschema 93e72a773fade158921402d6a24c819b48aba29d diff --git a/vendor/github.com/moby/buildkit/client/client.go b/vendor/github.com/moby/buildkit/client/client.go index ca4b34fdd5..c71c00c0e2 100644 --- a/vendor/github.com/moby/buildkit/client/client.go +++ b/vendor/github.com/moby/buildkit/client/client.go @@ -1,6 +1,7 @@ package client import ( + "context" "crypto/tls" "crypto/x509" "io/ioutil" @@ -24,7 +25,6 @@ type ClientOpt interface{} // New returns a new buildkit client. Address can be empty for the system-default address. func New(address string, opts ...ClientOpt) (*Client, error) { gopts := []grpc.DialOption{ - grpc.WithTimeout(30 * time.Second), grpc.WithDialer(dialer), grpc.FailOnNonTempDialError(true), } @@ -53,7 +53,11 @@ func New(address string, opts ...ClientOpt) (*Client, error) { if address == "" { address = appdefaults.Address } - conn, err := grpc.Dial(address, gopts...) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + conn, err := grpc.DialContext(ctx, address, gopts...) if err != nil { return nil, errors.Wrapf(err, "failed to dial %q . make sure buildkitd is running", address) } diff --git a/vendor/github.com/moby/buildkit/client/llb/source.go b/vendor/github.com/moby/buildkit/client/llb/source.go index e7e92abede..5b1bf061bf 100644 --- a/vendor/github.com/moby/buildkit/client/llb/source.go +++ b/vendor/github.com/moby/buildkit/client/llb/source.go @@ -210,6 +210,9 @@ func Local(name string, opts ...LocalOption) State { if gi.IncludePatterns != "" { attrs[pb.AttrIncludePatterns] = gi.IncludePatterns } + if gi.FollowPaths != "" { + attrs[pb.AttrFollowPaths] = gi.FollowPaths + } if gi.ExcludePatterns != "" { attrs[pb.AttrExcludePatterns] = gi.ExcludePatterns } @@ -248,6 +251,17 @@ func IncludePatterns(p []string) LocalOption { }) } +func FollowPaths(p []string) LocalOption { + return localOptionFunc(func(li *LocalInfo) { + if len(p) == 0 { + li.FollowPaths = "" + return + } + dt, _ := json.Marshal(p) // empty on error + li.FollowPaths = string(dt) + }) +} + func ExcludePatterns(p []string) LocalOption { return localOptionFunc(func(li *LocalInfo) { if len(p) == 0 { @@ -270,6 +284,7 @@ type LocalInfo struct { SessionID string IncludePatterns string ExcludePatterns string + FollowPaths string SharedKeyHint string } diff --git a/vendor/github.com/moby/buildkit/session/filesync/diffcopy.go b/vendor/github.com/moby/buildkit/session/filesync/diffcopy.go index 7f8bf3c0df..d0f8e76df3 100644 --- a/vendor/github.com/moby/buildkit/session/filesync/diffcopy.go +++ b/vendor/github.com/moby/buildkit/session/filesync/diffcopy.go @@ -12,10 +12,11 @@ import ( "google.golang.org/grpc" ) -func sendDiffCopy(stream grpc.Stream, dir string, includes, excludes []string, progress progressCb, _map func(*fsutil.Stat) bool) error { +func sendDiffCopy(stream grpc.Stream, dir string, includes, excludes, followPaths []string, progress progressCb, _map func(*fsutil.Stat) bool) error { return fsutil.Send(stream.Context(), stream, dir, &fsutil.WalkOpt{ ExcludePatterns: excludes, IncludePatterns: includes, + FollowPaths: followPaths, Map: _map, }, progress) } diff --git a/vendor/github.com/moby/buildkit/session/filesync/filesync.go b/vendor/github.com/moby/buildkit/session/filesync/filesync.go index 232a696d73..ee2668f06b 100644 --- a/vendor/github.com/moby/buildkit/session/filesync/filesync.go +++ b/vendor/github.com/moby/buildkit/session/filesync/filesync.go @@ -18,6 +18,7 @@ const ( keyOverrideExcludes = "override-excludes" keyIncludePatterns = "include-patterns" keyExcludePatterns = "exclude-patterns" + keyFollowPaths = "followpaths" keyDirName = "dir-name" ) @@ -87,6 +88,8 @@ func (sp *fsSyncProvider) handle(method string, stream grpc.ServerStream) (retEr } includes := opts[keyIncludePatterns] + followPaths := opts[keyFollowPaths] + var progress progressCb if sp.p != nil { progress = sp.p @@ -98,7 +101,7 @@ func (sp *fsSyncProvider) handle(method string, stream grpc.ServerStream) (retEr doneCh = sp.doneCh sp.doneCh = nil } - err := pr.sendFn(stream, dir.Dir, includes, excludes, progress, dir.Map) + err := pr.sendFn(stream, dir.Dir, includes, excludes, followPaths, progress, dir.Map) if doneCh != nil { if err != nil { doneCh <- err @@ -117,7 +120,7 @@ type progressCb func(int, bool) type protocol struct { name string - sendFn func(stream grpc.Stream, srcDir string, includes, excludes []string, progress progressCb, _map func(*fsutil.Stat) bool) error + sendFn func(stream grpc.Stream, srcDir string, includes, excludes, followPaths []string, progress progressCb, _map func(*fsutil.Stat) bool) error recvFn func(stream grpc.Stream, destDir string, cu CacheUpdater, progress progressCb) error } @@ -142,6 +145,7 @@ type FSSendRequestOpt struct { Name string IncludePatterns []string ExcludePatterns []string + FollowPaths []string OverrideExcludes bool // deprecated: this is used by docker/cli for automatically loading .dockerignore from the directory DestDir string CacheUpdater CacheUpdater @@ -181,6 +185,10 @@ func FSSync(ctx context.Context, c session.Caller, opt FSSendRequestOpt) error { opts[keyExcludePatterns] = opt.ExcludePatterns } + if opt.FollowPaths != nil { + opts[keyFollowPaths] = opt.FollowPaths + } + opts[keyDirName] = []string{opt.Name} ctx, cancel := context.WithCancel(ctx) @@ -261,7 +269,7 @@ func CopyToCaller(ctx context.Context, srcPath string, c session.Caller, progres return err } - return sendDiffCopy(cc, srcPath, nil, nil, progress, nil) + return sendDiffCopy(cc, srcPath, nil, nil, nil, progress, nil) } func CopyFileWriter(ctx context.Context, c session.Caller) (io.WriteCloser, error) { diff --git a/vendor/github.com/moby/buildkit/solver/pb/attr.go b/vendor/github.com/moby/buildkit/solver/pb/attr.go index ffbb67a778..cee4f1b326 100644 --- a/vendor/github.com/moby/buildkit/solver/pb/attr.go +++ b/vendor/github.com/moby/buildkit/solver/pb/attr.go @@ -4,6 +4,7 @@ const AttrKeepGitDir = "git.keepgitdir" const AttrFullRemoteURL = "git.fullurl" const AttrLocalSessionID = "local.session" const AttrIncludePatterns = "local.includepattern" +const AttrFollowPaths = "local.followpaths" const AttrExcludePatterns = "local.excludepatterns" const AttrSharedKeyHint = "local.sharedkeyhint" const AttrLLBDefinitionFilename = "llbbuild.filename" diff --git a/vendor/github.com/moby/buildkit/util/progress/progressui/display.go b/vendor/github.com/moby/buildkit/util/progress/progressui/display.go index 3f27a6f304..fd7a8c23c3 100644 --- a/vendor/github.com/moby/buildkit/util/progress/progressui/display.go +++ b/vendor/github.com/moby/buildkit/util/progress/progressui/display.go @@ -1,6 +1,7 @@ package progressui import ( + "bytes" "context" "fmt" "io" @@ -15,17 +16,21 @@ import ( "golang.org/x/time/rate" ) -func DisplaySolveStatus(ctx context.Context, c console.Console, ch chan *client.SolveStatus) error { - disp := &display{c: c} +func DisplaySolveStatus(ctx context.Context, c console.Console, w io.Writer, ch chan *client.SolveStatus) error { - t := newTrace() + modeConsole := c != nil + + disp := &display{c: c} + printer := &textMux{w: w} + + t := newTrace(w) + + var done bool ticker := time.NewTicker(100 * time.Millisecond) defer ticker.Stop() displayLimiter := rate.NewLimiter(rate.Every(70*time.Millisecond), 1) - var done bool - for { select { case <-ctx.Done(): @@ -39,12 +44,21 @@ func DisplaySolveStatus(ctx context.Context, c console.Console, ch chan *client. } } - if done { - disp.print(t.displayInfo(), true) - t.printErrorLogs(c) - return nil - } else if displayLimiter.Allow() { - disp.print(t.displayInfo(), false) + if modeConsole { + if done { + disp.print(t.displayInfo(), true) + t.printErrorLogs(c) + return nil + } else if displayLimiter.Allow() { + disp.print(t.displayInfo(), false) + } + } else { + if done || displayLimiter.Allow() { + printer.print(t) + if done { + return nil + } + } } } } @@ -66,37 +80,108 @@ type job struct { } type trace struct { + w io.Writer localTimeDiff time.Duration vertexes []*vertex byDigest map[digest.Digest]*vertex + nextIndex int + updates map[digest.Digest]struct{} } type vertex struct { *client.Vertex statuses []*status byID map[string]*status - logs []*client.VertexLog indent string + index int + + logs [][]byte + logsPartial bool + logsOffset int + prev *client.Vertex + events []string + lastBlockTime *time.Time + count int + statusUpdates map[string]struct{} +} + +func (v *vertex) update(c int) { + if v.count == 0 { + now := time.Now() + v.lastBlockTime = &now + } + v.count += c } type status struct { *client.VertexStatus } -func newTrace() *trace { +func newTrace(w io.Writer) *trace { return &trace{ byDigest: make(map[digest.Digest]*vertex), + updates: make(map[digest.Digest]struct{}), + w: w, } } +func (t *trace) triggerVertexEvent(v *client.Vertex) { + if v.Started == nil { + return + } + + var old client.Vertex + vtx := t.byDigest[v.Digest] + if v := vtx.prev; v != nil { + old = *v + } + + var ev []string + if v.Digest != old.Digest { + ev = append(ev, fmt.Sprintf("%13s %s", "digest:", v.Digest)) + } + if v.Name != old.Name { + ev = append(ev, fmt.Sprintf("%13s %q", "name:", v.Name)) + } + if v.Started != old.Started { + if v.Started != nil && old.Started == nil || !v.Started.Equal(*old.Started) { + ev = append(ev, fmt.Sprintf("%13s %v", "started:", v.Started)) + } + } + if v.Completed != old.Completed && v.Completed != nil { + ev = append(ev, fmt.Sprintf("%13s %v", "completed:", v.Completed)) + if v.Started != nil { + ev = append(ev, fmt.Sprintf("%13s %v", "duration:", v.Completed.Sub(*v.Started))) + } + } + if v.Cached != old.Cached { + ev = append(ev, fmt.Sprintf("%13s %v", "cached:", v.Cached)) + } + if v.Error != old.Error { + ev = append(ev, fmt.Sprintf("%13s %q", "error:", v.Error)) + } + + if len(ev) > 0 { + vtx.events = append(vtx.events, ev...) + vtx.update(len(ev)) + t.updates[v.Digest] = struct{}{} + } + + t.byDigest[v.Digest].prev = v +} + func (t *trace) update(s *client.SolveStatus) { for _, v := range s.Vertexes { prev, ok := t.byDigest[v.Digest] if !ok { + t.nextIndex++ t.byDigest[v.Digest] = &vertex{ - byID: make(map[string]*status), + byID: make(map[string]*status), + statusUpdates: make(map[string]struct{}), + index: t.nextIndex, } } + t.triggerVertexEvent(v) if v.Started != nil && (prev == nil || prev.Started == nil) { if t.localTimeDiff == 0 { t.localTimeDiff = time.Since(*v.Started) @@ -118,13 +203,29 @@ func (t *trace) update(s *client.SolveStatus) { v.statuses = append(v.statuses, v.byID[s.ID]) } v.byID[s.ID].VertexStatus = s + v.statusUpdates[s.ID] = struct{}{} + t.updates[v.Digest] = struct{}{} + v.update(1) } for _, l := range s.Logs { v, ok := t.byDigest[l.Vertex] if !ok { continue // shouldn't happen } - v.logs = append(v.logs, l) + complete := split(l.Data, byte('\n'), func(dt []byte) { + if v.logsPartial && len(v.logs) != 0 { + v.logs[len(v.logs)-1] = append(v.logs[len(v.logs)-1], dt...) + } else { + ts := time.Duration(0) + if v.Started != nil { + ts = l.Timestamp.Sub(*v.Started) + } + v.logs = append(v.logs, []byte(fmt.Sprintf("#%d %s %s", v.index, fmt.Sprintf("%#.4g", ts.Seconds())[:5], dt))) + } + }) + v.logsPartial = !complete + t.updates[v.Digest] = struct{}{} + v.update(1) } } @@ -134,12 +235,8 @@ func (t *trace) printErrorLogs(f io.Writer) { fmt.Fprintln(f, "------") fmt.Fprintf(f, " > %s:\n", v.Name) for _, l := range v.logs { - switch l.Stream { - case 1: - f.Write(l.Data) - case 2: - f.Write(l.Data) - } + f.Write(l) + fmt.Fprintln(f) } fmt.Fprintln(f, "------") } @@ -196,6 +293,24 @@ func (t *trace) displayInfo() (d displayInfo) { return d } +func split(dt []byte, sep byte, fn func([]byte)) bool { + if len(dt) == 0 { + return false + } + for { + if len(dt) == 0 { + return true + } + idx := bytes.IndexByte(dt, sep) + if idx == -1 { + fn(dt) + return false + } + fn(dt[:idx]) + dt = dt[idx+1:] + } +} + func addTime(tm *time.Time, d time.Duration) *time.Time { if tm == nil { return nil diff --git a/vendor/github.com/moby/buildkit/util/progress/progressui/printer.go b/vendor/github.com/moby/buildkit/util/progress/progressui/printer.go new file mode 100644 index 0000000000..f789d58385 --- /dev/null +++ b/vendor/github.com/moby/buildkit/util/progress/progressui/printer.go @@ -0,0 +1,214 @@ +package progressui + +import ( + "fmt" + "io" + "time" + + digest "github.com/opencontainers/go-digest" + "github.com/tonistiigi/units" +) + +const antiFlicker = 5 * time.Second +const maxDelay = 10 * time.Second + +type textMux struct { + w io.Writer + current digest.Digest +} + +func (p *textMux) printVtx(t *trace, dgst digest.Digest) { + v, ok := t.byDigest[dgst] + if !ok { + return + } + + if dgst != p.current { + if p.current != "" { + old := t.byDigest[p.current] + if old.logsPartial { + fmt.Fprintln(p.w, "") + } + old.logsOffset = 0 + old.count = 0 + fmt.Fprintf(p.w, "#%d ...\n", v.index) + } + + fmt.Fprintf(p.w, "\n#%d %s\n", v.index, limitString(v.Name, 72)) + } + + if len(v.events) != 0 { + v.logsOffset = 0 + } + for _, ev := range v.events { + fmt.Fprintf(p.w, "#%d %s\n", v.index, ev) + } + v.events = v.events[:0] + + for _, s := range v.statuses { + if _, ok := v.statusUpdates[s.ID]; ok { + var bytes string + if s.Total != 0 { + bytes = fmt.Sprintf(" %.2f / %.2f", units.Bytes(s.Current), units.Bytes(s.Total)) + } else if s.Current != 0 { + bytes = fmt.Sprintf(" %.2f", units.Bytes(s.Current)) + } + var tm string + endTime := s.Timestamp + if s.Completed != nil { + endTime = *s.Completed + } + if s.Started != nil { + diff := endTime.Sub(*s.Started).Seconds() + if diff > 0.01 { + tm = fmt.Sprintf(" %.1fs", diff) + } + } + if s.Completed != nil { + tm += " done" + } + fmt.Fprintf(p.w, "#%d %s%s%s\n", v.index, s.ID, bytes, tm) + } + } + v.statusUpdates = map[string]struct{}{} + + for i, l := range v.logs { + if i == 0 { + l = l[v.logsOffset:] + } + fmt.Fprintf(p.w, "%s", []byte(l)) + if i != len(v.logs)-1 || !v.logsPartial { + fmt.Fprintln(p.w, "") + } + } + + if len(v.logs) > 0 { + if v.logsPartial { + v.logs = v.logs[len(v.logs)-1:] + v.logsOffset = len(v.logs[0]) + } else { + v.logs = nil + v.logsOffset = 0 + } + } + + p.current = dgst + + if v.Completed != nil { + p.current = "" + v.count = 0 + fmt.Fprintf(p.w, "\n") + } + + delete(t.updates, dgst) +} + +func (p *textMux) print(t *trace) { + + completed := map[digest.Digest]struct{}{} + rest := map[digest.Digest]struct{}{} + + for dgst := range t.updates { + v, ok := t.byDigest[dgst] + if !ok { + continue + } + if v.Vertex.Completed != nil { + completed[dgst] = struct{}{} + } else { + rest[dgst] = struct{}{} + } + } + + current := p.current + + // items that have completed need to be printed first + if _, ok := completed[current]; ok { + p.printVtx(t, current) + } + + for dgst := range completed { + if dgst != current { + p.printVtx(t, dgst) + } + } + + if len(rest) == 0 { + if current != "" { + if v := t.byDigest[current]; v.Started != nil && v.Completed == nil { + return + } + } + // make any open vertex active + for dgst, v := range t.byDigest { + if v.Started != nil && v.Completed == nil { + p.printVtx(t, dgst) + return + } + } + return + } + + // now print the active one + if _, ok := rest[current]; ok { + p.printVtx(t, current) + } + + stats := map[digest.Digest]*vtxStat{} + now := time.Now() + sum := 0.0 + var max digest.Digest + if current != "" { + rest[current] = struct{}{} + } + for dgst := range rest { + v, ok := t.byDigest[dgst] + if !ok { + continue + } + tm := now.Sub(*v.lastBlockTime) + speed := float64(v.count) / tm.Seconds() + overLimit := tm > maxDelay && dgst != current + stats[dgst] = &vtxStat{blockTime: tm, speed: speed, overLimit: overLimit} + sum += speed + if overLimit || max == "" || stats[max].speed < speed { + max = dgst + } + } + for dgst := range stats { + stats[dgst].share = stats[dgst].speed / sum + } + + if _, ok := completed[current]; ok || current == "" { + p.printVtx(t, max) + return + } + + // show items that were hidden + for dgst := range rest { + if stats[dgst].overLimit { + p.printVtx(t, dgst) + return + } + } + + // fair split between vertexes + if 1.0/(1.0-stats[current].share)*antiFlicker.Seconds() < stats[current].blockTime.Seconds() { + p.printVtx(t, max) + return + } +} + +type vtxStat struct { + blockTime time.Duration + speed float64 + share float64 + overLimit bool +} + +func limitString(s string, l int) string { + if len(s) > l { + return s[:l] + "..." + } + return s +} diff --git a/vendor/github.com/moby/buildkit/util/system/seccomp_linux.go b/vendor/github.com/moby/buildkit/util/system/seccomp_linux.go index 663625c8d9..62afa03fef 100644 --- a/vendor/github.com/moby/buildkit/util/system/seccomp_linux.go +++ b/vendor/github.com/moby/buildkit/util/system/seccomp_linux.go @@ -1,4 +1,4 @@ -// +build linux +// +build linux,seccomp package system diff --git a/vendor/github.com/moby/buildkit/util/system/seccomp_nolinux.go b/vendor/github.com/moby/buildkit/util/system/seccomp_nolinux.go index 305d6c9ef0..e348c379a9 100644 --- a/vendor/github.com/moby/buildkit/util/system/seccomp_nolinux.go +++ b/vendor/github.com/moby/buildkit/util/system/seccomp_nolinux.go @@ -1,4 +1,4 @@ -// +build !linux +// +build !linux,seccomp package system diff --git a/vendor/github.com/moby/buildkit/util/system/seccomp_noseccomp.go b/vendor/github.com/moby/buildkit/util/system/seccomp_noseccomp.go new file mode 100644 index 0000000000..84cfb7fa83 --- /dev/null +++ b/vendor/github.com/moby/buildkit/util/system/seccomp_noseccomp.go @@ -0,0 +1,7 @@ +// +build !seccomp + +package system + +func SeccompSupported() bool { + return false +} diff --git a/vendor/github.com/moby/buildkit/vendor.conf b/vendor/github.com/moby/buildkit/vendor.conf index 6f29ab57f0..ee1375da27 100644 --- a/vendor/github.com/moby/buildkit/vendor.conf +++ b/vendor/github.com/moby/buildkit/vendor.conf @@ -6,7 +6,7 @@ github.com/davecgh/go-spew v1.1.0 github.com/pmezard/go-difflib v1.0.0 golang.org/x/sys 314a259e304ff91bd6985da2a7149bbf91237993 -github.com/containerd/containerd e1428ef05460da40720d622c803262e6fc8d3477 +github.com/containerd/containerd 63522d9eaa5a0443d225642c4b6f4f5fdedf932b github.com/containerd/typeurl f6943554a7e7e88b3c14aad190bf05932da84788 golang.org/x/sync 450f422ab23cf9881c94e2db30cac0eb1b7cf80c github.com/sirupsen/logrus v1.0.0 @@ -23,7 +23,7 @@ github.com/Microsoft/go-winio v0.4.7 github.com/containerd/fifo 3d5202aec260678c48179c56f40e6f38a095738c github.com/opencontainers/runtime-spec v1.0.1 github.com/containerd/go-runc f271fa2021de855d4d918dbef83c5fe19db1bdd5 -github.com/containerd/console cb7008ab3d8359b78c5f464cb7cf160107ad5925 +github.com/containerd/console 9290d21dc56074581f619579c43d970b4514bc08 google.golang.org/genproto d80a6e20e776b0b17a324d0ba1ab50a39c8e8944 golang.org/x/text 19e51611da83d6be54ddafce4a4af510cb3e9ea4 github.com/docker/go-events 9461782956ad83b30282bf90e31fa6a70c255ba9 @@ -36,11 +36,10 @@ github.com/docker/go-units v0.3.1 github.com/google/shlex 6f45313302b9c56850fc17f99e40caebce98c716 golang.org/x/time f51c12702a4d776e4c1fa9b0fabab841babae631 -github.com/BurntSushi/locker a6e239ea1c69bff1cfdb20c4b73dadf52f784b6a github.com/docker/docker 71cd53e4a197b303c6ba086bd584ffd67a884281 github.com/pkg/profile 5b67d428864e92711fcbd2f8629456121a56d91f -github.com/tonistiigi/fsutil dc68c74458923f357474a9178bd198aa3ed11a5f +github.com/tonistiigi/fsutil 8839685ae8c3c8bd67d0ce28e9b3157b23c1c7a5 github.com/hashicorp/go-immutable-radix 826af9ccf0feeee615d546d69b11f8e98da8c8f1 git://github.com/tonistiigi/go-immutable-radix.git github.com/hashicorp/golang-lru a0d98a5f288019575c6d1f4bb1573fef2d1fcdc4 github.com/mitchellh/hashstructure 2bca23e0e452137f789efbc8610126fd8b94f73b diff --git a/vendor/github.com/tonistiigi/fsutil/chtimes_nolinux.go b/vendor/github.com/tonistiigi/fsutil/chtimes_nolinux.go index 39bfdfee5b..cdd80ec9a7 100644 --- a/vendor/github.com/tonistiigi/fsutil/chtimes_nolinux.go +++ b/vendor/github.com/tonistiigi/fsutil/chtimes_nolinux.go @@ -9,5 +9,12 @@ import ( func chtimes(path string, un int64) error { mtime := time.Unix(0, un) + fi, err := os.Lstat(path) + if err != nil { + return err + } + if fi.Mode()&os.ModeSymlink != 0 { + return nil + } return os.Chtimes(path, mtime, mtime) } diff --git a/vendor/github.com/tonistiigi/fsutil/diff.go b/vendor/github.com/tonistiigi/fsutil/diff.go index 6125ef73af..340a0e48a4 100644 --- a/vendor/github.com/tonistiigi/fsutil/diff.go +++ b/vendor/github.com/tonistiigi/fsutil/diff.go @@ -1,10 +1,9 @@ package fsutil import ( + "context" "hash" "os" - - "golang.org/x/net/context" ) type walkerFn func(ctx context.Context, pathC chan<- *currentPath) error diff --git a/vendor/github.com/tonistiigi/fsutil/diff_containerd.go b/vendor/github.com/tonistiigi/fsutil/diff_containerd.go index c7c9788e85..2722cef4ac 100644 --- a/vendor/github.com/tonistiigi/fsutil/diff_containerd.go +++ b/vendor/github.com/tonistiigi/fsutil/diff_containerd.go @@ -1,10 +1,10 @@ package fsutil import ( + "context" "os" "strings" - "golang.org/x/net/context" "golang.org/x/sync/errgroup" ) diff --git a/vendor/github.com/tonistiigi/fsutil/diskwriter.go b/vendor/github.com/tonistiigi/fsutil/diskwriter.go index e2d034c75e..aa1974f243 100644 --- a/vendor/github.com/tonistiigi/fsutil/diskwriter.go +++ b/vendor/github.com/tonistiigi/fsutil/diskwriter.go @@ -1,6 +1,7 @@ package fsutil import ( + "context" "hash" "io" "os" @@ -9,9 +10,8 @@ import ( "sync" "time" - digest "github.com/opencontainers/go-digest" + "github.com/opencontainers/go-digest" "github.com/pkg/errors" - "golang.org/x/net/context" "golang.org/x/sync/errgroup" ) @@ -80,9 +80,7 @@ func (dw *DiskWriter) HandleChange(kind ChangeKind, p string, fi os.FileInfo, er } }() - p = filepath.FromSlash(p) - - destPath := filepath.Join(dw.dest, p) + destPath := filepath.Join(dw.dest, filepath.FromSlash(p)) if kind == ChangeKindDelete { // todo: no need to validate if diff is trusted but is it always? @@ -102,8 +100,10 @@ func (dw *DiskWriter) HandleChange(kind ChangeKind, p string, fi os.FileInfo, er return errors.Errorf("%s invalid change without stat information", p) } + statCopy := *stat + if dw.filter != nil { - if ok := dw.filter(stat); !ok { + if ok := dw.filter(&statCopy); !ok { return nil } } @@ -122,7 +122,7 @@ func (dw *DiskWriter) HandleChange(kind ChangeKind, p string, fi os.FileInfo, er } if oldFi != nil && fi.IsDir() && oldFi.IsDir() { - if err := rewriteMetadata(destPath, stat); err != nil { + if err := rewriteMetadata(destPath, &statCopy); err != nil { return errors.Wrapf(err, "error setting dir metadata for %s", destPath) } return nil @@ -141,16 +141,16 @@ func (dw *DiskWriter) HandleChange(kind ChangeKind, p string, fi os.FileInfo, er return errors.Wrapf(err, "failed to create dir %s", newPath) } case fi.Mode()&os.ModeDevice != 0 || fi.Mode()&os.ModeNamedPipe != 0: - if err := handleTarTypeBlockCharFifo(newPath, stat); err != nil { + if err := handleTarTypeBlockCharFifo(newPath, &statCopy); err != nil { return errors.Wrapf(err, "failed to create device %s", newPath) } case fi.Mode()&os.ModeSymlink != 0: - if err := os.Symlink(stat.Linkname, newPath); err != nil { + if err := os.Symlink(statCopy.Linkname, newPath); err != nil { return errors.Wrapf(err, "failed to symlink %s", newPath) } - case stat.Linkname != "": - if err := os.Link(filepath.Join(dw.dest, stat.Linkname), newPath); err != nil { - return errors.Wrapf(err, "failed to link %s to %s", newPath, stat.Linkname) + case statCopy.Linkname != "": + if err := os.Link(filepath.Join(dw.dest, statCopy.Linkname), newPath); err != nil { + return errors.Wrapf(err, "failed to link %s to %s", newPath, statCopy.Linkname) } default: isRegularFile = true @@ -170,7 +170,7 @@ func (dw *DiskWriter) HandleChange(kind ChangeKind, p string, fi os.FileInfo, er } } - if err := rewriteMetadata(newPath, stat); err != nil { + if err := rewriteMetadata(newPath, &statCopy); err != nil { return errors.Wrapf(err, "error setting metadata for %s", newPath) } @@ -272,14 +272,27 @@ func (hw *hashedWriter) Digest() digest.Digest { } type lazyFileWriter struct { - dest string - ctx context.Context - f *os.File + dest string + ctx context.Context + f *os.File + fileMode *os.FileMode } func (lfw *lazyFileWriter) Write(dt []byte) (int, error) { if lfw.f == nil { file, err := os.OpenFile(lfw.dest, os.O_WRONLY, 0) //todo: windows + if os.IsPermission(err) { + // retry after chmod + fi, er := os.Stat(lfw.dest) + if er == nil { + mode := fi.Mode() + lfw.fileMode = &mode + er = os.Chmod(lfw.dest, mode|0222) + if er == nil { + file, err = os.OpenFile(lfw.dest, os.O_WRONLY, 0) + } + } + } if err != nil { return 0, errors.Wrapf(err, "failed to open %s", lfw.dest) } @@ -289,10 +302,14 @@ func (lfw *lazyFileWriter) Write(dt []byte) (int, error) { } func (lfw *lazyFileWriter) Close() error { + var err error if lfw.f != nil { - return lfw.f.Close() + err = lfw.f.Close() } - return nil + if err == nil && lfw.fileMode != nil { + err = os.Chmod(lfw.dest, *lfw.fileMode) + } + return err } func mkdev(major int64, minor int64) uint32 { diff --git a/vendor/github.com/tonistiigi/fsutil/followlinks.go b/vendor/github.com/tonistiigi/fsutil/followlinks.go new file mode 100644 index 0000000000..ed4af6e816 --- /dev/null +++ b/vendor/github.com/tonistiigi/fsutil/followlinks.go @@ -0,0 +1,150 @@ +package fsutil + +import ( + "io/ioutil" + "os" + "path/filepath" + "runtime" + "sort" + strings "strings" + + "github.com/pkg/errors" +) + +func FollowLinks(root string, paths []string) ([]string, error) { + r := &symlinkResolver{root: root, resolved: map[string]struct{}{}} + for _, p := range paths { + if err := r.append(p); err != nil { + return nil, err + } + } + res := make([]string, 0, len(r.resolved)) + for r := range r.resolved { + res = append(res, r) + } + sort.Strings(res) + return dedupePaths(res), nil +} + +type symlinkResolver struct { + root string + resolved map[string]struct{} +} + +func (r *symlinkResolver) append(p string) error { + p = filepath.Join(".", p) + current := "." + for { + parts := strings.SplitN(p, string(filepath.Separator), 2) + current = filepath.Join(current, parts[0]) + + targets, err := r.readSymlink(current, true) + if err != nil { + return err + } + + p = "" + if len(parts) == 2 { + p = parts[1] + } + + if p == "" || targets != nil { + if _, ok := r.resolved[current]; ok { + return nil + } + } + + if targets != nil { + r.resolved[current] = struct{}{} + for _, target := range targets { + if err := r.append(filepath.Join(target, p)); err != nil { + return err + } + } + return nil + } + + if p == "" { + r.resolved[current] = struct{}{} + return nil + } + } +} + +func (r *symlinkResolver) readSymlink(p string, allowWildcard bool) ([]string, error) { + realPath := filepath.Join(r.root, p) + base := filepath.Base(p) + if allowWildcard && containsWildcards(base) { + fis, err := ioutil.ReadDir(filepath.Dir(realPath)) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, errors.Wrapf(err, "failed to read dir %s", filepath.Dir(realPath)) + } + var out []string + for _, f := range fis { + if ok, _ := filepath.Match(base, f.Name()); ok { + res, err := r.readSymlink(filepath.Join(filepath.Dir(p), f.Name()), false) + if err != nil { + return nil, err + } + out = append(out, res...) + } + } + return out, nil + } + + fi, err := os.Lstat(realPath) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, errors.Wrapf(err, "failed to lstat %s", realPath) + } + if fi.Mode()&os.ModeSymlink == 0 { + return nil, nil + } + link, err := os.Readlink(realPath) + if err != nil { + return nil, errors.Wrapf(err, "failed to readlink %s", realPath) + } + link = filepath.Clean(link) + if filepath.IsAbs(link) { + return []string{link}, nil + } + return []string{ + filepath.Join(string(filepath.Separator), filepath.Join(filepath.Dir(p), link)), + }, nil +} + +func containsWildcards(name string) bool { + isWindows := runtime.GOOS == "windows" + for i := 0; i < len(name); i++ { + ch := name[i] + if ch == '\\' && !isWindows { + i++ + } else if ch == '*' || ch == '?' || ch == '[' { + return true + } + } + return false +} + +// dedupePaths expects input as a sorted list +func dedupePaths(in []string) []string { + out := make([]string, 0, len(in)) + var last string + for _, s := range in { + // if one of the paths is root there is no filter + if s == "." { + return nil + } + if strings.HasPrefix(s, last+string(filepath.Separator)) { + continue + } + out = append(out, s) + last = s + } + return out +} diff --git a/vendor/github.com/tonistiigi/fsutil/receive.go b/vendor/github.com/tonistiigi/fsutil/receive.go index 5867c984ee..14ccb6c7bf 100644 --- a/vendor/github.com/tonistiigi/fsutil/receive.go +++ b/vendor/github.com/tonistiigi/fsutil/receive.go @@ -1,12 +1,12 @@ package fsutil import ( + "context" "io" "os" "sync" "github.com/pkg/errors" - "golang.org/x/net/context" "golang.org/x/sync/errgroup" ) diff --git a/vendor/github.com/tonistiigi/fsutil/send.go b/vendor/github.com/tonistiigi/fsutil/send.go index d3cda0bb64..61f6170a4c 100644 --- a/vendor/github.com/tonistiigi/fsutil/send.go +++ b/vendor/github.com/tonistiigi/fsutil/send.go @@ -1,13 +1,13 @@ package fsutil import ( + "context" "io" "os" "path/filepath" "sync" "github.com/pkg/errors" - "golang.org/x/net/context" "golang.org/x/sync/errgroup" ) diff --git a/vendor/github.com/tonistiigi/fsutil/walker.go b/vendor/github.com/tonistiigi/fsutil/walker.go index d05a42dbed..3a44172db9 100644 --- a/vendor/github.com/tonistiigi/fsutil/walker.go +++ b/vendor/github.com/tonistiigi/fsutil/walker.go @@ -1,6 +1,7 @@ package fsutil import ( + "context" "os" "path/filepath" "runtime" @@ -9,13 +10,15 @@ import ( "github.com/docker/docker/pkg/fileutils" "github.com/pkg/errors" - "golang.org/x/net/context" ) type WalkOpt struct { IncludePatterns []string ExcludePatterns []string - Map func(*Stat) bool + // FollowPaths contains symlinks that are resolved into include patterns + // before performing the fs walk + FollowPaths []string + Map func(*Stat) bool } func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) error { @@ -39,8 +42,25 @@ func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) err } } + var includePatterns []string + if opt != nil && opt.IncludePatterns != nil { + includePatterns = make([]string, len(opt.IncludePatterns)) + for k := range opt.IncludePatterns { + includePatterns[k] = filepath.Clean(opt.IncludePatterns[k]) + } + } + if opt != nil && opt.FollowPaths != nil { + targets, err := FollowLinks(p, opt.FollowPaths) + if err != nil { + return err + } + if targets != nil { + includePatterns = append(includePatterns, targets...) + includePatterns = dedupePaths(includePatterns) + } + } + var lastIncludedDir string - var includePatternPrefixes []string seenFiles := make(map[uint64]string) return filepath.Walk(root, func(path string, fi os.FileInfo, err error) (retErr error) { @@ -66,34 +86,31 @@ func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) err } if opt != nil { - if opt.IncludePatterns != nil { - if includePatternPrefixes == nil { - includePatternPrefixes = patternPrefixes(opt.IncludePatterns) - } - matched := false + if includePatterns != nil { + skip := false if lastIncludedDir != "" { if strings.HasPrefix(path, lastIncludedDir+string(filepath.Separator)) { - matched = true + skip = true } } - if !matched { - for _, p := range opt.IncludePatterns { - if m, _ := filepath.Match(p, path); m { + + if !skip { + matched := false + partial := true + for _, p := range includePatterns { + if ok, p := matchPrefix(p, path); ok { matched = true - break + if !p { + partial = false + break + } } } - if matched && fi.IsDir() { - lastIncludedDir = path - } - } - if !matched { - if !fi.IsDir() { + if !matched { return nil - } else { - if noPossiblePrefixMatch(path, includePatternPrefixes) { - return filepath.SkipDir - } + } + if !partial && fi.IsDir() { + lastIncludedDir = path } } } @@ -131,13 +148,13 @@ func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) err stat := &Stat{ Path: path, Mode: uint32(fi.Mode()), - Size_: fi.Size(), ModTime: fi.ModTime().UnixNano(), } setUnixOpt(fi, stat, path, seenFiles) if !fi.IsDir() { + stat.Size_ = fi.Size() if fi.Mode()&os.ModeSymlink != 0 { link, err := os.Readlink(origpath) if err != nil { @@ -199,29 +216,28 @@ func (s *StatInfo) Sys() interface{} { return s.Stat } -func patternPrefixes(patterns []string) []string { - pfxs := make([]string, 0, len(patterns)) - for _, ptrn := range patterns { - idx := strings.IndexFunc(ptrn, func(ch rune) bool { - return ch == '*' || ch == '?' || ch == '[' || ch == '\\' - }) - if idx == -1 { - idx = len(ptrn) - } - pfxs = append(pfxs, ptrn[:idx]) +func matchPrefix(pattern, name string) (bool, bool) { + count := strings.Count(name, string(filepath.Separator)) + partial := false + if strings.Count(pattern, string(filepath.Separator)) > count { + pattern = trimUntilIndex(pattern, string(filepath.Separator), count) + partial = true } - return pfxs + m, _ := filepath.Match(pattern, name) + return m, partial } -func noPossiblePrefixMatch(p string, pfxs []string) bool { - for _, pfx := range pfxs { - chk := p - if len(pfx) < len(p) { - chk = p[:len(pfx)] - } - if strings.HasPrefix(pfx, chk) { - return false +func trimUntilIndex(str, sep string, count int) string { + s := str + i := 0 + c := 0 + for { + idx := strings.Index(s, sep) + s = s[idx+len(sep):] + i += idx + len(sep) + c++ + if c >= count { + return str[:i-len(sep)] } } - return true }