From d6b05747ddaa0a18328a21487e17941e75a15060 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Tue, 15 Sep 2020 11:24:13 +0200 Subject: [PATCH 1/2] [19.03] vendor: tonistiigi/fsutil v0.0.0-20191018213012-0f039a052ca1 full diff: https://github.com/tonistiigi/fsutil/compare/3d2716dd0a4d06ff854241c7e8b6f3f904e1719f...0f039a052ca1da01626278199624b62aed9b3729 - receive: use filter on receive diff - prevents incremental transfers with userns because the metadata on disk is always different than the one being transferred. Signed-off-by: Sebastiaan van Stijn --- vendor.conf | 2 +- vendor/github.com/tonistiigi/fsutil/diff.go | 8 +++- .../tonistiigi/fsutil/diff_containerd.go | 47 +++++++++---------- .../tonistiigi/fsutil/diskwriter.go | 6 +-- vendor/github.com/tonistiigi/fsutil/go.mod | 2 +- .../github.com/tonistiigi/fsutil/receive.go | 8 ++-- .../tonistiigi/fsutil/types/stat.go | 7 +++ 7 files changed, 45 insertions(+), 35 deletions(-) create mode 100644 vendor/github.com/tonistiigi/fsutil/types/stat.go diff --git a/vendor.conf b/vendor.conf index ffb8d8609d..27147c1473 100755 --- a/vendor.conf +++ b/vendor.conf @@ -75,7 +75,7 @@ github.com/spf13/cobra ef82de70bb3f60c65fb8eebacbb2 github.com/spf13/pflag 4cb166e4f25ac4e8016a3595bbf7ea2e9aa85a2c https://github.com/thaJeztah/pflag.git # temporary fork with https://github.com/spf13/pflag/pull/170 applied, which isn't merged yet upstream github.com/syndtr/gocapability d98352740cb2c55f81556b63d4a1ec64c5a319c2 github.com/theupdateframework/notary d6e1431feb32348e0650bf7551ac5cffd01d857b # v0.6.1 -github.com/tonistiigi/fsutil 3d2716dd0a4d06ff854241c7e8b6f3f904e1719f +github.com/tonistiigi/fsutil 0f039a052ca1da01626278199624b62aed9b3729 github.com/tonistiigi/units 6950e57a87eaf136bbe44ef2ec8e75b9e3569de2 github.com/xeipuuv/gojsonpointer 4e3ac2762d5f479393488629ee9370b50873b3a6 github.com/xeipuuv/gojsonreference bd5ef7bd5415a7ac448318e64f11a24cd21e594b diff --git a/vendor/github.com/tonistiigi/fsutil/diff.go b/vendor/github.com/tonistiigi/fsutil/diff.go index acad9f4f22..1cbc32b306 100644 --- a/vendor/github.com/tonistiigi/fsutil/diff.go +++ b/vendor/github.com/tonistiigi/fsutil/diff.go @@ -5,6 +5,7 @@ import ( "hash" "os" + "github.com/pkg/errors" "github.com/tonistiigi/fsutil/types" ) @@ -25,9 +26,14 @@ func GetWalkerFn(root string) walkerFn { return err } + stat, ok := f.Sys().(*types.Stat) + if !ok { + return errors.Errorf("%T invalid file without stat information", f.Sys()) + } + p := ¤tPath{ path: path, - f: f, + stat: stat, } select { diff --git a/vendor/github.com/tonistiigi/fsutil/diff_containerd.go b/vendor/github.com/tonistiigi/fsutil/diff_containerd.go index e63ee916a6..370271523b 100644 --- a/vendor/github.com/tonistiigi/fsutil/diff_containerd.go +++ b/vendor/github.com/tonistiigi/fsutil/diff_containerd.go @@ -37,12 +37,12 @@ type ChangeFunc func(ChangeKind, string, os.FileInfo, error) error type currentPath struct { path string - f os.FileInfo + stat *types.Stat // fullPath string } // doubleWalkDiff walks both directories to create a diff -func doubleWalkDiff(ctx context.Context, changeFn ChangeFunc, a, b walkerFn) (err error) { +func doubleWalkDiff(ctx context.Context, changeFn ChangeFunc, a, b walkerFn, filter FilterFunc) (err error) { g, ctx := errgroup.WithContext(ctx) var ( @@ -86,14 +86,22 @@ func doubleWalkDiff(ctx context.Context, changeFn ChangeFunc, a, b walkerFn) (er continue } - var f os.FileInfo - k, p := pathChange(f1, f2) + var f *types.Stat + var f2copy *currentPath + if f2 != nil { + statCopy := *f2.stat + if filter != nil { + filter(f2.path, &statCopy) + } + f2copy = ¤tPath{path: f2.path, stat: &statCopy} + } + k, p := pathChange(f1, f2copy) switch k { case ChangeKindAdd: if rmdir != "" { rmdir = "" } - f = f2.f + f = f2.stat f2 = nil case ChangeKindDelete: // Check if this file is already removed by being @@ -101,30 +109,30 @@ func doubleWalkDiff(ctx context.Context, changeFn ChangeFunc, a, b walkerFn) (er if rmdir != "" && strings.HasPrefix(f1.path, rmdir) { f1 = nil continue - } else if rmdir == "" && f1.f.IsDir() { + } else if rmdir == "" && f1.stat.IsDir() { rmdir = f1.path + string(os.PathSeparator) } else if rmdir != "" { rmdir = "" } f1 = nil case ChangeKindModify: - same, err := sameFile(f1, f2) + same, err := sameFile(f1, f2copy) if err != nil { return err } - if f1.f.IsDir() && !f2.f.IsDir() { + if f1.stat.IsDir() && !f2copy.stat.IsDir() { rmdir = f1.path + string(os.PathSeparator) } else if rmdir != "" { rmdir = "" } - f = f2.f + f = f2.stat f1 = nil f2 = nil if same { continue loop0 } } - if err := changeFn(k, p, f, nil); err != nil { + if err := changeFn(k, p, &StatInfo{f}, nil); err != nil { return err } } @@ -159,28 +167,17 @@ func pathChange(lower, upper *currentPath) (ChangeKind, string) { func sameFile(f1, f2 *currentPath) (same bool, retErr error) { // If not a directory also check size, modtime, and content - if !f1.f.IsDir() { - if f1.f.Size() != f2.f.Size() { + if !f1.stat.IsDir() { + if f1.stat.Size_ != f2.stat.Size_ { return false, nil } - t1 := f1.f.ModTime() - t2 := f2.f.ModTime() - if t1.UnixNano() != t2.UnixNano() { + if f1.stat.ModTime != f2.stat.ModTime { return false, nil } } - ls1, ok := f1.f.Sys().(*types.Stat) - if !ok { - return false, nil - } - ls2, ok := f2.f.Sys().(*types.Stat) - if !ok { - return false, nil - } - - return compareStat(ls1, ls2) + return compareStat(f1.stat, f2.stat) } // compareStat returns whether the stats are equivalent, diff --git a/vendor/github.com/tonistiigi/fsutil/diskwriter.go b/vendor/github.com/tonistiigi/fsutil/diskwriter.go index 79a1673b78..70323c88c9 100644 --- a/vendor/github.com/tonistiigi/fsutil/diskwriter.go +++ b/vendor/github.com/tonistiigi/fsutil/diskwriter.go @@ -194,7 +194,7 @@ func (dw *DiskWriter) HandleChange(kind ChangeKind, p string, fi os.FileInfo, er if isRegularFile { if dw.opt.AsyncDataCb != nil { - dw.requestAsyncFileData(p, destPath, fi) + dw.requestAsyncFileData(p, destPath, fi, &statCopy) } } else { return dw.processChange(kind, p, fi, nil) @@ -203,7 +203,7 @@ func (dw *DiskWriter) HandleChange(kind ChangeKind, p string, fi os.FileInfo, er return nil } -func (dw *DiskWriter) requestAsyncFileData(p, dest string, fi os.FileInfo) { +func (dw *DiskWriter) requestAsyncFileData(p, dest string, fi os.FileInfo, st *types.Stat) { // todo: limit worker threads dw.eg.Go(func() error { if err := dw.processChange(ChangeKindAdd, p, fi, &lazyFileWriter{ @@ -211,7 +211,7 @@ func (dw *DiskWriter) requestAsyncFileData(p, dest string, fi os.FileInfo) { }); err != nil { return err } - return chtimes(dest, fi.ModTime().UnixNano()) // TODO: parent dirs + return chtimes(dest, st.ModTime) // TODO: parent dirs }) } diff --git a/vendor/github.com/tonistiigi/fsutil/go.mod b/vendor/github.com/tonistiigi/fsutil/go.mod index 0e26f72085..c6485f6155 100644 --- a/vendor/github.com/tonistiigi/fsutil/go.mod +++ b/vendor/github.com/tonistiigi/fsutil/go.mod @@ -19,7 +19,7 @@ require ( github.com/pkg/errors v0.8.1 github.com/sirupsen/logrus v1.0.3 // indirect github.com/stretchr/testify v1.3.0 - golang.org/x/crypto v0.0.0-20190129210102-0709b304e793 // indirect + golang.org/x/crypto v0.0.0-20180904163835-0709b304e793 // indirect golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e gopkg.in/airbrake/gobrake.v2 v2.0.9 // indirect diff --git a/vendor/github.com/tonistiigi/fsutil/receive.go b/vendor/github.com/tonistiigi/fsutil/receive.go index 4de7ec5f0b..0210dcdb1a 100644 --- a/vendor/github.com/tonistiigi/fsutil/receive.go +++ b/vendor/github.com/tonistiigi/fsutil/receive.go @@ -133,7 +133,7 @@ func (r *receiver) run(ctx context.Context) error { if !r.merge { destWalker = GetWalkerFn(r.dest) } - err := doubleWalkDiff(ctx, dw.HandleChange, destWalker, w.fill) + err := doubleWalkDiff(ctx, dw.HandleChange, destWalker, w.fill, r.filter) if err != nil { return err } @@ -180,11 +180,11 @@ func (r *receiver) run(ctx context.Context) error { r.mu.Unlock() } i++ - cp := ¤tPath{path: p.Stat.Path, f: &StatInfo{p.Stat}} - if err := r.orderValidator.HandleChange(ChangeKindAdd, cp.path, cp.f, nil); err != nil { + cp := ¤tPath{path: p.Stat.Path, stat: p.Stat} + if err := r.orderValidator.HandleChange(ChangeKindAdd, cp.path, &StatInfo{cp.stat}, nil); err != nil { return err } - if err := r.hlValidator.HandleChange(ChangeKindAdd, cp.path, cp.f, nil); err != nil { + if err := r.hlValidator.HandleChange(ChangeKindAdd, cp.path, &StatInfo{cp.stat}, nil); err != nil { return err } if err := w.update(cp); err != nil { diff --git a/vendor/github.com/tonistiigi/fsutil/types/stat.go b/vendor/github.com/tonistiigi/fsutil/types/stat.go new file mode 100644 index 0000000000..b79fd2bd76 --- /dev/null +++ b/vendor/github.com/tonistiigi/fsutil/types/stat.go @@ -0,0 +1,7 @@ +package types + +import "os" + +func (s Stat) IsDir() bool { + return os.FileMode(s.Mode).IsDir() +} From aebe8b0660480af068b8005b73d19fa7dfe910a9 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Tue, 15 Sep 2020 11:20:29 +0200 Subject: [PATCH 2/2] [19.03] vendor: buildkit v0.6.4-32-gdf89d4dc full diff: https://github.com/moby/buildkit/compare/ae10b292fefb00e0fbf9fecd1419c5f252e58895...v0.6.4-32-gdf89d4dc - progressui: fix panic on terminals with zero height Signed-off-by: Sebastiaan van Stijn --- vendor.conf | 2 +- .../moby/buildkit/frontend/gateway/pb/caps.go | 10 ++++++++++ vendor/github.com/moby/buildkit/go.mod | 5 +++-- .../moby/buildkit/session/filesync/filesync.go | 9 +++++++-- .../github.com/moby/buildkit/session/sshforward/ssh.go | 4 ++++ .../session/sshforward/sshprovider/agentprovider.go | 6 +++++- .../moby/buildkit/util/progress/progressui/display.go | 3 +++ 7 files changed, 33 insertions(+), 6 deletions(-) diff --git a/vendor.conf b/vendor.conf index 27147c1473..d6b7bc7612 100755 --- a/vendor.conf +++ b/vendor.conf @@ -54,7 +54,7 @@ github.com/Microsoft/go-winio 84b4ab48a50763fe7b3abcef38e5 github.com/Microsoft/hcsshim 672e52e9209d1e53718c1b6a7d68cc9272654ab5 github.com/miekg/pkcs11 cb39313ec884f2cd77f4762875fe96aecf68f8e3 # v1.0.2 github.com/mitchellh/mapstructure fa473d140ef3c6adf42d6b391fe76707f1f243c8 # v1.0.0 -github.com/moby/buildkit ae10b292fefb00e0fbf9fecd1419c5f252e58895 +github.com/moby/buildkit df89d4dcf73ce414cd76837bfb0e9a0cc0ef3386 # v0.6.4-32-gdf89d4dc github.com/modern-go/concurrent bacd9c7ef1dd9b15be4a9909b8ac7a4e313eec94 # 1.0.3 github.com/modern-go/reflect2 4b7aa43c6742a2c18fdef89dd197aaae7dac7ccd # 1.0.1 github.com/morikuni/aec 39771216ff4c63d11f5e604076f9c45e8be1067b diff --git a/vendor/github.com/moby/buildkit/frontend/gateway/pb/caps.go b/vendor/github.com/moby/buildkit/frontend/gateway/pb/caps.go index 1acde729a3..29ffc912e0 100644 --- a/vendor/github.com/moby/buildkit/frontend/gateway/pb/caps.go +++ b/vendor/github.com/moby/buildkit/frontend/gateway/pb/caps.go @@ -19,6 +19,9 @@ const ( CapReadDir apicaps.CapID = "readdir" CapStatFile apicaps.CapID = "statfile" CapImportCaches apicaps.CapID = "importcaches" + + // CapGatewaySolveMetadata can be used to check if solve calls from gateway reliably return metadata + CapGatewaySolveMetadata apicaps.CapID = "gateway.solve.metadata" ) func init() { @@ -92,4 +95,11 @@ func init() { Enabled: true, Status: apicaps.CapStatusExperimental, }) + + Caps.Init(apicaps.Cap{ + ID: CapGatewaySolveMetadata, + Name: "gateway metadata", + Enabled: true, + Status: apicaps.CapStatusExperimental, + }) } diff --git a/vendor/github.com/moby/buildkit/go.mod b/vendor/github.com/moby/buildkit/go.mod index 0d238c760f..cf1c7d6711 100644 --- a/vendor/github.com/moby/buildkit/go.mod +++ b/vendor/github.com/moby/buildkit/go.mod @@ -1,10 +1,11 @@ module github.com/moby/buildkit -go 1.11 +go 1.12 require ( github.com/BurntSushi/toml v0.3.1 github.com/Microsoft/go-winio v0.4.13-0.20190408173621-84b4ab48a507 + github.com/Microsoft/hcsshim v0.8.5 // indirect github.com/apache/thrift v0.0.0-20161221203622-b2a4d4ae21c7 // indirect github.com/codahale/hdrhistogram v0.0.0-20160425231609-f8ad88b59a58 // indirect github.com/containerd/cgroups v0.0.0-20190226200435-dbea6f2bd416 // indirect @@ -53,7 +54,7 @@ require ( github.com/sirupsen/logrus v1.3.0 github.com/stretchr/testify v1.3.0 github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2 // indirect - github.com/tonistiigi/fsutil v0.0.0-20190819224149-3d2716dd0a4d + github.com/tonistiigi/fsutil v0.0.0-20200128191323-6c909ab392c1 github.com/tonistiigi/units v0.0.0-20180711220420-6950e57a87ea github.com/uber/jaeger-client-go v0.0.0-20180103221425-e02c85f9069e github.com/uber/jaeger-lib v1.2.1 // indirect diff --git a/vendor/github.com/moby/buildkit/session/filesync/filesync.go b/vendor/github.com/moby/buildkit/session/filesync/filesync.go index a45abe02e7..51dd3c5383 100644 --- a/vendor/github.com/moby/buildkit/session/filesync/filesync.go +++ b/vendor/github.com/moby/buildkit/session/filesync/filesync.go @@ -255,7 +255,7 @@ func (sp *fsSyncTarget) Register(server *grpc.Server) { RegisterFileSendServer(server, sp) } -func (sp *fsSyncTarget) DiffCopy(stream FileSend_DiffCopyServer) error { +func (sp *fsSyncTarget) DiffCopy(stream FileSend_DiffCopyServer) (err error) { if sp.outdir != "" { return syncTargetDiffCopy(stream, sp.outdir) } @@ -277,7 +277,12 @@ func (sp *fsSyncTarget) DiffCopy(stream FileSend_DiffCopyServer) error { if wc == nil { return status.Errorf(codes.AlreadyExists, "target already exists") } - defer wc.Close() + defer func() { + err1 := wc.Close() + if err != nil { + err = err1 + } + }() return writeTargetFile(stream, wc) } diff --git a/vendor/github.com/moby/buildkit/session/sshforward/ssh.go b/vendor/github.com/moby/buildkit/session/sshforward/ssh.go index 0001f59b5f..a7a4c2e228 100644 --- a/vendor/github.com/moby/buildkit/session/sshforward/ssh.go +++ b/vendor/github.com/moby/buildkit/session/sshforward/ssh.go @@ -75,6 +75,10 @@ func MountSSHSocket(ctx context.Context, c session.Caller, opt SocketOpt) (sockP } }() + if err := os.Chmod(dir, 0711); err != nil { + return "", nil, errors.WithStack(err) + } + sockPath = filepath.Join(dir, "ssh_auth_sock") l, err := net.Listen("unix", sockPath) diff --git a/vendor/github.com/moby/buildkit/session/sshforward/sshprovider/agentprovider.go b/vendor/github.com/moby/buildkit/session/sshforward/sshprovider/agentprovider.go index 7aa3e3dfc2..4f569799f7 100644 --- a/vendor/github.com/moby/buildkit/session/sshforward/sshprovider/agentprovider.go +++ b/vendor/github.com/moby/buildkit/session/sshforward/sshprovider/agentprovider.go @@ -178,7 +178,7 @@ type sock struct { } type readOnlyAgent struct { - agent.Agent + agent.ExtendedAgent } func (a *readOnlyAgent) Add(_ agent.AddedKey) error { @@ -196,3 +196,7 @@ func (a *readOnlyAgent) RemoveAll() error { func (a *readOnlyAgent) Lock(_ []byte) error { return errors.Errorf("locking agent not allowed by buildkit") } + +func (a *readOnlyAgent) Extension(_ string, _ []byte) ([]byte, error) { + return nil, errors.Errorf("extensions not allowed by buildkit") +} diff --git a/vendor/github.com/moby/buildkit/util/progress/progressui/display.go b/vendor/github.com/moby/buildkit/util/progress/progressui/display.go index e7fee54df5..93e5b6a211 100644 --- a/vendor/github.com/moby/buildkit/util/progress/progressui/display.go +++ b/vendor/github.com/moby/buildkit/util/progress/progressui/display.go @@ -548,6 +548,9 @@ func align(l, r string, w int) string { } func wrapHeight(j []*job, limit int) []*job { + if limit < 0 { + return nil + } var wrapped []*job wrapped = append(wrapped, j...) if len(j) > limit {