Guest24897
/
DockerCLI
mirror of https://github.com/docker/cli.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

build: add experimental --no-console flag to support non-tty human-readable output with buildkit 

Unfortunately, this is for now the only way to see the output of RUN commands when using buildkit.
It is equivalent to `DOCKER_BUILDKIT=1 docker build . 2>&1 | cat`

Signed-off-by: Tibor Vass <tibor@docker.com>
This commit is contained in:
Tibor Vass 2018-06-09 01:07:42 +00:00
parent 584d59d8f5
commit ed75f6202b
22 changed files with 667 additions and 118 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
cli/command/image/build.go
View File

@ -57,6 +57,7 @@ type buildOptions struct {
isolation string
quiet bool
noCache bool
noConsole bool
rm bool
forceRm bool
pull bool
@ -151,6 +152,9 @@ func NewBuildCommand(dockerCli command.Cli) *cobra.Command {
flags.SetAnnotation("stream", "experimental", nil)
flags.SetAnnotation("stream", "version", []string{"1.31"})
flags.BoolVar(&options.noConsole, "no-console", false, "Show non-console output (with buildkit only)")
flags.SetAnnotation("no-console", "experimental", nil)
flags.SetAnnotation("no-console", "version", []string{"1.38"})
return cmd
}

24
cli/command/image/build_buildkit.go
View File

@ -212,23 +212,15 @@ func runBuildBuildKit(dockerCli command.Cli, options buildOptions) error {
ssArr := []*client.SolveStatus{}
displayStatus := func(displayCh chan *client.SolveStatus) {
if c, err := console.ConsoleFromFile(os.Stderr); err == nil {
// not using shared context to not disrupt display but let is finish reporting errors
eg.Go(func() error {
return progressui.DisplaySolveStatus(context.TODO(), c, displayCh)
})
} else {
// read from t.displayCh and send json to Stderr
eg.Go(func() error {
enc := json.NewEncoder(os.Stderr)
for ss := range displayCh {
if err := enc.Encode(ss); err != nil {
return err
}
}
return nil
})
var c console.Console
out := os.Stderr
if cons, err := console.ConsoleFromFile(out); err == nil && !options.noConsole {
c = cons
}
// not using shared context to not disrupt display but let is finish reporting errors
eg.Go(func() error {
return progressui.DisplaySolveStatus(context.TODO(), c, out, displayCh)
})
}
if options.quiet {

4
vendor.conf
View File

@ -49,7 +49,7 @@ github.com/matttproud/golang_protobuf_extensions v1.0.0
github.com/Microsoft/go-winio v0.4.6
github.com/miekg/pkcs11 5f6e0d0dad6f472df908c8e968a98ef00c9224bb
github.com/mitchellh/mapstructure f3009df150dadf309fdee4a54ed65c124afad715
github.com/moby/buildkit 43e758232a0ac7d50c6a11413186e16684fc1e4f
github.com/moby/buildkit 8ebd8cbe691a7047a1206e87564f363567d2f77b
github.com/morikuni/aec 39771216ff4c63d11f5e604076f9c45e8be1067b
github.com/Nvveen/Gotty a8b993ba6abdb0e0c12b0125c603323a71c7790c https://github.com/ijc25/Gotty
github.com/opencontainers/go-digest v1.0.0-rc1
@ -71,7 +71,7 @@ github.com/sirupsen/logrus v1.0.3
github.com/spf13/cobra v0.0.3
github.com/spf13/pflag v1.0.1
github.com/theupdateframework/notary v0.6.1
github.com/tonistiigi/fsutil dc68c74458923f357474a9178bd198aa3ed11a5f
github.com/tonistiigi/fsutil 8839685ae8c3c8bd67d0ce28e9b3157b23c1c7a5
github.com/xeipuuv/gojsonpointer e0fe6f68307607d540ed8eac07a342c33fa1b54a
github.com/xeipuuv/gojsonreference e02fc20de94c78484cd5ffb007f8af96be030a45
github.com/xeipuuv/gojsonschema 93e72a773fade158921402d6a24c819b48aba29d

8
vendor/github.com/moby/buildkit/client/client.go generated vendored
View File

@ -1,6 +1,7 @@
package client
import (
"context"
"crypto/tls"
"crypto/x509"
"io/ioutil"
@ -24,7 +25,6 @@ type ClientOpt interface{}
// New returns a new buildkit client. Address can be empty for the system-default address.
func New(address string, opts ...ClientOpt) (*Client, error) {
gopts := []grpc.DialOption{
grpc.WithTimeout(30 * time.Second),
grpc.WithDialer(dialer),
grpc.FailOnNonTempDialError(true),
}
@ -53,7 +53,11 @@ func New(address string, opts ...ClientOpt) (*Client, error) {
if address == "" {
address = appdefaults.Address
}
conn, err := grpc.Dial(address, gopts...)
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
conn, err := grpc.DialContext(ctx, address, gopts...)
if err != nil {
return nil, errors.Wrapf(err, "failed to dial %q . make sure buildkitd is running", address)
}

15
vendor/github.com/moby/buildkit/client/llb/source.go generated vendored
View File

@ -210,6 +210,9 @@ func Local(name string, opts ...LocalOption) State {
if gi.IncludePatterns != "" {
attrs[pb.AttrIncludePatterns] = gi.IncludePatterns
}
if gi.FollowPaths != "" {
attrs[pb.AttrFollowPaths] = gi.FollowPaths
}
if gi.ExcludePatterns != "" {
attrs[pb.AttrExcludePatterns] = gi.ExcludePatterns
}
@ -248,6 +251,17 @@ func IncludePatterns(p []string) LocalOption {
})
}
func FollowPaths(p []string) LocalOption {
return localOptionFunc(func(li *LocalInfo) {
if len(p) == 0 {
li.FollowPaths = ""
return
}
dt, _ := json.Marshal(p) // empty on error
li.FollowPaths = string(dt)
})
}
func ExcludePatterns(p []string) LocalOption {
return localOptionFunc(func(li *LocalInfo) {
if len(p) == 0 {
@ -270,6 +284,7 @@ type LocalInfo struct {
SessionID string
IncludePatterns string
ExcludePatterns string
FollowPaths string
SharedKeyHint string
}

3
vendor/github.com/moby/buildkit/session/filesync/diffcopy.go generated vendored
View File

@ -12,10 +12,11 @@ import (
"google.golang.org/grpc"
)
func sendDiffCopy(stream grpc.Stream, dir string, includes, excludes []string, progress progressCb, _map func(*fsutil.Stat) bool) error {
func sendDiffCopy(stream grpc.Stream, dir string, includes, excludes, followPaths []string, progress progressCb, _map func(*fsutil.Stat) bool) error {
return fsutil.Send(stream.Context(), stream, dir, &fsutil.WalkOpt{
ExcludePatterns: excludes,
IncludePatterns: includes,
FollowPaths: followPaths,
Map: _map,
}, progress)
}

14
vendor/github.com/moby/buildkit/session/filesync/filesync.go generated vendored
View File

@ -18,6 +18,7 @@ const (
keyOverrideExcludes = "override-excludes"
keyIncludePatterns = "include-patterns"
keyExcludePatterns = "exclude-patterns"
keyFollowPaths = "followpaths"
keyDirName = "dir-name"
)
@ -87,6 +88,8 @@ func (sp *fsSyncProvider) handle(method string, stream grpc.ServerStream) (retEr
}
includes := opts[keyIncludePatterns]
followPaths := opts[keyFollowPaths]
var progress progressCb
if sp.p != nil {
progress = sp.p
@ -98,7 +101,7 @@ func (sp *fsSyncProvider) handle(method string, stream grpc.ServerStream) (retEr
doneCh = sp.doneCh
sp.doneCh = nil
}
err := pr.sendFn(stream, dir.Dir, includes, excludes, progress, dir.Map)
err := pr.sendFn(stream, dir.Dir, includes, excludes, followPaths, progress, dir.Map)
if doneCh != nil {
if err != nil {
doneCh <- err
@ -117,7 +120,7 @@ type progressCb func(int, bool)
type protocol struct {
name string
sendFn func(stream grpc.Stream, srcDir string, includes, excludes []string, progress progressCb, _map func(*fsutil.Stat) bool) error
sendFn func(stream grpc.Stream, srcDir string, includes, excludes, followPaths []string, progress progressCb, _map func(*fsutil.Stat) bool) error
recvFn func(stream grpc.Stream, destDir string, cu CacheUpdater, progress progressCb) error
}
@ -142,6 +145,7 @@ type FSSendRequestOpt struct {
Name string
IncludePatterns []string
ExcludePatterns []string
FollowPaths []string
OverrideExcludes bool // deprecated: this is used by docker/cli for automatically loading .dockerignore from the directory
DestDir string
CacheUpdater CacheUpdater
@ -181,6 +185,10 @@ func FSSync(ctx context.Context, c session.Caller, opt FSSendRequestOpt) error {
opts[keyExcludePatterns] = opt.ExcludePatterns
}
if opt.FollowPaths != nil {
opts[keyFollowPaths] = opt.FollowPaths
}
opts[keyDirName] = []string{opt.Name}
ctx, cancel := context.WithCancel(ctx)
@ -261,7 +269,7 @@ func CopyToCaller(ctx context.Context, srcPath string, c session.Caller, progres
return err
}
return sendDiffCopy(cc, srcPath, nil, nil, progress, nil)
return sendDiffCopy(cc, srcPath, nil, nil, nil, progress, nil)
}
func CopyFileWriter(ctx context.Context, c session.Caller) (io.WriteCloser, error) {

1
vendor/github.com/moby/buildkit/solver/pb/attr.go generated vendored
View File

@ -4,6 +4,7 @@ const AttrKeepGitDir = "git.keepgitdir"
const AttrFullRemoteURL = "git.fullurl"
const AttrLocalSessionID = "local.session"
const AttrIncludePatterns = "local.includepattern"
const AttrFollowPaths = "local.followpaths"
const AttrExcludePatterns = "local.excludepatterns"
const AttrSharedKeyHint = "local.sharedkeyhint"
const AttrLLBDefinitionFilename = "llbbuild.filename"

157
vendor/github.com/moby/buildkit/util/progress/progressui/display.go generated vendored
View File

@ -1,6 +1,7 @@
package progressui
import (
"bytes"
"context"
"fmt"
"io"
@ -15,17 +16,21 @@ import (
"golang.org/x/time/rate"
)
func DisplaySolveStatus(ctx context.Context, c console.Console, ch chan *client.SolveStatus) error {
disp := &display{c: c}
func DisplaySolveStatus(ctx context.Context, c console.Console, w io.Writer, ch chan *client.SolveStatus) error {
t := newTrace()
modeConsole := c != nil
disp := &display{c: c}
printer := &textMux{w: w}
t := newTrace(w)
var done bool
ticker := time.NewTicker(100 * time.Millisecond)
defer ticker.Stop()
displayLimiter := rate.NewLimiter(rate.Every(70*time.Millisecond), 1)
var done bool
for {
select {
case <-ctx.Done():
@ -39,12 +44,21 @@ func DisplaySolveStatus(ctx context.Context, c console.Console, ch chan *client.
}
}
if done {
disp.print(t.displayInfo(), true)
t.printErrorLogs(c)
return nil
} else if displayLimiter.Allow() {
disp.print(t.displayInfo(), false)
if modeConsole {
if done {
disp.print(t.displayInfo(), true)
t.printErrorLogs(c)
return nil
} else if displayLimiter.Allow() {
disp.print(t.displayInfo(), false)
}
} else {
if done || displayLimiter.Allow() {
printer.print(t)
if done {
return nil
}
}
}
}
}
@ -66,37 +80,108 @@ type job struct {
}
type trace struct {
w io.Writer
localTimeDiff time.Duration
vertexes []*vertex
byDigest map[digest.Digest]*vertex
nextIndex int
updates map[digest.Digest]struct{}
}
type vertex struct {
*client.Vertex
statuses []*status
byID map[string]*status
logs []*client.VertexLog
indent string
index int
logs [][]byte
logsPartial bool
logsOffset int
prev *client.Vertex
events []string
lastBlockTime *time.Time
count int
statusUpdates map[string]struct{}
}
func (v *vertex) update(c int) {
if v.count == 0 {
now := time.Now()
v.lastBlockTime = &now
}
v.count += c
}
type status struct {
*client.VertexStatus
}
func newTrace() *trace {
func newTrace(w io.Writer) *trace {
return &trace{
byDigest: make(map[digest.Digest]*vertex),
updates: make(map[digest.Digest]struct{}),
w: w,
}
}
func (t *trace) triggerVertexEvent(v *client.Vertex) {
if v.Started == nil {
return
}
var old client.Vertex
vtx := t.byDigest[v.Digest]
if v := vtx.prev; v != nil {
old = *v
}
var ev []string
if v.Digest != old.Digest {
ev = append(ev, fmt.Sprintf("%13s %s", "digest:", v.Digest))
}
if v.Name != old.Name {
ev = append(ev, fmt.Sprintf("%13s %q", "name:", v.Name))
}
if v.Started != old.Started {
if v.Started != nil && old.Started == nil || !v.Started.Equal(*old.Started) {
ev = append(ev, fmt.Sprintf("%13s %v", "started:", v.Started))
}
}
if v.Completed != old.Completed && v.Completed != nil {
ev = append(ev, fmt.Sprintf("%13s %v", "completed:", v.Completed))
if v.Started != nil {
ev = append(ev, fmt.Sprintf("%13s %v", "duration:", v.Completed.Sub(*v.Started)))
}
}
if v.Cached != old.Cached {
ev = append(ev, fmt.Sprintf("%13s %v", "cached:", v.Cached))
}
if v.Error != old.Error {
ev = append(ev, fmt.Sprintf("%13s %q", "error:", v.Error))
}
if len(ev) > 0 {
vtx.events = append(vtx.events, ev...)
vtx.update(len(ev))
t.updates[v.Digest] = struct{}{}
}
t.byDigest[v.Digest].prev = v
}
func (t *trace) update(s *client.SolveStatus) {
for _, v := range s.Vertexes {
prev, ok := t.byDigest[v.Digest]
if !ok {
t.nextIndex++
t.byDigest[v.Digest] = &vertex{
byID: make(map[string]*status),
byID: make(map[string]*status),
statusUpdates: make(map[string]struct{}),
index: t.nextIndex,
}
}
t.triggerVertexEvent(v)
if v.Started != nil && (prev == nil || prev.Started == nil) {
if t.localTimeDiff == 0 {
t.localTimeDiff = time.Since(*v.Started)
@ -118,13 +203,29 @@ func (t *trace) update(s *client.SolveStatus) {
v.statuses = append(v.statuses, v.byID[s.ID])
}
v.byID[s.ID].VertexStatus = s
v.statusUpdates[s.ID] = struct{}{}
t.updates[v.Digest] = struct{}{}
v.update(1)
}
for _, l := range s.Logs {
v, ok := t.byDigest[l.Vertex]
if !ok {
continue // shouldn't happen
}
v.logs = append(v.logs, l)
complete := split(l.Data, byte('\n'), func(dt []byte) {
if v.logsPartial && len(v.logs) != 0 {
v.logs[len(v.logs)-1] = append(v.logs[len(v.logs)-1], dt...)
} else {
ts := time.Duration(0)
if v.Started != nil {
ts = l.Timestamp.Sub(*v.Started)
}
v.logs = append(v.logs, []byte(fmt.Sprintf("#%d %s %s", v.index, fmt.Sprintf("%#.4g", ts.Seconds())[:5], dt)))
}
})
v.logsPartial = !complete
t.updates[v.Digest] = struct{}{}
v.update(1)
}
}
@ -134,12 +235,8 @@ func (t *trace) printErrorLogs(f io.Writer) {
fmt.Fprintln(f, "------")
fmt.Fprintf(f, " > %s:\n", v.Name)
for _, l := range v.logs {
switch l.Stream {
case 1:
f.Write(l.Data)
case 2:
f.Write(l.Data)
}
f.Write(l)
fmt.Fprintln(f)
}
fmt.Fprintln(f, "------")
}
@ -196,6 +293,24 @@ func (t *trace) displayInfo() (d displayInfo) {
return d
}
func split(dt []byte, sep byte, fn func([]byte)) bool {
if len(dt) == 0 {
return false
}
for {
if len(dt) == 0 {
return true
}
idx := bytes.IndexByte(dt, sep)
if idx == -1 {
fn(dt)
return false
}
fn(dt[:idx])
dt = dt[idx+1:]
}
}
func addTime(tm *time.Time, d time.Duration) *time.Time {
if tm == nil {
return nil

214
vendor/github.com/moby/buildkit/util/progress/progressui/printer.go generated vendored Normal file
View File

@ -0,0 +1,214 @@
package progressui
import (
"fmt"
"io"
"time"
digest "github.com/opencontainers/go-digest"
"github.com/tonistiigi/units"
)
const antiFlicker = 5 * time.Second
const maxDelay = 10 * time.Second
type textMux struct {
w io.Writer
current digest.Digest
}
func (p *textMux) printVtx(t *trace, dgst digest.Digest) {
v, ok := t.byDigest[dgst]
if !ok {
return
}
if dgst != p.current {
if p.current != "" {
old := t.byDigest[p.current]
if old.logsPartial {
fmt.Fprintln(p.w, "")
}
old.logsOffset = 0
old.count = 0
fmt.Fprintf(p.w, "#%d ...\n", v.index)
}
fmt.Fprintf(p.w, "\n#%d %s\n", v.index, limitString(v.Name, 72))
}
if len(v.events) != 0 {
v.logsOffset = 0
}
for _, ev := range v.events {
fmt.Fprintf(p.w, "#%d %s\n", v.index, ev)
}
v.events = v.events[:0]
for _, s := range v.statuses {
if _, ok := v.statusUpdates[s.ID]; ok {
var bytes string
if s.Total != 0 {
bytes = fmt.Sprintf(" %.2f / %.2f", units.Bytes(s.Current), units.Bytes(s.Total))
} else if s.Current != 0 {
bytes = fmt.Sprintf(" %.2f", units.Bytes(s.Current))
}
var tm string
endTime := s.Timestamp
if s.Completed != nil {
endTime = *s.Completed
}
if s.Started != nil {
diff := endTime.Sub(*s.Started).Seconds()
if diff > 0.01 {
tm = fmt.Sprintf(" %.1fs", diff)
}
}
if s.Completed != nil {
tm += " done"
}
fmt.Fprintf(p.w, "#%d %s%s%s\n", v.index, s.ID, bytes, tm)
}
}
v.statusUpdates = map[string]struct{}{}
for i, l := range v.logs {
if i == 0 {
l = l[v.logsOffset:]
}
fmt.Fprintf(p.w, "%s", []byte(l))
if i != len(v.logs)-1 || !v.logsPartial {
fmt.Fprintln(p.w, "")
}
}
if len(v.logs) > 0 {
if v.logsPartial {
v.logs = v.logs[len(v.logs)-1:]
v.logsOffset = len(v.logs[0])
} else {
v.logs = nil
v.logsOffset = 0
}
}
p.current = dgst
if v.Completed != nil {
p.current = ""
v.count = 0
fmt.Fprintf(p.w, "\n")
}
delete(t.updates, dgst)
}
func (p *textMux) print(t *trace) {
completed := map[digest.Digest]struct{}{}
rest := map[digest.Digest]struct{}{}
for dgst := range t.updates {
v, ok := t.byDigest[dgst]
if !ok {
continue
}
if v.Vertex.Completed != nil {
completed[dgst] = struct{}{}
} else {
rest[dgst] = struct{}{}
}
}
current := p.current
// items that have completed need to be printed first
if _, ok := completed[current]; ok {
p.printVtx(t, current)
}
for dgst := range completed {
if dgst != current {
p.printVtx(t, dgst)
}
}
if len(rest) == 0 {
if current != "" {
if v := t.byDigest[current]; v.Started != nil && v.Completed == nil {
return
}
}
// make any open vertex active
for dgst, v := range t.byDigest {
if v.Started != nil && v.Completed == nil {
p.printVtx(t, dgst)
return
}
}
return
}
// now print the active one
if _, ok := rest[current]; ok {
p.printVtx(t, current)
}
stats := map[digest.Digest]*vtxStat{}
now := time.Now()
sum := 0.0
var max digest.Digest
if current != "" {
rest[current] = struct{}{}
}
for dgst := range rest {
v, ok := t.byDigest[dgst]
if !ok {
continue
}
tm := now.Sub(*v.lastBlockTime)
speed := float64(v.count) / tm.Seconds()
overLimit := tm > maxDelay && dgst != current
stats[dgst] = &vtxStat{blockTime: tm, speed: speed, overLimit: overLimit}
sum += speed
if overLimit || max == "" || stats[max].speed < speed {
max = dgst
}
}
for dgst := range stats {
stats[dgst].share = stats[dgst].speed / sum
}
if _, ok := completed[current]; ok || current == "" {
p.printVtx(t, max)
return
}
// show items that were hidden
for dgst := range rest {
if stats[dgst].overLimit {
p.printVtx(t, dgst)
return
}
}
// fair split between vertexes
if 1.0/(1.0-stats[current].share)*antiFlicker.Seconds() < stats[current].blockTime.Seconds() {
p.printVtx(t, max)
return
}
}
type vtxStat struct {
blockTime time.Duration
speed float64
share float64
overLimit bool
}
func limitString(s string, l int) string {
if len(s) > l {
return s[:l] + "..."
}
return s
}

2
vendor/github.com/moby/buildkit/util/system/seccomp_linux.go generated vendored
View File

@ -1,4 +1,4 @@
// +build linux
// +build linux,seccomp
package system

2
vendor/github.com/moby/buildkit/util/system/seccomp_nolinux.go generated vendored
View File

@ -1,4 +1,4 @@
// +build !linux
// +build !linux,seccomp
package system

7
vendor/github.com/moby/buildkit/util/system/seccomp_noseccomp.go generated vendored Normal file
View File

@ -0,0 +1,7 @@
// +build !seccomp
package system
func SeccompSupported() bool {
return false
}

7
vendor/github.com/moby/buildkit/vendor.conf generated vendored
View File

@ -6,7 +6,7 @@ github.com/davecgh/go-spew v1.1.0
github.com/pmezard/go-difflib v1.0.0
golang.org/x/sys 314a259e304ff91bd6985da2a7149bbf91237993
github.com/containerd/containerd e1428ef05460da40720d622c803262e6fc8d3477
github.com/containerd/containerd 63522d9eaa5a0443d225642c4b6f4f5fdedf932b
github.com/containerd/typeurl f6943554a7e7e88b3c14aad190bf05932da84788
golang.org/x/sync 450f422ab23cf9881c94e2db30cac0eb1b7cf80c
github.com/sirupsen/logrus v1.0.0
@ -23,7 +23,7 @@ github.com/Microsoft/go-winio v0.4.7
github.com/containerd/fifo 3d5202aec260678c48179c56f40e6f38a095738c
github.com/opencontainers/runtime-spec v1.0.1
github.com/containerd/go-runc f271fa2021de855d4d918dbef83c5fe19db1bdd5
github.com/containerd/console cb7008ab3d8359b78c5f464cb7cf160107ad5925
github.com/containerd/console 9290d21dc56074581f619579c43d970b4514bc08
google.golang.org/genproto d80a6e20e776b0b17a324d0ba1ab50a39c8e8944
golang.org/x/text 19e51611da83d6be54ddafce4a4af510cb3e9ea4
github.com/docker/go-events 9461782956ad83b30282bf90e31fa6a70c255ba9
@ -36,11 +36,10 @@ github.com/docker/go-units v0.3.1
github.com/google/shlex 6f45313302b9c56850fc17f99e40caebce98c716
golang.org/x/time f51c12702a4d776e4c1fa9b0fabab841babae631
github.com/BurntSushi/locker a6e239ea1c69bff1cfdb20c4b73dadf52f784b6a
github.com/docker/docker 71cd53e4a197b303c6ba086bd584ffd67a884281
github.com/pkg/profile 5b67d428864e92711fcbd2f8629456121a56d91f
github.com/tonistiigi/fsutil dc68c74458923f357474a9178bd198aa3ed11a5f
github.com/tonistiigi/fsutil 8839685ae8c3c8bd67d0ce28e9b3157b23c1c7a5
github.com/hashicorp/go-immutable-radix 826af9ccf0feeee615d546d69b11f8e98da8c8f1 git://github.com/tonistiigi/go-immutable-radix.git
github.com/hashicorp/golang-lru a0d98a5f288019575c6d1f4bb1573fef2d1fcdc4
github.com/mitchellh/hashstructure 2bca23e0e452137f789efbc8610126fd8b94f73b

7
vendor/github.com/tonistiigi/fsutil/chtimes_nolinux.go generated vendored
View File

@ -9,5 +9,12 @@ import (
func chtimes(path string, un int64) error {
mtime := time.Unix(0, un)
fi, err := os.Lstat(path)
if err != nil {
return err
}
if fi.Mode()&os.ModeSymlink != 0 {
return nil
}
return os.Chtimes(path, mtime, mtime)
}

3
vendor/github.com/tonistiigi/fsutil/diff.go generated vendored
View File

@ -1,10 +1,9 @@
package fsutil
import (
"context"
"hash"
"os"
"golang.org/x/net/context"
)
type walkerFn func(ctx context.Context, pathC chan<- *currentPath) error

2
vendor/github.com/tonistiigi/fsutil/diff_containerd.go generated vendored
View File

@ -1,10 +1,10 @@
package fsutil
import (
"context"
"os"
"strings"
"golang.org/x/net/context"
"golang.org/x/sync/errgroup"
)

53
vendor/github.com/tonistiigi/fsutil/diskwriter.go generated vendored
View File

@ -1,6 +1,7 @@
package fsutil
import (
"context"
"hash"
"io"
"os"
@ -9,9 +10,8 @@ import (
"sync"
"time"
digest "github.com/opencontainers/go-digest"
"github.com/opencontainers/go-digest"
"github.com/pkg/errors"
"golang.org/x/net/context"
"golang.org/x/sync/errgroup"
)
@ -80,9 +80,7 @@ func (dw *DiskWriter) HandleChange(kind ChangeKind, p string, fi os.FileInfo, er
}
}()
p = filepath.FromSlash(p)
destPath := filepath.Join(dw.dest, p)
destPath := filepath.Join(dw.dest, filepath.FromSlash(p))
if kind == ChangeKindDelete {
// todo: no need to validate if diff is trusted but is it always?
@ -102,8 +100,10 @@ func (dw *DiskWriter) HandleChange(kind ChangeKind, p string, fi os.FileInfo, er
return errors.Errorf("%s invalid change without stat information", p)
}
statCopy := *stat
if dw.filter != nil {
if ok := dw.filter(stat); !ok {
if ok := dw.filter(&statCopy); !ok {
return nil
}
}
@ -122,7 +122,7 @@ func (dw *DiskWriter) HandleChange(kind ChangeKind, p string, fi os.FileInfo, er
}
if oldFi != nil && fi.IsDir() && oldFi.IsDir() {
if err := rewriteMetadata(destPath, stat); err != nil {
if err := rewriteMetadata(destPath, &statCopy); err != nil {
return errors.Wrapf(err, "error setting dir metadata for %s", destPath)
}
return nil
@ -141,16 +141,16 @@ func (dw *DiskWriter) HandleChange(kind ChangeKind, p string, fi os.FileInfo, er
return errors.Wrapf(err, "failed to create dir %s", newPath)
}
case fi.Mode()&os.ModeDevice != 0 || fi.Mode()&os.ModeNamedPipe != 0:
if err := handleTarTypeBlockCharFifo(newPath, stat); err != nil {
if err := handleTarTypeBlockCharFifo(newPath, &statCopy); err != nil {
return errors.Wrapf(err, "failed to create device %s", newPath)
}
case fi.Mode()&os.ModeSymlink != 0:
if err := os.Symlink(stat.Linkname, newPath); err != nil {
if err := os.Symlink(statCopy.Linkname, newPath); err != nil {
return errors.Wrapf(err, "failed to symlink %s", newPath)
}
case stat.Linkname != "":
if err := os.Link(filepath.Join(dw.dest, stat.Linkname), newPath); err != nil {
return errors.Wrapf(err, "failed to link %s to %s", newPath, stat.Linkname)
case statCopy.Linkname != "":
if err := os.Link(filepath.Join(dw.dest, statCopy.Linkname), newPath); err != nil {
return errors.Wrapf(err, "failed to link %s to %s", newPath, statCopy.Linkname)
}
default:
isRegularFile = true
@ -170,7 +170,7 @@ func (dw *DiskWriter) HandleChange(kind ChangeKind, p string, fi os.FileInfo, er
}
}
if err := rewriteMetadata(newPath, stat); err != nil {
if err := rewriteMetadata(newPath, &statCopy); err != nil {
return errors.Wrapf(err, "error setting metadata for %s", newPath)
}
@ -272,14 +272,27 @@ func (hw *hashedWriter) Digest() digest.Digest {
}
type lazyFileWriter struct {
dest string
ctx context.Context
f *os.File
dest string
ctx context.Context
f *os.File
fileMode *os.FileMode
}
func (lfw *lazyFileWriter) Write(dt []byte) (int, error) {
if lfw.f == nil {
file, err := os.OpenFile(lfw.dest, os.O_WRONLY, 0) //todo: windows
if os.IsPermission(err) {
// retry after chmod
fi, er := os.Stat(lfw.dest)
if er == nil {
mode := fi.Mode()
lfw.fileMode = &mode
er = os.Chmod(lfw.dest, mode|0222)
if er == nil {
file, err = os.OpenFile(lfw.dest, os.O_WRONLY, 0)
}
}
}
if err != nil {
return 0, errors.Wrapf(err, "failed to open %s", lfw.dest)
}
@ -289,10 +302,14 @@ func (lfw *lazyFileWriter) Write(dt []byte) (int, error) {
}
func (lfw *lazyFileWriter) Close() error {
var err error
if lfw.f != nil {
return lfw.f.Close()
err = lfw.f.Close()
}
return nil
if err == nil && lfw.fileMode != nil {
err = os.Chmod(lfw.dest, *lfw.fileMode)
}
return err
}
func mkdev(major int64, minor int64) uint32 {

150
vendor/github.com/tonistiigi/fsutil/followlinks.go generated vendored Normal file
View File

@ -0,0 +1,150 @@
package fsutil
import (
"io/ioutil"
"os"
"path/filepath"
"runtime"
"sort"
strings "strings"
"github.com/pkg/errors"
)
func FollowLinks(root string, paths []string) ([]string, error) {
r := &symlinkResolver{root: root, resolved: map[string]struct{}{}}
for _, p := range paths {
if err := r.append(p); err != nil {
return nil, err
}
}
res := make([]string, 0, len(r.resolved))
for r := range r.resolved {
res = append(res, r)
}
sort.Strings(res)
return dedupePaths(res), nil
}
type symlinkResolver struct {
root string
resolved map[string]struct{}
}
func (r *symlinkResolver) append(p string) error {
p = filepath.Join(".", p)
current := "."
for {
parts := strings.SplitN(p, string(filepath.Separator), 2)
current = filepath.Join(current, parts[0])
targets, err := r.readSymlink(current, true)
if err != nil {
return err
}
p = ""
if len(parts) == 2 {
p = parts[1]
}
if p == "" || targets != nil {
if _, ok := r.resolved[current]; ok {
return nil
}
}
if targets != nil {
r.resolved[current] = struct{}{}
for _, target := range targets {
if err := r.append(filepath.Join(target, p)); err != nil {
return err
}
}
return nil
}
if p == "" {
r.resolved[current] = struct{}{}
return nil
}
}
}
func (r *symlinkResolver) readSymlink(p string, allowWildcard bool) ([]string, error) {
realPath := filepath.Join(r.root, p)
base := filepath.Base(p)
if allowWildcard && containsWildcards(base) {
fis, err := ioutil.ReadDir(filepath.Dir(realPath))
if err != nil {
if os.IsNotExist(err) {
return nil, nil
}
return nil, errors.Wrapf(err, "failed to read dir %s", filepath.Dir(realPath))
}
var out []string
for _, f := range fis {
if ok, _ := filepath.Match(base, f.Name()); ok {
res, err := r.readSymlink(filepath.Join(filepath.Dir(p), f.Name()), false)
if err != nil {
return nil, err
}
out = append(out, res...)
}
}
return out, nil
}
fi, err := os.Lstat(realPath)
if err != nil {
if os.IsNotExist(err) {
return nil, nil
}
return nil, errors.Wrapf(err, "failed to lstat %s", realPath)
}
if fi.Mode()&os.ModeSymlink == 0 {
return nil, nil
}
link, err := os.Readlink(realPath)
if err != nil {
return nil, errors.Wrapf(err, "failed to readlink %s", realPath)
}
link = filepath.Clean(link)
if filepath.IsAbs(link) {
return []string{link}, nil
}
return []string{
filepath.Join(string(filepath.Separator), filepath.Join(filepath.Dir(p), link)),
}, nil
}
func containsWildcards(name string) bool {
isWindows := runtime.GOOS == "windows"
for i := 0; i < len(name); i++ {
ch := name[i]
if ch == '\\' && !isWindows {
i++
} else if ch == '*' || ch == '?' || ch == '[' {
return true
}
}
return false
}
// dedupePaths expects input as a sorted list
func dedupePaths(in []string) []string {
out := make([]string, 0, len(in))
var last string
for _, s := range in {
// if one of the paths is root there is no filter
if s == "." {
return nil
}
if strings.HasPrefix(s, last+string(filepath.Separator)) {
continue
}
out = append(out, s)
last = s
}
return out
}

2
vendor/github.com/tonistiigi/fsutil/receive.go generated vendored
View File

@ -1,12 +1,12 @@
package fsutil
import (
"context"
"io"
"os"
"sync"
"github.com/pkg/errors"
"golang.org/x/net/context"
"golang.org/x/sync/errgroup"
)

2
vendor/github.com/tonistiigi/fsutil/send.go generated vendored
View File

@ -1,13 +1,13 @@
package fsutil
import (
"context"
"io"
"os"
"path/filepath"
"sync"
"github.com/pkg/errors"
"golang.org/x/net/context"
"golang.org/x/sync/errgroup"
)

104
vendor/github.com/tonistiigi/fsutil/walker.go generated vendored
View File

@ -1,6 +1,7 @@
package fsutil
import (
"context"
"os"
"path/filepath"
"runtime"
@ -9,13 +10,15 @@ import (
"github.com/docker/docker/pkg/fileutils"
"github.com/pkg/errors"
"golang.org/x/net/context"
)
type WalkOpt struct {
IncludePatterns []string
ExcludePatterns []string
Map func(*Stat) bool
// FollowPaths contains symlinks that are resolved into include patterns
// before performing the fs walk
FollowPaths []string
Map func(*Stat) bool
}
func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) error {
@ -39,8 +42,25 @@ func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) err
}
}
var includePatterns []string
if opt != nil && opt.IncludePatterns != nil {
includePatterns = make([]string, len(opt.IncludePatterns))
for k := range opt.IncludePatterns {
includePatterns[k] = filepath.Clean(opt.IncludePatterns[k])
}
}
if opt != nil && opt.FollowPaths != nil {
targets, err := FollowLinks(p, opt.FollowPaths)
if err != nil {
return err
}
if targets != nil {
includePatterns = append(includePatterns, targets...)
includePatterns = dedupePaths(includePatterns)
}
}
var lastIncludedDir string
var includePatternPrefixes []string
seenFiles := make(map[uint64]string)
return filepath.Walk(root, func(path string, fi os.FileInfo, err error) (retErr error) {
@ -66,34 +86,31 @@ func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) err
}
if opt != nil {
if opt.IncludePatterns != nil {
if includePatternPrefixes == nil {
includePatternPrefixes = patternPrefixes(opt.IncludePatterns)
}
matched := false
if includePatterns != nil {
skip := false
if lastIncludedDir != "" {
if strings.HasPrefix(path, lastIncludedDir+string(filepath.Separator)) {
matched = true
skip = true
}
}
if !matched {
for _, p := range opt.IncludePatterns {
if m, _ := filepath.Match(p, path); m {
if !skip {
matched := false
partial := true
for _, p := range includePatterns {
if ok, p := matchPrefix(p, path); ok {
matched = true
break
if !p {
partial = false
break
}
}
}
if matched && fi.IsDir() {
lastIncludedDir = path
}
}
if !matched {
if !fi.IsDir() {
if !matched {
return nil
} else {
if noPossiblePrefixMatch(path, includePatternPrefixes) {
return filepath.SkipDir
}
}
if !partial && fi.IsDir() {
lastIncludedDir = path
}
}
}
@ -131,13 +148,13 @@ func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) err
stat := &Stat{
Path: path,
Mode: uint32(fi.Mode()),
Size_: fi.Size(),
ModTime: fi.ModTime().UnixNano(),
}
setUnixOpt(fi, stat, path, seenFiles)
if !fi.IsDir() {
stat.Size_ = fi.Size()
if fi.Mode()&os.ModeSymlink != 0 {
link, err := os.Readlink(origpath)
if err != nil {
@ -199,29 +216,28 @@ func (s *StatInfo) Sys() interface{} {
return s.Stat
}
func patternPrefixes(patterns []string) []string {
pfxs := make([]string, 0, len(patterns))
for _, ptrn := range patterns {
idx := strings.IndexFunc(ptrn, func(ch rune) bool {
return ch == '*' || ch == '?' || ch == '[' || ch == '\\'
})
if idx == -1 {
idx = len(ptrn)
}
pfxs = append(pfxs, ptrn[:idx])
func matchPrefix(pattern, name string) (bool, bool) {
count := strings.Count(name, string(filepath.Separator))
partial := false
if strings.Count(pattern, string(filepath.Separator)) > count {
pattern = trimUntilIndex(pattern, string(filepath.Separator), count)
partial = true
}
return pfxs
m, _ := filepath.Match(pattern, name)
return m, partial
}
func noPossiblePrefixMatch(p string, pfxs []string) bool {
for _, pfx := range pfxs {
chk := p
if len(pfx) < len(p) {
chk = p[:len(pfx)]
}
if strings.HasPrefix(pfx, chk) {
return false
func trimUntilIndex(str, sep string, count int) string {
s := str
i := 0
c := 0
for {
idx := strings.Index(s, sep)
s = s[idx+len(sep):]
i += idx + len(sep)
c++
if c >= count {
return str[:i-len(sep)]
}
}
return true
}