vendor: buildkit 4d1f260e8490ec438ab66e08bb105577aca0ce06

full diff: df35e9818d...4d1f260e84

- moby/buildkit#1551 session: track sessions with a group construct
- moby/buildkit#1534 secrets: allow providing secrets with env
- moby/buildkit#1533 git: support for token authentication
- moby/buildkit#1549 progressui: fix logs time formatting

Signed-off-by: Sebastiaan van Stijn <github@gone.nl>

cli/command/image/build_buildkit.go

@@ -427,7 +427,7 @@ func (t *tracer) write(msg jsonmessage.JSONMessage) {
 }
 
 func parseSecretSpecs(sl []string) (session.Attachable, error) {
-	fs := make([]secretsprovider.FileSource, 0, len(sl))
+	fs := make([]secretsprovider.Source, 0, len(sl))
 	for _, v := range sl {
 		s, err := parseSecret(v)
 		if err != nil {
@@ -435,22 +435,23 @@ func parseSecretSpecs(sl []string) (session.Attachable, error) {
 		}
 		fs = append(fs, *s)
 	}
-	store, err := secretsprovider.NewFileStore(fs)
+	store, err := secretsprovider.NewStore(fs)
 	if err != nil {
 		return nil, err
 	}
 	return secretsprovider.NewSecretProvider(store), nil
 }
 
-func parseSecret(value string) (*secretsprovider.FileSource, error) {
+func parseSecret(value string) (*secretsprovider.Source, error) {
 	csvReader := csv.NewReader(strings.NewReader(value))
 	fields, err := csvReader.Read()
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to parse csv secret")
 	}
 
-	fs := secretsprovider.FileSource{}
+	fs := secretsprovider.Source{}
 
+	var typ string
 	for _, field := range fields {
 		parts := strings.SplitN(field, "=", 2)
 		key := strings.ToLower(parts[0])
@@ -462,17 +463,24 @@ func parseSecret(value string) (*secretsprovider.FileSource, error) {
 		value := parts[1]
 		switch key {
 		case "type":
-			if value != "file" {
+			if value != "file" && value != "env" {
 				return nil, errors.Errorf("unsupported secret type %q", value)
 			}
+			typ = value
 		case "id":
 			fs.ID = value
 		case "source", "src":
 			fs.FilePath = value
+		case "env":
+			fs.Env = value
 		default:
 			return nil, errors.Errorf("unexpected key '%s' in '%s'", key, field)
 		}
 	}
+	if typ == "env" && fs.Env == "" {
+		fs.Env = fs.FilePath
+		fs.FilePath = ""
+	}
 	return &fs, nil
 }
 

cli/command/image/build_test.go

@@ -187,7 +187,7 @@ func TestParseSecret(t *testing.T) {
 		value       string
 		errExpected bool
 		errMatch    string
-		filesource  *secretsprovider.FileSource
+		source      *secretsprovider.Source
 	}
 	var testcases = []testcase{
 		{
@@ -206,23 +206,32 @@ func TestParseSecret(t *testing.T) {
 			errExpected: true,
 			errMatch:    "unexpected key",
 		}, {
-			value:      "src=somefile",
-			filesource: &secretsprovider.FileSource{FilePath: "somefile"},
+			value:  "src=somefile",
+			source: &secretsprovider.Source{FilePath: "somefile"},
 		}, {
-			value:      "source=somefile",
-			filesource: &secretsprovider.FileSource{FilePath: "somefile"},
+			value:  "source=somefile",
+			source: &secretsprovider.Source{FilePath: "somefile"},
 		}, {
-			value:      "id=mysecret",
-			filesource: &secretsprovider.FileSource{ID: "mysecret"},
+			value:  "id=mysecret",
+			source: &secretsprovider.Source{ID: "mysecret"},
 		}, {
-			value:      "id=mysecret,src=somefile",
-			filesource: &secretsprovider.FileSource{ID: "mysecret", FilePath: "somefile"},
+			value:  "id=mysecret,src=somefile",
+			source: &secretsprovider.Source{ID: "mysecret", FilePath: "somefile"},
 		}, {
-			value:      "id=mysecret,source=somefile,type=file",
-			filesource: &secretsprovider.FileSource{ID: "mysecret", FilePath: "somefile"},
+			value:  "id=mysecret,source=somefile,type=file",
+			source: &secretsprovider.Source{ID: "mysecret", FilePath: "somefile"},
 		}, {
-			value:      "id=mysecret,src=somefile,src=othersecretfile",
-			filesource: &secretsprovider.FileSource{ID: "mysecret", FilePath: "othersecretfile"},
+			value:  "id=mysecret,src=somefile,src=othersecretfile",
+			source: &secretsprovider.Source{ID: "mysecret", FilePath: "othersecretfile"},
+		}, {
+			value:  "id=mysecret,src=somefile,env=SECRET",
+			source: &secretsprovider.Source{ID: "mysecret", FilePath: "somefile", Env: "SECRET"},
+		}, {
+			value:  "type=file",
+			source: &secretsprovider.Source{},
+		}, {
+			value:  "type=env",
+			source: &secretsprovider.Source{},
 		}, {
 			value:       "type=invalid",
 			errExpected: true,
@@ -237,7 +246,7 @@ func TestParseSecret(t *testing.T) {
 			if tc.errMatch != "" {
 				assert.ErrorContains(t, err, tc.errMatch)
 			}
-			assert.DeepEqual(t, secret, tc.filesource)
+			assert.DeepEqual(t, secret, tc.source)
 		})
 	}
 }

vendor.conf

@@ -47,7 +47,7 @@ github.com/Microsoft/go-winio                       6c72808b55902eae4c5943626030
 github.com/Microsoft/hcsshim                        5bc557dd210ff2caf615e6e22d398123de77fc11 # v0.8.9
 github.com/miekg/pkcs11                             210dc1e16747c5ba98a03bcbcf728c38086ea357 # v1.0.3
 github.com/mitchellh/mapstructure                   d16e9488127408e67948eb43b6d3fbb9f222da10 # v1.3.2
-github.com/moby/buildkit                            df35e9818d1f9066e616e03f4b8d727c97562e5b
+github.com/moby/buildkit                            4d1f260e8490ec438ab66e08bb105577aca0ce06
 github.com/moby/sys                                 6154f11e6840c0d6b0dbb23f4125a6134b3013c9 # mountinfo/v0.1.3
 github.com/moby/term                                73f35e472e8f0a3f91347164138ce6bd73b756a9
 github.com/modern-go/concurrent                     bacd9c7ef1dd9b15be4a9909b8ac7a4e313eec94 # 1.0.3

vendor/github.com/moby/buildkit/client/llb/source.go

@@ -215,7 +215,10 @@ func Git(remote, ref string, opts ...GitOption) State {
 		id += "#" + ref
 	}
 
-	gi := &GitInfo{}
+	gi := &GitInfo{
+		AuthHeaderSecret: "GIT_AUTH_HEADER",
+		AuthTokenSecret:  "GIT_AUTH_TOKEN",
+	}
 	for _, o := range opts {
 		o.SetGitOption(gi)
 	}
@@ -228,6 +231,14 @@ func Git(remote, ref string, opts ...GitOption) State {
 		attrs[pb.AttrFullRemoteURL] = url
 		addCap(&gi.Constraints, pb.CapSourceGitFullURL)
 	}
+	if gi.AuthTokenSecret != "" {
+		attrs[pb.AttrAuthTokenSecret] = gi.AuthTokenSecret
+		addCap(&gi.Constraints, pb.CapSourceGitHttpAuth)
+	}
+	if gi.AuthHeaderSecret != "" {
+		attrs[pb.AttrAuthHeaderSecret] = gi.AuthHeaderSecret
+		addCap(&gi.Constraints, pb.CapSourceGitHttpAuth)
+	}
 
 	addCap(&gi.Constraints, pb.CapSourceGit)
 
@@ -246,7 +257,9 @@ func (fn gitOptionFunc) SetGitOption(gi *GitInfo) {
 
 type GitInfo struct {
 	constraintsWrapper
-	KeepGitDir bool
+	KeepGitDir       bool
+	AuthTokenSecret  string
+	AuthHeaderSecret string
 }
 
 func KeepGitDir() GitOption {
@@ -255,6 +268,18 @@ func KeepGitDir() GitOption {
 	})
 }
 
+func AuthTokenSecret(v string) GitOption {
+	return gitOptionFunc(func(gi *GitInfo) {
+		gi.AuthTokenSecret = v
+	})
+}
+
+func AuthHeaderSecret(v string) GitOption {
+	return gitOptionFunc(func(gi *GitInfo) {
+		gi.AuthHeaderSecret = v
+	})
+}
+
 func Scratch() State {
 	return NewState(nil)
 }

vendor/github.com/moby/buildkit/frontend/gateway/grpcclient/client.go

@@ -32,7 +32,7 @@ type GrpcClient interface {
 }
 
 func New(ctx context.Context, opts map[string]string, session, product string, c pb.LLBBridgeClient, w []client.WorkerInfo) (GrpcClient, error) {
-	ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
+	ctx, cancel := context.WithTimeout(ctx, 15*time.Second)
 	defer cancel()
 	resp, err := c.Ping(ctx, &pb.PingRequest{})
 	if err != nil {

vendor/github.com/moby/buildkit/session/auth/auth.go

@@ -8,19 +8,28 @@ import (
 	"google.golang.org/grpc/codes"
 )
 
-func CredentialsFunc(ctx context.Context, c session.Caller) func(string) (string, string, error) {
+func CredentialsFunc(sm *session.Manager, g session.Group) func(string) (string, string, error) {
 	return func(host string) (string, string, error) {
-		client := NewAuthClient(c.Conn())
+		var user, secret string
+		err := sm.Any(context.TODO(), g, func(ctx context.Context, _ string, c session.Caller) error {
+			client := NewAuthClient(c.Conn())
 
-		resp, err := client.Credentials(ctx, &CredentialsRequest{
-			Host: host,
+			resp, err := client.Credentials(ctx, &CredentialsRequest{
+				Host: host,
+			})
+			if err != nil {
+				if grpcerrors.Code(err) == codes.Unimplemented {
+					return nil
+				}
+				return err
+			}
+			user = resp.Username
+			secret = resp.Secret
+			return nil
 		})
 		if err != nil {
-			if grpcerrors.Code(err) == codes.Unimplemented {
-				return "", "", nil
-			}
 			return "", "", err
 		}
-		return resp.Username, resp.Secret, nil
+		return user, secret, nil
 	}
 }

vendor/github.com/moby/buildkit/session/context.go

@@ -1,22 +0,0 @@
-package session
-
-import "context"
-
-type contextKeyT string
-
-var contextKey = contextKeyT("buildkit/session-id")
-
-func NewContext(ctx context.Context, id string) context.Context {
-	if id != "" {
-		return context.WithValue(ctx, contextKey, id)
-	}
-	return ctx
-}
-
-func FromContext(ctx context.Context) string {
-	v := ctx.Value(contextKey)
-	if v == nil {
-		return ""
-	}
-	return v.(string)
-}

vendor/github.com/moby/buildkit/session/group.go

@@ -0,0 +1,88 @@
+package session
+
+import (
+	"context"
+	"time"
+
+	"github.com/pkg/errors"
+)
+
+type Group interface {
+	SessionIterator() Iterator
+}
+type Iterator interface {
+	NextSession() string
+}
+
+func NewGroup(ids ...string) Group {
+	return &group{ids: ids}
+}
+
+type group struct {
+	ids []string
+}
+
+func (g *group) SessionIterator() Iterator {
+	return &group{ids: g.ids}
+}
+
+func (g *group) NextSession() string {
+	if len(g.ids) == 0 {
+		return ""
+	}
+	v := g.ids[0]
+	g.ids = g.ids[1:]
+	return v
+}
+
+func AllSessionIDs(g Group) (out []string) {
+	if g == nil {
+		return nil
+	}
+	it := g.SessionIterator()
+	if it == nil {
+		return nil
+	}
+	for {
+		v := it.NextSession()
+		if v == "" {
+			return
+		}
+		out = append(out, v)
+	}
+}
+
+func (sm *Manager) Any(ctx context.Context, g Group, f func(context.Context, string, Caller) error) error {
+	if g == nil {
+		return nil
+	}
+
+	iter := g.SessionIterator()
+	if iter == nil {
+		return nil
+	}
+
+	var lastErr error
+	for {
+		id := iter.NextSession()
+		if id == "" {
+			if lastErr != nil {
+				return lastErr
+			}
+			return errors.Errorf("no active sessions")
+		}
+
+		timeoutCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
+		defer cancel()
+		c, err := sm.Get(timeoutCtx, id)
+		if err != nil {
+			lastErr = err
+			continue
+		}
+		if err := f(ctx, id, c); err != nil {
+			lastErr = err
+			continue
+		}
+		return nil
+	}
+}

vendor/github.com/moby/buildkit/session/secrets/secretsprovider/file.go

@@ -1,54 +0,0 @@
-package secretsprovider
-
-import (
-	"context"
-	"io/ioutil"
-	"os"
-
-	"github.com/moby/buildkit/session/secrets"
-	"github.com/pkg/errors"
-)
-
-type FileSource struct {
-	ID       string
-	FilePath string
-}
-
-func NewFileStore(files []FileSource) (secrets.SecretStore, error) {
-	m := map[string]FileSource{}
-	for _, f := range files {
-		if f.ID == "" {
-			return nil, errors.Errorf("secret missing ID")
-		}
-		if f.FilePath == "" {
-			f.FilePath = f.ID
-		}
-		fi, err := os.Stat(f.FilePath)
-		if err != nil {
-			return nil, errors.Wrapf(err, "failed to stat %s", f.FilePath)
-		}
-		if fi.Size() > MaxSecretSize {
-			return nil, errors.Errorf("secret %s too big. max size 500KB", f.ID)
-		}
-		m[f.ID] = f
-	}
-	return &fileStore{
-		m: m,
-	}, nil
-}
-
-type fileStore struct {
-	m map[string]FileSource
-}
-
-func (fs *fileStore) GetSecret(ctx context.Context, id string) ([]byte, error) {
-	v, ok := fs.m[id]
-	if !ok {
-		return nil, errors.WithStack(secrets.ErrNotFound)
-	}
-	dt, err := ioutil.ReadFile(v.FilePath)
-	if err != nil {
-		return nil, err
-	}
-	return dt, nil
-}

vendor/github.com/moby/buildkit/session/secrets/secretsprovider/store.go

@@ -0,0 +1,86 @@
+package secretsprovider
+
+import (
+	"context"
+	"io/ioutil"
+	"os"
+	"runtime"
+	"strings"
+
+	"github.com/moby/buildkit/session/secrets"
+	"github.com/pkg/errors"
+	"github.com/tonistiigi/units"
+)
+
+type Source struct {
+	ID       string
+	FilePath string
+	Env      string
+}
+
+func NewStore(files []Source) (secrets.SecretStore, error) {
+	m := map[string]Source{}
+	for _, f := range files {
+		if f.ID == "" {
+			return nil, errors.Errorf("secret missing ID")
+		}
+		if f.Env == "" && f.FilePath == "" {
+			if hasEnv(f.ID) {
+				f.Env = f.ID
+			} else {
+				f.FilePath = f.ID
+			}
+		}
+		if f.FilePath != "" {
+			fi, err := os.Stat(f.FilePath)
+			if err != nil {
+				return nil, errors.Wrapf(err, "failed to stat %s", f.FilePath)
+			}
+			if fi.Size() > MaxSecretSize {
+				return nil, errors.Errorf("secret %s too big. max size %#.f", f.ID, MaxSecretSize*units.B)
+			}
+		}
+		m[f.ID] = f
+	}
+	return &fileStore{
+		m: m,
+	}, nil
+}
+
+type fileStore struct {
+	m map[string]Source
+}
+
+func (fs *fileStore) GetSecret(ctx context.Context, id string) ([]byte, error) {
+	v, ok := fs.m[id]
+	if !ok {
+		return nil, errors.WithStack(secrets.ErrNotFound)
+	}
+	if v.Env != "" {
+		return []byte(os.Getenv(v.Env)), nil
+	}
+	dt, err := ioutil.ReadFile(v.FilePath)
+	if err != nil {
+		return nil, err
+	}
+	return dt, nil
+}
+
+func hasEnv(name string) bool {
+	for _, entry := range os.Environ() {
+		idx := strings.IndexRune(entry, '=')
+		if idx == -1 {
+			continue
+		}
+		if runtime.GOOS == "windows" {
+			// Environment variable are case-insensitive on Windows. PaTh, path and PATH are equivalent.
+			if strings.EqualFold(entry[:idx], name) {
+				return true
+			}
+		}
+		if entry[:idx] == name {
+			return true
+		}
+	}
+	return false
+}

vendor/github.com/moby/buildkit/solver/pb/attr.go

@@ -2,6 +2,8 @@ package pb
 
 const AttrKeepGitDir = "git.keepgitdir"
 const AttrFullRemoteURL = "git.fullurl"
+const AttrAuthHeaderSecret = "git.authheadersecret"
+const AttrAuthTokenSecret = "git.authtokensecret"
 const AttrLocalSessionID = "local.session"
 const AttrLocalUniqueID = "local.unique"
 const AttrIncludePatterns = "local.includepattern"

vendor/github.com/moby/buildkit/solver/pb/caps.go

@@ -19,9 +19,10 @@ const (
 	CapSourceLocalExcludePatterns apicaps.CapID = "source.local.excludepatterns"
 	CapSourceLocalSharedKeyHint   apicaps.CapID = "source.local.sharedkeyhint"
 
-	CapSourceGit        apicaps.CapID = "source.git"
-	CapSourceGitKeepDir apicaps.CapID = "source.git.keepgitdir"
-	CapSourceGitFullURL apicaps.CapID = "source.git.fullurl"
+	CapSourceGit         apicaps.CapID = "source.git"
+	CapSourceGitKeepDir  apicaps.CapID = "source.git.keepgitdir"
+	CapSourceGitFullURL  apicaps.CapID = "source.git.fullurl"
+	CapSourceGitHttpAuth apicaps.CapID = "source.git.httpauth"
 
 	CapSourceHTTP         apicaps.CapID = "source.http"
 	CapSourceHTTPChecksum apicaps.CapID = "source.http.checksum"
@@ -131,6 +132,12 @@ func init() {
 		Status:  apicaps.CapStatusExperimental,
 	})
 
+	Caps.Init(apicaps.Cap{
+		ID:      CapSourceGitHttpAuth,
+		Enabled: true,
+		Status:  apicaps.CapStatusExperimental,
+	})
+
 	Caps.Init(apicaps.Cap{
 		ID:      CapSourceHTTP,
 		Enabled: true,

vendor/github.com/moby/buildkit/util/progress/progressui/display.go

@@ -273,7 +273,14 @@ func (t *trace) update(s *client.SolveStatus, termWidth int) {
 				if v.Started != nil {
 					ts = l.Timestamp.Sub(*v.Started)
 				}
-				v.logs = append(v.logs, []byte(fmt.Sprintf("#%d %s %s", v.index, fmt.Sprintf("%#.4g", ts.Seconds())[:5], dt)))
+				prec := 1
+				sec := ts.Seconds()
+				if sec < 10 {
+					prec = 3
+				} else if sec < 100 {
+					prec = 2
+				}
+				v.logs = append(v.logs, []byte(fmt.Sprintf("#%d %s %s", v.index, fmt.Sprintf("%.[2]*[1]f", sec, prec), dt)))
 			}
 			i++
 		})