From 924af54d98c46d3a5d521d1854def994a8064536 Mon Sep 17 00:00:00 2001 From: David Sheets Date: Tue, 21 Feb 2017 12:07:45 -0800 Subject: [PATCH 1/3] build: accept -f - to read Dockerfile from stdin Heavily based on implementation by David Sheets Signed-off-by: David Sheets Signed-off-by: Tonis Tiigi --- command/image/build.go | 66 ++++++++++++++++++++++++++++++---- command/image/build/context.go | 31 ++++++++++------ 2 files changed, 80 insertions(+), 17 deletions(-) diff --git a/command/image/build.go b/command/image/build.go index b14b0356ca..f6984619c1 100644 --- a/command/image/build.go +++ b/command/image/build.go @@ -6,10 +6,12 @@ import ( "bytes" "fmt" "io" + "io/ioutil" "os" "path/filepath" "regexp" "runtime" + "time" "github.com/docker/distribution/reference" "github.com/docker/docker/api" @@ -25,6 +27,7 @@ import ( "github.com/docker/docker/pkg/jsonmessage" "github.com/docker/docker/pkg/progress" "github.com/docker/docker/pkg/streamformatter" + "github.com/docker/docker/pkg/stringid" "github.com/docker/docker/pkg/urlutil" runconfigopts "github.com/docker/docker/runconfig/opts" units "github.com/docker/go-units" @@ -141,6 +144,7 @@ func (out *lastProgressOutput) WriteProgress(prog progress.Progress) error { func runBuild(dockerCli *command.DockerCli, options buildOptions) error { var ( buildCtx io.ReadCloser + dockerfileCtx io.ReadCloser err error contextDir string tempDir string @@ -157,6 +161,13 @@ func runBuild(dockerCli *command.DockerCli, options buildOptions) error { buildBuff = bytes.NewBuffer(nil) } + if options.dockerfileName == "-" { + if specifiedContext == "-" { + return errors.New("invalid argument: can't use stdin for both build context and dockerfile") + } + dockerfileCtx = dockerCli.In() + } + switch { case specifiedContext == "-": buildCtx, relDockerfile, err = build.GetContextFromReader(dockerCli.In(), options.dockerfileName) @@ -214,11 +225,11 @@ func runBuild(dockerCli *command.DockerCli, options buildOptions) error { // removed. The daemon will remove them for us, if needed, after it // parses the Dockerfile. Ignore errors here, as they will have been // caught by validateContextDirectory above. - var includes = []string{"."} - keepThem1, _ := fileutils.Matches(".dockerignore", excludes) - keepThem2, _ := fileutils.Matches(relDockerfile, excludes) - if keepThem1 || keepThem2 { - includes = append(includes, ".dockerignore", relDockerfile) + if keep, _ := fileutils.Matches(".dockerignore", excludes); keep { + excludes = append(excludes, "!.dockerignore") + } + if keep, _ := fileutils.Matches(relDockerfile, excludes); keep && dockerfileCtx == nil { + excludes = append(excludes, "!"+relDockerfile) } compression := archive.Uncompressed @@ -228,13 +239,56 @@ func runBuild(dockerCli *command.DockerCli, options buildOptions) error { buildCtx, err = archive.TarWithOptions(contextDir, &archive.TarOptions{ Compression: compression, ExcludePatterns: excludes, - IncludeFiles: includes, }) if err != nil { return err } } + // replace Dockerfile if added dynamically + if dockerfileCtx != nil { + file, err := ioutil.ReadAll(dockerfileCtx) + dockerfileCtx.Close() + if err != nil { + return err + } + now := time.Now() + hdrTmpl := &tar.Header{ + Mode: 0600, + Uid: 0, + Gid: 0, + ModTime: now, + Typeflag: tar.TypeReg, + AccessTime: now, + ChangeTime: now, + } + randomName := ".dockerfile." + stringid.GenerateRandomID()[:20] + + buildCtx = archive.ReplaceFileTarWrapper(buildCtx, map[string]archive.TarModifierFunc{ + randomName: func(_ string, h *tar.Header, content io.Reader) (*tar.Header, []byte, error) { + return hdrTmpl, file, nil + }, + ".dockerignore": func(_ string, h *tar.Header, content io.Reader) (*tar.Header, []byte, error) { + if h == nil { + h = hdrTmpl + } + extraIgnore := randomName + "\n" + b := &bytes.Buffer{} + if content != nil { + _, err := b.ReadFrom(content) + if err != nil { + return nil, nil, err + } + } else { + extraIgnore += ".dockerignore\n" + } + b.Write([]byte("\n" + extraIgnore)) + return h, b.Bytes(), nil + }, + }) + relDockerfile = randomName + } + ctx := context.Background() var resolvedTags []*resolvedTag diff --git a/command/image/build/context.go b/command/image/build/context.go index 85d319e0b7..348c721931 100644 --- a/command/image/build/context.go +++ b/command/image/build/context.go @@ -89,6 +89,10 @@ func GetContextFromReader(r io.ReadCloser, dockerfileName string) (out io.ReadCl return ioutils.NewReadCloserWrapper(buf, func() error { return r.Close() }), dockerfileName, nil } + if dockerfileName == "-" { + return nil, "", errors.New("build context is not an archive") + } + // Input should be read as a Dockerfile. tmpDir, err := ioutil.TempDir("", "docker-build-context-") if err != nil { @@ -166,7 +170,7 @@ func GetContextFromLocalDir(localDir, dockerfileName string) (absContextDir, rel // When using a local context directory, when the Dockerfile is specified // with the `-f/--file` option then it is considered relative to the // current directory and not the context directory. - if dockerfileName != "" { + if dockerfileName != "" && dockerfileName != "-" { if dockerfileName, err = filepath.Abs(dockerfileName); err != nil { return "", "", errors.Errorf("unable to get absolute path to Dockerfile: %v", err) } @@ -220,6 +224,8 @@ func getDockerfileRelPath(givenContextDir, givenDockerfile string) (absContextDi absDockerfile = altPath } } + } else if absDockerfile == "-" { + absDockerfile = filepath.Join(absContextDir, DefaultDockerfileName) } // If not already an absolute path, the Dockerfile path should be joined to @@ -234,18 +240,21 @@ func getDockerfileRelPath(givenContextDir, givenDockerfile string) (absContextDi // an issue in golang. On Windows, EvalSymLinks does not work on UNC file // paths (those starting with \\). This hack means that when using links // on UNC paths, they will not be followed. - if !isUNC(absDockerfile) { - absDockerfile, err = filepath.EvalSymlinks(absDockerfile) - if err != nil { - return "", "", errors.Errorf("unable to evaluate symlinks in Dockerfile path: %v", err) - } - } + if givenDockerfile != "-" { + if !isUNC(absDockerfile) { + absDockerfile, err = filepath.EvalSymlinks(absDockerfile) + if err != nil { + return "", "", errors.Errorf("unable to evaluate symlinks in Dockerfile path: %v", err) - if _, err := os.Lstat(absDockerfile); err != nil { - if os.IsNotExist(err) { - return "", "", errors.Errorf("Cannot locate Dockerfile: %q", absDockerfile) + } + } + + if _, err := os.Lstat(absDockerfile); err != nil { + if os.IsNotExist(err) { + return "", "", errors.Errorf("Cannot locate Dockerfile: %q", absDockerfile) + } + return "", "", errors.Errorf("unable to stat Dockerfile: %v", err) } - return "", "", errors.Errorf("unable to stat Dockerfile: %v", err) } if relDockerfile, err = filepath.Rel(absContextDir, absDockerfile); err != nil { From 596cd38a6e3415bcf5ec12b7bec2dda36770c72c Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 5 Apr 2017 12:09:26 -0400 Subject: [PATCH 2/3] Factor out adding dockerfile from stdin. Signed-off-by: Daniel Nephin --- command/image/build.go | 83 +++++++++++++++++++++++------------------- 1 file changed, 46 insertions(+), 37 deletions(-) diff --git a/command/image/build.go b/command/image/build.go index f6984619c1..965acb4b51 100644 --- a/command/image/build.go +++ b/command/image/build.go @@ -247,46 +247,10 @@ func runBuild(dockerCli *command.DockerCli, options buildOptions) error { // replace Dockerfile if added dynamically if dockerfileCtx != nil { - file, err := ioutil.ReadAll(dockerfileCtx) - dockerfileCtx.Close() + buildCtx, relDockerfile, err = addDockerfileToBuildContext(dockerfileCtx, buildCtx) if err != nil { return err } - now := time.Now() - hdrTmpl := &tar.Header{ - Mode: 0600, - Uid: 0, - Gid: 0, - ModTime: now, - Typeflag: tar.TypeReg, - AccessTime: now, - ChangeTime: now, - } - randomName := ".dockerfile." + stringid.GenerateRandomID()[:20] - - buildCtx = archive.ReplaceFileTarWrapper(buildCtx, map[string]archive.TarModifierFunc{ - randomName: func(_ string, h *tar.Header, content io.Reader) (*tar.Header, []byte, error) { - return hdrTmpl, file, nil - }, - ".dockerignore": func(_ string, h *tar.Header, content io.Reader) (*tar.Header, []byte, error) { - if h == nil { - h = hdrTmpl - } - extraIgnore := randomName + "\n" - b := &bytes.Buffer{} - if content != nil { - _, err := b.ReadFrom(content) - if err != nil { - return nil, nil, err - } - } else { - extraIgnore += ".dockerignore\n" - } - b.Write([]byte("\n" + extraIgnore)) - return h, b.Bytes(), nil - }, - }) - relDockerfile = randomName } ctx := context.Background() @@ -392,6 +356,51 @@ func runBuild(dockerCli *command.DockerCli, options buildOptions) error { return nil } +func addDockerfileToBuildContext(dockerfileCtx io.ReadCloser, buildCtx io.ReadCloser) (io.ReadCloser, string, error) { + file, err := ioutil.ReadAll(dockerfileCtx) + dockerfileCtx.Close() + if err != nil { + return nil, "", err + } + now := time.Now() + hdrTmpl := &tar.Header{ + Mode: 0600, + Uid: 0, + Gid: 0, + ModTime: now, + Typeflag: tar.TypeReg, + AccessTime: now, + ChangeTime: now, + } + randomName := ".dockerfile." + stringid.GenerateRandomID()[:20] + + buildCtx = archive.ReplaceFileTarWrapper(buildCtx, map[string]archive.TarModifierFunc{ + // Add the dockerfile with a random filename + randomName: func(_ string, h *tar.Header, content io.Reader) (*tar.Header, []byte, error) { + return hdrTmpl, file, nil + }, + // Update .dockerignore to include the random filename + ".dockerignore": func(_ string, h *tar.Header, content io.Reader) (*tar.Header, []byte, error) { + if h == nil { + h = hdrTmpl + } + extraIgnore := randomName + "\n" + b := &bytes.Buffer{} + if content != nil { + _, err := b.ReadFrom(content) + if err != nil { + return nil, nil, err + } + } else { + extraIgnore += ".dockerignore\n" + } + b.Write([]byte("\n" + extraIgnore)) + return h, b.Bytes(), nil + }, + }) + return buildCtx, randomName, nil +} + func isLocalDir(c string) bool { _, err := os.Stat(c) return err == nil From b30ad6dc6e29179abcf2c8b409c534b53ab18103 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 5 Apr 2017 18:25:29 -0400 Subject: [PATCH 3/3] Upadte archive.ReplaceFileTarWrapper() to not expect a sorted archive Improve test coverage of ReplaceFileTarWrapper() Signed-off-by: Daniel Nephin --- command/image/build.go | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/command/image/build.go b/command/image/build.go index 965acb4b51..5268cbc254 100644 --- a/command/image/build.go +++ b/command/image/build.go @@ -218,13 +218,14 @@ func runBuild(dockerCli *command.DockerCli, options buildOptions) error { return errors.Errorf("Error checking context: '%s'.", err) } - // If .dockerignore mentions .dockerignore or the Dockerfile - // then make sure we send both files over to the daemon - // because Dockerfile is, obviously, needed no matter what, and - // .dockerignore is needed to know if either one needs to be - // removed. The daemon will remove them for us, if needed, after it - // parses the Dockerfile. Ignore errors here, as they will have been - // caught by validateContextDirectory above. + // If .dockerignore mentions .dockerignore or the Dockerfile then make + // sure we send both files over to the daemon because Dockerfile is, + // obviously, needed no matter what, and .dockerignore is needed to know + // if either one needs to be removed. The daemon will remove them + // if necessary, after it parses the Dockerfile. Ignore errors here, as + // they will have been caught by validateContextDirectory above. + // Excludes are used instead of includes to maintain the order of files + // in the archive. if keep, _ := fileutils.Matches(".dockerignore", excludes); keep { excludes = append(excludes, "!.dockerignore") } @@ -384,17 +385,16 @@ func addDockerfileToBuildContext(dockerfileCtx io.ReadCloser, buildCtx io.ReadCl if h == nil { h = hdrTmpl } - extraIgnore := randomName + "\n" + b := &bytes.Buffer{} if content != nil { - _, err := b.ReadFrom(content) - if err != nil { + if _, err := b.ReadFrom(content); err != nil { return nil, nil, err } } else { - extraIgnore += ".dockerignore\n" + b.WriteString(".dockerignore") } - b.Write([]byte("\n" + extraIgnore)) + b.WriteString("\n" + randomName + "\n") return h, b.Bytes(), nil }, })