Move builder cli helper functions to own pkg

Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>
This commit is contained in:
Tonis Tiigi 2016-12-22 13:25:02 -08:00
parent bcc61e1300
commit c41bfce39a
5 changed files with 683 additions and 7 deletions

View File

@ -16,10 +16,10 @@ import (
"github.com/docker/docker/api" "github.com/docker/docker/api"
"github.com/docker/docker/api/types" "github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/container"
"github.com/docker/docker/builder"
"github.com/docker/docker/builder/dockerignore" "github.com/docker/docker/builder/dockerignore"
"github.com/docker/docker/cli" "github.com/docker/docker/cli"
"github.com/docker/docker/cli/command" "github.com/docker/docker/cli/command"
"github.com/docker/docker/cli/command/image/build"
"github.com/docker/docker/opts" "github.com/docker/docker/opts"
"github.com/docker/docker/pkg/archive" "github.com/docker/docker/pkg/archive"
"github.com/docker/docker/pkg/fileutils" "github.com/docker/docker/pkg/fileutils"
@ -29,7 +29,7 @@ import (
"github.com/docker/docker/pkg/urlutil" "github.com/docker/docker/pkg/urlutil"
"github.com/docker/docker/reference" "github.com/docker/docker/reference"
runconfigopts "github.com/docker/docker/runconfig/opts" runconfigopts "github.com/docker/docker/runconfig/opts"
"github.com/docker/go-units" units "github.com/docker/go-units"
"github.com/spf13/cobra" "github.com/spf13/cobra"
) )
@ -156,13 +156,13 @@ func runBuild(dockerCli *command.DockerCli, options buildOptions) error {
switch { switch {
case specifiedContext == "-": case specifiedContext == "-":
buildCtx, relDockerfile, err = builder.GetContextFromReader(dockerCli.In(), options.dockerfileName) buildCtx, relDockerfile, err = build.GetContextFromReader(dockerCli.In(), options.dockerfileName)
case urlutil.IsGitURL(specifiedContext): case urlutil.IsGitURL(specifiedContext):
tempDir, relDockerfile, err = builder.GetContextFromGitURL(specifiedContext, options.dockerfileName) tempDir, relDockerfile, err = build.GetContextFromGitURL(specifiedContext, options.dockerfileName)
case urlutil.IsURL(specifiedContext): case urlutil.IsURL(specifiedContext):
buildCtx, relDockerfile, err = builder.GetContextFromURL(progBuff, specifiedContext, options.dockerfileName) buildCtx, relDockerfile, err = build.GetContextFromURL(progBuff, specifiedContext, options.dockerfileName)
default: default:
contextDir, relDockerfile, err = builder.GetContextFromLocalDir(specifiedContext, options.dockerfileName) contextDir, relDockerfile, err = build.GetContextFromLocalDir(specifiedContext, options.dockerfileName)
} }
if err != nil { if err != nil {
@ -198,7 +198,7 @@ func runBuild(dockerCli *command.DockerCli, options buildOptions) error {
} }
} }
if err := builder.ValidateContextDirectory(contextDir, excludes); err != nil { if err := build.ValidateContextDirectory(contextDir, excludes); err != nil {
return fmt.Errorf("Error checking context: '%s'.", err) return fmt.Errorf("Error checking context: '%s'.", err)
} }

View File

@ -0,0 +1,265 @@
package build
import (
"bufio"
"fmt"
"io"
"io/ioutil"
"os"
"os/exec"
"path/filepath"
"runtime"
"strings"
"github.com/docker/docker/pkg/archive"
"github.com/docker/docker/pkg/fileutils"
"github.com/docker/docker/pkg/gitutils"
"github.com/docker/docker/pkg/httputils"
"github.com/docker/docker/pkg/ioutils"
"github.com/docker/docker/pkg/progress"
"github.com/docker/docker/pkg/streamformatter"
)
const (
// DefaultDockerfileName is the Default filename with Docker commands, read by docker build
DefaultDockerfileName string = "Dockerfile"
)
// ValidateContextDirectory checks if all the contents of the directory
// can be read and returns an error if some files can't be read
// symlinks which point to non-existing files don't trigger an error
func ValidateContextDirectory(srcPath string, excludes []string) error {
contextRoot, err := getContextRoot(srcPath)
if err != nil {
return err
}
return filepath.Walk(contextRoot, func(filePath string, f os.FileInfo, err error) error {
if err != nil {
if os.IsPermission(err) {
return fmt.Errorf("can't stat '%s'", filePath)
}
if os.IsNotExist(err) {
return nil
}
return err
}
// skip this directory/file if it's not in the path, it won't get added to the context
if relFilePath, err := filepath.Rel(contextRoot, filePath); err != nil {
return err
} else if skip, err := fileutils.Matches(relFilePath, excludes); err != nil {
return err
} else if skip {
if f.IsDir() {
return filepath.SkipDir
}
return nil
}
// skip checking if symlinks point to non-existing files, such symlinks can be useful
// also skip named pipes, because they hanging on open
if f.Mode()&(os.ModeSymlink|os.ModeNamedPipe) != 0 {
return nil
}
if !f.IsDir() {
currentFile, err := os.Open(filePath)
if err != nil && os.IsPermission(err) {
return fmt.Errorf("no permission to read from '%s'", filePath)
}
currentFile.Close()
}
return nil
})
}
// GetContextFromReader will read the contents of the given reader as either a
// Dockerfile or tar archive. Returns a tar archive used as a context and a
// path to the Dockerfile inside the tar.
func GetContextFromReader(r io.ReadCloser, dockerfileName string) (out io.ReadCloser, relDockerfile string, err error) {
buf := bufio.NewReader(r)
magic, err := buf.Peek(archive.HeaderSize)
if err != nil && err != io.EOF {
return nil, "", fmt.Errorf("failed to peek context header from STDIN: %v", err)
}
if archive.IsArchive(magic) {
return ioutils.NewReadCloserWrapper(buf, func() error { return r.Close() }), dockerfileName, nil
}
// Input should be read as a Dockerfile.
tmpDir, err := ioutil.TempDir("", "docker-build-context-")
if err != nil {
return nil, "", fmt.Errorf("unbale to create temporary context directory: %v", err)
}
f, err := os.Create(filepath.Join(tmpDir, DefaultDockerfileName))
if err != nil {
return nil, "", err
}
_, err = io.Copy(f, buf)
if err != nil {
f.Close()
return nil, "", err
}
if err := f.Close(); err != nil {
return nil, "", err
}
if err := r.Close(); err != nil {
return nil, "", err
}
tar, err := archive.Tar(tmpDir, archive.Uncompressed)
if err != nil {
return nil, "", err
}
return ioutils.NewReadCloserWrapper(tar, func() error {
err := tar.Close()
os.RemoveAll(tmpDir)
return err
}), DefaultDockerfileName, nil
}
// GetContextFromGitURL uses a Git URL as context for a `docker build`. The
// git repo is cloned into a temporary directory used as the context directory.
// Returns the absolute path to the temporary context directory, the relative
// path of the dockerfile in that context directory, and a non-nil error on
// success.
func GetContextFromGitURL(gitURL, dockerfileName string) (absContextDir, relDockerfile string, err error) {
if _, err := exec.LookPath("git"); err != nil {
return "", "", fmt.Errorf("unable to find 'git': %v", err)
}
if absContextDir, err = gitutils.Clone(gitURL); err != nil {
return "", "", fmt.Errorf("unable to 'git clone' to temporary context directory: %v", err)
}
return getDockerfileRelPath(absContextDir, dockerfileName)
}
// GetContextFromURL uses a remote URL as context for a `docker build`. The
// remote resource is downloaded as either a Dockerfile or a tar archive.
// Returns the tar archive used for the context and a path of the
// dockerfile inside the tar.
func GetContextFromURL(out io.Writer, remoteURL, dockerfileName string) (io.ReadCloser, string, error) {
response, err := httputils.Download(remoteURL)
if err != nil {
return nil, "", fmt.Errorf("unable to download remote context %s: %v", remoteURL, err)
}
progressOutput := streamformatter.NewStreamFormatter().NewProgressOutput(out, true)
// Pass the response body through a progress reader.
progReader := progress.NewProgressReader(response.Body, progressOutput, response.ContentLength, "", fmt.Sprintf("Downloading build context from remote url: %s", remoteURL))
return GetContextFromReader(ioutils.NewReadCloserWrapper(progReader, func() error { return response.Body.Close() }), dockerfileName)
}
// GetContextFromLocalDir uses the given local directory as context for a
// `docker build`. Returns the absolute path to the local context directory,
// the relative path of the dockerfile in that context directory, and a non-nil
// error on success.
func GetContextFromLocalDir(localDir, dockerfileName string) (absContextDir, relDockerfile string, err error) {
// When using a local context directory, when the Dockerfile is specified
// with the `-f/--file` option then it is considered relative to the
// current directory and not the context directory.
if dockerfileName != "" {
if dockerfileName, err = filepath.Abs(dockerfileName); err != nil {
return "", "", fmt.Errorf("unable to get absolute path to Dockerfile: %v", err)
}
}
return getDockerfileRelPath(localDir, dockerfileName)
}
// getDockerfileRelPath uses the given context directory for a `docker build`
// and returns the absolute path to the context directory, the relative path of
// the dockerfile in that context directory, and a non-nil error on success.
func getDockerfileRelPath(givenContextDir, givenDockerfile string) (absContextDir, relDockerfile string, err error) {
if absContextDir, err = filepath.Abs(givenContextDir); err != nil {
return "", "", fmt.Errorf("unable to get absolute context directory of given context directory %q: %v", givenContextDir, err)
}
// The context dir might be a symbolic link, so follow it to the actual
// target directory.
//
// FIXME. We use isUNC (always false on non-Windows platforms) to workaround
// an issue in golang. On Windows, EvalSymLinks does not work on UNC file
// paths (those starting with \\). This hack means that when using links
// on UNC paths, they will not be followed.
if !isUNC(absContextDir) {
absContextDir, err = filepath.EvalSymlinks(absContextDir)
if err != nil {
return "", "", fmt.Errorf("unable to evaluate symlinks in context path: %v", err)
}
}
stat, err := os.Lstat(absContextDir)
if err != nil {
return "", "", fmt.Errorf("unable to stat context directory %q: %v", absContextDir, err)
}
if !stat.IsDir() {
return "", "", fmt.Errorf("context must be a directory: %s", absContextDir)
}
absDockerfile := givenDockerfile
if absDockerfile == "" {
// No -f/--file was specified so use the default relative to the
// context directory.
absDockerfile = filepath.Join(absContextDir, DefaultDockerfileName)
// Just to be nice ;-) look for 'dockerfile' too but only
// use it if we found it, otherwise ignore this check
if _, err = os.Lstat(absDockerfile); os.IsNotExist(err) {
altPath := filepath.Join(absContextDir, strings.ToLower(DefaultDockerfileName))
if _, err = os.Lstat(altPath); err == nil {
absDockerfile = altPath
}
}
}
// If not already an absolute path, the Dockerfile path should be joined to
// the base directory.
if !filepath.IsAbs(absDockerfile) {
absDockerfile = filepath.Join(absContextDir, absDockerfile)
}
// Evaluate symlinks in the path to the Dockerfile too.
//
// FIXME. We use isUNC (always false on non-Windows platforms) to workaround
// an issue in golang. On Windows, EvalSymLinks does not work on UNC file
// paths (those starting with \\). This hack means that when using links
// on UNC paths, they will not be followed.
if !isUNC(absDockerfile) {
absDockerfile, err = filepath.EvalSymlinks(absDockerfile)
if err != nil {
return "", "", fmt.Errorf("unable to evaluate symlinks in Dockerfile path: %v", err)
}
}
if _, err := os.Lstat(absDockerfile); err != nil {
if os.IsNotExist(err) {
return "", "", fmt.Errorf("Cannot locate Dockerfile: %q", absDockerfile)
}
return "", "", fmt.Errorf("unable to stat Dockerfile: %v", err)
}
if relDockerfile, err = filepath.Rel(absContextDir, absDockerfile); err != nil {
return "", "", fmt.Errorf("unable to get relative Dockerfile path: %v", err)
}
if strings.HasPrefix(relDockerfile, ".."+string(filepath.Separator)) {
return "", "", fmt.Errorf("The Dockerfile (%s) must be within the build context (%s)", givenDockerfile, givenContextDir)
}
return absContextDir, relDockerfile, nil
}
// isUNC returns true if the path is UNC (one starting \\). It always returns
// false on Linux.
func isUNC(path string) bool {
return runtime.GOOS == "windows" && strings.HasPrefix(path, `\\`)
}

View File

@ -0,0 +1,383 @@
package build
import (
"archive/tar"
"bytes"
"io"
"io/ioutil"
"os"
"path/filepath"
"runtime"
"strings"
"testing"
"github.com/docker/docker/pkg/archive"
)
const dockerfileContents = "FROM busybox"
var prepareEmpty = func(t *testing.T) (string, func()) {
return "", func() {}
}
var prepareNoFiles = func(t *testing.T) (string, func()) {
return createTestTempDir(t, "", "builder-context-test")
}
var prepareOneFile = func(t *testing.T) (string, func()) {
contextDir, cleanup := createTestTempDir(t, "", "builder-context-test")
createTestTempFile(t, contextDir, DefaultDockerfileName, dockerfileContents, 0777)
return contextDir, cleanup
}
func testValidateContextDirectory(t *testing.T, prepare func(t *testing.T) (string, func()), excludes []string) {
contextDir, cleanup := prepare(t)
defer cleanup()
err := ValidateContextDirectory(contextDir, excludes)
if err != nil {
t.Fatalf("Error should be nil, got: %s", err)
}
}
func TestGetContextFromLocalDirNoDockerfile(t *testing.T) {
contextDir, cleanup := createTestTempDir(t, "", "builder-context-test")
defer cleanup()
absContextDir, relDockerfile, err := GetContextFromLocalDir(contextDir, "")
if err == nil {
t.Fatalf("Error should not be nil")
}
if absContextDir != "" {
t.Fatalf("Absolute directory path should be empty, got: %s", absContextDir)
}
if relDockerfile != "" {
t.Fatalf("Relative path to Dockerfile should be empty, got: %s", relDockerfile)
}
}
func TestGetContextFromLocalDirNotExistingDir(t *testing.T) {
contextDir, cleanup := createTestTempDir(t, "", "builder-context-test")
defer cleanup()
fakePath := filepath.Join(contextDir, "fake")
absContextDir, relDockerfile, err := GetContextFromLocalDir(fakePath, "")
if err == nil {
t.Fatalf("Error should not be nil")
}
if absContextDir != "" {
t.Fatalf("Absolute directory path should be empty, got: %s", absContextDir)
}
if relDockerfile != "" {
t.Fatalf("Relative path to Dockerfile should be empty, got: %s", relDockerfile)
}
}
func TestGetContextFromLocalDirNotExistingDockerfile(t *testing.T) {
contextDir, cleanup := createTestTempDir(t, "", "builder-context-test")
defer cleanup()
fakePath := filepath.Join(contextDir, "fake")
absContextDir, relDockerfile, err := GetContextFromLocalDir(contextDir, fakePath)
if err == nil {
t.Fatalf("Error should not be nil")
}
if absContextDir != "" {
t.Fatalf("Absolute directory path should be empty, got: %s", absContextDir)
}
if relDockerfile != "" {
t.Fatalf("Relative path to Dockerfile should be empty, got: %s", relDockerfile)
}
}
func TestGetContextFromLocalDirWithNoDirectory(t *testing.T) {
contextDir, dirCleanup := createTestTempDir(t, "", "builder-context-test")
defer dirCleanup()
createTestTempFile(t, contextDir, DefaultDockerfileName, dockerfileContents, 0777)
chdirCleanup := chdir(t, contextDir)
defer chdirCleanup()
absContextDir, relDockerfile, err := GetContextFromLocalDir(contextDir, "")
if err != nil {
t.Fatalf("Error when getting context from local dir: %s", err)
}
if absContextDir != contextDir {
t.Fatalf("Absolute directory path should be equal to %s, got: %s", contextDir, absContextDir)
}
if relDockerfile != DefaultDockerfileName {
t.Fatalf("Relative path to dockerfile should be equal to %s, got: %s", DefaultDockerfileName, relDockerfile)
}
}
func TestGetContextFromLocalDirWithDockerfile(t *testing.T) {
contextDir, cleanup := createTestTempDir(t, "", "builder-context-test")
defer cleanup()
createTestTempFile(t, contextDir, DefaultDockerfileName, dockerfileContents, 0777)
absContextDir, relDockerfile, err := GetContextFromLocalDir(contextDir, "")
if err != nil {
t.Fatalf("Error when getting context from local dir: %s", err)
}
if absContextDir != contextDir {
t.Fatalf("Absolute directory path should be equal to %s, got: %s", contextDir, absContextDir)
}
if relDockerfile != DefaultDockerfileName {
t.Fatalf("Relative path to dockerfile should be equal to %s, got: %s", DefaultDockerfileName, relDockerfile)
}
}
func TestGetContextFromLocalDirLocalFile(t *testing.T) {
contextDir, cleanup := createTestTempDir(t, "", "builder-context-test")
defer cleanup()
createTestTempFile(t, contextDir, DefaultDockerfileName, dockerfileContents, 0777)
testFilename := createTestTempFile(t, contextDir, "tmpTest", "test", 0777)
absContextDir, relDockerfile, err := GetContextFromLocalDir(testFilename, "")
if err == nil {
t.Fatalf("Error should not be nil")
}
if absContextDir != "" {
t.Fatalf("Absolute directory path should be empty, got: %s", absContextDir)
}
if relDockerfile != "" {
t.Fatalf("Relative path to Dockerfile should be empty, got: %s", relDockerfile)
}
}
func TestGetContextFromLocalDirWithCustomDockerfile(t *testing.T) {
contextDir, cleanup := createTestTempDir(t, "", "builder-context-test")
defer cleanup()
chdirCleanup := chdir(t, contextDir)
defer chdirCleanup()
createTestTempFile(t, contextDir, DefaultDockerfileName, dockerfileContents, 0777)
absContextDir, relDockerfile, err := GetContextFromLocalDir(contextDir, DefaultDockerfileName)
if err != nil {
t.Fatalf("Error when getting context from local dir: %s", err)
}
if absContextDir != contextDir {
t.Fatalf("Absolute directory path should be equal to %s, got: %s", contextDir, absContextDir)
}
if relDockerfile != DefaultDockerfileName {
t.Fatalf("Relative path to dockerfile should be equal to %s, got: %s", DefaultDockerfileName, relDockerfile)
}
}
func TestGetContextFromReaderString(t *testing.T) {
tarArchive, relDockerfile, err := GetContextFromReader(ioutil.NopCloser(strings.NewReader(dockerfileContents)), "")
if err != nil {
t.Fatalf("Error when executing GetContextFromReader: %s", err)
}
tarReader := tar.NewReader(tarArchive)
_, err = tarReader.Next()
if err != nil {
t.Fatalf("Error when reading tar archive: %s", err)
}
buff := new(bytes.Buffer)
buff.ReadFrom(tarReader)
contents := buff.String()
_, err = tarReader.Next()
if err != io.EOF {
t.Fatalf("Tar stream too long: %s", err)
}
if err = tarArchive.Close(); err != nil {
t.Fatalf("Error when closing tar stream: %s", err)
}
if dockerfileContents != contents {
t.Fatalf("Uncompressed tar archive does not equal: %s, got: %s", dockerfileContents, contents)
}
if relDockerfile != DefaultDockerfileName {
t.Fatalf("Relative path not equals %s, got: %s", DefaultDockerfileName, relDockerfile)
}
}
func TestGetContextFromReaderTar(t *testing.T) {
contextDir, cleanup := createTestTempDir(t, "", "builder-context-test")
defer cleanup()
createTestTempFile(t, contextDir, DefaultDockerfileName, dockerfileContents, 0777)
tarStream, err := archive.Tar(contextDir, archive.Uncompressed)
if err != nil {
t.Fatalf("Error when creating tar: %s", err)
}
tarArchive, relDockerfile, err := GetContextFromReader(tarStream, DefaultDockerfileName)
if err != nil {
t.Fatalf("Error when executing GetContextFromReader: %s", err)
}
tarReader := tar.NewReader(tarArchive)
header, err := tarReader.Next()
if err != nil {
t.Fatalf("Error when reading tar archive: %s", err)
}
if header.Name != DefaultDockerfileName {
t.Fatalf("Dockerfile name should be: %s, got: %s", DefaultDockerfileName, header.Name)
}
buff := new(bytes.Buffer)
buff.ReadFrom(tarReader)
contents := buff.String()
_, err = tarReader.Next()
if err != io.EOF {
t.Fatalf("Tar stream too long: %s", err)
}
if err = tarArchive.Close(); err != nil {
t.Fatalf("Error when closing tar stream: %s", err)
}
if dockerfileContents != contents {
t.Fatalf("Uncompressed tar archive does not equal: %s, got: %s", dockerfileContents, contents)
}
if relDockerfile != DefaultDockerfileName {
t.Fatalf("Relative path not equals %s, got: %s", DefaultDockerfileName, relDockerfile)
}
}
func TestValidateContextDirectoryEmptyContext(t *testing.T) {
// This isn't a valid test on Windows. See https://play.golang.org/p/RR6z6jxR81.
// The test will ultimately end up calling filepath.Abs(""). On Windows,
// golang will error. On Linux, golang will return /. Due to there being
// drive letters on Windows, this is probably the correct behaviour for
// Windows.
if runtime.GOOS == "windows" {
t.Skip("Invalid test on Windows")
}
testValidateContextDirectory(t, prepareEmpty, []string{})
}
func TestValidateContextDirectoryContextWithNoFiles(t *testing.T) {
testValidateContextDirectory(t, prepareNoFiles, []string{})
}
func TestValidateContextDirectoryWithOneFile(t *testing.T) {
testValidateContextDirectory(t, prepareOneFile, []string{})
}
func TestValidateContextDirectoryWithOneFileExcludes(t *testing.T) {
testValidateContextDirectory(t, prepareOneFile, []string{DefaultDockerfileName})
}
// createTestTempDir creates a temporary directory for testing.
// It returns the created path and a cleanup function which is meant to be used as deferred call.
// When an error occurs, it terminates the test.
func createTestTempDir(t *testing.T, dir, prefix string) (string, func()) {
path, err := ioutil.TempDir(dir, prefix)
if err != nil {
t.Fatalf("Error when creating directory %s with prefix %s: %s", dir, prefix, err)
}
return path, func() {
err = os.RemoveAll(path)
if err != nil {
t.Fatalf("Error when removing directory %s: %s", path, err)
}
}
}
// createTestTempSubdir creates a temporary directory for testing.
// It returns the created path but doesn't provide a cleanup function,
// so createTestTempSubdir should be used only for creating temporary subdirectories
// whose parent directories are properly cleaned up.
// When an error occurs, it terminates the test.
func createTestTempSubdir(t *testing.T, dir, prefix string) string {
path, err := ioutil.TempDir(dir, prefix)
if err != nil {
t.Fatalf("Error when creating directory %s with prefix %s: %s", dir, prefix, err)
}
return path
}
// createTestTempFile creates a temporary file within dir with specific contents and permissions.
// When an error occurs, it terminates the test
func createTestTempFile(t *testing.T, dir, filename, contents string, perm os.FileMode) string {
filePath := filepath.Join(dir, filename)
err := ioutil.WriteFile(filePath, []byte(contents), perm)
if err != nil {
t.Fatalf("Error when creating %s file: %s", filename, err)
}
return filePath
}
// chdir changes current working directory to dir.
// It returns a function which changes working directory back to the previous one.
// This function is meant to be executed as a deferred call.
// When an error occurs, it terminates the test.
func chdir(t *testing.T, dir string) func() {
workingDirectory, err := os.Getwd()
if err != nil {
t.Fatalf("Error when retrieving working directory: %s", err)
}
err = os.Chdir(dir)
if err != nil {
t.Fatalf("Error when changing directory to %s: %s", dir, err)
}
return func() {
err = os.Chdir(workingDirectory)
if err != nil {
t.Fatalf("Error when changing back to working directory (%s): %s", workingDirectory, err)
}
}
}

View File

@ -0,0 +1,11 @@
// +build !windows
package build
import (
"path/filepath"
)
func getContextRoot(srcPath string) (string, error) {
return filepath.Join(srcPath, "."), nil
}

View File

@ -0,0 +1,17 @@
// +build windows
package build
import (
"path/filepath"
"github.com/docker/docker/pkg/longpath"
)
func getContextRoot(srcPath string) (string, error) {
cr, err := filepath.Abs(srcPath)
if err != nil {
return "", err
}
return longpath.AddPrefix(cr), nil
}