diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 0000000000..c5fd505597 --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,12 @@ +wrappedNode(label: 'linux && x86_64', cleanWorkspace: true) { + timeout(time: 60, unit: 'MINUTES') { + stage "Git Checkout" + checkout scm + + stage "Run end-to-end test suite" + sh "docker version" + sh "E2E_UNIQUE_ID=clie2e${BUILD_NUMBER} \ + IMAGE_TAG=clie2e${BUILD_NUMBER} \ + make -f docker.Makefile test-e2e" + } +} diff --git a/Makefile b/Makefile index ccf36b4665..17ba8e5b2d 100644 --- a/Makefile +++ b/Makefile @@ -10,13 +10,13 @@ _:=$(shell ./scripts/warn-outside-container $(MAKECMDGOALS)) clean: ## remove build artifacts rm -rf ./build/* cli/winresources/rsrc_* ./man/man[1-9] docs/yaml/gen -.PHONY: test -test: ## run go test - ./scripts/test/unit $(shell go list ./... | grep -v '/vendor/') +.PHONY: test-unit +test-unit: ## run unit test + ./scripts/test/unit $(shell go list ./... | grep -vE '/vendor/|/e2e/') .PHONY: test-coverage test-coverage: ## run test coverage - ./scripts/test/unit-with-coverage $(shell go list ./... | grep -v '/vendor/') + ./scripts/test/unit-with-coverage $(shell go list ./... | grep -vE '/vendor/|/e2e/') .PHONY: lint lint: ## run all the lint tools diff --git a/cli/command/stack/remove.go b/cli/command/stack/remove.go index d95171aabf..157f71ea12 100644 --- a/cli/command/stack/remove.go +++ b/cli/command/stack/remove.go @@ -2,6 +2,7 @@ package stack import ( "fmt" + "sort" "strings" "github.com/docker/cli/cli" @@ -88,12 +89,19 @@ func runRemove(dockerCli command.Cli, opts removeOptions) error { return nil } +func sortServiceByName(services []swarm.Service) func(i, j int) bool { + return func(i, j int) bool { + return services[i].Spec.Name < services[j].Spec.Name + } +} + func removeServices( ctx context.Context, dockerCli command.Cli, services []swarm.Service, ) bool { var hasError bool + sort.Slice(services, sortServiceByName(services)) for _, service := range services { fmt.Fprintf(dockerCli.Err(), "Removing service %s\n", service.Spec.Name) if err := dockerCli.Client().ServiceRemove(ctx, service.ID); err != nil { diff --git a/docker.Makefile b/docker.Makefile index b7a34da314..097ad49399 100644 --- a/docker.Makefile +++ b/docker.Makefile @@ -4,10 +4,10 @@ # Makefile for developing using Docker # -DEV_DOCKER_IMAGE_NAME = docker-cli-dev -LINTER_IMAGE_NAME = docker-cli-lint -CROSS_IMAGE_NAME = docker-cli-cross -VALIDATE_IMAGE_NAME = docker-cli-shell-validate +DEV_DOCKER_IMAGE_NAME = docker-cli-dev$(IMAGE_TAG) +LINTER_IMAGE_NAME = docker-cli-lint$(IMAGE_TAG) +CROSS_IMAGE_NAME = docker-cli-cross$(IMAGE_TAG) +VALIDATE_IMAGE_NAME = docker-cli-shell-validate$(IMAGE_TAG) MOUNTS = -v "$(CURDIR)":/go/src/github.com/docker/cli VERSION = $(shell cat VERSION) ENVVARS = -e VERSION=$(VERSION) -e GITCOMMIT @@ -42,9 +42,9 @@ clean: build_docker_image docker run --rm $(ENVVARS) $(MOUNTS) $(DEV_DOCKER_IMAGE_NAME) make clean # run go test -.PHONY: test -test: build_docker_image - docker run --rm $(ENVVARS) $(MOUNTS) $(DEV_DOCKER_IMAGE_NAME) make test +.PHONY: test-unit +test-unit: build_docker_image + docker run --rm $(ENVVARS) $(MOUNTS) $(DEV_DOCKER_IMAGE_NAME) make test-unit # build the CLI for multiple architectures using a container .PHONY: cross @@ -90,3 +90,7 @@ yamldocs: build_docker_image .PHONY: shellcheck shellcheck: build_shell_validate_image docker run -ti --rm $(ENVVARS) $(MOUNTS) $(VALIDATE_IMAGE_NAME) make shellcheck + +.PHONY: test-e2e +test-e2e: binary + ./scripts/test/e2e/wrapper diff --git a/dockerfiles/Dockerfile.test-e2e-env b/dockerfiles/Dockerfile.test-e2e-env new file mode 100644 index 0000000000..3c672f4e8c --- /dev/null +++ b/dockerfiles/Dockerfile.test-e2e-env @@ -0,0 +1,17 @@ +FROM docker/compose:1.15.0 + +RUN apk add -U bash curl + +ARG DOCKER_CHANNEL=edge +ARG DOCKER_VERSION=17.06.0-ce +RUN export URL=https://download.docker.com/linux/static; \ + curl -Ls $URL/$DOCKER_CHANNEL/x86_64/docker-$DOCKER_VERSION.tgz | \ + tar -xz docker/docker && \ + mv docker/docker /usr/local/bin/ && \ + rmdir docker +ENV DISABLE_WARN_OUTSIDE_CONTAINER=1 +WORKDIR /work +COPY scripts/test/e2e scripts/test/e2e +COPY e2e/compose-env.yaml e2e/compose-env.yaml + +ENTRYPOINT ["bash", "/work/scripts/test/e2e/run"] diff --git a/e2e/compose-env.yaml b/e2e/compose-env.yaml new file mode 100644 index 0000000000..afc95e3af0 --- /dev/null +++ b/e2e/compose-env.yaml @@ -0,0 +1,10 @@ +version: '2.1' + +services: + registry: + image: 'registry:2' + + engine: + image: 'docker:${TEST_ENGINE_VERSION:-edge-dind}' + privileged: true + command: ['--insecure-registry=registry:5000'] diff --git a/e2e/stack/main_test.go b/e2e/stack/main_test.go new file mode 100644 index 0000000000..74081f457b --- /dev/null +++ b/e2e/stack/main_test.go @@ -0,0 +1,26 @@ +package stack + +import ( + "fmt" + "os" + "testing" + + "github.com/pkg/errors" +) + +func TestMain(m *testing.M) { + if err := setupTestEnv(); err != nil { + fmt.Println(err.Error()) + os.Exit(3) + } + os.Exit(m.Run()) +} + +// TODO: move to shared internal package +func setupTestEnv() error { + dockerHost := os.Getenv("TEST_DOCKER_HOST") + if dockerHost == "" { + return errors.New("$TEST_DOCKER_HOST must be set") + } + return os.Setenv("DOCKER_HOST", dockerHost) +} diff --git a/e2e/stack/remove_test.go b/e2e/stack/remove_test.go new file mode 100644 index 0000000000..77d95c793d --- /dev/null +++ b/e2e/stack/remove_test.go @@ -0,0 +1,89 @@ +package stack + +import ( + "fmt" + "strings" + "testing" + "time" + + shlex "github.com/flynn-archive/go-shlex" + "github.com/gotestyourself/gotestyourself/golden" + "github.com/gotestyourself/gotestyourself/icmd" + "github.com/stretchr/testify/require" +) + +func TestRemove(t *testing.T) { + stackname := "test-stack-remove" + deployFullStack(t, stackname) + defer cleanupFullStack(t, stackname) + + result := icmd.RunCmd(shell(t, "docker stack rm %s", stackname)) + + result.Assert(t, icmd.Expected{Out: icmd.None}) + golden.Assert(t, result.Stderr(), "stack-remove-success.golden") +} + +func deployFullStack(t *testing.T, stackname string) { + // TODO: this stack should have full options not minimal options + result := icmd.RunCmd(shell(t, + "docker stack deploy --compose-file=./testdata/full-stack.yml %s", stackname)) + result.Assert(t, icmd.Success) + + waitOn(t, taskCount(stackname, 2), 0) +} + +func cleanupFullStack(t *testing.T, stackname string) { + result := icmd.RunCmd(shell(t, "docker stack rm %s", stackname)) + result.Assert(t, icmd.Success) + waitOn(t, taskCount(stackname, 0), 0) +} + +func taskCount(stackname string, expected int) func() (bool, error) { + return func() (bool, error) { + result := icmd.RunCommand( + "docker", "stack", "ps", "-f=desired-state=running", stackname) + count := lines(result.Stdout()) - 1 + return count == expected, nil + } +} + +func lines(out string) int { + return len(strings.Split(strings.TrimSpace(out), "\n")) +} + +// TODO: move to gotestyourself +func shell(t *testing.T, format string, args ...interface{}) icmd.Cmd { + cmd, err := shlex.Split(fmt.Sprintf(format, args...)) + require.NoError(t, err) + return icmd.Cmd{Command: cmd} +} + +// TODO: move to gotestyourself +func waitOn(t *testing.T, check func() (bool, error), timeout time.Duration) { + if timeout == time.Duration(0) { + timeout = defaultTimeout() + } + + after := time.After(timeout) + for { + select { + case <-after: + // TODO: include check function name in error message + t.Fatalf("timeout hit after %s", timeout) + default: + // TODO: maybe return a failure message as well? + done, err := check() + if done { + return + } + if err != nil { + t.Fatal(err.Error()) + } + } + } +} + +func defaultTimeout() time.Duration { + // TODO: support override from environment variable + return 10 * time.Second +} diff --git a/e2e/stack/testdata/full-stack.yml b/e2e/stack/testdata/full-stack.yml new file mode 100644 index 0000000000..8c4d06f854 --- /dev/null +++ b/e2e/stack/testdata/full-stack.yml @@ -0,0 +1,9 @@ +version: '3.3' + +services: + one: + image: registry:5000/alpine:3.6 + command: top + two: + image: registry:5000/alpine:3.6 + command: top diff --git a/e2e/stack/testdata/stack-remove-success.golden b/e2e/stack/testdata/stack-remove-success.golden new file mode 100644 index 0000000000..f41a891702 --- /dev/null +++ b/e2e/stack/testdata/stack-remove-success.golden @@ -0,0 +1,3 @@ +Removing service test-stack-remove_one +Removing service test-stack-remove_two +Removing network test-stack-remove_default diff --git a/scripts/test/e2e/load-alpine b/scripts/test/e2e/load-alpine new file mode 100755 index 0000000000..5b75f0989c --- /dev/null +++ b/scripts/test/e2e/load-alpine @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -eu -o pipefail + +src=alpine:3.6 +dest=registry:5000/alpine:3.6 +docker pull $src +docker tag $src $dest +docker push $dest diff --git a/scripts/test/e2e/run b/scripts/test/e2e/run new file mode 100755 index 0000000000..936b5898fd --- /dev/null +++ b/scripts/test/e2e/run @@ -0,0 +1,85 @@ +#!/usr/bin/env bash +# Run integration tests against the latest docker-ce dind +set -eu -o pipefail + +function container_ip { + local cid=$1 + local network=$2 + docker inspect \ + -f "{{.NetworkSettings.Networks.${network}.IPAddress}}" "$cid" +} + +function setup { + local project=$1 + COMPOSE_PROJECT_NAME=$1 COMPOSE_FILE=$2 docker-compose up -d >&2 + + local network="${project}_default" + # TODO: only run if inside a container + docker network connect "$network" "$(hostname)" + + engine_ip="$(container_ip "${project}_engine_1" "$network")" + engine_host="tcp://$engine_ip:2375" + ( + export DOCKER_HOST="$engine_host" + timeout -t 200 ./scripts/test/e2e/wait-on-daemon + ./scripts/test/e2e/load-alpine + is_swarm_enabled || docker swarm init + ) >&2 + echo "$engine_host" +} + +function is_swarm_enabled { + docker info 2> /dev/null | grep -q 'Swarm: active' +} + +function cleanup { + COMPOSE_PROJECT_NAME=$1 COMPOSE_FILE=$2 docker-compose down >&2 +} + +function runtests { + local engine_host=$1 + + env -i \ + TEST_DOCKER_HOST="$engine_host" \ + GOPATH="$GOPATH" \ + PATH="$PWD/build/" \ + "$(which go)" test -v ./e2e/... +} + +export unique_id="${E2E_UNIQUE_ID:-cliendtoendsuite}" +compose_env_file=./e2e/compose-env.yaml + +cmd=${1-} + +case "$cmd" in + setup) + setup "$unique_id" "$compose_env_file" + exit + ;; + cleanup) + cleanup "$unique_id" "$compose_env_file" + exit + ;; + test) + engine_host=${2-} + if [[ -z "${engine_host}" ]]; then + echo "missing parameter docker engine host" + echo "Usage: $0 test ENGINE_HOST" + exit 3 + fi + runtests "$engine_host" + ;; + run|"") + engine_host="$(setup "$unique_id" "$compose_env_file")" + testexit=0 + runtests "$engine_host" || testexit=$? + cleanup "$unique_id" "$compose_env_file" + exit $testexit + ;; + *) + echo "Unknown command: $cmd" + echo "Usage: " + echo " $0 [setup | cleanup | test | run] [engine_host]" + exit 1 + ;; +esac diff --git a/scripts/test/e2e/wait-on-daemon b/scripts/test/e2e/wait-on-daemon new file mode 100755 index 0000000000..d1dd5c39f2 --- /dev/null +++ b/scripts/test/e2e/wait-on-daemon @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -eu -o pipefail + +echo "Waiting for docker daemon to become available at $DOCKER_HOST" +while ! docker version > /dev/null; do + sleep 0.3 +done + +docker version diff --git a/scripts/test/e2e/wrapper b/scripts/test/e2e/wrapper new file mode 100755 index 0000000000..a3a4f00b67 --- /dev/null +++ b/scripts/test/e2e/wrapper @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# Setup, run and teardown e2e test suite in containers. +set -eu -o pipefail + +unique_id="${E2E_UNIQUE_ID:-cliendtoendsuite}" +e2e_env_image=docker-cli-e2e-env:$unique_id +dev_image=docker-cli-dev:$unique_id + +function run_in_env { + local cmd=$1 + docker run -i --rm \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -e E2E_UNIQUE_ID \ + "$e2e_env_image" "$cmd" +} + +docker build \ + -t "$e2e_env_image" \ + -f dockerfiles/Dockerfile.test-e2e-env . + +docker build \ + -t "$dev_image" \ + -f dockerfiles/Dockerfile.dev . + +engine_host=$(run_in_env setup) +testexit=0 +docker run -i --rm \ + -v "$PWD:/go/src/github.com/docker/cli" \ + --network "${unique_id}_default" \ + "$dev_image" \ + ./scripts/test/e2e/run test "$engine_host" || testexit="$?" +run_in_env cleanup +exit "$testexit" diff --git a/scripts/test/watch b/scripts/test/watch index 6c9745aead..264eb7c249 100755 --- a/scripts/test/watch +++ b/scripts/test/watch @@ -1,3 +1,3 @@ #!/bin/sh # shellcheck disable=SC2016 -exec filewatcher -L 6 -x build -x script go test -timeout 10s -v './${dir}' +exec filewatcher -L 6 -x build -x script go test -timeout 30s -v './${dir}' diff --git a/vendor/github.com/gotestyourself/gotestyourself/icmd/command.go b/vendor/github.com/gotestyourself/gotestyourself/icmd/command.go new file mode 100644 index 0000000000..2470fdc662 --- /dev/null +++ b/vendor/github.com/gotestyourself/gotestyourself/icmd/command.go @@ -0,0 +1,277 @@ +/*Package icmd executes binaries and provides convenient assertions for testing the results. + */ +package icmd + +import ( + "bytes" + "fmt" + "io" + "os/exec" + "path/filepath" + "runtime" + "strings" + "sync" + "time" +) + +type testingT interface { + Fatalf(string, ...interface{}) +} + +const ( + // None is a token to inform Result.Assert that the output should be empty + None string = "" +) + +type lockedBuffer struct { + m sync.RWMutex + buf bytes.Buffer +} + +func (buf *lockedBuffer) Write(b []byte) (int, error) { + buf.m.Lock() + defer buf.m.Unlock() + return buf.buf.Write(b) +} + +func (buf *lockedBuffer) String() string { + buf.m.RLock() + defer buf.m.RUnlock() + return buf.buf.String() +} + +// Result stores the result of running a command +type Result struct { + Cmd *exec.Cmd + ExitCode int + Error error + // Timeout is true if the command was killed because it ran for too long + Timeout bool + outBuffer *lockedBuffer + errBuffer *lockedBuffer +} + +// Assert compares the Result against the Expected struct, and fails the test if +// any of the expectations are not met. +func (r *Result) Assert(t testingT, exp Expected) *Result { + err := r.Compare(exp) + if err == nil { + return r + } + _, file, line, ok := runtime.Caller(1) + if ok { + t.Fatalf("at %s:%d - %s\n", filepath.Base(file), line, err.Error()) + } else { + t.Fatalf("(no file/line info) - %s", err.Error()) + } + return nil +} + +// Compare returns a formatted error with the command, stdout, stderr, exit +// code, and any failed expectations +// nolint: gocyclo +func (r *Result) Compare(exp Expected) error { + errors := []string{} + add := func(format string, args ...interface{}) { + errors = append(errors, fmt.Sprintf(format, args...)) + } + + if exp.ExitCode != r.ExitCode { + add("ExitCode was %d expected %d", r.ExitCode, exp.ExitCode) + } + if exp.Timeout != r.Timeout { + if exp.Timeout { + add("Expected command to timeout") + } else { + add("Expected command to finish, but it hit the timeout") + } + } + if !matchOutput(exp.Out, r.Stdout()) { + add("Expected stdout to contain %q", exp.Out) + } + if !matchOutput(exp.Err, r.Stderr()) { + add("Expected stderr to contain %q", exp.Err) + } + switch { + // If a non-zero exit code is expected there is going to be an error. + // Don't require an error message as well as an exit code because the + // error message is going to be "exit status which is not useful + case exp.Error == "" && exp.ExitCode != 0: + case exp.Error == "" && r.Error != nil: + add("Expected no error") + case exp.Error != "" && r.Error == nil: + add("Expected error to contain %q, but there was no error", exp.Error) + case exp.Error != "" && !strings.Contains(r.Error.Error(), exp.Error): + add("Expected error to contain %q", exp.Error) + } + + if len(errors) == 0 { + return nil + } + return fmt.Errorf("%s\nFailures:\n%s", r, strings.Join(errors, "\n")) +} + +func matchOutput(expected string, actual string) bool { + switch expected { + case None: + return actual == "" + default: + return strings.Contains(actual, expected) + } +} + +func (r *Result) String() string { + var timeout string + if r.Timeout { + timeout = " (timeout)" + } + + return fmt.Sprintf(` +Command: %s +ExitCode: %d%s +Error: %v +Stdout: %v +Stderr: %v +`, + strings.Join(r.Cmd.Args, " "), + r.ExitCode, + timeout, + r.Error, + r.Stdout(), + r.Stderr()) +} + +// Expected is the expected output from a Command. This struct is compared to a +// Result struct by Result.Assert(). +type Expected struct { + ExitCode int + Timeout bool + Error string + Out string + Err string +} + +// Success is the default expected result. A Success result is one with a 0 +// ExitCode. +var Success = Expected{} + +// Stdout returns the stdout of the process as a string +func (r *Result) Stdout() string { + return r.outBuffer.String() +} + +// Stderr returns the stderr of the process as a string +func (r *Result) Stderr() string { + return r.errBuffer.String() +} + +// Combined returns the stdout and stderr combined into a single string +func (r *Result) Combined() string { + return r.outBuffer.String() + r.errBuffer.String() +} + +// SetExitError sets Error and ExitCode based on Error +func (r *Result) SetExitError(err error) { + if err == nil { + return + } + r.Error = err + r.ExitCode = processExitCode(err) +} + +// Cmd contains the arguments and options for a process to run as part of a test +// suite. +type Cmd struct { + Command []string + Timeout time.Duration + Stdin io.Reader + Stdout io.Writer + Dir string + Env []string +} + +// Command create a simple Cmd with the specified command and arguments +func Command(command string, args ...string) Cmd { + return Cmd{Command: append([]string{command}, args...)} +} + +// RunCmd runs a command and returns a Result +func RunCmd(cmd Cmd, cmdOperators ...func(*Cmd)) *Result { + for _, op := range cmdOperators { + op(&cmd) + } + result := StartCmd(cmd) + if result.Error != nil { + return result + } + return WaitOnCmd(cmd.Timeout, result) +} + +// RunCommand parses a command line and runs it, returning a result +func RunCommand(command string, args ...string) *Result { + return RunCmd(Command(command, args...)) +} + +// StartCmd starts a command, but doesn't wait for it to finish +func StartCmd(cmd Cmd) *Result { + result := buildCmd(cmd) + if result.Error != nil { + return result + } + result.SetExitError(result.Cmd.Start()) + return result +} + +func buildCmd(cmd Cmd) *Result { + var execCmd *exec.Cmd + switch len(cmd.Command) { + case 1: + execCmd = exec.Command(cmd.Command[0]) + default: + execCmd = exec.Command(cmd.Command[0], cmd.Command[1:]...) + } + outBuffer := new(lockedBuffer) + errBuffer := new(lockedBuffer) + + execCmd.Stdin = cmd.Stdin + execCmd.Dir = cmd.Dir + execCmd.Env = cmd.Env + if cmd.Stdout != nil { + execCmd.Stdout = io.MultiWriter(outBuffer, cmd.Stdout) + } else { + execCmd.Stdout = outBuffer + } + execCmd.Stderr = errBuffer + return &Result{ + Cmd: execCmd, + outBuffer: outBuffer, + errBuffer: errBuffer, + } +} + +// WaitOnCmd waits for a command to complete. If timeout is non-nil then +// only wait until the timeout. +func WaitOnCmd(timeout time.Duration, result *Result) *Result { + if timeout == time.Duration(0) { + result.SetExitError(result.Cmd.Wait()) + return result + } + + done := make(chan error, 1) + // Wait for command to exit in a goroutine + go func() { + done <- result.Cmd.Wait() + }() + + select { + case <-time.After(timeout): + killErr := result.Cmd.Process.Kill() + if killErr != nil { + fmt.Printf("failed to kill (pid=%d): %v\n", result.Cmd.Process.Pid, killErr) + } + result.Timeout = true + case err := <-done: + result.SetExitError(err) + } + return result +} diff --git a/vendor/github.com/gotestyourself/gotestyourself/icmd/exitcode.go b/vendor/github.com/gotestyourself/gotestyourself/icmd/exitcode.go new file mode 100644 index 0000000000..32272b4bbb --- /dev/null +++ b/vendor/github.com/gotestyourself/gotestyourself/icmd/exitcode.go @@ -0,0 +1,32 @@ +package icmd + +import ( + "os/exec" + "syscall" + + "github.com/pkg/errors" +) + +// GetExitCode returns the ExitStatus of a process from the error returned by +// exec.Run(). If the exit status could not be parsed an error is returned. +func GetExitCode(err error) (int, error) { + if exiterr, ok := err.(*exec.ExitError); ok { + if procExit, ok := exiterr.Sys().(syscall.WaitStatus); ok { + return procExit.ExitStatus(), nil + } + } + return 0, errors.Wrap(err, "failed to get exit code") +} + +func processExitCode(err error) (exitCode int) { + if err == nil { + return 0 + } + exitCode, exiterr := GetExitCode(err) + if exiterr != nil { + // TODO: Fix this so we check the error's text. + // we've failed to retrieve exit code, so we set it to 127 + return 127 + } + return exitCode +}