Merge pull request #445 from dnephin/add-end-to-end-suite

Add end to end suite

Jenkinsfile

@@ -0,0 +1,12 @@
+wrappedNode(label: 'linux && x86_64', cleanWorkspace: true) {
+  timeout(time: 60, unit: 'MINUTES') {
+    stage "Git Checkout"
+    checkout scm
+
+    stage "Run end-to-end test suite"
+    sh "docker version"
+    sh "E2E_UNIQUE_ID=clie2e${BUILD_NUMBER} \
+        IMAGE_TAG=clie2e${BUILD_NUMBER} \
+        make -f docker.Makefile test-e2e"
+  }
+}

Makefile

@@ -10,13 +10,13 @@ _:=$(shell ./scripts/warn-outside-container $(MAKECMDGOALS))
 clean: ## remove build artifacts
 	rm -rf ./build/* cli/winresources/rsrc_* ./man/man[1-9] docs/yaml/gen
 
-.PHONY: test
+.PHONY: test-unit
-test: ## run go test
+test-unit: ## run unit test
-	./scripts/test/unit $(shell go list ./... | grep -v '/vendor/')
+	./scripts/test/unit $(shell go list ./... | grep -vE '/vendor/|/e2e/')
 
 .PHONY: test-coverage
 test-coverage: ## run test coverage
-	./scripts/test/unit-with-coverage $(shell go list ./... | grep -v '/vendor/')
+	./scripts/test/unit-with-coverage $(shell go list ./... | grep -vE '/vendor/|/e2e/')
 
 .PHONY: lint
 lint: ## run all the lint tools

cli/command/stack/remove.go

@@ -2,6 +2,7 @@ package stack
 
 import (
 	"fmt"
+	"sort"
 	"strings"
 
 	"github.com/docker/cli/cli"
@@ -88,12 +89,19 @@ func runRemove(dockerCli command.Cli, opts removeOptions) error {
 	return nil
 }
 
+func sortServiceByName(services []swarm.Service) func(i, j int) bool {
+	return func(i, j int) bool {
+		return services[i].Spec.Name < services[j].Spec.Name
+	}
+}
+
 func removeServices(
 	ctx context.Context,
 	dockerCli command.Cli,
 	services []swarm.Service,
 ) bool {
 	var hasError bool
+	sort.Slice(services, sortServiceByName(services))
 	for _, service := range services {
 		fmt.Fprintf(dockerCli.Err(), "Removing service %s\n", service.Spec.Name)
 		if err := dockerCli.Client().ServiceRemove(ctx, service.ID); err != nil {

docker.Makefile

@@ -4,10 +4,10 @@
 # Makefile for developing using Docker
 #
 
-DEV_DOCKER_IMAGE_NAME = docker-cli-dev
+DEV_DOCKER_IMAGE_NAME = docker-cli-dev$(IMAGE_TAG)
-LINTER_IMAGE_NAME = docker-cli-lint
+LINTER_IMAGE_NAME = docker-cli-lint$(IMAGE_TAG)
-CROSS_IMAGE_NAME = docker-cli-cross
+CROSS_IMAGE_NAME = docker-cli-cross$(IMAGE_TAG)
-VALIDATE_IMAGE_NAME = docker-cli-shell-validate
+VALIDATE_IMAGE_NAME = docker-cli-shell-validate$(IMAGE_TAG)
 MOUNTS = -v "$(CURDIR)":/go/src/github.com/docker/cli
 VERSION = $(shell cat VERSION)
 ENVVARS = -e VERSION=$(VERSION) -e GITCOMMIT
@@ -42,9 +42,9 @@ clean: build_docker_image
 	docker run --rm $(ENVVARS) $(MOUNTS) $(DEV_DOCKER_IMAGE_NAME) make clean
 
 # run go test
-.PHONY: test
+.PHONY: test-unit
-test: build_docker_image
+test-unit: build_docker_image
-	docker run --rm $(ENVVARS) $(MOUNTS) $(DEV_DOCKER_IMAGE_NAME) make test
+	docker run --rm $(ENVVARS) $(MOUNTS) $(DEV_DOCKER_IMAGE_NAME) make test-unit
 
 # build the CLI for multiple architectures using a container
 .PHONY: cross
@@ -90,3 +90,7 @@ yamldocs: build_docker_image
 .PHONY: shellcheck
 shellcheck: build_shell_validate_image
 	docker run -ti --rm $(ENVVARS) $(MOUNTS) $(VALIDATE_IMAGE_NAME) make shellcheck
+
+.PHONY: test-e2e
+test-e2e: binary
+	./scripts/test/e2e/wrapper

dockerfiles/Dockerfile.test-e2e-env

@@ -0,0 +1,17 @@
+FROM    docker/compose:1.15.0
+
+RUN     apk add -U bash curl
+
+ARG     DOCKER_CHANNEL=edge
+ARG     DOCKER_VERSION=17.06.0-ce
+RUN     export URL=https://download.docker.com/linux/static; \
+        curl -Ls $URL/$DOCKER_CHANNEL/x86_64/docker-$DOCKER_VERSION.tgz | \
+        tar -xz docker/docker && \
+        mv docker/docker /usr/local/bin/ && \
+        rmdir docker
+ENV     DISABLE_WARN_OUTSIDE_CONTAINER=1
+WORKDIR /work
+COPY    scripts/test/e2e scripts/test/e2e
+COPY    e2e/compose-env.yaml e2e/compose-env.yaml
+
+ENTRYPOINT ["bash", "/work/scripts/test/e2e/run"]

e2e/compose-env.yaml

@@ -0,0 +1,10 @@
+version: '2.1'
+
+services:
+  registry:
+    image: 'registry:2'
+
+  engine:
+      image: 'docker:${TEST_ENGINE_VERSION:-edge-dind}'
+      privileged: true
+      command: ['--insecure-registry=registry:5000']

e2e/stack/main_test.go

@@ -0,0 +1,26 @@
+package stack
+
+import (
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/pkg/errors"
+)
+
+func TestMain(m *testing.M) {
+	if err := setupTestEnv(); err != nil {
+		fmt.Println(err.Error())
+		os.Exit(3)
+	}
+	os.Exit(m.Run())
+}
+
+// TODO: move to shared internal package
+func setupTestEnv() error {
+	dockerHost := os.Getenv("TEST_DOCKER_HOST")
+	if dockerHost == "" {
+		return errors.New("$TEST_DOCKER_HOST must be set")
+	}
+	return os.Setenv("DOCKER_HOST", dockerHost)
+}

e2e/stack/remove_test.go

@@ -0,0 +1,89 @@
+package stack
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+
+	shlex "github.com/flynn-archive/go-shlex"
+	"github.com/gotestyourself/gotestyourself/golden"
+	"github.com/gotestyourself/gotestyourself/icmd"
+	"github.com/stretchr/testify/require"
+)
+
+func TestRemove(t *testing.T) {
+	stackname := "test-stack-remove"
+	deployFullStack(t, stackname)
+	defer cleanupFullStack(t, stackname)
+
+	result := icmd.RunCmd(shell(t, "docker stack rm %s", stackname))
+
+	result.Assert(t, icmd.Expected{Out: icmd.None})
+	golden.Assert(t, result.Stderr(), "stack-remove-success.golden")
+}
+
+func deployFullStack(t *testing.T, stackname string) {
+	// TODO: this stack should have full options not minimal options
+	result := icmd.RunCmd(shell(t,
+		"docker stack deploy --compose-file=./testdata/full-stack.yml %s", stackname))
+	result.Assert(t, icmd.Success)
+
+	waitOn(t, taskCount(stackname, 2), 0)
+}
+
+func cleanupFullStack(t *testing.T, stackname string) {
+	result := icmd.RunCmd(shell(t, "docker stack rm %s", stackname))
+	result.Assert(t, icmd.Success)
+	waitOn(t, taskCount(stackname, 0), 0)
+}
+
+func taskCount(stackname string, expected int) func() (bool, error) {
+	return func() (bool, error) {
+		result := icmd.RunCommand(
+			"docker", "stack", "ps", "-f=desired-state=running", stackname)
+		count := lines(result.Stdout()) - 1
+		return count == expected, nil
+	}
+}
+
+func lines(out string) int {
+	return len(strings.Split(strings.TrimSpace(out), "\n"))
+}
+
+// TODO: move to gotestyourself
+func shell(t *testing.T, format string, args ...interface{}) icmd.Cmd {
+	cmd, err := shlex.Split(fmt.Sprintf(format, args...))
+	require.NoError(t, err)
+	return icmd.Cmd{Command: cmd}
+}
+
+// TODO: move to gotestyourself
+func waitOn(t *testing.T, check func() (bool, error), timeout time.Duration) {
+	if timeout == time.Duration(0) {
+		timeout = defaultTimeout()
+	}
+
+	after := time.After(timeout)
+	for {
+		select {
+		case <-after:
+			// TODO: include check function name in error message
+			t.Fatalf("timeout hit after %s", timeout)
+		default:
+			// TODO: maybe return a failure message as well?
+			done, err := check()
+			if done {
+				return
+			}
+			if err != nil {
+				t.Fatal(err.Error())
+			}
+		}
+	}
+}
+
+func defaultTimeout() time.Duration {
+	// TODO: support override from environment variable
+	return 10 * time.Second
+}

e2e/stack/testdata/full-stack.yml

@@ -0,0 +1,9 @@
+version: '3.3'
+
+services:
+  one:
+    image: registry:5000/alpine:3.6
+    command: top
+  two:
+    image: registry:5000/alpine:3.6
+    command: top

e2e/stack/testdata/stack-remove-success.golden

@@ -0,0 +1,3 @@
+Removing service test-stack-remove_one
+Removing service test-stack-remove_two
+Removing network test-stack-remove_default

scripts/test/e2e/load-alpine

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -eu -o pipefail
+
+src=alpine:3.6
+dest=registry:5000/alpine:3.6
+docker pull $src
+docker tag $src $dest
+docker push $dest

scripts/test/e2e/run

@@ -0,0 +1,85 @@
+#!/usr/bin/env bash
+# Run integration tests against the latest docker-ce dind
+set -eu -o pipefail
+
+function container_ip {
+    local cid=$1
+    local network=$2
+    docker inspect \
+        -f "{{.NetworkSettings.Networks.${network}.IPAddress}}" "$cid"
+}
+
+function setup {
+    local project=$1
+    COMPOSE_PROJECT_NAME=$1 COMPOSE_FILE=$2 docker-compose up -d >&2
+
+    local network="${project}_default"
+    # TODO: only run if inside a container
+    docker network connect "$network" "$(hostname)"
+
+    engine_ip="$(container_ip "${project}_engine_1" "$network")"
+    engine_host="tcp://$engine_ip:2375"
+    (
+        export DOCKER_HOST="$engine_host"
+        timeout -t 200 ./scripts/test/e2e/wait-on-daemon
+        ./scripts/test/e2e/load-alpine
+        is_swarm_enabled || docker swarm init
+    ) >&2
+    echo "$engine_host"
+}
+
+function is_swarm_enabled {
+    docker info 2> /dev/null | grep -q 'Swarm: active'
+}
+
+function cleanup {
+    COMPOSE_PROJECT_NAME=$1 COMPOSE_FILE=$2 docker-compose down >&2
+}
+
+function runtests {
+    local engine_host=$1
+
+    env -i \
+        TEST_DOCKER_HOST="$engine_host" \
+        GOPATH="$GOPATH" \
+        PATH="$PWD/build/" \
+        "$(which go)" test -v ./e2e/...
+}
+
+export unique_id="${E2E_UNIQUE_ID:-cliendtoendsuite}"
+compose_env_file=./e2e/compose-env.yaml
+
+cmd=${1-}
+
+case "$cmd" in
+    setup)
+        setup "$unique_id" "$compose_env_file"
+        exit
+        ;;
+    cleanup)
+        cleanup "$unique_id" "$compose_env_file"
+        exit
+        ;;
+    test)
+        engine_host=${2-}
+        if [[ -z "${engine_host}" ]]; then
+            echo "missing parameter docker engine host"
+            echo "Usage: $0 test ENGINE_HOST"
+            exit 3
+        fi
+        runtests "$engine_host"
+        ;;
+    run|"")
+        engine_host="$(setup "$unique_id" "$compose_env_file")"
+        testexit=0
+        runtests "$engine_host" || testexit=$?
+        cleanup "$unique_id" "$compose_env_file"
+        exit $testexit
+        ;;
+    *)
+        echo "Unknown command: $cmd"
+        echo "Usage: "
+        echo "    $0 [setup | cleanup | test | run] [engine_host]"
+        exit 1
+        ;;
+esac

scripts/test/e2e/wait-on-daemon

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+set -eu -o pipefail
+
+echo "Waiting for docker daemon to become available at $DOCKER_HOST"
+while ! docker version > /dev/null; do
+    sleep 0.3
+done
+
+docker version

scripts/test/e2e/wrapper

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+# Setup, run and teardown e2e test suite in containers.
+set -eu -o pipefail
+
+unique_id="${E2E_UNIQUE_ID:-cliendtoendsuite}"
+e2e_env_image=docker-cli-e2e-env:$unique_id
+dev_image=docker-cli-dev:$unique_id
+
+function run_in_env {
+    local cmd=$1
+    docker run -i --rm \
+        -v /var/run/docker.sock:/var/run/docker.sock \
+        -e E2E_UNIQUE_ID \
+        "$e2e_env_image" "$cmd"
+}
+
+docker build \
+    -t "$e2e_env_image" \
+    -f dockerfiles/Dockerfile.test-e2e-env .
+
+docker build \
+    -t "$dev_image" \
+    -f dockerfiles/Dockerfile.dev .
+
+engine_host=$(run_in_env setup)
+testexit=0
+docker run -i --rm \
+    -v "$PWD:/go/src/github.com/docker/cli" \
+    --network "${unique_id}_default" \
+    "$dev_image" \
+    ./scripts/test/e2e/run test "$engine_host" || testexit="$?"
+run_in_env cleanup
+exit "$testexit"

scripts/test/watch

@@ -1,3 +1,3 @@
 #!/bin/sh
 # shellcheck disable=SC2016
-exec filewatcher -L 6 -x build -x script go test -timeout 10s -v './${dir}'
+exec filewatcher -L 6 -x build -x script go test -timeout 30s -v './${dir}'

vendor/github.com/gotestyourself/gotestyourself/icmd/command.go

@@ -0,0 +1,277 @@
+/*Package icmd executes binaries and provides convenient assertions for testing the results.
+ */
+package icmd
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"sync"
+	"time"
+)
+
+type testingT interface {
+	Fatalf(string, ...interface{})
+}
+
+const (
+	// None is a token to inform Result.Assert that the output should be empty
+	None string = "<NOTHING>"
+)
+
+type lockedBuffer struct {
+	m   sync.RWMutex
+	buf bytes.Buffer
+}
+
+func (buf *lockedBuffer) Write(b []byte) (int, error) {
+	buf.m.Lock()
+	defer buf.m.Unlock()
+	return buf.buf.Write(b)
+}
+
+func (buf *lockedBuffer) String() string {
+	buf.m.RLock()
+	defer buf.m.RUnlock()
+	return buf.buf.String()
+}
+
+// Result stores the result of running a command
+type Result struct {
+	Cmd      *exec.Cmd
+	ExitCode int
+	Error    error
+	// Timeout is true if the command was killed because it ran for too long
+	Timeout   bool
+	outBuffer *lockedBuffer
+	errBuffer *lockedBuffer
+}
+
+// Assert compares the Result against the Expected struct, and fails the test if
+// any of the expectations are not met.
+func (r *Result) Assert(t testingT, exp Expected) *Result {
+	err := r.Compare(exp)
+	if err == nil {
+		return r
+	}
+	_, file, line, ok := runtime.Caller(1)
+	if ok {
+		t.Fatalf("at %s:%d - %s\n", filepath.Base(file), line, err.Error())
+	} else {
+		t.Fatalf("(no file/line info) - %s", err.Error())
+	}
+	return nil
+}
+
+// Compare returns a formatted error with the command, stdout, stderr, exit
+// code, and any failed expectations
+// nolint: gocyclo
+func (r *Result) Compare(exp Expected) error {
+	errors := []string{}
+	add := func(format string, args ...interface{}) {
+		errors = append(errors, fmt.Sprintf(format, args...))
+	}
+
+	if exp.ExitCode != r.ExitCode {
+		add("ExitCode was %d expected %d", r.ExitCode, exp.ExitCode)
+	}
+	if exp.Timeout != r.Timeout {
+		if exp.Timeout {
+			add("Expected command to timeout")
+		} else {
+			add("Expected command to finish, but it hit the timeout")
+		}
+	}
+	if !matchOutput(exp.Out, r.Stdout()) {
+		add("Expected stdout to contain %q", exp.Out)
+	}
+	if !matchOutput(exp.Err, r.Stderr()) {
+		add("Expected stderr to contain %q", exp.Err)
+	}
+	switch {
+	// If a non-zero exit code is expected there is going to be an error.
+	// Don't require an error message as well as an exit code because the
+	// error message is going to be "exit status <code> which is not useful
+	case exp.Error == "" && exp.ExitCode != 0:
+	case exp.Error == "" && r.Error != nil:
+		add("Expected no error")
+	case exp.Error != "" && r.Error == nil:
+		add("Expected error to contain %q, but there was no error", exp.Error)
+	case exp.Error != "" && !strings.Contains(r.Error.Error(), exp.Error):
+		add("Expected error to contain %q", exp.Error)
+	}
+
+	if len(errors) == 0 {
+		return nil
+	}
+	return fmt.Errorf("%s\nFailures:\n%s", r, strings.Join(errors, "\n"))
+}
+
+func matchOutput(expected string, actual string) bool {
+	switch expected {
+	case None:
+		return actual == ""
+	default:
+		return strings.Contains(actual, expected)
+	}
+}
+
+func (r *Result) String() string {
+	var timeout string
+	if r.Timeout {
+		timeout = " (timeout)"
+	}
+
+	return fmt.Sprintf(`
+Command:  %s
+ExitCode: %d%s
+Error:    %v
+Stdout:   %v
+Stderr:   %v
+`,
+		strings.Join(r.Cmd.Args, " "),
+		r.ExitCode,
+		timeout,
+		r.Error,
+		r.Stdout(),
+		r.Stderr())
+}
+
+// Expected is the expected output from a Command. This struct is compared to a
+// Result struct by Result.Assert().
+type Expected struct {
+	ExitCode int
+	Timeout  bool
+	Error    string
+	Out      string
+	Err      string
+}
+
+// Success is the default expected result. A Success result is one with a 0
+// ExitCode.
+var Success = Expected{}
+
+// Stdout returns the stdout of the process as a string
+func (r *Result) Stdout() string {
+	return r.outBuffer.String()
+}
+
+// Stderr returns the stderr of the process as a string
+func (r *Result) Stderr() string {
+	return r.errBuffer.String()
+}
+
+// Combined returns the stdout and stderr combined into a single string
+func (r *Result) Combined() string {
+	return r.outBuffer.String() + r.errBuffer.String()
+}
+
+// SetExitError sets Error and ExitCode based on Error
+func (r *Result) SetExitError(err error) {
+	if err == nil {
+		return
+	}
+	r.Error = err
+	r.ExitCode = processExitCode(err)
+}
+
+// Cmd contains the arguments and options for a process to run as part of a test
+// suite.
+type Cmd struct {
+	Command []string
+	Timeout time.Duration
+	Stdin   io.Reader
+	Stdout  io.Writer
+	Dir     string
+	Env     []string
+}
+
+// Command create a simple Cmd with the specified command and arguments
+func Command(command string, args ...string) Cmd {
+	return Cmd{Command: append([]string{command}, args...)}
+}
+
+// RunCmd runs a command and returns a Result
+func RunCmd(cmd Cmd, cmdOperators ...func(*Cmd)) *Result {
+	for _, op := range cmdOperators {
+		op(&cmd)
+	}
+	result := StartCmd(cmd)
+	if result.Error != nil {
+		return result
+	}
+	return WaitOnCmd(cmd.Timeout, result)
+}
+
+// RunCommand parses a command line and runs it, returning a result
+func RunCommand(command string, args ...string) *Result {
+	return RunCmd(Command(command, args...))
+}
+
+// StartCmd starts a command, but doesn't wait for it to finish
+func StartCmd(cmd Cmd) *Result {
+	result := buildCmd(cmd)
+	if result.Error != nil {
+		return result
+	}
+	result.SetExitError(result.Cmd.Start())
+	return result
+}
+
+func buildCmd(cmd Cmd) *Result {
+	var execCmd *exec.Cmd
+	switch len(cmd.Command) {
+	case 1:
+		execCmd = exec.Command(cmd.Command[0])
+	default:
+		execCmd = exec.Command(cmd.Command[0], cmd.Command[1:]...)
+	}
+	outBuffer := new(lockedBuffer)
+	errBuffer := new(lockedBuffer)
+
+	execCmd.Stdin = cmd.Stdin
+	execCmd.Dir = cmd.Dir
+	execCmd.Env = cmd.Env
+	if cmd.Stdout != nil {
+		execCmd.Stdout = io.MultiWriter(outBuffer, cmd.Stdout)
+	} else {
+		execCmd.Stdout = outBuffer
+	}
+	execCmd.Stderr = errBuffer
+	return &Result{
+		Cmd:       execCmd,
+		outBuffer: outBuffer,
+		errBuffer: errBuffer,
+	}
+}
+
+// WaitOnCmd waits for a command to complete. If timeout is non-nil then
+// only wait until the timeout.
+func WaitOnCmd(timeout time.Duration, result *Result) *Result {
+	if timeout == time.Duration(0) {
+		result.SetExitError(result.Cmd.Wait())
+		return result
+	}
+
+	done := make(chan error, 1)
+	// Wait for command to exit in a goroutine
+	go func() {
+		done <- result.Cmd.Wait()
+	}()
+
+	select {
+	case <-time.After(timeout):
+		killErr := result.Cmd.Process.Kill()
+		if killErr != nil {
+			fmt.Printf("failed to kill (pid=%d): %v\n", result.Cmd.Process.Pid, killErr)
+		}
+		result.Timeout = true
+	case err := <-done:
+		result.SetExitError(err)
+	}
+	return result
+}

vendor/github.com/gotestyourself/gotestyourself/icmd/exitcode.go

@@ -0,0 +1,32 @@
+package icmd
+
+import (
+	"os/exec"
+	"syscall"
+
+	"github.com/pkg/errors"
+)
+
+// GetExitCode returns the ExitStatus of a process from the error returned by
+// exec.Run(). If the exit status could not be parsed an error is returned.
+func GetExitCode(err error) (int, error) {
+	if exiterr, ok := err.(*exec.ExitError); ok {
+		if procExit, ok := exiterr.Sys().(syscall.WaitStatus); ok {
+			return procExit.ExitStatus(), nil
+		}
+	}
+	return 0, errors.Wrap(err, "failed to get exit code")
+}
+
+func processExitCode(err error) (exitCode int) {
+	if err == nil {
+		return 0
+	}
+	exitCode, exiterr := GetExitCode(err)
+	if exiterr != nil {
+		// TODO: Fix this so we check the error's text.
+		// we've failed to retrieve exit code, so we set it to 127
+		return 127
+	}
+	return exitCode
+}