Add scripts for setting up e2e test environment.

Signed-off-by: Daniel Nephin <dnephin@docker.com>

Makefile

@@ -10,13 +10,13 @@ _:=$(shell ./scripts/warn-outside-container $(MAKECMDGOALS))
 clean: ## remove build artifacts
 	rm -rf ./build/* cli/winresources/rsrc_* ./man/man[1-9] docs/yaml/gen
 
-.PHONY: test
+.PHONY: test-unit
-test: ## run go test
+test-unit: ## run unit test
-	./scripts/test/unit $(shell go list ./... | grep -v '/vendor/')
+	./scripts/test/unit $(shell go list ./... | grep -vE '/vendor/|/e2e/')
 
 .PHONY: test-coverage
 test-coverage: ## run test coverage
-	./scripts/test/unit-with-coverage $(shell go list ./... | grep -v '/vendor/')
+	./scripts/test/unit-with-coverage $(shell go list ./... | grep -vE '/vendor/|/e2e/')
 
 .PHONY: lint
 lint: ## run all the lint tools

docker.Makefile

@@ -42,9 +42,9 @@ clean: build_docker_image
 	docker run --rm $(ENVVARS) $(MOUNTS) $(DEV_DOCKER_IMAGE_NAME) make clean
 
 # run go test
-.PHONY: test
+.PHONY: test-unit
-test: build_docker_image
+test-unit: build_docker_image
-	docker run --rm $(ENVVARS) $(MOUNTS) $(DEV_DOCKER_IMAGE_NAME) make test
+	docker run --rm $(ENVVARS) $(MOUNTS) $(DEV_DOCKER_IMAGE_NAME) make test-unit
 
 # build the CLI for multiple architectures using a container
 .PHONY: cross
@@ -90,3 +90,7 @@ yamldocs: build_docker_image
 .PHONY: shellcheck
 shellcheck: build_shell_validate_image
 	docker run -ti --rm $(ENVVARS) $(MOUNTS) $(VALIDATE_IMAGE_NAME) make shellcheck
+
+.PHONY: test-e2e:
+test-e2e: binary
+	./scripts/test/e2e/wrapper

dockerfiles/Dockerfile.test-e2e-env

@@ -0,0 +1,15 @@
+FROM    docker/compose:1.15.0
+
+RUN     apk add -U bash curl
+
+RUN     curl -Ls https://download.docker.com/linux/static/edge/x86_64/docker-17.06.0-ce.tgz | \
+        tar -xz docker/docker && \
+        mv docker/docker /usr/local/bin/ && \
+        rmdir docker
+ENV     DISABLE_WARN_OUTSIDE_CONTAINER=1
+WORKDIR /work
+COPY    scripts/test/e2e scripts/test/e2e
+COPY    e2e/compose-env.yaml e2e/compose-env.yaml
+
+ENTRYPOINT ["bash", "/work/scripts/test/e2e/run"]
+CMD     []

e2e/compose-env.yaml

@@ -0,0 +1,10 @@
+version: '3.3'
+
+services:
+  registry:
+    image: 'registry:2'
+
+  engine:
+      image: 'docker:${TEST_ENGINE_VERSION:-edge-dind}'
+      privileged: true
+      command: ['--insecure-registry=registry:5000']

scripts/test/e2e/load-alpine

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -eu -o pipefail
+
+src=alpine:3.6
+dest=registry:5000/alpine:3.6
+docker pull $src
+docker tag $src $dest
+docker push $dest

scripts/test/e2e/run

@@ -0,0 +1,85 @@
+#!/usr/bin/env bash
+# Run integration tests against the latest docker-ce dind
+set -eu -o pipefail
+
+function container_ip {
+    local cid=$1
+    local network=$2
+    docker inspect \
+        -f "{{.NetworkSettings.Networks.${network}.IPAddress}}" "$cid"
+}
+
+function setup {
+    local project=$1
+    COMPOSE_PROJECT_NAME=$1 COMPOSE_FILE=$2 docker-compose up -d >&2
+
+    local network="${project}_default"
+    # TODO: only run if inside a container
+    docker network connect "$network" "$(hostname)"
+
+    engine_ip="$(container_ip "${project}_engine_1" "$network")"
+    engine_host="tcp://$engine_ip:2375"
+    (
+        export DOCKER_HOST="$engine_host"
+        timeout -t 200 ./scripts/test/e2e/wait-on-daemon
+        ./scripts/test/e2e/load-alpine
+        is_swarm_enabled || docker swarm init
+    ) >&2
+    echo "$engine_host"
+}
+
+function is_swarm_enabled {
+    docker info 2> /dev/null | grep -q 'Swarm: active'
+}
+
+function cleanup {
+    COMPOSE_PROJECT_NAME=$1 COMPOSE_FILE=$2 docker-compose down >&2
+}
+
+function runtests {
+    local engine_host=$1
+
+    env -i \
+        TEST_DOCKER_HOST="$engine_host" \
+        GOPATH="$GOPATH" \
+        PATH="$PWD/build/" \
+        "$(which go)" test -v ./e2e/...
+}
+
+export unique_id="${E2E_UNIQUE_ID:-cliendtoendsuite}"
+compose_env_file=./e2e/compose-env.yaml
+
+cmd=${1-}
+
+case "$cmd" in
+    setup)
+        setup "$unique_id" "$compose_env_file"
+        exit
+        ;;
+    cleanup)
+        cleanup "$unique_id" "$compose_env_file"
+        exit
+        ;;
+    test)
+        engine_host=${2-}
+        if [[ -z "${engine_host}" ]]; then
+            echo "missing parameter docker engine host"
+            echo "Usage: $0 test ENGINE_HOST"
+            exit 3
+        fi
+        runtests "$engine_host"
+        ;;
+    run|"")
+        engine_host="$(setup "$unique_id" "$compose_env_file")"
+        testexit=0
+        runtests "$engine_host" || testexit=$?
+        cleanup "$unique_id" "$compose_env_file"
+        exit $testexit
+        ;;
+    *)
+        echo "Unknown command: $cmd"
+        echo "Usage: "
+        echo "    $0 [setup | cleanup | test | run] [engine_host]"
+        exit 1
+        ;;
+esac

scripts/test/e2e/wait-on-daemon

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+set -eu -o pipefail
+
+echo "Waiting for docker daemon to become available at $DOCKER_HOST"
+while ! docker version > /dev/null; do
+    sleep 0.3
+done
+
+docker version

scripts/test/e2e/wrapper

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+# Setup, run and teardown e2e test suite in containers.
+set -eu -o pipefail
+
+unique_id="${E2E_UNIQUE_ID:-cliendtoendsuite}"
+e2e_env_image=docker-cli-e2e-env:$unique_id
+dev_image=docker-cli-dev:$unique_id
+
+function run_in_env {
+    local cmd=$1
+    docker run -i --rm \
+        -v /var/run/docker.sock:/var/run/docker.sock \
+        -e E2E_UNIQUE_ID \
+        "$e2e_env_image" "$cmd"
+}
+
+docker build \
+    -t "$e2e_env_image" \
+    -f dockerfiles/Dockerfile.test-e2e-env .
+
+docker build \
+    -t "$dev_image" \
+    -f dockerfiles/Dockerfile.dev .
+
+engine_host=$(run_in_env setup)
+testexit=0
+docker run -i --rm \
+    -v "$PWD:/go/src/github.com/docker/cli" \
+    --network "${unique_id}_default" \
+    "$dev_image" \
+    ./scripts/test/e2e/run test "$engine_host" || testexit="$?"
+run_in_env cleanup
+exit "$testexit"

scripts/test/watch

@@ -1,3 +1,3 @@
 #!/bin/sh
 # shellcheck disable=SC2016
-exec filewatcher -L 6 -x build -x script go test -timeout 10s -v './${dir}'
+exec filewatcher -L 6 -x build -x script go test -timeout 30s -v './${dir}'