diff --git a/Makefile b/Makefile index ccf36b4665..17ba8e5b2d 100644 --- a/Makefile +++ b/Makefile @@ -10,13 +10,13 @@ _:=$(shell ./scripts/warn-outside-container $(MAKECMDGOALS)) clean: ## remove build artifacts rm -rf ./build/* cli/winresources/rsrc_* ./man/man[1-9] docs/yaml/gen -.PHONY: test -test: ## run go test - ./scripts/test/unit $(shell go list ./... | grep -v '/vendor/') +.PHONY: test-unit +test-unit: ## run unit test + ./scripts/test/unit $(shell go list ./... | grep -vE '/vendor/|/e2e/') .PHONY: test-coverage test-coverage: ## run test coverage - ./scripts/test/unit-with-coverage $(shell go list ./... | grep -v '/vendor/') + ./scripts/test/unit-with-coverage $(shell go list ./... | grep -vE '/vendor/|/e2e/') .PHONY: lint lint: ## run all the lint tools diff --git a/docker.Makefile b/docker.Makefile index b7a34da314..2054c89209 100644 --- a/docker.Makefile +++ b/docker.Makefile @@ -42,9 +42,9 @@ clean: build_docker_image docker run --rm $(ENVVARS) $(MOUNTS) $(DEV_DOCKER_IMAGE_NAME) make clean # run go test -.PHONY: test -test: build_docker_image - docker run --rm $(ENVVARS) $(MOUNTS) $(DEV_DOCKER_IMAGE_NAME) make test +.PHONY: test-unit +test-unit: build_docker_image + docker run --rm $(ENVVARS) $(MOUNTS) $(DEV_DOCKER_IMAGE_NAME) make test-unit # build the CLI for multiple architectures using a container .PHONY: cross @@ -90,3 +90,7 @@ yamldocs: build_docker_image .PHONY: shellcheck shellcheck: build_shell_validate_image docker run -ti --rm $(ENVVARS) $(MOUNTS) $(VALIDATE_IMAGE_NAME) make shellcheck + +.PHONY: test-e2e: +test-e2e: binary + ./scripts/test/e2e/wrapper diff --git a/dockerfiles/Dockerfile.test-e2e-env b/dockerfiles/Dockerfile.test-e2e-env new file mode 100644 index 0000000000..c16b914bdb --- /dev/null +++ b/dockerfiles/Dockerfile.test-e2e-env @@ -0,0 +1,15 @@ +FROM docker/compose:1.15.0 + +RUN apk add -U bash curl + +RUN curl -Ls https://download.docker.com/linux/static/edge/x86_64/docker-17.06.0-ce.tgz | \ + tar -xz docker/docker && \ + mv docker/docker /usr/local/bin/ && \ + rmdir docker +ENV DISABLE_WARN_OUTSIDE_CONTAINER=1 +WORKDIR /work +COPY scripts/test/e2e scripts/test/e2e +COPY e2e/compose-env.yaml e2e/compose-env.yaml + +ENTRYPOINT ["bash", "/work/scripts/test/e2e/run"] +CMD [] diff --git a/e2e/compose-env.yaml b/e2e/compose-env.yaml new file mode 100644 index 0000000000..f16a23c608 --- /dev/null +++ b/e2e/compose-env.yaml @@ -0,0 +1,10 @@ +version: '3.3' + +services: + registry: + image: 'registry:2' + + engine: + image: 'docker:${TEST_ENGINE_VERSION:-edge-dind}' + privileged: true + command: ['--insecure-registry=registry:5000'] diff --git a/scripts/test/e2e/load-alpine b/scripts/test/e2e/load-alpine new file mode 100755 index 0000000000..5b75f0989c --- /dev/null +++ b/scripts/test/e2e/load-alpine @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -eu -o pipefail + +src=alpine:3.6 +dest=registry:5000/alpine:3.6 +docker pull $src +docker tag $src $dest +docker push $dest diff --git a/scripts/test/e2e/run b/scripts/test/e2e/run new file mode 100755 index 0000000000..936b5898fd --- /dev/null +++ b/scripts/test/e2e/run @@ -0,0 +1,85 @@ +#!/usr/bin/env bash +# Run integration tests against the latest docker-ce dind +set -eu -o pipefail + +function container_ip { + local cid=$1 + local network=$2 + docker inspect \ + -f "{{.NetworkSettings.Networks.${network}.IPAddress}}" "$cid" +} + +function setup { + local project=$1 + COMPOSE_PROJECT_NAME=$1 COMPOSE_FILE=$2 docker-compose up -d >&2 + + local network="${project}_default" + # TODO: only run if inside a container + docker network connect "$network" "$(hostname)" + + engine_ip="$(container_ip "${project}_engine_1" "$network")" + engine_host="tcp://$engine_ip:2375" + ( + export DOCKER_HOST="$engine_host" + timeout -t 200 ./scripts/test/e2e/wait-on-daemon + ./scripts/test/e2e/load-alpine + is_swarm_enabled || docker swarm init + ) >&2 + echo "$engine_host" +} + +function is_swarm_enabled { + docker info 2> /dev/null | grep -q 'Swarm: active' +} + +function cleanup { + COMPOSE_PROJECT_NAME=$1 COMPOSE_FILE=$2 docker-compose down >&2 +} + +function runtests { + local engine_host=$1 + + env -i \ + TEST_DOCKER_HOST="$engine_host" \ + GOPATH="$GOPATH" \ + PATH="$PWD/build/" \ + "$(which go)" test -v ./e2e/... +} + +export unique_id="${E2E_UNIQUE_ID:-cliendtoendsuite}" +compose_env_file=./e2e/compose-env.yaml + +cmd=${1-} + +case "$cmd" in + setup) + setup "$unique_id" "$compose_env_file" + exit + ;; + cleanup) + cleanup "$unique_id" "$compose_env_file" + exit + ;; + test) + engine_host=${2-} + if [[ -z "${engine_host}" ]]; then + echo "missing parameter docker engine host" + echo "Usage: $0 test ENGINE_HOST" + exit 3 + fi + runtests "$engine_host" + ;; + run|"") + engine_host="$(setup "$unique_id" "$compose_env_file")" + testexit=0 + runtests "$engine_host" || testexit=$? + cleanup "$unique_id" "$compose_env_file" + exit $testexit + ;; + *) + echo "Unknown command: $cmd" + echo "Usage: " + echo " $0 [setup | cleanup | test | run] [engine_host]" + exit 1 + ;; +esac diff --git a/scripts/test/e2e/wait-on-daemon b/scripts/test/e2e/wait-on-daemon new file mode 100755 index 0000000000..d1dd5c39f2 --- /dev/null +++ b/scripts/test/e2e/wait-on-daemon @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -eu -o pipefail + +echo "Waiting for docker daemon to become available at $DOCKER_HOST" +while ! docker version > /dev/null; do + sleep 0.3 +done + +docker version diff --git a/scripts/test/e2e/wrapper b/scripts/test/e2e/wrapper new file mode 100755 index 0000000000..a3a4f00b67 --- /dev/null +++ b/scripts/test/e2e/wrapper @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# Setup, run and teardown e2e test suite in containers. +set -eu -o pipefail + +unique_id="${E2E_UNIQUE_ID:-cliendtoendsuite}" +e2e_env_image=docker-cli-e2e-env:$unique_id +dev_image=docker-cli-dev:$unique_id + +function run_in_env { + local cmd=$1 + docker run -i --rm \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -e E2E_UNIQUE_ID \ + "$e2e_env_image" "$cmd" +} + +docker build \ + -t "$e2e_env_image" \ + -f dockerfiles/Dockerfile.test-e2e-env . + +docker build \ + -t "$dev_image" \ + -f dockerfiles/Dockerfile.dev . + +engine_host=$(run_in_env setup) +testexit=0 +docker run -i --rm \ + -v "$PWD:/go/src/github.com/docker/cli" \ + --network "${unique_id}_default" \ + "$dev_image" \ + ./scripts/test/e2e/run test "$engine_host" || testexit="$?" +run_in_env cleanup +exit "$testexit" diff --git a/scripts/test/watch b/scripts/test/watch index 6c9745aead..264eb7c249 100755 --- a/scripts/test/watch +++ b/scripts/test/watch @@ -1,3 +1,3 @@ #!/bin/sh # shellcheck disable=SC2016 -exec filewatcher -L 6 -x build -x script go test -timeout 10s -v './${dir}' +exec filewatcher -L 6 -x build -x script go test -timeout 30s -v './${dir}'