Merge pull request #1064 from chris-crone/e2e-test-image

Create standalone e2e test image

docker.Makefile

@@ -9,6 +9,7 @@ BINARY_NATIVE_IMAGE_NAME = docker-cli-native$(IMAGE_TAG)
 LINTER_IMAGE_NAME = docker-cli-lint$(IMAGE_TAG)
 CROSS_IMAGE_NAME = docker-cli-cross$(IMAGE_TAG)
 VALIDATE_IMAGE_NAME = docker-cli-shell-validate$(IMAGE_TAG)
+E2E_IMAGE_NAME = docker-cli-e2e$(IMAGE_TAG)
 MOUNTS = -v "$(CURDIR)":/go/src/github.com/docker/cli
 VERSION = $(shell cat VERSION)
 ENVVARS = -e VERSION=$(VERSION) -e GITCOMMIT -e PLATFORM
@@ -35,6 +36,10 @@ build_shell_validate_image:
 build_binary_native_image:
 	docker build -t $(BINARY_NATIVE_IMAGE_NAME) -f ./dockerfiles/Dockerfile.binary-native .
 
+.PHONY: build_e2e_image
+build_e2e_image:
+	docker build -t $(E2E_IMAGE_NAME) --build-arg VERSION=$(VERSION) --build-arg GITCOMMIT=$(GITCOMMIT) -f ./dockerfiles/Dockerfile.e2e .
+
 
 # build executable using a container
 binary: build_binary_native_image
@@ -114,5 +119,5 @@ shellcheck: build_shell_validate_image
 	docker run -ti --rm $(ENVVARS) $(MOUNTS) $(VALIDATE_IMAGE_NAME) make shellcheck
 
 .PHONY: test-e2e
-test-e2e: binary
-	./scripts/test/e2e/wrapper
+test-e2e: build_e2e_image
+	docker run --rm -v /var/run/docker.sock:/var/run/docker.sock $(E2E_IMAGE_NAME)

dockerfiles/Dockerfile.dev

@@ -17,12 +17,6 @@ RUN     go get -d github.com/mjibson/esc && \
         go build -v -o /usr/bin/esc . && \
         rm -rf /go/src/* /go/pkg/* /go/bin/*
 
-# FIXME(vdemeester) only used for e2e, could be in e2e special image in the future
-ARG     NOTARY_VERSION=v0.6.0
-RUN     export URL=https://github.com/theupdateframework/notary/releases/download; \
-        curl -Ls $URL/${NOTARY_VERSION}/notary-Linux-amd64 -o /usr/local/bin/notary && \
-        chmod +x /usr/local/bin/notary
-
 ENV     CGO_ENABLED=0 \
         PATH=$PATH:/go/src/github.com/docker/cli/build \
         DISABLE_WARN_OUTSIDE_CONTAINER=1

dockerfiles/Dockerfile.e2e

@@ -0,0 +1,34 @@
+ARG GO_VERSION=1.10.2
+# Use Debian based image as docker-compose requires glibc.
+FROM golang:${GO_VERSION}
+
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    curl \
+    openssl \
+    && rm -rf /var/lib/apt/lists/*
+
+ARG COMPOSE_VERSION=1.21.2
+RUN curl -L https://github.com/docker/compose/releases/download/${COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` -o /usr/local/bin/docker-compose \
+    && chmod +x /usr/local/bin/docker-compose
+
+ARG NOTARY_VERSION=v0.6.1
+RUN curl -Ls https://github.com/theupdateframework/notary/releases/download/${NOTARY_VERSION}/notary-Linux-amd64 -o /usr/local/bin/notary \
+    && chmod +x /usr/local/bin/notary
+
+ENV CGO_ENABLED=0 \
+    DISABLE_WARN_OUTSIDE_CONTAINER=1 \
+    PATH=/go/src/github.com/docker/cli/build:$PATH
+WORKDIR /go/src/github.com/docker/cli
+
+# Trust notary CA cert.
+COPY e2e/testdata/notary/root-ca.cert /usr/share/ca-certificates/notary.cert
+RUN echo 'notary.cert' >> /etc/ca-certificates.conf && update-ca-certificates
+
+COPY . .
+ARG VERSION
+ARG GITCOMMIT
+ENV VERSION=${VERSION} GITCOMMIT=${GITCOMMIT}
+RUN ./scripts/build/binary
+
+CMD ./scripts/test/e2e/entry

dockerfiles/Dockerfile.test-e2e-env

@@ -1,17 +0,0 @@
-FROM    docker/compose:1.15.0
-
-RUN     apk add -U bash curl
-
-ARG     DOCKER_CHANNEL=edge
-ARG     DOCKER_VERSION=17.06.0-ce
-RUN     export URL=https://download.docker.com/linux/static; \
-        curl -Ls $URL/$DOCKER_CHANNEL/x86_64/docker-$DOCKER_VERSION.tgz | \
-        tar -xz docker/docker && \
-        mv docker/docker /usr/local/bin/ && \
-        rmdir docker
-ENV     DISABLE_WARN_OUTSIDE_CONTAINER=1
-WORKDIR /work
-COPY    scripts/test/e2e scripts/test/e2e
-COPY    e2e/compose-env.yaml e2e/compose-env.yaml
-
-ENTRYPOINT ["bash", "/work/scripts/test/e2e/run"]

e2e/compose-env.yaml

@@ -10,12 +10,9 @@ services:
       command: ['--insecure-registry=registry:5000']
 
   notary-server:
-      image: 'notary:server-0.4.2'
+      build:
+        context: ./testdata
+        dockerfile: Dockerfile.notary-server
       ports:
         - 4443:4443
-      volumes:
-       - notary-fixtures:/fixtures
       command: ['notary-server', '-config=/fixtures/notary-config.json']
-
-volumes:
-  notary-fixtures: {}

e2e/container/create_test.go

@@ -5,10 +5,14 @@ import (
 	"testing"
 
 	"github.com/docker/cli/e2e/internal/fixtures"
+	"github.com/docker/cli/internal/test/environment"
 	"github.com/gotestyourself/gotestyourself/icmd"
+	"github.com/gotestyourself/gotestyourself/skip"
 )
 
 func TestCreateWithContentTrust(t *testing.T) {
+	skip.If(t, environment.RemoteDaemon())
+
 	dir := fixtures.SetupConfigFile(t)
 	defer dir.Remove()
 	image := fixtures.CreateMaskedTrustedRemoteImage(t, registryPrefix, "trust-create", "latest")

e2e/container/run_test.go

@@ -5,15 +5,19 @@ import (
 	"testing"
 
 	"github.com/docker/cli/e2e/internal/fixtures"
+	"github.com/docker/cli/internal/test/environment"
 	"github.com/gotestyourself/gotestyourself/assert"
 	is "github.com/gotestyourself/gotestyourself/assert/cmp"
 	"github.com/gotestyourself/gotestyourself/golden"
 	"github.com/gotestyourself/gotestyourself/icmd"
+	"github.com/gotestyourself/gotestyourself/skip"
 )
 
 const registryPrefix = "registry:5000"
 
 func TestRunAttachedFromRemoteImageAndRemove(t *testing.T) {
+	skip.If(t, environment.RemoteDaemon())
+
 	image := createRemoteImage(t)
 
 	result := icmd.RunCommand("docker", "run", "--rm", image,
@@ -25,6 +29,8 @@ func TestRunAttachedFromRemoteImageAndRemove(t *testing.T) {
 }
 
 func TestRunWithContentTrust(t *testing.T) {
+	skip.If(t, environment.RemoteDaemon())
+
 	dir := fixtures.SetupConfigFile(t)
 	defer dir.Remove()
 	image := fixtures.CreateMaskedTrustedRemoteImage(t, registryPrefix, "trust-run", "latest")

e2e/image/build_test.go

@@ -5,9 +5,11 @@ import (
 	"testing"
 
 	"github.com/docker/cli/e2e/internal/fixtures"
+	"github.com/docker/cli/internal/test/environment"
 	"github.com/docker/cli/internal/test/output"
 	"github.com/gotestyourself/gotestyourself/fs"
 	"github.com/gotestyourself/gotestyourself/icmd"
+	"github.com/gotestyourself/gotestyourself/skip"
 )
 
 func TestBuildFromContextDirectoryWithTag(t *testing.T) {
@@ -25,22 +27,25 @@ func TestBuildFromContextDirectoryWithTag(t *testing.T) {
 	result := icmd.RunCmd(
 		icmd.Command("docker", "build", "-t", "myimage", "."),
 		withWorkingDir(dir))
+	defer icmd.RunCommand("docker", "image", "rm", "myimage")
 
 	result.Assert(t, icmd.Expected{Err: icmd.None})
 	output.Assert(t, result.Stdout(), map[int]func(string) error{
 		0:  output.Prefix("Sending build context to Docker daemon"),
-		1:  output.Equals("Step 1/4 : FROM\tregistry:5000/alpine:3.6"),
-		3:  output.Equals("Step 2/4 : COPY\trun /usr/bin/run"),
-		5:  output.Equals("Step 3/4 : RUN\t\trun"),
-		7:  output.Equals("running"),
-		8:  output.Prefix("Removing intermediate container "),
-		10: output.Equals("Step 4/4 : COPY\tdata /data"),
-		12: output.Prefix("Successfully built "),
-		13: output.Equals("Successfully tagged myimage:latest"),
+		1:  output.Suffix("Step 1/4 : FROM registry:5000/alpine:3.6"),
+		3:  output.Suffix("Step 2/4 : COPY run /usr/bin/run"),
+		5:  output.Suffix("Step 3/4 : RUN run"),
+		7:  output.Suffix("running"),
+		8:  output.Contains("Removing intermediate container"),
+		10: output.Suffix("Step 4/4 : COPY data /data"),
+		12: output.Contains("Successfully built "),
+		13: output.Suffix("Successfully tagged myimage:latest"),
 	})
 }
 
 func TestTrustedBuild(t *testing.T) {
+	skip.If(t, environment.RemoteDaemon())
+
 	dir := fixtures.SetupConfigFile(t)
 	defer dir.Remove()
 	image1 := fixtures.CreateMaskedTrustedRemoteImage(t, registryPrefix, "trust-build1", "latest")
@@ -73,6 +78,8 @@ func TestTrustedBuild(t *testing.T) {
 }
 
 func TestTrustedBuildUntrustedImage(t *testing.T) {
+	skip.If(t, environment.RemoteDaemon())
+
 	dir := fixtures.SetupConfigFile(t)
 	defer dir.Remove()
 	buildDir := fs.NewDir(t, "test-trusted-build-context-dir",

e2e/image/pull_test.go

@@ -4,13 +4,17 @@ import (
 	"testing"
 
 	"github.com/docker/cli/e2e/internal/fixtures"
+	"github.com/docker/cli/internal/test/environment"
 	"github.com/gotestyourself/gotestyourself/golden"
 	"github.com/gotestyourself/gotestyourself/icmd"
+	"github.com/gotestyourself/gotestyourself/skip"
 )
 
 const registryPrefix = "registry:5000"
 
 func TestPullWithContentTrust(t *testing.T) {
+	skip.If(t, environment.RemoteDaemon())
+
 	dir := fixtures.SetupConfigFile(t)
 	defer dir.Remove()
 	image := fixtures.CreateMaskedTrustedRemoteImage(t, registryPrefix, "trust-pull", "latest")
@@ -29,6 +33,8 @@ func TestPullWithContentTrust(t *testing.T) {
 }
 
 func TestPullWithContentTrustUsesCacheWhenNotaryUnavailable(t *testing.T) {
+	skip.If(t, environment.RemoteDaemon())
+
 	dir := fixtures.SetupConfigFile(t)
 	defer dir.Remove()
 	image := fixtures.CreateMaskedTrustedRemoteImage(t, registryPrefix, "trust-pull-unreachable", "latest")

e2e/image/push_test.go

@@ -8,11 +8,13 @@ import (
 	"testing"
 
 	"github.com/docker/cli/e2e/internal/fixtures"
+	"github.com/docker/cli/internal/test/environment"
 	"github.com/docker/cli/internal/test/output"
 	"github.com/gotestyourself/gotestyourself/assert"
 	"github.com/gotestyourself/gotestyourself/fs"
 	"github.com/gotestyourself/gotestyourself/golden"
 	"github.com/gotestyourself/gotestyourself/icmd"
+	"github.com/gotestyourself/gotestyourself/skip"
 )
 
 const (
@@ -29,6 +31,8 @@ const (
 )
 
 func TestPushWithContentTrust(t *testing.T) {
+	skip.If(t, environment.RemoteDaemon())
+
 	dir := fixtures.SetupConfigFile(t)
 	defer dir.Remove()
 	image := createImage(t, registryPrefix, "trust-push", "latest")
@@ -52,6 +56,8 @@ func TestPushWithContentTrust(t *testing.T) {
 }
 
 func TestPushWithContentTrustUnreachableServer(t *testing.T) {
+	skip.If(t, environment.RemoteDaemon())
+
 	dir := fixtures.SetupConfigFile(t)
 	defer dir.Remove()
 	image := createImage(t, registryPrefix, "trust-push-unreachable", "latest")
@@ -68,6 +74,8 @@ func TestPushWithContentTrustUnreachableServer(t *testing.T) {
 }
 
 func TestPushWithContentTrustExistingTag(t *testing.T) {
+	skip.If(t, environment.RemoteDaemon())
+
 	dir := fixtures.SetupConfigFile(t)
 	defer dir.Remove()
 	image := createImage(t, registryPrefix, "trust-push-existing", "latest")
@@ -98,6 +106,8 @@ func TestPushWithContentTrustExistingTag(t *testing.T) {
 }
 
 func TestPushWithContentTrustReleasesDelegationOnly(t *testing.T) {
+	skip.If(t, environment.RemoteDaemon())
+
 	role := "targets/releases"
 
 	dir := fixtures.SetupConfigFile(t)
@@ -147,6 +157,8 @@ func TestPushWithContentTrustReleasesDelegationOnly(t *testing.T) {
 }
 
 func TestPushWithContentTrustSignsAllFirstLevelRolesWeHaveKeysFor(t *testing.T) {
+	skip.If(t, environment.RemoteDaemon())
+
 	dir := fixtures.SetupConfigFile(t)
 	defer dir.Remove()
 	copyPrivateKey(t, dir.Join("trust", "private"), privkey1)
@@ -211,6 +223,8 @@ func TestPushWithContentTrustSignsAllFirstLevelRolesWeHaveKeysFor(t *testing.T)
 }
 
 func TestPushWithContentTrustSignsForRolesWithKeysAndValidPaths(t *testing.T) {
+	skip.If(t, environment.RemoteDaemon())
+
 	dir := fixtures.SetupConfigFile(t)
 	defer dir.Remove()
 	copyPrivateKey(t, dir.Join("trust", "private"), privkey1)

e2e/plugin/trust_test.go

@@ -9,16 +9,20 @@ import (
 	"testing"
 
 	"github.com/docker/cli/e2e/internal/fixtures"
+	"github.com/docker/cli/internal/test/environment"
 	"github.com/docker/docker/api/types"
 	"github.com/gotestyourself/gotestyourself/assert"
 	"github.com/gotestyourself/gotestyourself/fs"
 	"github.com/gotestyourself/gotestyourself/icmd"
+	"github.com/gotestyourself/gotestyourself/skip"
 	"github.com/pkg/errors"
 )
 
 const registryPrefix = "registry:5000"
 
 func TestInstallWithContentTrust(t *testing.T) {
+	skip.If(t, environment.SkipPluginTests())
+
 	pluginName := fmt.Sprintf("%s/plugin-content-trust", registryPrefix)
 
 	dir := fixtures.SetupConfigFile(t)
@@ -51,6 +55,8 @@ func TestInstallWithContentTrust(t *testing.T) {
 }
 
 func TestInstallWithContentTrustUntrusted(t *testing.T) {
+	skip.If(t, environment.SkipPluginTests())
+
 	dir := fixtures.SetupConfigFile(t)
 	defer dir.Remove()
 

e2e/stack/deploy_test.go

@@ -1,25 +1,43 @@
 package stack
 
 import (
+	"fmt"
 	"sort"
 	"strings"
 	"testing"
 
+	"github.com/docker/cli/internal/test/environment"
 	"github.com/gotestyourself/gotestyourself/assert"
 	"github.com/gotestyourself/gotestyourself/golden"
 	"github.com/gotestyourself/gotestyourself/icmd"
+	"github.com/gotestyourself/gotestyourself/skip"
 )
 
 func TestDeployWithNamedResources(t *testing.T) {
-	stackname := "test-stack-deploy-with-names"
+	t.Run("Swarm", func(t *testing.T) {
+		testDeployWithNamedResources(t, "swarm")
+	})
+	t.Run("Kubernetes", func(t *testing.T) {
+		// FIXME(chris-crone): currently does not work with compose for kubernetes.
+		t.Skip("FIXME(chris-crone): currently does not work with compose for kubernetes.")
+		skip.If(t, !environment.KubernetesEnabled())
+
+		testDeployWithNamedResources(t, "kubernetes")
+	})
+}
+
+func testDeployWithNamedResources(t *testing.T, orchestrator string) {
+	stackname := fmt.Sprintf("test-stack-deploy-with-names-%s", orchestrator)
 	composefile := golden.Path("stack-with-named-resources.yml")
 
-	result := icmd.RunCommand(
-		"docker", "stack", "deploy", "-c", composefile, stackname)
+	result := icmd.RunCommand("docker", "--orchestrator", orchestrator,
+		"stack", "deploy", "-c", composefile, stackname)
+	defer icmd.RunCommand("docker", "--orchestrator", orchestrator,
+		"stack", "rm", stackname)
 
 	result.Assert(t, icmd.Success)
 	stdout := strings.Split(result.Stdout(), "\n")
-	expected := strings.Split(string(golden.Get(t, "stack-deploy-with-names.golden")), "\n")
+	expected := strings.Split(string(golden.Get(t, fmt.Sprintf("stack-deploy-with-names-%s.golden", orchestrator))), "\n")
 	sort.Strings(stdout)
 	sort.Strings(expected)
 	assert.DeepEqual(t, stdout, expected)

e2e/stack/remove_test.go

@@ -1,6 +1,7 @@
 package stack
 
 import (
+	"fmt"
 	"strings"
 	"testing"
 
@@ -8,50 +9,69 @@ import (
 	"github.com/gotestyourself/gotestyourself/golden"
 	"github.com/gotestyourself/gotestyourself/icmd"
 	"github.com/gotestyourself/gotestyourself/poll"
+	"github.com/gotestyourself/gotestyourself/skip"
 )
 
 var pollSettings = environment.DefaultPollSettings
 
 func TestRemove(t *testing.T) {
-	stackname := "test-stack-remove"
-	deployFullStack(t, stackname)
-	defer cleanupFullStack(t, stackname)
+	t.Run("Swarm", func(t *testing.T) {
+		testRemove(t, "swarm")
+	})
+	t.Run("Kubernetes", func(t *testing.T) {
+		skip.If(t, !environment.KubernetesEnabled())
 
-	result := icmd.RunCommand("docker", "stack", "rm", stackname)
-
-	result.Assert(t, icmd.Expected{Err: icmd.None})
-	golden.Assert(t, result.Stdout(), "stack-remove-success.golden")
+		testRemove(t, "kubernetes")
+	})
 }
 
-func deployFullStack(t *testing.T, stackname string) {
+func testRemove(t *testing.T, orchestrator string) {
+	stackname := "test-stack-remove-" + orchestrator
+	deployFullStack(t, orchestrator, stackname)
+	defer cleanupFullStack(t, orchestrator, stackname)
+	result := icmd.RunCommand("docker", "--orchestrator", orchestrator,
+		"stack", "rm", stackname)
+	result.Assert(t, icmd.Expected{Err: icmd.None})
+	golden.Assert(t, result.Stdout(),
+		fmt.Sprintf("stack-remove-%s-success.golden", orchestrator))
+}
+
+func deployFullStack(t *testing.T, orchestrator, stackname string) {
 	// TODO: this stack should have full options not minimal options
-	result := icmd.RunCommand("docker", "stack", "deploy",
-		"--compose-file=./testdata/full-stack.yml", stackname)
+	result := icmd.RunCommand("docker", "--orchestrator", orchestrator,
+		"stack", "deploy", "--compose-file=./testdata/full-stack.yml", stackname)
 	result.Assert(t, icmd.Success)
 
-	poll.WaitOn(t, taskCount(stackname, 2), pollSettings)
+	poll.WaitOn(t, taskCount(orchestrator, stackname, 2), pollSettings)
 }
 
-func cleanupFullStack(t *testing.T, stackname string) {
+func cleanupFullStack(t *testing.T, orchestrator, stackname string) {
 	// FIXME(vdemeester) we shouldn't have to do that. it is hidding a race on docker stack rm
-	poll.WaitOn(t, stackRm(stackname), pollSettings)
-	poll.WaitOn(t, taskCount(stackname, 0), pollSettings)
+	poll.WaitOn(t, stackRm(orchestrator, stackname), pollSettings)
+	poll.WaitOn(t, taskCount(orchestrator, stackname, 0), pollSettings)
 }
 
-func stackRm(stackname string) func(t poll.LogT) poll.Result {
+func stackRm(orchestrator, stackname string) func(t poll.LogT) poll.Result {
 	return func(poll.LogT) poll.Result {
-		result := icmd.RunCommand("docker", "stack", "rm", stackname)
+		result := icmd.RunCommand("docker", "--orchestrator", orchestrator, "stack", "rm", stackname)
 		if result.Error != nil {
+			if strings.Contains(result.Stderr(), "not found") {
+				return poll.Success()
+			}
 			return poll.Continue("docker stack rm %s failed : %v", stackname, result.Error)
 		}
 		return poll.Success()
 	}
 }
 
-func taskCount(stackname string, expected int) func(t poll.LogT) poll.Result {
+func taskCount(orchestrator, stackname string, expected int) func(t poll.LogT) poll.Result {
 	return func(poll.LogT) poll.Result {
-		result := icmd.RunCommand(
-			"docker", "stack", "ps", "-f=desired-state=running", stackname)
+		args := []string{"--orchestrator", orchestrator, "stack", "ps", stackname}
+		// FIXME(chris-crone): remove when we support filtering by desired-state on kubernetes
+		if orchestrator == "swarm" {
+			args = append(args, "-f=desired-state=running")
+		}
+		result := icmd.RunCommand("docker", args...)
 		count := lines(result.Stdout()) - 1
 		if count == expected {
 			return poll.Success()

e2e/stack/testdata/stack-deploy-with-names-kubernetes.golden

@@ -0,0 +1,7 @@
+Creating network test-stack-deploy-with-names_network2
+Creating network named-network
+Creating secret named-secret
+Creating secret test-stack-deploy-with-names_secret2
+Creating config test-stack-deploy-with-names_config2
+Creating config named-config
+Creating service test-stack-deploy-with-names_web

e2e/stack/testdata/stack-deploy-with-names-swarm.golden

@@ -0,0 +1,7 @@
+Creating network test-stack-deploy-with-names-swarm_network2
+Creating network named-network
+Creating secret named-secret
+Creating secret test-stack-deploy-with-names-swarm_secret2
+Creating config test-stack-deploy-with-names-swarm_config2
+Creating config named-config
+Creating service test-stack-deploy-with-names-swarm_web

e2e/stack/testdata/stack-remove-kubernetes-success.golden

@@ -0,0 +1 @@
+Removing stack: test-stack-remove-kubernetes

e2e/stack/testdata/stack-remove-success.golden

@@ -1,3 +0,0 @@
-Removing service test-stack-remove_one
-Removing service test-stack-remove_two
-Removing network test-stack-remove_default

e2e/stack/testdata/stack-remove-swarm-success.golden

@@ -0,0 +1,3 @@
+Removing service test-stack-remove-swarm_one
+Removing service test-stack-remove-swarm_two
+Removing network test-stack-remove-swarm_default

e2e/testdata/Dockerfile.notary-server

@@ -0,0 +1,4 @@
+ARG NOTARY_VERSION=0.5.0
+FROM notary:server-${NOTARY_VERSION}
+
+COPY ./notary/ /fixtures/

e2e/trust/revoke_test.go

@@ -5,10 +5,12 @@ import (
 	"testing"
 
 	"github.com/docker/cli/e2e/internal/fixtures"
+	"github.com/docker/cli/internal/test/environment"
 	"github.com/gotestyourself/gotestyourself/assert"
 	is "github.com/gotestyourself/gotestyourself/assert/cmp"
 	"github.com/gotestyourself/gotestyourself/fs"
 	"github.com/gotestyourself/gotestyourself/icmd"
+	"github.com/gotestyourself/gotestyourself/skip"
 )
 
 const (
@@ -17,6 +19,8 @@ const (
 )
 
 func TestRevokeImage(t *testing.T) {
+	skip.If(t, environment.RemoteDaemon())
+
 	dir := fixtures.SetupConfigFile(t)
 	defer dir.Remove()
 	setupTrustedImagesForRevoke(t, dir)
@@ -29,6 +33,8 @@ func TestRevokeImage(t *testing.T) {
 }
 
 func TestRevokeRepo(t *testing.T) {
+	skip.If(t, environment.RemoteDaemon())
+
 	dir := fixtures.SetupConfigFile(t)
 	defer dir.Remove()
 	setupTrustedImagesForRevokeRepo(t, dir)

e2e/trust/sign_test.go

@@ -5,10 +5,12 @@ import (
 	"testing"
 
 	"github.com/docker/cli/e2e/internal/fixtures"
+	"github.com/docker/cli/internal/test/environment"
 	"github.com/gotestyourself/gotestyourself/assert"
 	is "github.com/gotestyourself/gotestyourself/assert/cmp"
 	"github.com/gotestyourself/gotestyourself/fs"
 	"github.com/gotestyourself/gotestyourself/icmd"
+	"github.com/gotestyourself/gotestyourself/skip"
 )
 
 const (
@@ -17,6 +19,8 @@ const (
 )
 
 func TestSignLocalImage(t *testing.T) {
+	skip.If(t, environment.RemoteDaemon())
+
 	dir := fixtures.SetupConfigFile(t)
 	defer dir.Remove()
 	icmd.RunCmd(icmd.Command("docker", "pull", fixtures.AlpineImage)).Assert(t, icmd.Success)
@@ -31,6 +35,8 @@ func TestSignLocalImage(t *testing.T) {
 }
 
 func TestSignWithLocalFlag(t *testing.T) {
+	skip.If(t, environment.RemoteDaemon())
+
 	dir := fixtures.SetupConfigFile(t)
 	defer dir.Remove()
 	setupTrustedImageForOverwrite(t, dir)

internal/test/environment/testenv.go

@@ -2,6 +2,7 @@ package environment
 
 import (
 	"os"
+	"strings"
 	"time"
 
 	"github.com/gotestyourself/gotestyourself/poll"
@@ -14,7 +15,63 @@ func Setup() error {
 	if dockerHost == "" {
 		return errors.New("$TEST_DOCKER_HOST must be set")
 	}
-	return os.Setenv("DOCKER_HOST", dockerHost)
+	if err := os.Setenv("DOCKER_HOST", dockerHost); err != nil {
+		return err
+	}
+
+	if dockerCertPath := os.Getenv("TEST_DOCKER_CERT_PATH"); dockerCertPath != "" {
+		if err := os.Setenv("DOCKER_CERT_PATH", dockerCertPath); err != nil {
+			return err
+		}
+		if err := os.Setenv("DOCKER_TLS_VERIFY", "1"); err != nil {
+			return err
+		}
+	}
+
+	if kubeConfig := os.Getenv("TEST_KUBECONFIG"); kubeConfig != "" {
+		if err := os.Setenv("KUBECONFIG", kubeConfig); err != nil {
+			return err
+		}
+	}
+
+	if val := boolFromString(os.Getenv("TEST_REMOTE_DAEMON")); val {
+		if err := os.Setenv("REMOTE_DAEMON", "1"); err != nil {
+			return err
+		}
+	}
+
+	if val := boolFromString(os.Getenv("TEST_SKIP_PLUGIN_TESTS")); val {
+		if err := os.Setenv("SKIP_PLUGIN_TESTS", "1"); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// KubernetesEnabled returns if Kubernetes testing is enabled
+func KubernetesEnabled() bool {
+	return os.Getenv("KUBECONFIG") != ""
+}
+
+// RemoteDaemon returns true if running against a remote daemon
+func RemoteDaemon() bool {
+	return os.Getenv("REMOTE_DAEMON") != ""
+}
+
+// SkipPluginTests returns if plugin tests should be skipped
+func SkipPluginTests() bool {
+	return os.Getenv("SKIP_PLUGIN_TESTS") != ""
+}
+
+// boolFromString determines boolean value from string
+func boolFromString(val string) bool {
+	switch strings.ToLower(val) {
+	case "true", "1":
+		return true
+	default:
+		return false
+	}
 }
 
 // DefaultPollSettings used with gotestyourself/poll

internal/test/output/output.go

@@ -7,7 +7,7 @@ import (
 	"github.com/pkg/errors"
 )
 
-// Assert checks wether the output contains the specified lines
+// Assert checks output lines at specified locations
 func Assert(t *testing.T, actual string, expectedLines map[int]func(string) error) {
 	t.Helper()
 	for i, line := range strings.Split(actual, "\n") {
@@ -24,13 +24,33 @@ func Assert(t *testing.T, actual string, expectedLines map[int]func(string) erro
 	}
 }
 
-// Prefix returns whether if the line has the specified string as prefix
+// Prefix returns whether the line has the specified string as prefix
 func Prefix(expected string) func(string) error {
 	return func(actual string) error {
 		if strings.HasPrefix(actual, expected) {
 			return nil
 		}
-		return errors.Errorf("expected %s to start with %s", actual, expected)
+		return errors.Errorf("expected %q to start with %q", actual, expected)
+	}
+}
+
+// Suffix returns whether the line has the specified string as suffix
+func Suffix(expected string) func(string) error {
+	return func(actual string) error {
+		if strings.HasSuffix(actual, expected) {
+			return nil
+		}
+		return errors.Errorf("expected %q to end with %q", actual, expected)
+	}
+}
+
+// Contains returns whether the line contains the specified string
+func Contains(expected string) func(string) error {
+	return func(actual string) error {
+		if strings.Contains(actual, expected) {
+			return nil
+		}
+		return errors.Errorf("expected %q to contain %q", actual, expected)
 	}
 }
 
@@ -40,6 +60,6 @@ func Equals(expected string) func(string) error {
 		if expected == actual {
 			return nil
 		}
-		return errors.Errorf("got %s, expected %s", actual, expected)
+		return errors.Errorf("got %q, expected %q", actual, expected)
 	}
 }

scripts/test/e2e/entry

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+set -eu -o pipefail
+
+if [[ -n "${REMOTE_DAEMON-}" ]]; then
+  # Run tests against a remote daemon.
+  ./scripts/test/e2e/run fetch-images
+  ./scripts/test/e2e/run test "${DOCKER_HOST-}"
+else
+  # Run tests against dind.
+  ./scripts/test/e2e/wrapper
+fi

scripts/test/e2e/load-alpine

@@ -1,8 +0,0 @@
-#!/usr/bin/env bash
-set -eu -o pipefail
-
-src=alpine@sha256:f006ecbb824d87947d0b51ab8488634bf69fe4094959d935c0c103f4820a417d
-dest=registry:5000/alpine:3.6
-docker pull $src
-docker tag $src $dest
-docker push $dest

scripts/test/e2e/load-busybox

@@ -1,8 +0,0 @@
-#!/usr/bin/env bash
-set -eu -o pipefail
-
-src=busybox@sha256:3e8fa85ddfef1af9ca85a5cfb714148956984e02f00bec3f7f49d3925a91e0e7
-dest=registry:5000/busybox:1.27.2
-docker pull $src
-docker tag $src $dest
-docker push $dest

scripts/test/e2e/load-image

@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+# Fetch images used for e2e testing
+set -eu -o pipefail
+
+alpine_src=alpine@sha256:f006ecbb824d87947d0b51ab8488634bf69fe4094959d935c0c103f4820a417d
+alpine_dest=registry:5000/alpine:3.6
+
+busybox_src=busybox@sha256:3e8fa85ddfef1af9ca85a5cfb714148956984e02f00bec3f7f49d3925a91e0e7
+busybox_dest=registry:5000/busybox:1.27.2
+
+function fetch_tag_image {
+    local src=$1
+    local dest=$2
+    docker pull "$src"
+    docker tag "$src" "$dest"
+}
+
+function push_image {
+  local img=$1
+  docker push "$img"
+}
+
+cmd=${1-}
+case "$cmd" in
+    alpine)
+        fetch_tag_image "$alpine_src" "$alpine_dest"
+        push_image "$alpine_dest"
+        exit
+        ;;
+    busybox)
+        fetch_tag_image "$busybox_src" "$busybox_dest"
+        push_image "$busybox_dest"
+        exit
+        ;;
+    all|"")
+        fetch_tag_image "$alpine_src" "$alpine_dest"
+        push_image "$alpine_dest"
+        fetch_tag_image "$busybox_src" "$busybox_dest"
+        push_image "$busybox_dest"
+        exit
+        ;;
+    fetch-only)
+        fetch_tag_image "$alpine_src" "$alpine_dest"
+        fetch_tag_image "$busybox_src" "$busybox_dest"
+        exit
+        ;;
+    *)
+        echo "Unknown command: $cmd"
+        echo "Usage:"
+        echo "    $0 [alpine | busybox | all | fetch-only]"
+        exit 1
+        ;;
+esac

scripts/test/e2e/run

@@ -9,9 +9,13 @@ function container_ip {
         -f "{{.NetworkSettings.Networks.${network}.IPAddress}}" "$cid"
 }
 
+function fetch_images {
+    ./scripts/test/e2e/load-image fetch-only
+}
+
 function setup {
     local project=$1
-    COMPOSE_PROJECT_NAME=$1 COMPOSE_FILE=$2 docker-compose up -d >&2
+    COMPOSE_PROJECT_NAME=$1 COMPOSE_FILE=$2 docker-compose up --build -d >&2
 
     local network="${project}_default"
     # TODO: only run if inside a container
@@ -21,9 +25,8 @@ function setup {
     engine_host="tcp://$engine_ip:2375"
     (
         export DOCKER_HOST="$engine_host"
-        timeout -t 200 ./scripts/test/e2e/wait-on-daemon
-        ./scripts/test/e2e/load-alpine
-        ./scripts/test/e2e/load-busybox
+        timeout 200 ./scripts/test/e2e/wait-on-daemon
+        ./scripts/test/e2e/load-image
         is_swarm_enabled || docker swarm init
     ) >&2
     echo "$engine_host"
@@ -34,17 +37,22 @@ function is_swarm_enabled {
 }
 
 function cleanup {
-    COMPOSE_PROJECT_NAME=$1 COMPOSE_FILE=$2 docker-compose down -v >&2
+    local project=$1
+    local network="${project}_default"
+    docker network disconnect "$network" "$(hostname)"
+    COMPOSE_PROJECT_NAME=$1 COMPOSE_FILE=$2 docker-compose down -v --rmi local >&2
 }
 
 function runtests {
     local engine_host=$1
 
-    # TODO: only run if inside a container
-    update-ca-certificates
     # shellcheck disable=SC2086
     env -i \
         TEST_DOCKER_HOST="$engine_host" \
+        TEST_DOCKER_CERT_PATH="${DOCKER_CERT_PATH-}" \
+        TEST_KUBECONFIG="${KUBECONFIG-}" \
+        TEST_REMOTE_DAEMON="${REMOTE_DAEMON-}" \
+        TEST_SKIP_PLUGIN_TESTS="${SKIP_PLUGIN_TESTS-}" \
         GOPATH="$GOPATH" \
         PATH="$PWD/build/" \
         "$(which go)" test -v ./e2e/... ${TESTFLAGS-}
@@ -64,6 +72,10 @@ case "$cmd" in
         cleanup "$unique_id" "$compose_env_file"
         exit
         ;;
+    fetch-images)
+        fetch_images
+        exit
+        ;;
     test)
         engine_host=${2-}
         if [[ -z "${engine_host}" ]]; then

scripts/test/e2e/wrapper

@@ -2,35 +2,7 @@
 # Setup, run and teardown e2e test suite in containers.
 set -eu -o pipefail
 
-unique_id="${E2E_UNIQUE_ID:-cliendtoendsuite}"
-e2e_env_image=docker-cli-e2e-env:$unique_id
-dev_image=docker-cli-dev:$unique_id
-
-function run_in_env {
-    local cmd=$1
-    docker run -i --rm \
-        -v /var/run/docker.sock:/var/run/docker.sock \
-        -e E2E_UNIQUE_ID \
-        "$e2e_env_image" "$cmd"
-}
-
-docker build \
-    -t "$e2e_env_image" \
-    -f dockerfiles/Dockerfile.test-e2e-env .
-
-docker build \
-    -t "$dev_image" \
-    -f dockerfiles/Dockerfile.dev .
-
-notary_volume="${unique_id}_notary-fixtures"
-docker volume create --name "$notary_volume"
-docker run --rm \
-    -v "$PWD:/go/src/github.com/docker/cli" \
-    -v "$notary_volume:/data" \
-    "$dev_image" \
-    cp -r ./e2e/testdata/notary/* /data/
-
-engine_host=$(run_in_env setup)
+engine_host=$(./scripts/test/e2e/run setup)
 testexit=0
 
 
@@ -39,14 +11,7 @@ if [[ -n "${TEST_DEBUG-}" ]]; then
     test_cmd="shell"
 fi
 
-docker run -i --rm \
-    -v "$PWD:/go/src/github.com/docker/cli" \
-    -v "$PWD/e2e/testdata/notary/root-ca.cert:/usr/local/share/ca-certificates/notary.cert" \
-    --network "${unique_id}_default" \
-    -e TESTFLAGS \
-    -e ENGINE_HOST="$engine_host" \
-    "$dev_image" \
 ./scripts/test/e2e/run "$test_cmd" "$engine_host" || testexit="$?"
 
-run_in_env cleanup
+./scripts/test/e2e/run cleanup
 exit "$testexit"