Update e2e test for image pull to check stdout

Also add TEST_DEBUG env variable for debugging E2E tests.
And change icmd environment helpers to fit the CmdOp interface os they
can be passed to 'icmd.RunCmd()'

Signed-off-by: Daniel Nephin <dnephin@docker.com>

e2e/image/pull_test.go

@@ -3,11 +3,10 @@ package image
 import (
 	"fmt"
 	"os"
-	"strings"
 	"testing"
 
+	"github.com/gotestyourself/gotestyourself/golden"
 	"github.com/gotestyourself/gotestyourself/icmd"
-	"github.com/stretchr/testify/require"
 )
 
 const notaryURL = "https://notary-server:4443"
@@ -17,36 +16,29 @@ const alpineImage = "registry:5000/alpine:3.6"
 const busyboxImage = "registry:5000/busybox:1.27.2"
 
 func TestPullWithContentTrust(t *testing.T) {
-	image := createTrustedRemoteImage(t, "trust", "latest")
+	image := createMaskedTrustedRemoteImage(t, "trust", "latest")
 
-	// test that pulling without the tag defaults to latest
+	result := icmd.RunCmd(icmd.Command("docker", "pull", image), withTrustNoPassphrase)
-	imageWithoutTag := strings.TrimSuffix(image, ":latest")
+	result.Assert(t, icmd.Expected{Err: icmd.None})
-	icmd.RunCmd(trustedCmdNoPassphrases(icmd.Command("docker", "pull", imageWithoutTag))).Assert(t, icmd.Success)
+	golden.Assert(t, result.Stdout(), "pull-with-content-trust.golden")
-	icmd.RunCommand("docker", "rmi", image).Assert(t, icmd.Success)
+}
 
-	// try pulling with the tag, record output for comparison later
+// createMaskedTrustedRemoteImage creates a remote image that is signed with
-	result := icmd.RunCmd(trustedCmdNoPassphrases(icmd.Command("docker", "pull", image)))
+// content trust, then pushes a different untrusted image at the same tag.
-	result.Assert(t, icmd.Success)
+func createMaskedTrustedRemoteImage(t *testing.T, repo, tag string) string {
-	firstPullOutput := result.String()
+	image := createTrustedRemoteImage(t, repo, tag)
-	icmd.RunCommand("docker", "rmi", image).Assert(t, icmd.Success)
-
-	// push an unsigned image on the same reference name, but with different content (busybox)
 	createNamedUnsignedImageFromBusyBox(t, image)
-
+	return image
-	// now pull with content trust
-	result = icmd.RunCmd(trustedCmdNoPassphrases(icmd.Command("docker", "pull", image)))
-	result.Assert(t, icmd.Success)
-	secondPullOutput := result.String()
-
-	// assert that the digest and other output is the same since we ignore the unsigned image
-	require.Equal(t, firstPullOutput, secondPullOutput)
 }
 
 func createTrustedRemoteImage(t *testing.T, repo, tag string) string {
 	image := fmt.Sprintf("%s/%s:%s", registryPrefix, repo, tag)
 	icmd.RunCommand("docker", "pull", alpineImage).Assert(t, icmd.Success)
 	icmd.RunCommand("docker", "tag", alpineImage, image).Assert(t, icmd.Success)
-	icmd.RunCmd(trustedCmdWithPassphrases(icmd.Command("docker", "push", image), "root_password", "repo_password")).Assert(t, icmd.Success)
+	result := icmd.RunCmd(
+		icmd.Command("docker", "push", image),
+		withTrustAndPassphrase("root_password", "repo_password"))
+	result.Assert(t, icmd.Success)
 	icmd.RunCommand("docker", "rmi", image).Assert(t, icmd.Success)
 	return image
 }
@@ -58,22 +50,22 @@ func createNamedUnsignedImageFromBusyBox(t *testing.T, image string) {
 	icmd.RunCommand("docker", "rmi", image).Assert(t, icmd.Success)
 }
 
-func trustedCmdWithPassphrases(cmd icmd.Cmd, rootPwd, repositoryPwd string) icmd.Cmd {
+func withTrustAndPassphrase(rootPwd, repositoryPwd string) func(cmd *icmd.Cmd) {
-	env := append(os.Environ(), []string{
+	return func(cmd *icmd.Cmd) {
+		env := append(os.Environ(),
 			"DOCKER_CONTENT_TRUST=1",
 			"DOCKER_CONTENT_TRUST_SERVER="+notaryURL,
 			"DOCKER_CONTENT_TRUST_ROOT_PASSPHRASE="+rootPwd,
 			"DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE="+repositoryPwd,
-	}...)
+		)
 		cmd.Env = append(cmd.Env, env...)
-	return cmd
+	}
 }
 
-func trustedCmdNoPassphrases(cmd icmd.Cmd) icmd.Cmd {
+func withTrustNoPassphrase(cmd *icmd.Cmd) {
-	env := append(os.Environ(), []string{
+	env := append(os.Environ(),
 		"DOCKER_CONTENT_TRUST=1",
 		"DOCKER_CONTENT_TRUST_SERVER="+notaryURL,
-	}...)
+	)
 	cmd.Env = append(cmd.Env, env...)
-	return cmd
 }

e2e/image/testdata/pull-with-content-trust.golden

@@ -0,0 +1,5 @@
+Pull (1 of 1): registry:5000/trust:latest@sha256:641b95ddb2ea9dc2af1a0113b6b348ebc20872ba615204fbe12148e98fd6f23d
+sha256:641b95ddb2ea9dc2af1a0113b6b348ebc20872ba615204fbe12148e98fd6f23d: Pulling from trust
+Digest: sha256:641b95ddb2ea9dc2af1a0113b6b348ebc20872ba615204fbe12148e98fd6f23d
+Status: Downloaded newer image for registry:5000/trust@sha256:641b95ddb2ea9dc2af1a0113b6b348ebc20872ba615204fbe12148e98fd6f23d
+Tagging registry:5000/trust@sha256:641b95ddb2ea9dc2af1a0113b6b348ebc20872ba615204fbe12148e98fd6f23d as registry:5000/trust:latest

scripts/test/e2e/run

@@ -80,6 +80,9 @@ case "$cmd" in
         cleanup "$unique_id" "$compose_env_file"
         exit $testexit
         ;;
+    shell)
+        $SHELL
+        ;;
     *)
         echo "Unknown command: $cmd"
         echo "Usage: "

scripts/test/e2e/wrapper

@@ -32,13 +32,21 @@ docker run --rm \
 
 engine_host=$(run_in_env setup)
 testexit=0
+
+
+test_cmd="test"
+if [[ -n "${TEST_DEBUG-}" ]]; then
+    test_cmd="shell"
+fi
+
 docker run -i --rm \
     -v "$PWD:/go/src/github.com/docker/cli" \
     -v "$PWD/e2e/testdata/notary/root-ca.cert:/usr/local/share/ca-certificates/notary.cert" \
     --network "${unique_id}_default" \
     -e TESTFLAGS \
+    -e ENGINE_HOST="$engine_host" \
     "$dev_image" \
-    ./scripts/test/e2e/run test "$engine_host" || testexit="$?"
+    ./scripts/test/e2e/run "$test_cmd" "$engine_host" || testexit="$?"
 
 run_in_env cleanup
 exit "$testexit"