Add some simple e2e tests for executing CLI plugins

To help with this add a bad plugin which produces invalid metadata and arrange
for it to be built in the e2e container.

Signed-off-by: Ian Campbell <ijc@docker.com>

dockerfiles/Dockerfile.e2e

@@ -38,6 +38,6 @@ ARG VERSION
 ARG GITCOMMIT
 ENV VERSION=${VERSION} GITCOMMIT=${GITCOMMIT}
 RUN ./scripts/build/binary
-RUN ./scripts/build/plugins
+RUN ./scripts/build/plugins e2e/cli-plugins/plugins/*
 
 CMD ./scripts/test/e2e/entry

e2e/cli-plugins/plugins/badmeta/main.go

@@ -0,0 +1,19 @@
+package main
+
+// This is not a real plugin, but just returns malformated metadata
+// from the subcommand and otherwise exits with failure.
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/docker/cli/cli-plugins/manager"
+)
+
+func main() {
+	if len(os.Args) == 2 && os.Args[1] == manager.MetadataSubcommandName {
+		fmt.Println(`{invalid-json}`)
+		os.Exit(0)
+	}
+	os.Exit(1)
+}

e2e/cli-plugins/run_test.go

@@ -0,0 +1,60 @@
+package cliplugins
+
+import (
+	"testing"
+
+	"gotest.tools/assert"
+	is "gotest.tools/assert/cmp"
+	"gotest.tools/golden"
+	"gotest.tools/icmd"
+)
+
+// TestRunNonexisting ensures correct behaviour when running a nonexistent plugin.
+func TestRunNonexisting(t *testing.T) {
+	run, cleanup := prepare(t)
+	defer cleanup()
+
+	res := icmd.RunCmd(run("nonexistent"))
+	res.Assert(t, icmd.Expected{
+		ExitCode: 1,
+	})
+	assert.Assert(t, is.Equal(res.Stdout(), ""))
+	golden.Assert(t, res.Stderr(), "docker-nonexistent-err.golden")
+}
+
+// TestRunBad ensures correct behaviour when running an existent but invalid plugin
+func TestRunBad(t *testing.T) {
+	run, cleanup := prepare(t)
+	defer cleanup()
+
+	res := icmd.RunCmd(run("badmeta"))
+	res.Assert(t, icmd.Expected{
+		ExitCode: 1,
+	})
+	assert.Assert(t, is.Equal(res.Stdout(), ""))
+	golden.Assert(t, res.Stderr(), "docker-badmeta-err.golden")
+}
+
+// TestRunGood ensures correct behaviour when running a valid plugin
+func TestRunGood(t *testing.T) {
+	run, cleanup := prepare(t)
+	defer cleanup()
+
+	res := icmd.RunCmd(run("helloworld"))
+	res.Assert(t, icmd.Expected{
+		ExitCode: 0,
+		Out:      "Hello World!",
+	})
+}
+
+// TestRunGoodSubcommand ensures correct behaviour when running a valid plugin with a subcommand
+func TestRunGoodSubcommand(t *testing.T) {
+	run, cleanup := prepare(t)
+	defer cleanup()
+
+	res := icmd.RunCmd(run("helloworld", "goodbye"))
+	res.Assert(t, icmd.Expected{
+		ExitCode: 0,
+		Out:      "Goodbye World!",
+	})
+}

e2e/cli-plugins/testdata/docker-badmeta-err.golden

@@ -0,0 +1,2 @@
+docker: 'badmeta' is not a docker command.
+See 'docker --help'

e2e/cli-plugins/testdata/docker-nonexistent-err.golden

@@ -0,0 +1,2 @@
+docker: 'nonexistent' is not a docker command.
+See 'docker --help'

e2e/cli-plugins/util_test.go

@@ -0,0 +1,24 @@
+package cliplugins
+
+import (
+	"fmt"
+	"os"
+	"testing"
+
+	"gotest.tools/fs"
+	"gotest.tools/icmd"
+)
+
+func prepare(t *testing.T) (func(args ...string) icmd.Cmd, func()) {
+	cfg := fs.NewDir(t, "plugin-test",
+		fs.WithFile("config.json", fmt.Sprintf(`{"cliPluginsExtraDirs": [%q]}`, os.Getenv("DOCKER_CLI_E2E_PLUGINS_EXTRA_DIRS"))),
+	)
+	run := func(args ...string) icmd.Cmd {
+		return icmd.Command("docker", append([]string{"--config", cfg.Path()}, args...)...)
+	}
+	cleanup := func() {
+		cfg.Remove()
+	}
+	return run, cleanup
+
+}

scripts/build/plugins

@@ -8,7 +8,7 @@ set -eu -o pipefail
 source ./scripts/build/.variables
 
 mkdir -p "build/plugins-${GOOS}-${GOARCH}"
-for p in cli-plugins/examples/* ; do
+for p in cli-plugins/examples/* "$@" ; do
     [ -d "$p" ] || continue
 
     n=$(basename "$p")

scripts/test/e2e/run

@@ -69,6 +69,7 @@ function runtests {
         TEST_SKIP_PLUGIN_TESTS="${SKIP_PLUGIN_TESTS-}" \
         GOPATH="$GOPATH" \
         PATH="$PWD/build/:/usr/bin" \
+        DOCKER_CLI_E2E_PLUGINS_EXTRA_DIRS="$PWD/build/plugins-linux-amd64" \
         "$(which go)" test -v ./e2e/... ${TESTFLAGS-}
 }