Merge pull request #445 from dnephin/add-end-to-end-suite

Add end to end suite
This commit is contained in:
Tibor Vass 2017-08-23 16:16:09 -07:00 committed by GitHub
commit 6c3d93bbb6
17 changed files with 634 additions and 12 deletions

12
Jenkinsfile vendored Normal file
View File

@ -0,0 +1,12 @@
wrappedNode(label: 'linux && x86_64', cleanWorkspace: true) {
timeout(time: 60, unit: 'MINUTES') {
stage "Git Checkout"
checkout scm
stage "Run end-to-end test suite"
sh "docker version"
sh "E2E_UNIQUE_ID=clie2e${BUILD_NUMBER} \
IMAGE_TAG=clie2e${BUILD_NUMBER} \
make -f docker.Makefile test-e2e"
}
}

View File

@ -10,13 +10,13 @@ _:=$(shell ./scripts/warn-outside-container $(MAKECMDGOALS))
clean: ## remove build artifacts
rm -rf ./build/* cli/winresources/rsrc_* ./man/man[1-9] docs/yaml/gen
.PHONY: test
test: ## run go test
./scripts/test/unit $(shell go list ./... | grep -v '/vendor/')
.PHONY: test-unit
test-unit: ## run unit test
./scripts/test/unit $(shell go list ./... | grep -vE '/vendor/|/e2e/')
.PHONY: test-coverage
test-coverage: ## run test coverage
./scripts/test/unit-with-coverage $(shell go list ./... | grep -v '/vendor/')
./scripts/test/unit-with-coverage $(shell go list ./... | grep -vE '/vendor/|/e2e/')
.PHONY: lint
lint: ## run all the lint tools

View File

@ -2,6 +2,7 @@ package stack
import (
"fmt"
"sort"
"strings"
"github.com/docker/cli/cli"
@ -88,12 +89,19 @@ func runRemove(dockerCli command.Cli, opts removeOptions) error {
return nil
}
func sortServiceByName(services []swarm.Service) func(i, j int) bool {
return func(i, j int) bool {
return services[i].Spec.Name < services[j].Spec.Name
}
}
func removeServices(
ctx context.Context,
dockerCli command.Cli,
services []swarm.Service,
) bool {
var hasError bool
sort.Slice(services, sortServiceByName(services))
for _, service := range services {
fmt.Fprintf(dockerCli.Err(), "Removing service %s\n", service.Spec.Name)
if err := dockerCli.Client().ServiceRemove(ctx, service.ID); err != nil {

View File

@ -4,10 +4,10 @@
# Makefile for developing using Docker
#
DEV_DOCKER_IMAGE_NAME = docker-cli-dev
LINTER_IMAGE_NAME = docker-cli-lint
CROSS_IMAGE_NAME = docker-cli-cross
VALIDATE_IMAGE_NAME = docker-cli-shell-validate
DEV_DOCKER_IMAGE_NAME = docker-cli-dev$(IMAGE_TAG)
LINTER_IMAGE_NAME = docker-cli-lint$(IMAGE_TAG)
CROSS_IMAGE_NAME = docker-cli-cross$(IMAGE_TAG)
VALIDATE_IMAGE_NAME = docker-cli-shell-validate$(IMAGE_TAG)
MOUNTS = -v "$(CURDIR)":/go/src/github.com/docker/cli
VERSION = $(shell cat VERSION)
ENVVARS = -e VERSION=$(VERSION) -e GITCOMMIT
@ -42,9 +42,9 @@ clean: build_docker_image
docker run --rm $(ENVVARS) $(MOUNTS) $(DEV_DOCKER_IMAGE_NAME) make clean
# run go test
.PHONY: test
test: build_docker_image
docker run --rm $(ENVVARS) $(MOUNTS) $(DEV_DOCKER_IMAGE_NAME) make test
.PHONY: test-unit
test-unit: build_docker_image
docker run --rm $(ENVVARS) $(MOUNTS) $(DEV_DOCKER_IMAGE_NAME) make test-unit
# build the CLI for multiple architectures using a container
.PHONY: cross
@ -90,3 +90,7 @@ yamldocs: build_docker_image
.PHONY: shellcheck
shellcheck: build_shell_validate_image
docker run -ti --rm $(ENVVARS) $(MOUNTS) $(VALIDATE_IMAGE_NAME) make shellcheck
.PHONY: test-e2e
test-e2e: binary
./scripts/test/e2e/wrapper

View File

@ -0,0 +1,17 @@
FROM docker/compose:1.15.0
RUN apk add -U bash curl
ARG DOCKER_CHANNEL=edge
ARG DOCKER_VERSION=17.06.0-ce
RUN export URL=https://download.docker.com/linux/static; \
curl -Ls $URL/$DOCKER_CHANNEL/x86_64/docker-$DOCKER_VERSION.tgz | \
tar -xz docker/docker && \
mv docker/docker /usr/local/bin/ && \
rmdir docker
ENV DISABLE_WARN_OUTSIDE_CONTAINER=1
WORKDIR /work
COPY scripts/test/e2e scripts/test/e2e
COPY e2e/compose-env.yaml e2e/compose-env.yaml
ENTRYPOINT ["bash", "/work/scripts/test/e2e/run"]

10
e2e/compose-env.yaml Normal file
View File

@ -0,0 +1,10 @@
version: '2.1'
services:
registry:
image: 'registry:2'
engine:
image: 'docker:${TEST_ENGINE_VERSION:-edge-dind}'
privileged: true
command: ['--insecure-registry=registry:5000']

26
e2e/stack/main_test.go Normal file
View File

@ -0,0 +1,26 @@
package stack
import (
"fmt"
"os"
"testing"
"github.com/pkg/errors"
)
func TestMain(m *testing.M) {
if err := setupTestEnv(); err != nil {
fmt.Println(err.Error())
os.Exit(3)
}
os.Exit(m.Run())
}
// TODO: move to shared internal package
func setupTestEnv() error {
dockerHost := os.Getenv("TEST_DOCKER_HOST")
if dockerHost == "" {
return errors.New("$TEST_DOCKER_HOST must be set")
}
return os.Setenv("DOCKER_HOST", dockerHost)
}

89
e2e/stack/remove_test.go Normal file
View File

@ -0,0 +1,89 @@
package stack
import (
"fmt"
"strings"
"testing"
"time"
shlex "github.com/flynn-archive/go-shlex"
"github.com/gotestyourself/gotestyourself/golden"
"github.com/gotestyourself/gotestyourself/icmd"
"github.com/stretchr/testify/require"
)
func TestRemove(t *testing.T) {
stackname := "test-stack-remove"
deployFullStack(t, stackname)
defer cleanupFullStack(t, stackname)
result := icmd.RunCmd(shell(t, "docker stack rm %s", stackname))
result.Assert(t, icmd.Expected{Out: icmd.None})
golden.Assert(t, result.Stderr(), "stack-remove-success.golden")
}
func deployFullStack(t *testing.T, stackname string) {
// TODO: this stack should have full options not minimal options
result := icmd.RunCmd(shell(t,
"docker stack deploy --compose-file=./testdata/full-stack.yml %s", stackname))
result.Assert(t, icmd.Success)
waitOn(t, taskCount(stackname, 2), 0)
}
func cleanupFullStack(t *testing.T, stackname string) {
result := icmd.RunCmd(shell(t, "docker stack rm %s", stackname))
result.Assert(t, icmd.Success)
waitOn(t, taskCount(stackname, 0), 0)
}
func taskCount(stackname string, expected int) func() (bool, error) {
return func() (bool, error) {
result := icmd.RunCommand(
"docker", "stack", "ps", "-f=desired-state=running", stackname)
count := lines(result.Stdout()) - 1
return count == expected, nil
}
}
func lines(out string) int {
return len(strings.Split(strings.TrimSpace(out), "\n"))
}
// TODO: move to gotestyourself
func shell(t *testing.T, format string, args ...interface{}) icmd.Cmd {
cmd, err := shlex.Split(fmt.Sprintf(format, args...))
require.NoError(t, err)
return icmd.Cmd{Command: cmd}
}
// TODO: move to gotestyourself
func waitOn(t *testing.T, check func() (bool, error), timeout time.Duration) {
if timeout == time.Duration(0) {
timeout = defaultTimeout()
}
after := time.After(timeout)
for {
select {
case <-after:
// TODO: include check function name in error message
t.Fatalf("timeout hit after %s", timeout)
default:
// TODO: maybe return a failure message as well?
done, err := check()
if done {
return
}
if err != nil {
t.Fatal(err.Error())
}
}
}
}
func defaultTimeout() time.Duration {
// TODO: support override from environment variable
return 10 * time.Second
}

9
e2e/stack/testdata/full-stack.yml vendored Normal file
View File

@ -0,0 +1,9 @@
version: '3.3'
services:
one:
image: registry:5000/alpine:3.6
command: top
two:
image: registry:5000/alpine:3.6
command: top

View File

@ -0,0 +1,3 @@
Removing service test-stack-remove_one
Removing service test-stack-remove_two
Removing network test-stack-remove_default

8
scripts/test/e2e/load-alpine Executable file
View File

@ -0,0 +1,8 @@
#!/usr/bin/env bash
set -eu -o pipefail
src=alpine:3.6
dest=registry:5000/alpine:3.6
docker pull $src
docker tag $src $dest
docker push $dest

85
scripts/test/e2e/run Executable file
View File

@ -0,0 +1,85 @@
#!/usr/bin/env bash
# Run integration tests against the latest docker-ce dind
set -eu -o pipefail
function container_ip {
local cid=$1
local network=$2
docker inspect \
-f "{{.NetworkSettings.Networks.${network}.IPAddress}}" "$cid"
}
function setup {
local project=$1
COMPOSE_PROJECT_NAME=$1 COMPOSE_FILE=$2 docker-compose up -d >&2
local network="${project}_default"
# TODO: only run if inside a container
docker network connect "$network" "$(hostname)"
engine_ip="$(container_ip "${project}_engine_1" "$network")"
engine_host="tcp://$engine_ip:2375"
(
export DOCKER_HOST="$engine_host"
timeout -t 200 ./scripts/test/e2e/wait-on-daemon
./scripts/test/e2e/load-alpine
is_swarm_enabled || docker swarm init
) >&2
echo "$engine_host"
}
function is_swarm_enabled {
docker info 2> /dev/null | grep -q 'Swarm: active'
}
function cleanup {
COMPOSE_PROJECT_NAME=$1 COMPOSE_FILE=$2 docker-compose down >&2
}
function runtests {
local engine_host=$1
env -i \
TEST_DOCKER_HOST="$engine_host" \
GOPATH="$GOPATH" \
PATH="$PWD/build/" \
"$(which go)" test -v ./e2e/...
}
export unique_id="${E2E_UNIQUE_ID:-cliendtoendsuite}"
compose_env_file=./e2e/compose-env.yaml
cmd=${1-}
case "$cmd" in
setup)
setup "$unique_id" "$compose_env_file"
exit
;;
cleanup)
cleanup "$unique_id" "$compose_env_file"
exit
;;
test)
engine_host=${2-}
if [[ -z "${engine_host}" ]]; then
echo "missing parameter docker engine host"
echo "Usage: $0 test ENGINE_HOST"
exit 3
fi
runtests "$engine_host"
;;
run|"")
engine_host="$(setup "$unique_id" "$compose_env_file")"
testexit=0
runtests "$engine_host" || testexit=$?
cleanup "$unique_id" "$compose_env_file"
exit $testexit
;;
*)
echo "Unknown command: $cmd"
echo "Usage: "
echo " $0 [setup | cleanup | test | run] [engine_host]"
exit 1
;;
esac

View File

@ -0,0 +1,9 @@
#!/usr/bin/env bash
set -eu -o pipefail
echo "Waiting for docker daemon to become available at $DOCKER_HOST"
while ! docker version > /dev/null; do
sleep 0.3
done
docker version

33
scripts/test/e2e/wrapper Executable file
View File

@ -0,0 +1,33 @@
#!/usr/bin/env bash
# Setup, run and teardown e2e test suite in containers.
set -eu -o pipefail
unique_id="${E2E_UNIQUE_ID:-cliendtoendsuite}"
e2e_env_image=docker-cli-e2e-env:$unique_id
dev_image=docker-cli-dev:$unique_id
function run_in_env {
local cmd=$1
docker run -i --rm \
-v /var/run/docker.sock:/var/run/docker.sock \
-e E2E_UNIQUE_ID \
"$e2e_env_image" "$cmd"
}
docker build \
-t "$e2e_env_image" \
-f dockerfiles/Dockerfile.test-e2e-env .
docker build \
-t "$dev_image" \
-f dockerfiles/Dockerfile.dev .
engine_host=$(run_in_env setup)
testexit=0
docker run -i --rm \
-v "$PWD:/go/src/github.com/docker/cli" \
--network "${unique_id}_default" \
"$dev_image" \
./scripts/test/e2e/run test "$engine_host" || testexit="$?"
run_in_env cleanup
exit "$testexit"

View File

@ -1,3 +1,3 @@
#!/bin/sh
# shellcheck disable=SC2016
exec filewatcher -L 6 -x build -x script go test -timeout 10s -v './${dir}'
exec filewatcher -L 6 -x build -x script go test -timeout 30s -v './${dir}'

View File

@ -0,0 +1,277 @@
/*Package icmd executes binaries and provides convenient assertions for testing the results.
*/
package icmd
import (
"bytes"
"fmt"
"io"
"os/exec"
"path/filepath"
"runtime"
"strings"
"sync"
"time"
)
type testingT interface {
Fatalf(string, ...interface{})
}
const (
// None is a token to inform Result.Assert that the output should be empty
None string = "<NOTHING>"
)
type lockedBuffer struct {
m sync.RWMutex
buf bytes.Buffer
}
func (buf *lockedBuffer) Write(b []byte) (int, error) {
buf.m.Lock()
defer buf.m.Unlock()
return buf.buf.Write(b)
}
func (buf *lockedBuffer) String() string {
buf.m.RLock()
defer buf.m.RUnlock()
return buf.buf.String()
}
// Result stores the result of running a command
type Result struct {
Cmd *exec.Cmd
ExitCode int
Error error
// Timeout is true if the command was killed because it ran for too long
Timeout bool
outBuffer *lockedBuffer
errBuffer *lockedBuffer
}
// Assert compares the Result against the Expected struct, and fails the test if
// any of the expectations are not met.
func (r *Result) Assert(t testingT, exp Expected) *Result {
err := r.Compare(exp)
if err == nil {
return r
}
_, file, line, ok := runtime.Caller(1)
if ok {
t.Fatalf("at %s:%d - %s\n", filepath.Base(file), line, err.Error())
} else {
t.Fatalf("(no file/line info) - %s", err.Error())
}
return nil
}
// Compare returns a formatted error with the command, stdout, stderr, exit
// code, and any failed expectations
// nolint: gocyclo
func (r *Result) Compare(exp Expected) error {
errors := []string{}
add := func(format string, args ...interface{}) {
errors = append(errors, fmt.Sprintf(format, args...))
}
if exp.ExitCode != r.ExitCode {
add("ExitCode was %d expected %d", r.ExitCode, exp.ExitCode)
}
if exp.Timeout != r.Timeout {
if exp.Timeout {
add("Expected command to timeout")
} else {
add("Expected command to finish, but it hit the timeout")
}
}
if !matchOutput(exp.Out, r.Stdout()) {
add("Expected stdout to contain %q", exp.Out)
}
if !matchOutput(exp.Err, r.Stderr()) {
add("Expected stderr to contain %q", exp.Err)
}
switch {
// If a non-zero exit code is expected there is going to be an error.
// Don't require an error message as well as an exit code because the
// error message is going to be "exit status <code> which is not useful
case exp.Error == "" && exp.ExitCode != 0:
case exp.Error == "" && r.Error != nil:
add("Expected no error")
case exp.Error != "" && r.Error == nil:
add("Expected error to contain %q, but there was no error", exp.Error)
case exp.Error != "" && !strings.Contains(r.Error.Error(), exp.Error):
add("Expected error to contain %q", exp.Error)
}
if len(errors) == 0 {
return nil
}
return fmt.Errorf("%s\nFailures:\n%s", r, strings.Join(errors, "\n"))
}
func matchOutput(expected string, actual string) bool {
switch expected {
case None:
return actual == ""
default:
return strings.Contains(actual, expected)
}
}
func (r *Result) String() string {
var timeout string
if r.Timeout {
timeout = " (timeout)"
}
return fmt.Sprintf(`
Command: %s
ExitCode: %d%s
Error: %v
Stdout: %v
Stderr: %v
`,
strings.Join(r.Cmd.Args, " "),
r.ExitCode,
timeout,
r.Error,
r.Stdout(),
r.Stderr())
}
// Expected is the expected output from a Command. This struct is compared to a
// Result struct by Result.Assert().
type Expected struct {
ExitCode int
Timeout bool
Error string
Out string
Err string
}
// Success is the default expected result. A Success result is one with a 0
// ExitCode.
var Success = Expected{}
// Stdout returns the stdout of the process as a string
func (r *Result) Stdout() string {
return r.outBuffer.String()
}
// Stderr returns the stderr of the process as a string
func (r *Result) Stderr() string {
return r.errBuffer.String()
}
// Combined returns the stdout and stderr combined into a single string
func (r *Result) Combined() string {
return r.outBuffer.String() + r.errBuffer.String()
}
// SetExitError sets Error and ExitCode based on Error
func (r *Result) SetExitError(err error) {
if err == nil {
return
}
r.Error = err
r.ExitCode = processExitCode(err)
}
// Cmd contains the arguments and options for a process to run as part of a test
// suite.
type Cmd struct {
Command []string
Timeout time.Duration
Stdin io.Reader
Stdout io.Writer
Dir string
Env []string
}
// Command create a simple Cmd with the specified command and arguments
func Command(command string, args ...string) Cmd {
return Cmd{Command: append([]string{command}, args...)}
}
// RunCmd runs a command and returns a Result
func RunCmd(cmd Cmd, cmdOperators ...func(*Cmd)) *Result {
for _, op := range cmdOperators {
op(&cmd)
}
result := StartCmd(cmd)
if result.Error != nil {
return result
}
return WaitOnCmd(cmd.Timeout, result)
}
// RunCommand parses a command line and runs it, returning a result
func RunCommand(command string, args ...string) *Result {
return RunCmd(Command(command, args...))
}
// StartCmd starts a command, but doesn't wait for it to finish
func StartCmd(cmd Cmd) *Result {
result := buildCmd(cmd)
if result.Error != nil {
return result
}
result.SetExitError(result.Cmd.Start())
return result
}
func buildCmd(cmd Cmd) *Result {
var execCmd *exec.Cmd
switch len(cmd.Command) {
case 1:
execCmd = exec.Command(cmd.Command[0])
default:
execCmd = exec.Command(cmd.Command[0], cmd.Command[1:]...)
}
outBuffer := new(lockedBuffer)
errBuffer := new(lockedBuffer)
execCmd.Stdin = cmd.Stdin
execCmd.Dir = cmd.Dir
execCmd.Env = cmd.Env
if cmd.Stdout != nil {
execCmd.Stdout = io.MultiWriter(outBuffer, cmd.Stdout)
} else {
execCmd.Stdout = outBuffer
}
execCmd.Stderr = errBuffer
return &Result{
Cmd: execCmd,
outBuffer: outBuffer,
errBuffer: errBuffer,
}
}
// WaitOnCmd waits for a command to complete. If timeout is non-nil then
// only wait until the timeout.
func WaitOnCmd(timeout time.Duration, result *Result) *Result {
if timeout == time.Duration(0) {
result.SetExitError(result.Cmd.Wait())
return result
}
done := make(chan error, 1)
// Wait for command to exit in a goroutine
go func() {
done <- result.Cmd.Wait()
}()
select {
case <-time.After(timeout):
killErr := result.Cmd.Process.Kill()
if killErr != nil {
fmt.Printf("failed to kill (pid=%d): %v\n", result.Cmd.Process.Pid, killErr)
}
result.Timeout = true
case err := <-done:
result.SetExitError(err)
}
return result
}

View File

@ -0,0 +1,32 @@
package icmd
import (
"os/exec"
"syscall"
"github.com/pkg/errors"
)
// GetExitCode returns the ExitStatus of a process from the error returned by
// exec.Run(). If the exit status could not be parsed an error is returned.
func GetExitCode(err error) (int, error) {
if exiterr, ok := err.(*exec.ExitError); ok {
if procExit, ok := exiterr.Sys().(syscall.WaitStatus); ok {
return procExit.ExitStatus(), nil
}
}
return 0, errors.Wrap(err, "failed to get exit code")
}
func processExitCode(err error) (exitCode int) {
if err == nil {
return 0
}
exitCode, exiterr := GetExitCode(err)
if exiterr != nil {
// TODO: Fix this so we check the error's text.
// we've failed to retrieve exit code, so we set it to 127
return 127
}
return exitCode
}