From 592c146ccabade747a61b64bc22c58a0cd328f1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Mon, 12 Feb 2024 16:27:25 +0100 Subject: [PATCH 1/3] testenv: Add DaemonAPIVersion helper MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Allow tests to check the negotiated API version used by the client. Can be used to skip tests based on API versions, for example: ```go skip.If(t, versions.LessThan(environment.DaemonAPIVersion(t), "1.44")) ``` will skip the test if the API version is older than 1.44 Signed-off-by: Paweł Gronowski (cherry picked from commit 9831fea4db5d08c7e296ca356e6093e6de28dd48) Signed-off-by: Paweł Gronowski --- internal/test/environment/testenv.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/internal/test/environment/testenv.go b/internal/test/environment/testenv.go index 8b035fca1b..7fe6880244 100644 --- a/internal/test/environment/testenv.go +++ b/internal/test/environment/testenv.go @@ -108,3 +108,14 @@ func SkipIfNotPlatform(t *testing.T, platform string) { daemonPlatform := strings.TrimSpace(result.Stdout()) skip.If(t, daemonPlatform != platform, "running against a non %s daemon", platform) } + +// DaemonAPIVersion returns the negotiated daemon API version. +func DaemonAPIVersion(t *testing.T) string { + t.Helper() + // Use Client.APIVersion instead of Server.APIVersion. + // The latter is the maximum version that the server supports + // while the Client.APIVersion contains the negotiated version. + result := icmd.RunCmd(icmd.Command("docker", "version", "--format", "{{.Client.APIVersion}}")) + result.Assert(t, icmd.Expected{Err: icmd.None}) + return strings.TrimSpace(result.Stdout()) +} From 5a942fadcf63cd18cd15d41e7fb34fa795fcda35 Mon Sep 17 00:00:00 2001 From: Christopher Petito Date: Tue, 13 Feb 2024 13:30:09 +0000 Subject: [PATCH 2/3] Update gha runners and engines used in e2e tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - gha runners updated to ubuntu 22.04 - e2e now runs against moby 23.0, 24.0 and 25.0 - temporarily skip broken test for moby < 25 Signed-off-by: Christopher Petito (cherry picked from commit 6b67b95493ea48ae0f8b8209e164d4591bb92e1f) Signed-off-by: Paweł Gronowski --- .github/workflows/build.yml | 10 +++++----- .github/workflows/e2e.yml | 12 ++++++------ .github/workflows/test.yml | 2 +- .github/workflows/validate.yml | 6 +++--- e2e/image/testdata/pull-with-content-trust.golden | 2 +- e2e/plugin/trust_test.go | 5 ++++- 6 files changed, 20 insertions(+), 17 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8487e4d3c3..b535428086 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -19,7 +19,7 @@ on: jobs: prepare: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 outputs: matrix: ${{ steps.platforms.outputs.matrix }} steps: @@ -37,7 +37,7 @@ jobs: echo ${{ steps.platforms.outputs.matrix }} build: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 needs: - prepare strategy: @@ -90,7 +90,7 @@ jobs: if-no-files-found: error bin-image: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 if: ${{ github.event_name != 'pull_request' && github.repository == 'docker/cli' }} steps: - @@ -134,7 +134,7 @@ jobs: *.cache-to=type=gha,scope=bin-image,mode=max prepare-plugins: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 outputs: matrix: ${{ steps.platforms.outputs.matrix }} steps: @@ -152,7 +152,7 @@ jobs: echo ${{ steps.platforms.outputs.matrix }} plugins: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 needs: - prepare-plugins strategy: diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 67ffdff09d..227d105f8e 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -16,7 +16,7 @@ on: jobs: e2e: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 strategy: fail-fast: false matrix: @@ -28,11 +28,11 @@ jobs: - alpine - debian engine-version: -# - 20.10-dind # FIXME: Fails on 20.10 - - stable-dind # TODO: Use 20.10-dind, stable-dind is deprecated - include: - - target: non-experimental - engine-version: 19.03-dind + - 25.0 # latest + - 24.0 # latest - 1 + - 23.0 # mirantis lts + # TODO(krissetto) 19.03 needs a look, doesn't work ubuntu 22.04 (cgroup errors). + # we could have a separate job that tests it against ubuntu 20.04 steps: - name: Checkout diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 447c570914..db37d7db7a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -16,7 +16,7 @@ on: jobs: ctn: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 steps: - name: Checkout diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index ccd0493a32..41349713c8 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -16,7 +16,7 @@ on: jobs: validate: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 strategy: fail-fast: false matrix: @@ -37,7 +37,7 @@ jobs: # check that the generated Markdown and the checked-in files match validate-md: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 steps: - name: Checkout @@ -57,7 +57,7 @@ jobs: fi validate-make: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 strategy: fail-fast: false matrix: diff --git a/e2e/image/testdata/pull-with-content-trust.golden b/e2e/image/testdata/pull-with-content-trust.golden index b21407e117..582a4e0cc9 100644 --- a/e2e/image/testdata/pull-with-content-trust.golden +++ b/e2e/image/testdata/pull-with-content-trust.golden @@ -1,5 +1,5 @@ Pull (1 of 1): registry:5000/trust-pull:latest@sha256:e2e16842c9b54d985bf1ef9242a313f36b856181f188de21313820e177002501 -sha256:e2e16842c9b54d985bf1ef9242a313f36b856181f188de21313820e177002501: Pulling from trust-pull +registry:5000/trust-pull@sha256:e2e16842c9b54d985bf1ef9242a313f36b856181f188de21313820e177002501: Pulling from trust-pull Digest: sha256:e2e16842c9b54d985bf1ef9242a313f36b856181f188de21313820e177002501 Status: Downloaded newer image for registry:5000/trust-pull@sha256:e2e16842c9b54d985bf1ef9242a313f36b856181f188de21313820e177002501 registry:5000/trust-pull:latest diff --git a/e2e/plugin/trust_test.go b/e2e/plugin/trust_test.go index 38bdcc5e45..c6957d6f37 100644 --- a/e2e/plugin/trust_test.go +++ b/e2e/plugin/trust_test.go @@ -11,6 +11,7 @@ import ( "github.com/docker/cli/e2e/internal/fixtures" "github.com/docker/cli/internal/test/environment" "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/versions" "github.com/pkg/errors" "gotest.tools/v3/assert" "gotest.tools/v3/fs" @@ -21,6 +22,8 @@ import ( const registryPrefix = "registry:5000" func TestInstallWithContentTrust(t *testing.T) { + // TODO(krissetto): remove this skip once the fix (see https://github.com/moby/moby/pull/47299) is deployed to moby versions < 25 + skip.If(t, versions.LessThan(environment.DaemonAPIVersion(t), "1.44")) skip.If(t, environment.SkipPluginTests()) pluginName := fmt.Sprintf("%s/plugin-content-trust", registryPrefix) @@ -50,7 +53,7 @@ func TestInstallWithContentTrust(t *testing.T) { fixtures.WithNotary, ) result.Assert(t, icmd.Expected{ - Out: fmt.Sprintf("Status: Downloaded newer image for %s@sha", pluginName), + Out: fmt.Sprintf("Installed plugin %s", pluginName), }) } From 956d15c72338162b0b5177c8cf8b5ed93d5e4bd6 Mon Sep 17 00:00:00 2001 From: Christopher Petito Date: Tue, 13 Feb 2024 13:32:45 +0000 Subject: [PATCH 3/3] Cleanup of dockerfiles, compose files and env vars MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christopher Petito (cherry picked from commit 69ed6588a8dcd92921022ba0f42e562cd41bbb4e) Signed-off-by: Paweł Gronowski --- .github/workflows/e2e.yml | 2 +- docker.Makefile | 10 +++++----- e2e/compose-env.connhelper-ssh.yaml | 2 ++ e2e/compose-env.yaml | 4 +++- e2e/testdata/Dockerfile.connhelper-ssh | 15 +++++++++++++-- e2e/testdata/Dockerfile.evil-notary-server | 3 +++ e2e/testdata/Dockerfile.notary-server | 3 +++ 7 files changed, 30 insertions(+), 9 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 227d105f8e..9ce1a6d92a 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -55,7 +55,7 @@ jobs: make -f docker.Makefile test-e2e-${{ matrix.target }} env: BASE_VARIANT: ${{ matrix.base }} - E2E_ENGINE_VERSION: ${{ matrix.engine-version }} + ENGINE_VERSION: ${{ matrix.engine-version }} TESTFLAGS: -coverprofile=/tmp/coverage/coverage.txt - name: Send to Codecov diff --git a/docker.Makefile b/docker.Makefile index bddd91a2a5..011bfd9b68 100644 --- a/docker.Makefile +++ b/docker.Makefile @@ -14,13 +14,13 @@ PACKAGER_NAME ?= DEV_DOCKER_IMAGE_NAME = docker-cli-dev$(IMAGE_TAG) E2E_IMAGE_NAME = docker-cli-e2e -E2E_ENGINE_VERSION ?= +ENGINE_VERSION ?= CACHE_VOLUME_NAME := docker-cli-dev-cache ifeq ($(DOCKER_CLI_GO_BUILD_CACHE),y) DOCKER_CLI_MOUNTS += -v "$(CACHE_VOLUME_NAME):/root/.cache/go-build" endif VERSION = $(shell cat VERSION) -ENVVARS = -e VERSION=$(VERSION) -e GITCOMMIT -e PLATFORM -e TESTFLAGS -e TESTDIRS -e GOOS -e GOARCH -e GOARM -e TEST_ENGINE_VERSION=$(E2E_ENGINE_VERSION) +ENVVARS = -e VERSION=$(VERSION) -e GITCOMMIT -e PLATFORM -e TESTFLAGS -e TESTDIRS -e GOOS -e GOARCH -e GOARM -e ENGINE_VERSION # Some Dockerfiles use features that are only supported with BuildKit enabled export DOCKER_BUILDKIT=1 @@ -132,21 +132,21 @@ test-e2e: test-e2e-non-experimental test-e2e-experimental test-e2e-connhelper-ss .PHONY: test-e2e-experimental test-e2e-experimental: build-e2e-image # run experimental e2e tests - docker run --rm $(ENVVARS) -e DOCKERD_EXPERIMENTAL=1 -e TEST_ENGINE_VERSION=$(E2E_ENGINE_VERSION) \ + docker run --rm $(ENVVARS) -e DOCKERD_EXPERIMENTAL=1 \ --mount type=bind,src=$(CURDIR)/build/coverage,dst=/tmp/coverage \ --mount type=bind,src=/var/run/docker.sock,dst=/var/run/docker.sock \ $(E2E_IMAGE_NAME) .PHONY: test-e2e-non-experimental test-e2e-non-experimental: build-e2e-image # run non-experimental e2e tests - docker run --rm $(ENVVARS) -e TEST_ENGINE_VERSION=$(E2E_ENGINE_VERSION) \ + docker run --rm $(ENVVARS) \ --mount type=bind,src=$(CURDIR)/build/coverage,dst=/tmp/coverage \ --mount type=bind,src=/var/run/docker.sock,dst=/var/run/docker.sock \ $(E2E_IMAGE_NAME) .PHONY: test-e2e-connhelper-ssh test-e2e-connhelper-ssh: build-e2e-image # run experimental SSH-connection helper e2e tests - docker run --rm $(ENVVARS) -e DOCKERD_EXPERIMENTAL=1 -e TEST_ENGINE_VERSION=$(E2E_ENGINE_VERSION) -e TEST_CONNHELPER=ssh \ + docker run --rm $(ENVVARS) -e DOCKERD_EXPERIMENTAL=1 -e TEST_CONNHELPER=ssh \ --mount type=bind,src=$(CURDIR)/build/coverage,dst=/tmp/coverage \ --mount type=bind,src=/var/run/docker.sock,dst=/var/run/docker.sock \ $(E2E_IMAGE_NAME) diff --git a/e2e/compose-env.connhelper-ssh.yaml b/e2e/compose-env.connhelper-ssh.yaml index 2a91ab48fe..283e306fbc 100644 --- a/e2e/compose-env.connhelper-ssh.yaml +++ b/e2e/compose-env.connhelper-ssh.yaml @@ -3,5 +3,7 @@ services: build: context: ./testdata dockerfile: Dockerfile.connhelper-ssh + args: + - ENGINE_VERSION environment: - TEST_CONNHELPER_SSH_ID_RSA_PUB diff --git a/e2e/compose-env.yaml b/e2e/compose-env.yaml index 9eade791b0..2a30e1040e 100644 --- a/e2e/compose-env.yaml +++ b/e2e/compose-env.yaml @@ -1,9 +1,10 @@ services: + registry: image: 'registry:2' engine: - image: 'docker:${TEST_ENGINE_VERSION:-stable-dind}' + image: 'docker:${ENGINE_VERSION:-25.0}-dind' privileged: true command: ['--insecure-registry=registry:5000'] environment: @@ -16,6 +17,7 @@ services: ports: - 4443:4443 command: ['notary-server', '-config=/fixtures/notary-config.json'] + evil-notary-server: build: context: ./testdata diff --git a/e2e/testdata/Dockerfile.connhelper-ssh b/e2e/testdata/Dockerfile.connhelper-ssh index 64306b518c..3771f3e62f 100644 --- a/e2e/testdata/Dockerfile.connhelper-ssh +++ b/e2e/testdata/Dockerfile.connhelper-ssh @@ -1,5 +1,16 @@ -FROM docker:test-dind -RUN apk --no-cache add shadow openssh-server && \ +# syntax=docker/dockerfile:1 + +# ENGINE_VERSION is the version of the (docker-in-docker) Docker Engine to +# test against. +ARG ENGINE_VERSION=25.0 + +FROM docker:${ENGINE_VERSION}-dind + +# the openssh-client update is needed for security reasons when using docker:23.0-dind, currently maintained as an lts by mirantis +RUN apk --no-cache upgrade openssh-client && \ + apk --no-cache add shadow openssh-server && \ + # TODO(krissetto): `groupadd` can be removed once we only test against moby >= v24 + # see https://github.com/docker-library/docker/pull/470 groupadd -f docker && \ useradd --create-home --shell /bin/sh --password $(head -c32 /dev/urandom | base64) penguin && \ usermod -aG docker penguin && \ diff --git a/e2e/testdata/Dockerfile.evil-notary-server b/e2e/testdata/Dockerfile.evil-notary-server index d982d5a4eb..97f234e849 100644 --- a/e2e/testdata/Dockerfile.evil-notary-server +++ b/e2e/testdata/Dockerfile.evil-notary-server @@ -1,4 +1,7 @@ +# syntax=docker/dockerfile:1 + ARG NOTARY_VERSION=0.6.1 + FROM notary:server-${NOTARY_VERSION} COPY ./notary-evil/ /fixtures/ diff --git a/e2e/testdata/Dockerfile.notary-server b/e2e/testdata/Dockerfile.notary-server index 4bc59d2d88..846253e2fb 100644 --- a/e2e/testdata/Dockerfile.notary-server +++ b/e2e/testdata/Dockerfile.notary-server @@ -1,4 +1,7 @@ +# syntax=docker/dockerfile:1 + ARG NOTARY_VERSION=0.6.1 + FROM notary:server-${NOTARY_VERSION} COPY ./notary/ /fixtures/