From 6b67b95493ea48ae0f8b8209e164d4591bb92e1f Mon Sep 17 00:00:00 2001 From: Christopher Petito Date: Tue, 13 Feb 2024 13:30:09 +0000 Subject: [PATCH 1/2] Update gha runners and engines used in e2e tests - gha runners updated to ubuntu 22.04 - e2e now runs against moby 23.0, 24.0 and 25.0 - temporarily skip broken test for moby < 25 Signed-off-by: Christopher Petito --- .github/workflows/build.yml | 10 +++++----- .github/workflows/e2e.yml | 12 ++++++------ .github/workflows/test.yml | 2 +- .github/workflows/validate.yml | 6 +++--- e2e/image/testdata/pull-with-content-trust.golden | 2 +- e2e/plugin/trust_test.go | 5 ++++- 6 files changed, 20 insertions(+), 17 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8487e4d3c3..b535428086 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -19,7 +19,7 @@ on: jobs: prepare: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 outputs: matrix: ${{ steps.platforms.outputs.matrix }} steps: @@ -37,7 +37,7 @@ jobs: echo ${{ steps.platforms.outputs.matrix }} build: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 needs: - prepare strategy: @@ -90,7 +90,7 @@ jobs: if-no-files-found: error bin-image: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 if: ${{ github.event_name != 'pull_request' && github.repository == 'docker/cli' }} steps: - @@ -134,7 +134,7 @@ jobs: *.cache-to=type=gha,scope=bin-image,mode=max prepare-plugins: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 outputs: matrix: ${{ steps.platforms.outputs.matrix }} steps: @@ -152,7 +152,7 @@ jobs: echo ${{ steps.platforms.outputs.matrix }} plugins: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 needs: - prepare-plugins strategy: diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 67ffdff09d..227d105f8e 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -16,7 +16,7 @@ on: jobs: e2e: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 strategy: fail-fast: false matrix: @@ -28,11 +28,11 @@ jobs: - alpine - debian engine-version: -# - 20.10-dind # FIXME: Fails on 20.10 - - stable-dind # TODO: Use 20.10-dind, stable-dind is deprecated - include: - - target: non-experimental - engine-version: 19.03-dind + - 25.0 # latest + - 24.0 # latest - 1 + - 23.0 # mirantis lts + # TODO(krissetto) 19.03 needs a look, doesn't work ubuntu 22.04 (cgroup errors). + # we could have a separate job that tests it against ubuntu 20.04 steps: - name: Checkout diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5cf04b3372..1a8ce1f291 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -16,7 +16,7 @@ on: jobs: ctn: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 steps: - name: Checkout diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index ccd0493a32..41349713c8 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -16,7 +16,7 @@ on: jobs: validate: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 strategy: fail-fast: false matrix: @@ -37,7 +37,7 @@ jobs: # check that the generated Markdown and the checked-in files match validate-md: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 steps: - name: Checkout @@ -57,7 +57,7 @@ jobs: fi validate-make: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 strategy: fail-fast: false matrix: diff --git a/e2e/image/testdata/pull-with-content-trust.golden b/e2e/image/testdata/pull-with-content-trust.golden index b21407e117..582a4e0cc9 100644 --- a/e2e/image/testdata/pull-with-content-trust.golden +++ b/e2e/image/testdata/pull-with-content-trust.golden @@ -1,5 +1,5 @@ Pull (1 of 1): registry:5000/trust-pull:latest@sha256:e2e16842c9b54d985bf1ef9242a313f36b856181f188de21313820e177002501 -sha256:e2e16842c9b54d985bf1ef9242a313f36b856181f188de21313820e177002501: Pulling from trust-pull +registry:5000/trust-pull@sha256:e2e16842c9b54d985bf1ef9242a313f36b856181f188de21313820e177002501: Pulling from trust-pull Digest: sha256:e2e16842c9b54d985bf1ef9242a313f36b856181f188de21313820e177002501 Status: Downloaded newer image for registry:5000/trust-pull@sha256:e2e16842c9b54d985bf1ef9242a313f36b856181f188de21313820e177002501 registry:5000/trust-pull:latest diff --git a/e2e/plugin/trust_test.go b/e2e/plugin/trust_test.go index 38bdcc5e45..c6957d6f37 100644 --- a/e2e/plugin/trust_test.go +++ b/e2e/plugin/trust_test.go @@ -11,6 +11,7 @@ import ( "github.com/docker/cli/e2e/internal/fixtures" "github.com/docker/cli/internal/test/environment" "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/versions" "github.com/pkg/errors" "gotest.tools/v3/assert" "gotest.tools/v3/fs" @@ -21,6 +22,8 @@ import ( const registryPrefix = "registry:5000" func TestInstallWithContentTrust(t *testing.T) { + // TODO(krissetto): remove this skip once the fix (see https://github.com/moby/moby/pull/47299) is deployed to moby versions < 25 + skip.If(t, versions.LessThan(environment.DaemonAPIVersion(t), "1.44")) skip.If(t, environment.SkipPluginTests()) pluginName := fmt.Sprintf("%s/plugin-content-trust", registryPrefix) @@ -50,7 +53,7 @@ func TestInstallWithContentTrust(t *testing.T) { fixtures.WithNotary, ) result.Assert(t, icmd.Expected{ - Out: fmt.Sprintf("Status: Downloaded newer image for %s@sha", pluginName), + Out: fmt.Sprintf("Installed plugin %s", pluginName), }) } From 69ed6588a8dcd92921022ba0f42e562cd41bbb4e Mon Sep 17 00:00:00 2001 From: Christopher Petito Date: Tue, 13 Feb 2024 13:32:45 +0000 Subject: [PATCH 2/2] Cleanup of dockerfiles, compose files and env vars Signed-off-by: Christopher Petito --- .github/workflows/e2e.yml | 2 +- docker.Makefile | 10 +++++----- e2e/compose-env.connhelper-ssh.yaml | 2 ++ e2e/compose-env.yaml | 4 +++- e2e/testdata/Dockerfile.connhelper-ssh | 15 +++++++++++++-- e2e/testdata/Dockerfile.evil-notary-server | 3 +++ e2e/testdata/Dockerfile.notary-server | 3 +++ 7 files changed, 30 insertions(+), 9 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 227d105f8e..9ce1a6d92a 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -55,7 +55,7 @@ jobs: make -f docker.Makefile test-e2e-${{ matrix.target }} env: BASE_VARIANT: ${{ matrix.base }} - E2E_ENGINE_VERSION: ${{ matrix.engine-version }} + ENGINE_VERSION: ${{ matrix.engine-version }} TESTFLAGS: -coverprofile=/tmp/coverage/coverage.txt - name: Send to Codecov diff --git a/docker.Makefile b/docker.Makefile index bddd91a2a5..011bfd9b68 100644 --- a/docker.Makefile +++ b/docker.Makefile @@ -14,13 +14,13 @@ PACKAGER_NAME ?= DEV_DOCKER_IMAGE_NAME = docker-cli-dev$(IMAGE_TAG) E2E_IMAGE_NAME = docker-cli-e2e -E2E_ENGINE_VERSION ?= +ENGINE_VERSION ?= CACHE_VOLUME_NAME := docker-cli-dev-cache ifeq ($(DOCKER_CLI_GO_BUILD_CACHE),y) DOCKER_CLI_MOUNTS += -v "$(CACHE_VOLUME_NAME):/root/.cache/go-build" endif VERSION = $(shell cat VERSION) -ENVVARS = -e VERSION=$(VERSION) -e GITCOMMIT -e PLATFORM -e TESTFLAGS -e TESTDIRS -e GOOS -e GOARCH -e GOARM -e TEST_ENGINE_VERSION=$(E2E_ENGINE_VERSION) +ENVVARS = -e VERSION=$(VERSION) -e GITCOMMIT -e PLATFORM -e TESTFLAGS -e TESTDIRS -e GOOS -e GOARCH -e GOARM -e ENGINE_VERSION # Some Dockerfiles use features that are only supported with BuildKit enabled export DOCKER_BUILDKIT=1 @@ -132,21 +132,21 @@ test-e2e: test-e2e-non-experimental test-e2e-experimental test-e2e-connhelper-ss .PHONY: test-e2e-experimental test-e2e-experimental: build-e2e-image # run experimental e2e tests - docker run --rm $(ENVVARS) -e DOCKERD_EXPERIMENTAL=1 -e TEST_ENGINE_VERSION=$(E2E_ENGINE_VERSION) \ + docker run --rm $(ENVVARS) -e DOCKERD_EXPERIMENTAL=1 \ --mount type=bind,src=$(CURDIR)/build/coverage,dst=/tmp/coverage \ --mount type=bind,src=/var/run/docker.sock,dst=/var/run/docker.sock \ $(E2E_IMAGE_NAME) .PHONY: test-e2e-non-experimental test-e2e-non-experimental: build-e2e-image # run non-experimental e2e tests - docker run --rm $(ENVVARS) -e TEST_ENGINE_VERSION=$(E2E_ENGINE_VERSION) \ + docker run --rm $(ENVVARS) \ --mount type=bind,src=$(CURDIR)/build/coverage,dst=/tmp/coverage \ --mount type=bind,src=/var/run/docker.sock,dst=/var/run/docker.sock \ $(E2E_IMAGE_NAME) .PHONY: test-e2e-connhelper-ssh test-e2e-connhelper-ssh: build-e2e-image # run experimental SSH-connection helper e2e tests - docker run --rm $(ENVVARS) -e DOCKERD_EXPERIMENTAL=1 -e TEST_ENGINE_VERSION=$(E2E_ENGINE_VERSION) -e TEST_CONNHELPER=ssh \ + docker run --rm $(ENVVARS) -e DOCKERD_EXPERIMENTAL=1 -e TEST_CONNHELPER=ssh \ --mount type=bind,src=$(CURDIR)/build/coverage,dst=/tmp/coverage \ --mount type=bind,src=/var/run/docker.sock,dst=/var/run/docker.sock \ $(E2E_IMAGE_NAME) diff --git a/e2e/compose-env.connhelper-ssh.yaml b/e2e/compose-env.connhelper-ssh.yaml index 2a91ab48fe..283e306fbc 100644 --- a/e2e/compose-env.connhelper-ssh.yaml +++ b/e2e/compose-env.connhelper-ssh.yaml @@ -3,5 +3,7 @@ services: build: context: ./testdata dockerfile: Dockerfile.connhelper-ssh + args: + - ENGINE_VERSION environment: - TEST_CONNHELPER_SSH_ID_RSA_PUB diff --git a/e2e/compose-env.yaml b/e2e/compose-env.yaml index 9eade791b0..2a30e1040e 100644 --- a/e2e/compose-env.yaml +++ b/e2e/compose-env.yaml @@ -1,9 +1,10 @@ services: + registry: image: 'registry:2' engine: - image: 'docker:${TEST_ENGINE_VERSION:-stable-dind}' + image: 'docker:${ENGINE_VERSION:-25.0}-dind' privileged: true command: ['--insecure-registry=registry:5000'] environment: @@ -16,6 +17,7 @@ services: ports: - 4443:4443 command: ['notary-server', '-config=/fixtures/notary-config.json'] + evil-notary-server: build: context: ./testdata diff --git a/e2e/testdata/Dockerfile.connhelper-ssh b/e2e/testdata/Dockerfile.connhelper-ssh index 64306b518c..3771f3e62f 100644 --- a/e2e/testdata/Dockerfile.connhelper-ssh +++ b/e2e/testdata/Dockerfile.connhelper-ssh @@ -1,5 +1,16 @@ -FROM docker:test-dind -RUN apk --no-cache add shadow openssh-server && \ +# syntax=docker/dockerfile:1 + +# ENGINE_VERSION is the version of the (docker-in-docker) Docker Engine to +# test against. +ARG ENGINE_VERSION=25.0 + +FROM docker:${ENGINE_VERSION}-dind + +# the openssh-client update is needed for security reasons when using docker:23.0-dind, currently maintained as an lts by mirantis +RUN apk --no-cache upgrade openssh-client && \ + apk --no-cache add shadow openssh-server && \ + # TODO(krissetto): `groupadd` can be removed once we only test against moby >= v24 + # see https://github.com/docker-library/docker/pull/470 groupadd -f docker && \ useradd --create-home --shell /bin/sh --password $(head -c32 /dev/urandom | base64) penguin && \ usermod -aG docker penguin && \ diff --git a/e2e/testdata/Dockerfile.evil-notary-server b/e2e/testdata/Dockerfile.evil-notary-server index d982d5a4eb..97f234e849 100644 --- a/e2e/testdata/Dockerfile.evil-notary-server +++ b/e2e/testdata/Dockerfile.evil-notary-server @@ -1,4 +1,7 @@ +# syntax=docker/dockerfile:1 + ARG NOTARY_VERSION=0.6.1 + FROM notary:server-${NOTARY_VERSION} COPY ./notary-evil/ /fixtures/ diff --git a/e2e/testdata/Dockerfile.notary-server b/e2e/testdata/Dockerfile.notary-server index 4bc59d2d88..846253e2fb 100644 --- a/e2e/testdata/Dockerfile.notary-server +++ b/e2e/testdata/Dockerfile.notary-server @@ -1,4 +1,7 @@ +# syntax=docker/dockerfile:1 + ARG NOTARY_VERSION=0.6.1 + FROM notary:server-${NOTARY_VERSION} COPY ./notary/ /fixtures/