From eefb7634f9283744e33226d1044111f595f60bfb Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Fri, 25 Aug 2023 01:56:35 +0200 Subject: [PATCH 1/5] Dockerfile: use COPY --link where possible Signed-off-by: Sebastiaan van Stijn (cherry picked from commit af05a68828ede9bf58b82082a4d74af9fac74cae) Signed-off-by: Sebastiaan van Stijn --- Dockerfile | 22 +++++++++++----------- dockerfiles/Dockerfile.dev | 10 +++++----- dockerfiles/Dockerfile.lint | 2 +- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/Dockerfile b/Dockerfile index 424a83d69b..5b10a76d8b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,7 +11,7 @@ ARG BUILDX_VERSION=0.11.2 FROM --platform=$BUILDPLATFORM tonistiigi/xx:${XX_VERSION} AS xx FROM --platform=$BUILDPLATFORM golang:${GO_VERSION}-alpine${ALPINE_VERSION} AS build-base-alpine -COPY --from=xx / / +COPY --link --from=xx / / RUN apk add --no-cache bash clang lld llvm file git WORKDIR /go/src/github.com/docker/cli @@ -21,7 +21,7 @@ ARG TARGETPLATFORM RUN xx-apk add --no-cache musl-dev gcc FROM --platform=$BUILDPLATFORM golang:${GO_VERSION}-bullseye AS build-base-bullseye -COPY --from=xx / / +COPY --link --from=xx / / RUN apt-get update && apt-get install --no-install-recommends -y bash clang lld llvm file WORKDIR /go/src/github.com/docker/cli @@ -62,7 +62,7 @@ ARG CGO_ENABLED ARG VERSION # PACKAGER_NAME sets the company that produced the windows binary ARG PACKAGER_NAME -COPY --from=goversioninfo /out/goversioninfo /usr/bin/goversioninfo +COPY --link --from=goversioninfo /out/goversioninfo /usr/bin/goversioninfo # in bullseye arm64 target does not link with lld so configure it to use ld instead RUN [ ! -f /etc/alpine-release ] && xx-info is-cross && [ "$(xx-info arch)" = "arm64" ] && XX_CC_PREFER_LINKER=ld xx-clang --setup-target-triple || true RUN --mount=type=bind,target=.,ro \ @@ -76,7 +76,7 @@ RUN --mount=type=bind,target=.,ro \ xx-verify $([ "$GO_LINKMODE" = "static" ] && echo "--static") /out/docker FROM build-${BASE_VARIANT} AS test -COPY --from=gotestsum /out/gotestsum /usr/bin/gotestsum +COPY --link --from=gotestsum /out/gotestsum /usr/bin/gotestsum ENV GO111MODULE=auto RUN --mount=type=bind,target=.,rw \ --mount=type=cache,target=/root/.cache \ @@ -111,19 +111,19 @@ FROM docker/buildx-bin:${BUILDX_VERSION} AS buildx FROM e2e-base-${BASE_VARIANT} AS e2e ARG NOTARY_VERSION=v0.6.1 ADD --chmod=0755 https://github.com/theupdateframework/notary/releases/download/${NOTARY_VERSION}/notary-Linux-amd64 /usr/local/bin/notary -COPY e2e/testdata/notary/root-ca.cert /usr/share/ca-certificates/notary.cert +COPY --link e2e/testdata/notary/root-ca.cert /usr/share/ca-certificates/notary.cert RUN echo 'notary.cert' >> /etc/ca-certificates.conf && update-ca-certificates -COPY --from=gotestsum /out/gotestsum /usr/bin/gotestsum -COPY --from=build /out ./build/ -COPY --from=build-plugins /out ./build/ -COPY --from=buildx /buildx /usr/libexec/docker/cli-plugins/docker-buildx -COPY . . +COPY --link --from=gotestsum /out/gotestsum /usr/bin/gotestsum +COPY --link --from=build /out ./build/ +COPY --link --from=build-plugins /out ./build/ +COPY --link --from=buildx /buildx /usr/libexec/docker/cli-plugins/docker-buildx +COPY --link . . ENV DOCKER_BUILDKIT=1 ENV PATH=/go/src/github.com/docker/cli/build:$PATH CMD ./scripts/test/e2e/entry FROM build-base-${BASE_VARIANT} AS dev -COPY . . +COPY --link . . FROM scratch AS binary COPY --from=build /out . diff --git a/dockerfiles/Dockerfile.dev b/dockerfiles/Dockerfile.dev index 0ec149f3f1..65d8a63795 100644 --- a/dockerfiles/Dockerfile.dev +++ b/dockerfiles/Dockerfile.dev @@ -48,11 +48,11 @@ CMD bash ENV DISABLE_WARN_OUTSIDE_CONTAINER=1 ENV PATH=$PATH:/go/src/github.com/docker/cli/build -COPY --from=buildx /buildx /usr/libexec/docker/cli-plugins/docker-buildx -COPY --from=gofumpt /go/bin/* /go/bin/ -COPY --from=gotestsum /go/bin/* /go/bin/ -COPY --from=goversioninfo /go/bin/* /go/bin/ +COPY --link --from=buildx /buildx /usr/libexec/docker/cli-plugins/docker-buildx +COPY --link --from=gofumpt /go/bin/* /go/bin/ +COPY --link --from=gotestsum /go/bin/* /go/bin/ +COPY --link --from=goversioninfo /go/bin/* /go/bin/ WORKDIR /go/src/github.com/docker/cli ENV GO111MODULE=auto -COPY . . +COPY --link . . diff --git a/dockerfiles/Dockerfile.lint b/dockerfiles/Dockerfile.lint index 449a8d1a79..2390e2a6ec 100644 --- a/dockerfiles/Dockerfile.lint +++ b/dockerfiles/Dockerfile.lint @@ -11,7 +11,7 @@ ENV GO111MODULE=off ENV CGO_ENABLED=0 ENV GOGC=75 WORKDIR /go/src/github.com/docker/cli -COPY --from=golangci-lint /usr/bin/golangci-lint /usr/bin/golangci-lint +COPY --link --from=golangci-lint /usr/bin/golangci-lint /usr/bin/golangci-lint RUN --mount=type=bind,target=. \ --mount=type=cache,target=/root/.cache \ golangci-lint run From 4f35b26d400461f5fb409233db23b828a8f7faeb Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Tue, 26 Sep 2023 12:15:19 +0200 Subject: [PATCH 2/5] e2e: update to use compose v2, and don't depend on distro-packages We were depending on alpine's package repository to install compose, but for debian we used compose's GitHub releases. Depending on distro packages means that we don't know when updates will happen, and versions may diverge because of that; for example, alpine 3.18 updated to compose v2; On alpine 3.17: make -f docker.Makefile build-e2e-image docker run --rm docker-cli-e2e docker-compose --version docker-compose version 1.29.2, build unknown On alpine 3.18: make -f docker.Makefile build-e2e-image docker run --rm docker-cli-e2e docker-compose --version Docker Compose version v2.17.3 This caused our e2e script to fail, as it made assumptions about the name format created by compose, which changed from underscores to hyphens in v2; Container cliendtoendsuite-engine-1 Running Error: No such object: cliendtoendsuite_engine_1 This patch: - updates the Dockerfile to install compose from the compose-bin image - adjusts the e2e script for the new naming scheme format - removes the version field from the compose-files used in e2e, as they are no longer used by compose. Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 9e424af5da0566ac5b83deb73d71992f452f81aa) Signed-off-by: Sebastiaan van Stijn --- Dockerfile | 12 ++++++------ e2e/compose-env.connhelper-ssh.yaml | 2 -- e2e/compose-env.experimental.yaml | 3 --- e2e/compose-env.yaml | 3 --- scripts/test/e2e/run | 6 +++--- 5 files changed, 9 insertions(+), 17 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5b10a76d8b..72f2123aea 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,6 +7,7 @@ ARG XX_VERSION=1.1.1 ARG GOVERSIONINFO_VERSION=v1.3.0 ARG GOTESTSUM_VERSION=v1.10.0 ARG BUILDX_VERSION=0.11.2 +ARG COMPOSE_VERSION=v2.22.0 FROM --platform=$BUILDPLATFORM tonistiigi/xx:${XX_VERSION} AS xx @@ -98,15 +99,13 @@ RUN --mount=ro --mount=type=cache,target=/root/.cache \ TARGET=/out ./scripts/build/plugins e2e/cli-plugins/plugins/* FROM build-base-alpine AS e2e-base-alpine -RUN apk add --no-cache build-base curl docker-compose openssl openssh-client +RUN apk add --no-cache build-base curl openssl openssh-client FROM build-base-bullseye AS e2e-base-bullseye RUN apt-get update && apt-get install -y build-essential curl openssl openssh-client -ARG COMPOSE_VERSION=1.29.2 -RUN curl -fsSL https://github.com/docker/compose/releases/download/${COMPOSE_VERSION}/docker-compose-$(uname -s)-$(uname -m) -o /usr/local/bin/docker-compose && \ - chmod +x /usr/local/bin/docker-compose -FROM docker/buildx-bin:${BUILDX_VERSION} AS buildx +FROM docker/buildx-bin:${BUILDX_VERSION} AS buildx +FROM docker/compose-bin:${COMPOSE_VERSION} AS compose FROM e2e-base-${BASE_VARIANT} AS e2e ARG NOTARY_VERSION=v0.6.1 @@ -116,7 +115,8 @@ RUN echo 'notary.cert' >> /etc/ca-certificates.conf && update-ca-certificates COPY --link --from=gotestsum /out/gotestsum /usr/bin/gotestsum COPY --link --from=build /out ./build/ COPY --link --from=build-plugins /out ./build/ -COPY --link --from=buildx /buildx /usr/libexec/docker/cli-plugins/docker-buildx +COPY --link --from=buildx /buildx /usr/libexec/docker/cli-plugins/docker-buildx +COPY --link --from=compose /docker-compose /usr/libexec/docker/cli-plugins/docker-compose COPY --link . . ENV DOCKER_BUILDKIT=1 ENV PATH=/go/src/github.com/docker/cli/build:$PATH diff --git a/e2e/compose-env.connhelper-ssh.yaml b/e2e/compose-env.connhelper-ssh.yaml index a1baa114fe..2a91ab48fe 100644 --- a/e2e/compose-env.connhelper-ssh.yaml +++ b/e2e/compose-env.connhelper-ssh.yaml @@ -1,5 +1,3 @@ -version: '2.1' - services: engine: build: diff --git a/e2e/compose-env.experimental.yaml b/e2e/compose-env.experimental.yaml index cc99c46e32..0fac9fb20e 100644 --- a/e2e/compose-env.experimental.yaml +++ b/e2e/compose-env.experimental.yaml @@ -1,6 +1,3 @@ -version: '2.1' - services: engine: command: ["--insecure-registry=registry:5000", "--experimental"] - diff --git a/e2e/compose-env.yaml b/e2e/compose-env.yaml index a4c700c465..9eade791b0 100644 --- a/e2e/compose-env.yaml +++ b/e2e/compose-env.yaml @@ -1,5 +1,3 @@ -version: '2.1' - services: registry: image: 'registry:2' @@ -25,4 +23,3 @@ services: ports: - 4444:4443 command: ['notary-server', '-config=/fixtures/notary-config.json'] - diff --git a/scripts/test/e2e/run b/scripts/test/e2e/run index 9065884a02..54e1d61e8a 100755 --- a/scripts/test/e2e/run +++ b/scripts/test/e2e/run @@ -26,13 +26,13 @@ setup() { export TEST_CONNHELPER_SSH_ID_RSA_PUB file="${file}:./e2e/compose-env.connhelper-ssh.yaml" fi - COMPOSE_PROJECT_NAME=$project COMPOSE_FILE=$file docker-compose up --build -d >&2 + COMPOSE_PROJECT_NAME=$project COMPOSE_FILE=$file docker compose up --build -d >&2 local network="${project}_default" # TODO: only run if inside a container docker network connect "$network" "$(hostname)" - engine_ip="$(container_ip "${project}_engine_1" "$network")" + engine_ip="$(container_ip "${project}-engine-1" "$network")" engine_host="tcp://$engine_ip:2375" if [ "${TEST_CONNHELPER:-}" = "ssh" ];then engine_host="ssh://penguin@${engine_ip}" @@ -54,7 +54,7 @@ cleanup() { local project=$1 local network="${project}_default" docker network disconnect "$network" "$(hostname)" - COMPOSE_PROJECT_NAME=$1 COMPOSE_FILE=$2 docker-compose down -v --rmi local >&2 + COMPOSE_PROJECT_NAME=$1 COMPOSE_FILE=$2 docker compose down -v --rmi local >&2 } runtests() { From 9d193ea872a773e6fb968820ec1a4fbcb90d3296 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Tue, 26 Sep 2023 12:28:21 +0200 Subject: [PATCH 3/5] Dockerfile: build gotestsum and goversioninfo without cgo It's not needed to build these binaries. The Dockerfile.dev image already has CGO_ENABLED=0 as default in the golang image, so does not need updates. Signed-off-by: Sebastiaan van Stijn (cherry picked from commit f07e7e1eed3e7b1e760a0578485b3e011d4386aa) Signed-off-by: Sebastiaan van Stijn --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 72f2123aea..541d79a3bc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -41,13 +41,13 @@ FROM build-base-${BASE_VARIANT} AS goversioninfo ARG GOVERSIONINFO_VERSION RUN --mount=type=cache,target=/root/.cache/go-build \ --mount=type=cache,target=/go/pkg/mod \ - GOBIN=/out GO111MODULE=on go install "github.com/josephspurrier/goversioninfo/cmd/goversioninfo@${GOVERSIONINFO_VERSION}" + GOBIN=/out GO111MODULE=on CGO_ENABLED=0 go install "github.com/josephspurrier/goversioninfo/cmd/goversioninfo@${GOVERSIONINFO_VERSION}" FROM build-base-${BASE_VARIANT} AS gotestsum ARG GOTESTSUM_VERSION RUN --mount=type=cache,target=/root/.cache/go-build \ --mount=type=cache,target=/go/pkg/mod \ - GOBIN=/out GO111MODULE=on go install "gotest.tools/gotestsum@${GOTESTSUM_VERSION}" \ + GOBIN=/out GO111MODULE=on CGO_ENABLED=0 go install "gotest.tools/gotestsum@${GOTESTSUM_VERSION}" \ && /out/gotestsum --version FROM build-${BASE_VARIANT} AS build From 71c12cf295fdfe5e1406f7ba5451c101b5f04f2d Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Wed, 14 Jun 2023 13:30:40 +0200 Subject: [PATCH 4/5] Dockerfile: update ALPINE_VERSION to 3.18 This also moves `musl-dev` to the alpine-base stage, due to changes in Alpine 3.18 causing gotestsum build to fail because stdlib.h was missing; #17 5.065 # runtime/cgo #17 5.065 In file included from _cgo_export.c:3: #17 5.065 /usr/include/fortify/stdlib.h:23:15: fatal error: stdlib.h: No such file or directory #17 5.065 23 | #include_next #17 5.065 | ^~~~~~~~~~ alpine 3.17: / # find / | grep stdlib.h /usr/include/c++/12.2.1/tr1/stdlib.h /usr/include/c++/12.2.1/stdlib.h alpine 3.18 / # find / | grep stdlib.h /usr/lib/llvm16/lib/clang/16/include/__clang_hip_stdlib.h /usr/include/fortify/stdlib.h /usr/include/c++/12.2.1/tr1/stdlib.h /usr/include/c++/12.2.1/stdlib.h Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 6a74a63ee278123861121f3be41bbd3b733608ed) Signed-off-by: Sebastiaan van Stijn --- Dockerfile | 2 +- dockerfiles/Dockerfile.authors | 2 +- dockerfiles/Dockerfile.dev | 2 +- dockerfiles/Dockerfile.lint | 2 +- dockerfiles/Dockerfile.vendor | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 541d79a3bc..217215e7f8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ ARG BASE_VARIANT=alpine ARG GO_VERSION=1.20.12 -ARG ALPINE_VERSION=3.17 +ARG ALPINE_VERSION=3.18 ARG XX_VERSION=1.1.1 ARG GOVERSIONINFO_VERSION=v1.3.0 ARG GOTESTSUM_VERSION=v1.10.0 diff --git a/dockerfiles/Dockerfile.authors b/dockerfiles/Dockerfile.authors index 95f079a027..278ad769fe 100644 --- a/dockerfiles/Dockerfile.authors +++ b/dockerfiles/Dockerfile.authors @@ -1,6 +1,6 @@ # syntax=docker/dockerfile:1 -ARG ALPINE_VERSION=3.17 +ARG ALPINE_VERSION=3.18 FROM alpine:${ALPINE_VERSION} AS gen RUN apk add --no-cache bash git diff --git a/dockerfiles/Dockerfile.dev b/dockerfiles/Dockerfile.dev index 65d8a63795..6f4f8d496a 100644 --- a/dockerfiles/Dockerfile.dev +++ b/dockerfiles/Dockerfile.dev @@ -1,7 +1,7 @@ # syntax=docker/dockerfile:1 ARG GO_VERSION=1.20.12 -ARG ALPINE_VERSION=3.17 +ARG ALPINE_VERSION=3.18 ARG BUILDX_VERSION=0.11.2 FROM docker/buildx-bin:${BUILDX_VERSION} AS buildx diff --git a/dockerfiles/Dockerfile.lint b/dockerfiles/Dockerfile.lint index 2390e2a6ec..dead918ee6 100644 --- a/dockerfiles/Dockerfile.lint +++ b/dockerfiles/Dockerfile.lint @@ -1,7 +1,7 @@ # syntax=docker/dockerfile:1 ARG GO_VERSION=1.20.12 -ARG ALPINE_VERSION=3.17 +ARG ALPINE_VERSION=3.18 ARG GOLANGCI_LINT_VERSION=v1.52.2 FROM golangci/golangci-lint:${GOLANGCI_LINT_VERSION}-alpine AS golangci-lint diff --git a/dockerfiles/Dockerfile.vendor b/dockerfiles/Dockerfile.vendor index 63d4d86ec6..96f7ab6dfc 100644 --- a/dockerfiles/Dockerfile.vendor +++ b/dockerfiles/Dockerfile.vendor @@ -1,7 +1,7 @@ # syntax=docker/dockerfile:1 ARG GO_VERSION=1.20.12 -ARG ALPINE_VERSION=3.17 +ARG ALPINE_VERSION=3.18 ARG MODOUTDATED_VERSION=v0.8.0 FROM golang:${GO_VERSION}-alpine${ALPINE_VERSION} AS base From 5aa844d97b76fe4af3633df37ad7f0b803cf2ca5 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Wed, 24 Jan 2024 16:08:47 +0100 Subject: [PATCH 5/5] update to go1.20.13 go1.20.13 (released 2024-01-09) includes fixes to the runtime and the crypto/tls package. See the Go 1.20.13 milestone on our issue tracker for details: - https://github.com/golang/go/issues?q=milestone%3AGo1.20.13+label%3ACherryPickApproved - full diff: https://github.com/golang/go/compare/go1.20.12...go1.20.13 Signed-off-by: Sebastiaan van Stijn --- .github/workflows/test.yml | 2 +- Dockerfile | 2 +- docker-bake.hcl | 2 +- dockerfiles/Dockerfile.dev | 2 +- dockerfiles/Dockerfile.lint | 2 +- dockerfiles/Dockerfile.vendor | 2 +- e2e/testdata/Dockerfile.gencerts | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f0f242d08e..5a27a57353 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -63,7 +63,7 @@ jobs: name: Set up Go uses: actions/setup-go@v3 with: - go-version: 1.20.12 + go-version: 1.20.13 - name: Test run: | diff --git a/Dockerfile b/Dockerfile index 217215e7f8..5316fc3b60 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ # syntax=docker/dockerfile:1 ARG BASE_VARIANT=alpine -ARG GO_VERSION=1.20.12 +ARG GO_VERSION=1.20.13 ARG ALPINE_VERSION=3.18 ARG XX_VERSION=1.1.1 ARG GOVERSIONINFO_VERSION=v1.3.0 diff --git a/docker-bake.hcl b/docker-bake.hcl index c9144174b4..058b75bb9f 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -1,5 +1,5 @@ variable "GO_VERSION" { - default = "1.20.12" + default = "1.20.13" } variable "VERSION" { default = "" diff --git a/dockerfiles/Dockerfile.dev b/dockerfiles/Dockerfile.dev index 6f4f8d496a..4612c1b0cb 100644 --- a/dockerfiles/Dockerfile.dev +++ b/dockerfiles/Dockerfile.dev @@ -1,6 +1,6 @@ # syntax=docker/dockerfile:1 -ARG GO_VERSION=1.20.12 +ARG GO_VERSION=1.20.13 ARG ALPINE_VERSION=3.18 ARG BUILDX_VERSION=0.11.2 diff --git a/dockerfiles/Dockerfile.lint b/dockerfiles/Dockerfile.lint index dead918ee6..f3e927ae4c 100644 --- a/dockerfiles/Dockerfile.lint +++ b/dockerfiles/Dockerfile.lint @@ -1,6 +1,6 @@ # syntax=docker/dockerfile:1 -ARG GO_VERSION=1.20.12 +ARG GO_VERSION=1.20.13 ARG ALPINE_VERSION=3.18 ARG GOLANGCI_LINT_VERSION=v1.52.2 diff --git a/dockerfiles/Dockerfile.vendor b/dockerfiles/Dockerfile.vendor index 96f7ab6dfc..d8a0fab08d 100644 --- a/dockerfiles/Dockerfile.vendor +++ b/dockerfiles/Dockerfile.vendor @@ -1,6 +1,6 @@ # syntax=docker/dockerfile:1 -ARG GO_VERSION=1.20.12 +ARG GO_VERSION=1.20.13 ARG ALPINE_VERSION=3.18 ARG MODOUTDATED_VERSION=v0.8.0 diff --git a/e2e/testdata/Dockerfile.gencerts b/e2e/testdata/Dockerfile.gencerts index 5857fdc576..054ced337d 100644 --- a/e2e/testdata/Dockerfile.gencerts +++ b/e2e/testdata/Dockerfile.gencerts @@ -1,6 +1,6 @@ # syntax=docker/dockerfile:1 -ARG GO_VERSION=1.20.12 +ARG GO_VERSION=1.20.13 FROM golang:${GO_VERSION}-alpine AS generated RUN go install github.com/dmcgowan/quicktls@master