From 166856ab1b1681de107b5e064ea89ea4b27154fd Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Thu, 29 Nov 2018 01:06:10 +0100 Subject: [PATCH] Do not patch Dockerfiles in CI When building the Dockerfiles for development, those images are mainly used to create a reproducible build-environment. The source code is bind-mounted into the image at runtime; there is no need to create an image with the actual source code, and copying the source code into the image would lead to a new image being created for each code-change (possibly leading up to many "dangling" images for previous code-changes). However, when building (and using) the development images in CI, bind-mounting is not an option, because the daemon is running remotely. To make this work, the circle-ci script patched the Dockerfiles when CI is run; adding a `COPY` to the respective Dockerfiles. Patching Dockerfiles is not really a "best practice" and, even though the source code does not and up in the image, the source would still be _sent_ to the daemon for each build (unless BuildKit is used). This patch updates the makefiles, circle-ci script, and Dockerfiles; - When building the Dockerfiles locally, pipe the Dockerfile through stdin. Doing so, prevents the build-context from being sent to the daemon. This speeds up the build, and doesn't fill up the Docker "temp" directory with content that's not used - Now that no content is sent, add the COPY instructions to the Dockerfiles, and remove the code in the circle-ci script to "live patch" the Dockerfiles. Before this patch is applied (with cache): ``` $ time make -f docker.Makefile build_shell_validate_image docker build -t docker-cli-shell-validate -f ./dockerfiles/Dockerfile.shellcheck . Sending build context to Docker daemon 41MB Step 1/2 : FROM debian:stretch-slim ... Successfully built 81e14e8ad856 Successfully tagged docker-cli-shell-validate:latest 2.75 real 0.45 user 0.56 sys ``` After this patch is applied (with cache):: ``` $ time make -f docker.Makefile build_shell_validate_image cat ./dockerfiles/Dockerfile.shellcheck | docker build -t docker-cli-shell-validate - Sending build context to Docker daemon 2.048kB Step 1/2 : FROM debian:stretch-slim ... Successfully built 81e14e8ad856 Successfully tagged docker-cli-shell-validate:latest 0.33 real 0.07 user 0.08 sys ``` Signed-off-by: Sebastiaan van Stijn --- circle.yml | 20 +++++--------------- docker.Makefile | 15 ++++++++++----- dockerfiles/Dockerfile.cross | 1 + dockerfiles/Dockerfile.dev | 2 +- dockerfiles/Dockerfile.lint | 1 + dockerfiles/Dockerfile.shellcheck | 1 + 6 files changed, 19 insertions(+), 21 deletions(-) diff --git a/circle.yml b/circle.yml index 073d4cf69e..3b4d9259e9 100644 --- a/circle.yml +++ b/circle.yml @@ -16,9 +16,7 @@ jobs: - run: name: "Lint" command: | - dockerfile=dockerfiles/Dockerfile.lint - echo "COPY . ." >> $dockerfile - docker build -f $dockerfile --tag cli-linter:$CIRCLE_BUILD_NUM . + docker build -f dockerfiles/Dockerfile.lint --tag cli-linter:$CIRCLE_BUILD_NUM . docker run --rm cli-linter:$CIRCLE_BUILD_NUM cross: @@ -34,9 +32,7 @@ jobs: - run: name: "Cross" command: | - dockerfile=dockerfiles/Dockerfile.cross - echo "COPY . ." >> $dockerfile - docker build -f $dockerfile --tag cli-builder:$CIRCLE_BUILD_NUM . + docker build -f dockerfiles/Dockerfile.cross --tag cli-builder:$CIRCLE_BUILD_NUM . name=cross-$CIRCLE_BUILD_NUM-$CIRCLE_NODE_INDEX docker run \ -e CROSS_GROUP=$CIRCLE_NODE_INDEX \ @@ -60,9 +56,7 @@ jobs: - run: name: "Unit Test with Coverage" command: | - dockerfile=dockerfiles/Dockerfile.dev - echo "COPY . ." >> $dockerfile - docker build -f $dockerfile --tag cli-builder:$CIRCLE_BUILD_NUM . + docker build -f dockerfiles/Dockerfile.dev --tag cli-builder:$CIRCLE_BUILD_NUM . docker run --name \ test-$CIRCLE_BUILD_NUM cli-builder:$CIRCLE_BUILD_NUM \ make test-coverage @@ -89,10 +83,8 @@ jobs: - run: name: "Validate Vendor, Docs, and Code Generation" command: | - dockerfile=dockerfiles/Dockerfile.dev - echo "COPY . ." >> $dockerfile rm -f .dockerignore # include .git - docker build -f $dockerfile --tag cli-builder-with-git:$CIRCLE_BUILD_NUM . + docker build -f dockerfiles/Dockerfile.dev --tag cli-builder-with-git:$CIRCLE_BUILD_NUM . docker run --rm cli-builder-with-git:$CIRCLE_BUILD_NUM \ make ci-validate shellcheck: @@ -107,9 +99,7 @@ jobs: - run: name: "Run shellcheck" command: | - dockerfile=dockerfiles/Dockerfile.shellcheck - echo "COPY . ." >> $dockerfile - docker build -f $dockerfile --tag cli-validator:$CIRCLE_BUILD_NUM . + docker build -f dockerfiles/Dockerfile.shellcheck --tag cli-validator:$CIRCLE_BUILD_NUM . docker run --rm cli-validator:$CIRCLE_BUILD_NUM \ make shellcheck workflows: diff --git a/docker.Makefile b/docker.Makefile index 2ff2594a7f..628e989fb0 100644 --- a/docker.Makefile +++ b/docker.Makefile @@ -17,24 +17,29 @@ ENVVARS = -e VERSION=$(VERSION) -e GITCOMMIT -e PLATFORM # build docker image (dockerfiles/Dockerfile.build) .PHONY: build_docker_image build_docker_image: - docker build ${DOCKER_BUILD_ARGS} -t $(DEV_DOCKER_IMAGE_NAME) -f ./dockerfiles/Dockerfile.dev . + # build dockerfile from stdin so that we don't send the build-context; source is bind-mounted in the development environment + cat ./dockerfiles/Dockerfile.dev | docker build ${DOCKER_BUILD_ARGS} -t $(DEV_DOCKER_IMAGE_NAME) - # build docker image having the linting tools (dockerfiles/Dockerfile.lint) .PHONY: build_linter_image build_linter_image: - docker build ${DOCKER_BUILD_ARGS} -t $(LINTER_IMAGE_NAME) -f ./dockerfiles/Dockerfile.lint . + # build dockerfile from stdin so that we don't send the build-context; source is bind-mounted in the development environment + cat ./dockerfiles/Dockerfile.lint | docker build ${DOCKER_BUILD_ARGS} -t $(LINTER_IMAGE_NAME) - .PHONY: build_cross_image build_cross_image: - docker build ${DOCKER_BUILD_ARGS} -t $(CROSS_IMAGE_NAME) -f ./dockerfiles/Dockerfile.cross . + # build dockerfile from stdin so that we don't send the build-context; source is bind-mounted in the development environment + cat ./dockerfiles/Dockerfile.cross | docker build ${DOCKER_BUILD_ARGS} -t $(CROSS_IMAGE_NAME) - .PHONY: build_shell_validate_image build_shell_validate_image: - docker build -t $(VALIDATE_IMAGE_NAME) -f ./dockerfiles/Dockerfile.shellcheck . + # build dockerfile from stdin so that we don't send the build-context; source is bind-mounted in the development environment + cat ./dockerfiles/Dockerfile.shellcheck | docker build -t $(VALIDATE_IMAGE_NAME) - .PHONY: build_binary_native_image build_binary_native_image: - docker build -t $(BINARY_NATIVE_IMAGE_NAME) -f ./dockerfiles/Dockerfile.binary-native . + # build dockerfile from stdin so that we don't send the build-context; source is bind-mounted in the development environment + cat ./dockerfiles/Dockerfile.binary-native | docker build -t $(BINARY_NATIVE_IMAGE_NAME) - .PHONY: build_e2e_image build_e2e_image: diff --git a/dockerfiles/Dockerfile.cross b/dockerfiles/Dockerfile.cross index 78073218af..0f54c9f64a 100644 --- a/dockerfiles/Dockerfile.cross +++ b/dockerfiles/Dockerfile.cross @@ -1,3 +1,4 @@ FROM dockercore/golang-cross:1.11.1@sha256:57ce59bec5445d27cc00f9e27fc171bea44eed7a9890df6eea6540f214300a01 ENV DISABLE_WARN_OUTSIDE_CONTAINER=1 WORKDIR /go/src/github.com/docker/cli +COPY . . diff --git a/dockerfiles/Dockerfile.dev b/dockerfiles/Dockerfile.dev index 84399a6c8b..69de827da8 100644 --- a/dockerfiles/Dockerfile.dev +++ b/dockerfiles/Dockerfile.dev @@ -1,4 +1,3 @@ - FROM golang:1.11.1-alpine RUN apk add -U git make bash coreutils ca-certificates curl @@ -22,3 +21,4 @@ ENV CGO_ENABLED=0 \ DISABLE_WARN_OUTSIDE_CONTAINER=1 WORKDIR /go/src/github.com/docker/cli CMD sh +COPY . . diff --git a/dockerfiles/Dockerfile.lint b/dockerfiles/Dockerfile.lint index 953fd69206..10be9db265 100644 --- a/dockerfiles/Dockerfile.lint +++ b/dockerfiles/Dockerfile.lint @@ -15,3 +15,4 @@ ENV CGO_ENABLED=0 ENV DISABLE_WARN_OUTSIDE_CONTAINER=1 ENTRYPOINT ["/usr/local/bin/gometalinter"] CMD ["--config=gometalinter.json", "./..."] +COPY . . diff --git a/dockerfiles/Dockerfile.shellcheck b/dockerfiles/Dockerfile.shellcheck index 43112b314d..4d0ddd3332 100644 --- a/dockerfiles/Dockerfile.shellcheck +++ b/dockerfiles/Dockerfile.shellcheck @@ -7,3 +7,4 @@ RUN apt-get update && \ WORKDIR /go/src/github.com/docker/cli ENV DISABLE_WARN_OUTSIDE_CONTAINER=1 CMD bash +COPY . .