From c806eb49c96f012dbd6d98a34a00dba55fcdb16f Mon Sep 17 00:00:00 2001 From: Tibor Vass Date: Thu, 30 Aug 2018 01:56:17 +0000 Subject: [PATCH] build: add options to builder prune This patch adds --filter, --keep-storage, --all and --force to builder prune. Signed-off-by: Tibor Vass --- cli/command/builder/prune.go | 69 ++++++++++++++++++- cli/command/container/prune.go | 2 +- cli/command/formatter/disk_usage.go | 7 +- cli/command/network/prune.go | 2 +- cli/command/system/prune.go | 37 ++++------ cli/command/volume/prune.go | 2 +- vendor.conf | 2 +- .../docker/docker/api/types/types.go | 22 ++++-- .../docker/docker/client/build_prune.go | 19 ++++- .../docker/docker/client/interface.go | 2 +- vendor/github.com/docker/docker/vendor.conf | 8 +-- 11 files changed, 127 insertions(+), 45 deletions(-) diff --git a/cli/command/builder/prune.go b/cli/command/builder/prune.go index 55010d5424..6a53d7f788 100644 --- a/cli/command/builder/prune.go +++ b/cli/command/builder/prune.go @@ -3,29 +3,94 @@ package builder import ( "context" "fmt" + "strings" "github.com/docker/cli/cli" "github.com/docker/cli/cli/command" + "github.com/docker/cli/opts" + "github.com/docker/docker/api/types" units "github.com/docker/go-units" "github.com/spf13/cobra" ) +type pruneOptions struct { + force bool + all bool + filter opts.FilterOpt + keepStorage opts.MemBytes +} + // NewPruneCommand returns a new cobra prune command for images func NewPruneCommand(dockerCli command.Cli) *cobra.Command { + options := pruneOptions{filter: opts.NewFilterOpt()} + cmd := &cobra.Command{ Use: "prune", Short: "Remove build cache", Args: cli.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { - report, err := dockerCli.Client().BuildCachePrune(context.Background()) + spaceReclaimed, output, err := runPrune(dockerCli, options) if err != nil { return err } - fmt.Fprintln(dockerCli.Out(), "Total reclaimed space:", units.HumanSize(float64(report.SpaceReclaimed))) + if output != "" { + fmt.Fprintln(dockerCli.Out(), output) + } + fmt.Fprintln(dockerCli.Out(), "Total reclaimed space:", units.HumanSize(float64(spaceReclaimed))) return nil }, Annotations: map[string]string{"version": "1.39"}, } + flags := cmd.Flags() + flags.BoolVarP(&options.force, "force", "f", false, "Do not prompt for confirmation") + flags.BoolVarP(&options.all, "all", "a", false, "Remove all unused images, not just dangling ones") + flags.Var(&options.filter, "filter", "Provide filter values (e.g. 'max-age=24h')") + flags.Var(&options.keepStorage, "keep-storage", "Amount of disk space to keep for cache") + return cmd } + +const ( + normalWarning = `WARNING! This will remove all dangling build cache. Are you sure you want to continue?` + allCacheWarning = `WARNING! This will remove all build cache. Are you sure you want to continue?` +) + +func runPrune(dockerCli command.Cli, options pruneOptions) (spaceReclaimed uint64, output string, err error) { + pruneFilters := options.filter.Value() + pruneFilters = command.PruneFilters(dockerCli, pruneFilters) + + warning := normalWarning + if options.all { + warning = allCacheWarning + } + if !options.force && !command.PromptForConfirmation(dockerCli.In(), dockerCli.Out(), warning) { + return 0, "", nil + } + + report, err := dockerCli.Client().BuildCachePrune(context.Background(), types.BuildCachePruneOptions{ + All: options.all, + KeepStorage: options.keepStorage.Value(), + Filters: pruneFilters, + }) + if err != nil { + return 0, "", err + } + + if len(report.CachesDeleted) > 0 { + var sb strings.Builder + sb.WriteString("Deleted build cache objects:\n") + for _, id := range report.CachesDeleted { + sb.WriteString(id) + sb.WriteByte('\n') + } + output = sb.String() + } + + return report.SpaceReclaimed, output, nil +} + +// CachePrune executes a prune command for build cache +func CachePrune(dockerCli command.Cli, all bool, filter opts.FilterOpt) (uint64, string, error) { + return runPrune(dockerCli, pruneOptions{force: true, all: all, filter: filter}) +} diff --git a/cli/command/container/prune.go b/cli/command/container/prune.go index ba5f805534..1c820feeb9 100644 --- a/cli/command/container/prune.go +++ b/cli/command/container/prune.go @@ -73,6 +73,6 @@ func runPrune(dockerCli command.Cli, options pruneOptions) (spaceReclaimed uint6 // RunPrune calls the Container Prune API // This returns the amount of space reclaimed and a detailed output string -func RunPrune(dockerCli command.Cli, filter opts.FilterOpt) (uint64, string, error) { +func RunPrune(dockerCli command.Cli, all bool, filter opts.FilterOpt) (uint64, string, error) { return runPrune(dockerCli, pruneOptions{force: true, filter: filter}) } diff --git a/cli/command/formatter/disk_usage.go b/cli/command/formatter/disk_usage.go index d6389a14df..e7cd5166c2 100644 --- a/cli/command/formatter/disk_usage.go +++ b/cli/command/formatter/disk_usage.go @@ -18,12 +18,15 @@ const ( defaultDiskUsageTableFormat = "table {{.Type}}\t{{.TotalCount}}\t{{.Active}}\t{{.Size}}\t{{.Reclaimable}}" defaultBuildCacheVerboseFormat = ` ID: {{.ID}} +Parent: {{.Parent}} +Type: {{.Type}} Description: {{.Description}} -Mutable: {{.Mutable}} Size: {{.Size}} CreatedAt: {{.CreatedAt}} LastUsedAt: {{.LastUsedAt}} UsageCount: {{.UsageCount}} +InUse: {{.InUse}} +Shared: {{.Shared}} ` typeHeader = "TYPE" @@ -416,7 +419,7 @@ func (c *diskUsageBuilderContext) Size() string { func (c *diskUsageBuilderContext) Reclaimable() string { var inUseBytes int64 for _, bc := range c.buildCache { - if bc.InUse { + if bc.InUse && !bc.Shared { inUseBytes += bc.Size } } diff --git a/cli/command/network/prune.go b/cli/command/network/prune.go index b00e5cd21f..462d3616ea 100644 --- a/cli/command/network/prune.go +++ b/cli/command/network/prune.go @@ -70,7 +70,7 @@ func runPrune(dockerCli command.Cli, options pruneOptions) (output string, err e // RunPrune calls the Network Prune API // This returns the amount of space reclaimed and a detailed output string -func RunPrune(dockerCli command.Cli, filter opts.FilterOpt) (uint64, string, error) { +func RunPrune(dockerCli command.Cli, all bool, filter opts.FilterOpt) (uint64, string, error) { output, err := runPrune(dockerCli, pruneOptions{force: true, filter: filter}) return 0, output, err } diff --git a/cli/command/system/prune.go b/cli/command/system/prune.go index dcc7b68d44..3cb9f8b54b 100644 --- a/cli/command/system/prune.go +++ b/cli/command/system/prune.go @@ -2,12 +2,12 @@ package system import ( "bytes" - "context" "fmt" "text/template" "github.com/docker/cli/cli" "github.com/docker/cli/cli/command" + "github.com/docker/cli/cli/command/builder" "github.com/docker/cli/cli/command/container" "github.com/docker/cli/cli/command/image" "github.com/docker/cli/cli/command/network" @@ -21,20 +21,21 @@ import ( type pruneOptions struct { force bool all bool - pruneBuildCache bool pruneVolumes bool + pruneBuildCache bool filter opts.FilterOpt } // newPruneCommand creates a new cobra.Command for `docker prune` func newPruneCommand(dockerCli command.Cli) *cobra.Command { - options := pruneOptions{filter: opts.NewFilterOpt(), pruneBuildCache: true} + options := pruneOptions{filter: opts.NewFilterOpt()} cmd := &cobra.Command{ Use: "prune [OPTIONS]", Short: "Remove unused data", Args: cli.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { + options.pruneBuildCache = versions.GreaterThanOrEqualTo(dockerCli.Client().ClientVersion(), "1.31") return runPrune(dockerCli, options) }, Annotations: map[string]string{"version": "1.25"}, @@ -57,44 +58,30 @@ const confirmationTemplate = `WARNING! This will remove: {{- end }} Are you sure you want to continue?` -// runBuildCachePrune executes a prune command for build cache -func runBuildCachePrune(dockerCli command.Cli, _ opts.FilterOpt) (uint64, string, error) { - report, err := dockerCli.Client().BuildCachePrune(context.Background()) - if err != nil { - return 0, "", err - } - return report.SpaceReclaimed, "", nil -} - func runPrune(dockerCli command.Cli, options pruneOptions) error { // TODO version this once "until" filter is supported for volumes if options.pruneVolumes && options.filter.Value().Contains("until") { return fmt.Errorf(`ERROR: The "until" filter is not supported with "--volumes"`) } - if versions.LessThan(dockerCli.Client().ClientVersion(), "1.31") { - options.pruneBuildCache = false - } if !options.force && !command.PromptForConfirmation(dockerCli.In(), dockerCli.Out(), confirmationMessage(options)) { return nil } - imagePrune := func(dockerCli command.Cli, filter opts.FilterOpt) (uint64, string, error) { - return image.RunPrune(dockerCli, options.all, options.filter) - } - pruneFuncs := []func(dockerCli command.Cli, filter opts.FilterOpt) (uint64, string, error){ + pruneFuncs := []func(dockerCli command.Cli, all bool, filter opts.FilterOpt) (uint64, string, error){ container.RunPrune, network.RunPrune, } if options.pruneVolumes { pruneFuncs = append(pruneFuncs, volume.RunPrune) } - pruneFuncs = append(pruneFuncs, imagePrune) if options.pruneBuildCache { - pruneFuncs = append(pruneFuncs, runBuildCachePrune) + pruneFuncs = append(pruneFuncs, builder.CachePrune) } + // FIXME: modify image.RunPrune to not modify options.filter, otherwise this has to be last in the list. + pruneFuncs = append(pruneFuncs, image.RunPrune) var spaceReclaimed uint64 for _, pruneFn := range pruneFuncs { - spc, output, err := pruneFn(dockerCli, options.filter) + spc, output, err := pruneFn(dockerCli, options.all, options.filter) if err != nil { return err } @@ -126,7 +113,11 @@ func confirmationMessage(options pruneOptions) string { warnings = append(warnings, "all dangling images") } if options.pruneBuildCache { - warnings = append(warnings, "all build cache") + if options.all { + warnings = append(warnings, "all build cache") + } else { + warnings = append(warnings, "all dangling build cache") + } } if len(options.filter.String()) > 0 { warnings = append(warnings, "Elements to be pruned will be filtered with:") diff --git a/cli/command/volume/prune.go b/cli/command/volume/prune.go index 012c549f96..8e48eb973a 100644 --- a/cli/command/volume/prune.go +++ b/cli/command/volume/prune.go @@ -73,6 +73,6 @@ func runPrune(dockerCli command.Cli, options pruneOptions) (spaceReclaimed uint6 // RunPrune calls the Volume Prune API // This returns the amount of space reclaimed and a detailed output string -func RunPrune(dockerCli command.Cli, filter opts.FilterOpt) (uint64, string, error) { +func RunPrune(dockerCli command.Cli, all bool, filter opts.FilterOpt) (uint64, string, error) { return runPrune(dockerCli, pruneOptions{force: true, filter: filter}) } diff --git a/vendor.conf b/vendor.conf index aa6269b3b8..d15dc3b427 100755 --- a/vendor.conf +++ b/vendor.conf @@ -12,7 +12,7 @@ github.com/cpuguy83/go-md2man v1.0.8 github.com/davecgh/go-spew 346938d642f2ec3594ed81d874461961cd0faa76 # v1.1.0 github.com/dgrijalva/jwt-go a2c85815a77d0f951e33ba4db5ae93629a1530af github.com/docker/distribution 83389a148052d74ac602f5f1d62f86ff2f3c4aa5 -github.com/docker/docker 2629fe93266e82751af4f1c7568e21060f065b73 +github.com/docker/docker 6ba1e91877691c4043b57c97090668bc4fd874b6 github.com/docker/docker-credential-helpers 5241b46610f2491efdf9d1c85f1ddf5b02f6d962 # the docker/go package contains a customized version of canonical/json # and is used by Notary. The package is periodically rebased on current Go versions. diff --git a/vendor/github.com/docker/docker/api/types/types.go b/vendor/github.com/docker/docker/api/types/types.go index ed62fd41e5..a8fae3ba32 100644 --- a/vendor/github.com/docker/docker/api/types/types.go +++ b/vendor/github.com/docker/docker/api/types/types.go @@ -543,6 +543,7 @@ type ImagesPruneReport struct { // BuildCachePruneReport contains the response for Engine API: // POST "/build/prune" type BuildCachePruneReport struct { + CachesDeleted []string SpaceReclaimed uint64 } @@ -592,14 +593,21 @@ type BuildResult struct { // BuildCache contains information about a build cache record type BuildCache struct { - ID string - Mutable bool - InUse bool - Size int64 - + ID string + Parent string + Type string + Description string + InUse bool + Shared bool + Size int64 CreatedAt time.Time LastUsedAt *time.Time UsageCount int - Parent string - Description string +} + +// BuildCachePruneOptions hold parameters to prune the build cache +type BuildCachePruneOptions struct { + All bool + KeepStorage int64 + Filters filters.Args } diff --git a/vendor/github.com/docker/docker/client/build_prune.go b/vendor/github.com/docker/docker/client/build_prune.go index c4772a04e7..42bbf99ef1 100644 --- a/vendor/github.com/docker/docker/client/build_prune.go +++ b/vendor/github.com/docker/docker/client/build_prune.go @@ -4,19 +4,34 @@ import ( "context" "encoding/json" "fmt" + "net/url" "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/filters" + "github.com/pkg/errors" ) // BuildCachePrune requests the daemon to delete unused cache data -func (cli *Client) BuildCachePrune(ctx context.Context) (*types.BuildCachePruneReport, error) { +func (cli *Client) BuildCachePrune(ctx context.Context, opts types.BuildCachePruneOptions) (*types.BuildCachePruneReport, error) { if err := cli.NewVersionError("1.31", "build prune"); err != nil { return nil, err } report := types.BuildCachePruneReport{} - serverResp, err := cli.post(ctx, "/build/prune", nil, nil, nil) + query := url.Values{} + if opts.All { + query.Set("all", "1") + } + query.Set("keep-storage", fmt.Sprintf("%d", opts.KeepStorage)) + filters, err := filters.ToJSON(opts.Filters) + if err != nil { + return nil, errors.Wrap(err, "prune could not marshal filters option") + } + query.Set("filters", filters) + + serverResp, err := cli.post(ctx, "/build/prune", query, nil, nil) + if err != nil { return nil, err } diff --git a/vendor/github.com/docker/docker/client/interface.go b/vendor/github.com/docker/docker/client/interface.go index 663749f008..d190f8e58d 100644 --- a/vendor/github.com/docker/docker/client/interface.go +++ b/vendor/github.com/docker/docker/client/interface.go @@ -86,7 +86,7 @@ type DistributionAPIClient interface { // ImageAPIClient defines API client methods for the images type ImageAPIClient interface { ImageBuild(ctx context.Context, context io.Reader, options types.ImageBuildOptions) (types.ImageBuildResponse, error) - BuildCachePrune(ctx context.Context) (*types.BuildCachePruneReport, error) + BuildCachePrune(ctx context.Context, opts types.BuildCachePruneOptions) (*types.BuildCachePruneReport, error) BuildCancel(ctx context.Context, id string) error ImageCreate(ctx context.Context, parentReference string, options types.ImageCreateOptions) (io.ReadCloser, error) ImageHistory(ctx context.Context, image string) ([]image.HistoryResponseItem, error) diff --git a/vendor/github.com/docker/docker/vendor.conf b/vendor/github.com/docker/docker/vendor.conf index b13e022123..4798ae7255 100644 --- a/vendor/github.com/docker/docker/vendor.conf +++ b/vendor/github.com/docker/docker/vendor.conf @@ -1,7 +1,7 @@ # the following lines are in sorted order, FYI github.com/Azure/go-ansiterm d6e3b3328b783f23731bc4d058875b0371ff8109 -github.com/Microsoft/hcsshim v0.6.12 -github.com/Microsoft/go-winio v0.4.9 +github.com/Microsoft/hcsshim v0.6.14 +github.com/Microsoft/go-winio v0.4.10 github.com/docker/libtrust 9cbd2a1374f46905c68a4eb3694a130610adc62a github.com/go-check/check 4ed411733c5785b40214c70bce814c3a3a689609 https://github.com/cpuguy83/check.git github.com/golang/gddo 9b12a26f3fbd7397dee4e20939ddca719d840d2a @@ -26,7 +26,7 @@ github.com/imdario/mergo v0.3.6 golang.org/x/sync 1d60e4601c6fd243af51cc01ddf169918a5407ca # buildkit -github.com/moby/buildkit 46f9075ab68a07df2c40ae6e240ce4f9392b3a66 git://github.com/tiborvass/buildkit.git +github.com/moby/buildkit e1cd06ad6b74e4b747306c4408c451b3b6d87a89 github.com/tonistiigi/fsutil b19464cd1b6a00773b4f2eb7acf9c30426f9df42 github.com/grpc-ecosystem/grpc-opentracing 8e809c8a86450a29b90dcc9efbf062d0fe6d9746 github.com/opentracing/opentracing-go 1361b9cd60be79c4c3a7fa9841b3c132e40066a7 @@ -114,7 +114,7 @@ github.com/googleapis/gax-go v2.0.0 google.golang.org/genproto 694d95ba50e67b2e363f3483057db5d4910c18f9 # containerd -github.com/containerd/containerd v1.2.0-beta.0 +github.com/containerd/containerd 3f42445e38d1081f4b8c3b8d7d1ed1860198ed7a github.com/containerd/fifo 3d5202aec260678c48179c56f40e6f38a095738c github.com/containerd/continuity d3c23511c1bf5851696cba83143d9cbcd666869b github.com/containerd/cgroups 5e610833b72089b37d0e615de9a92dfc043757c2