Merge pull request #1327 from tiborvass/cmd-builder-prune-with-options

build: add options to builder prune
This commit is contained in:
Tibor Vass 2018-09-04 22:35:11 -07:00 committed by GitHub
commit 0198955105
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 316 additions and 66 deletions

View File

@ -3,29 +3,94 @@ package builder
import (
"context"
"fmt"
"strings"
"github.com/docker/cli/cli"
"github.com/docker/cli/cli/command"
"github.com/docker/cli/opts"
"github.com/docker/docker/api/types"
units "github.com/docker/go-units"
"github.com/spf13/cobra"
)
type pruneOptions struct {
force bool
all bool
filter opts.FilterOpt
keepStorage opts.MemBytes
}
// NewPruneCommand returns a new cobra prune command for images
func NewPruneCommand(dockerCli command.Cli) *cobra.Command {
options := pruneOptions{filter: opts.NewFilterOpt()}
cmd := &cobra.Command{
Use: "prune",
Short: "Remove build cache",
Args: cli.NoArgs,
RunE: func(cmd *cobra.Command, args []string) error {
report, err := dockerCli.Client().BuildCachePrune(context.Background())
spaceReclaimed, output, err := runPrune(dockerCli, options)
if err != nil {
return err
}
fmt.Fprintln(dockerCli.Out(), "Total reclaimed space:", units.HumanSize(float64(report.SpaceReclaimed)))
if output != "" {
fmt.Fprintln(dockerCli.Out(), output)
}
fmt.Fprintln(dockerCli.Out(), "Total reclaimed space:", units.HumanSize(float64(spaceReclaimed)))
return nil
},
Annotations: map[string]string{"version": "1.39"},
}
flags := cmd.Flags()
flags.BoolVarP(&options.force, "force", "f", false, "Do not prompt for confirmation")
flags.BoolVarP(&options.all, "all", "a", false, "Remove all unused images, not just dangling ones")
flags.Var(&options.filter, "filter", "Provide filter values (e.g. 'max-age=24h')")
flags.Var(&options.keepStorage, "keep-storage", "Amount of disk space to keep for cache")
return cmd
}
const (
normalWarning = `WARNING! This will remove all dangling build cache. Are you sure you want to continue?`
allCacheWarning = `WARNING! This will remove all build cache. Are you sure you want to continue?`
)
func runPrune(dockerCli command.Cli, options pruneOptions) (spaceReclaimed uint64, output string, err error) {
pruneFilters := options.filter.Value()
pruneFilters = command.PruneFilters(dockerCli, pruneFilters)
warning := normalWarning
if options.all {
warning = allCacheWarning
}
if !options.force && !command.PromptForConfirmation(dockerCli.In(), dockerCli.Out(), warning) {
return 0, "", nil
}
report, err := dockerCli.Client().BuildCachePrune(context.Background(), types.BuildCachePruneOptions{
All: options.all,
KeepStorage: options.keepStorage.Value(),
Filters: pruneFilters,
})
if err != nil {
return 0, "", err
}
if len(report.CachesDeleted) > 0 {
var sb strings.Builder
sb.WriteString("Deleted build cache objects:\n")
for _, id := range report.CachesDeleted {
sb.WriteString(id)
sb.WriteByte('\n')
}
output = sb.String()
}
return report.SpaceReclaimed, output, nil
}
// CachePrune executes a prune command for build cache
func CachePrune(dockerCli command.Cli, all bool, filter opts.FilterOpt) (uint64, string, error) {
return runPrune(dockerCli, pruneOptions{force: true, all: all, filter: filter})
}

View File

@ -73,6 +73,6 @@ func runPrune(dockerCli command.Cli, options pruneOptions) (spaceReclaimed uint6
// RunPrune calls the Container Prune API
// This returns the amount of space reclaimed and a detailed output string
func RunPrune(dockerCli command.Cli, filter opts.FilterOpt) (uint64, string, error) {
func RunPrune(dockerCli command.Cli, all bool, filter opts.FilterOpt) (uint64, string, error) {
return runPrune(dockerCli, pruneOptions{force: true, filter: filter})
}

View File

@ -0,0 +1,165 @@
package formatter
import (
"fmt"
"sort"
"strings"
"time"
"github.com/docker/docker/api/types"
"github.com/docker/docker/pkg/stringid"
"github.com/docker/go-units"
)
const (
defaultBuildCacheTableFormat = "table {{.ID}}\t{{.Type}}\t{{.Size}}\t{{.CreatedSince}}\t{{.LastUsedSince}}\t{{.UsageCount}}\t{{.Shared}}\t{{.Description}}"
cacheIDHeader = "CACHE ID"
parentHeader = "PARENT"
lastUsedSinceHeader = "LAST USED"
usageCountHeader = "USAGE"
inUseHeader = "IN USE"
sharedHeader = "SHARED"
)
// NewBuildCacheFormat returns a Format for rendering using a Context
func NewBuildCacheFormat(source string, quiet bool) Format {
switch source {
case TableFormatKey:
if quiet {
return defaultQuietFormat
}
return Format(defaultBuildCacheTableFormat)
case RawFormatKey:
if quiet {
return `build_cache_id: {{.ID}}`
}
format := `build_cache_id: {{.ID}}
parent_id: {{.Parent}}
type: {{.Type}}
description: {{.Description}}
created_at: {{.CreatedSince}}
last_used_at: {{.LastUsedSince}}
usage_count: {{.UsageCount}}
in_use: {{.InUse}}
shared: {{.Shared}}
`
return Format(format)
}
return Format(source)
}
func buildCacheSort(buildCache []*types.BuildCache) {
sort.Slice(buildCache, func(i, j int) bool {
lui, luj := buildCache[i].LastUsedAt, buildCache[j].LastUsedAt
switch {
case lui == nil && luj == nil:
return strings.Compare(buildCache[i].ID, buildCache[j].ID) < 0
case lui == nil:
return true
case luj == nil:
return false
case lui.Equal(*luj):
return strings.Compare(buildCache[i].ID, buildCache[j].ID) < 0
default:
return lui.Before(*luj)
}
})
}
// BuildCacheWrite renders the context for a list of containers
func BuildCacheWrite(ctx Context, buildCaches []*types.BuildCache) error {
render := func(format func(subContext subContext) error) error {
buildCacheSort(buildCaches)
for _, bc := range buildCaches {
err := format(&buildCacheContext{trunc: ctx.Trunc, v: bc})
if err != nil {
return err
}
}
return nil
}
return ctx.Write(newBuildCacheContext(), render)
}
type buildCacheHeaderContext map[string]string
type buildCacheContext struct {
HeaderContext
trunc bool
v *types.BuildCache
}
func newBuildCacheContext() *buildCacheContext {
buildCacheCtx := buildCacheContext{}
buildCacheCtx.header = buildCacheHeaderContext{
"ID": cacheIDHeader,
"Parent": parentHeader,
"Type": typeHeader,
"Size": sizeHeader,
"CreatedSince": createdSinceHeader,
"LastUsedSince": lastUsedSinceHeader,
"UsageCount": usageCountHeader,
"InUse": inUseHeader,
"Shared": sharedHeader,
"Description": descriptionHeader,
}
return &buildCacheCtx
}
func (c *buildCacheContext) MarshalJSON() ([]byte, error) {
return marshalJSON(c)
}
func (c *buildCacheContext) ID() string {
id := c.v.ID
if c.trunc {
id = stringid.TruncateID(c.v.ID)
}
if c.v.InUse {
return id + "*"
}
return id
}
func (c *buildCacheContext) Parent() string {
if c.trunc {
return stringid.TruncateID(c.v.Parent)
}
return c.v.Parent
}
func (c *buildCacheContext) Type() string {
return c.v.Type
}
func (c *buildCacheContext) Description() string {
return c.v.Description
}
func (c *buildCacheContext) Size() string {
return units.HumanSizeWithPrecision(float64(c.v.Size), 3)
}
func (c *buildCacheContext) CreatedSince() string {
return units.HumanDuration(time.Now().UTC().Sub(c.v.CreatedAt)) + " ago"
}
func (c *buildCacheContext) LastUsedSince() string {
if c.v.LastUsedAt == nil {
return ""
}
return units.HumanDuration(time.Now().UTC().Sub(*c.v.LastUsedAt)) + " ago"
}
func (c *buildCacheContext) UsageCount() string {
return fmt.Sprintf("%d", c.v.UsageCount)
}
func (c *buildCacheContext) InUse() string {
return fmt.Sprintf("%t", c.v.InUse)
}
func (c *buildCacheContext) Shared() string {
return fmt.Sprintf("%t", c.v.Shared)
}

View File

@ -12,19 +12,11 @@ import (
)
const (
defaultDiskUsageImageTableFormat = "table {{.Repository}}\t{{.Tag}}\t{{.ID}}\t{{.CreatedSince}} ago\t{{.VirtualSize}}\t{{.SharedSize}}\t{{.UniqueSize}}\t{{.Containers}}"
defaultDiskUsageContainerTableFormat = "table {{.ID}}\t{{.Image}}\t{{.Command}}\t{{.LocalVolumes}}\t{{.Size}}\t{{.RunningFor}} ago\t{{.Status}}\t{{.Names}}"
defaultDiskUsageImageTableFormat = "table {{.Repository}}\t{{.Tag}}\t{{.ID}}\t{{.CreatedSince}}\t{{.VirtualSize}}\t{{.SharedSize}}\t{{.UniqueSize}}\t{{.Containers}}"
defaultDiskUsageContainerTableFormat = "table {{.ID}}\t{{.Image}}\t{{.Command}}\t{{.LocalVolumes}}\t{{.Size}}\t{{.RunningFor}}\t{{.Status}}\t{{.Names}}"
defaultDiskUsageVolumeTableFormat = "table {{.Name}}\t{{.Links}}\t{{.Size}}"
defaultDiskUsageTableFormat = "table {{.Type}}\t{{.TotalCount}}\t{{.Active}}\t{{.Size}}\t{{.Reclaimable}}"
defaultBuildCacheVerboseFormat = `
ID: {{.ID}}
Description: {{.Description}}
Mutable: {{.Mutable}}
Size: {{.Size}}
CreatedAt: {{.CreatedAt}}
LastUsedAt: {{.LastUsedAt}}
UsageCount: {{.UsageCount}}
`
defaultDiskUsageBuildCacheTableFormat = "table {{.ID}}\t{{.Type}}\t{{.Size}}\t{{.CreatedSince}}\t{{.LastUsedSince}}\t{{.UsageCount}}\t{{.Shared}}"
typeHeader = "TYPE"
totalHeader = "TOTAL"
@ -32,7 +24,7 @@ UsageCount: {{.UsageCount}}
reclaimableHeader = "RECLAIMABLE"
containersHeader = "CONTAINERS"
sharedSizeHeader = "SHARED SIZE"
uniqueSizeHeader = "UNIQUE SiZE"
uniqueSizeHeader = "UNIQUE SIZE"
)
// DiskUsageContext contains disk usage specific information required by the formatter, encapsulate a Context struct.
@ -56,7 +48,6 @@ func (ctx *DiskUsageContext) startSubsection(format string) (*template.Template,
return ctx.parseFormat()
}
//
// NewDiskUsageFormat returns a format for rendering an DiskUsageContext
func NewDiskUsageFormat(source string) Format {
switch source {
@ -129,6 +120,7 @@ func (ctx *DiskUsageContext) Write() (err error) {
return err
}
// nolint: gocyclo
func (ctx *DiskUsageContext) verboseWrite() error {
// First images
tmpl, err := ctx.startSubsection(defaultDiskUsageImageTableFormat)
@ -196,11 +188,17 @@ func (ctx *DiskUsageContext) verboseWrite() error {
// And build cache
fmt.Fprintf(ctx.Output, "\nBuild cache usage: %s\n\n", units.HumanSize(float64(ctx.BuilderSize)))
t := template.Must(template.New("buildcache").Parse(defaultBuildCacheVerboseFormat))
for _, v := range ctx.BuildCache {
t.Execute(ctx.Output, *v)
tmpl, err = ctx.startSubsection(defaultDiskUsageBuildCacheTableFormat)
if err != nil {
return err
}
buildCacheSort(ctx.BuildCache)
for _, v := range ctx.BuildCache {
if err := ctx.contextFormat(tmpl, &buildCacheContext{v: v, trunc: true}); err != nil {
return err
}
}
ctx.postFormat(tmpl, newBuildCacheContext())
return nil
}
@ -416,7 +414,7 @@ func (c *diskUsageBuilderContext) Size() string {
func (c *diskUsageBuilderContext) Reclaimable() string {
var inUseBytes int64
for _, bc := range c.buildCache {
if bc.InUse {
if bc.InUse && !bc.Shared {
inUseBytes += bc.Size
}
}

View File

@ -32,11 +32,11 @@ Build Cache 0 0 0B
DiskUsageContext{Verbose: true},
`Images space usage:
REPOSITORY TAG IMAGE ID CREATED ago SIZE SHARED SIZE UNIQUE SiZE CONTAINERS
REPOSITORY TAG IMAGE ID CREATED SIZE SHARED SIZE UNIQUE SIZE CONTAINERS
Containers space usage:
CONTAINER ID IMAGE COMMAND LOCAL VOLUMES SIZE CREATED ago STATUS NAMES
CONTAINER ID IMAGE COMMAND LOCAL VOLUMES SIZE CREATED STATUS NAMES
Local Volumes space usage:
@ -44,6 +44,7 @@ VOLUME NAME LINKS SIZE
Build cache usage: 0B
CACHE ID TYPE SIZE CREATED LAST USED USAGE SHARED
`,
},
// Errors

View File

@ -70,7 +70,7 @@ func runPrune(dockerCli command.Cli, options pruneOptions) (output string, err e
// RunPrune calls the Network Prune API
// This returns the amount of space reclaimed and a detailed output string
func RunPrune(dockerCli command.Cli, filter opts.FilterOpt) (uint64, string, error) {
func RunPrune(dockerCli command.Cli, all bool, filter opts.FilterOpt) (uint64, string, error) {
output, err := runPrune(dockerCli, pruneOptions{force: true, filter: filter})
return 0, output, err
}

View File

@ -52,13 +52,20 @@ func runDiskUsage(dockerCli command.Cli, opts diskUsageOptions) error {
format = formatter.TableFormatKey
}
var bsz int64
for _, bc := range du.BuildCache {
if !bc.Shared {
bsz += bc.Size
}
}
duCtx := formatter.DiskUsageContext{
Context: formatter.Context{
Output: dockerCli.Out(),
Format: formatter.NewDiskUsageFormat(format),
},
LayersSize: du.LayersSize,
BuilderSize: du.BuilderSize,
BuilderSize: bsz,
BuildCache: du.BuildCache,
Images: du.Images,
Containers: du.Containers,

View File

@ -2,12 +2,12 @@ package system
import (
"bytes"
"context"
"fmt"
"text/template"
"github.com/docker/cli/cli"
"github.com/docker/cli/cli/command"
"github.com/docker/cli/cli/command/builder"
"github.com/docker/cli/cli/command/container"
"github.com/docker/cli/cli/command/image"
"github.com/docker/cli/cli/command/network"
@ -21,20 +21,21 @@ import (
type pruneOptions struct {
force bool
all bool
pruneBuildCache bool
pruneVolumes bool
pruneBuildCache bool
filter opts.FilterOpt
}
// newPruneCommand creates a new cobra.Command for `docker prune`
func newPruneCommand(dockerCli command.Cli) *cobra.Command {
options := pruneOptions{filter: opts.NewFilterOpt(), pruneBuildCache: true}
options := pruneOptions{filter: opts.NewFilterOpt()}
cmd := &cobra.Command{
Use: "prune [OPTIONS]",
Short: "Remove unused data",
Args: cli.NoArgs,
RunE: func(cmd *cobra.Command, args []string) error {
options.pruneBuildCache = versions.GreaterThanOrEqualTo(dockerCli.Client().ClientVersion(), "1.31")
return runPrune(dockerCli, options)
},
Annotations: map[string]string{"version": "1.25"},
@ -57,44 +58,30 @@ const confirmationTemplate = `WARNING! This will remove:
{{- end }}
Are you sure you want to continue?`
// runBuildCachePrune executes a prune command for build cache
func runBuildCachePrune(dockerCli command.Cli, _ opts.FilterOpt) (uint64, string, error) {
report, err := dockerCli.Client().BuildCachePrune(context.Background())
if err != nil {
return 0, "", err
}
return report.SpaceReclaimed, "", nil
}
func runPrune(dockerCli command.Cli, options pruneOptions) error {
// TODO version this once "until" filter is supported for volumes
if options.pruneVolumes && options.filter.Value().Contains("until") {
return fmt.Errorf(`ERROR: The "until" filter is not supported with "--volumes"`)
}
if versions.LessThan(dockerCli.Client().ClientVersion(), "1.31") {
options.pruneBuildCache = false
}
if !options.force && !command.PromptForConfirmation(dockerCli.In(), dockerCli.Out(), confirmationMessage(options)) {
return nil
}
imagePrune := func(dockerCli command.Cli, filter opts.FilterOpt) (uint64, string, error) {
return image.RunPrune(dockerCli, options.all, options.filter)
}
pruneFuncs := []func(dockerCli command.Cli, filter opts.FilterOpt) (uint64, string, error){
pruneFuncs := []func(dockerCli command.Cli, all bool, filter opts.FilterOpt) (uint64, string, error){
container.RunPrune,
network.RunPrune,
}
if options.pruneVolumes {
pruneFuncs = append(pruneFuncs, volume.RunPrune)
}
pruneFuncs = append(pruneFuncs, imagePrune)
if options.pruneBuildCache {
pruneFuncs = append(pruneFuncs, runBuildCachePrune)
pruneFuncs = append(pruneFuncs, builder.CachePrune)
}
// FIXME: modify image.RunPrune to not modify options.filter, otherwise this has to be last in the list.
pruneFuncs = append(pruneFuncs, image.RunPrune)
var spaceReclaimed uint64
for _, pruneFn := range pruneFuncs {
spc, output, err := pruneFn(dockerCli, options.filter)
spc, output, err := pruneFn(dockerCli, options.all, options.filter)
if err != nil {
return err
}
@ -126,7 +113,11 @@ func confirmationMessage(options pruneOptions) string {
warnings = append(warnings, "all dangling images")
}
if options.pruneBuildCache {
if options.all {
warnings = append(warnings, "all build cache")
} else {
warnings = append(warnings, "all dangling build cache")
}
}
if len(options.filter.String()) > 0 {
warnings = append(warnings, "Elements to be pruned will be filtered with:")

View File

@ -73,6 +73,6 @@ func runPrune(dockerCli command.Cli, options pruneOptions) (spaceReclaimed uint6
// RunPrune calls the Volume Prune API
// This returns the amount of space reclaimed and a detailed output string
func RunPrune(dockerCli command.Cli, filter opts.FilterOpt) (uint64, string, error) {
func RunPrune(dockerCli command.Cli, all bool, filter opts.FilterOpt) (uint64, string, error) {
return runPrune(dockerCli, pruneOptions{force: true, filter: filter})
}

View File

@ -12,7 +12,7 @@ github.com/cpuguy83/go-md2man v1.0.8
github.com/davecgh/go-spew 346938d642f2ec3594ed81d874461961cd0faa76 # v1.1.0
github.com/dgrijalva/jwt-go a2c85815a77d0f951e33ba4db5ae93629a1530af
github.com/docker/distribution 83389a148052d74ac602f5f1d62f86ff2f3c4aa5
github.com/docker/docker 2629fe93266e82751af4f1c7568e21060f065b73
github.com/docker/docker 6ba1e91877691c4043b57c97090668bc4fd874b6
github.com/docker/docker-credential-helpers 5241b46610f2491efdf9d1c85f1ddf5b02f6d962
# the docker/go package contains a customized version of canonical/json
# and is used by Notary. The package is periodically rebased on current Go versions.

View File

@ -543,6 +543,7 @@ type ImagesPruneReport struct {
// BuildCachePruneReport contains the response for Engine API:
// POST "/build/prune"
type BuildCachePruneReport struct {
CachesDeleted []string
SpaceReclaimed uint64
}
@ -593,13 +594,20 @@ type BuildResult struct {
// BuildCache contains information about a build cache record
type BuildCache struct {
ID string
Mutable bool
Parent string
Type string
Description string
InUse bool
Shared bool
Size int64
CreatedAt time.Time
LastUsedAt *time.Time
UsageCount int
Parent string
Description string
}
// BuildCachePruneOptions hold parameters to prune the build cache
type BuildCachePruneOptions struct {
All bool
KeepStorage int64
Filters filters.Args
}

View File

@ -4,19 +4,34 @@ import (
"context"
"encoding/json"
"fmt"
"net/url"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/filters"
"github.com/pkg/errors"
)
// BuildCachePrune requests the daemon to delete unused cache data
func (cli *Client) BuildCachePrune(ctx context.Context) (*types.BuildCachePruneReport, error) {
func (cli *Client) BuildCachePrune(ctx context.Context, opts types.BuildCachePruneOptions) (*types.BuildCachePruneReport, error) {
if err := cli.NewVersionError("1.31", "build prune"); err != nil {
return nil, err
}
report := types.BuildCachePruneReport{}
serverResp, err := cli.post(ctx, "/build/prune", nil, nil, nil)
query := url.Values{}
if opts.All {
query.Set("all", "1")
}
query.Set("keep-storage", fmt.Sprintf("%d", opts.KeepStorage))
filters, err := filters.ToJSON(opts.Filters)
if err != nil {
return nil, errors.Wrap(err, "prune could not marshal filters option")
}
query.Set("filters", filters)
serverResp, err := cli.post(ctx, "/build/prune", query, nil, nil)
if err != nil {
return nil, err
}

View File

@ -86,7 +86,7 @@ type DistributionAPIClient interface {
// ImageAPIClient defines API client methods for the images
type ImageAPIClient interface {
ImageBuild(ctx context.Context, context io.Reader, options types.ImageBuildOptions) (types.ImageBuildResponse, error)
BuildCachePrune(ctx context.Context) (*types.BuildCachePruneReport, error)
BuildCachePrune(ctx context.Context, opts types.BuildCachePruneOptions) (*types.BuildCachePruneReport, error)
BuildCancel(ctx context.Context, id string) error
ImageCreate(ctx context.Context, parentReference string, options types.ImageCreateOptions) (io.ReadCloser, error)
ImageHistory(ctx context.Context, image string) ([]image.HistoryResponseItem, error)

View File

@ -1,7 +1,7 @@
# the following lines are in sorted order, FYI
github.com/Azure/go-ansiterm d6e3b3328b783f23731bc4d058875b0371ff8109
github.com/Microsoft/hcsshim v0.6.12
github.com/Microsoft/go-winio v0.4.9
github.com/Microsoft/hcsshim v0.6.14
github.com/Microsoft/go-winio v0.4.10
github.com/docker/libtrust 9cbd2a1374f46905c68a4eb3694a130610adc62a
github.com/go-check/check 4ed411733c5785b40214c70bce814c3a3a689609 https://github.com/cpuguy83/check.git
github.com/golang/gddo 9b12a26f3fbd7397dee4e20939ddca719d840d2a
@ -26,7 +26,7 @@ github.com/imdario/mergo v0.3.6
golang.org/x/sync 1d60e4601c6fd243af51cc01ddf169918a5407ca
# buildkit
github.com/moby/buildkit 46f9075ab68a07df2c40ae6e240ce4f9392b3a66 git://github.com/tiborvass/buildkit.git
github.com/moby/buildkit e1cd06ad6b74e4b747306c4408c451b3b6d87a89
github.com/tonistiigi/fsutil b19464cd1b6a00773b4f2eb7acf9c30426f9df42
github.com/grpc-ecosystem/grpc-opentracing 8e809c8a86450a29b90dcc9efbf062d0fe6d9746
github.com/opentracing/opentracing-go 1361b9cd60be79c4c3a7fa9841b3c132e40066a7
@ -114,7 +114,7 @@ github.com/googleapis/gax-go v2.0.0
google.golang.org/genproto 694d95ba50e67b2e363f3483057db5d4910c18f9
# containerd
github.com/containerd/containerd v1.2.0-beta.0
github.com/containerd/containerd 3f42445e38d1081f4b8c3b8d7d1ed1860198ed7a
github.com/containerd/fifo 3d5202aec260678c48179c56f40e6f38a095738c
github.com/containerd/continuity d3c23511c1bf5851696cba83143d9cbcd666869b
github.com/containerd/cgroups 5e610833b72089b37d0e615de9a92dfc043757c2