add manifest command

Enable inspection (aka "shallow pull") of images' manifest info, and
also the creation of manifest lists (aka "fat manifests").

The workflow for creating a manifest list will be:

`docker manifest create new-list-ref-name image-ref [image-ref...]`
`docker manifest annotate new-list-ref-name image-ref --os linux --arch
arm`
`docker manifest push new-list-ref-name`

The annotate step is optional. Most architectures are fine by default.

There is also a `manifest inspect` command to allow for a "shallow pull"
of an image's manifest: `docker manifest inspect
manifest-or-manifest_list`.

To be more in line with the existing external manifest tool, there is
also a `-v` option for inspect that will show information depending on
what the reference maps to (list or single manifest).

Signed-off-by: Christy Perez <christy@linux.vnet.ibm.com>
Signed-off-by: Daniel Nephin <dnephin@docker.com>

cli/command/cli.go

@@ -5,16 +5,22 @@ import (
 	"net"
 	"net/http"
 	"os"
+	"path/filepath"
 	"runtime"
 	"time"
 
 	"github.com/docker/cli/cli"
+	"github.com/docker/cli/cli/config"
 	cliconfig "github.com/docker/cli/cli/config"
 	"github.com/docker/cli/cli/config/configfile"
 	cliflags "github.com/docker/cli/cli/flags"
+	manifeststore "github.com/docker/cli/cli/manifest/store"
+	registryclient "github.com/docker/cli/cli/registry/client"
 	"github.com/docker/cli/cli/trust"
 	dopts "github.com/docker/cli/opts"
 	"github.com/docker/docker/api"
+	"github.com/docker/docker/api/types"
+	registrytypes "github.com/docker/docker/api/types/registry"
 	"github.com/docker/docker/client"
 	"github.com/docker/go-connections/sockets"
 	"github.com/docker/go-connections/tlsconfig"
@@ -45,6 +51,8 @@ type Cli interface {
 	ClientInfo() ClientInfo
 	NotaryClient(imgRefAndAuth trust.ImageRefAndAuth, actions []string) (notaryclient.Repository, error)
 	DefaultVersion() string
+	ManifestStore() manifeststore.Store
+	RegistryClient(bool) registryclient.RegistryClient
 }
 
 // DockerCli is an instance the docker command line client.
@@ -114,6 +122,21 @@ func (cli *DockerCli) ClientInfo() ClientInfo {
 	return cli.clientInfo
 }
 
+// ManifestStore returns a store for local manifests
+func (cli *DockerCli) ManifestStore() manifeststore.Store {
+	// TODO: support override default location from config file
+	return manifeststore.NewStore(filepath.Join(config.Dir(), "manifests"))
+}
+
+// RegistryClient returns a client for communicating with a Docker distribution
+// registry
+func (cli *DockerCli) RegistryClient(allowInsecure bool) registryclient.RegistryClient {
+	resolver := func(ctx context.Context, index *registrytypes.IndexInfo) types.AuthConfig {
+		return ResolveAuthConfig(ctx, cli, index)
+	}
+	return registryclient.NewRegistryClient(resolver, UserAgent(), allowInsecure)
+}
+
 // Initialize the dockerCli runs initialization that must happen after command
 // line flags are parsed.
 func (cli *DockerCli) Initialize(opts *cliflags.ClientOptions) error {

cli/command/commands/commands.go

@@ -8,6 +8,7 @@ import (
 	"github.com/docker/cli/cli/command/config"
 	"github.com/docker/cli/cli/command/container"
 	"github.com/docker/cli/cli/command/image"
+	"github.com/docker/cli/cli/command/manifest"
 	"github.com/docker/cli/cli/command/network"
 	"github.com/docker/cli/cli/command/node"
 	"github.com/docker/cli/cli/command/plugin"
@@ -39,12 +40,15 @@ func AddCommands(cmd *cobra.Command, dockerCli *command.DockerCli) {
 		image.NewImageCommand(dockerCli),
 		image.NewBuildCommand(dockerCli),
 
-		// node
+		// manifest
-		node.NewNodeCommand(dockerCli),
+		manifest.NewManifestCommand(dockerCli),
 
 		// network
 		network.NewNetworkCommand(dockerCli),
 
+		// node
+		node.NewNodeCommand(dockerCli),
+
 		// plugin
 		plugin.NewPluginCommand(dockerCli),
 

cli/command/manifest/annotate.go

@@ -0,0 +1,93 @@
+package manifest
+
+import (
+	"fmt"
+
+	"github.com/docker/cli/cli"
+	"github.com/docker/cli/cli/command"
+	"github.com/docker/cli/cli/manifest/store"
+	"github.com/pkg/errors"
+	"github.com/spf13/cobra"
+)
+
+type annotateOptions struct {
+	target     string // the target manifest list name (also transaction ID)
+	image      string // the manifest to annotate within the list
+	variant    string // an architecture variant
+	os         string
+	arch       string
+	osFeatures []string
+}
+
+// NewAnnotateCommand creates a new `docker manifest annotate` command
+func newAnnotateCommand(dockerCli command.Cli) *cobra.Command {
+	var opts annotateOptions
+
+	cmd := &cobra.Command{
+		Use:   "annotate [OPTIONS] MANIFEST_LIST MANIFEST",
+		Short: "Add additional information to a local image manifest",
+		Args:  cli.ExactArgs(2),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			opts.target = args[0]
+			opts.image = args[1]
+			return runManifestAnnotate(dockerCli, opts)
+		},
+	}
+
+	flags := cmd.Flags()
+
+	flags.StringVar(&opts.os, "os", "", "Set operating system")
+	flags.StringVar(&opts.arch, "arch", "", "Set architecture")
+	flags.StringSliceVar(&opts.osFeatures, "os-features", []string{}, "Set operating system feature")
+	flags.StringVar(&opts.variant, "variant", "", "Set architecture variant")
+
+	return cmd
+}
+
+func runManifestAnnotate(dockerCli command.Cli, opts annotateOptions) error {
+	targetRef, err := normalizeReference(opts.target)
+	if err != nil {
+		return errors.Wrapf(err, "annotate: Error parsing name for manifest list (%s): %s", opts.target)
+	}
+	imgRef, err := normalizeReference(opts.image)
+	if err != nil {
+		return errors.Wrapf(err, "annotate: Error parsing name for manifest (%s): %s:", opts.image)
+	}
+
+	manifestStore := dockerCli.ManifestStore()
+	imageManifest, err := manifestStore.Get(targetRef, imgRef)
+	switch {
+	case store.IsNotFound(err):
+		return fmt.Errorf("manifest for image %s does not exist in %s", opts.image, opts.target)
+	case err != nil:
+		return err
+	}
+
+	// Update the mf
+	if opts.os != "" {
+		imageManifest.Platform.OS = opts.os
+	}
+	if opts.arch != "" {
+		imageManifest.Platform.Architecture = opts.arch
+	}
+	for _, osFeature := range opts.osFeatures {
+		imageManifest.Platform.OSFeatures = appendIfUnique(imageManifest.Platform.OSFeatures, osFeature)
+	}
+	if opts.variant != "" {
+		imageManifest.Platform.Variant = opts.variant
+	}
+
+	if !isValidOSArch(imageManifest.Platform.OS, imageManifest.Platform.Architecture) {
+		return errors.Errorf("manifest entry for image has unsupported os/arch combination: %s/%s", opts.os, opts.arch)
+	}
+	return manifestStore.Save(targetRef, imgRef, imageManifest)
+}
+
+func appendIfUnique(list []string, str string) []string {
+	for _, s := range list {
+		if s == str {
+			return list
+		}
+	}
+	return append(list, str)
+}

cli/command/manifest/client_test.go

@@ -0,0 +1,28 @@
+package manifest
+
+import (
+	manifesttypes "github.com/docker/cli/cli/manifest/types"
+	"github.com/docker/cli/cli/registry/client"
+	"github.com/docker/distribution/reference"
+	"golang.org/x/net/context"
+)
+
+type fakeRegistryClient struct {
+	client.RegistryClient
+	getManifestFunc     func(ctx context.Context, ref reference.Named) (manifesttypes.ImageManifest, error)
+	getManifestListFunc func(ctx context.Context, ref reference.Named) ([]manifesttypes.ImageManifest, error)
+}
+
+func (c *fakeRegistryClient) GetManifest(ctx context.Context, ref reference.Named) (manifesttypes.ImageManifest, error) {
+	if c.getManifestFunc != nil {
+		return c.getManifestFunc(ctx, ref)
+	}
+	return manifesttypes.ImageManifest{}, nil
+}
+
+func (c *fakeRegistryClient) GetManifestList(ctx context.Context, ref reference.Named) ([]manifesttypes.ImageManifest, error) {
+	if c.getManifestListFunc != nil {
+		return c.getManifestListFunc(ctx, ref)
+	}
+	return nil, nil
+}

cli/command/manifest/cmd.go

@@ -0,0 +1,44 @@
+package manifest
+
+import (
+	"fmt"
+
+	"github.com/docker/cli/cli"
+	"github.com/docker/cli/cli/command"
+
+	"github.com/spf13/cobra"
+)
+
+// NewManifestCommand returns a cobra command for `manifest` subcommands
+func NewManifestCommand(dockerCli command.Cli) *cobra.Command {
+	// use dockerCli as command.Cli
+	cmd := &cobra.Command{
+		Use:   "manifest COMMAND",
+		Short: "Manage Docker image manifests and manifest lists",
+		Long:  manifestDescription,
+		Args:  cli.NoArgs,
+		Run: func(cmd *cobra.Command, args []string) {
+			fmt.Fprintf(dockerCli.Err(), "\n"+cmd.UsageString())
+		},
+	}
+	cmd.AddCommand(
+		newCreateListCommand(dockerCli),
+		newInspectCommand(dockerCli),
+		newAnnotateCommand(dockerCli),
+		newPushListCommand(dockerCli),
+	)
+	return cmd
+}
+
+var manifestDescription = `
+The **docker manifest** command has subcommands for managing image manifests and
+manifest lists. A manifest list allows you to use one name to refer to the same image
+built for multiple architectures.
+
+To see help for a subcommand, use:
+
+    docker manifest CMD --help
+
+For full details on using docker manifest lists, see the registry v2 specification.
+
+`

cli/command/manifest/create_list.go

@@ -0,0 +1,82 @@
+package manifest
+
+import (
+	"fmt"
+
+	"github.com/docker/cli/cli"
+	"github.com/docker/cli/cli/command"
+	"github.com/docker/cli/cli/manifest/store"
+	"github.com/docker/docker/registry"
+	"github.com/pkg/errors"
+	"github.com/spf13/cobra"
+	"golang.org/x/net/context"
+)
+
+type createOpts struct {
+	amend    bool
+	insecure bool
+}
+
+func newCreateListCommand(dockerCli command.Cli) *cobra.Command {
+	opts := createOpts{}
+
+	cmd := &cobra.Command{
+		Use:   "create MANFEST_LIST MANIFEST [MANIFEST...]",
+		Short: "Create a local manifest list for annotating and pushing to a registry",
+		Args:  cli.RequiresMinArgs(2),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return createManifestList(dockerCli, args, opts)
+		},
+	}
+
+	flags := cmd.Flags()
+	flags.BoolVar(&opts.insecure, "insecure", false, "allow communication with an insecure registry")
+	flags.BoolVarP(&opts.amend, "amend", "a", false, "Amend an existing manifest list")
+	return cmd
+}
+
+func createManifestList(dockerCli command.Cli, args []string, opts createOpts) error {
+	newRef := args[0]
+	targetRef, err := normalizeReference(newRef)
+	if err != nil {
+		return errors.Wrapf(err, "error parsing name for manifest list (%s): %v", newRef)
+	}
+
+	_, err = registry.ParseRepositoryInfo(targetRef)
+	if err != nil {
+		return errors.Wrapf(err, "error parsing repository name for manifest list (%s): %v", newRef)
+	}
+
+	manifestStore := dockerCli.ManifestStore()
+	_, err = manifestStore.GetList(targetRef)
+	switch {
+	case store.IsNotFound(err):
+		// New manifest list
+	case err != nil:
+		return err
+	case !opts.amend:
+		return errors.Errorf("refusing to amend an existing manifest list with no --amend flag")
+	}
+
+	ctx := context.Background()
+	// Now create the local manifest list transaction by looking up the manifest schemas
+	// for the constituent images:
+	manifests := args[1:]
+	for _, manifestRef := range manifests {
+		namedRef, err := normalizeReference(manifestRef)
+		if err != nil {
+			// TODO: wrap error?
+			return err
+		}
+
+		manifest, err := getManifest(ctx, dockerCli, targetRef, namedRef, opts.insecure)
+		if err != nil {
+			return err
+		}
+		if err := manifestStore.Save(targetRef, namedRef, manifest); err != nil {
+			return err
+		}
+	}
+	fmt.Fprintf(dockerCli.Out(), "Created manifest list %s\n", targetRef.String())
+	return nil
+}

cli/command/manifest/inspect.go

@@ -0,0 +1,147 @@
+package manifest
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+
+	"github.com/docker/cli/cli"
+	"github.com/docker/cli/cli/command"
+	"github.com/docker/cli/cli/manifest/types"
+	"github.com/docker/distribution/manifest/manifestlist"
+	"github.com/docker/distribution/reference"
+	"github.com/docker/docker/registry"
+	"github.com/spf13/cobra"
+	"golang.org/x/net/context"
+)
+
+type inspectOptions struct {
+	ref      string
+	list     string
+	verbose  bool
+	insecure bool
+}
+
+// NewInspectCommand creates a new `docker manifest inspect` command
+func newInspectCommand(dockerCli command.Cli) *cobra.Command {
+	var opts inspectOptions
+
+	cmd := &cobra.Command{
+		Use:   "inspect [OPTIONS] [MANIFEST_LIST] MANIFEST",
+		Short: "Display an image manifest, or manifest list",
+		Args:  cli.RequiresRangeArgs(1, 2),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			switch len(args) {
+			case 1:
+				opts.ref = args[0]
+			case 2:
+				opts.list = args[0]
+				opts.ref = args[1]
+			}
+			return runInspect(dockerCli, opts)
+		},
+	}
+
+	flags := cmd.Flags()
+	flags.BoolVar(&opts.insecure, "insecure", false, "allow communication with an insecure registry")
+	flags.BoolVarP(&opts.verbose, "verbose", "v", false, "Output additional info including layers and platform")
+	return cmd
+}
+
+func runInspect(dockerCli command.Cli, opts inspectOptions) error {
+	namedRef, err := normalizeReference(opts.ref)
+	if err != nil {
+		return err
+	}
+
+	// If list reference is provided, display the local manifest in a list
+	if opts.list != "" {
+		listRef, err := normalizeReference(opts.list)
+		if err != nil {
+			return err
+		}
+
+		imageManifest, err := dockerCli.ManifestStore().Get(listRef, namedRef)
+		if err != nil {
+			return err
+		}
+		return printManifest(dockerCli, imageManifest, opts)
+	}
+
+	// Try a local manifest list first
+	localManifestList, err := dockerCli.ManifestStore().GetList(namedRef)
+	if err == nil {
+		return printManifestList(dockerCli, namedRef, localManifestList, opts)
+	}
+
+	// Next try a remote manifest
+	ctx := context.Background()
+	registryClient := dockerCli.RegistryClient(opts.insecure)
+	imageManifest, err := registryClient.GetManifest(ctx, namedRef)
+	if err == nil {
+		return printManifest(dockerCli, imageManifest, opts)
+	}
+
+	// Finally try a remote manifest list
+	manifestList, err := registryClient.GetManifestList(ctx, namedRef)
+	if err != nil {
+		return err
+	}
+	return printManifestList(dockerCli, namedRef, manifestList, opts)
+}
+
+func printManifest(dockerCli command.Cli, manifest types.ImageManifest, opts inspectOptions) error {
+	buffer := new(bytes.Buffer)
+	if !opts.verbose {
+		_, raw, err := manifest.Payload()
+		if err != nil {
+			return err
+		}
+		if err := json.Indent(buffer, raw, "", "\t"); err != nil {
+			return err
+		}
+		fmt.Fprintln(dockerCli.Out(), buffer.String())
+		return nil
+	}
+	jsonBytes, err := json.MarshalIndent(manifest, "", "\t")
+	if err != nil {
+		return err
+	}
+	dockerCli.Out().Write(append(jsonBytes, '\n'))
+	return nil
+}
+
+func printManifestList(dockerCli command.Cli, namedRef reference.Named, list []types.ImageManifest, opts inspectOptions) error {
+	if !opts.verbose {
+		targetRepo, err := registry.ParseRepositoryInfo(namedRef)
+		if err != nil {
+			return err
+		}
+
+		manifests := []manifestlist.ManifestDescriptor{}
+		// More than one response. This is a manifest list.
+		for _, img := range list {
+			mfd, err := buildManifestDescriptor(targetRepo, img)
+			if err != nil {
+				return fmt.Errorf("error assembling ManifestDescriptor")
+			}
+			manifests = append(manifests, mfd)
+		}
+		deserializedML, err := manifestlist.FromDescriptors(manifests)
+		if err != nil {
+			return err
+		}
+		jsonBytes, err := deserializedML.MarshalJSON()
+		if err != nil {
+			return err
+		}
+		fmt.Fprintln(dockerCli.Out(), string(jsonBytes))
+		return nil
+	}
+	jsonBytes, err := json.MarshalIndent(list, "", "\t")
+	if err != nil {
+		return err
+	}
+	dockerCli.Out().Write(append(jsonBytes, '\n'))
+	return nil
+}

cli/command/manifest/inspect_test.go

@@ -0,0 +1,131 @@
+package manifest
+
+import (
+	"io/ioutil"
+	"os"
+	"testing"
+
+	"github.com/docker/cli/cli/manifest/store"
+	"github.com/docker/cli/cli/manifest/types"
+	manifesttypes "github.com/docker/cli/cli/manifest/types"
+	"github.com/docker/cli/internal/test"
+	"github.com/docker/distribution"
+	"github.com/docker/distribution/manifest/schema2"
+	"github.com/docker/distribution/reference"
+	"github.com/gotestyourself/gotestyourself/golden"
+	"github.com/opencontainers/go-digest"
+	"github.com/pkg/errors"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/net/context"
+)
+
+func newTempManifestStore(t *testing.T) (store.Store, func()) {
+	tmpdir, err := ioutil.TempDir("", "test-manifest-storage")
+	require.NoError(t, err)
+
+	return store.NewStore(tmpdir), func() { os.RemoveAll(tmpdir) }
+}
+
+func ref(t *testing.T, name string) reference.Named {
+	named, err := reference.ParseNamed("example.com/" + name)
+	require.NoError(t, err)
+	return named
+}
+
+func fullImageManifest(t *testing.T, ref reference.Named) types.ImageManifest {
+	man, err := schema2.FromStruct(schema2.Manifest{
+		Versioned: schema2.SchemaVersion,
+		Config: distribution.Descriptor{
+			Digest:    "sha256:7328f6f8b41890597575cbaadc884e7386ae0acc53b747401ebce5cf0d624560",
+			Size:      1520,
+			MediaType: schema2.MediaTypeImageConfig,
+		},
+		Layers: []distribution.Descriptor{
+			{
+				MediaType: schema2.MediaTypeLayer,
+				Size:      1990402,
+				Digest:    "sha256:88286f41530e93dffd4b964e1db22ce4939fffa4a4c665dab8591fbab03d4926",
+			},
+		},
+	})
+	require.NoError(t, err)
+	// TODO: include image data for verbose inspect
+	return types.NewImageManifest(ref, digest.Digest("abcd"), types.Image{}, man)
+}
+
+func TestInspectCommandLocalManifestNotFound(t *testing.T) {
+	store, cleanup := newTempManifestStore(t)
+	defer cleanup()
+
+	cli := test.NewFakeCli(nil)
+	cli.SetManifestStore(store)
+
+	cmd := newInspectCommand(cli)
+	cmd.SetOutput(ioutil.Discard)
+	cmd.SetArgs([]string{"example.com/list:v1", "example.com/alpine:3.0"})
+	err := cmd.Execute()
+	assert.EqualError(t, err, "No such manifest: example.com/alpine:3.0")
+}
+
+func TestInspectCommandNotFound(t *testing.T) {
+	store, cleanup := newTempManifestStore(t)
+	defer cleanup()
+
+	cli := test.NewFakeCli(nil)
+	cli.SetManifestStore(store)
+	cli.SetRegistryClient(&fakeRegistryClient{
+		getManifestFunc: func(_ context.Context, _ reference.Named) (manifesttypes.ImageManifest, error) {
+			return manifesttypes.ImageManifest{}, errors.New("missing")
+		},
+		getManifestListFunc: func(ctx context.Context, ref reference.Named) ([]manifesttypes.ImageManifest, error) {
+			return nil, errors.Errorf("No such manifest: %s", ref)
+		},
+	})
+
+	cmd := newInspectCommand(cli)
+	cmd.SetOutput(ioutil.Discard)
+	cmd.SetArgs([]string{"example.com/alpine:3.0"})
+	err := cmd.Execute()
+	assert.EqualError(t, err, "No such manifest: example.com/alpine:3.0")
+}
+
+func TestInspectCommandLocalManifest(t *testing.T) {
+	store, cleanup := newTempManifestStore(t)
+	defer cleanup()
+
+	cli := test.NewFakeCli(nil)
+	cli.SetManifestStore(store)
+	namedRef := ref(t, "alpine:3.0")
+	imageManifest := fullImageManifest(t, namedRef)
+	err := store.Save(ref(t, "list:v1"), namedRef, imageManifest)
+	require.NoError(t, err)
+
+	cmd := newInspectCommand(cli)
+	cmd.SetArgs([]string{"example.com/list:v1", "example.com/alpine:3.0"})
+	require.NoError(t, cmd.Execute())
+	actual := cli.OutBuffer()
+	expected := golden.Get(t, "inspect-manifest.golden")
+	assert.Equal(t, string(expected), actual.String())
+}
+
+func TestInspectcommandRemoteManifest(t *testing.T) {
+	store, cleanup := newTempManifestStore(t)
+	defer cleanup()
+
+	cli := test.NewFakeCli(nil)
+	cli.SetManifestStore(store)
+	cli.SetRegistryClient(&fakeRegistryClient{
+		getManifestFunc: func(_ context.Context, ref reference.Named) (manifesttypes.ImageManifest, error) {
+			return fullImageManifest(t, ref), nil
+		},
+	})
+
+	cmd := newInspectCommand(cli)
+	cmd.SetOutput(ioutil.Discard)
+	cmd.SetArgs([]string{"example.com/alpine:3.0"})
+	require.NoError(t, cmd.Execute())
+	actual := cli.OutBuffer()
+	expected := golden.Get(t, "inspect-manifest.golden")
+	assert.Equal(t, string(expected), actual.String())
+}

cli/command/manifest/push.go

@@ -0,0 +1,272 @@
+package manifest
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+
+	"github.com/docker/cli/cli"
+	"github.com/docker/cli/cli/command"
+	"github.com/docker/cli/cli/manifest/types"
+	registryclient "github.com/docker/cli/cli/registry/client"
+	"github.com/docker/distribution/manifest/manifestlist"
+	"github.com/docker/distribution/manifest/schema2"
+	"github.com/docker/distribution/reference"
+	"github.com/docker/docker/registry"
+	"github.com/pkg/errors"
+	"github.com/spf13/cobra"
+	"golang.org/x/net/context"
+)
+
+type pushOpts struct {
+	insecure bool
+	purge    bool
+	target   string
+}
+
+type mountRequest struct {
+	ref      reference.Named
+	manifest types.ImageManifest
+}
+
+type manifestBlob struct {
+	canonical reference.Canonical
+	os        string
+}
+
+type pushRequest struct {
+	targetRef     reference.Named
+	list          *manifestlist.DeserializedManifestList
+	mountRequests []mountRequest
+	manifestBlobs []manifestBlob
+	insecure      bool
+}
+
+func newPushListCommand(dockerCli command.Cli) *cobra.Command {
+	opts := pushOpts{}
+
+	cmd := &cobra.Command{
+		Use:   "push [OPTIONS] MANIFEST_LIST",
+		Short: "Push a manifest list to a repository",
+		Args:  cli.ExactArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			opts.target = args[0]
+			return runPush(dockerCli, opts)
+		},
+	}
+
+	flags := cmd.Flags()
+	flags.BoolVarP(&opts.purge, "purge", "p", false, "Remove the local manifest list after push")
+	flags.BoolVar(&opts.insecure, "insecure", false, "Allow push to an insecure registry")
+	return cmd
+}
+
+func runPush(dockerCli command.Cli, opts pushOpts) error {
+
+	targetRef, err := normalizeReference(opts.target)
+	if err != nil {
+		return err
+	}
+
+	manifests, err := dockerCli.ManifestStore().GetList(targetRef)
+	if err != nil {
+		return err
+	}
+	if len(manifests) == 0 {
+		return errors.Errorf("%s not found", targetRef)
+	}
+
+	pushRequest, err := buildPushRequest(manifests, targetRef, opts.insecure)
+	if err != nil {
+		return err
+	}
+
+	ctx := context.Background()
+	if err := pushList(ctx, dockerCli, pushRequest); err != nil {
+		return err
+	}
+	if opts.purge {
+		return dockerCli.ManifestStore().Remove(targetRef)
+	}
+	return nil
+}
+
+func buildPushRequest(manifests []types.ImageManifest, targetRef reference.Named, insecure bool) (pushRequest, error) {
+	req := pushRequest{targetRef: targetRef, insecure: insecure}
+
+	var err error
+	req.list, err = buildManifestList(manifests, targetRef)
+	if err != nil {
+		return req, err
+	}
+
+	targetRepo, err := registry.ParseRepositoryInfo(targetRef)
+	if err != nil {
+		return req, err
+	}
+	targetRepoName, err := registryclient.RepoNameForReference(targetRepo.Name)
+	if err != nil {
+		return req, err
+	}
+
+	for _, imageManifest := range manifests {
+		manifestRepoName, err := registryclient.RepoNameForReference(imageManifest.Ref)
+		if err != nil {
+			return req, err
+		}
+
+		repoName, _ := reference.WithName(manifestRepoName)
+		if repoName.Name() != targetRepoName {
+			blobs, err := buildBlobRequestList(imageManifest, repoName)
+			if err != nil {
+				return req, err
+			}
+			req.manifestBlobs = append(req.manifestBlobs, blobs...)
+
+			manifestPush, err := buildPutManifestRequest(imageManifest, targetRef)
+			if err != nil {
+				return req, err
+			}
+			req.mountRequests = append(req.mountRequests, manifestPush)
+		}
+	}
+	return req, nil
+}
+
+func buildManifestList(manifests []types.ImageManifest, targetRef reference.Named) (*manifestlist.DeserializedManifestList, error) {
+	targetRepoInfo, err := registry.ParseRepositoryInfo(targetRef)
+	if err != nil {
+		return nil, err
+	}
+
+	descriptors := []manifestlist.ManifestDescriptor{}
+	for _, imageManifest := range manifests {
+		if imageManifest.Platform.Architecture == "" || imageManifest.Platform.OS == "" {
+			return nil, errors.Errorf(
+				"manifest %s must have an OS and Architecture to be pushed to a registry", imageManifest.Ref)
+		}
+		descriptor, err := buildManifestDescriptor(targetRepoInfo, imageManifest)
+		if err != nil {
+			return nil, err
+		}
+		descriptors = append(descriptors, descriptor)
+	}
+
+	return manifestlist.FromDescriptors(descriptors)
+}
+
+func buildManifestDescriptor(targetRepo *registry.RepositoryInfo, imageManifest types.ImageManifest) (manifestlist.ManifestDescriptor, error) {
+	repoInfo, err := registry.ParseRepositoryInfo(imageManifest.Ref)
+	if err != nil {
+		return manifestlist.ManifestDescriptor{}, err
+	}
+
+	manifestRepoHostname := reference.Domain(repoInfo.Name)
+	targetRepoHostname := reference.Domain(targetRepo.Name)
+	if manifestRepoHostname != targetRepoHostname {
+		return manifestlist.ManifestDescriptor{}, errors.Errorf("cannot use source images from a different registry than the target image: %s != %s", manifestRepoHostname, targetRepoHostname)
+	}
+
+	mediaType, raw, err := imageManifest.Payload()
+	if err != nil {
+		return manifestlist.ManifestDescriptor{}, err
+	}
+
+	manifest := manifestlist.ManifestDescriptor{
+		Platform: imageManifest.Platform,
+	}
+	manifest.Descriptor.Digest = imageManifest.Digest
+	manifest.Size = int64(len(raw))
+	manifest.MediaType = mediaType
+
+	if err = manifest.Descriptor.Digest.Validate(); err != nil {
+		return manifestlist.ManifestDescriptor{}, errors.Wrapf(err,
+			"digest parse of image %q failed with error: %v", imageManifest.Ref)
+	}
+
+	return manifest, nil
+}
+
+func buildBlobRequestList(imageManifest types.ImageManifest, repoName reference.Named) ([]manifestBlob, error) {
+	var blobReqs []manifestBlob
+
+	for _, blobDigest := range imageManifest.Blobs() {
+		canonical, err := reference.WithDigest(repoName, blobDigest)
+		if err != nil {
+			return nil, err
+		}
+		blobReqs = append(blobReqs, manifestBlob{canonical: canonical, os: imageManifest.Platform.OS})
+	}
+	return blobReqs, nil
+}
+
+func buildPutManifestRequest(imageManifest types.ImageManifest, targetRef reference.Named) (mountRequest, error) {
+	refWithoutTag, err := reference.WithName(targetRef.Name())
+	if err != nil {
+		return mountRequest{}, err
+	}
+	mountRef, err := reference.WithDigest(refWithoutTag, imageManifest.Digest)
+	if err != nil {
+		return mountRequest{}, err
+	}
+
+	// This indentation has to be added to ensure sha parity with the registry
+	v2ManifestBytes, err := json.MarshalIndent(imageManifest.SchemaV2Manifest, "", "   ")
+	if err != nil {
+		return mountRequest{}, err
+	}
+	// indent only the DeserializedManifest portion of this, in order to maintain parity with the registry
+	// and not alter the sha
+	var v2Manifest schema2.DeserializedManifest
+	if err = v2Manifest.UnmarshalJSON(v2ManifestBytes); err != nil {
+		return mountRequest{}, err
+	}
+	imageManifest.SchemaV2Manifest = &v2Manifest
+
+	return mountRequest{ref: mountRef, manifest: imageManifest}, err
+}
+
+func pushList(ctx context.Context, dockerCli command.Cli, req pushRequest) error {
+	rclient := dockerCli.RegistryClient(req.insecure)
+
+	if err := mountBlobs(ctx, rclient, req.targetRef, req.manifestBlobs); err != nil {
+		return err
+	}
+	if err := pushReferences(ctx, dockerCli.Out(), rclient, req.mountRequests); err != nil {
+		return err
+	}
+	dgst, err := rclient.PutManifest(ctx, req.targetRef, req.list)
+	if err != nil {
+		return err
+	}
+
+	fmt.Fprintln(dockerCli.Out(), dgst.String())
+	return nil
+}
+
+func pushReferences(ctx context.Context, out io.Writer, client registryclient.RegistryClient, mounts []mountRequest) error {
+	for _, mount := range mounts {
+		newDigest, err := client.PutManifest(ctx, mount.ref, mount.manifest)
+		if err != nil {
+			return err
+		}
+		fmt.Fprintf(out, "Pushed ref %s with digest: %s\n", mount.ref, newDigest)
+	}
+	return nil
+}
+
+func mountBlobs(ctx context.Context, client registryclient.RegistryClient, ref reference.Named, blobs []manifestBlob) error {
+	for _, blob := range blobs {
+		err := client.MountBlob(ctx, blob.canonical, ref)
+		switch err.(type) {
+		case nil:
+		case registryclient.ErrBlobCreated:
+			if blob.os != "windows" {
+				return fmt.Errorf("error mounting %s to %s", blob.canonical, ref)
+			}
+		default:
+			return err
+		}
+	}
+	return nil
+}

cli/command/manifest/testdata/inspect-manifest.golden

@@ -0,0 +1,16 @@
+{
+	"schemaVersion": 2,
+	"mediaType": "application/vnd.docker.distribution.manifest.v2+json",
+	"config": {
+		"mediaType": "application/vnd.docker.container.image.v1+json",
+		"size": 1520,
+		"digest": "sha256:7328f6f8b41890597575cbaadc884e7386ae0acc53b747401ebce5cf0d624560"
+	},
+	"layers": [
+		{
+			"mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
+			"size": 1990402,
+			"digest": "sha256:88286f41530e93dffd4b964e1db22ce4939fffa4a4c665dab8591fbab03d4926"
+		}
+	]
+}

cli/command/manifest/util.go

@@ -0,0 +1,79 @@
+package manifest
+
+import (
+	"github.com/docker/cli/cli/command"
+	"github.com/docker/cli/cli/manifest/store"
+	"github.com/docker/cli/cli/manifest/types"
+	"github.com/docker/distribution/reference"
+	"golang.org/x/net/context"
+)
+
+type osArch struct {
+	os   string
+	arch string
+}
+
+// Remove any unsupported os/arch combo
+// list of valid os/arch values (see "Optional Environment Variables" section
+// of https://golang.org/doc/install/source
+// Added linux/s390x as we know System z support already exists
+var validOSArches = map[osArch]bool{
+	{os: "darwin", arch: "386"}:      true,
+	{os: "darwin", arch: "amd64"}:    true,
+	{os: "darwin", arch: "arm"}:      true,
+	{os: "darwin", arch: "arm64"}:    true,
+	{os: "dragonfly", arch: "amd64"}: true,
+	{os: "freebsd", arch: "386"}:     true,
+	{os: "freebsd", arch: "amd64"}:   true,
+	{os: "freebsd", arch: "arm"}:     true,
+	{os: "linux", arch: "386"}:       true,
+	{os: "linux", arch: "amd64"}:     true,
+	{os: "linux", arch: "arm"}:       true,
+	{os: "linux", arch: "arm64"}:     true,
+	{os: "linux", arch: "ppc64le"}:   true,
+	{os: "linux", arch: "mips64"}:    true,
+	{os: "linux", arch: "mips64le"}:  true,
+	{os: "linux", arch: "s390x"}:     true,
+	{os: "netbsd", arch: "386"}:      true,
+	{os: "netbsd", arch: "amd64"}:    true,
+	{os: "netbsd", arch: "arm"}:      true,
+	{os: "openbsd", arch: "386"}:     true,
+	{os: "openbsd", arch: "amd64"}:   true,
+	{os: "openbsd", arch: "arm"}:     true,
+	{os: "plan9", arch: "386"}:       true,
+	{os: "plan9", arch: "amd64"}:     true,
+	{os: "solaris", arch: "amd64"}:   true,
+	{os: "windows", arch: "386"}:     true,
+	{os: "windows", arch: "amd64"}:   true,
+}
+
+func isValidOSArch(os string, arch string) bool {
+	// check for existence of this combo
+	_, ok := validOSArches[osArch{os, arch}]
+	return ok
+}
+
+func normalizeReference(ref string) (reference.Named, error) {
+	namedRef, err := reference.ParseNormalizedNamed(ref)
+	if err != nil {
+		return nil, err
+	}
+	if _, isDigested := namedRef.(reference.Canonical); !isDigested {
+		return reference.TagNameOnly(namedRef), nil
+	}
+	return namedRef, nil
+}
+
+// getManifest from the local store, and fallback to the remote registry if it
+//  doesn't exist locally
+func getManifest(ctx context.Context, dockerCli command.Cli, listRef, namedRef reference.Named, insecure bool) (types.ImageManifest, error) {
+	data, err := dockerCli.ManifestStore().Get(listRef, namedRef)
+	switch {
+	case store.IsNotFound(err):
+		return dockerCli.RegistryClient(insecure).GetManifest(ctx, namedRef)
+	case err != nil:
+		return types.ImageManifest{}, err
+	default:
+		return data, nil
+	}
+}

cli/command/registry.go

@@ -10,14 +10,13 @@ import (
 	"runtime"
 	"strings"
 
-	"golang.org/x/net/context"
-
 	"github.com/docker/distribution/reference"
 	"github.com/docker/docker/api/types"
 	registrytypes "github.com/docker/docker/api/types/registry"
 	"github.com/docker/docker/pkg/term"
 	"github.com/docker/docker/registry"
 	"github.com/pkg/errors"
+	"golang.org/x/net/context"
 )
 
 // ElectAuthServer returns the default registry to use (by asking the daemon)

cli/manifest/store/store.go

@@ -0,0 +1,147 @@
+package store
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/docker/cli/cli/manifest/types"
+	"github.com/docker/distribution/reference"
+)
+
+// Store manages local storage of image distribution manifests
+type Store interface {
+	Remove(listRef reference.Reference) error
+	Get(listRef reference.Reference, manifest reference.Reference) (types.ImageManifest, error)
+	GetList(listRef reference.Reference) ([]types.ImageManifest, error)
+	Save(listRef reference.Reference, manifest reference.Reference, image types.ImageManifest) error
+}
+
+// fsStore manages manifest files stored on the local filesystem
+type fsStore struct {
+	root string
+}
+
+// NewStore returns a new store for a local file path
+func NewStore(root string) Store {
+	return &fsStore{root: root}
+}
+
+// Remove a manifest list from local storage
+func (s *fsStore) Remove(listRef reference.Reference) error {
+	path := filepath.Join(s.root, makeFilesafeName(listRef.String()))
+	return os.RemoveAll(path)
+}
+
+// Get returns the local manifest
+func (s *fsStore) Get(listRef reference.Reference, manifest reference.Reference) (types.ImageManifest, error) {
+	filename := manifestToFilename(s.root, listRef.String(), manifest.String())
+	return s.getFromFilename(manifest, filename)
+}
+
+func (s *fsStore) getFromFilename(ref reference.Reference, filename string) (types.ImageManifest, error) {
+	bytes, err := ioutil.ReadFile(filename)
+	switch {
+	case os.IsNotExist(err):
+		return types.ImageManifest{}, newNotFoundError(ref.String())
+	case err != nil:
+		return types.ImageManifest{}, err
+	}
+	var manifestInfo types.ImageManifest
+	return manifestInfo, json.Unmarshal(bytes, &manifestInfo)
+}
+
+// GetList returns all the local manifests for a transaction
+func (s *fsStore) GetList(listRef reference.Reference) ([]types.ImageManifest, error) {
+	filenames, err := s.listManifests(listRef.String())
+	switch {
+	case err != nil:
+		return nil, err
+	case filenames == nil:
+		return nil, newNotFoundError(listRef.String())
+	}
+
+	manifests := []types.ImageManifest{}
+	for _, filename := range filenames {
+		filename = filepath.Join(s.root, makeFilesafeName(listRef.String()), filename)
+		manifest, err := s.getFromFilename(listRef, filename)
+		if err != nil {
+			return nil, err
+		}
+		manifests = append(manifests, manifest)
+	}
+	return manifests, nil
+}
+
+// listManifests stored in a transaction
+func (s *fsStore) listManifests(transaction string) ([]string, error) {
+	transactionDir := filepath.Join(s.root, makeFilesafeName(transaction))
+	fileInfos, err := ioutil.ReadDir(transactionDir)
+	switch {
+	case os.IsNotExist(err):
+		return nil, nil
+	case err != nil:
+		return nil, err
+	}
+
+	filenames := []string{}
+	for _, info := range fileInfos {
+		filenames = append(filenames, info.Name())
+	}
+	return filenames, nil
+}
+
+// Save a manifest as part of a local manifest list
+func (s *fsStore) Save(listRef reference.Reference, manifest reference.Reference, image types.ImageManifest) error {
+	if err := s.createManifestListDirectory(listRef.String()); err != nil {
+		return err
+	}
+	filename := manifestToFilename(s.root, listRef.String(), manifest.String())
+	bytes, err := json.Marshal(image)
+	if err != nil {
+		return err
+	}
+	return ioutil.WriteFile(filename, bytes, 0644)
+}
+
+func (s *fsStore) createManifestListDirectory(transaction string) error {
+	path := filepath.Join(s.root, makeFilesafeName(transaction))
+	return os.MkdirAll(path, 0755)
+}
+
+func manifestToFilename(root, manifestList, manifest string) string {
+	return filepath.Join(root, makeFilesafeName(manifestList), makeFilesafeName(manifest))
+}
+
+func makeFilesafeName(ref string) string {
+	fileName := strings.Replace(ref, ":", "-", -1)
+	return strings.Replace(fileName, "/", "_", -1)
+}
+
+type notFoundError struct {
+	object string
+}
+
+func newNotFoundError(ref string) *notFoundError {
+	return &notFoundError{object: ref}
+}
+
+func (n *notFoundError) Error() string {
+	return fmt.Sprintf("No such manifest: %s", n.object)
+}
+
+// NotFound interface
+func (n *notFoundError) NotFound() {}
+
+// IsNotFound returns true if the error is a not found error
+func IsNotFound(err error) bool {
+	_, ok := err.(notFound)
+	return ok
+}
+
+type notFound interface {
+	NotFound()
+}

cli/manifest/store/store_test.go

@@ -0,0 +1,131 @@
+package store
+
+import (
+	"io/ioutil"
+	"os"
+	"testing"
+
+	"github.com/docker/cli/cli/manifest/types"
+	"github.com/docker/distribution/reference"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+type fakeRef struct {
+	name string
+}
+
+func (f fakeRef) String() string {
+	return f.name
+}
+
+func (f fakeRef) Name() string {
+	return f.name
+}
+
+func ref(name string) fakeRef {
+	return fakeRef{name: name}
+}
+
+func sref(t *testing.T, name string) *types.SerializableNamed {
+	named, err := reference.ParseNamed("example.com/" + name)
+	require.NoError(t, err)
+	return &types.SerializableNamed{Named: named}
+}
+
+func newTestStore(t *testing.T) (Store, func()) {
+	tmpdir, err := ioutil.TempDir("", "manifest-store-test")
+	require.NoError(t, err)
+
+	return NewStore(tmpdir), func() { os.RemoveAll(tmpdir) }
+}
+
+func getFiles(t *testing.T, store Store) []os.FileInfo {
+	infos, err := ioutil.ReadDir(store.(*fsStore).root)
+	require.NoError(t, err)
+	return infos
+}
+
+func TestStoreRemove(t *testing.T) {
+	store, cleanup := newTestStore(t)
+	defer cleanup()
+
+	listRef := ref("list")
+	data := types.ImageManifest{Ref: sref(t, "abcdef")}
+	require.NoError(t, store.Save(listRef, ref("manifest"), data))
+	require.Len(t, getFiles(t, store), 1)
+
+	assert.NoError(t, store.Remove(listRef))
+	assert.Len(t, getFiles(t, store), 0)
+}
+
+func TestStoreSaveAndGet(t *testing.T) {
+	store, cleanup := newTestStore(t)
+	defer cleanup()
+
+	listRef := ref("list")
+	data := types.ImageManifest{Ref: sref(t, "abcdef")}
+	err := store.Save(listRef, ref("exists"), data)
+	require.NoError(t, err)
+
+	var testcases = []struct {
+		listRef     reference.Reference
+		manifestRef reference.Reference
+		expected    types.ImageManifest
+		expectedErr string
+	}{
+		{
+			listRef:     listRef,
+			manifestRef: ref("exists"),
+			expected:    data,
+		},
+		{
+			listRef:     listRef,
+			manifestRef: ref("exist:does-not"),
+			expectedErr: "No such manifest: exist:does-not",
+		},
+		{
+			listRef:     ref("list:does-not-exist"),
+			manifestRef: ref("manifest:does-not-exist"),
+			expectedErr: "No such manifest: manifest:does-not-exist",
+		},
+	}
+
+	for _, testcase := range testcases {
+		actual, err := store.Get(testcase.listRef, testcase.manifestRef)
+		if testcase.expectedErr != "" {
+			assert.EqualError(t, err, testcase.expectedErr)
+			assert.True(t, IsNotFound(err))
+			continue
+		}
+		if !assert.NoError(t, err, testcase.manifestRef.String()) {
+			continue
+		}
+		assert.Equal(t, testcase.expected, actual, testcase.manifestRef.String())
+	}
+}
+
+func TestStoreGetList(t *testing.T) {
+	store, cleanup := newTestStore(t)
+	defer cleanup()
+
+	listRef := ref("list")
+	first := types.ImageManifest{Ref: sref(t, "first")}
+	require.NoError(t, store.Save(listRef, ref("first"), first))
+	second := types.ImageManifest{Ref: sref(t, "second")}
+	require.NoError(t, store.Save(listRef, ref("exists"), second))
+
+	list, err := store.GetList(listRef)
+	require.NoError(t, err)
+	assert.Len(t, list, 2)
+}
+
+func TestStoreGetListDoesNotExist(t *testing.T) {
+	store, cleanup := newTestStore(t)
+	defer cleanup()
+
+	listRef := ref("list")
+	_, err := store.GetList(listRef)
+	assert.EqualError(t, err, "No such manifest: list")
+	assert.True(t, IsNotFound(err))
+}

cli/manifest/types/types.go

@@ -0,0 +1,107 @@
+package types
+
+import (
+	"encoding/json"
+
+	"github.com/docker/distribution"
+	"github.com/docker/distribution/manifest/manifestlist"
+	"github.com/docker/distribution/manifest/schema2"
+	"github.com/docker/distribution/reference"
+	"github.com/opencontainers/go-digest"
+	"github.com/pkg/errors"
+)
+
+// ImageManifest contains info to output for a manifest object.
+type ImageManifest struct {
+	Ref              *SerializableNamed
+	Digest           digest.Digest
+	SchemaV2Manifest *schema2.DeserializedManifest `json:",omitempty"`
+	Platform         manifestlist.PlatformSpec
+}
+
+// Blobs returns the digests for all the blobs referenced by this manifest
+func (i ImageManifest) Blobs() []digest.Digest {
+	digests := []digest.Digest{}
+	for _, descriptor := range i.SchemaV2Manifest.References() {
+		digests = append(digests, descriptor.Digest)
+	}
+	return digests
+}
+
+// Payload returns the media type and bytes for the manifest
+func (i ImageManifest) Payload() (string, []byte, error) {
+	switch {
+	case i.SchemaV2Manifest != nil:
+		return i.SchemaV2Manifest.Payload()
+	default:
+		return "", nil, errors.Errorf("%s has no payload", i.Ref)
+	}
+}
+
+// References implements the distribution.Manifest interface. It delegates to
+// the underlying manifest.
+func (i ImageManifest) References() []distribution.Descriptor {
+	switch {
+	case i.SchemaV2Manifest != nil:
+		return i.SchemaV2Manifest.References()
+	default:
+		return nil
+	}
+}
+
+// NewImageManifest returns a new ImageManifest object. The values for Platform
+// are initialized from those in the image
+func NewImageManifest(ref reference.Named, digest digest.Digest, img Image, manifest *schema2.DeserializedManifest) ImageManifest {
+	platform := manifestlist.PlatformSpec{
+		OS:           img.OS,
+		Architecture: img.Architecture,
+		OSVersion:    img.OSVersion,
+		OSFeatures:   img.OSFeatures,
+	}
+	return ImageManifest{
+		Ref:              &SerializableNamed{Named: ref},
+		Digest:           digest,
+		SchemaV2Manifest: manifest,
+		Platform:         platform,
+	}
+}
+
+// SerializableNamed is a reference.Named that can be serialzied and deserialized
+// from JSON
+type SerializableNamed struct {
+	reference.Named
+}
+
+// UnmarshalJSON loads the Named reference from JSON bytes
+func (s *SerializableNamed) UnmarshalJSON(b []byte) error {
+	var raw string
+	if err := json.Unmarshal(b, &raw); err != nil {
+		return errors.Wrapf(err, "invalid named reference bytes: %s", b)
+	}
+	var err error
+	s.Named, err = reference.ParseNamed(raw)
+	return err
+}
+
+// MarshalJSON returns the JSON bytes representation
+func (s *SerializableNamed) MarshalJSON() ([]byte, error) {
+	return json.Marshal(s.String())
+}
+
+// Image is the minimal set of fields required to set default platform settings
+// on a manifest.
+type Image struct {
+	Architecture string   `json:"architecture,omitempty"`
+	OS           string   `json:"os,omitempty"`
+	OSVersion    string   `json:"os.version,omitempty"`
+	OSFeatures   []string `json:"os.features,omitempty"`
+}
+
+// NewImageFromJSON creates an Image configuration from json.
+func NewImageFromJSON(src []byte) (*Image, error) {
+	img := &Image{}
+	if err := json.Unmarshal(src, img); err != nil {
+		return nil, err
+	}
+	return img, nil
+}

cli/registry/client/client.go

@@ -0,0 +1,183 @@
+package client
+
+import (
+	"fmt"
+	"net/http"
+	"strings"
+
+	manifesttypes "github.com/docker/cli/cli/manifest/types"
+	"github.com/docker/distribution"
+	"github.com/docker/distribution/reference"
+	distributionclient "github.com/docker/distribution/registry/client"
+	"github.com/docker/docker/api/types"
+	registrytypes "github.com/docker/docker/api/types/registry"
+	"github.com/opencontainers/go-digest"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+	"golang.org/x/net/context"
+)
+
+// RegistryClient is a client used to communicate with a Docker distribution
+// registry
+type RegistryClient interface {
+	GetManifest(ctx context.Context, ref reference.Named) (manifesttypes.ImageManifest, error)
+	GetManifestList(ctx context.Context, ref reference.Named) ([]manifesttypes.ImageManifest, error)
+	MountBlob(ctx context.Context, source reference.Canonical, target reference.Named) error
+	PutManifest(ctx context.Context, ref reference.Named, manifest distribution.Manifest) (digest.Digest, error)
+}
+
+// NewRegistryClient returns a new RegistryClient with a resolver
+func NewRegistryClient(resolver AuthConfigResolver, userAgent string, insecure bool) RegistryClient {
+	return &client{
+		authConfigResolver: resolver,
+		insecureRegistry:   insecure,
+		userAgent:          userAgent,
+	}
+}
+
+// AuthConfigResolver returns Auth Configuration for an index
+type AuthConfigResolver func(ctx context.Context, index *registrytypes.IndexInfo) types.AuthConfig
+
+// PutManifestOptions is the data sent to push a manifest
+type PutManifestOptions struct {
+	MediaType string
+	Payload   []byte
+}
+
+type client struct {
+	authConfigResolver AuthConfigResolver
+	insecureRegistry   bool
+	userAgent          string
+}
+
+// ErrBlobCreated returned when a blob mount request was created
+type ErrBlobCreated struct {
+	From   reference.Named
+	Target reference.Named
+}
+
+func (err ErrBlobCreated) Error() string {
+	return fmt.Sprintf("blob mounted from: %v to: %v",
+		err.From, err.Target)
+}
+
+// ErrHTTPProto returned if attempting to use TLS with a non-TLS registry
+type ErrHTTPProto struct {
+	OrigErr string
+}
+
+func (err ErrHTTPProto) Error() string {
+	return err.OrigErr
+}
+
+var _ RegistryClient = &client{}
+
+// MountBlob into the registry, so it can be referenced by a manifest
+func (c *client) MountBlob(ctx context.Context, sourceRef reference.Canonical, targetRef reference.Named) error {
+	repoEndpoint, err := newDefaultRepositoryEndpoint(targetRef, c.insecureRegistry)
+	if err != nil {
+		return err
+	}
+	repo, err := c.getRepositoryForReference(ctx, targetRef, repoEndpoint)
+	if err != nil {
+		return err
+	}
+	lu, err := repo.Blobs(ctx).Create(ctx, distributionclient.WithMountFrom(sourceRef))
+	switch err.(type) {
+	case distribution.ErrBlobMounted:
+		logrus.Debugf("mount of blob %s succeeded", sourceRef)
+		return nil
+	case nil:
+	default:
+		return errors.Wrapf(err, "failed to mount blob %s to %s", sourceRef, targetRef)
+	}
+	lu.Cancel(ctx)
+	logrus.Debugf("mount of blob %s created", sourceRef)
+	return ErrBlobCreated{From: sourceRef, Target: targetRef}
+}
+
+// PutManifest sends the manifest to a registry and returns the new digest
+func (c *client) PutManifest(ctx context.Context, ref reference.Named, manifest distribution.Manifest) (digest.Digest, error) {
+	repoEndpoint, err := newDefaultRepositoryEndpoint(ref, c.insecureRegistry)
+	if err != nil {
+		return digest.Digest(""), err
+	}
+
+	repo, err := c.getRepositoryForReference(ctx, ref, repoEndpoint)
+	if err != nil {
+		return digest.Digest(""), err
+	}
+
+	manifestService, err := repo.Manifests(ctx)
+	if err != nil {
+		return digest.Digest(""), err
+	}
+
+	_, opts, err := getManifestOptionsFromReference(ref)
+	if err != nil {
+		return digest.Digest(""), err
+	}
+
+	dgst, err := manifestService.Put(ctx, manifest, opts...)
+	return dgst, errors.Wrapf(err, "failed to put manifest %s", ref)
+}
+
+func (c *client) getRepositoryForReference(ctx context.Context, ref reference.Named, repoEndpoint repositoryEndpoint) (distribution.Repository, error) {
+	httpTransport, err := c.getHTTPTransportForRepoEndpoint(ctx, repoEndpoint)
+	if err != nil {
+		if strings.Contains(err.Error(), "server gave HTTP response to HTTPS client") {
+			return nil, ErrHTTPProto{OrigErr: err.Error()}
+		}
+	}
+	repoName, err := reference.WithName(repoEndpoint.Name())
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to parse repo name from %s", ref)
+	}
+	return distributionclient.NewRepository(ctx, repoName, repoEndpoint.BaseURL(), httpTransport)
+}
+
+func (c *client) getHTTPTransportForRepoEndpoint(ctx context.Context, repoEndpoint repositoryEndpoint) (http.RoundTripper, error) {
+	httpTransport, err := getHTTPTransport(
+		c.authConfigResolver(ctx, repoEndpoint.info.Index),
+		repoEndpoint.endpoint,
+		repoEndpoint.Name(),
+		c.userAgent)
+	return httpTransport, errors.Wrap(err, "failed to configure transport")
+}
+
+// GetManifest returns an ImageManifest for the reference
+func (c *client) GetManifest(ctx context.Context, ref reference.Named) (manifesttypes.ImageManifest, error) {
+	var result manifesttypes.ImageManifest
+	fetch := func(ctx context.Context, repo distribution.Repository, ref reference.Named) (bool, error) {
+		var err error
+		result, err = fetchManifest(ctx, repo, ref)
+		return result.Ref != nil, err
+	}
+
+	err := c.iterateEndpoints(ctx, ref, fetch)
+	return result, err
+}
+
+// GetManifestList returns a list of ImageManifest for the reference
+func (c *client) GetManifestList(ctx context.Context, ref reference.Named) ([]manifesttypes.ImageManifest, error) {
+	result := []manifesttypes.ImageManifest{}
+	fetch := func(ctx context.Context, repo distribution.Repository, ref reference.Named) (bool, error) {
+		var err error
+		result, err = fetchList(ctx, repo, ref)
+		return len(result) > 0, err
+	}
+
+	err := c.iterateEndpoints(ctx, ref, fetch)
+	return result, err
+}
+
+func getManifestOptionsFromReference(ref reference.Named) (digest.Digest, []distribution.ManifestServiceOption, error) {
+	if tagged, isTagged := ref.(reference.NamedTagged); isTagged {
+		tag := tagged.Tag()
+		return "", []distribution.ManifestServiceOption{distribution.WithTag(tag)}, nil
+	}
+	if digested, isDigested := ref.(reference.Canonical); isDigested {
+		return digested.Digest(), []distribution.ManifestServiceOption{}, nil
+	}
+	return "", nil, errors.Errorf("%s no tag or digest", ref)
+}

cli/registry/client/endpoint.go

@@ -0,0 +1,133 @@
+package client
+
+import (
+	"fmt"
+	"net"
+	"net/http"
+	"time"
+
+	"github.com/docker/distribution/reference"
+	"github.com/docker/distribution/registry/client/auth"
+	"github.com/docker/distribution/registry/client/transport"
+	authtypes "github.com/docker/docker/api/types"
+	"github.com/docker/docker/registry"
+	"github.com/pkg/errors"
+)
+
+type repositoryEndpoint struct {
+	info     *registry.RepositoryInfo
+	endpoint registry.APIEndpoint
+}
+
+// Name returns the repository name
+func (r repositoryEndpoint) Name() string {
+	repoName := r.info.Name.Name()
+	// If endpoint does not support CanonicalName, use the RemoteName instead
+	if r.endpoint.TrimHostname {
+		repoName = reference.Path(r.info.Name)
+	}
+	return repoName
+}
+
+// BaseURL returns the endpoint url
+func (r repositoryEndpoint) BaseURL() string {
+	return r.endpoint.URL.String()
+}
+
+func newDefaultRepositoryEndpoint(ref reference.Named, insecure bool) (repositoryEndpoint, error) {
+	repoInfo, err := registry.ParseRepositoryInfo(ref)
+	if err != nil {
+		return repositoryEndpoint{}, err
+	}
+	endpoint, err := getDefaultEndpointFromRepoInfo(repoInfo)
+	if err != nil {
+		return repositoryEndpoint{}, err
+	}
+	if insecure {
+		endpoint.TLSConfig.InsecureSkipVerify = true
+	}
+	return repositoryEndpoint{info: repoInfo, endpoint: endpoint}, nil
+}
+
+func getDefaultEndpointFromRepoInfo(repoInfo *registry.RepositoryInfo) (registry.APIEndpoint, error) {
+	var err error
+
+	options := registry.ServiceOptions{}
+	registryService, err := registry.NewService(options)
+	if err != nil {
+		return registry.APIEndpoint{}, err
+	}
+	endpoints, err := registryService.LookupPushEndpoints(reference.Domain(repoInfo.Name))
+	if err != nil {
+		return registry.APIEndpoint{}, err
+	}
+	// Default to the highest priority endpoint to return
+	endpoint := endpoints[0]
+	if !repoInfo.Index.Secure {
+		for _, ep := range endpoints {
+			if ep.URL.Scheme == "http" {
+				endpoint = ep
+			}
+		}
+	}
+	return endpoint, nil
+}
+
+// getHTTPTransport builds a transport for use in communicating with a registry
+func getHTTPTransport(authConfig authtypes.AuthConfig, endpoint registry.APIEndpoint, repoName string, userAgent string) (http.RoundTripper, error) {
+	// get the http transport, this will be used in a client to upload manifest
+	base := &http.Transport{
+		Proxy: http.ProxyFromEnvironment,
+		Dial: (&net.Dialer{
+			Timeout:   30 * time.Second,
+			KeepAlive: 30 * time.Second,
+			DualStack: true,
+		}).Dial,
+		TLSHandshakeTimeout: 10 * time.Second,
+		TLSClientConfig:     endpoint.TLSConfig,
+		DisableKeepAlives:   true,
+	}
+
+	modifiers := registry.Headers(userAgent, http.Header{})
+	authTransport := transport.NewTransport(base, modifiers...)
+	challengeManager, confirmedV2, err := registry.PingV2Registry(endpoint.URL, authTransport)
+	if err != nil {
+		return nil, errors.Wrap(err, "error pinging v2 registry")
+	}
+	if !confirmedV2 {
+		return nil, fmt.Errorf("unsupported registry version")
+	}
+	if authConfig.RegistryToken != "" {
+		passThruTokenHandler := &existingTokenHandler{token: authConfig.RegistryToken}
+		modifiers = append(modifiers, auth.NewAuthorizer(challengeManager, passThruTokenHandler))
+	} else {
+		creds := registry.NewStaticCredentialStore(&authConfig)
+		tokenHandler := auth.NewTokenHandler(authTransport, creds, repoName, "*")
+		basicHandler := auth.NewBasicHandler(creds)
+		modifiers = append(modifiers, auth.NewAuthorizer(challengeManager, tokenHandler, basicHandler))
+	}
+	return transport.NewTransport(base, modifiers...), nil
+}
+
+// RepoNameForReference returns the repository name from a reference
+func RepoNameForReference(ref reference.Named) (string, error) {
+	// insecure is fine since this only returns the name
+	repo, err := newDefaultRepositoryEndpoint(ref, false)
+	if err != nil {
+		return "", err
+	}
+	return repo.Name(), nil
+}
+
+type existingTokenHandler struct {
+	token string
+}
+
+func (th *existingTokenHandler) AuthorizeRequest(req *http.Request, params map[string]string) error {
+	req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", th.token))
+	return nil
+}
+
+func (th *existingTokenHandler) Scheme() string {
+	return "bearer"
+}

cli/registry/client/fetcher.go

@@ -0,0 +1,295 @@
+package client
+
+import (
+	"fmt"
+
+	"github.com/docker/cli/cli/manifest/types"
+	"github.com/docker/distribution"
+	"github.com/docker/distribution/manifest/manifestlist"
+	"github.com/docker/distribution/manifest/schema2"
+	"github.com/docker/distribution/reference"
+	"github.com/docker/distribution/registry/api/errcode"
+	"github.com/docker/distribution/registry/api/v2"
+	distclient "github.com/docker/distribution/registry/client"
+	"github.com/docker/docker/registry"
+	digest "github.com/opencontainers/go-digest"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+	"golang.org/x/net/context"
+)
+
+// fetchManifest pulls a manifest from a registry and returns it. An error
+// is returned if no manifest is found matching namedRef.
+func fetchManifest(ctx context.Context, repo distribution.Repository, ref reference.Named) (types.ImageManifest, error) {
+	manifest, err := getManifest(ctx, repo, ref)
+	if err != nil {
+		return types.ImageManifest{}, err
+	}
+
+	switch v := manifest.(type) {
+	// Removed Schema 1 support
+	case *schema2.DeserializedManifest:
+		imageManifest, err := pullManifestSchemaV2(ctx, ref, repo, *v)
+		if err != nil {
+			return types.ImageManifest{}, err
+		}
+		return imageManifest, nil
+	case *manifestlist.DeserializedManifestList:
+		return types.ImageManifest{}, errors.Errorf("%s is a manifest list", ref)
+	}
+	return types.ImageManifest{}, errors.Errorf("%s is not a manifest", ref)
+}
+
+func fetchList(ctx context.Context, repo distribution.Repository, ref reference.Named) ([]types.ImageManifest, error) {
+	manifest, err := getManifest(ctx, repo, ref)
+	if err != nil {
+		return nil, err
+	}
+
+	switch v := manifest.(type) {
+	case *manifestlist.DeserializedManifestList:
+		imageManifests, err := pullManifestList(ctx, ref, repo, *v)
+		if err != nil {
+			return nil, err
+		}
+		return imageManifests, nil
+	default:
+		return nil, errors.Errorf("unsupported manifest format: %v", v)
+	}
+}
+
+func getManifest(ctx context.Context, repo distribution.Repository, ref reference.Named) (distribution.Manifest, error) {
+	manSvc, err := repo.Manifests(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	dgst, opts, err := getManifestOptionsFromReference(ref)
+	if err != nil {
+		return nil, errors.Errorf("image manifest for %q does not exist", ref)
+	}
+	return manSvc.Get(ctx, dgst, opts...)
+}
+
+func pullManifestSchemaV2(ctx context.Context, ref reference.Named, repo distribution.Repository, mfst schema2.DeserializedManifest) (types.ImageManifest, error) {
+	manifestDigest, err := validateManifestDigest(ref, mfst)
+	if err != nil {
+		return types.ImageManifest{}, err
+	}
+	configJSON, err := pullManifestSchemaV2ImageConfig(ctx, mfst.Target().Digest, repo)
+	if err != nil {
+		return types.ImageManifest{}, err
+	}
+
+	img, err := types.NewImageFromJSON(configJSON)
+	if err != nil {
+		return types.ImageManifest{}, err
+	}
+	return types.NewImageManifest(ref, manifestDigest, *img, &mfst), nil
+}
+
+func pullManifestSchemaV2ImageConfig(ctx context.Context, dgst digest.Digest, repo distribution.Repository) ([]byte, error) {
+	blobs := repo.Blobs(ctx)
+	configJSON, err := blobs.Get(ctx, dgst)
+	if err != nil {
+		return nil, err
+	}
+
+	verifier := dgst.Verifier()
+	if err != nil {
+		return nil, err
+	}
+	if _, err := verifier.Write(configJSON); err != nil {
+		return nil, err
+	}
+	if !verifier.Verified() {
+		return nil, errors.Errorf("image config verification failed for digest %s", dgst)
+	}
+	return configJSON, nil
+}
+
+// validateManifestDigest computes the manifest digest, and, if pulling by
+// digest, ensures that it matches the requested digest.
+func validateManifestDigest(ref reference.Named, mfst distribution.Manifest) (digest.Digest, error) {
+	_, canonical, err := mfst.Payload()
+	if err != nil {
+		return "", err
+	}
+
+	// If pull by digest, then verify the manifest digest.
+	if digested, isDigested := ref.(reference.Canonical); isDigested {
+		verifier := digested.Digest().Verifier()
+		if err != nil {
+			return "", err
+		}
+		if _, err := verifier.Write(canonical); err != nil {
+			return "", err
+		}
+		if !verifier.Verified() {
+			err := fmt.Errorf("manifest verification failed for digest %s", digested.Digest())
+			return "", err
+		}
+		return digested.Digest(), nil
+	}
+
+	return digest.FromBytes(canonical), nil
+}
+
+// pullManifestList handles "manifest lists" which point to various
+// platform-specific manifests.
+func pullManifestList(ctx context.Context, ref reference.Named, repo distribution.Repository, mfstList manifestlist.DeserializedManifestList) ([]types.ImageManifest, error) {
+	infos := []types.ImageManifest{}
+
+	if _, err := validateManifestDigest(ref, mfstList); err != nil {
+		return nil, err
+	}
+
+	for _, manifestDescriptor := range mfstList.Manifests {
+		manSvc, err := repo.Manifests(ctx)
+		if err != nil {
+			return nil, err
+		}
+		manifest, err := manSvc.Get(ctx, manifestDescriptor.Digest)
+		if err != nil {
+			return nil, err
+		}
+		v, ok := manifest.(*schema2.DeserializedManifest)
+		if !ok {
+			return nil, fmt.Errorf("unsupported manifest format: %s", v)
+		}
+
+		manifestRef, err := reference.WithDigest(ref, manifestDescriptor.Digest)
+		if err != nil {
+			return nil, err
+		}
+		imageManifest, err := pullManifestSchemaV2(ctx, manifestRef, repo, *v)
+		if err != nil {
+			return nil, err
+		}
+		imageManifest.Platform = manifestDescriptor.Platform
+		infos = append(infos, imageManifest)
+	}
+	return infos, nil
+}
+
+func continueOnError(err error) bool {
+	switch v := err.(type) {
+	case errcode.Errors:
+		if len(v) == 0 {
+			return true
+		}
+		return continueOnError(v[0])
+	case errcode.Error:
+		e := err.(errcode.Error)
+		switch e.Code {
+		case errcode.ErrorCodeUnauthorized, v2.ErrorCodeManifestUnknown, v2.ErrorCodeNameUnknown:
+			return true
+		}
+		return false
+	case *distclient.UnexpectedHTTPResponseError:
+		return true
+	}
+	return false
+}
+
+func (c *client) iterateEndpoints(ctx context.Context, namedRef reference.Named, each func(context.Context, distribution.Repository, reference.Named) (bool, error)) error {
+	endpoints, err := allEndpoints(namedRef)
+	if err != nil {
+		return err
+	}
+
+	repoInfo, err := registry.ParseRepositoryInfo(namedRef)
+	if err != nil {
+		return err
+	}
+
+	confirmedTLSRegistries := make(map[string]bool)
+	for _, endpoint := range endpoints {
+
+		if endpoint.Version == registry.APIVersion1 {
+			logrus.Debugf("skipping v1 endpoint %s", endpoint.URL)
+			continue
+		}
+
+		if endpoint.URL.Scheme != "https" {
+			if _, confirmedTLS := confirmedTLSRegistries[endpoint.URL.Host]; confirmedTLS {
+				logrus.Debugf("skipping non-TLS endpoint %s for host/port that appears to use TLS", endpoint.URL)
+				continue
+			}
+		}
+
+		if c.insecureRegistry {
+			endpoint.TLSConfig.InsecureSkipVerify = true
+		}
+		repoEndpoint := repositoryEndpoint{endpoint: endpoint, info: repoInfo}
+		repo, err := c.getRepositoryForReference(ctx, namedRef, repoEndpoint)
+		if err != nil {
+			logrus.Debugf("error with repo endpoint %s: %s", repoEndpoint, err)
+			if _, ok := err.(ErrHTTPProto); ok {
+				continue
+			}
+			return err
+		}
+
+		if endpoint.URL.Scheme == "http" && !c.insecureRegistry {
+			logrus.Debugf("skipping non-tls registry endpoint: %s", endpoint.URL)
+			continue
+		}
+		done, err := each(ctx, repo, namedRef)
+		if err != nil {
+			if continueOnError(err) {
+				if endpoint.URL.Scheme == "https" {
+					confirmedTLSRegistries[endpoint.URL.Host] = true
+				}
+				logrus.Debugf("continuing on error (%T) %s", err, err)
+				continue
+			}
+			logrus.Debugf("not continuing on error (%T) %s", err, err)
+			return err
+		}
+		if done {
+			return nil
+		}
+	}
+	return newNotFoundError(namedRef.String())
+}
+
+// allEndpoints returns a list of endpoints ordered by priority (v2, https, v1).
+func allEndpoints(namedRef reference.Named) ([]registry.APIEndpoint, error) {
+	repoInfo, err := registry.ParseRepositoryInfo(namedRef)
+	if err != nil {
+		return nil, err
+	}
+	registryService, err := registry.NewService(registry.ServiceOptions{})
+	if err != nil {
+		return []registry.APIEndpoint{}, err
+	}
+	endpoints, err := registryService.LookupPullEndpoints(reference.Domain(repoInfo.Name))
+	logrus.Debugf("endpoints for %s: %v", namedRef, endpoints)
+	return endpoints, err
+}
+
+type notFoundError struct {
+	object string
+}
+
+func newNotFoundError(ref string) *notFoundError {
+	return &notFoundError{object: ref}
+}
+
+func (n *notFoundError) Error() string {
+	return fmt.Sprintf("no such manifest: %s", n.object)
+}
+
+// NotFound interface
+func (n *notFoundError) NotFound() {}
+
+// IsNotFound returns true if the error is a not found error
+func IsNotFound(err error) bool {
+	_, ok := err.(notFound)
+	return ok
+}
+
+type notFound interface {
+	NotFound()
+}

internal/test/cli.go

@@ -10,6 +10,8 @@ import (
 	"github.com/docker/cli/cli/command"
 	"github.com/docker/cli/cli/config/configfile"
 	"github.com/docker/cli/cli/trust"
+	manifeststore "github.com/docker/cli/cli/manifest/store"
+	registryclient "github.com/docker/cli/cli/registry/client"
 	"github.com/docker/docker/client"
 	notaryclient "github.com/theupdateframework/notary/client"
 )
@@ -27,8 +29,9 @@ type FakeCli struct {
 	err            *bytes.Buffer
 	in             *command.InStream
 	server         command.ServerInfo
-	clientInfoFunc   clientInfoFuncType
 	notaryClientFunc notaryClientFuncType
+	manifestStore  manifeststore.Store
+	registryClient registryclient.RegistryClient
 }
 
 // NewFakeCli returns a fake for the command.Cli interface
@@ -124,4 +127,23 @@ func (c *FakeCli) NotaryClient(imgRefAndAuth trust.ImageRefAndAuth, actions []st
 		return c.notaryClientFunc(imgRefAndAuth, actions)
 	}
 	return nil, fmt.Errorf("no notary client available unless defined")
+
+// ManifestStore returns a fake store used for testing
+func (c *FakeCli) ManifestStore() manifeststore.Store {
+	return c.manifestStore
+}
+
+// RegistryClient returns a fake client for testing
+func (c *FakeCli) RegistryClient(insecure bool) registryclient.RegistryClient {
+	return c.registryClient
+}
+
+// SetManifestStore on the fake cli
+func (c *FakeCli) SetManifestStore(store manifeststore.Store) {
+	c.manifestStore = store
+}
+
+// SetRegistryClient on the fake cli
+func (c *FakeCli) SetRegistryClient(client registryclient.RegistryClient) {
+	c.registryClient = client
 }