From 02719bdbb5fb47389e47575bb006509da86df344 Mon Sep 17 00:00:00 2001 From: Christy Perez Date: Thu, 15 Jun 2017 13:41:54 -0500 Subject: [PATCH] add manifest command Enable inspection (aka "shallow pull") of images' manifest info, and also the creation of manifest lists (aka "fat manifests"). The workflow for creating a manifest list will be: `docker manifest create new-list-ref-name image-ref [image-ref...]` `docker manifest annotate new-list-ref-name image-ref --os linux --arch arm` `docker manifest push new-list-ref-name` The annotate step is optional. Most architectures are fine by default. There is also a `manifest inspect` command to allow for a "shallow pull" of an image's manifest: `docker manifest inspect manifest-or-manifest_list`. To be more in line with the existing external manifest tool, there is also a `-v` option for inspect that will show information depending on what the reference maps to (list or single manifest). Signed-off-by: Christy Perez Signed-off-by: Daniel Nephin --- cli/command/cli.go | 23 ++ cli/command/commands/commands.go | 8 +- cli/command/manifest/annotate.go | 93 ++++++ cli/command/manifest/client_test.go | 28 ++ cli/command/manifest/cmd.go | 44 +++ cli/command/manifest/create_list.go | 82 +++++ cli/command/manifest/inspect.go | 147 +++++++++ cli/command/manifest/inspect_test.go | 131 ++++++++ cli/command/manifest/push.go | 272 ++++++++++++++++ .../manifest/testdata/inspect-manifest.golden | 16 + cli/command/manifest/util.go | 79 +++++ cli/command/registry.go | 3 +- cli/manifest/store/store.go | 147 +++++++++ cli/manifest/store/store_test.go | 131 ++++++++ cli/manifest/types/types.go | 107 +++++++ cli/registry/client/client.go | 183 +++++++++++ cli/registry/client/endpoint.go | 133 ++++++++ cli/registry/client/fetcher.go | 295 ++++++++++++++++++ internal/test/cli.go | 38 ++- 19 files changed, 1948 insertions(+), 12 deletions(-) create mode 100644 cli/command/manifest/annotate.go create mode 100644 cli/command/manifest/client_test.go create mode 100644 cli/command/manifest/cmd.go create mode 100644 cli/command/manifest/create_list.go create mode 100644 cli/command/manifest/inspect.go create mode 100644 cli/command/manifest/inspect_test.go create mode 100644 cli/command/manifest/push.go create mode 100644 cli/command/manifest/testdata/inspect-manifest.golden create mode 100644 cli/command/manifest/util.go create mode 100644 cli/manifest/store/store.go create mode 100644 cli/manifest/store/store_test.go create mode 100644 cli/manifest/types/types.go create mode 100644 cli/registry/client/client.go create mode 100644 cli/registry/client/endpoint.go create mode 100644 cli/registry/client/fetcher.go diff --git a/cli/command/cli.go b/cli/command/cli.go index 5d903ccead..484c8c537c 100644 --- a/cli/command/cli.go +++ b/cli/command/cli.go @@ -5,16 +5,22 @@ import ( "net" "net/http" "os" + "path/filepath" "runtime" "time" "github.com/docker/cli/cli" + "github.com/docker/cli/cli/config" cliconfig "github.com/docker/cli/cli/config" "github.com/docker/cli/cli/config/configfile" cliflags "github.com/docker/cli/cli/flags" + manifeststore "github.com/docker/cli/cli/manifest/store" + registryclient "github.com/docker/cli/cli/registry/client" "github.com/docker/cli/cli/trust" dopts "github.com/docker/cli/opts" "github.com/docker/docker/api" + "github.com/docker/docker/api/types" + registrytypes "github.com/docker/docker/api/types/registry" "github.com/docker/docker/client" "github.com/docker/go-connections/sockets" "github.com/docker/go-connections/tlsconfig" @@ -45,6 +51,8 @@ type Cli interface { ClientInfo() ClientInfo NotaryClient(imgRefAndAuth trust.ImageRefAndAuth, actions []string) (notaryclient.Repository, error) DefaultVersion() string + ManifestStore() manifeststore.Store + RegistryClient(bool) registryclient.RegistryClient } // DockerCli is an instance the docker command line client. @@ -114,6 +122,21 @@ func (cli *DockerCli) ClientInfo() ClientInfo { return cli.clientInfo } +// ManifestStore returns a store for local manifests +func (cli *DockerCli) ManifestStore() manifeststore.Store { + // TODO: support override default location from config file + return manifeststore.NewStore(filepath.Join(config.Dir(), "manifests")) +} + +// RegistryClient returns a client for communicating with a Docker distribution +// registry +func (cli *DockerCli) RegistryClient(allowInsecure bool) registryclient.RegistryClient { + resolver := func(ctx context.Context, index *registrytypes.IndexInfo) types.AuthConfig { + return ResolveAuthConfig(ctx, cli, index) + } + return registryclient.NewRegistryClient(resolver, UserAgent(), allowInsecure) +} + // Initialize the dockerCli runs initialization that must happen after command // line flags are parsed. func (cli *DockerCli) Initialize(opts *cliflags.ClientOptions) error { diff --git a/cli/command/commands/commands.go b/cli/command/commands/commands.go index 6645770095..a0e3465845 100644 --- a/cli/command/commands/commands.go +++ b/cli/command/commands/commands.go @@ -8,6 +8,7 @@ import ( "github.com/docker/cli/cli/command/config" "github.com/docker/cli/cli/command/container" "github.com/docker/cli/cli/command/image" + "github.com/docker/cli/cli/command/manifest" "github.com/docker/cli/cli/command/network" "github.com/docker/cli/cli/command/node" "github.com/docker/cli/cli/command/plugin" @@ -39,12 +40,15 @@ func AddCommands(cmd *cobra.Command, dockerCli *command.DockerCli) { image.NewImageCommand(dockerCli), image.NewBuildCommand(dockerCli), - // node - node.NewNodeCommand(dockerCli), + // manifest + manifest.NewManifestCommand(dockerCli), // network network.NewNetworkCommand(dockerCli), + // node + node.NewNodeCommand(dockerCli), + // plugin plugin.NewPluginCommand(dockerCli), diff --git a/cli/command/manifest/annotate.go b/cli/command/manifest/annotate.go new file mode 100644 index 0000000000..f8bd0e7590 --- /dev/null +++ b/cli/command/manifest/annotate.go @@ -0,0 +1,93 @@ +package manifest + +import ( + "fmt" + + "github.com/docker/cli/cli" + "github.com/docker/cli/cli/command" + "github.com/docker/cli/cli/manifest/store" + "github.com/pkg/errors" + "github.com/spf13/cobra" +) + +type annotateOptions struct { + target string // the target manifest list name (also transaction ID) + image string // the manifest to annotate within the list + variant string // an architecture variant + os string + arch string + osFeatures []string +} + +// NewAnnotateCommand creates a new `docker manifest annotate` command +func newAnnotateCommand(dockerCli command.Cli) *cobra.Command { + var opts annotateOptions + + cmd := &cobra.Command{ + Use: "annotate [OPTIONS] MANIFEST_LIST MANIFEST", + Short: "Add additional information to a local image manifest", + Args: cli.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + opts.target = args[0] + opts.image = args[1] + return runManifestAnnotate(dockerCli, opts) + }, + } + + flags := cmd.Flags() + + flags.StringVar(&opts.os, "os", "", "Set operating system") + flags.StringVar(&opts.arch, "arch", "", "Set architecture") + flags.StringSliceVar(&opts.osFeatures, "os-features", []string{}, "Set operating system feature") + flags.StringVar(&opts.variant, "variant", "", "Set architecture variant") + + return cmd +} + +func runManifestAnnotate(dockerCli command.Cli, opts annotateOptions) error { + targetRef, err := normalizeReference(opts.target) + if err != nil { + return errors.Wrapf(err, "annotate: Error parsing name for manifest list (%s): %s", opts.target) + } + imgRef, err := normalizeReference(opts.image) + if err != nil { + return errors.Wrapf(err, "annotate: Error parsing name for manifest (%s): %s:", opts.image) + } + + manifestStore := dockerCli.ManifestStore() + imageManifest, err := manifestStore.Get(targetRef, imgRef) + switch { + case store.IsNotFound(err): + return fmt.Errorf("manifest for image %s does not exist in %s", opts.image, opts.target) + case err != nil: + return err + } + + // Update the mf + if opts.os != "" { + imageManifest.Platform.OS = opts.os + } + if opts.arch != "" { + imageManifest.Platform.Architecture = opts.arch + } + for _, osFeature := range opts.osFeatures { + imageManifest.Platform.OSFeatures = appendIfUnique(imageManifest.Platform.OSFeatures, osFeature) + } + if opts.variant != "" { + imageManifest.Platform.Variant = opts.variant + } + + if !isValidOSArch(imageManifest.Platform.OS, imageManifest.Platform.Architecture) { + return errors.Errorf("manifest entry for image has unsupported os/arch combination: %s/%s", opts.os, opts.arch) + } + return manifestStore.Save(targetRef, imgRef, imageManifest) +} + +func appendIfUnique(list []string, str string) []string { + for _, s := range list { + if s == str { + return list + } + } + return append(list, str) +} diff --git a/cli/command/manifest/client_test.go b/cli/command/manifest/client_test.go new file mode 100644 index 0000000000..d319ea343d --- /dev/null +++ b/cli/command/manifest/client_test.go @@ -0,0 +1,28 @@ +package manifest + +import ( + manifesttypes "github.com/docker/cli/cli/manifest/types" + "github.com/docker/cli/cli/registry/client" + "github.com/docker/distribution/reference" + "golang.org/x/net/context" +) + +type fakeRegistryClient struct { + client.RegistryClient + getManifestFunc func(ctx context.Context, ref reference.Named) (manifesttypes.ImageManifest, error) + getManifestListFunc func(ctx context.Context, ref reference.Named) ([]manifesttypes.ImageManifest, error) +} + +func (c *fakeRegistryClient) GetManifest(ctx context.Context, ref reference.Named) (manifesttypes.ImageManifest, error) { + if c.getManifestFunc != nil { + return c.getManifestFunc(ctx, ref) + } + return manifesttypes.ImageManifest{}, nil +} + +func (c *fakeRegistryClient) GetManifestList(ctx context.Context, ref reference.Named) ([]manifesttypes.ImageManifest, error) { + if c.getManifestListFunc != nil { + return c.getManifestListFunc(ctx, ref) + } + return nil, nil +} diff --git a/cli/command/manifest/cmd.go b/cli/command/manifest/cmd.go new file mode 100644 index 0000000000..bf19f3dd14 --- /dev/null +++ b/cli/command/manifest/cmd.go @@ -0,0 +1,44 @@ +package manifest + +import ( + "fmt" + + "github.com/docker/cli/cli" + "github.com/docker/cli/cli/command" + + "github.com/spf13/cobra" +) + +// NewManifestCommand returns a cobra command for `manifest` subcommands +func NewManifestCommand(dockerCli command.Cli) *cobra.Command { + // use dockerCli as command.Cli + cmd := &cobra.Command{ + Use: "manifest COMMAND", + Short: "Manage Docker image manifests and manifest lists", + Long: manifestDescription, + Args: cli.NoArgs, + Run: func(cmd *cobra.Command, args []string) { + fmt.Fprintf(dockerCli.Err(), "\n"+cmd.UsageString()) + }, + } + cmd.AddCommand( + newCreateListCommand(dockerCli), + newInspectCommand(dockerCli), + newAnnotateCommand(dockerCli), + newPushListCommand(dockerCli), + ) + return cmd +} + +var manifestDescription = ` +The **docker manifest** command has subcommands for managing image manifests and +manifest lists. A manifest list allows you to use one name to refer to the same image +built for multiple architectures. + +To see help for a subcommand, use: + + docker manifest CMD --help + +For full details on using docker manifest lists, see the registry v2 specification. + +` diff --git a/cli/command/manifest/create_list.go b/cli/command/manifest/create_list.go new file mode 100644 index 0000000000..29d244005f --- /dev/null +++ b/cli/command/manifest/create_list.go @@ -0,0 +1,82 @@ +package manifest + +import ( + "fmt" + + "github.com/docker/cli/cli" + "github.com/docker/cli/cli/command" + "github.com/docker/cli/cli/manifest/store" + "github.com/docker/docker/registry" + "github.com/pkg/errors" + "github.com/spf13/cobra" + "golang.org/x/net/context" +) + +type createOpts struct { + amend bool + insecure bool +} + +func newCreateListCommand(dockerCli command.Cli) *cobra.Command { + opts := createOpts{} + + cmd := &cobra.Command{ + Use: "create MANFEST_LIST MANIFEST [MANIFEST...]", + Short: "Create a local manifest list for annotating and pushing to a registry", + Args: cli.RequiresMinArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + return createManifestList(dockerCli, args, opts) + }, + } + + flags := cmd.Flags() + flags.BoolVar(&opts.insecure, "insecure", false, "allow communication with an insecure registry") + flags.BoolVarP(&opts.amend, "amend", "a", false, "Amend an existing manifest list") + return cmd +} + +func createManifestList(dockerCli command.Cli, args []string, opts createOpts) error { + newRef := args[0] + targetRef, err := normalizeReference(newRef) + if err != nil { + return errors.Wrapf(err, "error parsing name for manifest list (%s): %v", newRef) + } + + _, err = registry.ParseRepositoryInfo(targetRef) + if err != nil { + return errors.Wrapf(err, "error parsing repository name for manifest list (%s): %v", newRef) + } + + manifestStore := dockerCli.ManifestStore() + _, err = manifestStore.GetList(targetRef) + switch { + case store.IsNotFound(err): + // New manifest list + case err != nil: + return err + case !opts.amend: + return errors.Errorf("refusing to amend an existing manifest list with no --amend flag") + } + + ctx := context.Background() + // Now create the local manifest list transaction by looking up the manifest schemas + // for the constituent images: + manifests := args[1:] + for _, manifestRef := range manifests { + namedRef, err := normalizeReference(manifestRef) + if err != nil { + // TODO: wrap error? + return err + } + + manifest, err := getManifest(ctx, dockerCli, targetRef, namedRef, opts.insecure) + if err != nil { + return err + } + if err := manifestStore.Save(targetRef, namedRef, manifest); err != nil { + return err + } + } + fmt.Fprintf(dockerCli.Out(), "Created manifest list %s\n", targetRef.String()) + return nil +} diff --git a/cli/command/manifest/inspect.go b/cli/command/manifest/inspect.go new file mode 100644 index 0000000000..23db92dd44 --- /dev/null +++ b/cli/command/manifest/inspect.go @@ -0,0 +1,147 @@ +package manifest + +import ( + "bytes" + "encoding/json" + "fmt" + + "github.com/docker/cli/cli" + "github.com/docker/cli/cli/command" + "github.com/docker/cli/cli/manifest/types" + "github.com/docker/distribution/manifest/manifestlist" + "github.com/docker/distribution/reference" + "github.com/docker/docker/registry" + "github.com/spf13/cobra" + "golang.org/x/net/context" +) + +type inspectOptions struct { + ref string + list string + verbose bool + insecure bool +} + +// NewInspectCommand creates a new `docker manifest inspect` command +func newInspectCommand(dockerCli command.Cli) *cobra.Command { + var opts inspectOptions + + cmd := &cobra.Command{ + Use: "inspect [OPTIONS] [MANIFEST_LIST] MANIFEST", + Short: "Display an image manifest, or manifest list", + Args: cli.RequiresRangeArgs(1, 2), + RunE: func(cmd *cobra.Command, args []string) error { + switch len(args) { + case 1: + opts.ref = args[0] + case 2: + opts.list = args[0] + opts.ref = args[1] + } + return runInspect(dockerCli, opts) + }, + } + + flags := cmd.Flags() + flags.BoolVar(&opts.insecure, "insecure", false, "allow communication with an insecure registry") + flags.BoolVarP(&opts.verbose, "verbose", "v", false, "Output additional info including layers and platform") + return cmd +} + +func runInspect(dockerCli command.Cli, opts inspectOptions) error { + namedRef, err := normalizeReference(opts.ref) + if err != nil { + return err + } + + // If list reference is provided, display the local manifest in a list + if opts.list != "" { + listRef, err := normalizeReference(opts.list) + if err != nil { + return err + } + + imageManifest, err := dockerCli.ManifestStore().Get(listRef, namedRef) + if err != nil { + return err + } + return printManifest(dockerCli, imageManifest, opts) + } + + // Try a local manifest list first + localManifestList, err := dockerCli.ManifestStore().GetList(namedRef) + if err == nil { + return printManifestList(dockerCli, namedRef, localManifestList, opts) + } + + // Next try a remote manifest + ctx := context.Background() + registryClient := dockerCli.RegistryClient(opts.insecure) + imageManifest, err := registryClient.GetManifest(ctx, namedRef) + if err == nil { + return printManifest(dockerCli, imageManifest, opts) + } + + // Finally try a remote manifest list + manifestList, err := registryClient.GetManifestList(ctx, namedRef) + if err != nil { + return err + } + return printManifestList(dockerCli, namedRef, manifestList, opts) +} + +func printManifest(dockerCli command.Cli, manifest types.ImageManifest, opts inspectOptions) error { + buffer := new(bytes.Buffer) + if !opts.verbose { + _, raw, err := manifest.Payload() + if err != nil { + return err + } + if err := json.Indent(buffer, raw, "", "\t"); err != nil { + return err + } + fmt.Fprintln(dockerCli.Out(), buffer.String()) + return nil + } + jsonBytes, err := json.MarshalIndent(manifest, "", "\t") + if err != nil { + return err + } + dockerCli.Out().Write(append(jsonBytes, '\n')) + return nil +} + +func printManifestList(dockerCli command.Cli, namedRef reference.Named, list []types.ImageManifest, opts inspectOptions) error { + if !opts.verbose { + targetRepo, err := registry.ParseRepositoryInfo(namedRef) + if err != nil { + return err + } + + manifests := []manifestlist.ManifestDescriptor{} + // More than one response. This is a manifest list. + for _, img := range list { + mfd, err := buildManifestDescriptor(targetRepo, img) + if err != nil { + return fmt.Errorf("error assembling ManifestDescriptor") + } + manifests = append(manifests, mfd) + } + deserializedML, err := manifestlist.FromDescriptors(manifests) + if err != nil { + return err + } + jsonBytes, err := deserializedML.MarshalJSON() + if err != nil { + return err + } + fmt.Fprintln(dockerCli.Out(), string(jsonBytes)) + return nil + } + jsonBytes, err := json.MarshalIndent(list, "", "\t") + if err != nil { + return err + } + dockerCli.Out().Write(append(jsonBytes, '\n')) + return nil +} diff --git a/cli/command/manifest/inspect_test.go b/cli/command/manifest/inspect_test.go new file mode 100644 index 0000000000..ae4a7fe992 --- /dev/null +++ b/cli/command/manifest/inspect_test.go @@ -0,0 +1,131 @@ +package manifest + +import ( + "io/ioutil" + "os" + "testing" + + "github.com/docker/cli/cli/manifest/store" + "github.com/docker/cli/cli/manifest/types" + manifesttypes "github.com/docker/cli/cli/manifest/types" + "github.com/docker/cli/internal/test" + "github.com/docker/distribution" + "github.com/docker/distribution/manifest/schema2" + "github.com/docker/distribution/reference" + "github.com/gotestyourself/gotestyourself/golden" + "github.com/opencontainers/go-digest" + "github.com/pkg/errors" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/net/context" +) + +func newTempManifestStore(t *testing.T) (store.Store, func()) { + tmpdir, err := ioutil.TempDir("", "test-manifest-storage") + require.NoError(t, err) + + return store.NewStore(tmpdir), func() { os.RemoveAll(tmpdir) } +} + +func ref(t *testing.T, name string) reference.Named { + named, err := reference.ParseNamed("example.com/" + name) + require.NoError(t, err) + return named +} + +func fullImageManifest(t *testing.T, ref reference.Named) types.ImageManifest { + man, err := schema2.FromStruct(schema2.Manifest{ + Versioned: schema2.SchemaVersion, + Config: distribution.Descriptor{ + Digest: "sha256:7328f6f8b41890597575cbaadc884e7386ae0acc53b747401ebce5cf0d624560", + Size: 1520, + MediaType: schema2.MediaTypeImageConfig, + }, + Layers: []distribution.Descriptor{ + { + MediaType: schema2.MediaTypeLayer, + Size: 1990402, + Digest: "sha256:88286f41530e93dffd4b964e1db22ce4939fffa4a4c665dab8591fbab03d4926", + }, + }, + }) + require.NoError(t, err) + // TODO: include image data for verbose inspect + return types.NewImageManifest(ref, digest.Digest("abcd"), types.Image{}, man) +} + +func TestInspectCommandLocalManifestNotFound(t *testing.T) { + store, cleanup := newTempManifestStore(t) + defer cleanup() + + cli := test.NewFakeCli(nil) + cli.SetManifestStore(store) + + cmd := newInspectCommand(cli) + cmd.SetOutput(ioutil.Discard) + cmd.SetArgs([]string{"example.com/list:v1", "example.com/alpine:3.0"}) + err := cmd.Execute() + assert.EqualError(t, err, "No such manifest: example.com/alpine:3.0") +} + +func TestInspectCommandNotFound(t *testing.T) { + store, cleanup := newTempManifestStore(t) + defer cleanup() + + cli := test.NewFakeCli(nil) + cli.SetManifestStore(store) + cli.SetRegistryClient(&fakeRegistryClient{ + getManifestFunc: func(_ context.Context, _ reference.Named) (manifesttypes.ImageManifest, error) { + return manifesttypes.ImageManifest{}, errors.New("missing") + }, + getManifestListFunc: func(ctx context.Context, ref reference.Named) ([]manifesttypes.ImageManifest, error) { + return nil, errors.Errorf("No such manifest: %s", ref) + }, + }) + + cmd := newInspectCommand(cli) + cmd.SetOutput(ioutil.Discard) + cmd.SetArgs([]string{"example.com/alpine:3.0"}) + err := cmd.Execute() + assert.EqualError(t, err, "No such manifest: example.com/alpine:3.0") +} + +func TestInspectCommandLocalManifest(t *testing.T) { + store, cleanup := newTempManifestStore(t) + defer cleanup() + + cli := test.NewFakeCli(nil) + cli.SetManifestStore(store) + namedRef := ref(t, "alpine:3.0") + imageManifest := fullImageManifest(t, namedRef) + err := store.Save(ref(t, "list:v1"), namedRef, imageManifest) + require.NoError(t, err) + + cmd := newInspectCommand(cli) + cmd.SetArgs([]string{"example.com/list:v1", "example.com/alpine:3.0"}) + require.NoError(t, cmd.Execute()) + actual := cli.OutBuffer() + expected := golden.Get(t, "inspect-manifest.golden") + assert.Equal(t, string(expected), actual.String()) +} + +func TestInspectcommandRemoteManifest(t *testing.T) { + store, cleanup := newTempManifestStore(t) + defer cleanup() + + cli := test.NewFakeCli(nil) + cli.SetManifestStore(store) + cli.SetRegistryClient(&fakeRegistryClient{ + getManifestFunc: func(_ context.Context, ref reference.Named) (manifesttypes.ImageManifest, error) { + return fullImageManifest(t, ref), nil + }, + }) + + cmd := newInspectCommand(cli) + cmd.SetOutput(ioutil.Discard) + cmd.SetArgs([]string{"example.com/alpine:3.0"}) + require.NoError(t, cmd.Execute()) + actual := cli.OutBuffer() + expected := golden.Get(t, "inspect-manifest.golden") + assert.Equal(t, string(expected), actual.String()) +} diff --git a/cli/command/manifest/push.go b/cli/command/manifest/push.go new file mode 100644 index 0000000000..fcc9015d77 --- /dev/null +++ b/cli/command/manifest/push.go @@ -0,0 +1,272 @@ +package manifest + +import ( + "encoding/json" + "fmt" + "io" + + "github.com/docker/cli/cli" + "github.com/docker/cli/cli/command" + "github.com/docker/cli/cli/manifest/types" + registryclient "github.com/docker/cli/cli/registry/client" + "github.com/docker/distribution/manifest/manifestlist" + "github.com/docker/distribution/manifest/schema2" + "github.com/docker/distribution/reference" + "github.com/docker/docker/registry" + "github.com/pkg/errors" + "github.com/spf13/cobra" + "golang.org/x/net/context" +) + +type pushOpts struct { + insecure bool + purge bool + target string +} + +type mountRequest struct { + ref reference.Named + manifest types.ImageManifest +} + +type manifestBlob struct { + canonical reference.Canonical + os string +} + +type pushRequest struct { + targetRef reference.Named + list *manifestlist.DeserializedManifestList + mountRequests []mountRequest + manifestBlobs []manifestBlob + insecure bool +} + +func newPushListCommand(dockerCli command.Cli) *cobra.Command { + opts := pushOpts{} + + cmd := &cobra.Command{ + Use: "push [OPTIONS] MANIFEST_LIST", + Short: "Push a manifest list to a repository", + Args: cli.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + opts.target = args[0] + return runPush(dockerCli, opts) + }, + } + + flags := cmd.Flags() + flags.BoolVarP(&opts.purge, "purge", "p", false, "Remove the local manifest list after push") + flags.BoolVar(&opts.insecure, "insecure", false, "Allow push to an insecure registry") + return cmd +} + +func runPush(dockerCli command.Cli, opts pushOpts) error { + + targetRef, err := normalizeReference(opts.target) + if err != nil { + return err + } + + manifests, err := dockerCli.ManifestStore().GetList(targetRef) + if err != nil { + return err + } + if len(manifests) == 0 { + return errors.Errorf("%s not found", targetRef) + } + + pushRequest, err := buildPushRequest(manifests, targetRef, opts.insecure) + if err != nil { + return err + } + + ctx := context.Background() + if err := pushList(ctx, dockerCli, pushRequest); err != nil { + return err + } + if opts.purge { + return dockerCli.ManifestStore().Remove(targetRef) + } + return nil +} + +func buildPushRequest(manifests []types.ImageManifest, targetRef reference.Named, insecure bool) (pushRequest, error) { + req := pushRequest{targetRef: targetRef, insecure: insecure} + + var err error + req.list, err = buildManifestList(manifests, targetRef) + if err != nil { + return req, err + } + + targetRepo, err := registry.ParseRepositoryInfo(targetRef) + if err != nil { + return req, err + } + targetRepoName, err := registryclient.RepoNameForReference(targetRepo.Name) + if err != nil { + return req, err + } + + for _, imageManifest := range manifests { + manifestRepoName, err := registryclient.RepoNameForReference(imageManifest.Ref) + if err != nil { + return req, err + } + + repoName, _ := reference.WithName(manifestRepoName) + if repoName.Name() != targetRepoName { + blobs, err := buildBlobRequestList(imageManifest, repoName) + if err != nil { + return req, err + } + req.manifestBlobs = append(req.manifestBlobs, blobs...) + + manifestPush, err := buildPutManifestRequest(imageManifest, targetRef) + if err != nil { + return req, err + } + req.mountRequests = append(req.mountRequests, manifestPush) + } + } + return req, nil +} + +func buildManifestList(manifests []types.ImageManifest, targetRef reference.Named) (*manifestlist.DeserializedManifestList, error) { + targetRepoInfo, err := registry.ParseRepositoryInfo(targetRef) + if err != nil { + return nil, err + } + + descriptors := []manifestlist.ManifestDescriptor{} + for _, imageManifest := range manifests { + if imageManifest.Platform.Architecture == "" || imageManifest.Platform.OS == "" { + return nil, errors.Errorf( + "manifest %s must have an OS and Architecture to be pushed to a registry", imageManifest.Ref) + } + descriptor, err := buildManifestDescriptor(targetRepoInfo, imageManifest) + if err != nil { + return nil, err + } + descriptors = append(descriptors, descriptor) + } + + return manifestlist.FromDescriptors(descriptors) +} + +func buildManifestDescriptor(targetRepo *registry.RepositoryInfo, imageManifest types.ImageManifest) (manifestlist.ManifestDescriptor, error) { + repoInfo, err := registry.ParseRepositoryInfo(imageManifest.Ref) + if err != nil { + return manifestlist.ManifestDescriptor{}, err + } + + manifestRepoHostname := reference.Domain(repoInfo.Name) + targetRepoHostname := reference.Domain(targetRepo.Name) + if manifestRepoHostname != targetRepoHostname { + return manifestlist.ManifestDescriptor{}, errors.Errorf("cannot use source images from a different registry than the target image: %s != %s", manifestRepoHostname, targetRepoHostname) + } + + mediaType, raw, err := imageManifest.Payload() + if err != nil { + return manifestlist.ManifestDescriptor{}, err + } + + manifest := manifestlist.ManifestDescriptor{ + Platform: imageManifest.Platform, + } + manifest.Descriptor.Digest = imageManifest.Digest + manifest.Size = int64(len(raw)) + manifest.MediaType = mediaType + + if err = manifest.Descriptor.Digest.Validate(); err != nil { + return manifestlist.ManifestDescriptor{}, errors.Wrapf(err, + "digest parse of image %q failed with error: %v", imageManifest.Ref) + } + + return manifest, nil +} + +func buildBlobRequestList(imageManifest types.ImageManifest, repoName reference.Named) ([]manifestBlob, error) { + var blobReqs []manifestBlob + + for _, blobDigest := range imageManifest.Blobs() { + canonical, err := reference.WithDigest(repoName, blobDigest) + if err != nil { + return nil, err + } + blobReqs = append(blobReqs, manifestBlob{canonical: canonical, os: imageManifest.Platform.OS}) + } + return blobReqs, nil +} + +func buildPutManifestRequest(imageManifest types.ImageManifest, targetRef reference.Named) (mountRequest, error) { + refWithoutTag, err := reference.WithName(targetRef.Name()) + if err != nil { + return mountRequest{}, err + } + mountRef, err := reference.WithDigest(refWithoutTag, imageManifest.Digest) + if err != nil { + return mountRequest{}, err + } + + // This indentation has to be added to ensure sha parity with the registry + v2ManifestBytes, err := json.MarshalIndent(imageManifest.SchemaV2Manifest, "", " ") + if err != nil { + return mountRequest{}, err + } + // indent only the DeserializedManifest portion of this, in order to maintain parity with the registry + // and not alter the sha + var v2Manifest schema2.DeserializedManifest + if err = v2Manifest.UnmarshalJSON(v2ManifestBytes); err != nil { + return mountRequest{}, err + } + imageManifest.SchemaV2Manifest = &v2Manifest + + return mountRequest{ref: mountRef, manifest: imageManifest}, err +} + +func pushList(ctx context.Context, dockerCli command.Cli, req pushRequest) error { + rclient := dockerCli.RegistryClient(req.insecure) + + if err := mountBlobs(ctx, rclient, req.targetRef, req.manifestBlobs); err != nil { + return err + } + if err := pushReferences(ctx, dockerCli.Out(), rclient, req.mountRequests); err != nil { + return err + } + dgst, err := rclient.PutManifest(ctx, req.targetRef, req.list) + if err != nil { + return err + } + + fmt.Fprintln(dockerCli.Out(), dgst.String()) + return nil +} + +func pushReferences(ctx context.Context, out io.Writer, client registryclient.RegistryClient, mounts []mountRequest) error { + for _, mount := range mounts { + newDigest, err := client.PutManifest(ctx, mount.ref, mount.manifest) + if err != nil { + return err + } + fmt.Fprintf(out, "Pushed ref %s with digest: %s\n", mount.ref, newDigest) + } + return nil +} + +func mountBlobs(ctx context.Context, client registryclient.RegistryClient, ref reference.Named, blobs []manifestBlob) error { + for _, blob := range blobs { + err := client.MountBlob(ctx, blob.canonical, ref) + switch err.(type) { + case nil: + case registryclient.ErrBlobCreated: + if blob.os != "windows" { + return fmt.Errorf("error mounting %s to %s", blob.canonical, ref) + } + default: + return err + } + } + return nil +} diff --git a/cli/command/manifest/testdata/inspect-manifest.golden b/cli/command/manifest/testdata/inspect-manifest.golden new file mode 100644 index 0000000000..7089d9bddc --- /dev/null +++ b/cli/command/manifest/testdata/inspect-manifest.golden @@ -0,0 +1,16 @@ +{ + "schemaVersion": 2, + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "config": { + "mediaType": "application/vnd.docker.container.image.v1+json", + "size": 1520, + "digest": "sha256:7328f6f8b41890597575cbaadc884e7386ae0acc53b747401ebce5cf0d624560" + }, + "layers": [ + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 1990402, + "digest": "sha256:88286f41530e93dffd4b964e1db22ce4939fffa4a4c665dab8591fbab03d4926" + } + ] +} diff --git a/cli/command/manifest/util.go b/cli/command/manifest/util.go new file mode 100644 index 0000000000..b8887c7968 --- /dev/null +++ b/cli/command/manifest/util.go @@ -0,0 +1,79 @@ +package manifest + +import ( + "github.com/docker/cli/cli/command" + "github.com/docker/cli/cli/manifest/store" + "github.com/docker/cli/cli/manifest/types" + "github.com/docker/distribution/reference" + "golang.org/x/net/context" +) + +type osArch struct { + os string + arch string +} + +// Remove any unsupported os/arch combo +// list of valid os/arch values (see "Optional Environment Variables" section +// of https://golang.org/doc/install/source +// Added linux/s390x as we know System z support already exists +var validOSArches = map[osArch]bool{ + {os: "darwin", arch: "386"}: true, + {os: "darwin", arch: "amd64"}: true, + {os: "darwin", arch: "arm"}: true, + {os: "darwin", arch: "arm64"}: true, + {os: "dragonfly", arch: "amd64"}: true, + {os: "freebsd", arch: "386"}: true, + {os: "freebsd", arch: "amd64"}: true, + {os: "freebsd", arch: "arm"}: true, + {os: "linux", arch: "386"}: true, + {os: "linux", arch: "amd64"}: true, + {os: "linux", arch: "arm"}: true, + {os: "linux", arch: "arm64"}: true, + {os: "linux", arch: "ppc64le"}: true, + {os: "linux", arch: "mips64"}: true, + {os: "linux", arch: "mips64le"}: true, + {os: "linux", arch: "s390x"}: true, + {os: "netbsd", arch: "386"}: true, + {os: "netbsd", arch: "amd64"}: true, + {os: "netbsd", arch: "arm"}: true, + {os: "openbsd", arch: "386"}: true, + {os: "openbsd", arch: "amd64"}: true, + {os: "openbsd", arch: "arm"}: true, + {os: "plan9", arch: "386"}: true, + {os: "plan9", arch: "amd64"}: true, + {os: "solaris", arch: "amd64"}: true, + {os: "windows", arch: "386"}: true, + {os: "windows", arch: "amd64"}: true, +} + +func isValidOSArch(os string, arch string) bool { + // check for existence of this combo + _, ok := validOSArches[osArch{os, arch}] + return ok +} + +func normalizeReference(ref string) (reference.Named, error) { + namedRef, err := reference.ParseNormalizedNamed(ref) + if err != nil { + return nil, err + } + if _, isDigested := namedRef.(reference.Canonical); !isDigested { + return reference.TagNameOnly(namedRef), nil + } + return namedRef, nil +} + +// getManifest from the local store, and fallback to the remote registry if it +// doesn't exist locally +func getManifest(ctx context.Context, dockerCli command.Cli, listRef, namedRef reference.Named, insecure bool) (types.ImageManifest, error) { + data, err := dockerCli.ManifestStore().Get(listRef, namedRef) + switch { + case store.IsNotFound(err): + return dockerCli.RegistryClient(insecure).GetManifest(ctx, namedRef) + case err != nil: + return types.ImageManifest{}, err + default: + return data, nil + } +} diff --git a/cli/command/registry.go b/cli/command/registry.go index f3958d8a46..f6e5ac4659 100644 --- a/cli/command/registry.go +++ b/cli/command/registry.go @@ -10,14 +10,13 @@ import ( "runtime" "strings" - "golang.org/x/net/context" - "github.com/docker/distribution/reference" "github.com/docker/docker/api/types" registrytypes "github.com/docker/docker/api/types/registry" "github.com/docker/docker/pkg/term" "github.com/docker/docker/registry" "github.com/pkg/errors" + "golang.org/x/net/context" ) // ElectAuthServer returns the default registry to use (by asking the daemon) diff --git a/cli/manifest/store/store.go b/cli/manifest/store/store.go new file mode 100644 index 0000000000..5fb57468b2 --- /dev/null +++ b/cli/manifest/store/store.go @@ -0,0 +1,147 @@ +package store + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" + + "github.com/docker/cli/cli/manifest/types" + "github.com/docker/distribution/reference" +) + +// Store manages local storage of image distribution manifests +type Store interface { + Remove(listRef reference.Reference) error + Get(listRef reference.Reference, manifest reference.Reference) (types.ImageManifest, error) + GetList(listRef reference.Reference) ([]types.ImageManifest, error) + Save(listRef reference.Reference, manifest reference.Reference, image types.ImageManifest) error +} + +// fsStore manages manifest files stored on the local filesystem +type fsStore struct { + root string +} + +// NewStore returns a new store for a local file path +func NewStore(root string) Store { + return &fsStore{root: root} +} + +// Remove a manifest list from local storage +func (s *fsStore) Remove(listRef reference.Reference) error { + path := filepath.Join(s.root, makeFilesafeName(listRef.String())) + return os.RemoveAll(path) +} + +// Get returns the local manifest +func (s *fsStore) Get(listRef reference.Reference, manifest reference.Reference) (types.ImageManifest, error) { + filename := manifestToFilename(s.root, listRef.String(), manifest.String()) + return s.getFromFilename(manifest, filename) +} + +func (s *fsStore) getFromFilename(ref reference.Reference, filename string) (types.ImageManifest, error) { + bytes, err := ioutil.ReadFile(filename) + switch { + case os.IsNotExist(err): + return types.ImageManifest{}, newNotFoundError(ref.String()) + case err != nil: + return types.ImageManifest{}, err + } + var manifestInfo types.ImageManifest + return manifestInfo, json.Unmarshal(bytes, &manifestInfo) +} + +// GetList returns all the local manifests for a transaction +func (s *fsStore) GetList(listRef reference.Reference) ([]types.ImageManifest, error) { + filenames, err := s.listManifests(listRef.String()) + switch { + case err != nil: + return nil, err + case filenames == nil: + return nil, newNotFoundError(listRef.String()) + } + + manifests := []types.ImageManifest{} + for _, filename := range filenames { + filename = filepath.Join(s.root, makeFilesafeName(listRef.String()), filename) + manifest, err := s.getFromFilename(listRef, filename) + if err != nil { + return nil, err + } + manifests = append(manifests, manifest) + } + return manifests, nil +} + +// listManifests stored in a transaction +func (s *fsStore) listManifests(transaction string) ([]string, error) { + transactionDir := filepath.Join(s.root, makeFilesafeName(transaction)) + fileInfos, err := ioutil.ReadDir(transactionDir) + switch { + case os.IsNotExist(err): + return nil, nil + case err != nil: + return nil, err + } + + filenames := []string{} + for _, info := range fileInfos { + filenames = append(filenames, info.Name()) + } + return filenames, nil +} + +// Save a manifest as part of a local manifest list +func (s *fsStore) Save(listRef reference.Reference, manifest reference.Reference, image types.ImageManifest) error { + if err := s.createManifestListDirectory(listRef.String()); err != nil { + return err + } + filename := manifestToFilename(s.root, listRef.String(), manifest.String()) + bytes, err := json.Marshal(image) + if err != nil { + return err + } + return ioutil.WriteFile(filename, bytes, 0644) +} + +func (s *fsStore) createManifestListDirectory(transaction string) error { + path := filepath.Join(s.root, makeFilesafeName(transaction)) + return os.MkdirAll(path, 0755) +} + +func manifestToFilename(root, manifestList, manifest string) string { + return filepath.Join(root, makeFilesafeName(manifestList), makeFilesafeName(manifest)) +} + +func makeFilesafeName(ref string) string { + fileName := strings.Replace(ref, ":", "-", -1) + return strings.Replace(fileName, "/", "_", -1) +} + +type notFoundError struct { + object string +} + +func newNotFoundError(ref string) *notFoundError { + return ¬FoundError{object: ref} +} + +func (n *notFoundError) Error() string { + return fmt.Sprintf("No such manifest: %s", n.object) +} + +// NotFound interface +func (n *notFoundError) NotFound() {} + +// IsNotFound returns true if the error is a not found error +func IsNotFound(err error) bool { + _, ok := err.(notFound) + return ok +} + +type notFound interface { + NotFound() +} diff --git a/cli/manifest/store/store_test.go b/cli/manifest/store/store_test.go new file mode 100644 index 0000000000..fdf51dd641 --- /dev/null +++ b/cli/manifest/store/store_test.go @@ -0,0 +1,131 @@ +package store + +import ( + "io/ioutil" + "os" + "testing" + + "github.com/docker/cli/cli/manifest/types" + "github.com/docker/distribution/reference" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type fakeRef struct { + name string +} + +func (f fakeRef) String() string { + return f.name +} + +func (f fakeRef) Name() string { + return f.name +} + +func ref(name string) fakeRef { + return fakeRef{name: name} +} + +func sref(t *testing.T, name string) *types.SerializableNamed { + named, err := reference.ParseNamed("example.com/" + name) + require.NoError(t, err) + return &types.SerializableNamed{Named: named} +} + +func newTestStore(t *testing.T) (Store, func()) { + tmpdir, err := ioutil.TempDir("", "manifest-store-test") + require.NoError(t, err) + + return NewStore(tmpdir), func() { os.RemoveAll(tmpdir) } +} + +func getFiles(t *testing.T, store Store) []os.FileInfo { + infos, err := ioutil.ReadDir(store.(*fsStore).root) + require.NoError(t, err) + return infos +} + +func TestStoreRemove(t *testing.T) { + store, cleanup := newTestStore(t) + defer cleanup() + + listRef := ref("list") + data := types.ImageManifest{Ref: sref(t, "abcdef")} + require.NoError(t, store.Save(listRef, ref("manifest"), data)) + require.Len(t, getFiles(t, store), 1) + + assert.NoError(t, store.Remove(listRef)) + assert.Len(t, getFiles(t, store), 0) +} + +func TestStoreSaveAndGet(t *testing.T) { + store, cleanup := newTestStore(t) + defer cleanup() + + listRef := ref("list") + data := types.ImageManifest{Ref: sref(t, "abcdef")} + err := store.Save(listRef, ref("exists"), data) + require.NoError(t, err) + + var testcases = []struct { + listRef reference.Reference + manifestRef reference.Reference + expected types.ImageManifest + expectedErr string + }{ + { + listRef: listRef, + manifestRef: ref("exists"), + expected: data, + }, + { + listRef: listRef, + manifestRef: ref("exist:does-not"), + expectedErr: "No such manifest: exist:does-not", + }, + { + listRef: ref("list:does-not-exist"), + manifestRef: ref("manifest:does-not-exist"), + expectedErr: "No such manifest: manifest:does-not-exist", + }, + } + + for _, testcase := range testcases { + actual, err := store.Get(testcase.listRef, testcase.manifestRef) + if testcase.expectedErr != "" { + assert.EqualError(t, err, testcase.expectedErr) + assert.True(t, IsNotFound(err)) + continue + } + if !assert.NoError(t, err, testcase.manifestRef.String()) { + continue + } + assert.Equal(t, testcase.expected, actual, testcase.manifestRef.String()) + } +} + +func TestStoreGetList(t *testing.T) { + store, cleanup := newTestStore(t) + defer cleanup() + + listRef := ref("list") + first := types.ImageManifest{Ref: sref(t, "first")} + require.NoError(t, store.Save(listRef, ref("first"), first)) + second := types.ImageManifest{Ref: sref(t, "second")} + require.NoError(t, store.Save(listRef, ref("exists"), second)) + + list, err := store.GetList(listRef) + require.NoError(t, err) + assert.Len(t, list, 2) +} + +func TestStoreGetListDoesNotExist(t *testing.T) { + store, cleanup := newTestStore(t) + defer cleanup() + + listRef := ref("list") + _, err := store.GetList(listRef) + assert.EqualError(t, err, "No such manifest: list") + assert.True(t, IsNotFound(err)) +} diff --git a/cli/manifest/types/types.go b/cli/manifest/types/types.go new file mode 100644 index 0000000000..f618fd2b03 --- /dev/null +++ b/cli/manifest/types/types.go @@ -0,0 +1,107 @@ +package types + +import ( + "encoding/json" + + "github.com/docker/distribution" + "github.com/docker/distribution/manifest/manifestlist" + "github.com/docker/distribution/manifest/schema2" + "github.com/docker/distribution/reference" + "github.com/opencontainers/go-digest" + "github.com/pkg/errors" +) + +// ImageManifest contains info to output for a manifest object. +type ImageManifest struct { + Ref *SerializableNamed + Digest digest.Digest + SchemaV2Manifest *schema2.DeserializedManifest `json:",omitempty"` + Platform manifestlist.PlatformSpec +} + +// Blobs returns the digests for all the blobs referenced by this manifest +func (i ImageManifest) Blobs() []digest.Digest { + digests := []digest.Digest{} + for _, descriptor := range i.SchemaV2Manifest.References() { + digests = append(digests, descriptor.Digest) + } + return digests +} + +// Payload returns the media type and bytes for the manifest +func (i ImageManifest) Payload() (string, []byte, error) { + switch { + case i.SchemaV2Manifest != nil: + return i.SchemaV2Manifest.Payload() + default: + return "", nil, errors.Errorf("%s has no payload", i.Ref) + } +} + +// References implements the distribution.Manifest interface. It delegates to +// the underlying manifest. +func (i ImageManifest) References() []distribution.Descriptor { + switch { + case i.SchemaV2Manifest != nil: + return i.SchemaV2Manifest.References() + default: + return nil + } +} + +// NewImageManifest returns a new ImageManifest object. The values for Platform +// are initialized from those in the image +func NewImageManifest(ref reference.Named, digest digest.Digest, img Image, manifest *schema2.DeserializedManifest) ImageManifest { + platform := manifestlist.PlatformSpec{ + OS: img.OS, + Architecture: img.Architecture, + OSVersion: img.OSVersion, + OSFeatures: img.OSFeatures, + } + return ImageManifest{ + Ref: &SerializableNamed{Named: ref}, + Digest: digest, + SchemaV2Manifest: manifest, + Platform: platform, + } +} + +// SerializableNamed is a reference.Named that can be serialzied and deserialized +// from JSON +type SerializableNamed struct { + reference.Named +} + +// UnmarshalJSON loads the Named reference from JSON bytes +func (s *SerializableNamed) UnmarshalJSON(b []byte) error { + var raw string + if err := json.Unmarshal(b, &raw); err != nil { + return errors.Wrapf(err, "invalid named reference bytes: %s", b) + } + var err error + s.Named, err = reference.ParseNamed(raw) + return err +} + +// MarshalJSON returns the JSON bytes representation +func (s *SerializableNamed) MarshalJSON() ([]byte, error) { + return json.Marshal(s.String()) +} + +// Image is the minimal set of fields required to set default platform settings +// on a manifest. +type Image struct { + Architecture string `json:"architecture,omitempty"` + OS string `json:"os,omitempty"` + OSVersion string `json:"os.version,omitempty"` + OSFeatures []string `json:"os.features,omitempty"` +} + +// NewImageFromJSON creates an Image configuration from json. +func NewImageFromJSON(src []byte) (*Image, error) { + img := &Image{} + if err := json.Unmarshal(src, img); err != nil { + return nil, err + } + return img, nil +} diff --git a/cli/registry/client/client.go b/cli/registry/client/client.go new file mode 100644 index 0000000000..19d45a55f1 --- /dev/null +++ b/cli/registry/client/client.go @@ -0,0 +1,183 @@ +package client + +import ( + "fmt" + "net/http" + "strings" + + manifesttypes "github.com/docker/cli/cli/manifest/types" + "github.com/docker/distribution" + "github.com/docker/distribution/reference" + distributionclient "github.com/docker/distribution/registry/client" + "github.com/docker/docker/api/types" + registrytypes "github.com/docker/docker/api/types/registry" + "github.com/opencontainers/go-digest" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "golang.org/x/net/context" +) + +// RegistryClient is a client used to communicate with a Docker distribution +// registry +type RegistryClient interface { + GetManifest(ctx context.Context, ref reference.Named) (manifesttypes.ImageManifest, error) + GetManifestList(ctx context.Context, ref reference.Named) ([]manifesttypes.ImageManifest, error) + MountBlob(ctx context.Context, source reference.Canonical, target reference.Named) error + PutManifest(ctx context.Context, ref reference.Named, manifest distribution.Manifest) (digest.Digest, error) +} + +// NewRegistryClient returns a new RegistryClient with a resolver +func NewRegistryClient(resolver AuthConfigResolver, userAgent string, insecure bool) RegistryClient { + return &client{ + authConfigResolver: resolver, + insecureRegistry: insecure, + userAgent: userAgent, + } +} + +// AuthConfigResolver returns Auth Configuration for an index +type AuthConfigResolver func(ctx context.Context, index *registrytypes.IndexInfo) types.AuthConfig + +// PutManifestOptions is the data sent to push a manifest +type PutManifestOptions struct { + MediaType string + Payload []byte +} + +type client struct { + authConfigResolver AuthConfigResolver + insecureRegistry bool + userAgent string +} + +// ErrBlobCreated returned when a blob mount request was created +type ErrBlobCreated struct { + From reference.Named + Target reference.Named +} + +func (err ErrBlobCreated) Error() string { + return fmt.Sprintf("blob mounted from: %v to: %v", + err.From, err.Target) +} + +// ErrHTTPProto returned if attempting to use TLS with a non-TLS registry +type ErrHTTPProto struct { + OrigErr string +} + +func (err ErrHTTPProto) Error() string { + return err.OrigErr +} + +var _ RegistryClient = &client{} + +// MountBlob into the registry, so it can be referenced by a manifest +func (c *client) MountBlob(ctx context.Context, sourceRef reference.Canonical, targetRef reference.Named) error { + repoEndpoint, err := newDefaultRepositoryEndpoint(targetRef, c.insecureRegistry) + if err != nil { + return err + } + repo, err := c.getRepositoryForReference(ctx, targetRef, repoEndpoint) + if err != nil { + return err + } + lu, err := repo.Blobs(ctx).Create(ctx, distributionclient.WithMountFrom(sourceRef)) + switch err.(type) { + case distribution.ErrBlobMounted: + logrus.Debugf("mount of blob %s succeeded", sourceRef) + return nil + case nil: + default: + return errors.Wrapf(err, "failed to mount blob %s to %s", sourceRef, targetRef) + } + lu.Cancel(ctx) + logrus.Debugf("mount of blob %s created", sourceRef) + return ErrBlobCreated{From: sourceRef, Target: targetRef} +} + +// PutManifest sends the manifest to a registry and returns the new digest +func (c *client) PutManifest(ctx context.Context, ref reference.Named, manifest distribution.Manifest) (digest.Digest, error) { + repoEndpoint, err := newDefaultRepositoryEndpoint(ref, c.insecureRegistry) + if err != nil { + return digest.Digest(""), err + } + + repo, err := c.getRepositoryForReference(ctx, ref, repoEndpoint) + if err != nil { + return digest.Digest(""), err + } + + manifestService, err := repo.Manifests(ctx) + if err != nil { + return digest.Digest(""), err + } + + _, opts, err := getManifestOptionsFromReference(ref) + if err != nil { + return digest.Digest(""), err + } + + dgst, err := manifestService.Put(ctx, manifest, opts...) + return dgst, errors.Wrapf(err, "failed to put manifest %s", ref) +} + +func (c *client) getRepositoryForReference(ctx context.Context, ref reference.Named, repoEndpoint repositoryEndpoint) (distribution.Repository, error) { + httpTransport, err := c.getHTTPTransportForRepoEndpoint(ctx, repoEndpoint) + if err != nil { + if strings.Contains(err.Error(), "server gave HTTP response to HTTPS client") { + return nil, ErrHTTPProto{OrigErr: err.Error()} + } + } + repoName, err := reference.WithName(repoEndpoint.Name()) + if err != nil { + return nil, errors.Wrapf(err, "failed to parse repo name from %s", ref) + } + return distributionclient.NewRepository(ctx, repoName, repoEndpoint.BaseURL(), httpTransport) +} + +func (c *client) getHTTPTransportForRepoEndpoint(ctx context.Context, repoEndpoint repositoryEndpoint) (http.RoundTripper, error) { + httpTransport, err := getHTTPTransport( + c.authConfigResolver(ctx, repoEndpoint.info.Index), + repoEndpoint.endpoint, + repoEndpoint.Name(), + c.userAgent) + return httpTransport, errors.Wrap(err, "failed to configure transport") +} + +// GetManifest returns an ImageManifest for the reference +func (c *client) GetManifest(ctx context.Context, ref reference.Named) (manifesttypes.ImageManifest, error) { + var result manifesttypes.ImageManifest + fetch := func(ctx context.Context, repo distribution.Repository, ref reference.Named) (bool, error) { + var err error + result, err = fetchManifest(ctx, repo, ref) + return result.Ref != nil, err + } + + err := c.iterateEndpoints(ctx, ref, fetch) + return result, err +} + +// GetManifestList returns a list of ImageManifest for the reference +func (c *client) GetManifestList(ctx context.Context, ref reference.Named) ([]manifesttypes.ImageManifest, error) { + result := []manifesttypes.ImageManifest{} + fetch := func(ctx context.Context, repo distribution.Repository, ref reference.Named) (bool, error) { + var err error + result, err = fetchList(ctx, repo, ref) + return len(result) > 0, err + } + + err := c.iterateEndpoints(ctx, ref, fetch) + return result, err +} + +func getManifestOptionsFromReference(ref reference.Named) (digest.Digest, []distribution.ManifestServiceOption, error) { + if tagged, isTagged := ref.(reference.NamedTagged); isTagged { + tag := tagged.Tag() + return "", []distribution.ManifestServiceOption{distribution.WithTag(tag)}, nil + } + if digested, isDigested := ref.(reference.Canonical); isDigested { + return digested.Digest(), []distribution.ManifestServiceOption{}, nil + } + return "", nil, errors.Errorf("%s no tag or digest", ref) +} diff --git a/cli/registry/client/endpoint.go b/cli/registry/client/endpoint.go new file mode 100644 index 0000000000..a2d9c3359d --- /dev/null +++ b/cli/registry/client/endpoint.go @@ -0,0 +1,133 @@ +package client + +import ( + "fmt" + "net" + "net/http" + "time" + + "github.com/docker/distribution/reference" + "github.com/docker/distribution/registry/client/auth" + "github.com/docker/distribution/registry/client/transport" + authtypes "github.com/docker/docker/api/types" + "github.com/docker/docker/registry" + "github.com/pkg/errors" +) + +type repositoryEndpoint struct { + info *registry.RepositoryInfo + endpoint registry.APIEndpoint +} + +// Name returns the repository name +func (r repositoryEndpoint) Name() string { + repoName := r.info.Name.Name() + // If endpoint does not support CanonicalName, use the RemoteName instead + if r.endpoint.TrimHostname { + repoName = reference.Path(r.info.Name) + } + return repoName +} + +// BaseURL returns the endpoint url +func (r repositoryEndpoint) BaseURL() string { + return r.endpoint.URL.String() +} + +func newDefaultRepositoryEndpoint(ref reference.Named, insecure bool) (repositoryEndpoint, error) { + repoInfo, err := registry.ParseRepositoryInfo(ref) + if err != nil { + return repositoryEndpoint{}, err + } + endpoint, err := getDefaultEndpointFromRepoInfo(repoInfo) + if err != nil { + return repositoryEndpoint{}, err + } + if insecure { + endpoint.TLSConfig.InsecureSkipVerify = true + } + return repositoryEndpoint{info: repoInfo, endpoint: endpoint}, nil +} + +func getDefaultEndpointFromRepoInfo(repoInfo *registry.RepositoryInfo) (registry.APIEndpoint, error) { + var err error + + options := registry.ServiceOptions{} + registryService, err := registry.NewService(options) + if err != nil { + return registry.APIEndpoint{}, err + } + endpoints, err := registryService.LookupPushEndpoints(reference.Domain(repoInfo.Name)) + if err != nil { + return registry.APIEndpoint{}, err + } + // Default to the highest priority endpoint to return + endpoint := endpoints[0] + if !repoInfo.Index.Secure { + for _, ep := range endpoints { + if ep.URL.Scheme == "http" { + endpoint = ep + } + } + } + return endpoint, nil +} + +// getHTTPTransport builds a transport for use in communicating with a registry +func getHTTPTransport(authConfig authtypes.AuthConfig, endpoint registry.APIEndpoint, repoName string, userAgent string) (http.RoundTripper, error) { + // get the http transport, this will be used in a client to upload manifest + base := &http.Transport{ + Proxy: http.ProxyFromEnvironment, + Dial: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + DualStack: true, + }).Dial, + TLSHandshakeTimeout: 10 * time.Second, + TLSClientConfig: endpoint.TLSConfig, + DisableKeepAlives: true, + } + + modifiers := registry.Headers(userAgent, http.Header{}) + authTransport := transport.NewTransport(base, modifiers...) + challengeManager, confirmedV2, err := registry.PingV2Registry(endpoint.URL, authTransport) + if err != nil { + return nil, errors.Wrap(err, "error pinging v2 registry") + } + if !confirmedV2 { + return nil, fmt.Errorf("unsupported registry version") + } + if authConfig.RegistryToken != "" { + passThruTokenHandler := &existingTokenHandler{token: authConfig.RegistryToken} + modifiers = append(modifiers, auth.NewAuthorizer(challengeManager, passThruTokenHandler)) + } else { + creds := registry.NewStaticCredentialStore(&authConfig) + tokenHandler := auth.NewTokenHandler(authTransport, creds, repoName, "*") + basicHandler := auth.NewBasicHandler(creds) + modifiers = append(modifiers, auth.NewAuthorizer(challengeManager, tokenHandler, basicHandler)) + } + return transport.NewTransport(base, modifiers...), nil +} + +// RepoNameForReference returns the repository name from a reference +func RepoNameForReference(ref reference.Named) (string, error) { + // insecure is fine since this only returns the name + repo, err := newDefaultRepositoryEndpoint(ref, false) + if err != nil { + return "", err + } + return repo.Name(), nil +} + +type existingTokenHandler struct { + token string +} + +func (th *existingTokenHandler) AuthorizeRequest(req *http.Request, params map[string]string) error { + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", th.token)) + return nil +} + +func (th *existingTokenHandler) Scheme() string { + return "bearer" +} diff --git a/cli/registry/client/fetcher.go b/cli/registry/client/fetcher.go new file mode 100644 index 0000000000..1e748f255d --- /dev/null +++ b/cli/registry/client/fetcher.go @@ -0,0 +1,295 @@ +package client + +import ( + "fmt" + + "github.com/docker/cli/cli/manifest/types" + "github.com/docker/distribution" + "github.com/docker/distribution/manifest/manifestlist" + "github.com/docker/distribution/manifest/schema2" + "github.com/docker/distribution/reference" + "github.com/docker/distribution/registry/api/errcode" + "github.com/docker/distribution/registry/api/v2" + distclient "github.com/docker/distribution/registry/client" + "github.com/docker/docker/registry" + digest "github.com/opencontainers/go-digest" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "golang.org/x/net/context" +) + +// fetchManifest pulls a manifest from a registry and returns it. An error +// is returned if no manifest is found matching namedRef. +func fetchManifest(ctx context.Context, repo distribution.Repository, ref reference.Named) (types.ImageManifest, error) { + manifest, err := getManifest(ctx, repo, ref) + if err != nil { + return types.ImageManifest{}, err + } + + switch v := manifest.(type) { + // Removed Schema 1 support + case *schema2.DeserializedManifest: + imageManifest, err := pullManifestSchemaV2(ctx, ref, repo, *v) + if err != nil { + return types.ImageManifest{}, err + } + return imageManifest, nil + case *manifestlist.DeserializedManifestList: + return types.ImageManifest{}, errors.Errorf("%s is a manifest list", ref) + } + return types.ImageManifest{}, errors.Errorf("%s is not a manifest", ref) +} + +func fetchList(ctx context.Context, repo distribution.Repository, ref reference.Named) ([]types.ImageManifest, error) { + manifest, err := getManifest(ctx, repo, ref) + if err != nil { + return nil, err + } + + switch v := manifest.(type) { + case *manifestlist.DeserializedManifestList: + imageManifests, err := pullManifestList(ctx, ref, repo, *v) + if err != nil { + return nil, err + } + return imageManifests, nil + default: + return nil, errors.Errorf("unsupported manifest format: %v", v) + } +} + +func getManifest(ctx context.Context, repo distribution.Repository, ref reference.Named) (distribution.Manifest, error) { + manSvc, err := repo.Manifests(ctx) + if err != nil { + return nil, err + } + + dgst, opts, err := getManifestOptionsFromReference(ref) + if err != nil { + return nil, errors.Errorf("image manifest for %q does not exist", ref) + } + return manSvc.Get(ctx, dgst, opts...) +} + +func pullManifestSchemaV2(ctx context.Context, ref reference.Named, repo distribution.Repository, mfst schema2.DeserializedManifest) (types.ImageManifest, error) { + manifestDigest, err := validateManifestDigest(ref, mfst) + if err != nil { + return types.ImageManifest{}, err + } + configJSON, err := pullManifestSchemaV2ImageConfig(ctx, mfst.Target().Digest, repo) + if err != nil { + return types.ImageManifest{}, err + } + + img, err := types.NewImageFromJSON(configJSON) + if err != nil { + return types.ImageManifest{}, err + } + return types.NewImageManifest(ref, manifestDigest, *img, &mfst), nil +} + +func pullManifestSchemaV2ImageConfig(ctx context.Context, dgst digest.Digest, repo distribution.Repository) ([]byte, error) { + blobs := repo.Blobs(ctx) + configJSON, err := blobs.Get(ctx, dgst) + if err != nil { + return nil, err + } + + verifier := dgst.Verifier() + if err != nil { + return nil, err + } + if _, err := verifier.Write(configJSON); err != nil { + return nil, err + } + if !verifier.Verified() { + return nil, errors.Errorf("image config verification failed for digest %s", dgst) + } + return configJSON, nil +} + +// validateManifestDigest computes the manifest digest, and, if pulling by +// digest, ensures that it matches the requested digest. +func validateManifestDigest(ref reference.Named, mfst distribution.Manifest) (digest.Digest, error) { + _, canonical, err := mfst.Payload() + if err != nil { + return "", err + } + + // If pull by digest, then verify the manifest digest. + if digested, isDigested := ref.(reference.Canonical); isDigested { + verifier := digested.Digest().Verifier() + if err != nil { + return "", err + } + if _, err := verifier.Write(canonical); err != nil { + return "", err + } + if !verifier.Verified() { + err := fmt.Errorf("manifest verification failed for digest %s", digested.Digest()) + return "", err + } + return digested.Digest(), nil + } + + return digest.FromBytes(canonical), nil +} + +// pullManifestList handles "manifest lists" which point to various +// platform-specific manifests. +func pullManifestList(ctx context.Context, ref reference.Named, repo distribution.Repository, mfstList manifestlist.DeserializedManifestList) ([]types.ImageManifest, error) { + infos := []types.ImageManifest{} + + if _, err := validateManifestDigest(ref, mfstList); err != nil { + return nil, err + } + + for _, manifestDescriptor := range mfstList.Manifests { + manSvc, err := repo.Manifests(ctx) + if err != nil { + return nil, err + } + manifest, err := manSvc.Get(ctx, manifestDescriptor.Digest) + if err != nil { + return nil, err + } + v, ok := manifest.(*schema2.DeserializedManifest) + if !ok { + return nil, fmt.Errorf("unsupported manifest format: %s", v) + } + + manifestRef, err := reference.WithDigest(ref, manifestDescriptor.Digest) + if err != nil { + return nil, err + } + imageManifest, err := pullManifestSchemaV2(ctx, manifestRef, repo, *v) + if err != nil { + return nil, err + } + imageManifest.Platform = manifestDescriptor.Platform + infos = append(infos, imageManifest) + } + return infos, nil +} + +func continueOnError(err error) bool { + switch v := err.(type) { + case errcode.Errors: + if len(v) == 0 { + return true + } + return continueOnError(v[0]) + case errcode.Error: + e := err.(errcode.Error) + switch e.Code { + case errcode.ErrorCodeUnauthorized, v2.ErrorCodeManifestUnknown, v2.ErrorCodeNameUnknown: + return true + } + return false + case *distclient.UnexpectedHTTPResponseError: + return true + } + return false +} + +func (c *client) iterateEndpoints(ctx context.Context, namedRef reference.Named, each func(context.Context, distribution.Repository, reference.Named) (bool, error)) error { + endpoints, err := allEndpoints(namedRef) + if err != nil { + return err + } + + repoInfo, err := registry.ParseRepositoryInfo(namedRef) + if err != nil { + return err + } + + confirmedTLSRegistries := make(map[string]bool) + for _, endpoint := range endpoints { + + if endpoint.Version == registry.APIVersion1 { + logrus.Debugf("skipping v1 endpoint %s", endpoint.URL) + continue + } + + if endpoint.URL.Scheme != "https" { + if _, confirmedTLS := confirmedTLSRegistries[endpoint.URL.Host]; confirmedTLS { + logrus.Debugf("skipping non-TLS endpoint %s for host/port that appears to use TLS", endpoint.URL) + continue + } + } + + if c.insecureRegistry { + endpoint.TLSConfig.InsecureSkipVerify = true + } + repoEndpoint := repositoryEndpoint{endpoint: endpoint, info: repoInfo} + repo, err := c.getRepositoryForReference(ctx, namedRef, repoEndpoint) + if err != nil { + logrus.Debugf("error with repo endpoint %s: %s", repoEndpoint, err) + if _, ok := err.(ErrHTTPProto); ok { + continue + } + return err + } + + if endpoint.URL.Scheme == "http" && !c.insecureRegistry { + logrus.Debugf("skipping non-tls registry endpoint: %s", endpoint.URL) + continue + } + done, err := each(ctx, repo, namedRef) + if err != nil { + if continueOnError(err) { + if endpoint.URL.Scheme == "https" { + confirmedTLSRegistries[endpoint.URL.Host] = true + } + logrus.Debugf("continuing on error (%T) %s", err, err) + continue + } + logrus.Debugf("not continuing on error (%T) %s", err, err) + return err + } + if done { + return nil + } + } + return newNotFoundError(namedRef.String()) +} + +// allEndpoints returns a list of endpoints ordered by priority (v2, https, v1). +func allEndpoints(namedRef reference.Named) ([]registry.APIEndpoint, error) { + repoInfo, err := registry.ParseRepositoryInfo(namedRef) + if err != nil { + return nil, err + } + registryService, err := registry.NewService(registry.ServiceOptions{}) + if err != nil { + return []registry.APIEndpoint{}, err + } + endpoints, err := registryService.LookupPullEndpoints(reference.Domain(repoInfo.Name)) + logrus.Debugf("endpoints for %s: %v", namedRef, endpoints) + return endpoints, err +} + +type notFoundError struct { + object string +} + +func newNotFoundError(ref string) *notFoundError { + return ¬FoundError{object: ref} +} + +func (n *notFoundError) Error() string { + return fmt.Sprintf("no such manifest: %s", n.object) +} + +// NotFound interface +func (n *notFoundError) NotFound() {} + +// IsNotFound returns true if the error is a not found error +func IsNotFound(err error) bool { + _, ok := err.(notFound) + return ok +} + +type notFound interface { + NotFound() +} diff --git a/internal/test/cli.go b/internal/test/cli.go index 5fffde64ee..e99e8e2fbd 100644 --- a/internal/test/cli.go +++ b/internal/test/cli.go @@ -10,6 +10,8 @@ import ( "github.com/docker/cli/cli/command" "github.com/docker/cli/cli/config/configfile" "github.com/docker/cli/cli/trust" + manifeststore "github.com/docker/cli/cli/manifest/store" + registryclient "github.com/docker/cli/cli/registry/client" "github.com/docker/docker/client" notaryclient "github.com/theupdateframework/notary/client" ) @@ -20,15 +22,16 @@ type clientInfoFuncType func() command.ClientInfo // FakeCli emulates the default DockerCli type FakeCli struct { command.DockerCli - client client.APIClient - configfile *configfile.ConfigFile - out *command.OutStream - outBuffer *bytes.Buffer - err *bytes.Buffer - in *command.InStream - server command.ServerInfo - clientInfoFunc clientInfoFuncType + client client.APIClient + configfile *configfile.ConfigFile + out *command.OutStream + outBuffer *bytes.Buffer + err *bytes.Buffer + in *command.InStream + server command.ServerInfo notaryClientFunc notaryClientFuncType + manifestStore manifeststore.Store + registryClient registryclient.RegistryClient } // NewFakeCli returns a fake for the command.Cli interface @@ -124,4 +127,23 @@ func (c *FakeCli) NotaryClient(imgRefAndAuth trust.ImageRefAndAuth, actions []st return c.notaryClientFunc(imgRefAndAuth, actions) } return nil, fmt.Errorf("no notary client available unless defined") + +// ManifestStore returns a fake store used for testing +func (c *FakeCli) ManifestStore() manifeststore.Store { + return c.manifestStore +} + +// RegistryClient returns a fake client for testing +func (c *FakeCli) RegistryClient(insecure bool) registryclient.RegistryClient { + return c.registryClient +} + +// SetManifestStore on the fake cli +func (c *FakeCli) SetManifestStore(store manifeststore.Store) { + c.manifestStore = store +} + +// SetRegistryClient on the fake cli +func (c *FakeCli) SetRegistryClient(client registryclient.RegistryClient) { + c.registryClient = client }