Merge pull request #3990 from jedevc/manifest-oci

Add OCI support to manifest subcommand

cli/command/manifest/push.go

@@ -12,6 +12,7 @@ import (
 	registryclient "github.com/docker/cli/cli/registry/client"
 	"github.com/docker/distribution"
 	"github.com/docker/distribution/manifest/manifestlist"
+	"github.com/docker/distribution/manifest/ocischema"
 	"github.com/docker/distribution/manifest/schema2"
 	"github.com/docker/distribution/reference"
 	"github.com/docker/docker/registry"
@@ -217,18 +218,53 @@ func buildPutManifestRequest(imageManifest types.ImageManifest, targetRef refere
 		return mountRequest{}, err
 	}
 
-	// This indentation has to be added to ensure sha parity with the registry
-	v2ManifestBytes, err := json.MarshalIndent(imageManifest.SchemaV2Manifest, "", "   ")
+	// Attempt to reconstruct indentation of the manifest to ensure sha parity
+	// with the registry - if we haven't preserved the raw content.
+	//
+	// This is necessary because our previous internal storage format did not
+	// preserve whitespace. If we don't have the newer format present, we can
+	// attempt the reconstruction like before, but explicitly error if the
+	// reconstruction failed!
+	switch {
+	case imageManifest.SchemaV2Manifest != nil:
+		dt := imageManifest.Raw
+		if len(dt) == 0 {
+			dt, err = json.MarshalIndent(imageManifest.SchemaV2Manifest, "", "   ")
 			if err != nil {
 				return mountRequest{}, err
 			}
-	// indent only the DeserializedManifest portion of this, in order to maintain parity with the registry
-	// and not alter the sha
-	var v2Manifest schema2.DeserializedManifest
-	if err = v2Manifest.UnmarshalJSON(v2ManifestBytes); err != nil {
+		}
+
+		dig := imageManifest.Descriptor.Digest
+		if dig2 := dig.Algorithm().FromBytes(dt); dig != dig2 {
+			return mountRequest{}, errors.Errorf("internal digest mismatch for %s: expected %s, got %s", imageManifest.Ref, dig, dig2)
+		}
+
+		var manifest schema2.DeserializedManifest
+		if err = manifest.UnmarshalJSON(dt); err != nil {
 			return mountRequest{}, err
 		}
-	imageManifest.SchemaV2Manifest = &v2Manifest
+		imageManifest.SchemaV2Manifest = &manifest
+	case imageManifest.OCIManifest != nil:
+		dt := imageManifest.Raw
+		if len(dt) == 0 {
+			dt, err = json.MarshalIndent(imageManifest.OCIManifest, "", "  ")
+			if err != nil {
+				return mountRequest{}, err
+			}
+		}
+
+		dig := imageManifest.Descriptor.Digest
+		if dig2 := dig.Algorithm().FromBytes(dt); dig != dig2 {
+			return mountRequest{}, errors.Errorf("internal digest mismatch for %s: expected %s, got %s", imageManifest.Ref, dig, dig2)
+		}
+
+		var manifest ocischema.DeserializedManifest
+		if err = manifest.UnmarshalJSON(dt); err != nil {
+			return mountRequest{}, err
+		}
+		imageManifest.OCIManifest = &manifest
+	}
 
 	return mountRequest{ref: mountRef, manifest: imageManifest}, err
 }

cli/command/manifest/testdata/inspect-annotate.golden

@@ -14,6 +14,7 @@
 			"variant": "v7"
 		}
 	},
+	"Raw": "ewogICAic2NoZW1hVmVyc2lvbiI6IDIsCiAgICJtZWRpYVR5cGUiOiAiYXBwbGljYXRpb24vdm5kLmRvY2tlci5kaXN0cmlidXRpb24ubWFuaWZlc3QudjIranNvbiIsCiAgICJjb25maWciOiB7CiAgICAgICJtZWRpYVR5cGUiOiAiYXBwbGljYXRpb24vdm5kLmRvY2tlci5jb250YWluZXIuaW1hZ2UudjEranNvbiIsCiAgICAgICJzaXplIjogMTUyMCwKICAgICAgImRpZ2VzdCI6ICJzaGEyNTY6NzMyOGY2ZjhiNDE4OTA1OTc1NzVjYmFhZGM4ODRlNzM4NmFlMGFjYzUzYjc0NzQwMWViY2U1Y2YwZDYyNDU2MCIKICAgfSwKICAgImxheWVycyI6IFsKICAgICAgewogICAgICAgICAibWVkaWFUeXBlIjogImFwcGxpY2F0aW9uL3ZuZC5kb2NrZXIuaW1hZ2Uucm9vdGZzLmRpZmYudGFyLmd6aXAiLAogICAgICAgICAic2l6ZSI6IDE5OTA0MDIsCiAgICAgICAgICJkaWdlc3QiOiAic2hhMjU2Ojg4Mjg2ZjQxNTMwZTkzZGZmZDRiOTY0ZTFkYjIyY2U0OTM5ZmZmYTRhNGM2NjVkYWI4NTkxZmJhYjAzZDQ5MjYiCiAgICAgIH0KICAgXQp9",
 	"SchemaV2Manifest": {
 		"schemaVersion": 2,
 		"mediaType": "application/vnd.docker.distribution.manifest.v2+json",

cli/command/manifest/util.go

@@ -76,6 +76,8 @@ func getManifest(ctx context.Context, dockerCli command.Cli, listRef, namedRef r
 		return dockerCli.RegistryClient(insecure).GetManifest(ctx, namedRef)
 	case err != nil:
 		return types.ImageManifest{}, err
+	case len(data.Raw) == 0:
+		return dockerCli.RegistryClient(insecure).GetManifest(ctx, namedRef)
 	default:
 		return data, nil
 	}

cli/manifest/types/types.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/docker/distribution"
 	"github.com/docker/distribution/manifest/manifestlist"
+	"github.com/docker/distribution/manifest/ocischema"
 	"github.com/docker/distribution/manifest/schema2"
 	"github.com/docker/distribution/reference"
 	"github.com/opencontainers/go-digest"
@@ -16,10 +17,12 @@ import (
 type ImageManifest struct {
 	Ref        *SerializableNamed
 	Descriptor ocispec.Descriptor
+	Raw        []byte `json:",omitempty"`
 
 	// SchemaV2Manifest is used for inspection
-	// TODO: Deprecate this and store manifest blobs
 	SchemaV2Manifest *schema2.DeserializedManifest `json:",omitempty"`
+	// OCIManifest is used for inspection
+	OCIManifest *ocischema.DeserializedManifest `json:",omitempty"`
 }
 
 // OCIPlatform creates an OCI platform from a manifest list platform spec
@@ -53,9 +56,16 @@ func PlatformSpecFromOCI(p *ocispec.Platform) *manifestlist.PlatformSpec {
 // Blobs returns the digests for all the blobs referenced by this manifest
 func (i ImageManifest) Blobs() []digest.Digest {
 	digests := []digest.Digest{}
+	switch {
+	case i.SchemaV2Manifest != nil:
 		for _, descriptor := range i.SchemaV2Manifest.References() {
 			digests = append(digests, descriptor.Digest)
 		}
+	case i.OCIManifest != nil:
+		for _, descriptor := range i.OCIManifest.References() {
+			digests = append(digests, descriptor.Digest)
+		}
+	}
 	return digests
 }
 
@@ -65,6 +75,8 @@ func (i ImageManifest) Payload() (string, []byte, error) {
 	switch {
 	case i.SchemaV2Manifest != nil:
 		return i.SchemaV2Manifest.Payload()
+	case i.OCIManifest != nil:
+		return i.OCIManifest.Payload()
 	default:
 		return "", nil, errors.Errorf("%s has no payload", i.Ref)
 	}
@@ -76,6 +88,8 @@ func (i ImageManifest) References() []distribution.Descriptor {
 	switch {
 	case i.SchemaV2Manifest != nil:
 		return i.SchemaV2Manifest.References()
+	case i.OCIManifest != nil:
+		return i.OCIManifest.References()
 	default:
 		return nil
 	}
@@ -84,13 +98,35 @@ func (i ImageManifest) References() []distribution.Descriptor {
 // NewImageManifest returns a new ImageManifest object. The values for Platform
 // are initialized from those in the image
 func NewImageManifest(ref reference.Named, desc ocispec.Descriptor, manifest *schema2.DeserializedManifest) ImageManifest {
+	raw, err := manifest.MarshalJSON()
+	if err != nil {
+		raw = nil
+	}
+
 	return ImageManifest{
 		Ref:              &SerializableNamed{Named: ref},
 		Descriptor:       desc,
+		Raw:              raw,
 		SchemaV2Manifest: manifest,
 	}
 }
 
+// NewOCIImageManifest returns a new ImageManifest object. The values for
+// Platform are initialized from those in the image
+func NewOCIImageManifest(ref reference.Named, desc ocispec.Descriptor, manifest *ocischema.DeserializedManifest) ImageManifest {
+	raw, err := manifest.MarshalJSON()
+	if err != nil {
+		raw = nil
+	}
+
+	return ImageManifest{
+		Ref:         &SerializableNamed{Named: ref},
+		Descriptor:  desc,
+		Raw:         raw,
+		OCIManifest: manifest,
+	}
+}
+
 // SerializableNamed is a reference.Named that can be serialized and deserialized
 // from JSON
 type SerializableNamed struct {

cli/registry/client/fetcher.go

@@ -7,6 +7,7 @@ import (
 	"github.com/docker/cli/cli/manifest/types"
 	"github.com/docker/distribution"
 	"github.com/docker/distribution/manifest/manifestlist"
+	"github.com/docker/distribution/manifest/ocischema"
 	"github.com/docker/distribution/manifest/schema2"
 	"github.com/docker/distribution/reference"
 	"github.com/docker/distribution/registry/api/errcode"
@@ -35,6 +36,12 @@ func fetchManifest(ctx context.Context, repo distribution.Repository, ref refere
 			return types.ImageManifest{}, err
 		}
 		return imageManifest, nil
+	case *ocischema.DeserializedManifest:
+		imageManifest, err := pullManifestOCISchema(ctx, ref, repo, *v)
+		if err != nil {
+			return types.ImageManifest{}, err
+		}
+		return imageManifest, nil
 	case *manifestlist.DeserializedManifestList:
 		return types.ImageManifest{}, errors.Errorf("%s is a manifest list", ref)
 	}
@@ -94,6 +101,28 @@ func pullManifestSchemaV2(ctx context.Context, ref reference.Named, repo distrib
 	return types.NewImageManifest(ref, manifestDesc, &mfst), nil
 }
 
+func pullManifestOCISchema(ctx context.Context, ref reference.Named, repo distribution.Repository, mfst ocischema.DeserializedManifest) (types.ImageManifest, error) {
+	manifestDesc, err := validateManifestDigest(ref, mfst)
+	if err != nil {
+		return types.ImageManifest{}, err
+	}
+	configJSON, err := pullManifestSchemaV2ImageConfig(ctx, mfst.Target().Digest, repo)
+	if err != nil {
+		return types.ImageManifest{}, err
+	}
+
+	if manifestDesc.Platform == nil {
+		manifestDesc.Platform = &ocispec.Platform{}
+	}
+
+	// Fill in os and architecture fields from config JSON
+	if err := json.Unmarshal(configJSON, manifestDesc.Platform); err != nil {
+		return types.ImageManifest{}, err
+	}
+
+	return types.NewOCIImageManifest(ref, manifestDesc, &mfst), nil
+}
+
 func pullManifestSchemaV2ImageConfig(ctx context.Context, dgst digest.Digest, repo distribution.Repository) ([]byte, error) {
 	blobs := repo.Blobs(ctx)
 	configJSON, err := blobs.Get(ctx, dgst)
@@ -153,16 +182,21 @@ func pullManifestList(ctx context.Context, ref reference.Named, repo distributio
 		if err != nil {
 			return nil, err
 		}
-		v, ok := manifest.(*schema2.DeserializedManifest)
-		if !ok {
-			return nil, errors.Errorf("unsupported manifest format: %v", v)
-		}
 
 		manifestRef, err := reference.WithDigest(ref, manifestDescriptor.Digest)
 		if err != nil {
 			return nil, err
 		}
-		imageManifest, err := pullManifestSchemaV2(ctx, manifestRef, repo, *v)
+
+		var imageManifest types.ImageManifest
+		switch v := manifest.(type) {
+		case *schema2.DeserializedManifest:
+			imageManifest, err = pullManifestSchemaV2(ctx, manifestRef, repo, *v)
+		case *ocischema.DeserializedManifest:
+			imageManifest, err = pullManifestOCISchema(ctx, manifestRef, repo, *v)
+		default:
+			err = errors.Errorf("unsupported manifest type: %T", manifest)
+		}
 		if err != nil {
 			return nil, err
 		}

vendor/github.com/docker/distribution/manifest/ocischema/builder.go

@@ -0,0 +1,107 @@
+package ocischema
+
+import (
+	"context"
+	"errors"
+
+	"github.com/docker/distribution"
+	"github.com/docker/distribution/manifest"
+	"github.com/opencontainers/go-digest"
+	v1 "github.com/opencontainers/image-spec/specs-go/v1"
+)
+
+// Builder is a type for constructing manifests.
+type Builder struct {
+	// bs is a BlobService used to publish the configuration blob.
+	bs distribution.BlobService
+
+	// configJSON references
+	configJSON []byte
+
+	// layers is a list of layer descriptors that gets built by successive
+	// calls to AppendReference.
+	layers []distribution.Descriptor
+
+	// Annotations contains arbitrary metadata relating to the targeted content.
+	annotations map[string]string
+
+	// For testing purposes
+	mediaType string
+}
+
+// NewManifestBuilder is used to build new manifests for the current schema
+// version. It takes a BlobService so it can publish the configuration blob
+// as part of the Build process, and annotations.
+func NewManifestBuilder(bs distribution.BlobService, configJSON []byte, annotations map[string]string) distribution.ManifestBuilder {
+	mb := &Builder{
+		bs:          bs,
+		configJSON:  make([]byte, len(configJSON)),
+		annotations: annotations,
+		mediaType:   v1.MediaTypeImageManifest,
+	}
+	copy(mb.configJSON, configJSON)
+
+	return mb
+}
+
+// SetMediaType assigns the passed mediatype or error if the mediatype is not a
+// valid media type for oci image manifests currently: "" or "application/vnd.oci.image.manifest.v1+json"
+func (mb *Builder) SetMediaType(mediaType string) error {
+	if mediaType != "" && mediaType != v1.MediaTypeImageManifest {
+		return errors.New("invalid media type for OCI image manifest")
+	}
+
+	mb.mediaType = mediaType
+	return nil
+}
+
+// Build produces a final manifest from the given references.
+func (mb *Builder) Build(ctx context.Context) (distribution.Manifest, error) {
+	m := Manifest{
+		Versioned: manifest.Versioned{
+			SchemaVersion: 2,
+			MediaType:     mb.mediaType,
+		},
+		Layers:      make([]distribution.Descriptor, len(mb.layers)),
+		Annotations: mb.annotations,
+	}
+	copy(m.Layers, mb.layers)
+
+	configDigest := digest.FromBytes(mb.configJSON)
+
+	var err error
+	m.Config, err = mb.bs.Stat(ctx, configDigest)
+	switch err {
+	case nil:
+		// Override MediaType, since Put always replaces the specified media
+		// type with application/octet-stream in the descriptor it returns.
+		m.Config.MediaType = v1.MediaTypeImageConfig
+		return FromStruct(m)
+	case distribution.ErrBlobUnknown:
+		// nop
+	default:
+		return nil, err
+	}
+
+	// Add config to the blob store
+	m.Config, err = mb.bs.Put(ctx, v1.MediaTypeImageConfig, mb.configJSON)
+	// Override MediaType, since Put always replaces the specified media
+	// type with application/octet-stream in the descriptor it returns.
+	m.Config.MediaType = v1.MediaTypeImageConfig
+	if err != nil {
+		return nil, err
+	}
+
+	return FromStruct(m)
+}
+
+// AppendReference adds a reference to the current ManifestBuilder.
+func (mb *Builder) AppendReference(d distribution.Describable) error {
+	mb.layers = append(mb.layers, d.Descriptor())
+	return nil
+}
+
+// References returns the current references added to this builder.
+func (mb *Builder) References() []distribution.Descriptor {
+	return mb.layers
+}

vendor/github.com/docker/distribution/manifest/ocischema/manifest.go

@@ -0,0 +1,146 @@
+package ocischema
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+
+	"github.com/docker/distribution"
+	"github.com/docker/distribution/manifest"
+	"github.com/opencontainers/go-digest"
+	v1 "github.com/opencontainers/image-spec/specs-go/v1"
+)
+
+var (
+	// SchemaVersion provides a pre-initialized version structure for this
+	// packages version of the manifest.
+	SchemaVersion = manifest.Versioned{
+		SchemaVersion: 2, // historical value here.. does not pertain to OCI or docker version
+		MediaType:     v1.MediaTypeImageManifest,
+	}
+)
+
+func init() {
+	ocischemaFunc := func(b []byte) (distribution.Manifest, distribution.Descriptor, error) {
+		if err := validateManifest(b); err != nil {
+			return nil, distribution.Descriptor{}, err
+		}
+		m := new(DeserializedManifest)
+		err := m.UnmarshalJSON(b)
+		if err != nil {
+			return nil, distribution.Descriptor{}, err
+		}
+
+		dgst := digest.FromBytes(b)
+		return m, distribution.Descriptor{Digest: dgst, Size: int64(len(b)), MediaType: v1.MediaTypeImageManifest}, err
+	}
+	err := distribution.RegisterManifestSchema(v1.MediaTypeImageManifest, ocischemaFunc)
+	if err != nil {
+		panic(fmt.Sprintf("Unable to register manifest: %s", err))
+	}
+}
+
+// Manifest defines a ocischema manifest.
+type Manifest struct {
+	manifest.Versioned
+
+	// Config references the image configuration as a blob.
+	Config distribution.Descriptor `json:"config"`
+
+	// Layers lists descriptors for the layers referenced by the
+	// configuration.
+	Layers []distribution.Descriptor `json:"layers"`
+
+	// Annotations contains arbitrary metadata for the image manifest.
+	Annotations map[string]string `json:"annotations,omitempty"`
+}
+
+// References returns the descriptors of this manifests references.
+func (m Manifest) References() []distribution.Descriptor {
+	references := make([]distribution.Descriptor, 0, 1+len(m.Layers))
+	references = append(references, m.Config)
+	references = append(references, m.Layers...)
+	return references
+}
+
+// Target returns the target of this manifest.
+func (m Manifest) Target() distribution.Descriptor {
+	return m.Config
+}
+
+// DeserializedManifest wraps Manifest with a copy of the original JSON.
+// It satisfies the distribution.Manifest interface.
+type DeserializedManifest struct {
+	Manifest
+
+	// canonical is the canonical byte representation of the Manifest.
+	canonical []byte
+}
+
+// FromStruct takes a Manifest structure, marshals it to JSON, and returns a
+// DeserializedManifest which contains the manifest and its JSON representation.
+func FromStruct(m Manifest) (*DeserializedManifest, error) {
+	var deserialized DeserializedManifest
+	deserialized.Manifest = m
+
+	var err error
+	deserialized.canonical, err = json.MarshalIndent(&m, "", "   ")
+	return &deserialized, err
+}
+
+// UnmarshalJSON populates a new Manifest struct from JSON data.
+func (m *DeserializedManifest) UnmarshalJSON(b []byte) error {
+	m.canonical = make([]byte, len(b))
+	// store manifest in canonical
+	copy(m.canonical, b)
+
+	// Unmarshal canonical JSON into Manifest object
+	var manifest Manifest
+	if err := json.Unmarshal(m.canonical, &manifest); err != nil {
+		return err
+	}
+
+	if manifest.MediaType != "" && manifest.MediaType != v1.MediaTypeImageManifest {
+		return fmt.Errorf("if present, mediaType in manifest should be '%s' not '%s'",
+			v1.MediaTypeImageManifest, manifest.MediaType)
+	}
+
+	m.Manifest = manifest
+
+	return nil
+}
+
+// MarshalJSON returns the contents of canonical. If canonical is empty,
+// marshals the inner contents.
+func (m *DeserializedManifest) MarshalJSON() ([]byte, error) {
+	if len(m.canonical) > 0 {
+		return m.canonical, nil
+	}
+
+	return nil, errors.New("JSON representation not initialized in DeserializedManifest")
+}
+
+// Payload returns the raw content of the manifest. The contents can be used to
+// calculate the content identifier.
+func (m DeserializedManifest) Payload() (string, []byte, error) {
+	return v1.MediaTypeImageManifest, m.canonical, nil
+}
+
+// unknownDocument represents a manifest, manifest list, or index that has not
+// yet been validated
+type unknownDocument struct {
+	Manifests interface{} `json:"manifests,omitempty"`
+}
+
+// validateManifest returns an error if the byte slice is invalid JSON or if it
+// contains fields that belong to a index
+func validateManifest(b []byte) error {
+	var doc unknownDocument
+	if err := json.Unmarshal(b, &doc); err != nil {
+		return err
+	}
+	if doc.Manifests != nil {
+		return errors.New("ocimanifest: expected manifest but found index")
+	}
+	return nil
+}

vendor/modules.txt

@@ -27,6 +27,7 @@ github.com/docker/distribution
 github.com/docker/distribution/digestset
 github.com/docker/distribution/manifest
 github.com/docker/distribution/manifest/manifestlist
+github.com/docker/distribution/manifest/ocischema
 github.com/docker/distribution/manifest/schema2
 github.com/docker/distribution/metrics
 github.com/docker/distribution/reference