diff --git a/cli/command/manifest/push.go b/cli/command/manifest/push.go index 9ab2e246b6..41ab4941d8 100644 --- a/cli/command/manifest/push.go +++ b/cli/command/manifest/push.go @@ -12,6 +12,7 @@ import ( registryclient "github.com/docker/cli/cli/registry/client" "github.com/docker/distribution" "github.com/docker/distribution/manifest/manifestlist" + "github.com/docker/distribution/manifest/ocischema" "github.com/docker/distribution/manifest/schema2" "github.com/docker/distribution/reference" "github.com/docker/docker/registry" @@ -217,18 +218,53 @@ func buildPutManifestRequest(imageManifest types.ImageManifest, targetRef refere return mountRequest{}, err } - // This indentation has to be added to ensure sha parity with the registry - v2ManifestBytes, err := json.MarshalIndent(imageManifest.SchemaV2Manifest, "", " ") - if err != nil { - return mountRequest{}, err + // Attempt to reconstruct indentation of the manifest to ensure sha parity + // with the registry - if we haven't preserved the raw content. + // + // This is necessary because our previous internal storage format did not + // preserve whitespace. If we don't have the newer format present, we can + // attempt the reconstruction like before, but explicitly error if the + // reconstruction failed! + switch { + case imageManifest.SchemaV2Manifest != nil: + dt := imageManifest.Raw + if len(dt) == 0 { + dt, err = json.MarshalIndent(imageManifest.SchemaV2Manifest, "", " ") + if err != nil { + return mountRequest{}, err + } + } + + dig := imageManifest.Descriptor.Digest + if dig2 := dig.Algorithm().FromBytes(dt); dig != dig2 { + return mountRequest{}, errors.Errorf("internal digest mismatch for %s: expected %s, got %s", imageManifest.Ref, dig, dig2) + } + + var manifest schema2.DeserializedManifest + if err = manifest.UnmarshalJSON(dt); err != nil { + return mountRequest{}, err + } + imageManifest.SchemaV2Manifest = &manifest + case imageManifest.OCIManifest != nil: + dt := imageManifest.Raw + if len(dt) == 0 { + dt, err = json.MarshalIndent(imageManifest.OCIManifest, "", " ") + if err != nil { + return mountRequest{}, err + } + } + + dig := imageManifest.Descriptor.Digest + if dig2 := dig.Algorithm().FromBytes(dt); dig != dig2 { + return mountRequest{}, errors.Errorf("internal digest mismatch for %s: expected %s, got %s", imageManifest.Ref, dig, dig2) + } + + var manifest ocischema.DeserializedManifest + if err = manifest.UnmarshalJSON(dt); err != nil { + return mountRequest{}, err + } + imageManifest.OCIManifest = &manifest } - // indent only the DeserializedManifest portion of this, in order to maintain parity with the registry - // and not alter the sha - var v2Manifest schema2.DeserializedManifest - if err = v2Manifest.UnmarshalJSON(v2ManifestBytes); err != nil { - return mountRequest{}, err - } - imageManifest.SchemaV2Manifest = &v2Manifest return mountRequest{ref: mountRef, manifest: imageManifest}, err } diff --git a/cli/command/manifest/testdata/inspect-annotate.golden b/cli/command/manifest/testdata/inspect-annotate.golden index 5fe5bd5a9b..f594d518cb 100644 --- a/cli/command/manifest/testdata/inspect-annotate.golden +++ b/cli/command/manifest/testdata/inspect-annotate.golden @@ -14,6 +14,7 @@ "variant": "v7" } }, + "Raw": "ewogICAic2NoZW1hVmVyc2lvbiI6IDIsCiAgICJtZWRpYVR5cGUiOiAiYXBwbGljYXRpb24vdm5kLmRvY2tlci5kaXN0cmlidXRpb24ubWFuaWZlc3QudjIranNvbiIsCiAgICJjb25maWciOiB7CiAgICAgICJtZWRpYVR5cGUiOiAiYXBwbGljYXRpb24vdm5kLmRvY2tlci5jb250YWluZXIuaW1hZ2UudjEranNvbiIsCiAgICAgICJzaXplIjogMTUyMCwKICAgICAgImRpZ2VzdCI6ICJzaGEyNTY6NzMyOGY2ZjhiNDE4OTA1OTc1NzVjYmFhZGM4ODRlNzM4NmFlMGFjYzUzYjc0NzQwMWViY2U1Y2YwZDYyNDU2MCIKICAgfSwKICAgImxheWVycyI6IFsKICAgICAgewogICAgICAgICAibWVkaWFUeXBlIjogImFwcGxpY2F0aW9uL3ZuZC5kb2NrZXIuaW1hZ2Uucm9vdGZzLmRpZmYudGFyLmd6aXAiLAogICAgICAgICAic2l6ZSI6IDE5OTA0MDIsCiAgICAgICAgICJkaWdlc3QiOiAic2hhMjU2Ojg4Mjg2ZjQxNTMwZTkzZGZmZDRiOTY0ZTFkYjIyY2U0OTM5ZmZmYTRhNGM2NjVkYWI4NTkxZmJhYjAzZDQ5MjYiCiAgICAgIH0KICAgXQp9", "SchemaV2Manifest": { "schemaVersion": 2, "mediaType": "application/vnd.docker.distribution.manifest.v2+json", diff --git a/cli/command/manifest/util.go b/cli/command/manifest/util.go index 1d530d9f2f..16aba15c74 100644 --- a/cli/command/manifest/util.go +++ b/cli/command/manifest/util.go @@ -76,6 +76,8 @@ func getManifest(ctx context.Context, dockerCli command.Cli, listRef, namedRef r return dockerCli.RegistryClient(insecure).GetManifest(ctx, namedRef) case err != nil: return types.ImageManifest{}, err + case len(data.Raw) == 0: + return dockerCli.RegistryClient(insecure).GetManifest(ctx, namedRef) default: return data, nil } diff --git a/cli/manifest/types/types.go b/cli/manifest/types/types.go index 5b094f5142..ca2a3e7866 100644 --- a/cli/manifest/types/types.go +++ b/cli/manifest/types/types.go @@ -5,6 +5,7 @@ import ( "github.com/docker/distribution" "github.com/docker/distribution/manifest/manifestlist" + "github.com/docker/distribution/manifest/ocischema" "github.com/docker/distribution/manifest/schema2" "github.com/docker/distribution/reference" "github.com/opencontainers/go-digest" @@ -16,10 +17,12 @@ import ( type ImageManifest struct { Ref *SerializableNamed Descriptor ocispec.Descriptor + Raw []byte `json:",omitempty"` // SchemaV2Manifest is used for inspection - // TODO: Deprecate this and store manifest blobs SchemaV2Manifest *schema2.DeserializedManifest `json:",omitempty"` + // OCIManifest is used for inspection + OCIManifest *ocischema.DeserializedManifest `json:",omitempty"` } // OCIPlatform creates an OCI platform from a manifest list platform spec @@ -53,8 +56,15 @@ func PlatformSpecFromOCI(p *ocispec.Platform) *manifestlist.PlatformSpec { // Blobs returns the digests for all the blobs referenced by this manifest func (i ImageManifest) Blobs() []digest.Digest { digests := []digest.Digest{} - for _, descriptor := range i.SchemaV2Manifest.References() { - digests = append(digests, descriptor.Digest) + switch { + case i.SchemaV2Manifest != nil: + for _, descriptor := range i.SchemaV2Manifest.References() { + digests = append(digests, descriptor.Digest) + } + case i.OCIManifest != nil: + for _, descriptor := range i.OCIManifest.References() { + digests = append(digests, descriptor.Digest) + } } return digests } @@ -65,6 +75,8 @@ func (i ImageManifest) Payload() (string, []byte, error) { switch { case i.SchemaV2Manifest != nil: return i.SchemaV2Manifest.Payload() + case i.OCIManifest != nil: + return i.OCIManifest.Payload() default: return "", nil, errors.Errorf("%s has no payload", i.Ref) } @@ -76,6 +88,8 @@ func (i ImageManifest) References() []distribution.Descriptor { switch { case i.SchemaV2Manifest != nil: return i.SchemaV2Manifest.References() + case i.OCIManifest != nil: + return i.OCIManifest.References() default: return nil } @@ -84,13 +98,35 @@ func (i ImageManifest) References() []distribution.Descriptor { // NewImageManifest returns a new ImageManifest object. The values for Platform // are initialized from those in the image func NewImageManifest(ref reference.Named, desc ocispec.Descriptor, manifest *schema2.DeserializedManifest) ImageManifest { + raw, err := manifest.MarshalJSON() + if err != nil { + raw = nil + } + return ImageManifest{ Ref: &SerializableNamed{Named: ref}, Descriptor: desc, + Raw: raw, SchemaV2Manifest: manifest, } } +// NewOCIImageManifest returns a new ImageManifest object. The values for +// Platform are initialized from those in the image +func NewOCIImageManifest(ref reference.Named, desc ocispec.Descriptor, manifest *ocischema.DeserializedManifest) ImageManifest { + raw, err := manifest.MarshalJSON() + if err != nil { + raw = nil + } + + return ImageManifest{ + Ref: &SerializableNamed{Named: ref}, + Descriptor: desc, + Raw: raw, + OCIManifest: manifest, + } +} + // SerializableNamed is a reference.Named that can be serialized and deserialized // from JSON type SerializableNamed struct { diff --git a/cli/registry/client/fetcher.go b/cli/registry/client/fetcher.go index ef8011d1fb..acae274a44 100644 --- a/cli/registry/client/fetcher.go +++ b/cli/registry/client/fetcher.go @@ -7,6 +7,7 @@ import ( "github.com/docker/cli/cli/manifest/types" "github.com/docker/distribution" "github.com/docker/distribution/manifest/manifestlist" + "github.com/docker/distribution/manifest/ocischema" "github.com/docker/distribution/manifest/schema2" "github.com/docker/distribution/reference" "github.com/docker/distribution/registry/api/errcode" @@ -35,6 +36,12 @@ func fetchManifest(ctx context.Context, repo distribution.Repository, ref refere return types.ImageManifest{}, err } return imageManifest, nil + case *ocischema.DeserializedManifest: + imageManifest, err := pullManifestOCISchema(ctx, ref, repo, *v) + if err != nil { + return types.ImageManifest{}, err + } + return imageManifest, nil case *manifestlist.DeserializedManifestList: return types.ImageManifest{}, errors.Errorf("%s is a manifest list", ref) } @@ -94,6 +101,28 @@ func pullManifestSchemaV2(ctx context.Context, ref reference.Named, repo distrib return types.NewImageManifest(ref, manifestDesc, &mfst), nil } +func pullManifestOCISchema(ctx context.Context, ref reference.Named, repo distribution.Repository, mfst ocischema.DeserializedManifest) (types.ImageManifest, error) { + manifestDesc, err := validateManifestDigest(ref, mfst) + if err != nil { + return types.ImageManifest{}, err + } + configJSON, err := pullManifestSchemaV2ImageConfig(ctx, mfst.Target().Digest, repo) + if err != nil { + return types.ImageManifest{}, err + } + + if manifestDesc.Platform == nil { + manifestDesc.Platform = &ocispec.Platform{} + } + + // Fill in os and architecture fields from config JSON + if err := json.Unmarshal(configJSON, manifestDesc.Platform); err != nil { + return types.ImageManifest{}, err + } + + return types.NewOCIImageManifest(ref, manifestDesc, &mfst), nil +} + func pullManifestSchemaV2ImageConfig(ctx context.Context, dgst digest.Digest, repo distribution.Repository) ([]byte, error) { blobs := repo.Blobs(ctx) configJSON, err := blobs.Get(ctx, dgst) @@ -153,16 +182,21 @@ func pullManifestList(ctx context.Context, ref reference.Named, repo distributio if err != nil { return nil, err } - v, ok := manifest.(*schema2.DeserializedManifest) - if !ok { - return nil, errors.Errorf("unsupported manifest format: %v", v) - } manifestRef, err := reference.WithDigest(ref, manifestDescriptor.Digest) if err != nil { return nil, err } - imageManifest, err := pullManifestSchemaV2(ctx, manifestRef, repo, *v) + + var imageManifest types.ImageManifest + switch v := manifest.(type) { + case *schema2.DeserializedManifest: + imageManifest, err = pullManifestSchemaV2(ctx, manifestRef, repo, *v) + case *ocischema.DeserializedManifest: + imageManifest, err = pullManifestOCISchema(ctx, manifestRef, repo, *v) + default: + err = errors.Errorf("unsupported manifest type: %T", manifest) + } if err != nil { return nil, err } diff --git a/vendor/github.com/docker/distribution/manifest/ocischema/builder.go b/vendor/github.com/docker/distribution/manifest/ocischema/builder.go new file mode 100644 index 0000000000..b89bf5b714 --- /dev/null +++ b/vendor/github.com/docker/distribution/manifest/ocischema/builder.go @@ -0,0 +1,107 @@ +package ocischema + +import ( + "context" + "errors" + + "github.com/docker/distribution" + "github.com/docker/distribution/manifest" + "github.com/opencontainers/go-digest" + v1 "github.com/opencontainers/image-spec/specs-go/v1" +) + +// Builder is a type for constructing manifests. +type Builder struct { + // bs is a BlobService used to publish the configuration blob. + bs distribution.BlobService + + // configJSON references + configJSON []byte + + // layers is a list of layer descriptors that gets built by successive + // calls to AppendReference. + layers []distribution.Descriptor + + // Annotations contains arbitrary metadata relating to the targeted content. + annotations map[string]string + + // For testing purposes + mediaType string +} + +// NewManifestBuilder is used to build new manifests for the current schema +// version. It takes a BlobService so it can publish the configuration blob +// as part of the Build process, and annotations. +func NewManifestBuilder(bs distribution.BlobService, configJSON []byte, annotations map[string]string) distribution.ManifestBuilder { + mb := &Builder{ + bs: bs, + configJSON: make([]byte, len(configJSON)), + annotations: annotations, + mediaType: v1.MediaTypeImageManifest, + } + copy(mb.configJSON, configJSON) + + return mb +} + +// SetMediaType assigns the passed mediatype or error if the mediatype is not a +// valid media type for oci image manifests currently: "" or "application/vnd.oci.image.manifest.v1+json" +func (mb *Builder) SetMediaType(mediaType string) error { + if mediaType != "" && mediaType != v1.MediaTypeImageManifest { + return errors.New("invalid media type for OCI image manifest") + } + + mb.mediaType = mediaType + return nil +} + +// Build produces a final manifest from the given references. +func (mb *Builder) Build(ctx context.Context) (distribution.Manifest, error) { + m := Manifest{ + Versioned: manifest.Versioned{ + SchemaVersion: 2, + MediaType: mb.mediaType, + }, + Layers: make([]distribution.Descriptor, len(mb.layers)), + Annotations: mb.annotations, + } + copy(m.Layers, mb.layers) + + configDigest := digest.FromBytes(mb.configJSON) + + var err error + m.Config, err = mb.bs.Stat(ctx, configDigest) + switch err { + case nil: + // Override MediaType, since Put always replaces the specified media + // type with application/octet-stream in the descriptor it returns. + m.Config.MediaType = v1.MediaTypeImageConfig + return FromStruct(m) + case distribution.ErrBlobUnknown: + // nop + default: + return nil, err + } + + // Add config to the blob store + m.Config, err = mb.bs.Put(ctx, v1.MediaTypeImageConfig, mb.configJSON) + // Override MediaType, since Put always replaces the specified media + // type with application/octet-stream in the descriptor it returns. + m.Config.MediaType = v1.MediaTypeImageConfig + if err != nil { + return nil, err + } + + return FromStruct(m) +} + +// AppendReference adds a reference to the current ManifestBuilder. +func (mb *Builder) AppendReference(d distribution.Describable) error { + mb.layers = append(mb.layers, d.Descriptor()) + return nil +} + +// References returns the current references added to this builder. +func (mb *Builder) References() []distribution.Descriptor { + return mb.layers +} diff --git a/vendor/github.com/docker/distribution/manifest/ocischema/manifest.go b/vendor/github.com/docker/distribution/manifest/ocischema/manifest.go new file mode 100644 index 0000000000..d51f8debb1 --- /dev/null +++ b/vendor/github.com/docker/distribution/manifest/ocischema/manifest.go @@ -0,0 +1,146 @@ +package ocischema + +import ( + "encoding/json" + "errors" + "fmt" + + "github.com/docker/distribution" + "github.com/docker/distribution/manifest" + "github.com/opencontainers/go-digest" + v1 "github.com/opencontainers/image-spec/specs-go/v1" +) + +var ( + // SchemaVersion provides a pre-initialized version structure for this + // packages version of the manifest. + SchemaVersion = manifest.Versioned{ + SchemaVersion: 2, // historical value here.. does not pertain to OCI or docker version + MediaType: v1.MediaTypeImageManifest, + } +) + +func init() { + ocischemaFunc := func(b []byte) (distribution.Manifest, distribution.Descriptor, error) { + if err := validateManifest(b); err != nil { + return nil, distribution.Descriptor{}, err + } + m := new(DeserializedManifest) + err := m.UnmarshalJSON(b) + if err != nil { + return nil, distribution.Descriptor{}, err + } + + dgst := digest.FromBytes(b) + return m, distribution.Descriptor{Digest: dgst, Size: int64(len(b)), MediaType: v1.MediaTypeImageManifest}, err + } + err := distribution.RegisterManifestSchema(v1.MediaTypeImageManifest, ocischemaFunc) + if err != nil { + panic(fmt.Sprintf("Unable to register manifest: %s", err)) + } +} + +// Manifest defines a ocischema manifest. +type Manifest struct { + manifest.Versioned + + // Config references the image configuration as a blob. + Config distribution.Descriptor `json:"config"` + + // Layers lists descriptors for the layers referenced by the + // configuration. + Layers []distribution.Descriptor `json:"layers"` + + // Annotations contains arbitrary metadata for the image manifest. + Annotations map[string]string `json:"annotations,omitempty"` +} + +// References returns the descriptors of this manifests references. +func (m Manifest) References() []distribution.Descriptor { + references := make([]distribution.Descriptor, 0, 1+len(m.Layers)) + references = append(references, m.Config) + references = append(references, m.Layers...) + return references +} + +// Target returns the target of this manifest. +func (m Manifest) Target() distribution.Descriptor { + return m.Config +} + +// DeserializedManifest wraps Manifest with a copy of the original JSON. +// It satisfies the distribution.Manifest interface. +type DeserializedManifest struct { + Manifest + + // canonical is the canonical byte representation of the Manifest. + canonical []byte +} + +// FromStruct takes a Manifest structure, marshals it to JSON, and returns a +// DeserializedManifest which contains the manifest and its JSON representation. +func FromStruct(m Manifest) (*DeserializedManifest, error) { + var deserialized DeserializedManifest + deserialized.Manifest = m + + var err error + deserialized.canonical, err = json.MarshalIndent(&m, "", " ") + return &deserialized, err +} + +// UnmarshalJSON populates a new Manifest struct from JSON data. +func (m *DeserializedManifest) UnmarshalJSON(b []byte) error { + m.canonical = make([]byte, len(b)) + // store manifest in canonical + copy(m.canonical, b) + + // Unmarshal canonical JSON into Manifest object + var manifest Manifest + if err := json.Unmarshal(m.canonical, &manifest); err != nil { + return err + } + + if manifest.MediaType != "" && manifest.MediaType != v1.MediaTypeImageManifest { + return fmt.Errorf("if present, mediaType in manifest should be '%s' not '%s'", + v1.MediaTypeImageManifest, manifest.MediaType) + } + + m.Manifest = manifest + + return nil +} + +// MarshalJSON returns the contents of canonical. If canonical is empty, +// marshals the inner contents. +func (m *DeserializedManifest) MarshalJSON() ([]byte, error) { + if len(m.canonical) > 0 { + return m.canonical, nil + } + + return nil, errors.New("JSON representation not initialized in DeserializedManifest") +} + +// Payload returns the raw content of the manifest. The contents can be used to +// calculate the content identifier. +func (m DeserializedManifest) Payload() (string, []byte, error) { + return v1.MediaTypeImageManifest, m.canonical, nil +} + +// unknownDocument represents a manifest, manifest list, or index that has not +// yet been validated +type unknownDocument struct { + Manifests interface{} `json:"manifests,omitempty"` +} + +// validateManifest returns an error if the byte slice is invalid JSON or if it +// contains fields that belong to a index +func validateManifest(b []byte) error { + var doc unknownDocument + if err := json.Unmarshal(b, &doc); err != nil { + return err + } + if doc.Manifests != nil { + return errors.New("ocimanifest: expected manifest but found index") + } + return nil +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 07476c714f..162dd82240 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -27,6 +27,7 @@ github.com/docker/distribution github.com/docker/distribution/digestset github.com/docker/distribution/manifest github.com/docker/distribution/manifest/manifestlist +github.com/docker/distribution/manifest/ocischema github.com/docker/distribution/manifest/schema2 github.com/docker/distribution/metrics github.com/docker/distribution/reference