Add compose support for cluster volumes

Signed-off-by: Drew Erny <derny@mirantis.com>

cli/compose/convert/volume.go

@@ -1,6 +1,8 @@
 package convert
 
 import (
+	"strings"
+
 	composetypes "github.com/docker/cli/cli/compose/types"
 	"github.com/docker/docker/api/types/mount"
 	"github.com/pkg/errors"
@@ -45,6 +47,9 @@ func handleVolumeToMount(
 	if volume.Bind != nil {
 		return mount.Mount{}, errors.New("bind options are incompatible with type volume")
 	}
+	if volume.Cluster != nil {
+		return mount.Mount{}, errors.New("cluster options are incompatible with type volume")
+	}
 	// Anonymous volumes
 	if volume.Source == "" {
 		return result, nil
@@ -94,6 +99,9 @@ func handleBindToMount(volume composetypes.ServiceVolumeConfig) (mount.Mount, er
 	if volume.Tmpfs != nil {
 		return mount.Mount{}, errors.New("tmpfs options are incompatible with type bind")
 	}
+	if volume.Cluster != nil {
+		return mount.Mount{}, errors.New("cluster options are incompatible with type bind")
+	}
 	if volume.Bind != nil {
 		result.BindOptions = &mount.BindOptions{
 			Propagation: mount.Propagation(volume.Bind.Propagation),
@@ -114,6 +122,9 @@ func handleTmpfsToMount(volume composetypes.ServiceVolumeConfig) (mount.Mount, e
 	if volume.Volume != nil {
 		return mount.Mount{}, errors.New("volume options are incompatible with type tmpfs")
 	}
+	if volume.Cluster != nil {
+		return mount.Mount{}, errors.New("cluster options are incompatible with type tmpfs")
+	}
 	if volume.Tmpfs != nil {
 		result.TmpfsOptions = &mount.TmpfsOptions{
 			SizeBytes: volume.Tmpfs.Size,
@@ -142,6 +153,49 @@ func handleNpipeToMount(volume composetypes.ServiceVolumeConfig) (mount.Mount, e
 	return result, nil
 }
 
+func handleClusterToMount(
+	volume composetypes.ServiceVolumeConfig,
+	stackVolumes volumes,
+	namespace Namespace,
+) (mount.Mount, error) {
+	if volume.Source == "" {
+		return mount.Mount{}, errors.New("invalid cluster source, source cannot be empty")
+	}
+	if volume.Tmpfs != nil {
+		return mount.Mount{}, errors.New("tmpfs options are incompatible with type cluster")
+	}
+	if volume.Bind != nil {
+		return mount.Mount{}, errors.New("bind options are incompatible with type cluster")
+	}
+	if volume.Volume != nil {
+		return mount.Mount{}, errors.New("volume options are incompatible with type cluster")
+	}
+
+	result := createMountFromVolume(volume)
+	result.ClusterOptions = &mount.ClusterOptions{}
+
+	if !strings.HasPrefix(volume.Source, "group:") {
+		// if the volume is a cluster volume and the source is a volumegroup, we
+		// will ignore checking to see if such a volume is defined. the volume
+		// group isn't namespaced, and there's no simple way to indicate that
+		// external volumes with a given group exist.
+		stackVolume, exists := stackVolumes[volume.Source]
+		if !exists {
+			return mount.Mount{}, errors.Errorf("undefined volume %q", volume.Source)
+		}
+
+		// if the volume is not specified with a group source, we may namespace
+		// the name, if one is not otherwise specified.
+		if stackVolume.Name != "" {
+			result.Source = stackVolume.Name
+		} else {
+			result.Source = namespace.Scope(volume.Source)
+		}
+	}
+
+	return result, nil
+}
+
 func convertVolumeToMount(
 	volume composetypes.ServiceVolumeConfig,
 	stackVolumes volumes,
@@ -156,6 +210,8 @@ func convertVolumeToMount(
 		return handleTmpfsToMount(volume)
 	case "npipe":
 		return handleNpipeToMount(volume)
+	case "cluster":
+		return handleClusterToMount(volume, stackVolumes, namespace)
 	}
-	return mount.Mount{}, errors.New("volume type must be volume, bind, tmpfs or npipe")
+	return mount.Mount{}, errors.New("volume type must be volume, bind, tmpfs, npipe, or cluster")
 }

cli/compose/convert/volume_test.go

@@ -41,7 +41,7 @@ func TestConvertVolumeToMountUnapprovedType(t *testing.T) {
 		Target: "/foo/bar",
 	}
 	_, err := convertVolumeToMount(config, volumes{}, NewNamespace("foo"))
-	assert.Error(t, err, "volume type must be volume, bind, tmpfs or npipe")
+	assert.Error(t, err, "volume type must be volume, bind, tmpfs, npipe, or cluster")
 }
 
 func TestConvertVolumeToMountConflictingOptionsBindInVolume(t *testing.T) {
@@ -359,3 +359,71 @@ func TestConvertVolumeToMountAnonymousNpipe(t *testing.T) {
 	assert.NilError(t, err)
 	assert.Check(t, is.DeepEqual(expected, mount))
 }
+
+func TestConvertVolumeMountClusterName(t *testing.T) {
+	stackVolumes := volumes{
+		"my-csi": composetypes.VolumeConfig{
+			Driver: "mycsidriver",
+			Spec: &composetypes.ClusterVolumeSpec{
+				Group: "mygroup",
+				AccessMode: &composetypes.AccessMode{
+					Scope:       "single",
+					Sharing:     "none",
+					BlockVolume: &composetypes.BlockVolume{},
+				},
+				Availability: "active",
+			},
+		},
+	}
+
+	config := composetypes.ServiceVolumeConfig{
+		Type:   "cluster",
+		Source: "my-csi",
+		Target: "/srv",
+	}
+
+	expected := mount.Mount{
+		Type:           mount.TypeCluster,
+		Source:         "foo_my-csi",
+		Target:         "/srv",
+		ClusterOptions: &mount.ClusterOptions{},
+	}
+
+	mount, err := convertVolumeToMount(config, stackVolumes, NewNamespace("foo"))
+	assert.NilError(t, err)
+	assert.Check(t, is.DeepEqual(expected, mount))
+}
+
+func TestConvertVolumeMountClusterGroup(t *testing.T) {
+	stackVolumes := volumes{
+		"my-csi": composetypes.VolumeConfig{
+			Driver: "mycsidriver",
+			Spec: &composetypes.ClusterVolumeSpec{
+				Group: "mygroup",
+				AccessMode: &composetypes.AccessMode{
+					Scope:       "single",
+					Sharing:     "none",
+					BlockVolume: &composetypes.BlockVolume{},
+				},
+				Availability: "active",
+			},
+		},
+	}
+
+	config := composetypes.ServiceVolumeConfig{
+		Type:   "cluster",
+		Source: "group:mygroup",
+		Target: "/srv",
+	}
+
+	expected := mount.Mount{
+		Type:           mount.TypeCluster,
+		Source:         "group:mygroup",
+		Target:         "/srv",
+		ClusterOptions: &mount.ClusterOptions{},
+	}
+
+	mount, err := convertVolumeToMount(config, stackVolumes, NewNamespace("foo"))
+	assert.NilError(t, err)
+	assert.Check(t, is.DeepEqual(expected, mount))
+}

cli/compose/loader/full-example.yml

@@ -1,4 +1,4 @@
-version: "3.9"
+version: "3.10"
 
 services:
   foo:
@@ -287,6 +287,9 @@ services:
         target: /opt
         tmpfs:
           size: 10000
+      - type: cluster
+        source: group:mygroup
+        target: /srv
 
     working_dir: /code
     x-bar: baz
@@ -379,6 +382,36 @@ volumes:
     x-bar: baz
     x-foo: bar
 
+  cluster-volume:
+    driver: my-csi-driver
+    x-cluster-spec:
+      group: mygroup
+      access_mode:
+        scope: single
+        sharing: none
+        block_volume: {}
+      accessibility_requirements:
+        requisite:
+        - segments:
+          - region=R1
+          - zone=Z1
+        - segments:
+            region: R1
+            zone: Z2
+        preferred:
+        - segments:
+            region: R1
+            zone: Z1
+      capacity_range:
+        required_bytes: 1G
+        limit_bytes: 8G
+      secrets:
+      - key: mycsisecret
+        secret: secret1
+      - key: mycsisecret2
+        secret: secret4
+      availability: active
+
 configs:
   config1:
     file: ./config_data

cli/compose/loader/full-struct_test.go

@@ -10,7 +10,7 @@ import (
 
 func fullExampleConfig(workingDir, homeDir string) *types.Config {
 	return &types.Config{
-		Version:  "3.9",
+		Version:  "3.10",
 		Services: services(workingDir, homeDir),
 		Networks: networks(),
 		Volumes:  volumes(),
@@ -379,6 +379,7 @@ func services(workingDir, homeDir string) []types.ServiceConfig {
 				{Target: "/opt", Type: "tmpfs", Tmpfs: &types.ServiceVolumeTmpfs{
 					Size: int64(10000),
 				}},
+				{Source: "group:mygroup", Target: "/srv", Type: "cluster"},
 			},
 			WorkingDir: "/code",
 		},
@@ -460,6 +461,41 @@ func volumes() map[string]types.VolumeConfig {
 				"x-foo": "bar",
 			},
 		},
+		"cluster-volume": {
+			Driver: "my-csi-driver",
+			Spec: &types.ClusterVolumeSpec{
+				Group: "mygroup",
+				AccessMode: &types.AccessMode{
+					Scope:       "single",
+					Sharing:     "none",
+					BlockVolume: &types.BlockVolume{},
+				},
+				AccessibilityRequirements: &types.TopologyRequirement{
+					Requisite: []types.Topology{
+						{
+							Segments: types.Mapping{"region": "R1", "zone": "Z1"},
+						},
+						{
+							Segments: types.Mapping{"region": "R1", "zone": "Z2"},
+						},
+					},
+					Preferred: []types.Topology{
+						{
+							Segments: types.Mapping{"region": "R1", "zone": "Z1"},
+						},
+					},
+				},
+				CapacityRange: &types.CapacityRange{
+					RequiredBytes: types.UnitBytes(1 * 1024 * 1024 * 1024),
+					LimitBytes:    types.UnitBytes(8 * 1024 * 1024 * 1024),
+				},
+				Secrets: []types.VolumeSecret{
+					{Key: "mycsisecret", Secret: "secret1"},
+					{Key: "mycsisecret2", Secret: "secret4"},
+				},
+				Availability: "active",
+			},
+		},
 	}
 }
 
@@ -518,7 +554,7 @@ func secrets(workingDir string) map[string]types.SecretConfig {
 }
 
 func fullExampleYAML(workingDir string) string {
-	return fmt.Sprintf(`version: "3.9"
+	return fmt.Sprintf(`version: "3.10"
 services:
   foo:
     build:
@@ -811,6 +847,9 @@ services:
       target: /opt
       tmpfs:
         size: 10000
+    - type: cluster
+      source: group:mygroup
+      target: /srv
     working_dir: /code
     x-bar: baz
     x-foo: bar
@@ -843,6 +882,35 @@ volumes:
     driver_opts:
       baz: "1"
       foo: bar
+  cluster-volume:
+    driver: my-csi-driver
+    x-cluster-spec:
+      group: mygroup
+      access_mode:
+        scope: single
+        sharing: none
+        block_volume: {}
+      accessibility_requirements:
+        requisite:
+        - segments:
+            region: R1
+            zone: Z1
+        - segments:
+            region: R1
+            zone: Z2
+        preferred:
+        - segments:
+            region: R1
+            zone: Z1
+      capacity_range:
+        required_bytes: "1073741824"
+        limit_bytes: "8589934592"
+      secrets:
+      - key: mycsisecret
+        secret: secret1
+      - key: mycsisecret2
+        secret: secret4
+      availability: active
   external-volume:
     name: external-volume
     external: true
@@ -1409,12 +1477,17 @@ func fullExampleJSON(workingDir string) string {
           "tmpfs": {
             "size": 10000
           }
+        },
+        {
+          "type": "cluster",
+          "source": "group:mygroup",
+          "target": "/srv"
         }
       ],
       "working_dir": "/code"
     }
   },
-  "version": "3.9",
+  "version": "3.10",
   "volumes": {
     "another-volume": {
       "name": "user_specified_name",
@@ -1425,6 +1498,57 @@ func fullExampleJSON(workingDir string) string {
       },
       "external": false
     },
+    "cluster-volume": {
+      "driver": "my-csi-driver",
+      "external": false,
+      "x-cluster-spec": {
+        "group": "mygroup",
+        "access_mode": {
+          "scope": "single",
+          "sharing": "none",
+          "block_volume": {}
+        },
+        "accessibility_requirements": {
+          "requisite": [
+            {
+              "segments": {
+                "region": "R1",
+                "zone": "Z1"
+              }
+            },
+            {
+              "segments": {
+                "region": "R1",
+                "zone": "Z2"
+              }
+            }
+          ],
+          "preferred": [
+            {
+              "segments": {
+                "region": "R1",
+                "zone": "Z1"
+              }
+            }
+          ]
+        },
+        "capacity_range": {
+          "required_bytes": "1073741824",
+          "limit_bytes": "8589934592"
+        },
+        "secrets": [
+          {
+            "key": "mycsisecret",
+            "secret": "secret1"
+          },
+          {
+            "key": "mycsisecret2",
+            "secret": "secret4"
+          }
+        ],
+        "availability": "active"
+      }
+    },
     "external-volume": {
       "name": "external-volume",
       "external": true

cli/compose/loader/types_test.go

@@ -20,7 +20,7 @@ func TestMarshallConfig(t *testing.T) {
 	assert.Check(t, is.Equal(expected, string(actual)))
 
 	// Make sure the expected still
-	dict, err := ParseYAML([]byte("version: '3.9'\n" + expected))
+	dict, err := ParseYAML([]byte("version: '3.10'\n" + expected))
 	assert.NilError(t, err)
 	_, err = Load(buildConfigDetails(dict, map[string]string{}))
 	assert.NilError(t, err)

cli/compose/schema/data/config_schema_v3.10.json

@@ -523,7 +523,59 @@
           },
           "additionalProperties": false
         },
-        "labels": {"$ref": "#/definitions/list_or_dict"}
+        "labels": {"$ref": "#/definitions/list_or_dict"},
+        "x-cluster-spec": {
+          "type": "object",
+          "properties": {
+            "group": {"type": "string"},
+            "access_mode": {
+              "type": "object",
+              "properties": {
+                "scope": {"type": "string"},
+                "sharing": {"type": "string"},
+                "block_volume": {"type": "object"},
+                "mount_volume": {
+                  "type": "object",
+                  "properties": {
+                    "fs_type": {"type": "string"},
+                    "mount_flags": {"type": "array", "items": {"type": "string"}}
+                  }
+                }
+              }
+            },
+            "accessibility_requirements": {
+              "type": "object",
+              "properties": {
+                "requisite": {
+                  "type": "array",
+                  "items": {
+                    "type": "object",
+                    "properties": {
+                      "segments": {"$ref": "#/definitions/list_or_dict"}
+                    }
+                  }
+                },
+                "preferred": {
+                  "type": "array",
+                  "items": {
+                    "type": "object",
+                    "properties": {
+                      "segments": {"$ref": "#/definitions/list_or_dict"}
+                    }
+                  }
+                }
+              }
+            },
+            "capacity_range": {
+              "type": "object",
+              "properties": {
+                "required_bytes": {"type": "string"},
+                "limit_bytes": {"type": "string"}
+              }
+            },
+            "availability": {"type": "string"}
+          }
+        }
       },
       "patternProperties": {"^x-": {}},
       "additionalProperties": false

cli/compose/types/types.go

@@ -384,14 +384,15 @@ type ServicePortConfig struct {
 
 // ServiceVolumeConfig are references to a volume used by a service
 type ServiceVolumeConfig struct {
-	Type        string               `yaml:",omitempty" json:"type,omitempty"`
+	Type        string                `yaml:",omitempty" json:"type,omitempty"`
-	Source      string               `yaml:",omitempty" json:"source,omitempty"`
+	Source      string                `yaml:",omitempty" json:"source,omitempty"`
-	Target      string               `yaml:",omitempty" json:"target,omitempty"`
+	Target      string                `yaml:",omitempty" json:"target,omitempty"`
-	ReadOnly    bool                 `mapstructure:"read_only" yaml:"read_only,omitempty" json:"read_only,omitempty"`
+	ReadOnly    bool                  `mapstructure:"read_only" yaml:"read_only,omitempty" json:"read_only,omitempty"`
-	Consistency string               `yaml:",omitempty" json:"consistency,omitempty"`
+	Consistency string                `yaml:",omitempty" json:"consistency,omitempty"`
-	Bind        *ServiceVolumeBind   `yaml:",omitempty" json:"bind,omitempty"`
+	Bind        *ServiceVolumeBind    `yaml:",omitempty" json:"bind,omitempty"`
-	Volume      *ServiceVolumeVolume `yaml:",omitempty" json:"volume,omitempty"`
+	Volume      *ServiceVolumeVolume  `yaml:",omitempty" json:"volume,omitempty"`
-	Tmpfs       *ServiceVolumeTmpfs  `yaml:",omitempty" json:"tmpfs,omitempty"`
+	Tmpfs       *ServiceVolumeTmpfs   `yaml:",omitempty" json:"tmpfs,omitempty"`
+	Cluster     *ServiceVolumeCluster `yaml:",omitempty" json:"cluster,omitempty"`
 }
 
 // ServiceVolumeBind are options for a service volume of type bind
@@ -409,6 +410,10 @@ type ServiceVolumeTmpfs struct {
 	Size int64 `yaml:",omitempty" json:"size,omitempty"`
 }
 
+// ServiceVolumeCluster are options for a service volume of type cluster.
+// Deliberately left blank for future options, but unused now.
+type ServiceVolumeCluster struct{}
+
 // FileReferenceConfig for a reference to a swarm file object
 type FileReferenceConfig struct {
 	Source string  `yaml:",omitempty" json:"source,omitempty"`
@@ -480,6 +485,63 @@ type VolumeConfig struct {
 	External   External               `yaml:",omitempty" json:"external,omitempty"`
 	Labels     Labels                 `yaml:",omitempty" json:"labels,omitempty"`
 	Extras     map[string]interface{} `yaml:",inline" json:"-"`
+	Spec       *ClusterVolumeSpec     `mapstructure:"x-cluster-spec" yaml:"x-cluster-spec,omitempty" json:"x-cluster-spec,omitempty"`
+}
+
+// ClusterVolumeSpec defines all the configuration and options specific to a
+// cluster (CSI) volume.
+type ClusterVolumeSpec struct {
+	Group                     string               `yaml:",omitempty" json:"group,omitempty"`
+	AccessMode                *AccessMode          `mapstructure:"access_mode" yaml:"access_mode,omitempty" json:"access_mode,omitempty"`
+	AccessibilityRequirements *TopologyRequirement `mapstructure:"accessibility_requirements" yaml:"accessibility_requirements,omitempty" json:"accessibility_requirements,omitempty"`
+	CapacityRange             *CapacityRange       `mapstructure:"capacity_range" yaml:"capacity_range,omitempty" json:"capacity_range,omitempty"`
+
+	Secrets []VolumeSecret `yaml:",omitempty" json:"secrets,omitempty"`
+
+	Availability string `yaml:",omitempty" json:"availability,omitempty"`
+}
+
+// AccessMode defines the way a cluster volume is accessed by the tasks
+type AccessMode struct {
+	Scope   string `yaml:",omitempty" json:"scope,omitempty"`
+	Sharing string `yaml:",omitempty" json:"sharing,omitempty"`
+
+	MountVolume *MountVolume `mapstructure:"mount_volume" yaml:"mount_volume,omitempty" json:"mount_volume,omitempty"`
+	BlockVolume *BlockVolume `mapstructure:"block_volume" yaml:"block_volume,omitempty" json:"block_volume,omitempty"`
+}
+
+// MountVolume defines options for using a volume as a Mount
+type MountVolume struct {
+	FsType     string   `mapstructure:"fs_type" yaml:"fs_type,omitempty" json:"fs_type,omitempty"`
+	MountFlags []string `mapstructure:"mount_flags" yaml:"mount_flags,omitempty" json:"mount_flags,omitempty"`
+}
+
+// BlockVolume is deliberately empty
+type BlockVolume struct{}
+
+// TopologyRequirement defines the requirements for volume placement in the
+// cluster.
+type TopologyRequirement struct {
+	Requisite []Topology `yaml:",omitempty" json:"requisite,omitempty"`
+	Preferred []Topology `yaml:",omitempty" json:"preferred,omitempty"`
+}
+
+// Topology defines a particular topology group
+type Topology struct {
+	Segments Mapping `yaml:",omitempty" json:"segments,omitempty"`
+}
+
+// CapacityRange defines the minimum and maximum size of a volume.
+type CapacityRange struct {
+	RequiredBytes UnitBytes `mapstructure:"required_bytes" yaml:"required_bytes,omitempty" json:"required_bytes,omitempty"`
+	LimitBytes    UnitBytes `mapstructure:"limit_bytes" yaml:"limit_bytes,omitempty" json:"limit_bytes,omitempty"`
+}
+
+// VolumeSecret defines a secret that needs to be passed to the CSI plugin when
+// using the volume.
+type VolumeSecret struct {
+	Key    string `yaml:",omitempty" json:"key,omitempty"`
+	Secret string `yaml:",omitempty" json:"secret,omitempty"`
 }
 
 // External identifies a Volume or Network as a reference to a resource that is