Add support for configs to compose format

Signed-off-by: Brian Goff <cpuguy83@gmail.com>

cli/command/stack/deploy_composefile.go

@@ -79,6 +79,14 @@ func deployCompose(ctx context.Context, dockerCli command.Cli, opts deployOption
 		return err
 	}
 
+	configs, err := convert.Configs(namespace, config.Configs)
+	if err != nil {
+		return err
+	}
+	if err := createConfigs(ctx, dockerCli, namespace, configs); err != nil {
+		return err
+	}
+
 	services, err := convert.Services(namespace, config, dockerCli.Client())
 	if err != nil {
 		return err
@@ -208,6 +216,33 @@ func createSecrets(
 	return nil
 }
 
+func createConfigs(
+	ctx context.Context,
+	dockerCli command.Cli,
+	namespace convert.Namespace,
+	configs []swarm.ConfigSpec,
+) error {
+	client := dockerCli.Client()
+
+	for _, configSpec := range configs {
+		config, _, err := client.ConfigInspectWithRaw(ctx, configSpec.Name)
+		if err == nil {
+			// config already exists, then we update that
+			if err := client.ConfigUpdate(ctx, config.ID, config.Meta.Version, configSpec); err != nil {
+				return err
+			}
+		} else if apiclient.IsErrConfigNotFound(err) {
+			// config does not exist, then we create a new one.
+			if _, err := client.ConfigCreate(ctx, configSpec); err != nil {
+				return err
+			}
+		} else {
+			return err
+		}
+	}
+	return nil
+}
+
 func createNetworks(
 	ctx context.Context,
 	dockerCli command.Cli,

cli/compose/convert/compose.go

@@ -116,3 +116,27 @@ func Secrets(namespace Namespace, secrets map[string]composetypes.SecretConfig)
 	}
 	return result, nil
 }
+
+// Configs converts config objects from the Compose type to the engine API type
+func Configs(namespace Namespace, configs map[string]composetypes.ConfigObjConfig) ([]swarm.ConfigSpec, error) {
+	result := []swarm.ConfigSpec{}
+	for name, config := range configs {
+		if config.External.External {
+			continue
+		}
+
+		data, err := ioutil.ReadFile(config.File)
+		if err != nil {
+			return nil, err
+		}
+
+		result = append(result, swarm.ConfigSpec{
+			Annotations: swarm.Annotations{
+				Name:   namespace.Scope(name),
+				Labels: AddStackLabel(namespace, config.Labels),
+			},
+			Data: data,
+		})
+	}
+	return result, nil
+}

cli/compose/convert/compose_test.go

@@ -133,3 +133,34 @@ func TestSecrets(t *testing.T) {
 	}, secret.Labels)
 	assert.Equal(t, []byte(secretText), secret.Data)
 }
+
+func TestConfigs(t *testing.T) {
+	namespace := Namespace{name: "foo"}
+
+	configText := "this is the first config"
+	configFile := tempfile.NewTempFile(t, "convert-configs", configText)
+	defer configFile.Remove()
+
+	source := map[string]composetypes.ConfigObjConfig{
+		"one": {
+			File:   configFile.Name(),
+			Labels: map[string]string{"monster": "mash"},
+		},
+		"ext": {
+			External: composetypes.External{
+				External: true,
+			},
+		},
+	}
+
+	specs, err := Configs(namespace, source)
+	assert.NoError(t, err)
+	require.Len(t, specs, 1)
+	config := specs[0]
+	assert.Equal(t, "foo_one", config.Name)
+	assert.Equal(t, map[string]string{
+		"monster":      "mash",
+		LabelNamespace: "foo",
+	}, config.Labels)
+	assert.Equal(t, []byte(configText), config.Data)
+}

cli/compose/convert/service.go

@@ -37,7 +37,12 @@ func Services(
 		if err != nil {
 			return nil, errors.Wrapf(err, "service %s", service.Name)
 		}
-		serviceSpec, err := convertService(client.ClientVersion(), namespace, service, networks, volumes, secrets)
+		configs, err := convertServiceConfigObjs(client, namespace, service.Configs, config.Configs)
+		if err != nil {
+			return nil, errors.Wrapf(err, "service %s", service.Name)
+		}
+
+		serviceSpec, err := convertService(client.ClientVersion(), namespace, service, networks, volumes, secrets, configs)
 		if err != nil {
 			return nil, errors.Wrapf(err, "service %s", service.Name)
 		}
@@ -54,6 +59,7 @@ func convertService(
 	networkConfigs map[string]composetypes.NetworkConfig,
 	volumes map[string]composetypes.VolumeConfig,
 	secrets []*swarm.SecretReference,
+	configs []*swarm.ConfigReference,
 ) (swarm.ServiceSpec, error) {
 	name := namespace.Scope(service.Name)
 
@@ -277,6 +283,57 @@ func convertServiceSecrets(
 	return servicecli.ParseSecrets(client, refs)
 }
 
+// TODO: fix configs API so that ConfigsAPIClient is not required here
+func convertServiceConfigObjs(
+	client client.ConfigAPIClient,
+	namespace Namespace,
+	configs []composetypes.ServiceConfigObjConfig,
+	configSpecs map[string]composetypes.ConfigObjConfig,
+) ([]*swarm.ConfigReference, error) {
+	refs := []*swarm.ConfigReference{}
+	for _, config := range configs {
+		target := config.Target
+		if target == "" {
+			target = config.Source
+		}
+
+		configSpec, exists := configSpecs[config.Source]
+		if !exists {
+			return nil, errors.Errorf("undefined config %q", config.Source)
+		}
+
+		source := namespace.Scope(config.Source)
+		if configSpec.External.External {
+			source = configSpec.External.Name
+		}
+
+		uid := config.UID
+		gid := config.GID
+		if uid == "" {
+			uid = "0"
+		}
+		if gid == "" {
+			gid = "0"
+		}
+		mode := config.Mode
+		if mode == nil {
+			mode = uint32Ptr(0444)
+		}
+
+		refs = append(refs, &swarm.ConfigReference{
+			File: &swarm.ConfigReferenceFileTarget{
+				Name: target,
+				UID:  uid,
+				GID:  gid,
+				Mode: os.FileMode(*mode),
+			},
+			ConfigName: source,
+		})
+	}
+
+	return servicecli.ParseConfigs(client, refs)
+}
+
 func uint32Ptr(value uint32) *uint32 {
 	return &value
 }

cli/compose/loader/loader.go

@@ -66,69 +66,58 @@ func Load(configDetails types.ConfigDetails) (*types.Config, error) {
 	}
 
 	cfg := types.Config{}
-	lookupEnv := func(k string) (string, bool) {
-		v, ok := configDetails.Environment[k]
-		return v, ok
-	}
-	if services, ok := configDict["services"]; ok {
-		servicesConfig, err := interpolation.Interpolate(services.(map[string]interface{}), "service", lookupEnv)
-		if err != nil {
-			return nil, err
-		}
 
-		servicesList, err := LoadServices(servicesConfig, configDetails.WorkingDir, lookupEnv)
+	config, err := interpolateConfig(configDict, configDetails.LookupEnv)
-		if err != nil {
+	if err != nil {
-			return nil, err
+		return nil, err
-		}
-
-		cfg.Services = servicesList
 	}
 
-	if networks, ok := configDict["networks"]; ok {
+	cfg.Services, err = LoadServices(config["services"], configDetails.WorkingDir, configDetails.LookupEnv)
-		networksConfig, err := interpolation.Interpolate(networks.(map[string]interface{}), "network", lookupEnv)
+	if err != nil {
-		if err != nil {
+		return nil, err
-			return nil, err
-		}
-
-		networksMapping, err := LoadNetworks(networksConfig)
-		if err != nil {
-			return nil, err
-		}
-
-		cfg.Networks = networksMapping
 	}
 
-	if volumes, ok := configDict["volumes"]; ok {
+	cfg.Networks, err = LoadNetworks(config["networks"])
-		volumesConfig, err := interpolation.Interpolate(volumes.(map[string]interface{}), "volume", lookupEnv)
+	if err != nil {
-		if err != nil {
+		return nil, err
-			return nil, err
-		}
-
-		volumesMapping, err := LoadVolumes(volumesConfig)
-		if err != nil {
-			return nil, err
-		}
-
-		cfg.Volumes = volumesMapping
 	}
 
-	if secrets, ok := configDict["secrets"]; ok {
+	cfg.Volumes, err = LoadVolumes(config["volumes"])
-		secretsConfig, err := interpolation.Interpolate(secrets.(map[string]interface{}), "secret", lookupEnv)
+	if err != nil {
-		if err != nil {
+		return nil, err
-			return nil, err
+	}
-		}
 
-		secretsMapping, err := LoadSecrets(secretsConfig, configDetails.WorkingDir)
+	cfg.Secrets, err = LoadSecrets(config["secrets"], configDetails.WorkingDir)
-		if err != nil {
+	if err != nil {
-			return nil, err
+		return nil, err
-		}
+	}
 
-		cfg.Secrets = secretsMapping
+	cfg.Configs, err = LoadConfigObjs(config["configs"], configDetails.WorkingDir)
+	if err != nil {
+		return nil, err
 	}
 
 	return &cfg, nil
 }
 
+func interpolateConfig(configDict map[string]interface{}, lookupEnv template.Mapping) (map[string]map[string]interface{}, error) {
+	config := make(map[string]map[string]interface{})
+
+	for _, key := range []string{"services", "networks", "volumes", "secrets", "configs"} {
+		section, ok := configDict[key]
+		if !ok {
+			config[key] = make(map[string]interface{})
+			continue
+		}
+		var err error
+		config[key], err = interpolation.Interpolate(section.(map[string]interface{}), key, lookupEnv)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return config, nil
+}
+
 // GetUnsupportedProperties returns the list of any unsupported properties that are
 // used in the Compose files.
 func GetUnsupportedProperties(configDetails types.ConfigDetails) []string {
@@ -241,7 +230,9 @@ func transformHook(
 	case reflect.TypeOf([]types.ServicePortConfig{}):
 		return transformServicePort(data)
 	case reflect.TypeOf(types.ServiceSecretConfig{}):
-		return transformServiceSecret(data)
+		return transformStringSourceMap(data)
+	case reflect.TypeOf(types.ServiceConfigObjConfig{}):
+		return transformStringSourceMap(data)
 	case reflect.TypeOf(types.StringOrNumberList{}):
 		return transformStringOrNumberList(data)
 	case reflect.TypeOf(map[string]*types.ServiceNetworkConfig{}):
@@ -482,6 +473,25 @@ func LoadSecrets(source map[string]interface{}, workingDir string) (map[string]t
 	return secrets, nil
 }
 
+// LoadConfigObjs produces a ConfigObjConfig map from a compose file Dict
+// the source Dict is not validated if directly used. Use Load() to enable validation
+func LoadConfigObjs(source map[string]interface{}, workingDir string) (map[string]types.ConfigObjConfig, error) {
+	configs := make(map[string]types.ConfigObjConfig)
+	if err := transform(source, &configs); err != nil {
+		return configs, err
+	}
+	for name, config := range configs {
+		if config.External.External && config.External.Name == "" {
+			config.External.Name = name
+			configs[name] = config
+		}
+		if config.File != "" {
+			config.File = absPath(workingDir, config.File)
+		}
+	}
+	return configs, nil
+}
+
 func absPath(workingDir string, filepath string) string {
 	if path.IsAbs(filepath) {
 		return filepath
@@ -544,7 +554,7 @@ func transformServicePort(data interface{}) (interface{}, error) {
 	}
 }
 
-func transformServiceSecret(data interface{}) (interface{}, error) {
+func transformStringSourceMap(data interface{}) (interface{}, error) {
 	switch value := data.(type) {
 	case string:
 		return map[string]interface{}{"source": value}, nil

cli/compose/loader/loader_test.go

@@ -206,12 +206,17 @@ services:
     image: busybox
     credential_spec:
       File: "/foo"
+    configs: [super]
+configs:
+  super:
+    external: true
 `)
 	if !assert.NoError(t, err) {
 		return
 	}
 	assert.Equal(t, len(actual.Services), 1)
 	assert.Equal(t, actual.Services[0].CredentialSpec.File, "/foo")
+	assert.Equal(t, len(actual.Configs), 1)
 }
 
 func TestParseAndLoad(t *testing.T) {

cli/compose/schema/data/config_schema_v3.3.json

@@ -50,6 +50,17 @@
         }
       },
       "additionalProperties": false
+    },
+
+    "configs": {
+      "id": "#/properties/configs",
+      "type": "object",
+      "patternProperties": {
+        "^[a-zA-Z0-9._-]+$": {
+          "$ref": "#/definitions/config"
+        }
+      },
+      "additionalProperties": false
     }
   },
 
@@ -88,6 +99,24 @@
             {"type": "array", "items": {"type": "string"}}
           ]
         },
+        "configs": {
+          "type": "array",
+          "items": {
+            "oneOf": [
+              {"type": "string"},
+              {
+                "type": "object",
+                "properties": {
+                  "source": {"type": "string"},
+                  "target": {"type": "string"},
+                  "uid": {"type": "string"},
+                  "gid": {"type": "string"},
+                  "mode": {"type": "number"}
+                }
+              }
+            ]
+          }
+        },
         "container_name": {"type": "string"},
         "credential_spec": {"type": "object", "properties": {
           "file": {"type": "string"},
@@ -443,6 +472,22 @@
       "additionalProperties": false
     },
 
+    "config": {
+      "id": "#/definitions/config",
+      "type": "object",
+      "properties": {
+        "file": {"type": "string"},
+        "external": {
+          "type": ["boolean", "object"],
+          "properties": {
+            "name": {"type": "string"}
+          }
+        },
+        "labels": {"$ref": "#/definitions/list_or_dict"}
+      },
+      "additionalProperties": false
+    },
+
     "string_or_list": {
       "oneOf": [
         {"type": "string"},

cli/compose/types/types.go

@@ -60,12 +60,19 @@ type ConfigDetails struct {
 	Environment map[string]string
 }
 
+// LookupEnv provides a lookup function for environment variables
+func (cd ConfigDetails) LookupEnv(key string) (string, bool) {
+	v, ok := cd.Environment[key]
+	return v, ok
+}
+
 // Config is a full compose file configuration
 type Config struct {
 	Services []ServiceConfig
 	Networks map[string]NetworkConfig
 	Volumes  map[string]VolumeConfig
 	Secrets  map[string]SecretConfig
+	Configs  map[string]ConfigObjConfig
 }
 
 // ServiceConfig is the configuration of one service
@@ -76,6 +83,7 @@ type ServiceConfig struct {
 	CapDrop         []string `mapstructure:"cap_drop"`
 	CgroupParent    string   `mapstructure:"cgroup_parent"`
 	Command         ShellCommand
+	Configs         []ServiceConfigObjConfig
 	ContainerName   string               `mapstructure:"container_name"`
 	CredentialSpec  CredentialSpecConfig `mapstructure:"credential_spec"`
 	DependsOn       []string             `mapstructure:"depends_on"`
@@ -252,8 +260,7 @@ type ServiceVolumeVolume struct {
 	NoCopy bool `mapstructure:"nocopy"`
 }
 
-// ServiceSecretConfig is the secret configuration for a service
+type fileReferenceConfig struct {
-type ServiceSecretConfig struct {
 	Source string
 	Target string
 	UID    string
@@ -261,6 +268,12 @@ type ServiceSecretConfig struct {
 	Mode   *uint32
 }
 
+// ServiceConfigObjConfig is the config obj configuration for a service
+type ServiceConfigObjConfig fileReferenceConfig
+
+// ServiceSecretConfig is the secret configuration for a service
+type ServiceSecretConfig fileReferenceConfig
+
 // UlimitsConfig the ulimit configuration
 type UlimitsConfig struct {
 	Single int
@@ -305,15 +318,20 @@ type External struct {
 	External bool
 }
 
-// SecretConfig for a secret
-type SecretConfig struct {
-	File     string
-	External External
-	Labels   Labels
-}
-
 // CredentialSpecConfig for credential spec on Windows
 type CredentialSpecConfig struct {
 	File     string
 	Registry string
 }
+
+type fileObjectConfig struct {
+	File     string
+	External External
+	Labels   Labels
+}
+
+// SecretConfig for a secret
+type SecretConfig fileObjectConfig
+
+// ConfigObjConfig is the config for the swarm "Config" object
+type ConfigObjConfig fileObjectConfig