Implement secret types for compose file.

Signed-off-by: Daniel Nephin <dnephin@docker.com>

command/service/create.go

@@ -62,7 +62,7 @@ func runCreate(dockerCli *command.DockerCli, opts *serviceOptions) error {
 	specifiedSecrets := opts.secrets.Value()
 	if len(specifiedSecrets) > 0 {
 		// parse and validate secrets
-		secrets, err := parseSecrets(apiClient, specifiedSecrets)
+		secrets, err := ParseSecrets(apiClient, specifiedSecrets)
 		if err != nil {
 			return err
 		}

command/service/parse.go

@@ -10,9 +10,9 @@ import (
 	"golang.org/x/net/context"
 )
 
-// parseSecrets retrieves the secrets from the requested names and converts
+// ParseSecrets retrieves the secrets from the requested names and converts
 // them to secret references to use with the spec
-func parseSecrets(client client.SecretAPIClient, requestedSecrets []*types.SecretRequestOption) ([]*swarmtypes.SecretReference, error) {
+func ParseSecrets(client client.SecretAPIClient, requestedSecrets []*types.SecretRequestOption) ([]*swarmtypes.SecretReference, error) {
 	secretRefs := make(map[string]*swarmtypes.SecretReference)
 	ctx := context.Background()
 

command/service/update.go

@@ -443,7 +443,7 @@ func getUpdatedSecrets(apiClient client.SecretAPIClient, flags *pflag.FlagSet, s
 	if flags.Changed(flagSecretAdd) {
 		values := flags.Lookup(flagSecretAdd).Value.(*opts.SecretOpt).Value()
 
-		addSecrets, err := parseSecrets(apiClient, values)
+		addSecrets, err := ParseSecrets(apiClient, values)
 		if err != nil {
 			return nil, err
 		}

command/stack/deploy.go

@@ -126,7 +126,16 @@ func deployCompose(ctx context.Context, dockerCli *command.DockerCli, opts deplo
 	if err := createNetworks(ctx, dockerCli, namespace, networks); err != nil {
 		return err
 	}
-	services, err := convert.Services(namespace, config)
+
+	secrets, err := convert.Secrets(namespace, config.Secrets)
+	if err != nil {
+		return err
+	}
+	if err := createSecrets(ctx, dockerCli, namespace, secrets); err != nil {
+		return err
+	}
+
+	services, err := convert.Services(namespace, config, dockerCli.Client())
 	if err != nil {
 		return err
 	}
@@ -211,6 +220,24 @@ func validateExternalNetworks(
 	return nil
 }
 
+func createSecrets(
+	ctx context.Context,
+	dockerCli *command.DockerCli,
+	namespace convert.Namespace,
+	secrets []swarm.SecretSpec,
+) error {
+	client := dockerCli.Client()
+
+	for _, secret := range secrets {
+		fmt.Fprintf(dockerCli.Out(), "Creating secret %s\n", secret.Name)
+		_, err := client.SecretCreate(ctx, secret)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 func createNetworks(
 	ctx context.Context,
 	dockerCli *command.DockerCli,

compose/convert/compose.go

@@ -1,8 +1,11 @@
 package convert
 
 import (
+	"io/ioutil"
+
 	"github.com/docker/docker/api/types"
 	networktypes "github.com/docker/docker/api/types/network"
+	"github.com/docker/docker/api/types/swarm"
 	composetypes "github.com/docker/docker/cli/compose/types"
 )
 
@@ -82,3 +85,27 @@ func Networks(namespace Namespace, networks networkMap, servicesNetworks map[str
 
 	return result, externalNetworks
 }
+
+// Secrets converts secrets from the Compose type to the engine API type
+func Secrets(namespace Namespace, secrets map[string]composetypes.SecretConfig) ([]swarm.SecretSpec, error) {
+	result := []swarm.SecretSpec{}
+	for name, secret := range secrets {
+		if secret.External.External {
+			continue
+		}
+
+		data, err := ioutil.ReadFile(secret.File)
+		if err != nil {
+			return nil, err
+		}
+
+		result = append(result, swarm.SecretSpec{
+			Annotations: swarm.Annotations{
+				Name:   namespace.Scope(name),
+				Labels: AddStackLabel(namespace, secret.Labels),
+			},
+			Data: data,
+		})
+	}
+	return result, nil
+}

compose/convert/service.go

@@ -2,20 +2,26 @@ package convert
 
 import (
 	"fmt"
+	"os"
 	"time"
 
+	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/swarm"
+	servicecli "github.com/docker/docker/cli/command/service"
 	composetypes "github.com/docker/docker/cli/compose/types"
+	"github.com/docker/docker/client"
 	"github.com/docker/docker/opts"
 	runconfigopts "github.com/docker/docker/runconfig/opts"
 	"github.com/docker/go-connections/nat"
 )
 
 // Services from compose-file types to engine API types
+// TODO: fix secrets API so that SecretAPIClient is not required here
 func Services(
 	namespace Namespace,
 	config *composetypes.Config,
+	client client.SecretAPIClient,
 ) (map[string]swarm.ServiceSpec, error) {
 	result := make(map[string]swarm.ServiceSpec)
 
@@ -24,7 +30,12 @@ func Services(
 	networks := config.Networks
 
 	for _, service := range services {
-		serviceSpec, err := convertService(namespace, service, networks, volumes)
+
+		secrets, err := convertServiceSecrets(client, namespace, service.Secrets)
+		if err != nil {
+			return nil, err
+		}
+		serviceSpec, err := convertService(namespace, service, networks, volumes, secrets)
 		if err != nil {
 			return nil, err
 		}
@@ -39,6 +50,7 @@ func convertService(
 	service composetypes.ServiceConfig,
 	networkConfigs map[string]composetypes.NetworkConfig,
 	volumes map[string]composetypes.VolumeConfig,
+	secrets []*swarm.SecretReference,
 ) (swarm.ServiceSpec, error) {
 	name := namespace.Scope(service.Name)
 
@@ -108,6 +120,7 @@ func convertService(
 				StopGracePeriod: service.StopGracePeriod,
 				TTY:             service.Tty,
 				OpenStdin:       service.StdinOpen,
+				Secrets:         secrets,
 			},
 			LogDriver:     logDriver,
 			Resources:     resources,
@@ -163,6 +176,30 @@ func convertServiceNetworks(
 	return nets, nil
 }
 
+// TODO: fix secrets API so that SecretAPIClient is not required here
+func convertServiceSecrets(
+	client client.SecretAPIClient,
+	namespace Namespace,
+	secrets []composetypes.ServiceSecretConfig,
+) ([]*swarm.SecretReference, error) {
+	opts := []*types.SecretRequestOption{}
+	for _, secret := range secrets {
+		target := secret.Target
+		if target == "" {
+			target = secret.Source
+		}
+		opts = append(opts, &types.SecretRequestOption{
+			Source: namespace.Scope(secret.Source),
+			Target: target,
+			UID:    secret.UID,
+			GID:    secret.GID,
+			Mode:   os.FileMode(secret.Mode),
+		})
+	}
+
+	return servicecli.ParseSecrets(client, opts)
+}
+
 func convertExtraHosts(extraHosts map[string]string) []string {
 	hosts := []string{}
 	for host, ip := range extraHosts {

compose/loader/loader.go

@@ -109,6 +109,20 @@ func Load(configDetails types.ConfigDetails) (*types.Config, error) {
 		cfg.Volumes = volumesMapping
 	}
 
+	if secrets, ok := configDict["secrets"]; ok {
+		secretsConfig, err := interpolation.Interpolate(secrets.(types.Dict), "secret", os.LookupEnv)
+		if err != nil {
+			return nil, err
+		}
+
+		secretsMapping, err := loadSecrets(secretsConfig, configDetails.WorkingDir)
+		if err != nil {
+			return nil, err
+		}
+
+		cfg.Secrets = secretsMapping
+	}
+
 	return &cfg, nil
 }
 
@@ -210,13 +224,15 @@ func transformHook(
 ) (interface{}, error) {
 	switch target {
 	case reflect.TypeOf(types.External{}):
-		return transformExternal(source, target, data)
+		return transformExternal(data)
 	case reflect.TypeOf(make(map[string]string, 0)):
 		return transformMapStringString(source, target, data)
 	case reflect.TypeOf(types.UlimitsConfig{}):
-		return transformUlimits(source, target, data)
+		return transformUlimits(data)
 	case reflect.TypeOf(types.UnitBytes(0)):
 		return loadSize(data)
+	case reflect.TypeOf(types.ServiceSecretConfig{}):
+		return transformServiceSecret(data)
 	}
 	switch target.Kind() {
 	case reflect.Struct:
@@ -311,7 +327,7 @@ func resolveEnvironment(serviceConfig *types.ServiceConfig, serviceDict types.Di
 		var envVars []string
 
 		for _, file := range envFiles {
-			filePath := path.Join(workingDir, file)
+			filePath := absPath(workingDir, file)
 			fileVars, err := opts.ParseEnvFile(filePath)
 			if err != nil {
 				return err
@@ -341,7 +357,7 @@ func resolveVolumePaths(volumes []string, workingDir string) error {
 		}
 
 		if strings.HasPrefix(parts[0], ".") {
-			parts[0] = path.Join(workingDir, parts[0])
+			parts[0] = absPath(workingDir, parts[0])
 		}
 		parts[0] = expandUser(parts[0])
 
@@ -359,11 +375,7 @@ func expandUser(path string) string {
 	return path
 }
 
-func transformUlimits(
+func transformUlimits(data interface{}) (interface{}, error) {
-	source reflect.Type,
-	target reflect.Type,
-	data interface{},
-) (interface{}, error) {
 	switch value := data.(type) {
 	case int:
 		return types.UlimitsConfig{Single: value}, nil
@@ -407,6 +419,32 @@ func loadVolumes(source types.Dict) (map[string]types.VolumeConfig, error) {
 	return volumes, nil
 }
 
+// TODO: remove duplicate with networks/volumes
+func loadSecrets(source types.Dict, workingDir string) (map[string]types.SecretConfig, error) {
+	secrets := make(map[string]types.SecretConfig)
+	err := transform(source, &secrets)
+	if err != nil {
+		return secrets, err
+	}
+	for name, secret := range secrets {
+		if secret.External.External && secret.External.Name == "" {
+			secret.External.Name = name
+			secrets[name] = secret
+		}
+		if secret.File != "" {
+			secret.File = absPath(workingDir, secret.File)
+		}
+	}
+	return secrets, nil
+}
+
+func absPath(workingDir string, filepath string) string {
+	if path.IsAbs(filepath) {
+		return filepath
+	}
+	return path.Join(workingDir, filepath)
+}
+
 func transformStruct(
 	source reflect.Type,
 	target reflect.Type,
@@ -490,11 +528,7 @@ func convertField(
 	return data, nil
 }
 
-func transformExternal(
+func transformExternal(data interface{}) (interface{}, error) {
-	source reflect.Type,
-	target reflect.Type,
-	data interface{},
-) (interface{}, error) {
 	switch value := data.(type) {
 	case bool:
 		return map[string]interface{}{"external": value}, nil
@@ -507,6 +541,20 @@ func transformExternal(
 	}
 }
 
+func transformServiceSecret(data interface{}) (interface{}, error) {
+	switch value := data.(type) {
+	case string:
+		return map[string]interface{}{"source": value}, nil
+	case types.Dict:
+		return data, nil
+	case map[string]interface{}:
+		return data, nil
+	default:
+		return data, fmt.Errorf("invalid type %T for external", value)
+	}
+
+}
+
 func toYAMLName(name string) string {
 	nameParts := fieldNameRegexp.FindAllString(name, -1)
 	for i, p := range nameParts {

compose/loader/loader_test.go

@@ -163,6 +163,24 @@ func TestLoad(t *testing.T) {
 	assert.Equal(t, sampleConfig.Volumes, actual.Volumes)
 }
 
+func TestLoadV31(t *testing.T) {
+	actual, err := loadYAML(`
+version: "3.1"
+services:
+  foo:
+    image: busybox
+    secrets: [super]
+secrets:
+  super:
+    external: true
+`)
+	if !assert.NoError(t, err) {
+		return
+	}
+	assert.Equal(t, len(actual.Services), 1)
+	assert.Equal(t, len(actual.Secrets), 1)
+}
+
 func TestParseAndLoad(t *testing.T) {
 	actual, err := loadYAML(sampleYAML)
 	if !assert.NoError(t, err) {

compose/schema/data/config_schema_v3.1.json

@@ -374,9 +374,9 @@
           "properties": {
             "name": {"type": "string"}
           }
-        }
         },
-      "labels": {"$ref": "#/definitions/list_or_dict"},
+        "labels": {"$ref": "#/definitions/list_or_dict"}
+      },
       "additionalProperties": false
     },
 

compose/types/types.go

@@ -71,6 +71,7 @@ type Config struct {
 	Services []ServiceConfig
 	Networks map[string]NetworkConfig
 	Volumes  map[string]VolumeConfig
+	Secrets  map[string]SecretConfig
 }
 
 // ServiceConfig is the configuration of one service
@@ -108,6 +109,7 @@ type ServiceConfig struct {
 	Privileged      bool
 	ReadOnly        bool `mapstructure:"read_only"`
 	Restart         string
+	Secrets         []ServiceSecretConfig
 	SecurityOpt     []string       `mapstructure:"security_opt"`
 	StdinOpen       bool           `mapstructure:"stdin_open"`
 	StopGracePeriod *time.Duration `mapstructure:"stop_grace_period"`
@@ -191,6 +193,15 @@ type ServiceNetworkConfig struct {
 	Ipv6Address string `mapstructure:"ipv6_address"`
 }
 
+// ServiceSecretConfig is the secret configuration for a service
+type ServiceSecretConfig struct {
+	Source string
+	Target string
+	UID    string
+	GID    string
+	Mode   uint32
+}
+
 // UlimitsConfig the ulimit configuration
 type UlimitsConfig struct {
 	Single int
@@ -233,3 +244,10 @@ type External struct {
 	Name     string
 	External bool
 }
+
+// SecretConfig for a secret
+type SecretConfig struct {
+	File     string
+	External External
+	Labels   map[string]string `compose:"list_or_dict_equals"`
+}