Merge pull request #3544 from thaJeztah/carry_2740_add_config_command

Add stack config command (carry 2740)

cli/command/stack/cmd.go

@@ -34,6 +34,7 @@ func NewStackCommand(dockerCli command.Cli) *cobra.Command {
 		newPsCommand(dockerCli),
 		newRemoveCommand(dockerCli),
 		newServicesCommand(dockerCli),
+		newConfigCommand(dockerCli),
 	)
 	flags := cmd.PersistentFlags()
 	flags.String("orchestrator", "", "Orchestrator to use (swarm|all)")

cli/command/stack/config.go

@@ -0,0 +1,60 @@
+package stack
+
+import (
+	"fmt"
+
+	"github.com/docker/cli/cli"
+	"github.com/docker/cli/cli/command"
+	"github.com/docker/cli/cli/command/stack/loader"
+	"github.com/docker/cli/cli/command/stack/options"
+	composeLoader "github.com/docker/cli/cli/compose/loader"
+	composetypes "github.com/docker/cli/cli/compose/types"
+	"github.com/spf13/cobra"
+	yaml "gopkg.in/yaml.v2"
+)
+
+func newConfigCommand(dockerCli command.Cli) *cobra.Command {
+	var opts options.Config
+
+	cmd := &cobra.Command{
+		Use:   "config [OPTIONS]",
+		Short: "Outputs the final config file, after doing merges and interpolations",
+		Args:  cli.NoArgs,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			configDetails, err := loader.GetConfigDetails(opts.Composefiles, dockerCli.In())
+			if err != nil {
+				return err
+			}
+
+			cfg, err := outputConfig(configDetails, opts.SkipInterpolation)
+			if err != nil {
+				return err
+			}
+
+			_, err = fmt.Fprintf(dockerCli.Out(), "%s", cfg)
+			return err
+		},
+	}
+
+	flags := cmd.Flags()
+	flags.StringSliceVarP(&opts.Composefiles, "compose-file", "c", []string{}, `Path to a Compose file, or "-" to read from stdin`)
+	flags.BoolVar(&opts.SkipInterpolation, "skip-interpolation", false, "Skip interpolation and output only merged config")
+	return cmd
+}
+
+// outputConfig returns the merged and interpolated config file
+func outputConfig(configFiles composetypes.ConfigDetails, skipInterpolation bool) (string, error) {
+	optsFunc := func(options *composeLoader.Options) {
+		options.SkipInterpolation = skipInterpolation
+	}
+	config, err := composeLoader.Load(configFiles, optsFunc)
+	if err != nil {
+		return "", err
+	}
+
+	d, err := yaml.Marshal(&config)
+	if err != nil {
+		return "", err
+	}
+	return string(d), nil
+}

cli/command/stack/config_test.go

@@ -0,0 +1,106 @@
+package stack
+
+import (
+	"io"
+	"testing"
+
+	"github.com/docker/cli/cli/compose/loader"
+	composetypes "github.com/docker/cli/cli/compose/types"
+	"github.com/docker/cli/internal/test"
+	"gotest.tools/v3/assert"
+)
+
+func TestConfigWithEmptyComposeFile(t *testing.T) {
+	cmd := newConfigCommand(test.NewFakeCli(&fakeClient{}))
+	cmd.SetOut(io.Discard)
+
+	assert.ErrorContains(t, cmd.Execute(), `Please specify a Compose file`)
+}
+
+var configMergeTests = []struct {
+	name              string
+	skipInterpolation bool
+	first             string
+	second            string
+	merged            string
+}{
+	{
+		name:              "With Interpolation",
+		skipInterpolation: false,
+		first: `version: "3.7"
+services:
+  foo:
+    image: busybox:latest
+    command: cat file1.txt
+`,
+		second: `version: "3.7"
+services:
+  foo:
+    image: busybox:${VERSION}
+    command: cat file2.txt
+`,
+		merged: `version: "3.7"
+services:
+  foo:
+    command:
+    - cat
+    - file2.txt
+    image: busybox:1.0
+`,
+	},
+	{
+		name:              "Without Interpolation",
+		skipInterpolation: true,
+		first: `version: "3.7"
+services:
+  foo:
+    image: busybox:latest
+    command: cat file1.txt
+`,
+		second: `version: "3.7"
+services:
+  foo:
+    image: busybox:${VERSION}
+    command: cat file2.txt
+`,
+		merged: `version: "3.7"
+services:
+  foo:
+    command:
+    - cat
+    - file2.txt
+    image: busybox:${VERSION}
+`,
+	},
+}
+
+func TestConfigMergeInterpolation(t *testing.T) {
+
+	for _, tt := range configMergeTests {
+		t.Run(tt.name, func(t *testing.T) {
+			firstConfig := []byte(tt.first)
+			secondConfig := []byte(tt.second)
+
+			firstConfigData, err := loader.ParseYAML(firstConfig)
+			assert.NilError(t, err)
+			secondConfigData, err := loader.ParseYAML(secondConfig)
+			assert.NilError(t, err)
+
+			env := map[string]string{
+				"VERSION": "1.0",
+			}
+
+			cfg, err := outputConfig(composetypes.ConfigDetails{
+				ConfigFiles: []composetypes.ConfigFile{
+					{Config: firstConfigData, Filename: "firstConfig"},
+					{Config: secondConfigData, Filename: "secondConfig"},
+				},
+				Environment: env,
+			}, tt.skipInterpolation)
+			assert.NilError(t, err)
+
+			assert.Equal(t, cfg, tt.merged)
+		})
+	}
+
+}

cli/command/stack/loader/loader.go

@@ -18,7 +18,7 @@ import (
 
 // LoadComposefile parse the composefile specified in the cli and returns its Config and version.
 func LoadComposefile(dockerCli command.Cli, opts options.Deploy) (*composetypes.Config, error) {
-	configDetails, err := getConfigDetails(opts.Composefiles, dockerCli.In())
+	configDetails, err := GetConfigDetails(opts.Composefiles, dockerCli.In())
 	if err != nil {
 		return nil, err
 	}
@@ -68,7 +68,8 @@ func propertyWarnings(properties map[string]string) string {
 	return strings.Join(msgs, "\n\n")
 }
 
-func getConfigDetails(composefiles []string, stdin io.Reader) (composetypes.ConfigDetails, error) {
+// GetConfigDetails parse the composefiles specified in the cli and returns their ConfigDetails
+func GetConfigDetails(composefiles []string, stdin io.Reader) (composetypes.ConfigDetails, error) {
 	var details composetypes.ConfigDetails
 
 	if len(composefiles) == 0 {

cli/command/stack/loader/loader_test.go

@@ -21,7 +21,7 @@ services:
 	file := fs.NewFile(t, "test-get-config-details", fs.WithContent(content))
 	defer file.Remove()
 
-	details, err := getConfigDetails([]string{file.Path()}, nil)
+	details, err := GetConfigDetails([]string{file.Path()}, nil)
 	assert.NilError(t, err)
 	assert.Check(t, is.Equal(filepath.Dir(file.Path()), details.WorkingDir))
 	assert.Assert(t, is.Len(details.ConfigFiles, 1))
@@ -36,7 +36,7 @@ services:
   foo:
     image: alpine:3.5
 `
-	details, err := getConfigDetails([]string{"-"}, strings.NewReader(content))
+	details, err := GetConfigDetails([]string{"-"}, strings.NewReader(content))
 	assert.NilError(t, err)
 	cwd, err := os.Getwd()
 	assert.NilError(t, err)

cli/command/stack/options/opts.go

@@ -11,6 +11,12 @@ type Deploy struct {
 	Prune            bool
 }
 
+// Config holds docker stack config options
+type Config struct {
+	Composefiles      []string
+	SkipInterpolation bool
+}
+
 // List holds docker stack ls options
 type List struct {
 	Format        string

cli/compose/loader/merge.go

@@ -58,6 +58,8 @@ func mergeServices(base, override []types.ServiceConfig) ([]types.ServiceConfig,
 			reflect.TypeOf([]types.ServiceSecretConfig{}):    mergeSlice(toServiceSecretConfigsMap, toServiceSecretConfigsSlice),
 			reflect.TypeOf([]types.ServiceConfigObjConfig{}): mergeSlice(toServiceConfigObjConfigsMap, toSServiceConfigObjConfigsSlice),
 			reflect.TypeOf(&types.UlimitsConfig{}):           mergeUlimitsConfig,
+			reflect.TypeOf([]types.ServiceVolumeConfig{}):    mergeSlice(toServiceVolumeConfigsMap, toServiceVolumeConfigsSlice),
+			reflect.TypeOf(types.ShellCommand{}):             mergeShellCommand,
 			reflect.TypeOf(&types.ServiceNetworkConfig{}):    mergeServiceNetworkConfig,
 		},
 	}
@@ -116,6 +118,18 @@ func toServicePortConfigsMap(s interface{}) (map[interface{}]interface{}, error)
 	return m, nil
 }
 
+func toServiceVolumeConfigsMap(s interface{}) (map[interface{}]interface{}, error) {
+	volumes, ok := s.([]types.ServiceVolumeConfig)
+	if !ok {
+		return nil, errors.Errorf("not a serviceVolumeConfig slice: %v", s)
+	}
+	m := map[interface{}]interface{}{}
+	for _, v := range volumes {
+		m[v.Target] = v
+	}
+	return m, nil
+}
+
 func toServiceSecretConfigsSlice(dst reflect.Value, m map[interface{}]interface{}) error {
 	s := []types.ServiceSecretConfig{}
 	for _, v := range m {
@@ -146,6 +160,16 @@ func toServicePortConfigsSlice(dst reflect.Value, m map[interface{}]interface{})
 	return nil
 }
 
+func toServiceVolumeConfigsSlice(dst reflect.Value, m map[interface{}]interface{}) error {
+	s := []types.ServiceVolumeConfig{}
+	for _, v := range m {
+		s = append(s, v.(types.ServiceVolumeConfig))
+	}
+	sort.Slice(s, func(i, j int) bool { return s[i].Target < s[j].Target })
+	dst.Set(reflect.ValueOf(s))
+	return nil
+}
+
 type tomapFn func(s interface{}) (map[interface{}]interface{}, error)
 type writeValueFromMapFn func(reflect.Value, map[interface{}]interface{}) error
 
@@ -211,6 +235,14 @@ func mergeUlimitsConfig(dst, src reflect.Value) error {
 	return nil
 }
 
+//nolint: unparam
+func mergeShellCommand(dst, src reflect.Value) error {
+	if src.Len() != 0 {
+		dst.Set(src)
+	}
+	return nil
+}
+
 //nolint: unparam
 func mergeServiceNetworkConfig(dst, src reflect.Value) error {
 	if src.Interface() != reflect.Zero(reflect.TypeOf(src.Interface())).Interface() {

cli/compose/loader/merge_test.go

@@ -1017,6 +1017,134 @@ func TestLoadMultipleNetworks(t *testing.T) {
 	}, config)
 }
 
+func TestLoadMultipleServiceCommands(t *testing.T) {
+	base := map[string]interface{}{
+		"version": "3.7",
+		"services": map[string]interface{}{
+			"foo": map[string]interface{}{
+				"image":   "baz",
+				"command": "foo bar",
+			},
+		},
+		"volumes":  map[string]interface{}{},
+		"networks": map[string]interface{}{},
+		"secrets":  map[string]interface{}{},
+		"configs":  map[string]interface{}{},
+	}
+	override := map[string]interface{}{
+		"version": "3.7",
+		"services": map[string]interface{}{
+			"foo": map[string]interface{}{
+				"image":   "baz",
+				"command": "foo baz",
+			},
+		},
+		"volumes":  map[string]interface{}{},
+		"networks": map[string]interface{}{},
+		"secrets":  map[string]interface{}{},
+		"configs":  map[string]interface{}{},
+	}
+	configDetails := types.ConfigDetails{
+		ConfigFiles: []types.ConfigFile{
+			{Filename: "base.yml", Config: base},
+			{Filename: "override.yml", Config: override},
+		},
+	}
+	config, err := Load(configDetails)
+	assert.NilError(t, err)
+	assert.DeepEqual(t, &types.Config{
+		Filename: "base.yml",
+		Version:  "3.7",
+		Services: []types.ServiceConfig{
+			{
+				Name:        "foo",
+				Image:       "baz",
+				Command:     types.ShellCommand{"foo", "baz"},
+				Environment: types.MappingWithEquals{},
+			},
+		},
+		Volumes:  map[string]types.VolumeConfig{},
+		Secrets:  map[string]types.SecretConfig{},
+		Configs:  map[string]types.ConfigObjConfig{},
+		Networks: map[string]types.NetworkConfig{},
+	}, config)
+}
+
+func TestLoadMultipleServiceVolumes(t *testing.T) {
+	base := map[string]interface{}{
+		"version": "3.7",
+		"services": map[string]interface{}{
+			"foo": map[string]interface{}{
+				"image": "baz",
+				"volumes": []interface{}{
+					map[string]interface{}{
+						"type":   "volume",
+						"source": "sourceVolume",
+						"target": "/var/app",
+					},
+				},
+			},
+		},
+		"volumes": map[string]interface{}{
+			"sourceVolume": map[string]interface{}{},
+		},
+		"networks": map[string]interface{}{},
+		"secrets":  map[string]interface{}{},
+		"configs":  map[string]interface{}{},
+	}
+	override := map[string]interface{}{
+		"version": "3.7",
+		"services": map[string]interface{}{
+			"foo": map[string]interface{}{
+				"image": "baz",
+				"volumes": []interface{}{
+					map[string]interface{}{
+						"type":   "volume",
+						"source": "/local",
+						"target": "/var/app",
+					},
+				},
+			},
+		},
+		"volumes":  map[string]interface{}{},
+		"networks": map[string]interface{}{},
+		"secrets":  map[string]interface{}{},
+		"configs":  map[string]interface{}{},
+	}
+	configDetails := types.ConfigDetails{
+		ConfigFiles: []types.ConfigFile{
+			{Filename: "base.yml", Config: base},
+			{Filename: "override.yml", Config: override},
+		},
+	}
+	config, err := Load(configDetails)
+	assert.NilError(t, err)
+	assert.DeepEqual(t, &types.Config{
+		Filename: "base.yml",
+		Version:  "3.7",
+		Services: []types.ServiceConfig{
+			{
+				Name:        "foo",
+				Image:       "baz",
+				Environment: types.MappingWithEquals{},
+				Volumes: []types.ServiceVolumeConfig{
+					{
+						Type:   "volume",
+						Source: "/local",
+						Target: "/var/app",
+					},
+				},
+			},
+		},
+		Volumes: map[string]types.VolumeConfig{
+			"sourceVolume": {},
+		},
+		Secrets:  map[string]types.SecretConfig{},
+		Configs:  map[string]types.ConfigObjConfig{},
+		Networks: map[string]types.NetworkConfig{},
+	}, config)
+}
+
 func TestMergeUlimitsConfig(t *testing.T) {
 	specials := &specials{
 		m: map[reflect.Type]func(dst, src reflect.Value) error{

docs/reference/commandline/index.md

@@ -153,12 +153,13 @@ read the [`dockerd`](dockerd.md) reference page.
 ### Swarm stack commands
 
 | Command                             | Description                                             |
-|:------------------------------------|:-----------------------------------------------|
+|:------------------------------------|:--------------------------------------------------------|
 | [stack deploy](stack_deploy.md)     | Deploy a new stack or update an existing stack          |
 | [stack ls](stack_ls.md)             | List stacks in the swarm                                |
 | [stack ps](stack_ps.md)             | List the tasks in the stack                             |
 | [stack rm](stack_rm.md)             | Remove the stack from the swarm                         |
 | [stack services](stack_services.md) | List the services in the stack                          |
+| [stack config](stack_config.md)     | Output the Compose file after merges and interpolations |
 
 ### Plugin commands
 

docs/reference/commandline/stack_config.md

@@ -0,0 +1,68 @@
+---
+title: "stack config"
+description: "The stack config command description and usage"
+keywords: "stack, config"
+---
+
+# stack config
+
+```markdown
+Usage:	docker stack config [OPTIONS]
+
+Outputs the final config file, after doing merges and interpolations
+
+Aliases:
+  config, cfg
+
+Options:
+  -c, --compose-file strings   Path to a Compose file, or "-" to read from stdin
+      --orchestrator string    Orchestrator to use (swarm|kubernetes|all)
+      --skip-interpolation     Skip interpolation and output only merged config
+```
+
+## Description
+
+Outputs the final Compose file, after doing the merges and interpolations of the input Compose files.
+
+## Examples
+
+The following command outputs the result of the merge and interpolation of two Compose files.
+
+```bash
+$ docker stack config --compose-file docker-compose.yml --compose-file docker-compose.prod.yml
+```
+
+The Compose file can also be provided as standard input with `--compose-file -`:
+
+```bash
+$ cat docker-compose.yml | docker stack config --compose-file -
+```
+
+### Skipping interpolation
+
+In some cases, it might be useful to skip interpolation of environment variables.
+For example, when you want to pipe the output of this command back to `stack deploy`.
+
+If you have a regex for a redirect route in an environment variable for your webserver you would use two `$` signs to prevent `stack deploy` from interpolating `${1}`.
+
+```bash
+  service: webserver
+  environment:
+    REDIRECT_REGEX=http://host/redirect/$${1} 
+```
+
+With interpolation, the `stack config` command will replace the environment variable in the Compose file 
+with `REDIRECT_REGEX=http://host/redirect/${1}`, but then when piping it back to the `stack deploy` 
+command it will be interpolated again and result in undefined behavior. 
+That is why, when piping the output back to `stack deploy` one should always prefer the `--skip-interpolation` option.
+
+```
+$ docker stack config --compose-file web.yml --compose-file web.prod.yml --skip-interpolation | docker stack deploy --compose-file -
+```
+
+## Related commands
+
+* [stack deploy](stack_deploy.md)
+* [stack ps](stack_ps.md)
+* [stack rm](stack_rm.md)
+* [stack services](stack_services.md)

docs/reference/commandline/stack_deploy.md

@@ -111,3 +111,4 @@ axqh55ipl40h  vossibility_vossibility-collector  replicated  1/1       icecrime/
 * [stack ps](stack_ps.md)
 * [stack rm](stack_rm.md)
 * [stack services](stack_services.md)
+* [stack config](stack_config.md)

e2e/stack/config_test.go

@@ -0,0 +1,12 @@
+package stack
+
+import (
+	"testing"
+
+	"gotest.tools/v3/icmd"
+)
+
+func TestConfigFullStack(t *testing.T) {
+	result := icmd.RunCommand("docker", "stack", "config", "--compose-file=./testdata/full-stack.yml")
+	result.Assert(t, icmd.Success)
+}