DockerCLI/compose/loader/loader.go

612 lines
15 KiB
Go

package loader
import (
"fmt"
"os"
"path"
"reflect"
"regexp"
"sort"
"strings"
"github.com/docker/docker/cli/compose/interpolation"
"github.com/docker/docker/cli/compose/schema"
"github.com/docker/docker/cli/compose/types"
"github.com/docker/docker/runconfig/opts"
units "github.com/docker/go-units"
shellwords "github.com/mattn/go-shellwords"
"github.com/mitchellh/mapstructure"
yaml "gopkg.in/yaml.v2"
)
var (
fieldNameRegexp = regexp.MustCompile("[A-Z][a-z0-9]+")
)
// ParseYAML reads the bytes from a file, parses the bytes into a mapping
// structure, and returns it.
func ParseYAML(source []byte) (types.Dict, error) {
var cfg interface{}
if err := yaml.Unmarshal(source, &cfg); err != nil {
return nil, err
}
cfgMap, ok := cfg.(map[interface{}]interface{})
if !ok {
return nil, fmt.Errorf("Top-level object must be a mapping")
}
converted, err := convertToStringKeysRecursive(cfgMap, "")
if err != nil {
return nil, err
}
return converted.(types.Dict), nil
}
// Load reads a ConfigDetails and returns a fully loaded configuration
func Load(configDetails types.ConfigDetails) (*types.Config, error) {
if len(configDetails.ConfigFiles) < 1 {
return nil, fmt.Errorf("No files specified")
}
if len(configDetails.ConfigFiles) > 1 {
return nil, fmt.Errorf("Multiple files are not yet supported")
}
configDict := getConfigDict(configDetails)
if services, ok := configDict["services"]; ok {
if servicesDict, ok := services.(types.Dict); ok {
forbidden := getProperties(servicesDict, types.ForbiddenProperties)
if len(forbidden) > 0 {
return nil, &ForbiddenPropertiesError{Properties: forbidden}
}
}
}
if err := schema.Validate(configDict); err != nil {
return nil, err
}
cfg := types.Config{}
version := configDict["version"].(string)
if version != "3" && version != "3.0" {
return nil, fmt.Errorf(`Unsupported Compose file version: %#v. The only version supported is "3" (or "3.0")`, version)
}
if services, ok := configDict["services"]; ok {
servicesConfig, err := interpolation.Interpolate(services.(types.Dict), "service", os.LookupEnv)
if err != nil {
return nil, err
}
servicesList, err := loadServices(servicesConfig, configDetails.WorkingDir)
if err != nil {
return nil, err
}
cfg.Services = servicesList
}
if networks, ok := configDict["networks"]; ok {
networksConfig, err := interpolation.Interpolate(networks.(types.Dict), "network", os.LookupEnv)
if err != nil {
return nil, err
}
networksMapping, err := loadNetworks(networksConfig)
if err != nil {
return nil, err
}
cfg.Networks = networksMapping
}
if volumes, ok := configDict["volumes"]; ok {
volumesConfig, err := interpolation.Interpolate(volumes.(types.Dict), "volume", os.LookupEnv)
if err != nil {
return nil, err
}
volumesMapping, err := loadVolumes(volumesConfig)
if err != nil {
return nil, err
}
cfg.Volumes = volumesMapping
}
return &cfg, nil
}
// GetUnsupportedProperties returns the list of any unsupported properties that are
// used in the Compose files.
func GetUnsupportedProperties(configDetails types.ConfigDetails) []string {
unsupported := map[string]bool{}
for _, service := range getServices(getConfigDict(configDetails)) {
serviceDict := service.(types.Dict)
for _, property := range types.UnsupportedProperties {
if _, isSet := serviceDict[property]; isSet {
unsupported[property] = true
}
}
}
return sortedKeys(unsupported)
}
func sortedKeys(set map[string]bool) []string {
var keys []string
for key := range set {
keys = append(keys, key)
}
sort.Strings(keys)
return keys
}
// GetDeprecatedProperties returns the list of any deprecated properties that
// are used in the compose files.
func GetDeprecatedProperties(configDetails types.ConfigDetails) map[string]string {
return getProperties(getServices(getConfigDict(configDetails)), types.DeprecatedProperties)
}
func getProperties(services types.Dict, propertyMap map[string]string) map[string]string {
output := map[string]string{}
for _, service := range services {
if serviceDict, ok := service.(types.Dict); ok {
for property, description := range propertyMap {
if _, isSet := serviceDict[property]; isSet {
output[property] = description
}
}
}
}
return output
}
// ForbiddenPropertiesError is returned when there are properties in the Compose
// file that are forbidden.
type ForbiddenPropertiesError struct {
Properties map[string]string
}
func (e *ForbiddenPropertiesError) Error() string {
return "Configuration contains forbidden properties"
}
// TODO: resolve multiple files into a single config
func getConfigDict(configDetails types.ConfigDetails) types.Dict {
return configDetails.ConfigFiles[0].Config
}
func getServices(configDict types.Dict) types.Dict {
if services, ok := configDict["services"]; ok {
if servicesDict, ok := services.(types.Dict); ok {
return servicesDict
}
}
return types.Dict{}
}
func transform(source map[string]interface{}, target interface{}) error {
data := mapstructure.Metadata{}
config := &mapstructure.DecoderConfig{
DecodeHook: mapstructure.ComposeDecodeHookFunc(
transformHook,
mapstructure.StringToTimeDurationHookFunc()),
Result: target,
Metadata: &data,
}
decoder, err := mapstructure.NewDecoder(config)
if err != nil {
return err
}
err = decoder.Decode(source)
// TODO: log unused keys
return err
}
func transformHook(
source reflect.Type,
target reflect.Type,
data interface{},
) (interface{}, error) {
switch target {
case reflect.TypeOf(types.External{}):
return transformExternal(source, target, data)
case reflect.TypeOf(make(map[string]string, 0)):
return transformMapStringString(source, target, data)
case reflect.TypeOf(types.UlimitsConfig{}):
return transformUlimits(source, target, data)
case reflect.TypeOf(types.UnitBytes(0)):
return loadSize(data)
}
switch target.Kind() {
case reflect.Struct:
return transformStruct(source, target, data)
}
return data, nil
}
// keys needs to be converted to strings for jsonschema
// TODO: don't use types.Dict
func convertToStringKeysRecursive(value interface{}, keyPrefix string) (interface{}, error) {
if mapping, ok := value.(map[interface{}]interface{}); ok {
dict := make(types.Dict)
for key, entry := range mapping {
str, ok := key.(string)
if !ok {
var location string
if keyPrefix == "" {
location = "at top level"
} else {
location = fmt.Sprintf("in %s", keyPrefix)
}
return nil, fmt.Errorf("Non-string key %s: %#v", location, key)
}
var newKeyPrefix string
if keyPrefix == "" {
newKeyPrefix = str
} else {
newKeyPrefix = fmt.Sprintf("%s.%s", keyPrefix, str)
}
convertedEntry, err := convertToStringKeysRecursive(entry, newKeyPrefix)
if err != nil {
return nil, err
}
dict[str] = convertedEntry
}
return dict, nil
}
if list, ok := value.([]interface{}); ok {
var convertedList []interface{}
for index, entry := range list {
newKeyPrefix := fmt.Sprintf("%s[%d]", keyPrefix, index)
convertedEntry, err := convertToStringKeysRecursive(entry, newKeyPrefix)
if err != nil {
return nil, err
}
convertedList = append(convertedList, convertedEntry)
}
return convertedList, nil
}
return value, nil
}
func loadServices(servicesDict types.Dict, workingDir string) ([]types.ServiceConfig, error) {
var services []types.ServiceConfig
for name, serviceDef := range servicesDict {
serviceConfig, err := loadService(name, serviceDef.(types.Dict), workingDir)
if err != nil {
return nil, err
}
services = append(services, *serviceConfig)
}
return services, nil
}
func loadService(name string, serviceDict types.Dict, workingDir string) (*types.ServiceConfig, error) {
serviceConfig := &types.ServiceConfig{}
if err := transform(serviceDict, serviceConfig); err != nil {
return nil, err
}
serviceConfig.Name = name
if err := resolveEnvironment(serviceConfig, serviceDict, workingDir); err != nil {
return nil, err
}
if err := resolveVolumePaths(serviceConfig.Volumes, workingDir); err != nil {
return nil, err
}
return serviceConfig, nil
}
func resolveEnvironment(serviceConfig *types.ServiceConfig, serviceDict types.Dict, workingDir string) error {
environment := make(map[string]string)
if envFileVal, ok := serviceDict["env_file"]; ok {
envFiles := loadStringOrListOfStrings(envFileVal)
var envVars []string
for _, file := range envFiles {
filePath := path.Join(workingDir, file)
fileVars, err := opts.ParseEnvFile(filePath)
if err != nil {
return err
}
envVars = append(envVars, fileVars...)
}
for k, v := range opts.ConvertKVStringsToMap(envVars) {
environment[k] = v
}
}
for k, v := range serviceConfig.Environment {
environment[k] = v
}
serviceConfig.Environment = environment
return nil
}
func resolveVolumePaths(volumes []string, workingDir string) error {
for i, mapping := range volumes {
parts := strings.SplitN(mapping, ":", 2)
if len(parts) == 1 {
continue
}
if strings.HasPrefix(parts[0], ".") {
parts[0] = path.Join(workingDir, parts[0])
}
parts[0] = expandUser(parts[0])
volumes[i] = strings.Join(parts, ":")
}
return nil
}
// TODO: make this more robust
func expandUser(path string) string {
if strings.HasPrefix(path, "~") {
return strings.Replace(path, "~", os.Getenv("HOME"), 1)
}
return path
}
func transformUlimits(
source reflect.Type,
target reflect.Type,
data interface{},
) (interface{}, error) {
switch value := data.(type) {
case int:
return types.UlimitsConfig{Single: value}, nil
case types.Dict:
ulimit := types.UlimitsConfig{}
ulimit.Soft = value["soft"].(int)
ulimit.Hard = value["hard"].(int)
return ulimit, nil
default:
return data, fmt.Errorf("invalid type %T for ulimits", value)
}
}
func loadNetworks(source types.Dict) (map[string]types.NetworkConfig, error) {
networks := make(map[string]types.NetworkConfig)
err := transform(source, &networks)
if err != nil {
return networks, err
}
for name, network := range networks {
if network.External.External && network.External.Name == "" {
network.External.Name = name
networks[name] = network
}
}
return networks, nil
}
func loadVolumes(source types.Dict) (map[string]types.VolumeConfig, error) {
volumes := make(map[string]types.VolumeConfig)
err := transform(source, &volumes)
if err != nil {
return volumes, err
}
for name, volume := range volumes {
if volume.External.External && volume.External.Name == "" {
volume.External.Name = name
volumes[name] = volume
}
}
return volumes, nil
}
func transformStruct(
source reflect.Type,
target reflect.Type,
data interface{},
) (interface{}, error) {
structValue, ok := data.(map[string]interface{})
if !ok {
// FIXME: this is necessary because of convertToStringKeysRecursive
structValue, ok = data.(types.Dict)
if !ok {
panic(fmt.Sprintf(
"transformStruct called with non-map type: %T, %s", data, data))
}
}
var err error
for i := 0; i < target.NumField(); i++ {
field := target.Field(i)
fieldTag := field.Tag.Get("compose")
yamlName := toYAMLName(field.Name)
value, ok := structValue[yamlName]
if !ok {
continue
}
structValue[yamlName], err = convertField(
fieldTag, reflect.TypeOf(value), field.Type, value)
if err != nil {
return nil, fmt.Errorf("field %s: %s", yamlName, err.Error())
}
}
return structValue, nil
}
func transformMapStringString(
source reflect.Type,
target reflect.Type,
data interface{},
) (interface{}, error) {
switch value := data.(type) {
case map[string]interface{}:
return toMapStringString(value), nil
case types.Dict:
return toMapStringString(value), nil
case map[string]string:
return value, nil
default:
return data, fmt.Errorf("invalid type %T for map[string]string", value)
}
}
func convertField(
fieldTag string,
source reflect.Type,
target reflect.Type,
data interface{},
) (interface{}, error) {
switch fieldTag {
case "":
return data, nil
case "healthcheck":
return loadHealthcheck(data)
case "list_or_dict_equals":
return loadMappingOrList(data, "="), nil
case "list_or_dict_colon":
return loadMappingOrList(data, ":"), nil
case "list_or_struct_map":
return loadListOrStructMap(data, target)
case "string_or_list":
return loadStringOrListOfStrings(data), nil
case "list_of_strings_or_numbers":
return loadListOfStringsOrNumbers(data), nil
case "shell_command":
return loadShellCommand(data)
case "size":
return loadSize(data)
case "-":
return nil, nil
}
return data, nil
}
func transformExternal(
source reflect.Type,
target reflect.Type,
data interface{},
) (interface{}, error) {
switch value := data.(type) {
case bool:
return map[string]interface{}{"external": value}, nil
case types.Dict:
return map[string]interface{}{"external": true, "name": value["name"]}, nil
case map[string]interface{}:
return map[string]interface{}{"external": true, "name": value["name"]}, nil
default:
return data, fmt.Errorf("invalid type %T for external", value)
}
}
func toYAMLName(name string) string {
nameParts := fieldNameRegexp.FindAllString(name, -1)
for i, p := range nameParts {
nameParts[i] = strings.ToLower(p)
}
return strings.Join(nameParts, "_")
}
func loadListOrStructMap(value interface{}, target reflect.Type) (interface{}, error) {
if list, ok := value.([]interface{}); ok {
mapValue := map[interface{}]interface{}{}
for _, name := range list {
mapValue[name] = nil
}
return mapValue, nil
}
return value, nil
}
func loadListOfStringsOrNumbers(value interface{}) []string {
list := value.([]interface{})
result := make([]string, len(list))
for i, item := range list {
result[i] = fmt.Sprint(item)
}
return result
}
func loadStringOrListOfStrings(value interface{}) []string {
if list, ok := value.([]interface{}); ok {
result := make([]string, len(list))
for i, item := range list {
result[i] = fmt.Sprint(item)
}
return result
}
return []string{value.(string)}
}
func loadMappingOrList(mappingOrList interface{}, sep string) map[string]string {
if mapping, ok := mappingOrList.(types.Dict); ok {
return toMapStringString(mapping)
}
if list, ok := mappingOrList.([]interface{}); ok {
result := make(map[string]string)
for _, value := range list {
parts := strings.SplitN(value.(string), sep, 2)
if len(parts) == 1 {
result[parts[0]] = ""
} else {
result[parts[0]] = parts[1]
}
}
return result
}
panic(fmt.Errorf("expected a map or a slice, got: %#v", mappingOrList))
}
func loadShellCommand(value interface{}) (interface{}, error) {
if str, ok := value.(string); ok {
return shellwords.Parse(str)
}
return value, nil
}
func loadHealthcheck(value interface{}) (interface{}, error) {
if str, ok := value.(string); ok {
return append([]string{"CMD-SHELL"}, str), nil
}
return value, nil
}
func loadSize(value interface{}) (int64, error) {
switch value := value.(type) {
case int:
return int64(value), nil
case string:
return units.RAMInBytes(value)
}
panic(fmt.Errorf("invalid type for size %T", value))
}
func toMapStringString(value map[string]interface{}) map[string]string {
output := make(map[string]string)
for key, value := range value {
output[key] = toString(value)
}
return output
}
func toString(value interface{}) string {
if value == nil {
return ""
}
return fmt.Sprint(value)
}