From 7843aec98cbc1f8de4f2daec4f9019ff4b667719 Mon Sep 17 00:00:00 2001 From: Renaud Gaubert Date: Wed, 8 Nov 2017 15:06:43 +0100 Subject: [PATCH 1/7] Revendored swarmkit Signed-off-by: Renaud Gaubert --- vendor.conf | 2 +- .../swarmkit/api/genericresource/helpers.go | 111 +++++++ .../swarmkit/api/genericresource/parse.go | 111 +++++++ .../genericresource/resource_management.go | 202 ++++++++++++ .../swarmkit/api/genericresource/string.go | 54 ++++ .../swarmkit/api/genericresource/validate.go | 84 +++++ .../docker/swarmkit/api/specs.pb.go | 302 ++++++++++-------- .../docker/swarmkit/api/specs.proto | 4 + 8 files changed, 735 insertions(+), 135 deletions(-) create mode 100644 vendor/github.com/docker/swarmkit/api/genericresource/helpers.go create mode 100644 vendor/github.com/docker/swarmkit/api/genericresource/parse.go create mode 100644 vendor/github.com/docker/swarmkit/api/genericresource/resource_management.go create mode 100644 vendor/github.com/docker/swarmkit/api/genericresource/string.go create mode 100644 vendor/github.com/docker/swarmkit/api/genericresource/validate.go diff --git a/vendor.conf b/vendor.conf index 8c1ef581b1..97f8374527 100755 --- a/vendor.conf +++ b/vendor.conf @@ -14,7 +14,7 @@ github.com/docker/go d30aec9fd63c35133f8f79c3412ad91a3b08be06 github.com/docker/go-connections 3ede32e2033de7505e6500d6c868c2b9ed9f169d github.com/docker/go-events 9461782956ad83b30282bf90e31fa6a70c255ba9 github.com/docker/go-units 9e638d38cf6977a37a8ea0078f3ee75a7cdb2dd1 -github.com/docker/swarmkit 28f91d87bd3f75fd039dbb9be49bfd2381019261 +github.com/docker/swarmkit de950a7ed842c7b7e47e9451cde9bf8f96031894 github.com/flynn-archive/go-shlex 3f9db97f856818214da2e1057f8ad84803971cff github.com/gogo/protobuf v0.4 github.com/golang/protobuf 7a211bcf3bce0e3f1d74f9894916e6f116ae83b4 diff --git a/vendor/github.com/docker/swarmkit/api/genericresource/helpers.go b/vendor/github.com/docker/swarmkit/api/genericresource/helpers.go new file mode 100644 index 0000000000..350ab730c1 --- /dev/null +++ b/vendor/github.com/docker/swarmkit/api/genericresource/helpers.go @@ -0,0 +1,111 @@ +package genericresource + +import ( + "github.com/docker/swarmkit/api" +) + +// NewSet creates a set object +func NewSet(key string, vals ...string) []*api.GenericResource { + rs := make([]*api.GenericResource, 0, len(vals)) + + for _, v := range vals { + rs = append(rs, NewString(key, v)) + } + + return rs +} + +// NewString creates a String resource +func NewString(key, val string) *api.GenericResource { + return &api.GenericResource{ + Resource: &api.GenericResource_NamedResourceSpec{ + NamedResourceSpec: &api.NamedGenericResource{ + Kind: key, + Value: val, + }, + }, + } +} + +// NewDiscrete creates a Discrete resource +func NewDiscrete(key string, val int64) *api.GenericResource { + return &api.GenericResource{ + Resource: &api.GenericResource_DiscreteResourceSpec{ + DiscreteResourceSpec: &api.DiscreteGenericResource{ + Kind: key, + Value: val, + }, + }, + } +} + +// GetResource returns resources from the "resources" parameter matching the kind key +func GetResource(kind string, resources []*api.GenericResource) []*api.GenericResource { + var res []*api.GenericResource + + for _, r := range resources { + if Kind(r) != kind { + continue + } + + res = append(res, r) + } + + return res +} + +// ConsumeNodeResources removes "res" from nodeAvailableResources +func ConsumeNodeResources(nodeAvailableResources *[]*api.GenericResource, res []*api.GenericResource) { + if nodeAvailableResources == nil { + return + } + + w := 0 + +loop: + for _, na := range *nodeAvailableResources { + for _, r := range res { + if Kind(na) != Kind(r) { + continue + } + + if remove(na, r) { + continue loop + } + // If this wasn't the right element then + // we need to continue + } + + (*nodeAvailableResources)[w] = na + w++ + } + + *nodeAvailableResources = (*nodeAvailableResources)[:w] +} + +// Returns true if the element is to be removed from the list +func remove(na, r *api.GenericResource) bool { + switch tr := r.Resource.(type) { + case *api.GenericResource_DiscreteResourceSpec: + if na.GetDiscreteResourceSpec() == nil { + return false // Type change, ignore + } + + na.GetDiscreteResourceSpec().Value -= tr.DiscreteResourceSpec.Value + if na.GetDiscreteResourceSpec().Value <= 0 { + return true + } + case *api.GenericResource_NamedResourceSpec: + if na.GetNamedResourceSpec() == nil { + return false // Type change, ignore + } + + if tr.NamedResourceSpec.Value != na.GetNamedResourceSpec().Value { + return false // not the right item, ignore + } + + return true + } + + return false +} diff --git a/vendor/github.com/docker/swarmkit/api/genericresource/parse.go b/vendor/github.com/docker/swarmkit/api/genericresource/parse.go new file mode 100644 index 0000000000..f39a7077a8 --- /dev/null +++ b/vendor/github.com/docker/swarmkit/api/genericresource/parse.go @@ -0,0 +1,111 @@ +package genericresource + +import ( + "encoding/csv" + "fmt" + "strconv" + "strings" + + "github.com/docker/swarmkit/api" +) + +func newParseError(format string, args ...interface{}) error { + return fmt.Errorf("could not parse GenericResource: "+format, args...) +} + +// discreteResourceVal returns an int64 if the string is a discreteResource +// and an error if it isn't +func discreteResourceVal(res string) (int64, error) { + return strconv.ParseInt(res, 10, 64) +} + +// allNamedResources returns true if the array of resources are all namedResources +// e.g: res = [red, orange, green] +func allNamedResources(res []string) bool { + for _, v := range res { + if _, err := discreteResourceVal(v); err == nil { + return false + } + } + + return true +} + +// ParseCmd parses the Generic Resource command line argument +// and returns a list of *api.GenericResource +func ParseCmd(cmd string) ([]*api.GenericResource, error) { + if strings.Contains(cmd, "\n") { + return nil, newParseError("unexpected '\\n' character") + } + + r := csv.NewReader(strings.NewReader(cmd)) + records, err := r.ReadAll() + + if err != nil { + return nil, newParseError("%v", err) + } + + if len(records) != 1 { + return nil, newParseError("found multiple records while parsing cmd %v", records) + } + + return Parse(records[0]) +} + +// Parse parses a table of GenericResource resources +func Parse(cmds []string) ([]*api.GenericResource, error) { + tokens := make(map[string][]string) + + for _, term := range cmds { + kva := strings.Split(term, "=") + if len(kva) != 2 { + return nil, newParseError("incorrect term %s, missing"+ + " '=' or malformed expression", term) + } + + key := strings.TrimSpace(kva[0]) + val := strings.TrimSpace(kva[1]) + + tokens[key] = append(tokens[key], val) + } + + var rs []*api.GenericResource + for k, v := range tokens { + if u, ok := isDiscreteResource(v); ok { + if u < 0 { + return nil, newParseError("cannot ask for"+ + " negative resource %s", k) + } + + rs = append(rs, NewDiscrete(k, u)) + continue + } + + if allNamedResources(v) { + rs = append(rs, NewSet(k, v...)...) + continue + } + + return nil, newParseError("mixed discrete and named resources"+ + " in expression '%s=%s'", k, v) + } + + return rs, nil +} + +// isDiscreteResource returns true if the array of resources is a +// Discrete Resource. +// e.g: res = [1] +func isDiscreteResource(values []string) (int64, bool) { + if len(values) != 1 { + return int64(0), false + } + + u, err := discreteResourceVal(values[0]) + if err != nil { + return int64(0), false + } + + return u, true + +} diff --git a/vendor/github.com/docker/swarmkit/api/genericresource/resource_management.go b/vendor/github.com/docker/swarmkit/api/genericresource/resource_management.go new file mode 100644 index 0000000000..a89a118d62 --- /dev/null +++ b/vendor/github.com/docker/swarmkit/api/genericresource/resource_management.go @@ -0,0 +1,202 @@ +package genericresource + +import ( + "fmt" + "github.com/docker/swarmkit/api" +) + +// Claim assigns GenericResources to a task by taking them from the +// node's GenericResource list and storing them in the task's available list +func Claim(nodeAvailableResources, taskAssigned *[]*api.GenericResource, + taskReservations []*api.GenericResource) error { + var resSelected []*api.GenericResource + + for _, res := range taskReservations { + tr := res.GetDiscreteResourceSpec() + if tr == nil { + return fmt.Errorf("task should only hold Discrete type") + } + + // Select the resources + nrs, err := selectNodeResources(*nodeAvailableResources, tr) + if err != nil { + return err + } + + resSelected = append(resSelected, nrs...) + } + + ClaimResources(nodeAvailableResources, taskAssigned, resSelected) + return nil +} + +// ClaimResources adds the specified resources to the task's list +// and removes them from the node's generic resource list +func ClaimResources(nodeAvailableResources, taskAssigned *[]*api.GenericResource, + resSelected []*api.GenericResource) { + *taskAssigned = append(*taskAssigned, resSelected...) + ConsumeNodeResources(nodeAvailableResources, resSelected) +} + +func selectNodeResources(nodeRes []*api.GenericResource, + tr *api.DiscreteGenericResource) ([]*api.GenericResource, error) { + var nrs []*api.GenericResource + + for _, res := range nodeRes { + if Kind(res) != tr.Kind { + continue + } + + switch nr := res.Resource.(type) { + case *api.GenericResource_DiscreteResourceSpec: + if nr.DiscreteResourceSpec.Value >= tr.Value && tr.Value != 0 { + nrs = append(nrs, NewDiscrete(tr.Kind, tr.Value)) + } + + return nrs, nil + case *api.GenericResource_NamedResourceSpec: + nrs = append(nrs, res.Copy()) + + if int64(len(nrs)) == tr.Value { + return nrs, nil + } + } + } + + if len(nrs) == 0 { + return nil, fmt.Errorf("not enough resources available for task reservations: %+v", tr) + } + + return nrs, nil +} + +// Reclaim adds the resources taken by the task to the node's store +func Reclaim(nodeAvailableResources *[]*api.GenericResource, taskAssigned, nodeRes []*api.GenericResource) error { + err := reclaimResources(nodeAvailableResources, taskAssigned) + if err != nil { + return err + } + + sanitize(nodeRes, nodeAvailableResources) + + return nil +} + +func reclaimResources(nodeAvailableResources *[]*api.GenericResource, taskAssigned []*api.GenericResource) error { + // The node could have been updated + if nodeAvailableResources == nil { + return fmt.Errorf("node no longer has any resources") + } + + for _, res := range taskAssigned { + switch tr := res.Resource.(type) { + case *api.GenericResource_DiscreteResourceSpec: + nrs := GetResource(tr.DiscreteResourceSpec.Kind, *nodeAvailableResources) + + // If the resource went down to 0 it's no longer in the + // available list + if len(nrs) == 0 { + *nodeAvailableResources = append(*nodeAvailableResources, res.Copy()) + } + + if len(nrs) != 1 { + continue // Type change + } + + nr := nrs[0].GetDiscreteResourceSpec() + if nr == nil { + continue // Type change + } + + nr.Value += tr.DiscreteResourceSpec.Value + case *api.GenericResource_NamedResourceSpec: + *nodeAvailableResources = append(*nodeAvailableResources, res.Copy()) + } + } + + return nil +} + +// sanitize checks that nodeAvailableResources does not add resources unknown +// to the nodeSpec (nodeRes) or goes over the integer bound specified +// by the spec. +// Note this is because the user is able to update a node's resources +func sanitize(nodeRes []*api.GenericResource, nodeAvailableResources *[]*api.GenericResource) { + // - We add the sanitized resources at the end, after + // having removed the elements from the list + + // - When a set changes to a Discrete we also need + // to make sure that we don't add the Discrete multiple + // time hence, the need of a map to remember that + var sanitized []*api.GenericResource + kindSanitized := make(map[string]struct{}) + w := 0 + + for _, na := range *nodeAvailableResources { + ok, nrs := sanitizeResource(nodeRes, na) + if !ok { + if _, ok = kindSanitized[Kind(na)]; ok { + continue + } + + kindSanitized[Kind(na)] = struct{}{} + sanitized = append(sanitized, nrs...) + + continue + } + + (*nodeAvailableResources)[w] = na + w++ + } + + *nodeAvailableResources = (*nodeAvailableResources)[:w] + *nodeAvailableResources = append(*nodeAvailableResources, sanitized...) +} + +// Returns true if the element is in nodeRes and "sane" +// Returns false if the element isn't in nodeRes and "sane" and the element(s) that should be replacing it +func sanitizeResource(nodeRes []*api.GenericResource, res *api.GenericResource) (ok bool, nrs []*api.GenericResource) { + switch na := res.Resource.(type) { + case *api.GenericResource_DiscreteResourceSpec: + nrs := GetResource(na.DiscreteResourceSpec.Kind, nodeRes) + + // Type change or removed: reset + if len(nrs) != 1 { + return false, nrs + } + + // Type change: reset + nr := nrs[0].GetDiscreteResourceSpec() + if nr == nil { + return false, nrs + } + + // Amount change: reset + if na.DiscreteResourceSpec.Value > nr.Value { + return false, nrs + } + case *api.GenericResource_NamedResourceSpec: + nrs := GetResource(na.NamedResourceSpec.Kind, nodeRes) + + // Type change + if len(nrs) == 0 { + return false, nrs + } + + for _, nr := range nrs { + // Type change: reset + if nr.GetDiscreteResourceSpec() != nil { + return false, nrs + } + + if na.NamedResourceSpec.Value == nr.GetNamedResourceSpec().Value { + return true, nil + } + } + + // Removed + return false, nil + } + + return true, nil +} diff --git a/vendor/github.com/docker/swarmkit/api/genericresource/string.go b/vendor/github.com/docker/swarmkit/api/genericresource/string.go new file mode 100644 index 0000000000..5e388bebb0 --- /dev/null +++ b/vendor/github.com/docker/swarmkit/api/genericresource/string.go @@ -0,0 +1,54 @@ +package genericresource + +import ( + "strconv" + "strings" + + "github.com/docker/swarmkit/api" +) + +func discreteToString(d *api.GenericResource_DiscreteResourceSpec) string { + return strconv.FormatInt(d.DiscreteResourceSpec.Value, 10) +} + +// Kind returns the kind key as a string +func Kind(res *api.GenericResource) string { + switch r := res.Resource.(type) { + case *api.GenericResource_DiscreteResourceSpec: + return r.DiscreteResourceSpec.Kind + case *api.GenericResource_NamedResourceSpec: + return r.NamedResourceSpec.Kind + } + + return "" +} + +// Value returns the value key as a string +func Value(res *api.GenericResource) string { + switch res := res.Resource.(type) { + case *api.GenericResource_DiscreteResourceSpec: + return discreteToString(res) + case *api.GenericResource_NamedResourceSpec: + return res.NamedResourceSpec.Value + } + + return "" +} + +// EnvFormat returns the environment string version of the resource +func EnvFormat(res []*api.GenericResource, prefix string) []string { + envs := make(map[string][]string) + for _, v := range res { + key := Kind(v) + val := Value(v) + envs[key] = append(envs[key], val) + } + + env := make([]string, 0, len(res)) + for k, v := range envs { + k = strings.ToUpper(prefix + "_" + k) + env = append(env, k+"="+strings.Join(v, ",")) + } + + return env +} diff --git a/vendor/github.com/docker/swarmkit/api/genericresource/validate.go b/vendor/github.com/docker/swarmkit/api/genericresource/validate.go new file mode 100644 index 0000000000..eee3706c74 --- /dev/null +++ b/vendor/github.com/docker/swarmkit/api/genericresource/validate.go @@ -0,0 +1,84 @@ +package genericresource + +import ( + "fmt" + "github.com/docker/swarmkit/api" +) + +// ValidateTask validates that the task only uses integers +// for generic resources +func ValidateTask(resources *api.Resources) error { + for _, v := range resources.Generic { + if v.GetDiscreteResourceSpec() != nil { + continue + } + + return fmt.Errorf("invalid argument for resource %s", Kind(v)) + } + + return nil +} + +// HasEnough returns true if node can satisfy the task's GenericResource request +func HasEnough(nodeRes []*api.GenericResource, taskRes *api.GenericResource) (bool, error) { + t := taskRes.GetDiscreteResourceSpec() + if t == nil { + return false, fmt.Errorf("task should only hold Discrete type") + } + + if nodeRes == nil { + return false, nil + } + + nrs := GetResource(t.Kind, nodeRes) + if len(nrs) == 0 { + return false, nil + } + + switch nr := nrs[0].Resource.(type) { + case *api.GenericResource_DiscreteResourceSpec: + if t.Value > nr.DiscreteResourceSpec.Value { + return false, nil + } + case *api.GenericResource_NamedResourceSpec: + if t.Value > int64(len(nrs)) { + return false, nil + } + } + + return true, nil +} + +// HasResource checks if there is enough "res" in the "resources" argument +func HasResource(res *api.GenericResource, resources []*api.GenericResource) bool { + for _, r := range resources { + if Kind(res) != Kind(r) { + continue + } + + switch rtype := r.Resource.(type) { + case *api.GenericResource_DiscreteResourceSpec: + if res.GetDiscreteResourceSpec() == nil { + return false + } + + if res.GetDiscreteResourceSpec().Value < rtype.DiscreteResourceSpec.Value { + return false + } + + return true + case *api.GenericResource_NamedResourceSpec: + if res.GetNamedResourceSpec() == nil { + return false + } + + if res.GetNamedResourceSpec().Value != rtype.NamedResourceSpec.Value { + continue + } + + return true + } + } + + return false +} diff --git a/vendor/github.com/docker/swarmkit/api/specs.pb.go b/vendor/github.com/docker/swarmkit/api/specs.pb.go index b150d6f81e..dfd18a6d78 100644 --- a/vendor/github.com/docker/swarmkit/api/specs.pb.go +++ b/vendor/github.com/docker/swarmkit/api/specs.pb.go @@ -620,6 +620,9 @@ type ContainerSpec struct { // Isolation defines the isolation level for windows containers (default, process, hyperv). // Runtimes that don't support it ignore that field Isolation ContainerSpec_Isolation `protobuf:"varint,24,opt,name=isolation,proto3,enum=docker.swarmkit.v1.ContainerSpec_Isolation" json:"isolation,omitempty"` + // PidsLimit prevents from OS resource damage by applications inside the container + // using fork bomb attack. + PidsLimit int64 `protobuf:"varint,25,opt,name=pidsLimit,proto3" json:"pidsLimit,omitempty"` } func (m *ContainerSpec) Reset() { *m = ContainerSpec{} } @@ -2055,6 +2058,13 @@ func (m *ContainerSpec) MarshalTo(dAtA []byte) (int, error) { i++ i = encodeVarintSpecs(dAtA, i, uint64(m.Isolation)) } + if m.PidsLimit != 0 { + dAtA[i] = 0xc8 + i++ + dAtA[i] = 0x1 + i++ + i = encodeVarintSpecs(dAtA, i, uint64(m.PidsLimit)) + } return i, nil } @@ -2787,6 +2797,9 @@ func (m *ContainerSpec) Size() (n int) { if m.Isolation != 0 { n += 2 + sovSpecs(uint64(m.Isolation)) } + if m.PidsLimit != 0 { + n += 2 + sovSpecs(uint64(m.PidsLimit)) + } return n } @@ -3134,6 +3147,7 @@ func (this *ContainerSpec) String() string { `Privileges:` + strings.Replace(fmt.Sprintf("%v", this.Privileges), "Privileges", "Privileges", 1) + `,`, `Init:` + strings.Replace(fmt.Sprintf("%v", this.Init), "BoolValue", "google_protobuf4.BoolValue", 1) + `,`, `Isolation:` + fmt.Sprintf("%v", this.Isolation) + `,`, + `PidsLimit:` + fmt.Sprintf("%v", this.PidsLimit) + `,`, `}`, }, "") return s @@ -5261,6 +5275,25 @@ func (m *ContainerSpec) Unmarshal(dAtA []byte) error { break } } + case 25: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field PidsLimit", wireType) + } + m.PidsLimit = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowSpecs + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.PidsLimit |= (int64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipSpecs(dAtA[iNdEx:]) @@ -6572,138 +6605,139 @@ var ( func init() { proto.RegisterFile("github.com/docker/swarmkit/api/specs.proto", fileDescriptorSpecs) } var fileDescriptorSpecs = []byte{ - // 2114 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xcc, 0x58, 0x4f, 0x6f, 0xdb, 0xc8, - 0x15, 0xb7, 0x6c, 0x59, 0x96, 0x1e, 0xe5, 0x44, 0x9e, 0x4d, 0xb2, 0xb4, 0xb2, 0xb1, 0x15, 0x6d, - 0x36, 0xf5, 0xee, 0xa2, 0x32, 0xea, 0x2e, 0xb6, 0xd9, 0x4d, 0xb7, 0xad, 0x64, 0x69, 0x1d, 0x35, - 0x89, 0x2d, 0x8c, 0x1c, 0xb7, 0x01, 0x0a, 0x08, 0x63, 0x72, 0x2c, 0x11, 0xa6, 0x38, 0xec, 0x70, - 0xe8, 0x40, 0xb7, 0x1e, 0x17, 0xee, 0x67, 0x30, 0x7a, 0x28, 0x7a, 0x6f, 0xbf, 0x42, 0x4f, 0x39, - 0xf6, 0xd8, 0x5e, 0x8c, 0xae, 0xbf, 0x42, 0x6f, 0xbd, 0xb4, 0x98, 0xe1, 0x90, 0xa2, 0x1c, 0x3a, - 0x0e, 0xd0, 0x1c, 0x7a, 0x9b, 0x79, 0xfc, 0xfd, 0xde, 0xfc, 0xfb, 0xbd, 0x37, 0x6f, 0x08, 0x9f, - 0x0d, 0x1d, 0x31, 0x0a, 0x0f, 0x1b, 0x16, 0x1b, 0x6f, 0xda, 0xcc, 0x3a, 0xa6, 0x7c, 0x33, 0x78, - 0x45, 0xf8, 0xf8, 0xd8, 0x11, 0x9b, 0xc4, 0x77, 0x36, 0x03, 0x9f, 0x5a, 0x41, 0xc3, 0xe7, 0x4c, - 0x30, 0x84, 0x22, 0x40, 0x23, 0x06, 0x34, 0x4e, 0x7e, 0x54, 0xbd, 0x8e, 0x2f, 0x26, 0x3e, 0xd5, - 0xfc, 0xea, 0xad, 0x21, 0x1b, 0x32, 0xd5, 0xdc, 0x94, 0x2d, 0x6d, 0x5d, 0x1b, 0x32, 0x36, 0x74, - 0xe9, 0xa6, 0xea, 0x1d, 0x86, 0x47, 0x9b, 0x76, 0xc8, 0x89, 0x70, 0x98, 0xa7, 0xbf, 0xaf, 0x5e, - 0xfe, 0x4e, 0xbc, 0xc9, 0x55, 0xd4, 0x57, 0x9c, 0xf8, 0x3e, 0xe5, 0x7a, 0xc0, 0xfa, 0x59, 0x1e, - 0x8a, 0xbb, 0xcc, 0xa6, 0x7d, 0x9f, 0x5a, 0x68, 0x07, 0x0c, 0xe2, 0x79, 0x4c, 0x28, 0xdf, 0x81, - 0x99, 0xab, 0xe5, 0x36, 0x8c, 0xad, 0xf5, 0xc6, 0x9b, 0x6b, 0x6a, 0x34, 0xa7, 0xb0, 0x56, 0xfe, - 0xf5, 0xf9, 0xfa, 0x1c, 0x4e, 0x33, 0xd1, 0xcf, 0xa1, 0x6c, 0xd3, 0xc0, 0xe1, 0xd4, 0x1e, 0x70, - 0xe6, 0x52, 0x73, 0xbe, 0x96, 0xdb, 0xb8, 0xb1, 0xf5, 0x51, 0x96, 0x27, 0x39, 0x38, 0x66, 0x2e, - 0xc5, 0x86, 0x66, 0xc8, 0x0e, 0xda, 0x01, 0x18, 0xd3, 0xf1, 0x21, 0xe5, 0xc1, 0xc8, 0xf1, 0xcd, - 0x05, 0x45, 0xff, 0xc1, 0x55, 0x74, 0x39, 0xf7, 0xc6, 0xf3, 0x04, 0x8e, 0x53, 0x54, 0xf4, 0x1c, - 0xca, 0xe4, 0x84, 0x38, 0x2e, 0x39, 0x74, 0x5c, 0x47, 0x4c, 0xcc, 0xbc, 0x72, 0xf5, 0xe9, 0x5b, - 0x5d, 0x35, 0x53, 0x04, 0x3c, 0x43, 0xaf, 0xdb, 0x00, 0xd3, 0x81, 0xd0, 0x43, 0x58, 0xea, 0x75, - 0x76, 0xdb, 0xdd, 0xdd, 0x9d, 0xca, 0x5c, 0x75, 0xf5, 0xf4, 0xac, 0x76, 0x5b, 0xfa, 0x98, 0x02, - 0x7a, 0xd4, 0xb3, 0x1d, 0x6f, 0x88, 0x36, 0xa0, 0xd8, 0xdc, 0xde, 0xee, 0xf4, 0xf6, 0x3b, 0xed, - 0x4a, 0xae, 0x5a, 0x3d, 0x3d, 0xab, 0xdd, 0x99, 0x05, 0x36, 0x2d, 0x8b, 0xfa, 0x82, 0xda, 0xd5, - 0xfc, 0x77, 0x7f, 0x5c, 0x9b, 0xab, 0x7f, 0x97, 0x83, 0x72, 0x7a, 0x12, 0xe8, 0x21, 0x14, 0x9a, - 0xdb, 0xfb, 0xdd, 0x83, 0x4e, 0x65, 0x6e, 0x4a, 0x4f, 0x23, 0x9a, 0x96, 0x70, 0x4e, 0x28, 0x7a, - 0x00, 0x8b, 0xbd, 0xe6, 0x8b, 0x7e, 0xa7, 0x92, 0x9b, 0x4e, 0x27, 0x0d, 0xeb, 0x91, 0x30, 0x50, - 0xa8, 0x36, 0x6e, 0x76, 0x77, 0x2b, 0xf3, 0xd9, 0xa8, 0x36, 0x27, 0x8e, 0xa7, 0xa7, 0xf2, 0x87, - 0x3c, 0x18, 0x7d, 0xca, 0x4f, 0x1c, 0xeb, 0x3d, 0x4b, 0xe4, 0x4b, 0xc8, 0x0b, 0x12, 0x1c, 0x2b, - 0x69, 0x18, 0xd9, 0xd2, 0xd8, 0x27, 0xc1, 0xb1, 0x1c, 0x54, 0xd3, 0x15, 0x5e, 0x2a, 0x83, 0x53, - 0xdf, 0x75, 0x2c, 0x22, 0xa8, 0xad, 0x94, 0x61, 0x6c, 0x7d, 0x92, 0xc5, 0xc6, 0x09, 0x4a, 0xcf, - 0xff, 0xc9, 0x1c, 0x4e, 0x51, 0xd1, 0x63, 0x28, 0x0c, 0x5d, 0x76, 0x48, 0x5c, 0xa5, 0x09, 0x63, - 0xeb, 0x7e, 0x96, 0x93, 0x1d, 0x85, 0x98, 0x3a, 0xd0, 0x14, 0xf4, 0x08, 0x0a, 0xa1, 0x6f, 0x13, - 0x41, 0xcd, 0x82, 0x22, 0xd7, 0xb2, 0xc8, 0x2f, 0x14, 0x62, 0x9b, 0x79, 0x47, 0xce, 0x10, 0x6b, - 0x3c, 0x7a, 0x0a, 0x45, 0x8f, 0x8a, 0x57, 0x8c, 0x1f, 0x07, 0xe6, 0x52, 0x6d, 0x61, 0xc3, 0xd8, - 0xfa, 0x3c, 0x53, 0x8c, 0x11, 0xa6, 0x29, 0x04, 0xb1, 0x46, 0x63, 0xea, 0x89, 0xc8, 0x4d, 0x6b, - 0xde, 0xcc, 0xe1, 0xc4, 0x01, 0xfa, 0x29, 0x14, 0xa9, 0x67, 0xfb, 0xcc, 0xf1, 0x84, 0x59, 0xbc, - 0x7a, 0x22, 0x1d, 0x8d, 0x91, 0x9b, 0x89, 0x13, 0x86, 0x64, 0x73, 0xe6, 0xba, 0x87, 0xc4, 0x3a, - 0x36, 0x4b, 0xef, 0xb8, 0x8c, 0x84, 0xd1, 0x2a, 0x40, 0x7e, 0xcc, 0x6c, 0x5a, 0xdf, 0x84, 0x95, - 0x37, 0xb6, 0x1a, 0x55, 0xa1, 0xa8, 0xb7, 0x3a, 0xd2, 0x48, 0x1e, 0x27, 0xfd, 0xfa, 0x4d, 0x58, - 0x9e, 0xd9, 0xd6, 0xfa, 0x9f, 0x17, 0xa1, 0x18, 0x9f, 0x35, 0x6a, 0x42, 0xc9, 0x62, 0x9e, 0x20, - 0x8e, 0x47, 0xb9, 0x96, 0x57, 0xe6, 0xc9, 0x6c, 0xc7, 0x20, 0xc9, 0x7a, 0x32, 0x87, 0xa7, 0x2c, - 0xf4, 0x2d, 0x94, 0x38, 0x0d, 0x58, 0xc8, 0x2d, 0x1a, 0x68, 0x7d, 0x6d, 0x64, 0x2b, 0x24, 0x02, - 0x61, 0xfa, 0xdb, 0xd0, 0xe1, 0x54, 0xee, 0x72, 0x80, 0xa7, 0x54, 0xf4, 0x18, 0x96, 0x38, 0x0d, - 0x04, 0xe1, 0xe2, 0x6d, 0x12, 0xc1, 0x11, 0xa4, 0xc7, 0x5c, 0xc7, 0x9a, 0xe0, 0x98, 0x81, 0x1e, - 0x43, 0xc9, 0x77, 0x89, 0xa5, 0xbc, 0x9a, 0x8b, 0x8a, 0x7e, 0x2f, 0x8b, 0xde, 0x8b, 0x41, 0x78, - 0x8a, 0x47, 0x5f, 0x01, 0xb8, 0x6c, 0x38, 0xb0, 0xb9, 0x73, 0x42, 0xb9, 0x96, 0x58, 0x35, 0x8b, - 0xdd, 0x56, 0x08, 0x5c, 0x72, 0xd9, 0x30, 0x6a, 0xa2, 0x9d, 0xff, 0x49, 0x5f, 0x29, 0x6d, 0x3d, - 0x05, 0x20, 0xc9, 0x57, 0xad, 0xae, 0x4f, 0xdf, 0xc9, 0x95, 0x3e, 0x91, 0x14, 0x1d, 0xdd, 0x87, - 0xf2, 0x11, 0xe3, 0x16, 0x1d, 0xe8, 0xa8, 0x29, 0x29, 0x4d, 0x18, 0xca, 0x16, 0xe9, 0x0b, 0xb5, - 0x60, 0x69, 0x48, 0x3d, 0xca, 0x1d, 0xcb, 0x04, 0x35, 0xd8, 0xc3, 0xcc, 0x80, 0x8c, 0x20, 0x38, - 0xf4, 0x84, 0x33, 0xa6, 0x7a, 0xa4, 0x98, 0x88, 0x7e, 0x03, 0x1f, 0xc4, 0xc7, 0x37, 0xe0, 0xf4, - 0x88, 0x72, 0xea, 0x49, 0x0d, 0x18, 0x6a, 0x1f, 0x3e, 0x79, 0xbb, 0x06, 0x34, 0x5a, 0x27, 0x1b, - 0xc4, 0x2f, 0x7f, 0x08, 0x5a, 0x25, 0x58, 0xe2, 0xd1, 0xb8, 0xf5, 0xdf, 0xe7, 0xa4, 0xea, 0x2f, - 0x21, 0xd0, 0x26, 0x18, 0xc9, 0xf0, 0x8e, 0xad, 0xd4, 0x5b, 0x6a, 0xdd, 0xb8, 0x38, 0x5f, 0x87, - 0x18, 0xdb, 0x6d, 0xcb, 0x1c, 0xa4, 0xdb, 0x36, 0xea, 0xc0, 0x72, 0x42, 0x90, 0x65, 0x80, 0xbe, - 0x28, 0x6b, 0x6f, 0x9b, 0xe9, 0xfe, 0xc4, 0xa7, 0xb8, 0xcc, 0x53, 0xbd, 0xfa, 0xaf, 0x01, 0xbd, - 0xb9, 0x2f, 0x08, 0x41, 0xfe, 0xd8, 0xf1, 0xf4, 0x34, 0xb0, 0x6a, 0xa3, 0x06, 0x2c, 0xf9, 0x64, - 0xe2, 0x32, 0x62, 0xeb, 0xc0, 0xb8, 0xd5, 0x88, 0x0a, 0x84, 0x46, 0x5c, 0x20, 0x34, 0x9a, 0xde, - 0x04, 0xc7, 0xa0, 0xfa, 0x53, 0xb8, 0x9d, 0x79, 0xbc, 0x68, 0x0b, 0xca, 0x49, 0xc0, 0x4d, 0xd7, - 0x7a, 0xf3, 0xe2, 0x7c, 0xdd, 0x48, 0x22, 0xb3, 0xdb, 0xc6, 0x46, 0x02, 0xea, 0xda, 0xf5, 0xbf, - 0x1a, 0xb0, 0x3c, 0x13, 0xb6, 0xe8, 0x16, 0x2c, 0x3a, 0x63, 0x32, 0xa4, 0x7a, 0x8e, 0x51, 0x07, - 0x75, 0xa0, 0xe0, 0x92, 0x43, 0xea, 0xca, 0xe0, 0x95, 0x07, 0xf7, 0xc3, 0x6b, 0xe3, 0xbf, 0xf1, - 0x4c, 0xe1, 0x3b, 0x9e, 0xe0, 0x13, 0xac, 0xc9, 0xc8, 0x84, 0x25, 0x8b, 0x8d, 0xc7, 0xc4, 0x93, - 0xd7, 0xc4, 0xc2, 0x46, 0x09, 0xc7, 0x5d, 0xb9, 0x33, 0x84, 0x0f, 0x03, 0x33, 0xaf, 0xcc, 0xaa, - 0x8d, 0x2a, 0xb0, 0x40, 0xbd, 0x13, 0x73, 0x51, 0x99, 0x64, 0x53, 0x5a, 0x6c, 0x27, 0x8a, 0xbe, - 0x12, 0x96, 0x4d, 0xc9, 0x0b, 0x03, 0xca, 0xcd, 0xa5, 0x68, 0x47, 0x65, 0x1b, 0xfd, 0x04, 0x0a, - 0x63, 0x16, 0x7a, 0x22, 0x30, 0x8b, 0x6a, 0xb2, 0xab, 0x59, 0x93, 0x7d, 0x2e, 0x11, 0x5a, 0x59, - 0x1a, 0x8e, 0x3a, 0xb0, 0x12, 0x08, 0xe6, 0x0f, 0x86, 0x9c, 0x58, 0x74, 0xe0, 0x53, 0xee, 0x30, - 0x5b, 0xa7, 0xe1, 0xd5, 0x37, 0x0e, 0xa5, 0xad, 0x0b, 0x3e, 0x7c, 0x53, 0x72, 0x76, 0x24, 0xa5, - 0xa7, 0x18, 0xa8, 0x07, 0x65, 0x3f, 0x74, 0xdd, 0x01, 0xf3, 0xa3, 0x1b, 0x39, 0x8a, 0x9d, 0x77, - 0xd8, 0xb2, 0x5e, 0xe8, 0xba, 0x7b, 0x11, 0x09, 0x1b, 0xfe, 0xb4, 0x83, 0xee, 0x40, 0x61, 0xc8, - 0x59, 0xe8, 0x47, 0x71, 0x53, 0xc2, 0xba, 0x87, 0xbe, 0x81, 0xa5, 0x80, 0x5a, 0x9c, 0x8a, 0xc0, - 0x2c, 0xab, 0xa5, 0x7e, 0x9c, 0x35, 0x48, 0x5f, 0x41, 0x92, 0x98, 0xc0, 0x31, 0x07, 0xad, 0xc2, - 0x82, 0x10, 0x13, 0x73, 0xb9, 0x96, 0xdb, 0x28, 0xb6, 0x96, 0x2e, 0xce, 0xd7, 0x17, 0xf6, 0xf7, - 0x5f, 0x62, 0x69, 0x93, 0xb7, 0xc5, 0x88, 0x05, 0xc2, 0x23, 0x63, 0x6a, 0xde, 0x50, 0x7b, 0x9b, - 0xf4, 0xd1, 0x4b, 0x00, 0xdb, 0x0b, 0x06, 0x96, 0x4a, 0x4f, 0xe6, 0x4d, 0xb5, 0xba, 0xcf, 0xaf, - 0x5f, 0x5d, 0x7b, 0xb7, 0xaf, 0x6f, 0xcc, 0xe5, 0x8b, 0xf3, 0xf5, 0x52, 0xd2, 0xc5, 0x25, 0xdb, - 0x0b, 0xa2, 0x26, 0x6a, 0x81, 0x31, 0xa2, 0xc4, 0x15, 0x23, 0x6b, 0x44, 0xad, 0x63, 0xb3, 0x72, - 0xf5, 0x15, 0xf8, 0x44, 0xc1, 0xb4, 0x87, 0x34, 0x49, 0x2a, 0x58, 0x4e, 0x35, 0x30, 0x57, 0xd4, - 0x5e, 0x45, 0x1d, 0x74, 0x0f, 0x80, 0xf9, 0xd4, 0x1b, 0x04, 0xc2, 0x76, 0x3c, 0x13, 0xc9, 0x25, - 0xe3, 0x92, 0xb4, 0xf4, 0xa5, 0x01, 0xdd, 0x95, 0x17, 0x14, 0xb1, 0x07, 0xcc, 0x73, 0x27, 0xe6, - 0x07, 0xea, 0x6b, 0x51, 0x1a, 0xf6, 0x3c, 0x77, 0x82, 0xd6, 0xc1, 0x50, 0xba, 0x08, 0x9c, 0xa1, - 0x47, 0x5c, 0xf3, 0x96, 0xda, 0x0f, 0x90, 0xa6, 0xbe, 0xb2, 0xc8, 0x73, 0x88, 0x76, 0x23, 0x30, - 0x6f, 0x5f, 0x7d, 0x0e, 0x7a, 0xb2, 0xd3, 0x73, 0xd0, 0x1c, 0xf4, 0x33, 0x00, 0x9f, 0x3b, 0x27, - 0x8e, 0x4b, 0x87, 0x34, 0x30, 0xef, 0xa8, 0x45, 0xaf, 0x65, 0xde, 0x4c, 0x09, 0x0a, 0xa7, 0x18, - 0xa8, 0x01, 0x79, 0xc7, 0x73, 0x84, 0xf9, 0xa1, 0xbe, 0x95, 0x2e, 0x4b, 0xb5, 0xc5, 0x98, 0x7b, - 0x40, 0xdc, 0x90, 0x62, 0x85, 0x43, 0x5d, 0x28, 0x39, 0x01, 0x73, 0x95, 0x7c, 0x4d, 0x53, 0xe5, - 0xb7, 0x77, 0x38, 0xbf, 0x6e, 0x4c, 0xc1, 0x53, 0x76, 0xf5, 0x2b, 0x30, 0x52, 0x81, 0x2e, 0x03, - 0xf4, 0x98, 0x4e, 0x74, 0xee, 0x90, 0x4d, 0x79, 0x1a, 0x27, 0x72, 0x68, 0x95, 0xdc, 0x4a, 0x38, - 0xea, 0x7c, 0x3d, 0xff, 0x28, 0x57, 0xdd, 0x02, 0x23, 0x25, 0x78, 0xf4, 0xb1, 0x4c, 0xbc, 0x43, - 0x27, 0x10, 0x7c, 0x32, 0x20, 0xa1, 0x18, 0x99, 0xbf, 0x50, 0x84, 0x72, 0x6c, 0x6c, 0x86, 0x62, - 0x54, 0x1d, 0xc0, 0x54, 0x37, 0xa8, 0x06, 0x86, 0xd4, 0x63, 0x40, 0xf9, 0x09, 0xe5, 0xb2, 0xa8, - 0x91, 0xc7, 0x9d, 0x36, 0xc9, 0xb8, 0x09, 0x28, 0xe1, 0xd6, 0x48, 0xa5, 0xad, 0x12, 0xd6, 0x3d, - 0x99, 0x87, 0xe2, 0xe0, 0xd4, 0x79, 0x48, 0x77, 0xeb, 0x7f, 0xc9, 0x41, 0x29, 0x59, 0x28, 0xfa, - 0x02, 0x56, 0xba, 0xfd, 0xbd, 0x67, 0xcd, 0xfd, 0xee, 0xde, 0xee, 0xa0, 0xdd, 0xf9, 0xb6, 0xf9, - 0xe2, 0xd9, 0x7e, 0x65, 0xae, 0x7a, 0xef, 0xf4, 0xac, 0xb6, 0x3a, 0xcd, 0xa9, 0x31, 0xbc, 0x4d, - 0x8f, 0x48, 0xe8, 0x8a, 0x59, 0x56, 0x0f, 0xef, 0x6d, 0x77, 0xfa, 0xfd, 0x4a, 0xee, 0x2a, 0x56, - 0x8f, 0x33, 0x8b, 0x06, 0x01, 0xda, 0x82, 0xca, 0x94, 0xf5, 0xe4, 0x65, 0xaf, 0x83, 0x0f, 0x2a, - 0xf3, 0xd5, 0x8f, 0x4e, 0xcf, 0x6a, 0xe6, 0x9b, 0xa4, 0x27, 0x13, 0x9f, 0xf2, 0x03, 0xfd, 0x20, - 0xf8, 0x57, 0x0e, 0xca, 0xe9, 0x7a, 0x12, 0x6d, 0x47, 0x75, 0xa0, 0x3a, 0x86, 0x1b, 0x5b, 0x9b, - 0xd7, 0xd5, 0x9f, 0xea, 0x1e, 0x73, 0x43, 0xe9, 0xf7, 0xb9, 0x7c, 0xfa, 0x29, 0x32, 0xfa, 0x02, - 0x16, 0x7d, 0xc6, 0x45, 0x9c, 0xf1, 0xb3, 0xf5, 0xc8, 0x78, 0x5c, 0xa5, 0x44, 0xe0, 0xfa, 0x08, - 0x6e, 0xcc, 0x7a, 0x43, 0x0f, 0x60, 0xe1, 0xa0, 0xdb, 0xab, 0xcc, 0x55, 0xef, 0x9e, 0x9e, 0xd5, - 0x3e, 0x9c, 0xfd, 0x78, 0xe0, 0x70, 0x11, 0x12, 0xb7, 0xdb, 0x43, 0x9f, 0xc1, 0x62, 0x7b, 0xb7, - 0x8f, 0x71, 0x25, 0x57, 0x5d, 0x3f, 0x3d, 0xab, 0xdd, 0x9d, 0xc5, 0xc9, 0x4f, 0x2c, 0xf4, 0x6c, - 0xcc, 0x0e, 0x93, 0x67, 0xd0, 0xbf, 0xe7, 0xc1, 0xd0, 0x17, 0xe1, 0xfb, 0x7e, 0x29, 0x2f, 0x47, - 0x55, 0x5e, 0x9c, 0xe1, 0xe6, 0xaf, 0x2d, 0xf6, 0xca, 0x11, 0x41, 0xeb, 0xf2, 0x3e, 0x94, 0x1d, - 0xff, 0xe4, 0xcb, 0x01, 0xf5, 0xc8, 0xa1, 0xab, 0x5f, 0x44, 0x45, 0x6c, 0x48, 0x5b, 0x27, 0x32, - 0xc9, 0xf4, 0xea, 0x78, 0x82, 0x72, 0x4f, 0xbf, 0x75, 0x8a, 0x38, 0xe9, 0xa3, 0x6f, 0x20, 0xef, - 0xf8, 0x64, 0xac, 0x2b, 0xd4, 0xcc, 0x15, 0x74, 0x7b, 0xcd, 0xe7, 0x3a, 0x6e, 0x5a, 0xc5, 0x8b, - 0xf3, 0xf5, 0xbc, 0x34, 0x60, 0x45, 0x43, 0x6b, 0x71, 0x91, 0x28, 0x47, 0x52, 0x57, 0x65, 0x11, - 0xa7, 0x2c, 0x52, 0xfb, 0x8e, 0x37, 0xe4, 0x34, 0x08, 0xd4, 0xa5, 0x59, 0xc4, 0x71, 0x17, 0x55, - 0x61, 0x49, 0x97, 0x9a, 0xaa, 0xb6, 0x2c, 0xc9, 0x32, 0x4e, 0x1b, 0x5a, 0xcb, 0x60, 0x44, 0xbb, - 0x31, 0x38, 0xe2, 0x6c, 0x5c, 0xff, 0x4f, 0x1e, 0x8c, 0x6d, 0x37, 0x0c, 0x84, 0xae, 0x1a, 0xde, - 0xdb, 0xe6, 0xbf, 0x84, 0x15, 0xa2, 0x5e, 0xde, 0xc4, 0x93, 0x57, 0xb0, 0xaa, 0xe0, 0xf5, 0x01, - 0x3c, 0xc8, 0x74, 0x97, 0x80, 0xa3, 0x6a, 0xbf, 0x55, 0x90, 0x3e, 0xcd, 0x1c, 0xae, 0x90, 0x4b, - 0x5f, 0x50, 0x1f, 0x96, 0x19, 0xb7, 0x46, 0x34, 0x10, 0xd1, 0xc5, 0xad, 0x5f, 0xaa, 0x99, 0xff, - 0x30, 0xf6, 0xd2, 0x40, 0x7d, 0x6b, 0x45, 0xb3, 0x9d, 0xf5, 0x81, 0x1e, 0x41, 0x9e, 0x93, 0xa3, - 0xf8, 0x35, 0x92, 0x19, 0x24, 0x98, 0x1c, 0x89, 0x19, 0x17, 0x8a, 0x81, 0x7e, 0x09, 0x60, 0x3b, - 0x81, 0x4f, 0x84, 0x35, 0xa2, 0x5c, 0x1f, 0x76, 0xe6, 0x12, 0xdb, 0x09, 0x6a, 0xc6, 0x4b, 0x8a, - 0x8d, 0x9e, 0x42, 0xc9, 0x22, 0xb1, 0x5c, 0x0b, 0x57, 0x3f, 0xdf, 0xb7, 0x9b, 0xda, 0x45, 0x45, - 0xba, 0xb8, 0x38, 0x5f, 0x2f, 0xc6, 0x16, 0x5c, 0xb4, 0x88, 0x96, 0xef, 0x53, 0x58, 0x96, 0xcf, - 0xfa, 0x81, 0x1d, 0xa5, 0xb3, 0x48, 0x26, 0x57, 0xdc, 0xc2, 0xf2, 0x8d, 0xa8, 0xd3, 0x5e, 0x7c, - 0x9c, 0x65, 0x91, 0xb2, 0xa1, 0x5f, 0xc1, 0x0a, 0xf5, 0x2c, 0x3e, 0x51, 0x62, 0x8d, 0x67, 0x58, - 0xbc, 0x7a, 0xb1, 0x9d, 0x04, 0x3c, 0xb3, 0xd8, 0x0a, 0xbd, 0x64, 0xaf, 0xff, 0x23, 0x07, 0x10, - 0x15, 0x36, 0xef, 0x57, 0x80, 0x08, 0xf2, 0x36, 0x11, 0x44, 0x69, 0xae, 0x8c, 0x55, 0x1b, 0x7d, - 0x0d, 0x20, 0xe8, 0xd8, 0x97, 0xa9, 0xd7, 0x1b, 0x6a, 0xd9, 0xbc, 0x2d, 0x1d, 0xa4, 0xd0, 0x68, - 0x0b, 0x0a, 0xfa, 0xcd, 0x98, 0xbf, 0x96, 0xa7, 0x91, 0xf5, 0x3f, 0xe5, 0x00, 0xa2, 0x65, 0xfe, - 0x5f, 0xaf, 0xad, 0x65, 0xbe, 0xfe, 0x7e, 0x6d, 0xee, 0xef, 0xdf, 0xaf, 0xcd, 0xfd, 0xee, 0x62, - 0x2d, 0xf7, 0xfa, 0x62, 0x2d, 0xf7, 0xb7, 0x8b, 0xb5, 0xdc, 0x3f, 0x2f, 0xd6, 0x72, 0x87, 0x05, - 0x55, 0x7b, 0xfc, 0xf8, 0xbf, 0x01, 0x00, 0x00, 0xff, 0xff, 0x1a, 0xbd, 0x13, 0xac, 0xa9, 0x15, - 0x00, 0x00, + // 2131 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xcc, 0x58, 0x4f, 0x6f, 0x1b, 0xc7, + 0x15, 0x17, 0x25, 0x8a, 0x22, 0xdf, 0x52, 0x36, 0x35, 0x71, 0x9c, 0x15, 0x6d, 0x4b, 0x34, 0xe3, + 0xb8, 0x4a, 0x82, 0x52, 0xa8, 0x1a, 0xa4, 0x4e, 0xdc, 0xb4, 0x25, 0x45, 0x46, 0x66, 0x6d, 0x4b, + 0xc4, 0x50, 0x56, 0x6b, 0xa0, 0x00, 0x31, 0xda, 0x1d, 0x91, 0x03, 0x2d, 0x77, 0xb6, 0xb3, 0x43, + 0x19, 0xbc, 0xf5, 0x18, 0xa8, 0x9f, 0x41, 0xe8, 0xa1, 0xe8, 0xbd, 0xfd, 0x16, 0x3e, 0xf6, 0xd8, + 0x5e, 0x84, 0x44, 0x5f, 0xa1, 0xb7, 0x5e, 0x5a, 0xcc, 0xec, 0xec, 0x92, 0x94, 0x57, 0x96, 0x81, + 0xfa, 0xd0, 0xdb, 0xcc, 0xdb, 0xdf, 0xef, 0xcd, 0xbf, 0xdf, 0xbc, 0xf7, 0x66, 0xe1, 0xb3, 0x3e, + 0x93, 0x83, 0xd1, 0x61, 0xcd, 0xe1, 0xc3, 0x4d, 0x97, 0x3b, 0xc7, 0x54, 0x6c, 0x86, 0xaf, 0x88, + 0x18, 0x1e, 0x33, 0xb9, 0x49, 0x02, 0xb6, 0x19, 0x06, 0xd4, 0x09, 0x6b, 0x81, 0xe0, 0x92, 0x23, + 0x14, 0x01, 0x6a, 0x31, 0xa0, 0x76, 0xf2, 0x93, 0xf2, 0x75, 0x7c, 0x39, 0x0e, 0xa8, 0xe1, 0x97, + 0x6f, 0xf5, 0x79, 0x9f, 0xeb, 0xe6, 0xa6, 0x6a, 0x19, 0xeb, 0x5a, 0x9f, 0xf3, 0xbe, 0x47, 0x37, + 0x75, 0xef, 0x70, 0x74, 0xb4, 0xe9, 0x8e, 0x04, 0x91, 0x8c, 0xfb, 0xe6, 0xfb, 0xea, 0xe5, 0xef, + 0xc4, 0x1f, 0x5f, 0x45, 0x7d, 0x25, 0x48, 0x10, 0x50, 0x61, 0x06, 0xac, 0x9e, 0x65, 0x21, 0xbf, + 0xcb, 0x5d, 0xda, 0x0d, 0xa8, 0x83, 0x76, 0xc0, 0x22, 0xbe, 0xcf, 0xa5, 0xf6, 0x1d, 0xda, 0x99, + 0x4a, 0x66, 0xc3, 0xda, 0x5a, 0xaf, 0xbd, 0xb9, 0xa6, 0x5a, 0x7d, 0x02, 0x6b, 0x64, 0x5f, 0x9f, + 0xaf, 0xcf, 0xe1, 0x69, 0x26, 0xfa, 0x25, 0x14, 0x5d, 0x1a, 0x32, 0x41, 0xdd, 0x9e, 0xe0, 0x1e, + 0xb5, 0xe7, 0x2b, 0x99, 0x8d, 0x1b, 0x5b, 0x77, 0xd3, 0x3c, 0xa9, 0xc1, 0x31, 0xf7, 0x28, 0xb6, + 0x0c, 0x43, 0x75, 0xd0, 0x0e, 0xc0, 0x90, 0x0e, 0x0f, 0xa9, 0x08, 0x07, 0x2c, 0xb0, 0x17, 0x34, + 0xfd, 0x47, 0x57, 0xd1, 0xd5, 0xdc, 0x6b, 0xcf, 0x13, 0x38, 0x9e, 0xa2, 0xa2, 0xe7, 0x50, 0x24, + 0x27, 0x84, 0x79, 0xe4, 0x90, 0x79, 0x4c, 0x8e, 0xed, 0xac, 0x76, 0xf5, 0xe9, 0x5b, 0x5d, 0xd5, + 0xa7, 0x08, 0x78, 0x86, 0x5e, 0x75, 0x01, 0x26, 0x03, 0xa1, 0x87, 0xb0, 0xd4, 0x69, 0xed, 0x36, + 0xdb, 0xbb, 0x3b, 0xa5, 0xb9, 0xf2, 0xea, 0xe9, 0x59, 0xe5, 0x43, 0xe5, 0x63, 0x02, 0xe8, 0x50, + 0xdf, 0x65, 0x7e, 0x1f, 0x6d, 0x40, 0xbe, 0xbe, 0xbd, 0xdd, 0xea, 0xec, 0xb7, 0x9a, 0xa5, 0x4c, + 0xb9, 0x7c, 0x7a, 0x56, 0xb9, 0x3d, 0x0b, 0xac, 0x3b, 0x0e, 0x0d, 0x24, 0x75, 0xcb, 0xd9, 0xef, + 0xfe, 0xbc, 0x36, 0x57, 0xfd, 0x2e, 0x03, 0xc5, 0xe9, 0x49, 0xa0, 0x87, 0x90, 0xab, 0x6f, 0xef, + 0xb7, 0x0f, 0x5a, 0xa5, 0xb9, 0x09, 0x7d, 0x1a, 0x51, 0x77, 0x24, 0x3b, 0xa1, 0xe8, 0x01, 0x2c, + 0x76, 0xea, 0x2f, 0xba, 0xad, 0x52, 0x66, 0x32, 0x9d, 0x69, 0x58, 0x87, 0x8c, 0x42, 0x8d, 0x6a, + 0xe2, 0x7a, 0x7b, 0xb7, 0x34, 0x9f, 0x8e, 0x6a, 0x0a, 0xc2, 0x7c, 0x33, 0x95, 0x3f, 0x65, 0xc1, + 0xea, 0x52, 0x71, 0xc2, 0x9c, 0xf7, 0x2c, 0x91, 0x2f, 0x21, 0x2b, 0x49, 0x78, 0xac, 0xa5, 0x61, + 0xa5, 0x4b, 0x63, 0x9f, 0x84, 0xc7, 0x6a, 0x50, 0x43, 0xd7, 0x78, 0xa5, 0x0c, 0x41, 0x03, 0x8f, + 0x39, 0x44, 0x52, 0x57, 0x2b, 0xc3, 0xda, 0xfa, 0x24, 0x8d, 0x8d, 0x13, 0x94, 0x99, 0xff, 0x93, + 0x39, 0x3c, 0x45, 0x45, 0x8f, 0x21, 0xd7, 0xf7, 0xf8, 0x21, 0xf1, 0xb4, 0x26, 0xac, 0xad, 0xfb, + 0x69, 0x4e, 0x76, 0x34, 0x62, 0xe2, 0xc0, 0x50, 0xd0, 0x23, 0xc8, 0x8d, 0x02, 0x97, 0x48, 0x6a, + 0xe7, 0x34, 0xb9, 0x92, 0x46, 0x7e, 0xa1, 0x11, 0xdb, 0xdc, 0x3f, 0x62, 0x7d, 0x6c, 0xf0, 0xe8, + 0x29, 0xe4, 0x7d, 0x2a, 0x5f, 0x71, 0x71, 0x1c, 0xda, 0x4b, 0x95, 0x85, 0x0d, 0x6b, 0xeb, 0xf3, + 0x54, 0x31, 0x46, 0x98, 0xba, 0x94, 0xc4, 0x19, 0x0c, 0xa9, 0x2f, 0x23, 0x37, 0x8d, 0x79, 0x3b, + 0x83, 0x13, 0x07, 0xe8, 0xe7, 0x90, 0xa7, 0xbe, 0x1b, 0x70, 0xe6, 0x4b, 0x3b, 0x7f, 0xf5, 0x44, + 0x5a, 0x06, 0xa3, 0x36, 0x13, 0x27, 0x0c, 0xc5, 0x16, 0xdc, 0xf3, 0x0e, 0x89, 0x73, 0x6c, 0x17, + 0xde, 0x71, 0x19, 0x09, 0xa3, 0x91, 0x83, 0xec, 0x90, 0xbb, 0xb4, 0xba, 0x09, 0x2b, 0x6f, 0x6c, + 0x35, 0x2a, 0x43, 0xde, 0x6c, 0x75, 0xa4, 0x91, 0x2c, 0x4e, 0xfa, 0xd5, 0x9b, 0xb0, 0x3c, 0xb3, + 0xad, 0xd5, 0xbf, 0x2e, 0x42, 0x3e, 0x3e, 0x6b, 0x54, 0x87, 0x82, 0xc3, 0x7d, 0x49, 0x98, 0x4f, + 0x85, 0x91, 0x57, 0xea, 0xc9, 0x6c, 0xc7, 0x20, 0xc5, 0x7a, 0x32, 0x87, 0x27, 0x2c, 0xf4, 0x2d, + 0x14, 0x04, 0x0d, 0xf9, 0x48, 0x38, 0x34, 0x34, 0xfa, 0xda, 0x48, 0x57, 0x48, 0x04, 0xc2, 0xf4, + 0xf7, 0x23, 0x26, 0xa8, 0xda, 0xe5, 0x10, 0x4f, 0xa8, 0xe8, 0x31, 0x2c, 0x09, 0x1a, 0x4a, 0x22, + 0xe4, 0xdb, 0x24, 0x82, 0x23, 0x48, 0x87, 0x7b, 0xcc, 0x19, 0xe3, 0x98, 0x81, 0x1e, 0x43, 0x21, + 0xf0, 0x88, 0xa3, 0xbd, 0xda, 0x8b, 0x9a, 0x7e, 0x2f, 0x8d, 0xde, 0x89, 0x41, 0x78, 0x82, 0x47, + 0x5f, 0x01, 0x78, 0xbc, 0xdf, 0x73, 0x05, 0x3b, 0xa1, 0xc2, 0x48, 0xac, 0x9c, 0xc6, 0x6e, 0x6a, + 0x04, 0x2e, 0x78, 0xbc, 0x1f, 0x35, 0xd1, 0xce, 0xff, 0xa4, 0xaf, 0x29, 0x6d, 0x3d, 0x05, 0x20, + 0xc9, 0x57, 0xa3, 0xae, 0x4f, 0xdf, 0xc9, 0x95, 0x39, 0x91, 0x29, 0x3a, 0xba, 0x0f, 0xc5, 0x23, + 0x2e, 0x1c, 0xda, 0x33, 0xb7, 0xa6, 0xa0, 0x35, 0x61, 0x69, 0x5b, 0xa4, 0x2f, 0xd4, 0x80, 0xa5, + 0x3e, 0xf5, 0xa9, 0x60, 0x8e, 0x0d, 0x7a, 0xb0, 0x87, 0xa9, 0x17, 0x32, 0x82, 0xe0, 0x91, 0x2f, + 0xd9, 0x90, 0x9a, 0x91, 0x62, 0x22, 0xfa, 0x1d, 0x7c, 0x10, 0x1f, 0x5f, 0x4f, 0xd0, 0x23, 0x2a, + 0xa8, 0xaf, 0x34, 0x60, 0xe9, 0x7d, 0xf8, 0xe4, 0xed, 0x1a, 0x30, 0x68, 0x13, 0x6c, 0x90, 0xb8, + 0xfc, 0x21, 0x6c, 0x14, 0x60, 0x49, 0x44, 0xe3, 0x56, 0xff, 0x98, 0x51, 0xaa, 0xbf, 0x84, 0x40, + 0x9b, 0x60, 0x25, 0xc3, 0x33, 0x57, 0xab, 0xb7, 0xd0, 0xb8, 0x71, 0x71, 0xbe, 0x0e, 0x31, 0xb6, + 0xdd, 0x54, 0x31, 0xc8, 0xb4, 0x5d, 0xd4, 0x82, 0xe5, 0x84, 0xa0, 0xca, 0x00, 0x93, 0x28, 0x2b, + 0x6f, 0x9b, 0xe9, 0xfe, 0x38, 0xa0, 0xb8, 0x28, 0xa6, 0x7a, 0xd5, 0xdf, 0x02, 0x7a, 0x73, 0x5f, + 0x10, 0x82, 0xec, 0x31, 0xf3, 0xcd, 0x34, 0xb0, 0x6e, 0xa3, 0x1a, 0x2c, 0x05, 0x64, 0xec, 0x71, + 0xe2, 0x9a, 0x8b, 0x71, 0xab, 0x16, 0x15, 0x08, 0xb5, 0xb8, 0x40, 0xa8, 0xd5, 0xfd, 0x31, 0x8e, + 0x41, 0xd5, 0xa7, 0xf0, 0x61, 0xea, 0xf1, 0xa2, 0x2d, 0x28, 0x26, 0x17, 0x6e, 0xb2, 0xd6, 0x9b, + 0x17, 0xe7, 0xeb, 0x56, 0x72, 0x33, 0xdb, 0x4d, 0x6c, 0x25, 0xa0, 0xb6, 0x5b, 0xfd, 0xde, 0x82, + 0xe5, 0x99, 0x6b, 0x8b, 0x6e, 0xc1, 0x22, 0x1b, 0x92, 0x3e, 0x35, 0x73, 0x8c, 0x3a, 0xa8, 0x05, + 0x39, 0x8f, 0x1c, 0x52, 0x4f, 0x5d, 0x5e, 0x75, 0x70, 0x3f, 0xbe, 0xf6, 0xfe, 0xd7, 0x9e, 0x69, + 0x7c, 0xcb, 0x97, 0x62, 0x8c, 0x0d, 0x19, 0xd9, 0xb0, 0xe4, 0xf0, 0xe1, 0x90, 0xf8, 0x2a, 0x4d, + 0x2c, 0x6c, 0x14, 0x70, 0xdc, 0x55, 0x3b, 0x43, 0x44, 0x3f, 0xb4, 0xb3, 0xda, 0xac, 0xdb, 0xa8, + 0x04, 0x0b, 0xd4, 0x3f, 0xb1, 0x17, 0xb5, 0x49, 0x35, 0x95, 0xc5, 0x65, 0xd1, 0xed, 0x2b, 0x60, + 0xd5, 0x54, 0xbc, 0x51, 0x48, 0x85, 0xbd, 0x14, 0xed, 0xa8, 0x6a, 0xa3, 0x9f, 0x41, 0x6e, 0xc8, + 0x47, 0xbe, 0x0c, 0xed, 0xbc, 0x9e, 0xec, 0x6a, 0xda, 0x64, 0x9f, 0x2b, 0x84, 0x51, 0x96, 0x81, + 0xa3, 0x16, 0xac, 0x84, 0x92, 0x07, 0xbd, 0xbe, 0x20, 0x0e, 0xed, 0x05, 0x54, 0x30, 0xee, 0x9a, + 0x30, 0xbc, 0xfa, 0xc6, 0xa1, 0x34, 0x4d, 0xc1, 0x87, 0x6f, 0x2a, 0xce, 0x8e, 0xa2, 0x74, 0x34, + 0x03, 0x75, 0xa0, 0x18, 0x8c, 0x3c, 0xaf, 0xc7, 0x83, 0x28, 0x23, 0x47, 0x77, 0xe7, 0x1d, 0xb6, + 0xac, 0x33, 0xf2, 0xbc, 0xbd, 0x88, 0x84, 0xad, 0x60, 0xd2, 0x41, 0xb7, 0x21, 0xd7, 0x17, 0x7c, + 0x14, 0x44, 0xf7, 0xa6, 0x80, 0x4d, 0x0f, 0x7d, 0x03, 0x4b, 0x21, 0x75, 0x04, 0x95, 0xa1, 0x5d, + 0xd4, 0x4b, 0xfd, 0x38, 0x6d, 0x90, 0xae, 0x86, 0x24, 0x77, 0x02, 0xc7, 0x1c, 0xb4, 0x0a, 0x0b, + 0x52, 0x8e, 0xed, 0xe5, 0x4a, 0x66, 0x23, 0xdf, 0x58, 0xba, 0x38, 0x5f, 0x5f, 0xd8, 0xdf, 0x7f, + 0x89, 0x95, 0x4d, 0x65, 0x8b, 0x01, 0x0f, 0xa5, 0x4f, 0x86, 0xd4, 0xbe, 0xa1, 0xf7, 0x36, 0xe9, + 0xa3, 0x97, 0x00, 0xae, 0x1f, 0xf6, 0x1c, 0x1d, 0x9e, 0xec, 0x9b, 0x7a, 0x75, 0x9f, 0x5f, 0xbf, + 0xba, 0xe6, 0x6e, 0xd7, 0x64, 0xcc, 0xe5, 0x8b, 0xf3, 0xf5, 0x42, 0xd2, 0xc5, 0x05, 0xd7, 0x0f, + 0xa3, 0x26, 0x6a, 0x80, 0x35, 0xa0, 0xc4, 0x93, 0x03, 0x67, 0x40, 0x9d, 0x63, 0xbb, 0x74, 0x75, + 0x0a, 0x7c, 0xa2, 0x61, 0xc6, 0xc3, 0x34, 0x49, 0x29, 0x58, 0x4d, 0x35, 0xb4, 0x57, 0xf4, 0x5e, + 0x45, 0x1d, 0x74, 0x0f, 0x80, 0x07, 0xd4, 0xef, 0x85, 0xd2, 0x65, 0xbe, 0x8d, 0xd4, 0x92, 0x71, + 0x41, 0x59, 0xba, 0xca, 0x80, 0xee, 0xa8, 0x04, 0x45, 0xdc, 0x1e, 0xf7, 0xbd, 0xb1, 0xfd, 0x81, + 0xfe, 0x9a, 0x57, 0x86, 0x3d, 0xdf, 0x1b, 0xa3, 0x75, 0xb0, 0xb4, 0x2e, 0x42, 0xd6, 0xf7, 0x89, + 0x67, 0xdf, 0xd2, 0xfb, 0x01, 0xca, 0xd4, 0xd5, 0x16, 0x75, 0x0e, 0xd1, 0x6e, 0x84, 0xf6, 0x87, + 0x57, 0x9f, 0x83, 0x99, 0xec, 0xe4, 0x1c, 0x0c, 0x07, 0xfd, 0x02, 0x20, 0x10, 0xec, 0x84, 0x79, + 0xb4, 0x4f, 0x43, 0xfb, 0xb6, 0x5e, 0xf4, 0x5a, 0x6a, 0x66, 0x4a, 0x50, 0x78, 0x8a, 0x81, 0x6a, + 0x90, 0x65, 0x3e, 0x93, 0xf6, 0x47, 0x26, 0x2b, 0x5d, 0x96, 0x6a, 0x83, 0x73, 0xef, 0x80, 0x78, + 0x23, 0x8a, 0x35, 0x0e, 0xb5, 0xa1, 0xc0, 0x42, 0xee, 0x69, 0xf9, 0xda, 0xb6, 0x8e, 0x6f, 0xef, + 0x70, 0x7e, 0xed, 0x98, 0x82, 0x27, 0x6c, 0x74, 0x17, 0x0a, 0x01, 0x73, 0xc3, 0x67, 0x6c, 0xc8, + 0xa4, 0xbd, 0x5a, 0xc9, 0x6c, 0x2c, 0xe0, 0x89, 0xa1, 0xfc, 0x15, 0x58, 0x53, 0x61, 0x40, 0x5d, + 0xdf, 0x63, 0x3a, 0x36, 0x91, 0x45, 0x35, 0xd5, 0x59, 0x9d, 0xa8, 0x89, 0xe9, 0xd0, 0x57, 0xc0, + 0x51, 0xe7, 0xeb, 0xf9, 0x47, 0x99, 0xf2, 0x16, 0x58, 0x53, 0xd7, 0x01, 0x7d, 0xac, 0xc2, 0x72, + 0x9f, 0x85, 0x52, 0x8c, 0x7b, 0x64, 0x24, 0x07, 0xf6, 0xaf, 0x34, 0xa1, 0x18, 0x1b, 0xeb, 0x23, + 0x39, 0x28, 0xf7, 0x60, 0xa2, 0x2a, 0x54, 0x01, 0x4b, 0xa9, 0x35, 0xa4, 0xe2, 0x84, 0x0a, 0x55, + 0xf2, 0x28, 0x31, 0x4c, 0x9b, 0xd4, 0xad, 0x0a, 0x29, 0x11, 0xce, 0x40, 0x07, 0xb5, 0x02, 0x36, + 0x3d, 0x15, 0xa5, 0xe2, 0xab, 0x6b, 0xa2, 0x94, 0xe9, 0x56, 0xff, 0x96, 0x81, 0x42, 0xb2, 0x0d, + 0xe8, 0x0b, 0x58, 0x69, 0x77, 0xf7, 0x9e, 0xd5, 0xf7, 0xdb, 0x7b, 0xbb, 0xbd, 0x66, 0xeb, 0xdb, + 0xfa, 0x8b, 0x67, 0xfb, 0xa5, 0xb9, 0xf2, 0xbd, 0xd3, 0xb3, 0xca, 0xea, 0x24, 0xe2, 0xc6, 0xf0, + 0x26, 0x3d, 0x22, 0x23, 0x4f, 0xce, 0xb2, 0x3a, 0x78, 0x6f, 0xbb, 0xd5, 0xed, 0x96, 0x32, 0x57, + 0xb1, 0x3a, 0x82, 0x3b, 0x34, 0x0c, 0xd1, 0x16, 0x94, 0x26, 0xac, 0x27, 0x2f, 0x3b, 0x2d, 0x7c, + 0x50, 0x9a, 0x2f, 0xdf, 0x3d, 0x3d, 0xab, 0xd8, 0x6f, 0x92, 0x9e, 0x8c, 0x03, 0x2a, 0x0e, 0xcc, + 0x73, 0xe1, 0x5f, 0x19, 0x28, 0x4e, 0x57, 0x9b, 0x68, 0x3b, 0xaa, 0x12, 0xf5, 0x31, 0xdc, 0xd8, + 0xda, 0xbc, 0xae, 0x3a, 0xd5, 0x59, 0xce, 0x1b, 0x29, 0xbf, 0xcf, 0xd5, 0xc3, 0x50, 0x93, 0xd1, + 0x17, 0xb0, 0x18, 0x70, 0x21, 0xe3, 0x7c, 0x90, 0xae, 0x56, 0x2e, 0xe2, 0x1a, 0x26, 0x02, 0x57, + 0x07, 0x70, 0x63, 0xd6, 0x1b, 0x7a, 0x00, 0x0b, 0x07, 0xed, 0x4e, 0x69, 0xae, 0x7c, 0xe7, 0xf4, + 0xac, 0xf2, 0xd1, 0xec, 0xc7, 0x03, 0x26, 0xe4, 0x88, 0x78, 0xed, 0x0e, 0xfa, 0x0c, 0x16, 0x9b, + 0xbb, 0x5d, 0x8c, 0x4b, 0x99, 0xf2, 0xfa, 0xe9, 0x59, 0xe5, 0xce, 0x2c, 0x4e, 0x7d, 0xe2, 0x23, + 0xdf, 0xc5, 0xfc, 0x30, 0x79, 0x24, 0xfd, 0x7b, 0x1e, 0x2c, 0x93, 0x26, 0xdf, 0xf7, 0x3b, 0x7a, + 0x39, 0xaa, 0x01, 0xe3, 0xf8, 0x37, 0x7f, 0x6d, 0x29, 0x58, 0x8c, 0x08, 0x46, 0x97, 0xf7, 0xa1, + 0xc8, 0x82, 0x93, 0x2f, 0x7b, 0xd4, 0x27, 0x87, 0x9e, 0x79, 0x2f, 0xe5, 0xb1, 0xa5, 0x6c, 0xad, + 0xc8, 0xa4, 0x82, 0x2f, 0xf3, 0x25, 0x15, 0xbe, 0x79, 0x09, 0xe5, 0x71, 0xd2, 0x47, 0xdf, 0x40, + 0x96, 0x05, 0x64, 0x68, 0xea, 0xd7, 0xd4, 0x15, 0xb4, 0x3b, 0xf5, 0xe7, 0xe6, 0xde, 0x34, 0xf2, + 0x17, 0xe7, 0xeb, 0x59, 0x65, 0xc0, 0x9a, 0x86, 0xd6, 0xe2, 0x12, 0x52, 0x8d, 0xa4, 0x13, 0x69, + 0x1e, 0x4f, 0x59, 0x94, 0xf6, 0x99, 0xdf, 0x17, 0x34, 0x0c, 0x75, 0x4a, 0xcd, 0xe3, 0xb8, 0x8b, + 0xca, 0xb0, 0x64, 0x0a, 0x51, 0x5d, 0x79, 0x16, 0x54, 0x91, 0x67, 0x0c, 0x8d, 0x65, 0xb0, 0xa2, + 0xdd, 0xe8, 0x1d, 0x09, 0x3e, 0xac, 0xfe, 0x27, 0x0b, 0xd6, 0xb6, 0x37, 0x0a, 0xa5, 0xa9, 0x29, + 0xde, 0xdb, 0xe6, 0xbf, 0x84, 0x15, 0xa2, 0xdf, 0xe5, 0xc4, 0x57, 0x09, 0x5a, 0xd7, 0xf7, 0xe6, + 0x00, 0x1e, 0xa4, 0xba, 0x4b, 0xc0, 0xd1, 0x5b, 0xa0, 0x91, 0x53, 0x3e, 0xed, 0x0c, 0x2e, 0x91, + 0x4b, 0x5f, 0x50, 0x17, 0x96, 0xb9, 0x70, 0x06, 0x34, 0x94, 0x51, 0x5a, 0x37, 0xef, 0xd8, 0xd4, + 0x3f, 0x1c, 0x7b, 0xd3, 0x40, 0x93, 0xd3, 0xa2, 0xd9, 0xce, 0xfa, 0x40, 0x8f, 0x20, 0x2b, 0xc8, + 0x51, 0xfc, 0x56, 0x49, 0xbd, 0x24, 0x98, 0x1c, 0xc9, 0x19, 0x17, 0x9a, 0x81, 0x7e, 0x0d, 0xe0, + 0xb2, 0x30, 0x20, 0xd2, 0x19, 0x50, 0x61, 0x0e, 0x3b, 0x75, 0x89, 0xcd, 0x04, 0x35, 0xe3, 0x65, + 0x8a, 0x8d, 0x9e, 0x42, 0xc1, 0x21, 0xb1, 0x5c, 0x73, 0x57, 0x3f, 0xee, 0xb7, 0xeb, 0xc6, 0x45, + 0x49, 0xb9, 0xb8, 0x38, 0x5f, 0xcf, 0xc7, 0x16, 0x9c, 0x77, 0x88, 0x91, 0xef, 0x53, 0x58, 0x56, + 0x8f, 0xfe, 0x9e, 0x1b, 0x85, 0xb3, 0x48, 0x26, 0x57, 0xe4, 0x68, 0xf5, 0x82, 0x34, 0x61, 0x2f, + 0x3e, 0xce, 0xa2, 0x9c, 0xb2, 0xa1, 0xdf, 0xc0, 0x0a, 0xf5, 0x1d, 0x31, 0xd6, 0x62, 0x8d, 0x67, + 0x98, 0xbf, 0x7a, 0xb1, 0xad, 0x04, 0x3c, 0xb3, 0xd8, 0x12, 0xbd, 0x64, 0xaf, 0xfe, 0x33, 0x03, + 0x10, 0x95, 0x3d, 0xef, 0x57, 0x80, 0x08, 0xb2, 0x2e, 0x91, 0x44, 0x6b, 0xae, 0x88, 0x75, 0x1b, + 0x7d, 0x0d, 0x20, 0xe9, 0x30, 0x50, 0xa1, 0xd7, 0xef, 0x1b, 0xd9, 0xbc, 0x2d, 0x1c, 0x4c, 0xa1, + 0xd1, 0x16, 0xe4, 0xcc, 0x8b, 0x32, 0x7b, 0x2d, 0xcf, 0x20, 0xab, 0x7f, 0xc9, 0x00, 0x44, 0xcb, + 0xfc, 0xbf, 0x5e, 0x5b, 0xc3, 0x7e, 0xfd, 0xc3, 0xda, 0xdc, 0x3f, 0x7e, 0x58, 0x9b, 0xfb, 0xc3, + 0xc5, 0x5a, 0xe6, 0xf5, 0xc5, 0x5a, 0xe6, 0xef, 0x17, 0x6b, 0x99, 0xef, 0x2f, 0xd6, 0x32, 0x87, + 0x39, 0x5d, 0x99, 0xfc, 0xf4, 0xbf, 0x01, 0x00, 0x00, 0xff, 0xff, 0xb8, 0xa3, 0x85, 0xdc, 0xc7, + 0x15, 0x00, 0x00, } diff --git a/vendor/github.com/docker/swarmkit/api/specs.proto b/vendor/github.com/docker/swarmkit/api/specs.proto index 2b002c54f2..14448d0409 100644 --- a/vendor/github.com/docker/swarmkit/api/specs.proto +++ b/vendor/github.com/docker/swarmkit/api/specs.proto @@ -314,6 +314,10 @@ message ContainerSpec { // Isolation defines the isolation level for windows containers (default, process, hyperv). // Runtimes that don't support it ignore that field Isolation isolation = 24; + + // PidsLimit prevents from OS resource damage by applications inside the container + // using fork bomb attack. + int64 pidsLimit = 25; } // EndpointSpec defines the properties that can be configured to From 7ddd5f3434470cec983f319c5dc27c7a3fb6085b Mon Sep 17 00:00:00 2001 From: Renaud Gaubert Date: Wed, 8 Nov 2017 17:33:36 +0100 Subject: [PATCH 2/7] Updated GenericResource CLI Signed-off-by: Renaud Gaubert --- cli/command/service/create.go | 4 ++ cli/command/service/generic_resource_opts.go | 76 ++++++++++++++++++++ cli/command/service/opts.go | 30 +++++--- 3 files changed, 101 insertions(+), 9 deletions(-) create mode 100644 cli/command/service/generic_resource_opts.go diff --git a/cli/command/service/create.go b/cli/command/service/create.go index 84944fd23c..299cdc383f 100644 --- a/cli/command/service/create.go +++ b/cli/command/service/create.go @@ -5,6 +5,7 @@ import ( "github.com/docker/cli/cli" "github.com/docker/cli/cli/command" + cliopts "github.com/docker/cli/opts" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/versions" "github.com/spf13/cobra" @@ -58,6 +59,9 @@ func newCreateCommand(dockerCli command.Cli) *cobra.Command { flags.Var(&opts.hosts, flagHost, "Set one or more custom host-to-IP mappings (host:ip)") flags.SetAnnotation(flagHost, "version", []string{"1.25"}) + flags.Var(cliopts.NewListOptsRef(&opts.resources.resGenericResources, ValidateSingleGenericResource), "generic-resource", "User defined resources") + flags.SetAnnotation(flagHostAdd, "version", []string{"1.32"}) + flags.SetInterspersed(false) return cmd } diff --git a/cli/command/service/generic_resource_opts.go b/cli/command/service/generic_resource_opts.go new file mode 100644 index 0000000000..5effe83968 --- /dev/null +++ b/cli/command/service/generic_resource_opts.go @@ -0,0 +1,76 @@ +package service + +import ( + "fmt" + "strings" + + "github.com/pkg/errors" + + "github.com/docker/docker/api/types/swarm" + swarmapi "github.com/docker/swarmkit/api" + "github.com/docker/swarmkit/api/genericresource" +) + +// GenericResource is a concept that a user can use to advertise user-defined +// resources on a node and thus better place services based on these resources. +// E.g: NVIDIA GPUs, Intel FPGAs, ... +// See https://github.com/docker/swarmkit/blob/master/design/generic_resources.md + +// ValidateSingleGenericResource validates that a single entry in the +// generic resource list is valid. +// i.e 'GPU=UID1' is valid however 'GPU:UID1' or 'UID1' isn't +func ValidateSingleGenericResource(val string) (string, error) { + if strings.Count(val, "=") < 1 { + return "", fmt.Errorf("invalid generic-resource format `%s` expected `name=value`", val) + } + + return val, nil +} + +// ParseGenericResources parses an array of Generic resourceResources +// Requesting Named Generic Resources for a service is not supported this +// is filtered here. +func ParseGenericResources(value []string) ([]swarm.GenericResource, error) { + if len(value) == 0 { + return nil, nil + } + + resources, err := genericresource.Parse(value) + if err != nil { + return nil, errors.Wrapf(err, "invalid generic resource specification") + } + + swarmResources := genericResourcesFromGRPC(resources) + for _, res := range swarmResources { + if res.NamedResourceSpec != nil { + return nil, fmt.Errorf("invalid generic-resource request `%s=%s`, Named Generic Resources is not supported for service create or update", res.NamedResourceSpec.Kind, res.NamedResourceSpec.Value) + } + } + + return swarmResources, nil +} + +// genericResourcesFromGRPC converts a GRPC GenericResource to a GenericResource +func genericResourcesFromGRPC(genericRes []*swarmapi.GenericResource) []swarm.GenericResource { + var generic []swarm.GenericResource + for _, res := range genericRes { + var current swarm.GenericResource + + switch r := res.Resource.(type) { + case *swarmapi.GenericResource_DiscreteResourceSpec: + current.DiscreteResourceSpec = &swarm.DiscreteGenericResource{ + Kind: r.DiscreteResourceSpec.Kind, + Value: r.DiscreteResourceSpec.Value, + } + case *swarmapi.GenericResource_NamedResourceSpec: + current.NamedResourceSpec = &swarm.NamedGenericResource{ + Kind: r.NamedResourceSpec.Kind, + Value: r.NamedResourceSpec.Value, + } + } + + generic = append(generic, current) + } + + return generic +} diff --git a/cli/command/service/opts.go b/cli/command/service/opts.go index 071998607e..209167dfda 100644 --- a/cli/command/service/opts.go +++ b/cli/command/service/opts.go @@ -222,23 +222,30 @@ func (opts updateOptions) rollbackConfig(flags *pflag.FlagSet) *swarm.UpdateConf } type resourceOptions struct { - limitCPU opts.NanoCPUs - limitMemBytes opts.MemBytes - resCPU opts.NanoCPUs - resMemBytes opts.MemBytes + limitCPU opts.NanoCPUs + limitMemBytes opts.MemBytes + resCPU opts.NanoCPUs + resMemBytes opts.MemBytes + resGenericResources []string } -func (r *resourceOptions) ToResourceRequirements() *swarm.ResourceRequirements { +func (r *resourceOptions) ToResourceRequirements() (*swarm.ResourceRequirements, error) { + generic, err := ParseGenericResources(r.resGenericResources) + if err != nil { + return nil, err + } + return &swarm.ResourceRequirements{ Limits: &swarm.Resources{ NanoCPUs: r.limitCPU.Value(), MemoryBytes: r.limitMemBytes.Value(), }, Reservations: &swarm.Resources{ - NanoCPUs: r.resCPU.Value(), - MemoryBytes: r.resMemBytes.Value(), + NanoCPUs: r.resCPU.Value(), + MemoryBytes: r.resMemBytes.Value(), + GenericResources: generic, }, - } + }, nil } type restartPolicyOptions struct { @@ -588,6 +595,11 @@ func (options *serviceOptions) ToService(ctx context.Context, apiClient client.N return service, err } + resources, err := options.resources.ToResourceRequirements() + if err != nil { + return service, err + } + service = swarm.ServiceSpec{ Annotations: swarm.Annotations{ Name: options.name, @@ -619,7 +631,7 @@ func (options *serviceOptions) ToService(ctx context.Context, apiClient client.N Isolation: container.Isolation(options.isolation), }, Networks: networks, - Resources: options.resources.ToResourceRequirements(), + Resources: resources, RestartPolicy: options.restartPolicy.ToRestartPolicy(flags), Placement: &swarm.Placement{ Constraints: options.constraints.GetAll(), From 51c7cd91cf8530a81d1cc458db277c0e958a97d9 Mon Sep 17 00:00:00 2001 From: Renaud Gaubert Date: Wed, 8 Nov 2017 17:55:16 +0100 Subject: [PATCH 3/7] Added Generic Resource tests Signed-off-by: Renaud Gaubert --- .../service/generic_resource_opts_test.go | 22 +++++++++++ cli/command/service/opts_test.go | 38 +++++++++++++++++++ 2 files changed, 60 insertions(+) create mode 100644 cli/command/service/generic_resource_opts_test.go diff --git a/cli/command/service/generic_resource_opts_test.go b/cli/command/service/generic_resource_opts_test.go new file mode 100644 index 0000000000..99217e9f36 --- /dev/null +++ b/cli/command/service/generic_resource_opts_test.go @@ -0,0 +1,22 @@ +package service + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestValidateSingleGenericResource(t *testing.T) { + incorrect := []string{"foo", "fooo-bar"} + correct := []string{"foo=bar", "bar=1", "foo=barbar"} + + for _, v := range incorrect { + _, err := ValidateSingleGenericResource(v) + assert.Error(t, err) + } + + for _, v := range correct { + _, err := ValidateSingleGenericResource(v) + assert.NoError(t, err) + } +} diff --git a/cli/command/service/opts_test.go b/cli/command/service/opts_test.go index 6bf00ef6bb..e373c6479d 100644 --- a/cli/command/service/opts_test.go +++ b/cli/command/service/opts_test.go @@ -85,3 +85,41 @@ func TestHealthCheckOptionsToHealthConfigConflict(t *testing.T) { _, err := opt.toHealthConfig() assert.EqualError(t, err, "--no-healthcheck conflicts with --health-* options") } + +func TestResourceOptionsToResourceRequirements(t *testing.T) { + incorrectOptions := []resourceOptions{ + { + resGenericResources: []string{"foo=bar", "foo=1"}, + }, + { + resGenericResources: []string{"foo=bar", "foo=baz"}, + }, + { + resGenericResources: []string{"foo=bar"}, + }, + { + resGenericResources: []string{"foo=1", "foo=2"}, + }, + } + + for _, opt := range incorrectOptions { + _, err := opt.ToResourceRequirements() + assert.Error(t, err) + } + + correctOptions := []resourceOptions{ + { + resGenericResources: []string{"foo=1"}, + }, + { + resGenericResources: []string{"foo=1", "bar=2"}, + }, + } + + for _, opt := range correctOptions { + r, err := opt.ToResourceRequirements() + assert.NoError(t, err) + assert.Len(t, r.Reservations.GenericResources, len(opt.resGenericResources)) + } + +} From fe07ca70d6b5d699067f7bd65962134a7262c951 Mon Sep 17 00:00:00 2001 From: Renaud Gaubert Date: Wed, 8 Nov 2017 18:13:11 +0100 Subject: [PATCH 4/7] Added docs for dockerd Signed-off-by: Renaud Gaubert --- docs/reference/commandline/dockerd.md | 19 +++++++++++++++++++ man/dockerd.8.md | 10 ++++++++++ 2 files changed, 29 insertions(+) diff --git a/docs/reference/commandline/dockerd.md b/docs/reference/commandline/dockerd.md index f8573910f1..e95016bd6b 100644 --- a/docs/reference/commandline/dockerd.md +++ b/docs/reference/commandline/dockerd.md @@ -72,6 +72,7 @@ Options: --max-concurrent-uploads int Set the max concurrent uploads for each push (default 5) --metrics-addr string Set default address and port to serve the metrics api on --mtu int Set the containers network MTU + --node-generic-resources list Advertise user-defined resource --no-new-privileges Set no-new-privileges by default for new containers --oom-score-adjust int Set the oom_score_adj for the daemon (default -500) -p, --pidfile string Path to use for daemon PID file (default "/var/run/docker.pid") @@ -1237,6 +1238,23 @@ Please note that this feature is still marked as experimental as metrics and met names could change while this feature is still in experimental. Please provide feedback on what you would like to see collected in the API. +#### Node Generic Resources + +The `--node-generic-resources` option takes a list of key-value +pair (`key=value`) that allows you to advertise user defined resources +in a swarm cluster. + +The current expected use case is to advertise NVIDIA GPUs so that services +requesting `NVIDIA-GPU=[0-16]` can land on a node that has enough GPUs for +the task to run. + +Example of usage: +```json +{ + "node-generic-resources": ["NVIDIA-GPU=UUID1", "NVIDIA-GPU=UUID2"] +} +``` + ### Daemon configuration file The `--config-file` option allows you to set any configuration option @@ -1325,6 +1343,7 @@ This is a full example of the allowed configuration options on Linux: "no-new-privileges": false, "default-runtime": "runc", "oom-score-adjust": -500, + "node-generic-resources": ["NVIDIA-GPU=UUID1", "NVIDIA-GPU=UUID2"], "runtimes": { "cc-runtime": { "path": "/usr/bin/cc-runtime" diff --git a/man/dockerd.8.md b/man/dockerd.8.md index 5ff7dcd331..f4e2be4c99 100644 --- a/man/dockerd.8.md +++ b/man/dockerd.8.md @@ -56,6 +56,7 @@ dockerd - Enable daemon mode [**--mtu**[=*0*]] [**--max-concurrent-downloads**[=*3*]] [**--max-concurrent-uploads**[=*5*]] +[**--node-generic-resources**[=*[]*]] [**-p**|**--pidfile**[=*/var/run/docker.pid*]] [**--raw-logs**] [**--registry-mirror**[=*[]*]] @@ -326,6 +327,15 @@ unix://[/path/to/socket] to use. **--max-concurrent-uploads**=*5* Set the max concurrent uploads for each push. Default is `5`. +**--node-generic-resources**=*[]* + Advertise user-defined resource. Default is `[]`. + Use this if your swarm cluster has some nodes with custom + resources (e.g: NVIDIA GPU, SSD, ...) and you need your services to land on + nodes advertising these resources. + Usage example: `--node-generic-resources "NVIDIA-GPU=UUID1" + --node-generic-resources "NVIDIA-GPU=UUID2"` + + **-p**, **--pidfile**="" Path to use for daemon PID file. Default is `/var/run/docker.pid` From 4a6da88f7ac06d7bc09be8c197dbfbb8aa4894c5 Mon Sep 17 00:00:00 2001 From: Renaud Gaubert Date: Wed, 8 Nov 2017 18:21:54 +0100 Subject: [PATCH 5/7] Added docs for service create Signed-off-by: Renaud Gaubert --- docs/reference/commandline/service_create.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/reference/commandline/service_create.md b/docs/reference/commandline/service_create.md index f196a66996..5634d72cff 100644 --- a/docs/reference/commandline/service_create.md +++ b/docs/reference/commandline/service_create.md @@ -33,6 +33,7 @@ Options: --entrypoint command Overwrite the default ENTRYPOINT of the image -e, --env list Set environment variables --env-file list Read in a file of environment variables + --generic-resource list User defined resources request --group list Set one or more supplementary user groups for the container --health-cmd string Command to run to check health --health-interval duration Time between running the check (ms|s|m|h) @@ -915,6 +916,17 @@ Supported isolation modes on Windows are: - `process`: use process isolation (Windows server only) - `hyperv`: use Hyper-V isolation +### Create services requesting Generic Resources + +You can narrow the kind of nodes your task can land on through the using the +`--generic-resource` flag (if the nodes advertise these resources): + +```bash +$ docker service create --name cuda \ + --generic-resource "NVIDIA-GPU=2" \ + --generic-resource "SSD=1" \ + nvidia/cuda +``` ## Related commands From 20a6ff32ee0edaa6b87a493871a9d5e05065e1e9 Mon Sep 17 00:00:00 2001 From: Renaud Gaubert Date: Fri, 17 Nov 2017 23:05:44 +0100 Subject: [PATCH 6/7] Added support for generic resource update Signed-off-by: Renaud Gaubert --- cli/command/service/create.go | 2 + cli/command/service/generic_resource_opts.go | 29 +++++++ cli/command/service/opts.go | 2 + cli/command/service/update.go | 84 ++++++++++++++++++++ cli/command/service/update_test.go | 39 +++++++++ docs/reference/commandline/service_update.md | 2 + 6 files changed, 158 insertions(+) diff --git a/cli/command/service/create.go b/cli/command/service/create.go index 299cdc383f..6e19f8b748 100644 --- a/cli/command/service/create.go +++ b/cli/command/service/create.go @@ -77,6 +77,8 @@ func runCreate(dockerCli command.Cli, flags *pflag.FlagSet, opts *serviceOptions return err } + fmt.Printf("%v\n", service.TaskTemplate.Resources) + specifiedSecrets := opts.secrets.Value() if len(specifiedSecrets) > 0 { // parse and validate secrets diff --git a/cli/command/service/generic_resource_opts.go b/cli/command/service/generic_resource_opts.go index 5effe83968..66385888e1 100644 --- a/cli/command/service/generic_resource_opts.go +++ b/cli/command/service/generic_resource_opts.go @@ -74,3 +74,32 @@ func genericResourcesFromGRPC(genericRes []*swarmapi.GenericResource) []swarm.Ge return generic } + +func buildGenericResourceMap(genericRes []swarm.GenericResource) (map[string]swarm.GenericResource, error) { + m := make(map[string]swarm.GenericResource) + + for _, res := range genericRes { + if res.DiscreteResourceSpec == nil { + return nil, fmt.Errorf("invalid generic-resource `%+v` for service task", res) + } + + _, ok := m[res.DiscreteResourceSpec.Kind] + if ok { + return nil, fmt.Errorf("duplicate generic-resource `%+v` for service task", res.DiscreteResourceSpec.Kind) + } + + m[res.DiscreteResourceSpec.Kind] = res + } + + return m, nil +} + +func buildGenericResourceList(genericRes map[string]swarm.GenericResource) []swarm.GenericResource { + var l []swarm.GenericResource + + for _, res := range genericRes { + l = append(l, res) + } + + return l +} diff --git a/cli/command/service/opts.go b/cli/command/service/opts.go index 209167dfda..b57c5c8c4e 100644 --- a/cli/command/service/opts.go +++ b/cli/command/service/opts.go @@ -830,6 +830,8 @@ const ( flagEnvFile = "env-file" flagEnvRemove = "env-rm" flagEnvAdd = "env-add" + flagGenericResourcesRemove = "generic-resource-rm" + flagGenericResourcesAdd = "generic-resource-add" flagGroup = "group" flagGroupAdd = "group-add" flagGroupRemove = "group-rm" diff --git a/cli/command/service/update.go b/cli/command/service/update.go index a7c82ec0c1..d657871ff2 100644 --- a/cli/command/service/update.go +++ b/cli/command/service/update.go @@ -95,6 +95,12 @@ func newUpdateCommand(dockerCli command.Cli) *cobra.Command { flags.Var(&options.hosts, flagHostAdd, "Add a custom host-to-IP mapping (host:ip)") flags.SetAnnotation(flagHostAdd, "version", []string{"1.25"}) + // Add needs parsing, Remove only needs the key + flags.Var(newListOptsVar(), flagGenericResourcesRemove, "Remove a Generic resource") + flags.SetAnnotation(flagHostAdd, "version", []string{"1.32"}) + flags.Var(newListOptsVarWithValidator(ValidateSingleGenericResource), flagGenericResourcesAdd, "Add a Generic resource") + flags.SetAnnotation(flagHostAdd, "version", []string{"1.32"}) + return cmd } @@ -102,6 +108,10 @@ func newListOptsVar() *opts.ListOpts { return opts.NewListOptsRef(&[]string{}, nil) } +func newListOptsVarWithValidator(validator opts.ValidatorFctType) *opts.ListOpts { + return opts.NewListOptsRef(&[]string{}, validator) +} + // nolint: gocyclo func runUpdate(dockerCli command.Cli, flags *pflag.FlagSet, options *serviceOptions, serviceID string) error { apiClient := dockerCli.Client() @@ -314,6 +324,14 @@ func updateService(ctx context.Context, apiClient client.NetworkAPIClient, flags updateInt64Value(flagReserveMemory, &task.Resources.Reservations.MemoryBytes) } + if err := addGenericResources(flags, task); err != nil { + return err + } + + if err := removeGenericResources(flags, task); err != nil { + return err + } + updateDurationOpt(flagStopGracePeriod, &cspec.StopGracePeriod) if anyChanged(flags, flagRestartCondition, flagRestartDelay, flagRestartMaxAttempts, flagRestartWindow) { @@ -470,6 +488,72 @@ func anyChanged(flags *pflag.FlagSet, fields ...string) bool { return false } +func addGenericResources(flags *pflag.FlagSet, spec *swarm.TaskSpec) error { + if !flags.Changed(flagGenericResourcesAdd) { + return nil + } + + if spec.Resources == nil { + spec.Resources = &swarm.ResourceRequirements{} + } + + if spec.Resources.Reservations == nil { + spec.Resources.Reservations = &swarm.Resources{} + } + + values := flags.Lookup(flagGenericResourcesAdd).Value.(*opts.ListOpts).GetAll() + generic, err := ParseGenericResources(values) + if err != nil { + return err + } + + m, err := buildGenericResourceMap(spec.Resources.Reservations.GenericResources) + if err != nil { + return err + } + + for _, toAddRes := range generic { + m[toAddRes.DiscreteResourceSpec.Kind] = toAddRes + } + + spec.Resources.Reservations.GenericResources = buildGenericResourceList(m) + + return nil +} + +func removeGenericResources(flags *pflag.FlagSet, spec *swarm.TaskSpec) error { + // Can only be Discrete Resources + if !flags.Changed(flagGenericResourcesRemove) { + return nil + } + + if spec.Resources == nil { + spec.Resources = &swarm.ResourceRequirements{} + } + + if spec.Resources.Reservations == nil { + spec.Resources.Reservations = &swarm.Resources{} + } + + values := flags.Lookup(flagGenericResourcesRemove).Value.(*opts.ListOpts).GetAll() + + m, err := buildGenericResourceMap(spec.Resources.Reservations.GenericResources) + if err != nil { + return err + } + + for _, toRemoveRes := range values { + if _, ok := m[toRemoveRes]; !ok { + return fmt.Errorf("could not find generic-resource `%s` to remove it", toRemoveRes) + } + + delete(m, toRemoveRes) + } + + spec.Resources.Reservations.GenericResources = buildGenericResourceList(m) + return nil +} + func updatePlacementConstraints(flags *pflag.FlagSet, placement *swarm.Placement) { if flags.Changed(flagConstraintAdd) { values := flags.Lookup(flagConstraintAdd).Value.(*opts.ListOpts).GetAll() diff --git a/cli/command/service/update_test.go b/cli/command/service/update_test.go index 92b4d6bc59..ba55269cb0 100644 --- a/cli/command/service/update_test.go +++ b/cli/command/service/update_test.go @@ -547,3 +547,42 @@ func TestUpdateIsolationInvalid(t *testing.T) { require.NoError(t, err) assert.Equal(t, container.Isolation("test"), spec.TaskTemplate.ContainerSpec.Isolation) } + +func TestAddGenericResources(t *testing.T) { + task := &swarm.TaskSpec{} + flags := newUpdateCommand(nil).Flags() + + assert.Nil(t, addGenericResources(flags, task)) + + flags.Set(flagGenericResourcesAdd, "foo=1") + assert.NoError(t, addGenericResources(flags, task)) + assert.Len(t, task.Resources.Reservations.GenericResources, 1) + + // Checks that foo isn't added a 2nd time + flags = newUpdateCommand(nil).Flags() + flags.Set(flagGenericResourcesAdd, "bar=1") + assert.NoError(t, addGenericResources(flags, task)) + assert.Len(t, task.Resources.Reservations.GenericResources, 2) +} + +func TestRemoveGenericResources(t *testing.T) { + task := &swarm.TaskSpec{} + flags := newUpdateCommand(nil).Flags() + + assert.Nil(t, removeGenericResources(flags, task)) + + flags.Set(flagGenericResourcesRemove, "foo") + assert.Error(t, removeGenericResources(flags, task)) + + flags = newUpdateCommand(nil).Flags() + flags.Set(flagGenericResourcesAdd, "foo=1") + addGenericResources(flags, task) + flags = newUpdateCommand(nil).Flags() + flags.Set(flagGenericResourcesAdd, "bar=1") + addGenericResources(flags, task) + + flags = newUpdateCommand(nil).Flags() + flags.Set(flagGenericResourcesRemove, "foo") + assert.NoError(t, removeGenericResources(flags, task)) + assert.Len(t, task.Resources.Reservations.GenericResources, 1) +} diff --git a/docs/reference/commandline/service_update.md b/docs/reference/commandline/service_update.md index f57a26e802..ef9e0816d2 100644 --- a/docs/reference/commandline/service_update.md +++ b/docs/reference/commandline/service_update.md @@ -41,6 +41,8 @@ Options: --env-add list Add or update an environment variable --env-rm list Remove an environment variable --force Force update even if no changes require it + --generic-resource-add list Add an additional generic resource to the service's resources requirements + --generic-resource-rm list Remove a previously added generic resource to the service's resources requirements --group-add list Add an additional supplementary user group to the container --group-rm list Remove a previously added supplementary user group from the container --health-cmd string Command to run to check health From 1ff73f867df382cb5a19df4579da3570f4daaff5 Mon Sep 17 00:00:00 2001 From: Renaud Gaubert Date: Fri, 17 Nov 2017 23:05:59 +0100 Subject: [PATCH 7/7] Added support of Generic resources in compose file Signed-off-by: Renaud Gaubert --- cli/command/service/create.go | 2 - cli/compose/convert/service.go | 20 ++++++++- cli/compose/loader/full-example.yml | 10 ++++- cli/compose/loader/loader_test.go | 14 ++++++ cli/compose/schema/bindata.go | 2 +- .../schema/data/config_schema_v3.5.json | 44 ++++++++++++++----- cli/compose/types/types.go | 20 ++++++++- 7 files changed, 93 insertions(+), 19 deletions(-) diff --git a/cli/command/service/create.go b/cli/command/service/create.go index 6e19f8b748..299cdc383f 100644 --- a/cli/command/service/create.go +++ b/cli/command/service/create.go @@ -77,8 +77,6 @@ func runCreate(dockerCli command.Cli, flags *pflag.FlagSet, opts *serviceOptions return err } - fmt.Printf("%v\n", service.TaskTemplate.Resources) - specifiedSecrets := opts.secrets.Value() if len(specifiedSecrets) > 0 { // parse and validate secrets diff --git a/cli/compose/convert/service.go b/cli/compose/convert/service.go index 3c9a6a661a..e58c241c22 100644 --- a/cli/compose/convert/service.go +++ b/cli/compose/convert/service.go @@ -510,9 +510,25 @@ func convertResources(source composetypes.Resources) (*swarm.ResourceRequirement return nil, err } } + + var generic []swarm.GenericResource + for _, res := range source.Reservations.GenericResources { + var r swarm.GenericResource + + if res.DiscreteResourceSpec != nil { + r.DiscreteResourceSpec = &swarm.DiscreteGenericResource{ + Kind: res.DiscreteResourceSpec.Kind, + Value: res.DiscreteResourceSpec.Value, + } + } + + generic = append(generic, r) + } + resources.Reservations = &swarm.Resources{ - NanoCPUs: cpus, - MemoryBytes: int64(source.Reservations.MemoryBytes), + NanoCPUs: cpus, + MemoryBytes: int64(source.Reservations.MemoryBytes), + GenericResources: generic, } } return resources, nil diff --git a/cli/compose/loader/full-example.yml b/cli/compose/loader/full-example.yml index aefb56869e..0a52111c32 100644 --- a/cli/compose/loader/full-example.yml +++ b/cli/compose/loader/full-example.yml @@ -1,4 +1,4 @@ -version: "3.4" +version: "3.5" services: foo: @@ -16,7 +16,6 @@ services: labels: [FOO=BAR] - cap_add: - ALL @@ -54,6 +53,13 @@ services: reservations: cpus: '0.0001' memory: 20M + generic_resources: + - discrete_resource_spec: + kind: 'gpu' + value: 2 + - discrete_resource_spec: + kind: 'ssd' + value: 1 restart_policy: condition: on-failure delay: 5s diff --git a/cli/compose/loader/loader_test.go b/cli/compose/loader/loader_test.go index e56202d86c..d609311abc 100644 --- a/cli/compose/loader/loader_test.go +++ b/cli/compose/loader/loader_test.go @@ -879,6 +879,20 @@ func TestFullExample(t *testing.T) { Reservations: &types.Resource{ NanoCPUs: "0.0001", MemoryBytes: 20 * 1024 * 1024, + GenericResources: []types.GenericResource{ + { + DiscreteResourceSpec: &types.DiscreteGenericResource{ + Kind: "gpu", + Value: 2, + }, + }, + { + DiscreteResourceSpec: &types.DiscreteGenericResource{ + Kind: "ssd", + Value: 1, + }, + }, + }, }, }, RestartPolicy: &types.RestartPolicy{ diff --git a/cli/compose/schema/bindata.go b/cli/compose/schema/bindata.go index da71abb1a7..e234c2cddd 100644 --- a/cli/compose/schema/bindata.go +++ b/cli/compose/schema/bindata.go @@ -173,7 +173,7 @@ func dataConfig_schema_v34Json() (*asset, error) { return a, nil } -var _dataConfig_schema_v35Json = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xec\x1b\x4b\x73\xdb\xbc\xf1\xae\x5f\xc1\xe1\xf7\xdd\x22\xd9\x99\x69\xda\x99\xe6\xd6\x63\x4f\xed\xb9\x1e\x85\x03\x81\x2b\x09\x31\x08\x20\x0b\x50\xb6\x92\xf1\x7f\xef\xf0\x29\x90\x04\x09\x50\xa2\x63\xa7\xcd\xc9\x16\xb9\xbb\xc0\xbe\x1f\x00\x7f\xac\xa2\x28\xfe\x53\xd3\x23\x64\x24\xfe\x1c\xc5\x47\x63\xd4\xe7\xfb\xfb\xaf\x5a\x8a\x4d\xf5\xf4\x4e\xe2\xe1\x3e\x45\xb2\x37\x9b\x8f\x9f\xee\xab\x67\x7f\xc4\xeb\x02\x8f\xa5\x05\x0a\x95\x62\xcf\x0e\x49\xf5\x26\x39\xfd\xe5\xee\xaf\x77\x05\x7a\x05\x62\xce\x0a\x0a\x20\xb9\xfb\x0a\xd4\x54\xcf\x10\xbe\xe5\x0c\xa1\x40\x7e\x88\x4f\x80\x9a\x49\x11\x6f\xd7\xab\xe2\x9d\x42\xa9\x00\x0d\x03\x1d\x7f\x8e\x8a\xcd\x45\x51\x0b\xd2\x3c\xb0\xc8\x6a\x83\x4c\x1c\xe2\xf2\xf1\x4b\x49\x21\x8a\x62\x0d\x78\x62\xd4\xa2\xd0\x6e\xf5\x8f\xfb\x0b\xfd\xfb\x16\x6c\xdd\xa7\x6a\x6d\xb6\x7c\xae\x88\x31\x80\xe2\xdf\xc3\xbd\x95\xaf\xbf\x3c\x90\xcd\xf7\x7f\x6c\xfe\xf3\x71\xf3\xf7\xbb\x64\xb3\xfd\xf0\x67\xe7\x75\x21\x5f\x84\x7d\xb5\x7c\x0a\x7b\x26\x98\x61\x52\xb4\xeb\xc7\x2d\xe4\x4b\xfd\xdf\x4b\xbb\x30\x49\xd3\x12\x98\xf0\xce\xda\x7b\xc2\x35\x74\x79\x16\x60\x9e\x24\x3e\xfa\x78\x6e\xc1\xde\x88\xe7\x7a\x7d\x07\xcf\x5d\x76\x4e\x92\xe7\x99\x57\x83\x0d\xd4\x1b\x31\x53\x2d\xbf\x8c\xfe\x34\x50\x04\xe3\x37\xd9\x0a\xea\xcd\x2c\xb6\x58\x7e\x19\x86\xab\xa8\xe1\x63\xb8\x81\x7a\x23\x86\xab\xe5\x6f\x63\x78\xd5\x30\xed\xde\x63\xfc\xe5\x79\x53\xfc\x7d\x29\x69\x4e\xd2\xab\xa8\x58\xfb\x2b\x99\xe8\xc4\x3c\x97\x38\x5d\x31\x67\x5c\x9e\xad\x40\x47\x24\x99\x82\xe2\xf2\x5c\xee\xdc\x2d\xb3\x0a\x20\x03\x61\xe2\x56\x4c\x51\x14\xef\x72\xc6\xd3\xbe\xd4\xa5\x80\x7f\x15\x24\x1e\xac\x87\x51\xf4\xa3\x1f\xde\x2d\x3a\xe5\xfb\xce\xaf\x71\xa3\x68\xdf\x8f\xf0\xd2\xbe\xa7\x52\x18\x78\x36\x25\x53\xd3\x4b\x57\x22\x90\xf4\x11\x70\xcf\x38\x84\x62\x10\xac\x2c\x7d\x44\x64\x9c\x69\x93\x48\x4c\x52\x46\x8d\x13\x9f\x93\x1d\xf0\x9b\x28\x50\x42\x8f\x90\xec\x51\x66\x5e\x2a\xfb\xa4\xe2\x44\x3b\x09\x35\x11\x3c\x90\x73\x43\xf0\x00\x6e\xc9\xf6\x80\x07\xd8\x7e\xdf\x6a\x51\xad\x5f\xdb\x95\x83\x60\x4c\x89\x4a\x48\x9a\x76\xf6\x41\x10\xc9\x39\x5e\x47\x31\x33\x90\x69\x37\x43\x51\x9c\x0b\xf6\x2d\x87\x7f\xd6\x20\x06\x73\xe8\xd3\x4d\x51\xaa\xe5\x09\x1f\x50\xe6\x2a\x51\x04\x0b\x47\x9a\x16\x76\x4c\x65\x96\x11\xb1\x94\x77\xcd\xe1\x23\x40\xf2\x83\x38\x1f\xd9\x2e\x5b\xaf\x61\xbf\x6a\x57\xeb\x6c\x6b\x84\x1b\x3f\x3f\xc3\x78\xe1\x8f\x18\xfe\x98\x51\x84\x5c\x99\x23\x0d\x0d\x01\xd3\xae\xe0\x84\xcf\x59\x1a\x0e\x7c\x98\x03\x9c\xc9\xb4\xbb\x6f\x91\x67\x3b\xc0\x81\x4b\x76\x3d\x6b\xf8\x7b\xbb\x72\xbd\xe9\x69\xdf\x10\x26\x00\x13\x41\x32\x9f\xac\x62\x8a\x90\x82\x30\x8c\xf0\x44\x2b\xa0\x1d\xf0\x46\x53\x13\x9a\x89\x83\x42\x72\x8c\x70\x60\xda\xe0\x79\x3a\x28\xbd\xd8\x1b\x4b\x41\x81\x48\x75\x52\x35\x21\xf3\xa3\x67\x9c\x42\xdb\x91\x2c\x1a\x26\x52\x31\x95\x15\x2a\x32\x45\x5e\x28\xf6\x16\xf7\x10\x13\x0d\x04\xe9\xf1\x4a\x7c\x99\x11\x26\x42\x94\x0a\xc2\xe0\x59\x49\x56\x85\xb1\x77\x17\x9f\x40\x9c\x92\xd6\x6e\x66\x8b\x01\xc4\x89\xa1\x14\x59\x13\xa4\xc3\xb2\xb3\x85\xff\xac\xa4\x86\xdb\x83\x63\x8d\xf1\xd0\x30\xbe\x6e\x7d\x7a\xdb\x95\x5e\xbc\x97\x98\x91\x62\xb3\xcd\xda\xb6\x0f\x77\x96\x1a\x5a\x9e\x2d\x40\x9b\x87\xa2\xaa\x25\x3c\xe1\x4c\x3c\x2e\x6f\xe2\xf0\x6c\x90\x24\x47\xa9\xcd\x35\x05\x50\x7c\x04\xc2\xcd\x91\x1e\x81\x3e\x4e\xa0\xdb\x50\x1d\x6c\xa9\x4d\x88\x91\xb3\x8c\x1c\xfc\x40\x8a\x7a\x41\xb4\xe4\xc4\xd4\xc3\x8e\x29\xc0\xab\x2b\xc2\x78\x51\x2d\x59\x64\xe5\xe1\x50\x80\x8e\x99\xe6\xa0\xc3\xa8\x5f\xfb\x6a\xf3\x14\xd9\x09\x30\xb4\xdc\x94\xea\xd2\x18\xf5\x5f\x86\xa4\x7d\x6f\x27\xd9\x01\xfd\x72\x57\x35\x92\x13\xee\x57\xfe\xc7\x79\xbc\x1d\xe6\xd6\x61\x76\xed\x3f\xe9\x71\x18\x56\x10\x77\xb4\x92\x11\x5a\xd4\xbd\x08\x7a\x44\xaf\x17\xd0\xba\xb0\x4f\x06\xc5\xc1\x05\x76\x00\x3c\xc8\xc0\x63\x21\xfd\xaa\x7e\x63\x7e\x9f\x17\xa4\x3a\xef\x30\xc0\xc3\xcd\xd8\xf6\x42\xb7\x79\xd9\xae\xdf\xc4\x4a\x38\xc2\x19\xd1\xe0\x77\xf6\xc9\xc6\xad\xa5\xc6\xd4\xe9\x53\xa0\x4d\xb8\x70\xff\x36\x89\x3b\x82\x3a\x4a\x33\xbc\xc7\xf3\x90\xb2\x6b\x59\xce\x9d\x1b\xd9\xfa\xab\xdb\xd7\x6c\x41\x55\xb7\x42\xef\xc6\x8a\x32\x42\xd8\x0e\xa6\x24\x9a\x9f\xd2\x34\x5d\xe2\xd4\xa5\x32\xa8\x16\x1f\xf6\x51\x7d\x75\x07\x21\xbd\x4e\xf3\x35\x11\xa5\x1c\xd0\x8e\xd6\x8b\x09\x03\x87\xa2\xe7\x71\x27\x81\x7c\xc7\x99\x3e\x42\x3a\x07\x07\xa5\x91\x54\xf2\x30\xc7\x70\x8e\x89\xc2\x9d\x61\xa2\x11\xbb\xaa\x88\x53\xc8\x4e\x8c\xc3\xa1\xc7\xf1\x4e\x4a\x0e\x44\x74\x12\x05\x02\x49\x13\x29\xf8\x39\x00\x52\x1b\x82\xde\xf1\x85\x06\x9a\x23\x33\xe7\x44\x2a\xb3\x78\xf9\xa8\x8f\x59\xa2\xd9\x77\xe8\xfa\xde\xc5\xea\x6b\x42\xdb\xde\x86\x7a\xc3\xf8\xe8\xf7\xcc\xe2\xff\x66\x66\xa1\xcf\x9a\x9a\xeb\x6a\x6b\x6d\x52\x26\x12\xa9\x40\x78\x7d\x43\x1b\xa9\x92\x03\x12\x0a\x89\x02\x64\xd2\x29\x8a\x4e\x80\x4d\x73\xac\x5a\x83\x01\x19\xcd\x0e\x82\xb8\xe3\x8e\x05\x6a\x32\xb5\xbf\x72\x5a\x60\x8c\xdf\xd9\x73\xce\x32\x36\xee\x34\x0e\xab\x0d\xa8\xd7\xaa\x5a\xcd\x5d\xa2\x4d\x94\x67\x41\x21\x7b\xa2\x43\x98\x6e\x10\x02\x3a\x83\x23\xc1\x19\xa9\xa3\x74\xcc\xfd\x48\x7e\x72\xf5\x0d\xce\x7d\x75\x8e\xd5\x4b\x7a\xeb\x7a\x23\x5b\x27\xfc\xac\xd2\xab\xbf\x8d\xed\x68\xf5\xe3\x76\xaa\x5c\x7b\x9b\xb8\x12\x46\xe8\xa9\x06\xa4\x05\x1d\x9e\x0f\x47\xbf\x44\x84\xee\xe8\xa8\x04\x77\xe8\x26\x20\x8e\xd7\x2b\x05\xc6\xce\xd7\x8e\xfa\xc1\x15\x81\x85\x43\xa5\xd0\x4c\x1b\x10\xd4\x3d\x88\x75\x22\xed\xd8\xe0\x94\x63\x28\x94\xe9\xbe\x2b\xac\xeb\x2a\xa1\xc8\x61\x7c\x14\xe3\xee\x4d\x82\x7d\xb5\xbe\x3a\xf0\x53\x58\x11\x92\x4a\x35\xa2\x9a\x70\x36\xe6\xa6\xd9\xde\xe8\x62\xa2\x0e\x1d\x0b\x19\x4f\x12\x1f\x8b\x84\x94\x32\x77\xe4\x58\xf5\x50\x66\x5c\x3e\xe8\x0d\x05\x1b\x02\xae\x13\x73\x1b\xd4\x7b\x0b\x61\xfa\xf4\xbe\x06\x1a\x3d\x59\x67\x9a\xec\x7a\x07\x18\xae\x44\x5b\x64\x06\x3c\xb9\xf3\xbd\xbf\x60\x40\x30\xc8\x7a\x27\x11\x4d\x29\x65\x67\x7c\xd0\xef\x73\x5e\x6f\x58\x06\x32\x77\x06\xa7\x90\x6a\x89\xa0\x99\x5f\x6f\xad\x6c\x33\xad\xe9\xc5\xd6\x3d\x07\x8f\x09\x59\x90\x7d\x0b\x7a\xb0\xce\xb5\xaa\x29\x80\xd7\x4c\x42\xd2\x23\x88\xb4\x3c\x71\x09\xca\xa5\x08\x8a\x33\x4a\xb4\xaf\x5e\xb9\x61\xe6\x9c\xab\x94\x18\x48\xea\xeb\x34\x73\x2a\xc4\x89\xd2\x50\x11\x24\x9c\x03\x67\x3a\x0b\x29\xb5\xe2\x14\x38\x71\xe6\x1a\xaf\xdd\x94\xe8\x7b\xc2\x78\x8e\x90\x10\x1a\x30\x9f\xaf\x35\x25\x98\x91\xce\xe0\x15\xb6\x64\x46\x9e\x93\x66\xd9\x12\xc4\xe3\xb5\x25\x92\xc4\xd4\x5d\x6a\xad\x0b\xbb\xc8\x33\x47\xb1\x53\xf9\xc5\x66\xcf\x50\x9b\xaa\x27\x96\xaa\xfe\xd5\x0d\xea\x2f\xa3\x73\x86\xd0\xd1\xb4\x65\x75\x55\x55\x32\xaf\x61\x98\x30\x87\x4b\xfb\x31\x62\x9d\xcd\x8a\x03\x89\x21\xe8\x22\xa0\xb6\x27\x07\x5e\xfc\x45\xa5\x50\x85\x24\xc9\x59\x55\x08\x2d\x21\x0a\x2a\x45\xb5\x8f\x10\x2b\xbd\xd1\x2d\x0a\x1b\x2d\xda\xb8\x4c\x19\x6f\x04\x29\x11\x9e\x98\x48\xe5\xd3\xfc\xe8\xbb\x80\xb4\x15\x27\x14\x7a\x11\xfb\x56\x41\x6b\x83\x84\x09\x33\xfb\x44\xad\x2f\x16\x85\xb0\x07\x04\x31\xf4\x88\x68\xba\xb3\x89\xc6\xbb\x1b\x1f\x6f\x7e\x0e\x6b\x08\xad\x8a\x12\xff\x0d\x06\x9b\xb7\x2a\xff\x86\x1a\xb1\x75\x77\x4f\x76\x6f\xe1\xbc\xd5\xe1\x58\x46\xa7\x2a\xf7\x9e\xce\x65\x90\xc9\xe9\x2b\x2b\x37\xdc\x1a\xf7\xb1\xd8\x80\x2d\x50\xbd\x04\x1d\xe7\xd6\x50\x89\x54\xcb\xcf\x93\xfc\x47\xb6\x5b\xff\x34\x83\x29\x92\x2d\x15\x43\x82\x0f\xb8\x63\x67\xf9\x14\xbd\x83\xe8\x90\xef\x44\xd8\x1d\xcf\x77\x16\x1d\xec\x8a\xb9\xbe\xc4\x32\xa2\xd5\x87\xb6\x13\x5b\xb7\xb2\xda\x06\xab\x78\xf4\x06\xc9\x72\xfb\x2f\x9b\xc2\xfe\x10\xd8\xd5\x3d\x12\x63\x08\x3d\x06\x35\x9a\x33\xeb\xfd\x1b\xe2\xd0\x60\x1c\xe2\x0c\x43\x35\xd4\x02\x51\x28\xe4\x4a\xcf\xff\x46\xa4\xfa\xd5\xed\xfa\xe7\xd9\x60\xfd\x71\x8b\xf7\x03\x8a\x12\xea\xea\x5c\x1f\x62\x79\x01\xd7\x58\xdf\x81\x5e\xdf\x58\x5d\x83\x64\xe8\x54\x57\x0d\xf5\x5b\x5d\x6f\xad\xae\xde\xe1\xa2\xa5\xb6\xe1\x54\x71\x4a\x92\xc1\x37\xa0\x6a\x8c\x6d\x77\x1b\x7d\x30\xc7\x27\xa5\xdd\x1a\x6a\xea\xea\x41\x03\x32\x32\xdb\xee\x2d\x5a\x0b\x71\x9a\xf3\x05\xf3\xc7\xdd\x87\x89\x4a\x71\xea\xa6\xe2\x2b\x95\x58\x0b\x5c\xeb\x70\xeb\xb4\xd7\x84\x37\xd2\x1d\x7e\x11\x37\x1e\x23\x1a\xfc\xc1\xf7\x71\x05\x9f\xe2\x3c\x98\x7a\xff\xe8\x1e\xe4\x55\xdf\xb6\x6d\x3b\xf2\xe9\x81\x54\x17\x84\xad\x84\xbd\xb5\xe7\x12\xa3\x5f\x34\xb8\xbe\x9a\xeb\x1f\x23\x36\x5f\xaf\x8d\xdc\x6c\x58\xd9\x7f\xcb\xaf\x11\x57\x2f\xab\xff\x06\x00\x00\xff\xff\x9b\x23\x27\xbe\xf7\x3d\x00\x00") +var _dataConfig_schema_v35Json = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xec\x1b\xc9\x72\xec\xb6\xf1\x3e\x5f\xc1\x82\x7d\xb3\x16\x57\xc5\x49\x55\xde\x2d\xc7\x9c\x92\x73\x54\xf3\x58\x18\xb0\x87\x03\x0b\x04\x60\x00\x1c\x69\xfc\x4a\xff\x9e\xe2\x2a\x90\xc4\x46\x0d\xf5\x24\x27\x3e\x49\x43\x76\x37\x7a\x43\x6f\x00\xbf\xed\xb2\x0c\xfd\xa8\xc9\x09\x2a\x8c\xbe\x64\xe8\x64\x8c\xfc\x72\x7f\xff\xab\x16\xfc\xb6\x7b\x7a\x27\x54\x79\x5f\x28\x7c\x34\xb7\x3f\xff\x72\xdf\x3d\xfb\x01\xdd\x34\x78\xb4\x68\x50\x88\xe0\x47\x5a\xe6\xdd\x9b\xfc\xfc\x97\xbb\xbf\xde\x35\xe8\x1d\x88\xb9\x48\x68\x80\xc4\xe1\x57\x20\xa6\x7b\xa6\xe0\xb7\x9a\x2a\x68\x90\x1f\xd0\x19\x94\xa6\x82\xa3\xfd\xcd\xae\x79\x27\x95\x90\xa0\x0c\x05\x8d\xbe\x64\x0d\x73\x59\x36\x82\x0c\x0f\x2c\xb2\xda\x28\xca\x4b\xd4\x3e\x7e\x69\x29\x64\x19\xd2\xa0\xce\x94\x58\x14\x46\x56\x7f\xb8\x7f\xa5\x7f\x3f\x82\xdd\xcc\xa9\x5a\xcc\xb6\xcf\x25\x36\x06\x14\xff\xf7\x92\xb7\xf6\xf5\xd7\x07\x7c\xfb\xfb\x3f\x6e\xff\xf3\xf3\xed\xdf\xef\xf2\xdb\xfd\x4f\x3f\x4e\x5e\x37\xfa\x55\x70\xec\x96\x2f\xe0\x48\x39\x35\x54\xf0\x71\x7d\x34\x42\xbe\xf4\xff\xbd\x8c\x0b\xe3\xa2\x68\x81\x31\x9b\xac\x7d\xc4\x4c\xc3\x54\x66\x0e\xe6\x49\xa8\xc7\x98\xcc\x23\xd8\x07\xc9\xdc\xaf\xef\x90\x79\x2a\xce\x59\xb0\xba\x8a\x5a\x70\x80\xfa\x20\x61\xba\xe5\xb7\xb1\x9f\x06\xa2\xc0\xc4\x5d\xb6\x83\xfa\x30\x8f\x6d\x96\xdf\x46\xe0\x2e\x6a\xc4\x04\x1e\xa0\x3e\x48\xe0\x6e\xf9\xeb\x04\xde\x0d\x42\xbb\x79\x44\x5f\x9f\x6f\x9b\xbf\x2f\x2d\xcd\x20\xbd\x8e\x8a\xc5\x5f\x2b\xc4\x24\xe6\xb9\xd4\xe9\x8a\x39\x7e\x7d\x8e\x0a\xf5\x68\xb2\x00\xc9\xc4\xa5\xe5\xdc\xad\xb3\x0e\xa0\x02\x6e\xd0\xa8\xa6\x2c\x43\x87\x9a\xb2\x62\xae\x75\xc1\xe1\x5f\x0d\x89\x07\xeb\x61\x96\x7d\x9b\x87\x77\x8b\x4e\xfb\x7e\xf2\xcb\xef\x14\xe3\x7b\x8f\x2c\xe3\x7b\x22\xb8\x81\x67\xd3\x0a\x15\x5e\xba\x53\x81\x20\x8f\xa0\x8e\x94\x41\x2a\x06\x56\x9d\xa7\x7b\x54\xc6\xa8\x36\xb9\x50\x79\x41\x89\x71\xe2\x33\x7c\x00\x76\x15\x05\x82\xc9\x09\xf2\xa3\x12\x55\x94\xca\x31\xef\x24\xd1\x4e\x42\x43\x04\x4f\x94\xdc\x60\x55\x82\x5b\xb3\x33\xe0\x05\x76\x7c\x6f\x8d\xa8\xd6\xaf\xfd\xce\x41\x10\x11\x2c\x73\x5c\x14\x13\x3e\xb0\x52\xf8\x82\x6e\x32\x44\x0d\x54\xda\x2d\x50\x86\x6a\x4e\x7f\xab\xe1\x9f\x3d\x88\x51\x35\xcc\xe9\x16\x4a\xc8\xed\x09\x97\x4a\xd4\x32\x97\x58\x35\x1b\x29\xac\x6c\x44\x44\x55\x61\xbe\xd5\xee\x5a\x23\x47\x82\xe6\x17\x71\x3e\xb3\xb7\x6c\xbf\x86\xfd\x6a\x5c\x6d\xc2\x96\x47\x9a\xb8\x3c\xcb\x78\x11\x8f\x18\xf1\x98\xd1\x84\x5c\x51\x2b\x92\x1a\x02\xc2\x5b\xc1\x09\x5f\xd3\x22\x1d\xb8\x5c\x03\x5c\x89\x62\xca\x37\xaf\xab\x03\xa8\xc5\x96\x9c\xee\xac\xe5\xef\xfd\xce\xf5\x66\x66\x7d\x83\x29\x07\x95\x73\x5c\xc5\x74\x85\x88\x82\x02\xb8\xa1\x98\xe5\x5a\x02\x99\x80\x0f\x96\x0a\x58\x06\x25\x85\x64\xa4\xa0\xa4\xda\xa8\x4b\x38\x28\xbd\xd8\x8c\x15\x20\x81\x17\x3a\xef\x9a\x90\xf5\xd1\x13\x15\x30\x76\x24\x9b\x86\x89\x82\x87\xb2\x42\x47\xa6\xc9\x0b\x0d\x6f\x68\x86\x98\x6b\xc0\x8a\x9c\xde\x88\x2f\x2a\x4c\x79\x8a\x51\x81\x1b\x75\x91\x82\x76\x61\xec\xd3\xc5\x27\xe0\xe7\x7c\xf4\x9b\xd5\x6a\x00\x7e\xa6\x4a\xf0\x6a\x08\xd2\x69\xd9\xd9\xc2\x7f\x96\x42\xc3\xf5\xc1\xb1\xc7\x78\x18\x04\xbf\x19\xf7\xf4\x7e\xaa\x3d\x74\x14\xaa\xc2\x0d\xb3\xc3\xda\xf6\x1e\x9e\x2c\xb5\xf4\x3c\x5b\x81\xb6\x0c\x4d\x55\x8b\x59\xce\x28\x7f\xdc\xde\xc5\xe1\xd9\x28\x9c\x9f\x84\x36\x6f\x29\x80\xd0\x09\x30\x33\x27\x72\x02\xf2\x18\x40\xb7\xa1\x26\xd8\x42\x9b\x14\x27\xa7\x15\x2e\xe3\x40\x92\x44\x41\xb4\x60\xd8\xf4\xc3\x8e\x10\xe0\x9b\x2b\x42\xb4\xa9\x95\x2c\xb2\xa2\x2c\x1b\x50\x9f\x6b\x2e\x3a\x8c\xfe\x75\xac\x36\x2f\x14\x3d\x83\x4a\x2d\x37\x85\x7c\x6d\x8c\xe6\x2f\x53\xd2\x7e\xb4\x93\x9c\x80\x7e\xbd\xeb\x1a\xc9\xc0\xf6\x6b\xff\x63\x0c\xed\x97\xb9\x75\x99\x5d\xe7\x4f\x66\x12\xa6\x15\xc4\x13\xab\x54\x98\x34\x75\xaf\x02\xed\xb1\xeb\x2b\x68\x5f\xd8\xe7\x8b\xe2\xe0\x15\x76\x01\xbc\xc8\xc0\xbe\x90\xfe\xa6\x7e\x63\x7d\x9f\x97\x64\xba\xe8\x30\x20\x22\x8d\x8f\xbd\x54\x36\x5f\xd9\x8d\xbb\x58\x0b\x87\x19\xc5\x1a\xe2\x9b\x3d\xd8\xb8\x8d\xd4\xa8\x3c\xff\x92\xe8\x13\x2e\xdc\xbf\x05\x71\x3d\xa8\x5e\x9a\xe9\x3d\x5e\x84\x94\x5d\xcb\x32\xe6\x64\x64\x1f\xaf\x6e\xdf\xb3\x05\x95\xd3\x0a\x7d\x1a\x2b\xda\x08\x61\x6f\x30\x29\x94\xf9\x2e\x4d\xd3\x6b\x9c\x7a\xad\x0c\xba\xc5\x97\x7d\xd4\xdc\xdc\x49\x48\xef\xd3\x7c\x05\xa2\x94\x03\xda\xd1\x7a\x51\x6e\xa0\x6c\x7a\x1e\x77\x12\xa8\x0f\x8c\xea\x13\x14\x6b\x70\x94\x30\x82\x08\x96\xb6\x31\x9c\x63\xa2\xf4\xcd\x10\x68\xc4\xde\x54\xc4\x49\x45\xcf\x94\x41\x39\x93\xf8\x20\x04\x03\xcc\x27\x89\x42\x01\x2e\x72\xc1\xd9\x25\x01\x52\x1b\xac\xa2\xe3\x0b\x0d\xa4\x56\xd4\x5c\x72\x21\xcd\xe6\xe5\xa3\x3e\x55\xb9\xa6\xbf\xc3\x74\xef\xbd\x7a\x7d\x4f\x68\x3f\x63\x68\x36\x8c\xcf\xfe\x9c\x59\xfc\xdf\xcc\x2c\xf4\x45\x13\xf3\xb6\xda\x5a\x9b\x82\xf2\x5c\x48\xe0\xd1\xbd\xa1\x8d\x90\x79\xa9\x30\x81\x5c\x82\xa2\xc2\xa9\x8a\x49\x80\x2d\x6a\xd5\xb5\x06\x0b\x32\x9a\x96\x1c\xbb\xe3\x8e\x05\x6a\x2a\x79\x7c\xe3\xb4\xc0\x98\xf8\x66\xaf\x19\xad\xa8\x7f\xd3\x38\xbc\x36\xa1\x5e\xeb\x6a\x35\x77\x89\x16\x28\xcf\x92\x42\x76\xa0\x43\x08\x37\x08\x09\x9d\xc1\x09\xab\x15\xa9\xa3\xdd\x98\x47\x4f\x7e\x72\xf5\x0d\x4e\xbe\x26\xc7\xea\x2d\xbd\x9b\x9e\x91\xbd\x13\x7e\x55\xe9\x35\x67\x63\xef\xad\x7e\xdc\x9b\xaa\xd6\xd1\x26\xae\x85\xe1\x3a\xd4\x80\x8c\xa0\xcb\xf3\xe1\xec\x0f\x11\xa1\x27\x36\x6a\xc1\x1d\xb6\x49\x88\xe3\xfd\x4a\x89\xb1\xf3\xbd\xa3\x7e\x72\x45\x60\xe1\x10\xc1\x35\xd5\x06\x38\x71\x0f\x62\x9d\x48\x07\xba\x38\xe5\x58\x2a\x25\xdc\x77\xa5\x75\x5d\x2d\x14\x2e\xfd\xa3\x18\x77\x6f\x92\xbc\x57\xfb\xab\x03\xdf\x45\x14\x2e\x88\x90\x1e\xd3\xa4\x8b\xb1\x36\xcd\xce\x46\x17\x81\x3a\xd4\x17\x32\x9e\x84\x7a\x6c\x12\x52\x41\xdd\x91\x63\x37\x43\x59\x71\xf9\x60\x36\x14\x1c\x08\xb8\x4e\xcc\x6d\xd0\xe8\x2d\x84\xf0\xe9\x7d\x0f\xe4\x3d\x59\xa7\x1a\x1f\x66\x07\x18\xae\x44\xdb\x64\x06\x75\x76\xe7\xfb\x78\xc1\xa0\xc0\x28\x3a\x3b\x89\x18\x4a\x29\x3b\xe3\x83\xfe\x9c\xf3\x7a\x43\x2b\x10\xb5\x33\x38\xa5\x54\x4b\x58\x99\xf5\xf5\xd6\xce\x76\xd3\x9e\x1e\xb2\xee\x39\x44\x5c\xc8\x82\x9c\x7b\xd0\x83\x75\xae\xd5\x4d\x01\xa2\x6e\x92\x92\x1e\x81\x17\xed\x89\x4b\x52\x2e\x55\x20\x19\x25\x58\xc7\xea\x95\x2b\x66\xce\xb5\x2c\xb0\x81\xbc\xbf\x4e\xb3\xa6\x42\x0c\x94\x86\x12\x2b\xcc\x18\x30\xaa\xab\x94\x52\x0b\x15\xc0\xb0\x33\xd7\x44\xfd\xa6\x45\x3f\x62\xca\x6a\x05\x39\x26\x09\xf3\xf9\xde\x52\x9c\x1a\xe1\x0c\x5e\x69\x4b\x56\xf8\x39\x1f\x96\x6d\x41\x22\xbb\xb6\x45\x12\xaa\x70\x97\x5a\x37\x8d\x5f\xd4\x95\xa3\xd8\xe9\xf6\xc5\xed\x91\x2a\x6d\xba\x9e\x58\xc8\xfe\xd7\x34\xa8\xbf\x78\xe7\x0c\xa9\xa3\x69\xcb\xeb\xba\xaa\x64\x5d\xc3\x10\x70\x07\x67\xfb\x11\xa3\x18\xa3\xda\xbe\x27\xb2\x4e\x9d\x94\xa2\x0a\x2a\x11\x3b\x5a\x76\xe8\x2f\x5d\x87\x0e\xd4\x46\x97\x4d\x46\xf0\x1d\x7d\x7c\x16\x05\x38\xa0\x4b\xe0\xa0\x28\xc9\x27\xde\xe0\x89\x2e\x4b\xd8\x77\x9a\xdf\x5e\xef\xd9\x5d\x9a\x11\x8c\x76\xc5\xed\x16\xee\x4d\x04\xef\xf8\x48\x89\x3c\x57\x86\xba\x26\xee\x34\xad\x79\x25\x4d\x34\x2b\xb4\x08\x4f\x94\x17\xe2\x69\x7d\x46\xdd\x40\xdb\x92\x61\x02\xb3\x2c\x7c\xad\xa2\xb5\x51\x98\x72\xb3\xfa\x94\x74\xae\x16\xa9\xe0\x08\x0a\xf8\x32\xca\x65\xe1\x6e\x35\xf3\x77\xac\x31\xd9\xe2\x12\xf6\x10\x5a\x36\x6d\xdb\x07\x0c\xab\xaf\x35\xfe\x15\x75\xbf\x33\xdc\x84\x4a\xb7\x25\xc2\xa2\x07\x98\x5a\xcf\x61\x35\xbf\xb5\x02\x57\x99\x0a\xaa\x89\x02\x03\xe3\xca\xe3\x8d\xa8\xa9\x8b\x05\x3d\x21\xec\x05\xe8\xb1\xef\xa6\xa3\x81\x1a\x9d\x31\xab\x13\xa6\xaf\x6f\x3a\xaf\xf6\xb5\x7f\x09\xc8\xb3\x8f\x06\xac\xab\xa9\x21\x9b\x0e\x60\x1b\xd4\xe2\x49\x97\x13\x7a\xa8\x5c\xc8\xed\xa7\xa3\xf1\x0b\x08\xfb\xf8\x6c\x8e\x4a\x5c\x6d\x15\x3d\x93\xaf\x6b\x20\x67\x33\x90\x7d\x82\xb8\x58\x1f\x78\xda\x8d\xe5\x4f\x16\x17\xed\xfe\xaf\xbf\x92\xe5\xb1\xea\xc3\x38\x57\xb8\x19\x75\xb5\x4f\x36\xb1\xf7\x3e\xd4\x76\xfc\xb7\x23\x8e\xf9\x91\x86\x6b\x16\x82\x8d\xc1\xe4\x94\x34\x36\x59\xd9\xbd\x5e\x91\x65\x16\xc3\x3d\x67\x18\xea\xa1\x36\x88\x42\x29\x17\xd4\xfe\x37\x22\xd5\x1f\xdd\xaf\xbf\x9f\x0f\xf6\x9f\x6a\x45\x3f\x07\x6a\xa1\xa2\x73\xcd\x6b\x3c\x2f\xe1\x52\xf6\x27\xb0\xeb\x07\x9b\x6b\x91\x0c\x9d\xe6\xea\xa1\xfe\x34\xd7\x47\x9b\x6b\x76\x54\x6e\x99\x6d\x39\x23\x0f\x69\x32\xf9\x3e\x5f\x8f\xb1\x9f\xb2\x31\x07\x73\x7c\x20\xed\xeb\x4e\xbc\x4c\xf9\x4e\x6a\x66\x8b\xf6\x4a\x0c\x4b\xbe\x61\xfe\xb8\xfb\x29\x50\x29\x86\xee\xdd\xbe\x53\x89\xb5\xc1\x25\x25\xb7\x4d\x67\xe3\x87\x41\xbb\xcb\xef\x3b\xfd\x31\x62\xc0\x5f\x7c\xed\xd9\xc8\xc9\x2f\x8b\x33\x9c\x6f\xd3\x63\xe9\xee\x4b\xcd\xfd\x44\x3f\x33\x90\xee\xba\xbb\x95\xb0\xf7\x49\x4d\xad\xeb\x1b\xd0\xf9\xa1\xf8\xf0\x2d\xa6\xe7\x9e\xce\xce\xfe\xdb\x7e\x5b\xbb\x7b\xd9\xfd\x37\x00\x00\xff\xff\x8f\xfe\xaa\xe6\xc5\x40\x00\x00") func dataConfig_schema_v35JsonBytes() ([]byte, error) { return bindataRead( diff --git a/cli/compose/schema/data/config_schema_v3.5.json b/cli/compose/schema/data/config_schema_v3.5.json index bcd3b11091..d3352c7d52 100644 --- a/cli/compose/schema/data/config_schema_v3.5.json +++ b/cli/compose/schema/data/config_schema_v3.5.json @@ -354,8 +354,23 @@ "resources": { "type": "object", "properties": { - "limits": {"$ref": "#/definitions/resource"}, - "reservations": {"$ref": "#/definitions/resource"} + "limits": { + "type": "object", + "properties": { + "cpus": {"type": "string"}, + "memory": {"type": "string"} + }, + "additionalProperties": false + }, + "reservations": { + "type": "object", + "properties": { + "cpus": {"type": "string"}, + "memory": {"type": "string"}, + "generic_resources": {"$ref": "#/definitions/generic_resources"} + }, + "additionalProperties": false + } }, "additionalProperties": false }, @@ -390,14 +405,23 @@ "additionalProperties": false }, - "resource": { - "id": "#/definitions/resource", - "type": "object", - "properties": { - "cpus": {"type": "string"}, - "memory": {"type": "string"} - }, - "additionalProperties": false + "generic_resources": { + "id": "#/definitions/generic_resources", + "type": "array", + "items": { + "type": "object", + "properties": { + "discrete_resource_spec": { + "type": "object", + "properties": { + "kind": {"type": "string"}, + "value": {"type": "number"} + }, + "additionalProperties": false + } + }, + "additionalProperties": false + } }, "network": { diff --git a/cli/compose/types/types.go b/cli/compose/types/types.go index 27177718e2..c68187efe1 100644 --- a/cli/compose/types/types.go +++ b/cli/compose/types/types.go @@ -217,8 +217,24 @@ type Resources struct { // Resource is a resource to be limited or reserved type Resource struct { // TODO: types to convert from units and ratios - NanoCPUs string `mapstructure:"cpus"` - MemoryBytes UnitBytes `mapstructure:"memory"` + NanoCPUs string `mapstructure:"cpus"` + MemoryBytes UnitBytes `mapstructure:"memory"` + GenericResources []GenericResource `mapstructure:"generic_resources"` +} + +// GenericResource represents a "user defined" resource which can +// only be an integer (e.g: SSD=3) for a service +type GenericResource struct { + DiscreteResourceSpec *DiscreteGenericResource `mapstructure:"discrete_resource_spec"` +} + +// DiscreteGenericResource represents a "user defined" resource which is defined +// as an integer +// "Kind" is used to describe the Kind of a resource (e.g: "GPU", "FPGA", "SSD", ...) +// Value is used to count the resource (SSD=5, HDD=3, ...) +type DiscreteGenericResource struct { + Kind string + Value int64 } // UnitBytes is the bytes type