cli: Add options for Raft snapshotting

Add the following options to "swarm init" and "swarm update":

- --max-snapshots: Retain this many old Raft snapshots in addition
  to the latest one

- --snapshot-interval: Number of log entries between Raft snapshots

These options already existed in SwarmKit and the Docker API but were
never exposed in the CLI. I'm adding them here to fix this oversight.

--max-snapshots may be useful for debugging purposes and more
conservative users who want to store rolling backups of old versions of
the Raft state.

--snapshot-interval is most useful for performance tuning. The default
value of 10000 may not be ideal for some setups.

There is also a LogEntriesForSlowFollowers option that is not exposed. I
decided not to expose it along with these others because I don't think
it's generally useful (and I'm not sure what I would call the CLI flag).
But if people want, I can expose it for the sake of completeness.

Signed-off-by: Aaron Lehmann <aaron.lehmann@docker.com>

command/swarm/opts.go

@@ -24,6 +24,8 @@ const (
 	flagToken               = "token"
 	flagTaskHistoryLimit    = "task-history-limit"
 	flagExternalCA          = "external-ca"
+	flagMaxSnapshots        = "max-snapshots"
+	flagSnapshotInterval    = "snapshot-interval"
 )
 
 type swarmOptions struct {
@@ -31,6 +33,8 @@ type swarmOptions struct {
 	dispatcherHeartbeat time.Duration
 	nodeCertExpiry      time.Duration
 	externalCA          ExternalCAOption
+	maxSnapshots        uint64
+	snapshotInterval    uint64
 }
 
 // NodeAddrOption is a pflag.Value for listening addresses
@@ -167,11 +171,11 @@ func addSwarmFlags(flags *pflag.FlagSet, opts *swarmOptions) {
 	flags.DurationVar(&opts.dispatcherHeartbeat, flagDispatcherHeartbeat, time.Duration(5*time.Second), "Dispatcher heartbeat period")
 	flags.DurationVar(&opts.nodeCertExpiry, flagCertExpiry, time.Duration(90*24*time.Hour), "Validity period for node certificates")
 	flags.Var(&opts.externalCA, flagExternalCA, "Specifications of one or more certificate signing endpoints")
+	flags.Uint64Var(&opts.maxSnapshots, flagMaxSnapshots, 0, "Number of additional Raft snapshots to retain")
+	flags.Uint64Var(&opts.snapshotInterval, flagSnapshotInterval, 10000, "Number of log entries between Raft snapshots")
 }
 
-func (opts *swarmOptions) ToSpec(flags *pflag.FlagSet) swarm.Spec {
+func (opts *swarmOptions) mergeSwarmSpec(spec *swarm.Spec, flags *pflag.FlagSet) {
-	spec := swarm.Spec{}
-
 	if flags.Changed(flagTaskHistoryLimit) {
 		spec.Orchestration.TaskHistoryRetentionLimit = &opts.taskHistoryLimit
 	}
@@ -184,5 +188,16 @@ func (opts *swarmOptions) ToSpec(flags *pflag.FlagSet) swarm.Spec {
 	if flags.Changed(flagExternalCA) {
 		spec.CAConfig.ExternalCAs = opts.externalCA.Value()
 	}
+	if flags.Changed(flagMaxSnapshots) {
+		spec.Raft.KeepOldSnapshots = &opts.maxSnapshots
+	}
+	if flags.Changed(flagSnapshotInterval) {
+		spec.Raft.SnapshotInterval = opts.snapshotInterval
+	}
+}
+
+func (opts *swarmOptions) ToSpec(flags *pflag.FlagSet) swarm.Spec {
+	var spec swarm.Spec
+	opts.mergeSwarmSpec(&spec, flags)
 	return spec
 }

command/swarm/update.go

@@ -39,10 +39,7 @@ func runUpdate(dockerCli *command.DockerCli, flags *pflag.FlagSet, opts swarmOpt
 		return err
 	}
 
-	err = mergeSwarm(&swarm, flags)
+	opts.mergeSwarmSpec(&swarm.Spec, flags)
-	if err != nil {
-		return err
-	}
 
 	err = client.SwarmUpdate(ctx, swarm.Version, swarm.Spec, updateFlags)
 	if err != nil {
@@ -53,31 +50,3 @@ func runUpdate(dockerCli *command.DockerCli, flags *pflag.FlagSet, opts swarmOpt
 
 	return nil
 }
-
-func mergeSwarm(swarm *swarm.Swarm, flags *pflag.FlagSet) error {
-	spec := &swarm.Spec
-
-	if flags.Changed(flagTaskHistoryLimit) {
-		taskHistoryRetentionLimit, _ := flags.GetInt64(flagTaskHistoryLimit)
-		spec.Orchestration.TaskHistoryRetentionLimit = &taskHistoryRetentionLimit
-	}
-
-	if flags.Changed(flagDispatcherHeartbeat) {
-		if v, err := flags.GetDuration(flagDispatcherHeartbeat); err == nil {
-			spec.Dispatcher.HeartbeatPeriod = v
-		}
-	}
-
-	if flags.Changed(flagCertExpiry) {
-		if v, err := flags.GetDuration(flagCertExpiry); err == nil {
-			spec.CAConfig.NodeCertExpiry = v
-		}
-	}
-
-	if flags.Changed(flagExternalCA) {
-		value := flags.Lookup(flagExternalCA).Value.(*ExternalCAOption)
-		spec.CAConfig.ExternalCAs = value.Value()
-	}
-
-	return nil
-}

command/system/info.go

@@ -114,6 +114,9 @@ func prettyPrintInfo(dockerCli *command.DockerCli, info types.Info) error {
 			fmt.Fprintf(dockerCli.Out(), "  Task History Retention Limit: %d\n", taskHistoryRetentionLimit)
 			fmt.Fprintf(dockerCli.Out(), " Raft:\n")
 			fmt.Fprintf(dockerCli.Out(), "  Snapshot Interval: %d\n", info.Swarm.Cluster.Spec.Raft.SnapshotInterval)
+			if info.Swarm.Cluster.Spec.Raft.KeepOldSnapshots != nil {
+				fmt.Fprintf(dockerCli.Out(), "  Number of Old Snapshots to Retain: %d\n", *info.Swarm.Cluster.Spec.Raft.KeepOldSnapshots)
+			}
 			fmt.Fprintf(dockerCli.Out(), "  Heartbeat Tick: %d\n", info.Swarm.Cluster.Spec.Raft.HeartbeatTick)
 			fmt.Fprintf(dockerCli.Out(), "  Election Tick: %d\n", info.Swarm.Cluster.Spec.Raft.ElectionTick)
 			fmt.Fprintf(dockerCli.Out(), " Dispatcher:\n")