Merge pull request #28938 from elifa/master

Grace period option to health checks.

command/container/opts.go

@@ -113,6 +113,7 @@ type containerOptions struct {
 	healthCmd          string
 	healthInterval     time.Duration
 	healthTimeout      time.Duration
+	healthStartPeriod  time.Duration
 	healthRetries      int
 	runtime            string
 	autoRemove         bool
@@ -232,6 +233,8 @@ func addFlags(flags *pflag.FlagSet) *containerOptions {
 	flags.DurationVar(&copts.healthInterval, "health-interval", 0, "Time between running the check (ns|us|ms|s|m|h) (default 0s)")
 	flags.IntVar(&copts.healthRetries, "health-retries", 0, "Consecutive failures needed to report unhealthy")
 	flags.DurationVar(&copts.healthTimeout, "health-timeout", 0, "Maximum time to allow one check to run (ns|us|ms|s|m|h) (default 0s)")
+	flags.DurationVar(&copts.healthStartPeriod, "health-start-period", 0, "Start period for the container to initialize before starting health-retries countdown (ns|us|ms|s|m|h) (default 0s)")
+	flags.SetAnnotation("health-start-period", "version", []string{"1.29"})
 	flags.BoolVar(&copts.noHealthcheck, "no-healthcheck", false, "Disable any container-specified HEALTHCHECK")
 
 	// Resource management
@@ -464,6 +467,7 @@ func parse(flags *pflag.FlagSet, copts *containerOptions) (*containerConfig, err
 	haveHealthSettings := copts.healthCmd != "" ||
 		copts.healthInterval != 0 ||
 		copts.healthTimeout != 0 ||
+		copts.healthStartPeriod != 0 ||
 		copts.healthRetries != 0
 	if copts.noHealthcheck {
 		if haveHealthSettings {
@@ -486,12 +490,16 @@ func parse(flags *pflag.FlagSet, copts *containerOptions) (*containerConfig, err
 		if copts.healthRetries < 0 {
 			return nil, errors.Errorf("--health-retries cannot be negative")
 		}
+		if copts.healthStartPeriod < 0 {
+			return nil, fmt.Errorf("--health-start-period cannot be negative")
+		}
 
 		healthConfig = &container.HealthConfig{
-			Test:     probe,
+			Test:        probe,
-			Interval: copts.healthInterval,
+			Interval:    copts.healthInterval,
-			Timeout:  copts.healthTimeout,
+			Timeout:     copts.healthTimeout,
-			Retries:  copts.healthRetries,
+			StartPeriod: copts.healthStartPeriod,
+			Retries:     copts.healthRetries,
 		}
 	}
 

command/container/opts_test.go

@@ -501,8 +501,8 @@ func TestParseHealth(t *testing.T) {
 	checkError("--no-healthcheck conflicts with --health-* options",
 		"--no-healthcheck", "--health-cmd=/check.sh -q", "img", "cmd")
 
-	health = checkOk("--health-timeout=2s", "--health-retries=3", "--health-interval=4.5s", "img", "cmd")
+	health = checkOk("--health-timeout=2s", "--health-retries=3", "--health-interval=4.5s", "--health-start-period=5s", "img", "cmd")
-	if health.Timeout != 2*time.Second || health.Retries != 3 || health.Interval != 4500*time.Millisecond {
+	if health.Timeout != 2*time.Second || health.Retries != 3 || health.Interval != 4500*time.Millisecond || health.StartPeriod != 5*time.Second {
 		t.Fatalf("--health-*: got %#v", health)
 	}
 }

command/service/opts.go

@@ -282,6 +282,7 @@ type healthCheckOptions struct {
 	interval      PositiveDurationOpt
 	timeout       PositiveDurationOpt
 	retries       int
+	startPeriod   PositiveDurationOpt
 	noHealthcheck bool
 }
 
@@ -301,18 +302,22 @@ func (opts *healthCheckOptions) toHealthConfig() (*container.HealthConfig, error
 		if opts.cmd != "" {
 			test = []string{"CMD-SHELL", opts.cmd}
 		}
-		var interval, timeout time.Duration
+		var interval, timeout, startPeriod time.Duration
 		if ptr := opts.interval.Value(); ptr != nil {
 			interval = *ptr
 		}
 		if ptr := opts.timeout.Value(); ptr != nil {
 			timeout = *ptr
 		}
+		if ptr := opts.startPeriod.Value(); ptr != nil {
+			startPeriod = *ptr
+		}
 		healthConfig = &container.HealthConfig{
-			Test:     test,
+			Test:        test,
-			Interval: interval,
+			Interval:    interval,
-			Timeout:  timeout,
+			Timeout:     timeout,
-			Retries:  opts.retries,
+			Retries:     opts.retries,
+			StartPeriod: startPeriod,
 		}
 	}
 	return healthConfig, nil
@@ -555,6 +560,8 @@ func addServiceFlags(flags *pflag.FlagSet, opts *serviceOptions) {
 	flags.SetAnnotation(flagHealthTimeout, "version", []string{"1.25"})
 	flags.IntVar(&opts.healthcheck.retries, flagHealthRetries, 0, "Consecutive failures needed to report unhealthy")
 	flags.SetAnnotation(flagHealthRetries, "version", []string{"1.25"})
+	flags.Var(&opts.healthcheck.startPeriod, flagHealthStartPeriod, "Start period for the container to initialize before counting retries towards unstable (ns|us|ms|s|m|h)")
+	flags.SetAnnotation(flagHealthStartPeriod, "version", []string{"1.29"})
 	flags.BoolVar(&opts.healthcheck.noHealthcheck, flagNoHealthcheck, false, "Disable any container-specified HEALTHCHECK")
 	flags.SetAnnotation(flagNoHealthcheck, "version", []string{"1.25"})
 
@@ -644,6 +651,7 @@ const (
 	flagHealthInterval          = "health-interval"
 	flagHealthRetries           = "health-retries"
 	flagHealthTimeout           = "health-timeout"
+	flagHealthStartPeriod       = "health-start-period"
 	flagNoHealthcheck           = "no-healthcheck"
 	flagSecret                  = "secret"
 	flagSecretAdd               = "secret-add"

command/service/opts_test.go

@@ -71,18 +71,20 @@ func TestUint64OptSetAndValue(t *testing.T) {
 func TestHealthCheckOptionsToHealthConfig(t *testing.T) {
 	dur := time.Second
 	opt := healthCheckOptions{
-		cmd:      "curl",
+		cmd:         "curl",
-		interval: PositiveDurationOpt{DurationOpt{value: &dur}},
+		interval:    PositiveDurationOpt{DurationOpt{value: &dur}},
-		timeout:  PositiveDurationOpt{DurationOpt{value: &dur}},
+		timeout:     PositiveDurationOpt{DurationOpt{value: &dur}},
-		retries:  10,
+		startPeriod: PositiveDurationOpt{DurationOpt{value: &dur}},
+		retries:     10,
 	}
 	config, err := opt.toHealthConfig()
 	assert.NilError(t, err)
 	assert.Equal(t, reflect.DeepEqual(config, &container.HealthConfig{
-		Test:     []string{"CMD-SHELL", "curl"},
+		Test:        []string{"CMD-SHELL", "curl"},
-		Interval: time.Second,
+		Interval:    time.Second,
-		Timeout:  time.Second,
+		Timeout:     time.Second,
-		Retries:  10,
+		StartPeriod: time.Second,
+		Retries:     10,
 	}), true)
 }
 

command/service/update.go

@@ -898,7 +898,7 @@ func updateLogDriver(flags *pflag.FlagSet, taskTemplate *swarm.TaskSpec) error {
 }
 
 func updateHealthcheck(flags *pflag.FlagSet, containerSpec *swarm.ContainerSpec) error {
-	if !anyChanged(flags, flagNoHealthcheck, flagHealthCmd, flagHealthInterval, flagHealthRetries, flagHealthTimeout) {
+	if !anyChanged(flags, flagNoHealthcheck, flagHealthCmd, flagHealthInterval, flagHealthRetries, flagHealthTimeout, flagHealthStartPeriod) {
 		return nil
 	}
 	if containerSpec.Healthcheck == nil {
@@ -909,7 +909,7 @@ func updateHealthcheck(flags *pflag.FlagSet, containerSpec *swarm.ContainerSpec)
 		return err
 	}
 	if noHealthcheck {
-		if !anyChanged(flags, flagHealthCmd, flagHealthInterval, flagHealthRetries, flagHealthTimeout) {
+		if !anyChanged(flags, flagHealthCmd, flagHealthInterval, flagHealthRetries, flagHealthTimeout, flagHealthStartPeriod) {
 			containerSpec.Healthcheck = &container.HealthConfig{
 				Test: []string{"NONE"},
 			}
@@ -928,6 +928,10 @@ func updateHealthcheck(flags *pflag.FlagSet, containerSpec *swarm.ContainerSpec)
 		val := *flags.Lookup(flagHealthTimeout).Value.(*PositiveDurationOpt).Value()
 		containerSpec.Healthcheck.Timeout = val
 	}
+	if flags.Changed(flagHealthStartPeriod) {
+		val := *flags.Lookup(flagHealthStartPeriod).Value.(*PositiveDurationOpt).Value()
+		containerSpec.Healthcheck.StartPeriod = val
+	}
 	if flags.Changed(flagHealthRetries) {
 		containerSpec.Healthcheck.Retries, _ = flags.GetInt(flagHealthRetries)
 	}

command/service/update_test.go

@@ -311,6 +311,11 @@ func TestUpdateHealthcheckTable(t *testing.T) {
 			initial:  &container.HealthConfig{Test: []string{"CMD", "cmd1"}, Retries: 10},
 			expected: &container.HealthConfig{Test: []string{"CMD", "cmd1"}},
 		},
+		{
+			flags:    [][2]string{{"health-start-period", "1m"}},
+			initial:  &container.HealthConfig{Test: []string{"CMD", "cmd1"}},
+			expected: &container.HealthConfig{Test: []string{"CMD", "cmd1"}, StartPeriod: time.Minute},
+		},
 		{
 			flags: [][2]string{{"health-cmd", "cmd1"}, {"no-healthcheck", "true"}},
 			err:   "--no-healthcheck conflicts with --health-* options",

compose/convert/service.go

@@ -255,9 +255,9 @@ func convertHealthcheck(healthcheck *composetypes.HealthCheckConfig) (*container
 		return nil, nil
 	}
 	var (
-		err               error
+		err                            error
-		timeout, interval time.Duration
+		timeout, interval, startPeriod time.Duration
-		retries           int
+		retries                        int
 	)
 	if healthcheck.Disable {
 		if len(healthcheck.Test) != 0 {
@@ -280,14 +280,21 @@ func convertHealthcheck(healthcheck *composetypes.HealthCheckConfig) (*container
 			return nil, err
 		}
 	}
+	if healthcheck.StartPeriod != "" {
+		startPeriod, err = time.ParseDuration(healthcheck.StartPeriod)
+		if err != nil {
+			return nil, err
+		}
+	}
 	if healthcheck.Retries != nil {
 		retries = int(*healthcheck.Retries)
 	}
 	return &container.HealthConfig{
-		Test:     healthcheck.Test,
+		Test:        healthcheck.Test,
-		Timeout:  timeout,
+		Timeout:     timeout,
-		Interval: interval,
+		Interval:    interval,
-		Retries:  retries,
+		Retries:     retries,
+		StartPeriod: startPeriod,
 	}, nil
 }
 

compose/types/types.go

@@ -163,11 +163,12 @@ type DeployConfig struct {
 
 // HealthCheckConfig the healthcheck configuration for a service
 type HealthCheckConfig struct {
-	Test     HealthCheckTest
+	Test        HealthCheckTest
-	Timeout  string
+	Timeout     string
-	Interval string
+	Interval    string
-	Retries  *uint64
+	Retries     *uint64
-	Disable  bool
+	StartPeriod string
+	Disable     bool
 }
 
 // HealthCheckTest is the command run to test the health of a service