Merge pull request #948 from simonferquel/k8s-watch-stack-status

K8s: more robust stack error detection on deploy
2018-06-04 10:42:42 +02:00 · 2018-06-04 10:42:42 +02:00 · eaa9149e29
parent 2014e991ee f38510b2d8
commit eaa9149e29
16 changed files with 3085 additions and 88 deletions
--- a/cli/command/stack/kubernetes/cli.go
+++ b/cli/command/stack/kubernetes/cli.go
@ -7,6 +7,7 @@ import (

 	"github.com/docker/cli/cli/command"
 	"github.com/docker/cli/kubernetes"
+	cliv1beta1 "github.com/docker/cli/kubernetes/client/clientset/typed/compose/v1beta1"
 	flag "github.com/spf13/pflag"
 	kubeclient "k8s.io/client-go/kubernetes"
 	restclient "k8s.io/client-go/rest"
@ -112,3 +113,11 @@ func (c *KubeCli) checkHostsMatch() error {
 		"         Update $DOCKER_HOST (or pass -H), or use 'kubectl config use-context' to match.\n", daemonEndpoint.Hostname(), kubeEndpoint.Hostname())
 	return nil
 }
+
+func (c *KubeCli) stacksv1beta1() (cliv1beta1.StackInterface, error) {
+	raw, err := newStackV1Beta1(c.kubeConfig, c.kubeNamespace)
+	if err != nil {
+		return nil, err
+	}
+	return raw.stacks, nil
+}
--- a/cli/command/stack/kubernetes/deploy.go
+++ b/cli/command/stack/kubernetes/deploy.go
@ -2,9 +2,12 @@ package kubernetes

 import (
 	"fmt"
+	"io"

+	"github.com/docker/cli/cli/command"
 	"github.com/docker/cli/cli/command/stack/loader"
 	"github.com/docker/cli/cli/command/stack/options"
+	"github.com/morikuni/aec"
 	"github.com/pkg/errors"
 )

@ -39,10 +42,6 @@ func RunDeploy(dockerCli *KubeCli, opts options.Deploy) error {
 	configMaps := composeClient.ConfigMaps()
 	secrets := composeClient.Secrets()
 	services := composeClient.Services()
-	pods := composeClient.Pods()
-	watcher := DeployWatcher{
-		Pods: pods,
-	}

 	if err := stacks.IsColliding(services, stack); err != nil {
 		return err
@ -61,10 +60,109 @@ func RunDeploy(dockerCli *KubeCli, opts options.Deploy) error {
 	}

 	fmt.Fprintln(cmdOut, "Waiting for the stack to be stable and running...")
+	v1beta1Cli, err := dockerCli.stacksv1beta1()
+	if err != nil {
+		return err
+	}

-	<-watcher.Watch(stack.name, stack.getServices())
-
-	fmt.Fprintf(cmdOut, "Stack %s is stable and running\n\n", stack.name)
+	pods := composeClient.Pods()
+	watcher := &deployWatcher{
+		stacks: v1beta1Cli,
+		pods:   pods,
+	}
+	statusUpdates := make(chan serviceStatus)
+	displayDone := make(chan struct{})
+	go func() {
+		defer close(displayDone)
+		display := newStatusDisplay(dockerCli.Out())
+		for status := range statusUpdates {
+			display.OnStatus(status)
+		}
+	}()

+	err = watcher.Watch(stack.name, stack.getServices(), statusUpdates)
+	close(statusUpdates)
+	<-displayDone
+	if err != nil {
+		return err
+	}
+	fmt.Fprintf(cmdOut, "\nStack %s is stable and running\n\n", stack.name)
 	return nil
+
+}
+
+type statusDisplay interface {
+	OnStatus(serviceStatus)
+}
+type metaServiceState string
+
+const (
+	metaServiceStateReady   = metaServiceState("Ready")
+	metaServiceStatePending = metaServiceState("Pending")
+	metaServiceStateFailed  = metaServiceState("Failed")
+)
+
+func metaStateFromStatus(status serviceStatus) metaServiceState {
+	switch {
+	case status.podsReady > 0:
+		return metaServiceStateReady
+	case status.podsPending > 0:
+		return metaServiceStatePending
+	default:
+		return metaServiceStateFailed
+	}
+}
+
+type forwardOnlyStatusDisplay struct {
+	o      *command.OutStream
+	states map[string]metaServiceState
+}
+
+func (d *forwardOnlyStatusDisplay) OnStatus(status serviceStatus) {
+	state := metaStateFromStatus(status)
+	if d.states[status.name] != state {
+		d.states[status.name] = state
+		fmt.Fprintf(d.o, "%s: %s\n", status.name, state)
+	}
+}
+
+type interactiveStatusDisplay struct {
+	o        *command.OutStream
+	statuses []serviceStatus
+}
+
+func (d *interactiveStatusDisplay) OnStatus(status serviceStatus) {
+	b := aec.EmptyBuilder
+	for ix := 0; ix < len(d.statuses); ix++ {
+		b = b.Up(1).EraseLine(aec.EraseModes.All)
+	}
+	b = b.Column(0)
+	fmt.Fprint(d.o, b.ANSI)
+	updated := false
+	for ix, s := range d.statuses {
+		if s.name == status.name {
+			d.statuses[ix] = status
+			s = status
+			updated = true
+		}
+		displayInteractiveServiceStatus(s, d.o)
+	}
+	if !updated {
+		d.statuses = append(d.statuses, status)
+		displayInteractiveServiceStatus(status, d.o)
+	}
+}
+
+func displayInteractiveServiceStatus(status serviceStatus, o io.Writer) {
+	state := metaStateFromStatus(status)
+	totalFailed := status.podsFailed + status.podsSucceeded + status.podsUnknown
+	fmt.Fprintf(o, "%[1]s: %[2]s\t\t[pod status: %[3]d/%[6]d ready, %[4]d/%[6]d pending, %[5]d/%[6]d failed]\n", status.name, state,
+		status.podsReady, status.podsPending, totalFailed, status.podsTotal)
+}
+
+func newStatusDisplay(o *command.OutStream) statusDisplay {
+	if !o.IsTerminal() {
+		return &forwardOnlyStatusDisplay{o: o, states: map[string]metaServiceState{}}
+	}
+	return &interactiveStatusDisplay{o: o}
 }
--- a/cli/command/stack/kubernetes/stackclient.go
+++ b/cli/command/stack/kubernetes/stackclient.go
@ -33,7 +33,7 @@ type stackV1Beta1 struct {
 	stacks composev1beta1.StackInterface
 }

-func newStackV1Beta1(config *rest.Config, namespace string) (StackClient, error) {
+func newStackV1Beta1(config *rest.Config, namespace string) (*stackV1Beta1, error) {
 	client, err := composev1beta1.NewForConfig(config)
 	if err != nil {
 		return nil, err
@ -136,7 +136,7 @@ type stackV1Beta2 struct {
 	stacks composev1beta2.StackInterface
 }

-func newStackV1Beta2(config *rest.Config, namespace string) (StackClient, error) {
+func newStackV1Beta2(config *rest.Config, namespace string) (*stackV1Beta2, error) {
 	client, err := composev1beta2.NewForConfig(config)
 	if err != nil {
 		return nil, err
--- a/cli/command/stack/kubernetes/watcher.go
+++ b/cli/command/stack/kubernetes/watcher.go
@ -1,116 +1,255 @@
 package kubernetes

 import (
-	"fmt"
+	"context"
+	"sync"
 	"time"

+	apiv1beta1 "github.com/docker/cli/kubernetes/compose/v1beta1"
 	"github.com/docker/cli/kubernetes/labels"
+	"github.com/pkg/errors"
 	apiv1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	corev1 "k8s.io/client-go/kubernetes/typed/core/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	runtimeutil "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apimachinery/pkg/watch"
+	cache "k8s.io/client-go/tools/cache"
+	podutils "k8s.io/kubernetes/pkg/api/v1/pod"
 )

+type stackListWatch interface {
+	List(opts metav1.ListOptions) (*apiv1beta1.StackList, error)
+	Watch(opts metav1.ListOptions) (watch.Interface, error)
+}
+
+type podListWatch interface {
+	List(opts metav1.ListOptions) (*apiv1.PodList, error)
+	Watch(opts metav1.ListOptions) (watch.Interface, error)
+}
+
 // DeployWatcher watches a stack deployement
-type DeployWatcher struct {
-	Pods corev1.PodInterface
+type deployWatcher struct {
+	pods   podListWatch
+	stacks stackListWatch
 }

 // Watch watches a stuck deployement and return a chan that will holds the state of the stack
-func (w DeployWatcher) Watch(name string, serviceNames []string) chan bool {
-	stop := make(chan bool)
+func (w *deployWatcher) Watch(name string, serviceNames []string, statusUpdates chan serviceStatus) error {
+	errC := make(chan error, 1)
+	defer close(errC)

-	go w.waitForPods(name, serviceNames, stop)
+	handlers := runtimeutil.ErrorHandlers

-	return stop
-}
-
-func (w DeployWatcher) waitForPods(stackName string, serviceNames []string, stop chan bool) {
-	starts := map[string]int32{}
-
-	for {
-		time.Sleep(1 * time.Second)
-
-		list, err := w.Pods.List(metav1.ListOptions{
-			LabelSelector:        labels.SelectorForStack(stackName),
-			IncludeUninitialized: true,
+	// informer errors are reported using global error handlers
+	runtimeutil.ErrorHandlers = append(handlers, func(err error) {
+		errC <- err
 	})
-		if err != nil {
-			stop <- true
+	defer func() {
+		runtimeutil.ErrorHandlers = handlers
+	}()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	wg := sync.WaitGroup{}
+	defer func() {
+		cancel()
+		wg.Wait()
+	}()
+	wg.Add(2)
+	go func() {
+		defer wg.Done()
+		w.watchStackStatus(ctx, name, errC)
+	}()
+	go func() {
+		defer wg.Done()
+		w.waitForPods(ctx, name, serviceNames, errC, statusUpdates)
+	}()
+
+	return <-errC
+}
+
+type stackWatcher struct {
+	resultChan chan error
+	stackName  string
+}
+
+var _ cache.ResourceEventHandler = &stackWatcher{}
+
+func (sw *stackWatcher) OnAdd(obj interface{}) {
+	stack, ok := obj.(*apiv1beta1.Stack)
+	switch {
+	case !ok:
+		sw.resultChan <- errors.Errorf("stack %s has incorrect type", sw.stackName)
+	case stack.Status.Phase == apiv1beta1.StackFailure:
+		sw.resultChan <- errors.Errorf("stack %s failed with status %s: %s", sw.stackName, stack.Status.Phase, stack.Status.Message)
+	}
+}
+
+func (sw *stackWatcher) OnUpdate(oldObj, newObj interface{}) {
+	sw.OnAdd(newObj)
+}
+
+func (sw *stackWatcher) OnDelete(obj interface{}) {
+}
+
+func (w *deployWatcher) watchStackStatus(ctx context.Context, stackname string, e chan error) {
+	informer := newStackInformer(w.stacks, stackname)
+	sw := &stackWatcher{
+		resultChan: e,
+	}
+	informer.AddEventHandler(sw)
+	informer.Run(ctx.Done())
+}
+
+type serviceStatus struct {
+	name          string
+	podsPending   int
+	podsRunning   int
+	podsSucceeded int
+	podsFailed    int
+	podsUnknown   int
+	podsReady     int
+	podsTotal     int
+}
+
+type podWatcher struct {
+	stackName     string
+	services      map[string]serviceStatus
+	resultChan    chan error
+	starts        map[string]int32
+	indexer       cache.Indexer
+	statusUpdates chan serviceStatus
+}
+
+var _ cache.ResourceEventHandler = &podWatcher{}
+
+func (pw *podWatcher) handlePod(obj interface{}) {
+	pod, ok := obj.(*apiv1.Pod)
+	if !ok {
+		pw.resultChan <- errors.Errorf("Pod has incorrect type in stack %s", pw.stackName)
 		return
 	}
-
-		for i := range list.Items {
-			pod := list.Items[i]
-			if pod.Status.Phase != apiv1.PodRunning {
-				continue
-			}
-
-			startCount := startCount(pod)
 	serviceName := pod.Labels[labels.ForServiceName]
-			if startCount != starts[serviceName] {
-				if startCount == 1 {
-					fmt.Printf(" - Service %s has one container running\n", serviceName)
-				} else {
-					fmt.Printf(" - Service %s was restarted %d %s\n", serviceName, startCount-1, timeTimes(startCount-1))
-				}
-
-				starts[serviceName] = startCount
-			}
-		}
-
-		if allReady(list.Items, serviceNames) {
-			stop <- true
-			return
+	pw.updateServiceStatus(serviceName)
+	if pw.allReady() {
+		select {
+		case pw.resultChan <- nil:
+		default:
+			// result has already been reported, just don't block
 		}
 	}
 }

-func startCount(pod apiv1.Pod) int32 {
-	restart := int32(0)
-
-	for _, status := range pod.Status.ContainerStatuses {
-		restart += status.RestartCount
+func (pw *podWatcher) updateServiceStatus(serviceName string) {
+	pods, _ := pw.indexer.ByIndex("byservice", serviceName)
+	status := serviceStatus{name: serviceName}
+	for _, obj := range pods {
+		if pod, ok := obj.(*apiv1.Pod); ok {
+			switch pod.Status.Phase {
+			case apiv1.PodPending:
+				status.podsPending++
+			case apiv1.PodRunning:
+				status.podsRunning++
+			case apiv1.PodSucceeded:
+				status.podsSucceeded++
+			case apiv1.PodFailed:
+				status.podsFailed++
+			case apiv1.PodUnknown:
+				status.podsUnknown++
 			}
-
-	return 1 + restart
+			if podutils.IsPodReady(pod) {
+				status.podsReady++
+			}
+		}
+	}
+	status.podsTotal = len(pods)
+	oldStatus := pw.services[serviceName]
+	if oldStatus != status {
+		pw.statusUpdates <- status
+	}
+	pw.services[serviceName] = status
 }

-func allReady(pods []apiv1.Pod, serviceNames []string) bool {
-	serviceUp := map[string]bool{}
-
-	for _, pod := range pods {
-		if time.Since(pod.GetCreationTimestamp().Time) < 10*time.Second {
-			return false
-		}
-
-		ready := false
-		for _, cond := range pod.Status.Conditions {
-			if cond.Type == apiv1.PodReady && cond.Status == apiv1.ConditionTrue {
-				ready = true
-			}
-		}
-
-		if !ready {
-			return false
-		}
-
-		serviceName := pod.Labels[labels.ForServiceName]
-		serviceUp[serviceName] = true
-	}
-
-	for _, serviceName := range serviceNames {
-		if !serviceUp[serviceName] {
+func (pw *podWatcher) allReady() bool {
+	for _, status := range pw.services {
+		if status.podsReady == 0 {
 			return false
 		}
 	}
-
 	return true
 }

-func timeTimes(n int32) string {
-	if n == 1 {
-		return "time"
-	}
-
-	return "times"
+func (pw *podWatcher) OnAdd(obj interface{}) {
+	pw.handlePod(obj)
+}
+
+func (pw *podWatcher) OnUpdate(oldObj, newObj interface{}) {
+	pw.handlePod(newObj)
+}
+
+func (pw *podWatcher) OnDelete(obj interface{}) {
+	pw.handlePod(obj)
+}
+
+func (w *deployWatcher) waitForPods(ctx context.Context, stackName string, serviceNames []string, e chan error, statusUpdates chan serviceStatus) {
+	informer := newPodInformer(w.pods, stackName, cache.Indexers{
+		"byservice": func(obj interface{}) ([]string, error) {
+			pod, ok := obj.(*apiv1.Pod)
+			if !ok {
+				return nil, errors.Errorf("Pod has incorrect type in stack %s", stackName)
+			}
+			return []string{pod.Labels[labels.ForServiceName]}, nil
+		}})
+	services := map[string]serviceStatus{}
+	for _, name := range serviceNames {
+		services[name] = serviceStatus{name: name}
+	}
+	pw := &podWatcher{
+		stackName:     stackName,
+		services:      services,
+		resultChan:    e,
+		starts:        map[string]int32{},
+		indexer:       informer.GetIndexer(),
+		statusUpdates: statusUpdates,
+	}
+	informer.AddEventHandler(pw)
+	informer.Run(ctx.Done())
+}
+
+func newPodInformer(podsClient podListWatch, stackName string, indexers cache.Indexers) cache.SharedIndexInformer {
+	return cache.NewSharedIndexInformer(
+		&cache.ListWatch{
+			ListFunc: func(options metav1.ListOptions) (runtime.Object, error) {
+				options.LabelSelector = labels.SelectorForStack(stackName)
+				options.IncludeUninitialized = true
+				return podsClient.List(options)
+			},
+
+			WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
+				options.LabelSelector = labels.SelectorForStack(stackName)
+				options.IncludeUninitialized = true
+				return podsClient.Watch(options)
+			},
+		},
+		&apiv1.Pod{},
+		time.Second*5,
+		indexers,
+	)
+}
+
+func newStackInformer(stacksClient stackListWatch, stackName string) cache.SharedInformer {
+	return cache.NewSharedInformer(
+		&cache.ListWatch{
+			ListFunc: func(options metav1.ListOptions) (runtime.Object, error) {
+				options.LabelSelector = labels.SelectorForStack(stackName)
+				return stacksClient.List(options)
+			},
+
+			WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
+				options.LabelSelector = labels.SelectorForStack(stackName)
+				return stacksClient.Watch(options)
+			},
+		},
+		&apiv1beta1.Stack{},
+		time.Second*5,
+	)
 }
--- a/cli/command/stack/kubernetes/watcher_test.go
+++ b/cli/command/stack/kubernetes/watcher_test.go
@ -0,0 +1,218 @@
+package kubernetes
+
+import (
+	"testing"
+
+	apiv1beta1 "github.com/docker/cli/kubernetes/compose/v1beta1"
+	composelabels "github.com/docker/cli/kubernetes/labels"
+	"github.com/gotestyourself/gotestyourself/assert"
+	apiv1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/runtime/serializer"
+	"k8s.io/apimachinery/pkg/watch"
+	k8stesting "k8s.io/client-go/testing"
+)
+
+var podsResource = apiv1.SchemeGroupVersion.WithResource("pods")
+var podKind = apiv1.SchemeGroupVersion.WithKind("Pod")
+var stacksResource = apiv1beta1.SchemeGroupVersion.WithResource("stacks")
+var stackKind = apiv1beta1.SchemeGroupVersion.WithKind("Stack")
+
+type testPodAndStackRepository struct {
+	fake *k8stesting.Fake
+}
+
+func (r *testPodAndStackRepository) stackListWatchForNamespace(ns string) *testStackListWatch {
+	return &testStackListWatch{fake: r.fake, ns: ns}
+}
+func (r *testPodAndStackRepository) podListWatchForNamespace(ns string) *testPodListWatch {
+	return &testPodListWatch{fake: r.fake, ns: ns}
+}
+
+func newTestPodAndStackRepository(initialPods []apiv1.Pod, initialStacks []apiv1beta1.Stack, podWatchHandler, stackWatchHandler k8stesting.WatchReactionFunc) *testPodAndStackRepository {
+	var scheme = runtime.NewScheme()
+	var codecs = serializer.NewCodecFactory(scheme)
+	metav1.AddToGroupVersion(scheme, schema.GroupVersion{Version: "v1"})
+	apiv1.AddToScheme(scheme)
+	apiv1beta1.AddToScheme(scheme)
+
+	o := k8stesting.NewObjectTracker(scheme, codecs.UniversalDecoder())
+	for _, obj := range initialPods {
+		if err := o.Add(&obj); err != nil {
+			panic(err)
+		}
+	}
+	for _, obj := range initialStacks {
+		if err := o.Add(&obj); err != nil {
+			panic(err)
+		}
+	}
+	fakePtr := &k8stesting.Fake{}
+	fakePtr.AddReactor("*", "*", k8stesting.ObjectReaction(o))
+	if podWatchHandler != nil {
+		fakePtr.AddWatchReactor(podsResource.Resource, podWatchHandler)
+	}
+	if stackWatchHandler != nil {
+		fakePtr.AddWatchReactor(stacksResource.Resource, stackWatchHandler)
+	}
+	fakePtr.AddWatchReactor("*", k8stesting.DefaultWatchReactor(watch.NewFake(), nil))
+	return &testPodAndStackRepository{fake: fakePtr}
+}
+
+type testStackListWatch struct {
+	fake *k8stesting.Fake
+	ns   string
+}
+
+func (s *testStackListWatch) List(opts metav1.ListOptions) (*apiv1beta1.StackList, error) {
+	obj, err := s.fake.Invokes(k8stesting.NewListAction(stacksResource, stackKind, s.ns, opts), &apiv1beta1.StackList{})
+
+	if obj == nil {
+		return nil, err
+	}
+
+	label, _, _ := k8stesting.ExtractFromListOptions(opts)
+	if label == nil {
+		label = labels.Everything()
+	}
+	list := &apiv1beta1.StackList{}
+	for _, item := range obj.(*apiv1beta1.StackList).Items {
+		if label.Matches(labels.Set(item.Labels)) {
+			list.Items = append(list.Items, item)
+		}
+	}
+	return list, err
+}
+func (s *testStackListWatch) Watch(opts metav1.ListOptions) (watch.Interface, error) {
+	return s.fake.InvokesWatch(k8stesting.NewWatchAction(stacksResource, s.ns, opts))
+}
+
+type testPodListWatch struct {
+	fake *k8stesting.Fake
+	ns   string
+}
+
+func (p *testPodListWatch) List(opts metav1.ListOptions) (*apiv1.PodList, error) {
+	obj, err := p.fake.Invokes(k8stesting.NewListAction(podsResource, podKind, p.ns, opts), &apiv1.PodList{})
+
+	if obj == nil {
+		return nil, err
+	}
+
+	label, _, _ := k8stesting.ExtractFromListOptions(opts)
+	if label == nil {
+		label = labels.Everything()
+	}
+	list := &apiv1.PodList{}
+	for _, item := range obj.(*apiv1.PodList).Items {
+		if label.Matches(labels.Set(item.Labels)) {
+			list.Items = append(list.Items, item)
+		}
+	}
+	return list, err
+
+}
+func (p *testPodListWatch) Watch(opts metav1.ListOptions) (watch.Interface, error) {
+	return p.fake.InvokesWatch(k8stesting.NewWatchAction(podsResource, p.ns, opts))
+}
+
+func TestDeployWatchOk(t *testing.T) {
+	stack := apiv1beta1.Stack{
+		ObjectMeta: metav1.ObjectMeta{Name: "test-stack", Namespace: "test-ns"},
+	}
+
+	serviceNames := []string{"svc1", "svc2"}
+	testRepo := newTestPodAndStackRepository(nil, []apiv1beta1.Stack{stack}, func(action k8stesting.Action) (handled bool, ret watch.Interface, err error) {
+		res := watch.NewFake()
+		go func() {
+			pod1 := &apiv1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "test1",
+					Namespace: "test-ns",
+					Labels:    composelabels.ForService("test-stack", "svc1"),
+				},
+				Status: apiv1.PodStatus{
+					Phase: apiv1.PodRunning,
+					Conditions: []apiv1.PodCondition{
+						{
+							Type:   apiv1.PodReady,
+							Status: apiv1.ConditionTrue,
+						},
+					},
+				},
+			}
+			pod2 := &apiv1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "test2",
+					Namespace: "test-ns",
+					Labels:    composelabels.ForService("test-stack", "svc2"),
+				},
+				Status: apiv1.PodStatus{
+					Phase: apiv1.PodRunning,
+					Conditions: []apiv1.PodCondition{
+						{
+							Type:   apiv1.PodReady,
+							Status: apiv1.ConditionTrue,
+						},
+					},
+				},
+			}
+			res.Add(pod1)
+			res.Add(pod2)
+		}()
+
+		return true, res, nil
+	}, nil)
+
+	testee := &deployWatcher{
+		stacks: testRepo.stackListWatchForNamespace("test-ns"),
+		pods:   testRepo.podListWatchForNamespace("test-ns"),
+	}
+
+	statusUpdates := make(chan serviceStatus)
+	go func() {
+		for range statusUpdates {
+		}
+	}()
+	defer close(statusUpdates)
+	err := testee.Watch(stack.Name, serviceNames, statusUpdates)
+	assert.NilError(t, err)
+}
+
+func TestDeployReconcileFailure(t *testing.T) {
+	stack := apiv1beta1.Stack{
+		ObjectMeta: metav1.ObjectMeta{Name: "test-stack", Namespace: "test-ns"},
+	}
+
+	serviceNames := []string{"svc1", "svc2"}
+	testRepo := newTestPodAndStackRepository(nil, []apiv1beta1.Stack{stack}, nil, func(action k8stesting.Action) (handled bool, ret watch.Interface, err error) {
+		res := watch.NewFake()
+		go func() {
+			sfailed := stack
+			sfailed.Status = apiv1beta1.StackStatus{
+				Phase:   apiv1beta1.StackFailure,
+				Message: "test error",
+			}
+			res.Modify(&sfailed)
+		}()
+
+		return true, res, nil
+	})
+
+	testee := &deployWatcher{
+		stacks: testRepo.stackListWatchForNamespace("test-ns"),
+		pods:   testRepo.podListWatchForNamespace("test-ns"),
+	}
+
+	statusUpdates := make(chan serviceStatus)
+	go func() {
+		for range statusUpdates {
+		}
+	}()
+	defer close(statusUpdates)
+	err := testee.Watch(stack.Name, serviceNames, statusUpdates)
+	assert.ErrorContains(t, err, "Failure: test error")
+}
--- a/vendor.conf
+++ b/vendor.conf
@ -36,6 +36,7 @@ github.com/go-openapi/spec 6aced65f8501fe1217321abf0749d354824ba2ff
 github.com/go-openapi/swag 1d0bd113de87027671077d3c71eb3ac5d7dbba72
 github.com/gregjones/httpcache c1f8028e62adb3d518b823a2f8e6a95c38bdd3aa
 github.com/grpc-ecosystem/grpc-gateway 1a03ca3bad1e1ebadaedd3abb76bc58d4ac8143b
+github.com/hashicorp/golang-lru 0a025b7e63adc15a622f29b0b2c4c3848243bbf6
 github.com/howeyc/gopass 3ca23474a7c7203e0a0a070fd33508f6efdb9b3d
 github.com/imdario/mergo 9d5f1277e9a8ed20c3684bda8fde67c05628518c # v0.3.4
 github.com/inconshreveable/mousetrap 76626ae9c91c4f2a10f34cad8ce83ea42c93bb75
@ -48,6 +49,7 @@ github.com/Microsoft/go-winio v0.4.6
 github.com/miekg/pkcs11 5f6e0d0dad6f472df908c8e968a98ef00c9224bb
 github.com/mitchellh/mapstructure f3009df150dadf309fdee4a54ed65c124afad715
 github.com/moby/buildkit aaff9d591ef128560018433fe61beb802e149de8
+github.com/morikuni/aec 39771216ff4c63d11f5e604076f9c45e8be1067b
 github.com/Nvveen/Gotty a8b993ba6abdb0e0c12b0125c603323a71c7790c https://github.com/ijc25/Gotty
 github.com/opencontainers/go-digest 21dfd564fd89c944783d00d069f33e3e7123c448
 github.com/opencontainers/image-spec v1.0.1
@ -87,4 +89,3 @@ k8s.io/client-go kubernetes-1.8.2
 k8s.io/kubernetes v1.8.2
 k8s.io/kube-openapi 61b46af70dfed79c6d24530cd23b41440a7f22a5
 vbom.ml/util 928aaa586d7718c70f4090ddf83f2b34c16fdc8d
-github.com/hashicorp/golang-lru 0a025b7e63adc15a622f29b0b2c4c3848243bbf6
--- a/vendor/github.com/morikuni/aec/LICENSE
+++ b/vendor/github.com/morikuni/aec/LICENSE
@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 Taihei Morikuni
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
--- a/vendor/github.com/morikuni/aec/README.md
+++ b/vendor/github.com/morikuni/aec/README.md
@ -0,0 +1,178 @@
+# aec
+
+[![GoDoc](https://godoc.org/github.com/morikuni/aec?status.svg)](https://godoc.org/github.com/morikuni/aec)
+
+Go wrapper for ANSI escape code.
+
+## Install
+
+```bash
+go get github.com/morikuni/aec
+```
+
+## Features
+
+ANSI escape codes depend on terminal environment.  
+Some of these features may not work.  
+Check supported Font-Style/Font-Color features with [checkansi](./checkansi).
+
+[Wikipedia](https://en.wikipedia.org/wiki/ANSI_escape_code) for more detail.
+
+### Cursor
+
+- `Up(n)`
+- `Down(n)`
+- `Right(n)`
+- `Left(n)`
+- `NextLine(n)`
+- `PreviousLine(n)`
+- `Column(col)`
+- `Position(row, col)`
+- `Save`
+- `Restore`
+- `Hide`
+- `Show`
+- `Report`
+
+### Erase
+
+- `EraseDisplay(mode)`
+- `EraseLine(mode)`
+
+### Scroll
+
+- `ScrollUp(n)`
+- `ScrollDown(n)`
+
+### Font Style
+
+- `Bold`
+- `Faint`
+- `Italic`
+- `Underline`
+- `BlinkSlow`
+- `BlinkRapid`
+- `Inverse`
+- `Conceal`
+- `CrossOut`
+- `Frame`
+- `Encircle`
+- `Overline`
+
+### Font Color
+
+Foreground color.
+
+- `DefaultF`
+- `BlackF`
+- `RedF`
+- `GreenF`
+- `YellowF`
+- `BlueF`
+- `MagentaF`
+- `CyanF`
+- `WhiteF`
+- `LightBlackF`
+- `LightRedF`
+- `LightGreenF`
+- `LightYellowF`
+- `LightBlueF`
+- `LightMagentaF`
+- `LightCyanF`
+- `LightWhiteF`
+- `Color3BitF(color)`
+- `Color8BitF(color)`
+- `FullColorF(r, g, b)`
+
+Background color.
+
+- `DefaultB`
+- `BlackB`
+- `RedB`
+- `GreenB`
+- `YellowB`
+- `BlueB`
+- `MagentaB`
+- `CyanB`
+- `WhiteB`
+- `LightBlackB`
+- `LightRedB`
+- `LightGreenB`
+- `LightYellowB`
+- `LightBlueB`
+- `LightMagentaB`
+- `LightCyanB`
+- `LightWhiteB`
+- `Color3BitB(color)`
+- `Color8BitB(color)`
+- `FullColorB(r, g, b)`
+
+### Color Converter
+
+24bit RGB color to ANSI color.
+
+- `NewRGB3Bit(r, g, b)`
+- `NewRGB8Bit(r, g, b)`
+
+### Builder
+
+To mix these features.
+
+```go
+custom := aec.EmptyBuilder.Right(2).RGB8BitF(128, 255, 64).RedB().ANSI
+custom.Apply("Hello World")
+```
+
+## Usage
+
+1. Create ANSI by `aec.XXX().With(aec.YYY())` or `aec.EmptyBuilder.XXX().YYY().ANSI`
+2. Print ANSI by `fmt.Print(ansi, "some string", aec.Reset)` or `fmt.Print(ansi.Apply("some string"))`
+
+`aec.Reset` should be added when using font style or font color features.
+
+## Example
+
+Simple progressbar.
+
+![sample](./sample.gif)
+
+```go
+package main
+
+import (
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/morikuni/aec"
+)
+
+func main() {
+	const n = 20
+	builder := aec.EmptyBuilder
+
+	up2 := aec.Up(2)
+	col := aec.Column(n + 2)
+	bar := aec.Color8BitF(aec.NewRGB8Bit(64, 255, 64))
+	label := builder.LightRedF().Underline().With(col).Right(1).ANSI
+
+	// for up2
+	fmt.Println()
+	fmt.Println()
+
+	for i := 0; i <= n; i++ {
+		fmt.Print(up2)
+		fmt.Println(label.Apply(fmt.Sprint(i, "/", n)))
+		fmt.Print("[")
+		fmt.Print(bar.Apply(strings.Repeat("=", i)))
+		fmt.Println(col.Apply("]"))
+		time.Sleep(100 * time.Millisecond)
+	}
+}
+```
+
+## License
+
+[MIT](./LICENSE)
+
+
--- a/vendor/github.com/morikuni/aec/aec.go
+++ b/vendor/github.com/morikuni/aec/aec.go
@ -0,0 +1,137 @@
+package aec
+
+import "fmt"
+
+// EraseMode is listed in a variable EraseModes.
+type EraseMode uint
+
+var (
+	// EraseModes is a list of EraseMode.
+	EraseModes struct {
+		// All erase all.
+		All EraseMode
+
+		// Head erase to head.
+		Head EraseMode
+
+		// Tail erase to tail.
+		Tail EraseMode
+	}
+
+	// Save saves the cursor position.
+	Save ANSI
+
+	// Restore restores the cursor position.
+	Restore ANSI
+
+	// Hide hides the cursor.
+	Hide ANSI
+
+	// Show shows the cursor.
+	Show ANSI
+
+	// Report reports the cursor position.
+	Report ANSI
+)
+
+// Up moves up the cursor.
+func Up(n uint) ANSI {
+	if n == 0 {
+		return empty
+	}
+	return newAnsi(fmt.Sprintf(esc+"%dA", n))
+}
+
+// Down moves down the cursor.
+func Down(n uint) ANSI {
+	if n == 0 {
+		return empty
+	}
+	return newAnsi(fmt.Sprintf(esc+"%dB", n))
+}
+
+// Right moves right the cursor.
+func Right(n uint) ANSI {
+	if n == 0 {
+		return empty
+	}
+	return newAnsi(fmt.Sprintf(esc+"%dC", n))
+}
+
+// Left moves left the cursor.
+func Left(n uint) ANSI {
+	if n == 0 {
+		return empty
+	}
+	return newAnsi(fmt.Sprintf(esc+"%dD", n))
+}
+
+// NextLine moves down the cursor to head of a line.
+func NextLine(n uint) ANSI {
+	if n == 0 {
+		return empty
+	}
+	return newAnsi(fmt.Sprintf(esc+"%dE", n))
+}
+
+// PreviousLine moves up the cursor to head of a line.
+func PreviousLine(n uint) ANSI {
+	if n == 0 {
+		return empty
+	}
+	return newAnsi(fmt.Sprintf(esc+"%dF", n))
+}
+
+// Column set the cursor position to a given column.
+func Column(col uint) ANSI {
+	return newAnsi(fmt.Sprintf(esc+"%dG", col))
+}
+
+// Position set the cursor position to a given absolute position.
+func Position(row, col uint) ANSI {
+	return newAnsi(fmt.Sprintf(esc+"%d;%dH", row, col))
+}
+
+// EraseDisplay erases display by given EraseMode.
+func EraseDisplay(m EraseMode) ANSI {
+	return newAnsi(fmt.Sprintf(esc+"%dJ", m))
+}
+
+// EraseLine erases lines by given EraseMode.
+func EraseLine(m EraseMode) ANSI {
+	return newAnsi(fmt.Sprintf(esc+"%dK", m))
+}
+
+// ScrollUp scrolls up the page.
+func ScrollUp(n int) ANSI {
+	if n == 0 {
+		return empty
+	}
+	return newAnsi(fmt.Sprintf(esc+"%dS", n))
+}
+
+// ScrollDown scrolls down the page.
+func ScrollDown(n int) ANSI {
+	if n == 0 {
+		return empty
+	}
+	return newAnsi(fmt.Sprintf(esc+"%dT", n))
+}
+
+func init() {
+	EraseModes = struct {
+		All  EraseMode
+		Head EraseMode
+		Tail EraseMode
+	}{
+		Tail: 0,
+		Head: 1,
+		All:  2,
+	}
+
+	Save = newAnsi(esc + "s")
+	Restore = newAnsi(esc + "u")
+	Hide = newAnsi(esc + "?25l")
+	Show = newAnsi(esc + "?25h")
+	Report = newAnsi(esc + "6n")
+}
--- a/vendor/github.com/morikuni/aec/ansi.go
+++ b/vendor/github.com/morikuni/aec/ansi.go
@ -0,0 +1,59 @@
+package aec
+
+import (
+	"fmt"
+	"strings"
+)
+
+const esc = "\x1b["
+
+// Reset resets SGR effect.
+const Reset string = "\x1b[0m"
+
+var empty = newAnsi("")
+
+// ANSI represents ANSI escape code.
+type ANSI interface {
+	fmt.Stringer
+
+	// With adapts given ANSIs.
+	With(...ANSI) ANSI
+
+	// Apply wraps given string in ANSI.
+	Apply(string) string
+}
+
+type ansiImpl string
+
+func newAnsi(s string) *ansiImpl {
+	r := ansiImpl(s)
+	return &r
+}
+
+func (a *ansiImpl) With(ansi ...ANSI) ANSI {
+	return concat(append([]ANSI{a}, ansi...))
+}
+
+func (a *ansiImpl) Apply(s string) string {
+	return a.String() + s + Reset
+}
+
+func (a *ansiImpl) String() string {
+	return string(*a)
+}
+
+// Apply wraps given string in ANSIs.
+func Apply(s string, ansi ...ANSI) string {
+	if len(ansi) == 0 {
+		return s
+	}
+	return concat(ansi).Apply(s)
+}
+
+func concat(ansi []ANSI) ANSI {
+	strs := make([]string, 0, len(ansi))
+	for _, p := range ansi {
+		strs = append(strs, p.String())
+	}
+	return newAnsi(strings.Join(strs, ""))
+}
--- a/vendor/github.com/morikuni/aec/builder.go
+++ b/vendor/github.com/morikuni/aec/builder.go
@ -0,0 +1,388 @@
+package aec
+
+// Builder is a lightweight syntax to construct customized ANSI.
+type Builder struct {
+	ANSI ANSI
+}
+
+// EmptyBuilder is an initialized Builder.
+var EmptyBuilder *Builder
+
+// NewBuilder creates a Builder from existing ANSI.
+func NewBuilder(a ...ANSI) *Builder {
+	return &Builder{concat(a)}
+}
+
+// With is a syntax for With.
+func (builder *Builder) With(a ...ANSI) *Builder {
+	return NewBuilder(builder.ANSI.With(a...))
+}
+
+// Up is a syntax for Up.
+func (builder *Builder) Up(n uint) *Builder {
+	return builder.With(Up(n))
+}
+
+// Down is a syntax for Down.
+func (builder *Builder) Down(n uint) *Builder {
+	return builder.With(Down(n))
+}
+
+// Right is a syntax for Right.
+func (builder *Builder) Right(n uint) *Builder {
+	return builder.With(Right(n))
+}
+
+// Left is a syntax for Left.
+func (builder *Builder) Left(n uint) *Builder {
+	return builder.With(Left(n))
+}
+
+// NextLine is a syntax for NextLine.
+func (builder *Builder) NextLine(n uint) *Builder {
+	return builder.With(NextLine(n))
+}
+
+// PreviousLine is a syntax for PreviousLine.
+func (builder *Builder) PreviousLine(n uint) *Builder {
+	return builder.With(PreviousLine(n))
+}
+
+// Column is a syntax for Column.
+func (builder *Builder) Column(col uint) *Builder {
+	return builder.With(Column(col))
+}
+
+// Position is a syntax for Position.
+func (builder *Builder) Position(row, col uint) *Builder {
+	return builder.With(Position(row, col))
+}
+
+// EraseDisplay is a syntax for EraseDisplay.
+func (builder *Builder) EraseDisplay(m EraseMode) *Builder {
+	return builder.With(EraseDisplay(m))
+}
+
+// EraseLine is a syntax for EraseLine.
+func (builder *Builder) EraseLine(m EraseMode) *Builder {
+	return builder.With(EraseLine(m))
+}
+
+// ScrollUp is a syntax for ScrollUp.
+func (builder *Builder) ScrollUp(n int) *Builder {
+	return builder.With(ScrollUp(n))
+}
+
+// ScrollDown is a syntax for ScrollDown.
+func (builder *Builder) ScrollDown(n int) *Builder {
+	return builder.With(ScrollDown(n))
+}
+
+// Save is a syntax for Save.
+func (builder *Builder) Save() *Builder {
+	return builder.With(Save)
+}
+
+// Restore is a syntax for Restore.
+func (builder *Builder) Restore() *Builder {
+	return builder.With(Restore)
+}
+
+// Hide is a syntax for Hide.
+func (builder *Builder) Hide() *Builder {
+	return builder.With(Hide)
+}
+
+// Show is a syntax for Show.
+func (builder *Builder) Show() *Builder {
+	return builder.With(Show)
+}
+
+// Report is a syntax for Report.
+func (builder *Builder) Report() *Builder {
+	return builder.With(Report)
+}
+
+// Bold is a syntax for Bold.
+func (builder *Builder) Bold() *Builder {
+	return builder.With(Bold)
+}
+
+// Faint is a syntax for Faint.
+func (builder *Builder) Faint() *Builder {
+	return builder.With(Faint)
+}
+
+// Italic is a syntax for Italic.
+func (builder *Builder) Italic() *Builder {
+	return builder.With(Italic)
+}
+
+// Underline is a syntax for Underline.
+func (builder *Builder) Underline() *Builder {
+	return builder.With(Underline)
+}
+
+// BlinkSlow is a syntax for BlinkSlow.
+func (builder *Builder) BlinkSlow() *Builder {
+	return builder.With(BlinkSlow)
+}
+
+// BlinkRapid is a syntax for BlinkRapid.
+func (builder *Builder) BlinkRapid() *Builder {
+	return builder.With(BlinkRapid)
+}
+
+// Inverse is a syntax for Inverse.
+func (builder *Builder) Inverse() *Builder {
+	return builder.With(Inverse)
+}
+
+// Conceal is a syntax for Conceal.
+func (builder *Builder) Conceal() *Builder {
+	return builder.With(Conceal)
+}
+
+// CrossOut is a syntax for CrossOut.
+func (builder *Builder) CrossOut() *Builder {
+	return builder.With(CrossOut)
+}
+
+// BlackF is a syntax for BlackF.
+func (builder *Builder) BlackF() *Builder {
+	return builder.With(BlackF)
+}
+
+// RedF is a syntax for RedF.
+func (builder *Builder) RedF() *Builder {
+	return builder.With(RedF)
+}
+
+// GreenF is a syntax for GreenF.
+func (builder *Builder) GreenF() *Builder {
+	return builder.With(GreenF)
+}
+
+// YellowF is a syntax for YellowF.
+func (builder *Builder) YellowF() *Builder {
+	return builder.With(YellowF)
+}
+
+// BlueF is a syntax for BlueF.
+func (builder *Builder) BlueF() *Builder {
+	return builder.With(BlueF)
+}
+
+// MagentaF is a syntax for MagentaF.
+func (builder *Builder) MagentaF() *Builder {
+	return builder.With(MagentaF)
+}
+
+// CyanF is a syntax for CyanF.
+func (builder *Builder) CyanF() *Builder {
+	return builder.With(CyanF)
+}
+
+// WhiteF is a syntax for WhiteF.
+func (builder *Builder) WhiteF() *Builder {
+	return builder.With(WhiteF)
+}
+
+// DefaultF is a syntax for DefaultF.
+func (builder *Builder) DefaultF() *Builder {
+	return builder.With(DefaultF)
+}
+
+// BlackB is a syntax for BlackB.
+func (builder *Builder) BlackB() *Builder {
+	return builder.With(BlackB)
+}
+
+// RedB is a syntax for RedB.
+func (builder *Builder) RedB() *Builder {
+	return builder.With(RedB)
+}
+
+// GreenB is a syntax for GreenB.
+func (builder *Builder) GreenB() *Builder {
+	return builder.With(GreenB)
+}
+
+// YellowB is a syntax for YellowB.
+func (builder *Builder) YellowB() *Builder {
+	return builder.With(YellowB)
+}
+
+// BlueB is a syntax for BlueB.
+func (builder *Builder) BlueB() *Builder {
+	return builder.With(BlueB)
+}
+
+// MagentaB is a syntax for MagentaB.
+func (builder *Builder) MagentaB() *Builder {
+	return builder.With(MagentaB)
+}
+
+// CyanB is a syntax for CyanB.
+func (builder *Builder) CyanB() *Builder {
+	return builder.With(CyanB)
+}
+
+// WhiteB is a syntax for WhiteB.
+func (builder *Builder) WhiteB() *Builder {
+	return builder.With(WhiteB)
+}
+
+// DefaultB is a syntax for DefaultB.
+func (builder *Builder) DefaultB() *Builder {
+	return builder.With(DefaultB)
+}
+
+// Frame is a syntax for Frame.
+func (builder *Builder) Frame() *Builder {
+	return builder.With(Frame)
+}
+
+// Encircle is a syntax for Encircle.
+func (builder *Builder) Encircle() *Builder {
+	return builder.With(Encircle)
+}
+
+// Overline is a syntax for Overline.
+func (builder *Builder) Overline() *Builder {
+	return builder.With(Overline)
+}
+
+// LightBlackF is a syntax for LightBlueF.
+func (builder *Builder) LightBlackF() *Builder {
+	return builder.With(LightBlackF)
+}
+
+// LightRedF is a syntax for LightRedF.
+func (builder *Builder) LightRedF() *Builder {
+	return builder.With(LightRedF)
+}
+
+// LightGreenF is a syntax for LightGreenF.
+func (builder *Builder) LightGreenF() *Builder {
+	return builder.With(LightGreenF)
+}
+
+// LightYellowF is a syntax for LightYellowF.
+func (builder *Builder) LightYellowF() *Builder {
+	return builder.With(LightYellowF)
+}
+
+// LightBlueF is a syntax for LightBlueF.
+func (builder *Builder) LightBlueF() *Builder {
+	return builder.With(LightBlueF)
+}
+
+// LightMagentaF is a syntax for LightMagentaF.
+func (builder *Builder) LightMagentaF() *Builder {
+	return builder.With(LightMagentaF)
+}
+
+// LightCyanF is a syntax for LightCyanF.
+func (builder *Builder) LightCyanF() *Builder {
+	return builder.With(LightCyanF)
+}
+
+// LightWhiteF is a syntax for LightWhiteF.
+func (builder *Builder) LightWhiteF() *Builder {
+	return builder.With(LightWhiteF)
+}
+
+// LightBlackB is a syntax for LightBlackB.
+func (builder *Builder) LightBlackB() *Builder {
+	return builder.With(LightBlackB)
+}
+
+// LightRedB is a syntax for LightRedB.
+func (builder *Builder) LightRedB() *Builder {
+	return builder.With(LightRedB)
+}
+
+// LightGreenB is a syntax for LightGreenB.
+func (builder *Builder) LightGreenB() *Builder {
+	return builder.With(LightGreenB)
+}
+
+// LightYellowB is a syntax for LightYellowB.
+func (builder *Builder) LightYellowB() *Builder {
+	return builder.With(LightYellowB)
+}
+
+// LightBlueB is a syntax for LightBlueB.
+func (builder *Builder) LightBlueB() *Builder {
+	return builder.With(LightBlueB)
+}
+
+// LightMagentaB is a syntax for LightMagentaB.
+func (builder *Builder) LightMagentaB() *Builder {
+	return builder.With(LightMagentaB)
+}
+
+// LightCyanB is a syntax for LightCyanB.
+func (builder *Builder) LightCyanB() *Builder {
+	return builder.With(LightCyanB)
+}
+
+// LightWhiteB is a syntax for LightWhiteB.
+func (builder *Builder) LightWhiteB() *Builder {
+	return builder.With(LightWhiteB)
+}
+
+// Color3BitF is a syntax for Color3BitF.
+func (builder *Builder) Color3BitF(c RGB3Bit) *Builder {
+	return builder.With(Color3BitF(c))
+}
+
+// Color3BitB is a syntax for Color3BitB.
+func (builder *Builder) Color3BitB(c RGB3Bit) *Builder {
+	return builder.With(Color3BitB(c))
+}
+
+// Color8BitF is a syntax for Color8BitF.
+func (builder *Builder) Color8BitF(c RGB8Bit) *Builder {
+	return builder.With(Color8BitF(c))
+}
+
+// Color8BitB is a syntax for Color8BitB.
+func (builder *Builder) Color8BitB(c RGB8Bit) *Builder {
+	return builder.With(Color8BitB(c))
+}
+
+// FullColorF is a syntax for FullColorF.
+func (builder *Builder) FullColorF(r, g, b uint8) *Builder {
+	return builder.With(FullColorF(r, g, b))
+}
+
+// FullColorB is a syntax for FullColorB.
+func (builder *Builder) FullColorB(r, g, b uint8) *Builder {
+	return builder.With(FullColorB(r, g, b))
+}
+
+// RGB3BitF is a syntax for Color3BitF with NewRGB3Bit.
+func (builder *Builder) RGB3BitF(r, g, b uint8) *Builder {
+	return builder.Color3BitF(NewRGB3Bit(r, g, b))
+}
+
+// RGB3BitB is a syntax for Color3BitB with NewRGB3Bit.
+func (builder *Builder) RGB3BitB(r, g, b uint8) *Builder {
+	return builder.Color3BitB(NewRGB3Bit(r, g, b))
+}
+
+// RGB8BitF is a syntax for Color8BitF with NewRGB8Bit.
+func (builder *Builder) RGB8BitF(r, g, b uint8) *Builder {
+	return builder.Color8BitF(NewRGB8Bit(r, g, b))
+}
+
+// RGB8BitB is a syntax for Color8BitB with NewRGB8Bit.
+func (builder *Builder) RGB8BitB(r, g, b uint8) *Builder {
+	return builder.Color8BitB(NewRGB8Bit(r, g, b))
+}
+
+func init() {
+	EmptyBuilder = &Builder{empty}
+}
--- a/vendor/github.com/morikuni/aec/sgr.go
+++ b/vendor/github.com/morikuni/aec/sgr.go
@ -0,0 +1,202 @@
+package aec
+
+import (
+	"fmt"
+)
+
+// RGB3Bit is a 3bit RGB color.
+type RGB3Bit uint8
+
+// RGB8Bit is a 8bit RGB color.
+type RGB8Bit uint8
+
+func newSGR(n uint) ANSI {
+	return newAnsi(fmt.Sprintf(esc+"%dm", n))
+}
+
+// NewRGB3Bit create a RGB3Bit from given RGB.
+func NewRGB3Bit(r, g, b uint8) RGB3Bit {
+	return RGB3Bit((r >> 7) | ((g >> 6) & 0x2) | ((b >> 5) & 0x4))
+}
+
+// NewRGB8Bit create a RGB8Bit from given RGB.
+func NewRGB8Bit(r, g, b uint8) RGB8Bit {
+	return RGB8Bit(16 + 36*(r/43) + 6*(g/43) + b/43)
+}
+
+// Color3BitF set the foreground color of text.
+func Color3BitF(c RGB3Bit) ANSI {
+	return newAnsi(fmt.Sprintf(esc+"%dm", c+30))
+}
+
+// Color3BitB set the background color of text.
+func Color3BitB(c RGB3Bit) ANSI {
+	return newAnsi(fmt.Sprintf(esc+"%dm", c+40))
+}
+
+// Color8BitF set the foreground color of text.
+func Color8BitF(c RGB8Bit) ANSI {
+	return newAnsi(fmt.Sprintf(esc+"38;5;%dm", c))
+}
+
+// Color8BitB set the background color of text.
+func Color8BitB(c RGB8Bit) ANSI {
+	return newAnsi(fmt.Sprintf(esc+"48;5;%dm", c))
+}
+
+// FullColorF set the foreground color of text.
+func FullColorF(r, g, b uint8) ANSI {
+	return newAnsi(fmt.Sprintf(esc+"38;2;%d;%d;%dm", r, g, b))
+}
+
+// FullColorB set the foreground color of text.
+func FullColorB(r, g, b uint8) ANSI {
+	return newAnsi(fmt.Sprintf(esc+"48;2;%d;%d;%dm", r, g, b))
+}
+
+// Style
+var (
+	// Bold set the text style to bold or increased intensity.
+	Bold ANSI
+
+	// Faint set the text style to faint.
+	Faint ANSI
+
+	// Italic set the text style to italic.
+	Italic ANSI
+
+	// Underline set the text style to underline.
+	Underline ANSI
+
+	// BlinkSlow set the text style to slow blink.
+	BlinkSlow ANSI
+
+	// BlinkRapid set the text style to rapid blink.
+	BlinkRapid ANSI
+
+	// Inverse swap the foreground color and background color.
+	Inverse ANSI
+
+	// Conceal set the text style to conceal.
+	Conceal ANSI
+
+	// CrossOut set the text style to crossed out.
+	CrossOut ANSI
+
+	// Frame set the text style to framed.
+	Frame ANSI
+
+	// Encircle set the text style to encircled.
+	Encircle ANSI
+
+	// Overline set the text style to overlined.
+	Overline ANSI
+)
+
+// Foreground color of text.
+var (
+	// DefaultF is the default color of foreground.
+	DefaultF ANSI
+
+	// Normal color
+	BlackF   ANSI
+	RedF     ANSI
+	GreenF   ANSI
+	YellowF  ANSI
+	BlueF    ANSI
+	MagentaF ANSI
+	CyanF    ANSI
+	WhiteF   ANSI
+
+	// Light color
+	LightBlackF   ANSI
+	LightRedF     ANSI
+	LightGreenF   ANSI
+	LightYellowF  ANSI
+	LightBlueF    ANSI
+	LightMagentaF ANSI
+	LightCyanF    ANSI
+	LightWhiteF   ANSI
+)
+
+// Background color of text.
+var (
+	// DefaultB is the default color of background.
+	DefaultB ANSI
+
+	// Normal color
+	BlackB   ANSI
+	RedB     ANSI
+	GreenB   ANSI
+	YellowB  ANSI
+	BlueB    ANSI
+	MagentaB ANSI
+	CyanB    ANSI
+	WhiteB   ANSI
+
+	// Light color
+	LightBlackB   ANSI
+	LightRedB     ANSI
+	LightGreenB   ANSI
+	LightYellowB  ANSI
+	LightBlueB    ANSI
+	LightMagentaB ANSI
+	LightCyanB    ANSI
+	LightWhiteB   ANSI
+)
+
+func init() {
+	Bold = newSGR(1)
+	Faint = newSGR(2)
+	Italic = newSGR(3)
+	Underline = newSGR(4)
+	BlinkSlow = newSGR(5)
+	BlinkRapid = newSGR(6)
+	Inverse = newSGR(7)
+	Conceal = newSGR(8)
+	CrossOut = newSGR(9)
+
+	BlackF = newSGR(30)
+	RedF = newSGR(31)
+	GreenF = newSGR(32)
+	YellowF = newSGR(33)
+	BlueF = newSGR(34)
+	MagentaF = newSGR(35)
+	CyanF = newSGR(36)
+	WhiteF = newSGR(37)
+
+	DefaultF = newSGR(39)
+
+	BlackB = newSGR(40)
+	RedB = newSGR(41)
+	GreenB = newSGR(42)
+	YellowB = newSGR(43)
+	BlueB = newSGR(44)
+	MagentaB = newSGR(45)
+	CyanB = newSGR(46)
+	WhiteB = newSGR(47)
+
+	DefaultB = newSGR(49)
+
+	Frame = newSGR(51)
+	Encircle = newSGR(52)
+	Overline = newSGR(53)
+
+	LightBlackF = newSGR(90)
+	LightRedF = newSGR(91)
+	LightGreenF = newSGR(92)
+	LightYellowF = newSGR(93)
+	LightBlueF = newSGR(94)
+	LightMagentaF = newSGR(95)
+	LightCyanF = newSGR(96)
+	LightWhiteF = newSGR(97)
+
+	LightBlackB = newSGR(100)
+	LightRedB = newSGR(101)
+	LightGreenB = newSGR(102)
+	LightYellowB = newSGR(103)
+	LightBlueB = newSGR(104)
+	LightMagentaB = newSGR(105)
+	LightCyanB = newSGR(106)
+	LightWhiteB = newSGR(107)
+}
--- a/vendor/k8s.io/client-go/testing/actions.go
+++ b/vendor/k8s.io/client-go/testing/actions.go
@ -0,0 +1,528 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package testing
+
+import (
+	"fmt"
+	"path"
+	"strings"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+)
+
+func NewRootGetAction(resource schema.GroupVersionResource, name string) GetActionImpl {
+	action := GetActionImpl{}
+	action.Verb = "get"
+	action.Resource = resource
+	action.Name = name
+
+	return action
+}
+
+func NewGetAction(resource schema.GroupVersionResource, namespace, name string) GetActionImpl {
+	action := GetActionImpl{}
+	action.Verb = "get"
+	action.Resource = resource
+	action.Namespace = namespace
+	action.Name = name
+
+	return action
+}
+
+func NewGetSubresourceAction(resource schema.GroupVersionResource, namespace, subresource, name string) GetActionImpl {
+	action := GetActionImpl{}
+	action.Verb = "get"
+	action.Resource = resource
+	action.Subresource = subresource
+	action.Namespace = namespace
+	action.Name = name
+
+	return action
+}
+
+func NewRootListAction(resource schema.GroupVersionResource, kind schema.GroupVersionKind, opts interface{}) ListActionImpl {
+	action := ListActionImpl{}
+	action.Verb = "list"
+	action.Resource = resource
+	action.Kind = kind
+	labelSelector, fieldSelector, _ := ExtractFromListOptions(opts)
+	action.ListRestrictions = ListRestrictions{labelSelector, fieldSelector}
+
+	return action
+}
+
+func NewListAction(resource schema.GroupVersionResource, kind schema.GroupVersionKind, namespace string, opts interface{}) ListActionImpl {
+	action := ListActionImpl{}
+	action.Verb = "list"
+	action.Resource = resource
+	action.Kind = kind
+	action.Namespace = namespace
+	labelSelector, fieldSelector, _ := ExtractFromListOptions(opts)
+	action.ListRestrictions = ListRestrictions{labelSelector, fieldSelector}
+
+	return action
+}
+
+func NewListSubresourceAction(resource schema.GroupVersionResource, name, subresource string, kind schema.GroupVersionKind, namespace string, opts interface{}) ListActionImpl {
+	action := ListActionImpl{}
+	action.Verb = "list"
+	action.Resource = resource
+	action.Subresource = subresource
+	action.Kind = kind
+	action.Namespace = namespace
+	action.Name = name
+	labelSelector, fieldSelector, _ := ExtractFromListOptions(opts)
+	action.ListRestrictions = ListRestrictions{labelSelector, fieldSelector}
+
+	return action
+}
+
+func NewRootCreateAction(resource schema.GroupVersionResource, object runtime.Object) CreateActionImpl {
+	action := CreateActionImpl{}
+	action.Verb = "create"
+	action.Resource = resource
+	action.Object = object
+
+	return action
+}
+
+func NewCreateAction(resource schema.GroupVersionResource, namespace string, object runtime.Object) CreateActionImpl {
+	action := CreateActionImpl{}
+	action.Verb = "create"
+	action.Resource = resource
+	action.Namespace = namespace
+	action.Object = object
+
+	return action
+}
+
+func NewCreateSubresourceAction(resource schema.GroupVersionResource, name, subresource string, namespace string, object runtime.Object) CreateActionImpl {
+	action := CreateActionImpl{}
+	action.Verb = "create"
+	action.Resource = resource
+	action.Subresource = subresource
+	action.Namespace = namespace
+	action.Name = name
+	action.Object = object
+
+	return action
+}
+
+func NewRootUpdateAction(resource schema.GroupVersionResource, object runtime.Object) UpdateActionImpl {
+	action := UpdateActionImpl{}
+	action.Verb = "update"
+	action.Resource = resource
+	action.Object = object
+
+	return action
+}
+
+func NewUpdateAction(resource schema.GroupVersionResource, namespace string, object runtime.Object) UpdateActionImpl {
+	action := UpdateActionImpl{}
+	action.Verb = "update"
+	action.Resource = resource
+	action.Namespace = namespace
+	action.Object = object
+
+	return action
+}
+
+func NewRootPatchAction(resource schema.GroupVersionResource, name string, patch []byte) PatchActionImpl {
+	action := PatchActionImpl{}
+	action.Verb = "patch"
+	action.Resource = resource
+	action.Name = name
+	action.Patch = patch
+
+	return action
+}
+
+func NewPatchAction(resource schema.GroupVersionResource, namespace string, name string, patch []byte) PatchActionImpl {
+	action := PatchActionImpl{}
+	action.Verb = "patch"
+	action.Resource = resource
+	action.Namespace = namespace
+	action.Name = name
+	action.Patch = patch
+
+	return action
+}
+
+func NewRootPatchSubresourceAction(resource schema.GroupVersionResource, name string, patch []byte, subresources ...string) PatchActionImpl {
+	action := PatchActionImpl{}
+	action.Verb = "patch"
+	action.Resource = resource
+	action.Subresource = path.Join(subresources...)
+	action.Name = name
+	action.Patch = patch
+
+	return action
+}
+
+func NewPatchSubresourceAction(resource schema.GroupVersionResource, namespace, name string, patch []byte, subresources ...string) PatchActionImpl {
+	action := PatchActionImpl{}
+	action.Verb = "patch"
+	action.Resource = resource
+	action.Subresource = path.Join(subresources...)
+	action.Namespace = namespace
+	action.Name = name
+	action.Patch = patch
+
+	return action
+}
+
+func NewRootUpdateSubresourceAction(resource schema.GroupVersionResource, subresource string, object runtime.Object) UpdateActionImpl {
+	action := UpdateActionImpl{}
+	action.Verb = "update"
+	action.Resource = resource
+	action.Subresource = subresource
+	action.Object = object
+
+	return action
+}
+func NewUpdateSubresourceAction(resource schema.GroupVersionResource, subresource string, namespace string, object runtime.Object) UpdateActionImpl {
+	action := UpdateActionImpl{}
+	action.Verb = "update"
+	action.Resource = resource
+	action.Subresource = subresource
+	action.Namespace = namespace
+	action.Object = object
+
+	return action
+}
+
+func NewRootDeleteAction(resource schema.GroupVersionResource, name string) DeleteActionImpl {
+	action := DeleteActionImpl{}
+	action.Verb = "delete"
+	action.Resource = resource
+	action.Name = name
+
+	return action
+}
+
+func NewDeleteAction(resource schema.GroupVersionResource, namespace, name string) DeleteActionImpl {
+	action := DeleteActionImpl{}
+	action.Verb = "delete"
+	action.Resource = resource
+	action.Namespace = namespace
+	action.Name = name
+
+	return action
+}
+
+func NewRootDeleteCollectionAction(resource schema.GroupVersionResource, opts interface{}) DeleteCollectionActionImpl {
+	action := DeleteCollectionActionImpl{}
+	action.Verb = "delete-collection"
+	action.Resource = resource
+	labelSelector, fieldSelector, _ := ExtractFromListOptions(opts)
+	action.ListRestrictions = ListRestrictions{labelSelector, fieldSelector}
+
+	return action
+}
+
+func NewDeleteCollectionAction(resource schema.GroupVersionResource, namespace string, opts interface{}) DeleteCollectionActionImpl {
+	action := DeleteCollectionActionImpl{}
+	action.Verb = "delete-collection"
+	action.Resource = resource
+	action.Namespace = namespace
+	labelSelector, fieldSelector, _ := ExtractFromListOptions(opts)
+	action.ListRestrictions = ListRestrictions{labelSelector, fieldSelector}
+
+	return action
+}
+
+func NewRootWatchAction(resource schema.GroupVersionResource, opts interface{}) WatchActionImpl {
+	action := WatchActionImpl{}
+	action.Verb = "watch"
+	action.Resource = resource
+	labelSelector, fieldSelector, resourceVersion := ExtractFromListOptions(opts)
+	action.WatchRestrictions = WatchRestrictions{labelSelector, fieldSelector, resourceVersion}
+
+	return action
+}
+
+func ExtractFromListOptions(opts interface{}) (labelSelector labels.Selector, fieldSelector fields.Selector, resourceVersion string) {
+	var err error
+	switch t := opts.(type) {
+	case metav1.ListOptions:
+		labelSelector, err = labels.Parse(t.LabelSelector)
+		if err != nil {
+			panic(fmt.Errorf("invalid selector %q: %v", t.LabelSelector, err))
+		}
+		fieldSelector, err = fields.ParseSelector(t.FieldSelector)
+		if err != nil {
+			panic(fmt.Errorf("invalid selector %q: %v", t.FieldSelector, err))
+		}
+		resourceVersion = t.ResourceVersion
+	default:
+		panic(fmt.Errorf("expect a ListOptions %T", opts))
+	}
+	if labelSelector == nil {
+		labelSelector = labels.Everything()
+	}
+	if fieldSelector == nil {
+		fieldSelector = fields.Everything()
+	}
+	return labelSelector, fieldSelector, resourceVersion
+}
+
+func NewWatchAction(resource schema.GroupVersionResource, namespace string, opts interface{}) WatchActionImpl {
+	action := WatchActionImpl{}
+	action.Verb = "watch"
+	action.Resource = resource
+	action.Namespace = namespace
+	labelSelector, fieldSelector, resourceVersion := ExtractFromListOptions(opts)
+	action.WatchRestrictions = WatchRestrictions{labelSelector, fieldSelector, resourceVersion}
+
+	return action
+}
+
+func NewProxyGetAction(resource schema.GroupVersionResource, namespace, scheme, name, port, path string, params map[string]string) ProxyGetActionImpl {
+	action := ProxyGetActionImpl{}
+	action.Verb = "get"
+	action.Resource = resource
+	action.Namespace = namespace
+	action.Scheme = scheme
+	action.Name = name
+	action.Port = port
+	action.Path = path
+	action.Params = params
+	return action
+}
+
+type ListRestrictions struct {
+	Labels labels.Selector
+	Fields fields.Selector
+}
+type WatchRestrictions struct {
+	Labels          labels.Selector
+	Fields          fields.Selector
+	ResourceVersion string
+}
+
+type Action interface {
+	GetNamespace() string
+	GetVerb() string
+	GetResource() schema.GroupVersionResource
+	GetSubresource() string
+	Matches(verb, resource string) bool
+}
+
+type GenericAction interface {
+	Action
+	GetValue() interface{}
+}
+
+type GetAction interface {
+	Action
+	GetName() string
+}
+
+type ListAction interface {
+	Action
+	GetListRestrictions() ListRestrictions
+}
+
+type CreateAction interface {
+	Action
+	GetObject() runtime.Object
+}
+
+type UpdateAction interface {
+	Action
+	GetObject() runtime.Object
+}
+
+type DeleteAction interface {
+	Action
+	GetName() string
+}
+
+type DeleteCollectionAction interface {
+	Action
+	GetListRestrictions() ListRestrictions
+}
+
+type PatchAction interface {
+	Action
+	GetName() string
+	GetPatch() []byte
+}
+
+type WatchAction interface {
+	Action
+	GetWatchRestrictions() WatchRestrictions
+}
+
+type ProxyGetAction interface {
+	Action
+	GetScheme() string
+	GetName() string
+	GetPort() string
+	GetPath() string
+	GetParams() map[string]string
+}
+
+type ActionImpl struct {
+	Namespace   string
+	Verb        string
+	Resource    schema.GroupVersionResource
+	Subresource string
+}
+
+func (a ActionImpl) GetNamespace() string {
+	return a.Namespace
+}
+func (a ActionImpl) GetVerb() string {
+	return a.Verb
+}
+func (a ActionImpl) GetResource() schema.GroupVersionResource {
+	return a.Resource
+}
+func (a ActionImpl) GetSubresource() string {
+	return a.Subresource
+}
+func (a ActionImpl) Matches(verb, resource string) bool {
+	return strings.ToLower(verb) == strings.ToLower(a.Verb) &&
+		strings.ToLower(resource) == strings.ToLower(a.Resource.Resource)
+}
+
+type GenericActionImpl struct {
+	ActionImpl
+	Value interface{}
+}
+
+func (a GenericActionImpl) GetValue() interface{} {
+	return a.Value
+}
+
+type GetActionImpl struct {
+	ActionImpl
+	Name string
+}
+
+func (a GetActionImpl) GetName() string {
+	return a.Name
+}
+
+type ListActionImpl struct {
+	ActionImpl
+	Kind             schema.GroupVersionKind
+	Name             string
+	ListRestrictions ListRestrictions
+}
+
+func (a ListActionImpl) GetKind() schema.GroupVersionKind {
+	return a.Kind
+}
+
+func (a ListActionImpl) GetListRestrictions() ListRestrictions {
+	return a.ListRestrictions
+}
+
+type CreateActionImpl struct {
+	ActionImpl
+	Name   string
+	Object runtime.Object
+}
+
+func (a CreateActionImpl) GetObject() runtime.Object {
+	return a.Object
+}
+
+type UpdateActionImpl struct {
+	ActionImpl
+	Object runtime.Object
+}
+
+func (a UpdateActionImpl) GetObject() runtime.Object {
+	return a.Object
+}
+
+type PatchActionImpl struct {
+	ActionImpl
+	Name  string
+	Patch []byte
+}
+
+func (a PatchActionImpl) GetName() string {
+	return a.Name
+}
+
+func (a PatchActionImpl) GetPatch() []byte {
+	return a.Patch
+}
+
+type DeleteActionImpl struct {
+	ActionImpl
+	Name string
+}
+
+func (a DeleteActionImpl) GetName() string {
+	return a.Name
+}
+
+type DeleteCollectionActionImpl struct {
+	ActionImpl
+	ListRestrictions ListRestrictions
+}
+
+func (a DeleteCollectionActionImpl) GetListRestrictions() ListRestrictions {
+	return a.ListRestrictions
+}
+
+type WatchActionImpl struct {
+	ActionImpl
+	WatchRestrictions WatchRestrictions
+}
+
+func (a WatchActionImpl) GetWatchRestrictions() WatchRestrictions {
+	return a.WatchRestrictions
+}
+
+type ProxyGetActionImpl struct {
+	ActionImpl
+	Scheme string
+	Name   string
+	Port   string
+	Path   string
+	Params map[string]string
+}
+
+func (a ProxyGetActionImpl) GetScheme() string {
+	return a.Scheme
+}
+
+func (a ProxyGetActionImpl) GetName() string {
+	return a.Name
+}
+
+func (a ProxyGetActionImpl) GetPort() string {
+	return a.Port
+}
+
+func (a ProxyGetActionImpl) GetPath() string {
+	return a.Path
+}
+
+func (a ProxyGetActionImpl) GetParams() map[string]string {
+	return a.Params
+}
--- a/vendor/k8s.io/client-go/testing/fake.go
+++ b/vendor/k8s.io/client-go/testing/fake.go
@ -0,0 +1,259 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package testing
+
+import (
+	"fmt"
+	"sync"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/version"
+	"k8s.io/apimachinery/pkg/watch"
+	kubeversion "k8s.io/client-go/pkg/version"
+	restclient "k8s.io/client-go/rest"
+)
+
+// Fake implements client.Interface. Meant to be embedded into a struct to get
+// a default implementation. This makes faking out just the method you want to
+// test easier.
+type Fake struct {
+	sync.RWMutex
+	actions []Action // these may be castable to other types, but "Action" is the minimum
+
+	// ReactionChain is the list of reactors that will be attempted for every
+	// request in the order they are tried.
+	ReactionChain []Reactor
+	// WatchReactionChain is the list of watch reactors that will be attempted
+	// for every request in the order they are tried.
+	WatchReactionChain []WatchReactor
+	// ProxyReactionChain is the list of proxy reactors that will be attempted
+	// for every request in the order they are tried.
+	ProxyReactionChain []ProxyReactor
+
+	Resources []*metav1.APIResourceList
+}
+
+// Reactor is an interface to allow the composition of reaction functions.
+type Reactor interface {
+	// Handles indicates whether or not this Reactor deals with a given
+	// action.
+	Handles(action Action) bool
+	// React handles the action and returns results.  It may choose to
+	// delegate by indicated handled=false.
+	React(action Action) (handled bool, ret runtime.Object, err error)
+}
+
+// WatchReactor is an interface to allow the composition of watch functions.
+type WatchReactor interface {
+	// Handles indicates whether or not this Reactor deals with a given
+	// action.
+	Handles(action Action) bool
+	// React handles a watch action and returns results.  It may choose to
+	// delegate by indicating handled=false.
+	React(action Action) (handled bool, ret watch.Interface, err error)
+}
+
+// ProxyReactor is an interface to allow the composition of proxy get
+// functions.
+type ProxyReactor interface {
+	// Handles indicates whether or not this Reactor deals with a given
+	// action.
+	Handles(action Action) bool
+	// React handles a watch action and returns results.  It may choose to
+	// delegate by indicating handled=false.
+	React(action Action) (handled bool, ret restclient.ResponseWrapper, err error)
+}
+
+// ReactionFunc is a function that returns an object or error for a given
+// Action.  If "handled" is false, then the test client will ignore the
+// results and continue to the next ReactionFunc.  A ReactionFunc can describe
+// reactions on subresources by testing the result of the action's
+// GetSubresource() method.
+type ReactionFunc func(action Action) (handled bool, ret runtime.Object, err error)
+
+// WatchReactionFunc is a function that returns a watch interface.  If
+// "handled" is false, then the test client will ignore the results and
+// continue to the next ReactionFunc.
+type WatchReactionFunc func(action Action) (handled bool, ret watch.Interface, err error)
+
+// ProxyReactionFunc is a function that returns a ResponseWrapper interface
+// for a given Action.  If "handled" is false, then the test client will
+// ignore the results and continue to the next ProxyReactionFunc.
+type ProxyReactionFunc func(action Action) (handled bool, ret restclient.ResponseWrapper, err error)
+
+// AddReactor appends a reactor to the end of the chain.
+func (c *Fake) AddReactor(verb, resource string, reaction ReactionFunc) {
+	c.ReactionChain = append(c.ReactionChain, &SimpleReactor{verb, resource, reaction})
+}
+
+// PrependReactor adds a reactor to the beginning of the chain.
+func (c *Fake) PrependReactor(verb, resource string, reaction ReactionFunc) {
+	c.ReactionChain = append([]Reactor{&SimpleReactor{verb, resource, reaction}}, c.ReactionChain...)
+}
+
+// AddWatchReactor appends a reactor to the end of the chain.
+func (c *Fake) AddWatchReactor(resource string, reaction WatchReactionFunc) {
+	c.WatchReactionChain = append(c.WatchReactionChain, &SimpleWatchReactor{resource, reaction})
+}
+
+// PrependWatchReactor adds a reactor to the beginning of the chain.
+func (c *Fake) PrependWatchReactor(resource string, reaction WatchReactionFunc) {
+	c.WatchReactionChain = append([]WatchReactor{&SimpleWatchReactor{resource, reaction}}, c.WatchReactionChain...)
+}
+
+// AddProxyReactor appends a reactor to the end of the chain.
+func (c *Fake) AddProxyReactor(resource string, reaction ProxyReactionFunc) {
+	c.ProxyReactionChain = append(c.ProxyReactionChain, &SimpleProxyReactor{resource, reaction})
+}
+
+// PrependProxyReactor adds a reactor to the beginning of the chain.
+func (c *Fake) PrependProxyReactor(resource string, reaction ProxyReactionFunc) {
+	c.ProxyReactionChain = append([]ProxyReactor{&SimpleProxyReactor{resource, reaction}}, c.ProxyReactionChain...)
+}
+
+// Invokes records the provided Action and then invokes the ReactionFunc that
+// handles the action if one exists. defaultReturnObj is expected to be of the
+// same type a normal call would return.
+func (c *Fake) Invokes(action Action, defaultReturnObj runtime.Object) (runtime.Object, error) {
+	c.Lock()
+	defer c.Unlock()
+
+	c.actions = append(c.actions, action)
+	for _, reactor := range c.ReactionChain {
+		if !reactor.Handles(action) {
+			continue
+		}
+
+		handled, ret, err := reactor.React(action)
+		if !handled {
+			continue
+		}
+
+		return ret, err
+	}
+
+	return defaultReturnObj, nil
+}
+
+// InvokesWatch records the provided Action and then invokes the ReactionFunc
+// that handles the action if one exists.
+func (c *Fake) InvokesWatch(action Action) (watch.Interface, error) {
+	c.Lock()
+	defer c.Unlock()
+
+	c.actions = append(c.actions, action)
+	for _, reactor := range c.WatchReactionChain {
+		if !reactor.Handles(action) {
+			continue
+		}
+
+		handled, ret, err := reactor.React(action)
+		if !handled {
+			continue
+		}
+
+		return ret, err
+	}
+
+	return nil, fmt.Errorf("unhandled watch: %#v", action)
+}
+
+// InvokesProxy records the provided Action and then invokes the ReactionFunc
+// that handles the action if one exists.
+func (c *Fake) InvokesProxy(action Action) restclient.ResponseWrapper {
+	c.Lock()
+	defer c.Unlock()
+
+	c.actions = append(c.actions, action)
+	for _, reactor := range c.ProxyReactionChain {
+		if !reactor.Handles(action) {
+			continue
+		}
+
+		handled, ret, err := reactor.React(action)
+		if !handled || err != nil {
+			continue
+		}
+
+		return ret
+	}
+
+	return nil
+}
+
+// ClearActions clears the history of actions called on the fake client.
+func (c *Fake) ClearActions() {
+	c.Lock()
+	defer c.Unlock()
+
+	c.actions = make([]Action, 0)
+}
+
+// Actions returns a chronologically ordered slice fake actions called on the
+// fake client.
+func (c *Fake) Actions() []Action {
+	c.RLock()
+	defer c.RUnlock()
+	fa := make([]Action, len(c.actions))
+	copy(fa, c.actions)
+	return fa
+}
+
+// TODO: this probably should be moved to somewhere else.
+type FakeDiscovery struct {
+	*Fake
+}
+
+func (c *FakeDiscovery) ServerResourcesForGroupVersion(groupVersion string) (*metav1.APIResourceList, error) {
+	action := ActionImpl{
+		Verb:     "get",
+		Resource: schema.GroupVersionResource{Resource: "resource"},
+	}
+	c.Invokes(action, nil)
+	for _, rl := range c.Resources {
+		if rl.GroupVersion == groupVersion {
+			return rl, nil
+		}
+	}
+
+	return nil, fmt.Errorf("GroupVersion %q not found", groupVersion)
+}
+
+func (c *FakeDiscovery) ServerResources() ([]*metav1.APIResourceList, error) {
+	action := ActionImpl{
+		Verb:     "get",
+		Resource: schema.GroupVersionResource{Resource: "resource"},
+	}
+	c.Invokes(action, nil)
+	return c.Resources, nil
+}
+
+func (c *FakeDiscovery) ServerGroups() (*metav1.APIGroupList, error) {
+	return nil, nil
+}
+
+func (c *FakeDiscovery) ServerVersion() (*version.Info, error) {
+	action := ActionImpl{}
+	action.Verb = "get"
+	action.Resource = schema.GroupVersionResource{Resource: "version"}
+
+	c.Invokes(action, nil)
+	versionInfo := kubeversion.Get()
+	return &versionInfo, nil
+}
--- a/vendor/k8s.io/client-go/testing/fixture.go
+++ b/vendor/k8s.io/client-go/testing/fixture.go
@ -0,0 +1,464 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package testing
+
+import (
+	"fmt"
+	"sync"
+
+	"k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/api/meta"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/watch"
+	restclient "k8s.io/client-go/rest"
+)
+
+// ObjectTracker keeps track of objects. It is intended to be used to
+// fake calls to a server by returning objects based on their kind,
+// namespace and name.
+type ObjectTracker interface {
+	// Add adds an object to the tracker. If object being added
+	// is a list, its items are added separately.
+	Add(obj runtime.Object) error
+
+	// Get retrieves the object by its kind, namespace and name.
+	Get(gvr schema.GroupVersionResource, ns, name string) (runtime.Object, error)
+
+	// Create adds an object to the tracker in the specified namespace.
+	Create(gvr schema.GroupVersionResource, obj runtime.Object, ns string) error
+
+	// Update updates an existing object in the tracker in the specified namespace.
+	Update(gvr schema.GroupVersionResource, obj runtime.Object, ns string) error
+
+	// List retrieves all objects of a given kind in the given
+	// namespace. Only non-List kinds are accepted.
+	List(gvr schema.GroupVersionResource, gvk schema.GroupVersionKind, ns string) (runtime.Object, error)
+
+	// Delete deletes an existing object from the tracker. If object
+	// didn't exist in the tracker prior to deletion, Delete returns
+	// no error.
+	Delete(gvr schema.GroupVersionResource, ns, name string) error
+}
+
+// ObjectScheme abstracts the implementation of common operations on objects.
+type ObjectScheme interface {
+	runtime.ObjectCreater
+	runtime.ObjectCopier
+	runtime.ObjectTyper
+}
+
+// ObjectReaction returns a ReactionFunc that applies core.Action to
+// the given tracker.
+func ObjectReaction(tracker ObjectTracker) ReactionFunc {
+	return func(action Action) (bool, runtime.Object, error) {
+		ns := action.GetNamespace()
+		gvr := action.GetResource()
+
+		// Here and below we need to switch on implementation types,
+		// not on interfaces, as some interfaces are identical
+		// (e.g. UpdateAction and CreateAction), so if we use them,
+		// updates and creates end up matching the same case branch.
+		switch action := action.(type) {
+
+		case ListActionImpl:
+			obj, err := tracker.List(gvr, action.GetKind(), ns)
+			return true, obj, err
+
+		case GetActionImpl:
+			obj, err := tracker.Get(gvr, ns, action.GetName())
+			return true, obj, err
+
+		case CreateActionImpl:
+			objMeta, err := meta.Accessor(action.GetObject())
+			if err != nil {
+				return true, nil, err
+			}
+			if action.GetSubresource() == "" {
+				err = tracker.Create(gvr, action.GetObject(), ns)
+			} else {
+				// TODO: Currently we're handling subresource creation as an update
+				// on the enclosing resource. This works for some subresources but
+				// might not be generic enough.
+				err = tracker.Update(gvr, action.GetObject(), ns)
+			}
+			if err != nil {
+				return true, nil, err
+			}
+			obj, err := tracker.Get(gvr, ns, objMeta.GetName())
+			return true, obj, err
+
+		case UpdateActionImpl:
+			objMeta, err := meta.Accessor(action.GetObject())
+			if err != nil {
+				return true, nil, err
+			}
+			err = tracker.Update(gvr, action.GetObject(), ns)
+			if err != nil {
+				return true, nil, err
+			}
+			obj, err := tracker.Get(gvr, ns, objMeta.GetName())
+			return true, obj, err
+
+		case DeleteActionImpl:
+			err := tracker.Delete(gvr, ns, action.GetName())
+			if err != nil {
+				return true, nil, err
+			}
+			return true, nil, nil
+
+		default:
+			return false, nil, fmt.Errorf("no reaction implemented for %s", action)
+		}
+	}
+}
+
+type tracker struct {
+	scheme  ObjectScheme
+	decoder runtime.Decoder
+	lock    sync.RWMutex
+	objects map[schema.GroupVersionResource][]runtime.Object
+}
+
+var _ ObjectTracker = &tracker{}
+
+// NewObjectTracker returns an ObjectTracker that can be used to keep track
+// of objects for the fake clientset. Mostly useful for unit tests.
+func NewObjectTracker(scheme ObjectScheme, decoder runtime.Decoder) ObjectTracker {
+	return &tracker{
+		scheme:  scheme,
+		decoder: decoder,
+		objects: make(map[schema.GroupVersionResource][]runtime.Object),
+	}
+}
+
+func (t *tracker) List(gvr schema.GroupVersionResource, gvk schema.GroupVersionKind, ns string) (runtime.Object, error) {
+	// Heuristic for list kind: original kind + List suffix. Might
+	// not always be true but this tracker has a pretty limited
+	// understanding of the actual API model.
+	listGVK := gvk
+	listGVK.Kind = listGVK.Kind + "List"
+	// GVK does have the concept of "internal version". The scheme recognizes
+	// the runtime.APIVersionInternal, but not the empty string.
+	if listGVK.Version == "" {
+		listGVK.Version = runtime.APIVersionInternal
+	}
+
+	list, err := t.scheme.New(listGVK)
+	if err != nil {
+		return nil, err
+	}
+
+	if !meta.IsListType(list) {
+		return nil, fmt.Errorf("%q is not a list type", listGVK.Kind)
+	}
+
+	t.lock.RLock()
+	defer t.lock.RUnlock()
+
+	objs, ok := t.objects[gvr]
+	if !ok {
+		return list, nil
+	}
+
+	matchingObjs, err := filterByNamespaceAndName(objs, ns, "")
+	if err != nil {
+		return nil, err
+	}
+	if err := meta.SetList(list, matchingObjs); err != nil {
+		return nil, err
+	}
+	if list, err = t.scheme.Copy(list); err != nil {
+		return nil, err
+	}
+	return list, nil
+}
+
+func (t *tracker) Get(gvr schema.GroupVersionResource, ns, name string) (runtime.Object, error) {
+	errNotFound := errors.NewNotFound(gvr.GroupResource(), name)
+
+	t.lock.RLock()
+	defer t.lock.RUnlock()
+
+	objs, ok := t.objects[gvr]
+	if !ok {
+		return nil, errNotFound
+	}
+
+	matchingObjs, err := filterByNamespaceAndName(objs, ns, name)
+	if err != nil {
+		return nil, err
+	}
+	if len(matchingObjs) == 0 {
+		return nil, errNotFound
+	}
+	if len(matchingObjs) > 1 {
+		return nil, fmt.Errorf("more than one object matched gvr %s, ns: %q name: %q", gvr, ns, name)
+	}
+
+	// Only one object should match in the tracker if it works
+	// correctly, as Add/Update methods enforce kind/namespace/name
+	// uniqueness.
+	obj, err := t.scheme.Copy(matchingObjs[0])
+	if err != nil {
+		return nil, err
+	}
+
+	if status, ok := obj.(*metav1.Status); ok {
+		if status.Status != metav1.StatusSuccess {
+			return nil, &errors.StatusError{ErrStatus: *status}
+		}
+	}
+
+	return obj, nil
+}
+
+func (t *tracker) Add(obj runtime.Object) error {
+	if meta.IsListType(obj) {
+		return t.addList(obj, false)
+	}
+	objMeta, err := meta.Accessor(obj)
+	if err != nil {
+		return err
+	}
+	gvks, _, err := t.scheme.ObjectKinds(obj)
+	if err != nil {
+		return err
+	}
+	if len(gvks) == 0 {
+		return fmt.Errorf("no registered kinds for %v", obj)
+	}
+	for _, gvk := range gvks {
+		// NOTE: UnsafeGuessKindToResource is a heuristic and default match. The
+		// actual registration in apiserver can specify arbitrary route for a
+		// gvk. If a test uses such objects, it cannot preset the tracker with
+		// objects via Add(). Instead, it should trigger the Create() function
+		// of the tracker, where an arbitrary gvr can be specified.
+		gvr, _ := meta.UnsafeGuessKindToResource(gvk)
+		// Resource doesn't have the concept of "__internal" version, just set it to "".
+		if gvr.Version == runtime.APIVersionInternal {
+			gvr.Version = ""
+		}
+
+		err := t.add(gvr, obj, objMeta.GetNamespace(), false)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (t *tracker) Create(gvr schema.GroupVersionResource, obj runtime.Object, ns string) error {
+	return t.add(gvr, obj, ns, false)
+}
+
+func (t *tracker) Update(gvr schema.GroupVersionResource, obj runtime.Object, ns string) error {
+	return t.add(gvr, obj, ns, true)
+}
+
+func (t *tracker) add(gvr schema.GroupVersionResource, obj runtime.Object, ns string, replaceExisting bool) error {
+	t.lock.Lock()
+	defer t.lock.Unlock()
+
+	gr := gvr.GroupResource()
+
+	// To avoid the object from being accidentally modified by caller
+	// after it's been added to the tracker, we always store the deep
+	// copy.
+	obj, err := t.scheme.Copy(obj)
+	if err != nil {
+		return err
+	}
+
+	newMeta, err := meta.Accessor(obj)
+	if err != nil {
+		return err
+	}
+
+	// Propagate namespace to the new object if hasn't already been set.
+	if len(newMeta.GetNamespace()) == 0 {
+		newMeta.SetNamespace(ns)
+	}
+
+	if ns != newMeta.GetNamespace() {
+		msg := fmt.Sprintf("request namespace does not match object namespace, request: %q object: %q", ns, newMeta.GetNamespace())
+		return errors.NewBadRequest(msg)
+	}
+
+	for i, existingObj := range t.objects[gvr] {
+		oldMeta, err := meta.Accessor(existingObj)
+		if err != nil {
+			return err
+		}
+		if oldMeta.GetNamespace() == newMeta.GetNamespace() && oldMeta.GetName() == newMeta.GetName() {
+			if replaceExisting {
+				t.objects[gvr][i] = obj
+				return nil
+			}
+			return errors.NewAlreadyExists(gr, newMeta.GetName())
+		}
+	}
+
+	if replaceExisting {
+		// Tried to update but no matching object was found.
+		return errors.NewNotFound(gr, newMeta.GetName())
+	}
+
+	t.objects[gvr] = append(t.objects[gvr], obj)
+
+	return nil
+}
+
+func (t *tracker) addList(obj runtime.Object, replaceExisting bool) error {
+	list, err := meta.ExtractList(obj)
+	if err != nil {
+		return err
+	}
+	errs := runtime.DecodeList(list, t.decoder)
+	if len(errs) > 0 {
+		return errs[0]
+	}
+	for _, obj := range list {
+		if err := t.Add(obj); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (t *tracker) Delete(gvr schema.GroupVersionResource, ns, name string) error {
+	t.lock.Lock()
+	defer t.lock.Unlock()
+
+	found := false
+
+	for i, existingObj := range t.objects[gvr] {
+		objMeta, err := meta.Accessor(existingObj)
+		if err != nil {
+			return err
+		}
+		if objMeta.GetNamespace() == ns && objMeta.GetName() == name {
+			t.objects[gvr] = append(t.objects[gvr][:i], t.objects[gvr][i+1:]...)
+			found = true
+			break
+		}
+	}
+
+	if found {
+		return nil
+	}
+
+	return errors.NewNotFound(gvr.GroupResource(), name)
+}
+
+// filterByNamespaceAndName returns all objects in the collection that
+// match provided namespace and name. Empty namespace matches
+// non-namespaced objects.
+func filterByNamespaceAndName(objs []runtime.Object, ns, name string) ([]runtime.Object, error) {
+	var res []runtime.Object
+
+	for _, obj := range objs {
+		acc, err := meta.Accessor(obj)
+		if err != nil {
+			return nil, err
+		}
+		if ns != "" && acc.GetNamespace() != ns {
+			continue
+		}
+		if name != "" && acc.GetName() != name {
+			continue
+		}
+		res = append(res, obj)
+	}
+
+	return res, nil
+}
+
+func DefaultWatchReactor(watchInterface watch.Interface, err error) WatchReactionFunc {
+	return func(action Action) (bool, watch.Interface, error) {
+		return true, watchInterface, err
+	}
+}
+
+// SimpleReactor is a Reactor.  Each reaction function is attached to a given verb,resource tuple.  "*" in either field matches everything for that value.
+// For instance, *,pods matches all verbs on pods.  This allows for easier composition of reaction functions
+type SimpleReactor struct {
+	Verb     string
+	Resource string
+
+	Reaction ReactionFunc
+}
+
+func (r *SimpleReactor) Handles(action Action) bool {
+	verbCovers := r.Verb == "*" || r.Verb == action.GetVerb()
+	if !verbCovers {
+		return false
+	}
+	resourceCovers := r.Resource == "*" || r.Resource == action.GetResource().Resource
+	if !resourceCovers {
+		return false
+	}
+
+	return true
+}
+
+func (r *SimpleReactor) React(action Action) (bool, runtime.Object, error) {
+	return r.Reaction(action)
+}
+
+// SimpleWatchReactor is a WatchReactor.  Each reaction function is attached to a given resource.  "*" matches everything for that value.
+// For instance, *,pods matches all verbs on pods.  This allows for easier composition of reaction functions
+type SimpleWatchReactor struct {
+	Resource string
+
+	Reaction WatchReactionFunc
+}
+
+func (r *SimpleWatchReactor) Handles(action Action) bool {
+	resourceCovers := r.Resource == "*" || r.Resource == action.GetResource().Resource
+	if !resourceCovers {
+		return false
+	}
+
+	return true
+}
+
+func (r *SimpleWatchReactor) React(action Action) (bool, watch.Interface, error) {
+	return r.Reaction(action)
+}
+
+// SimpleProxyReactor is a ProxyReactor.  Each reaction function is attached to a given resource.  "*" matches everything for that value.
+// For instance, *,pods matches all verbs on pods.  This allows for easier composition of reaction functions.
+type SimpleProxyReactor struct {
+	Resource string
+
+	Reaction ProxyReactionFunc
+}
+
+func (r *SimpleProxyReactor) Handles(action Action) bool {
+	resourceCovers := r.Resource == "*" || r.Resource == action.GetResource().Resource
+	if !resourceCovers {
+		return false
+	}
+
+	return true
+}
+
+func (r *SimpleProxyReactor) React(action Action) (bool, restclient.ResponseWrapper, error) {
+	return r.Reaction(action)
+}
--- a/vendor/k8s.io/kubernetes/pkg/api/v1/pod/util.go
+++ b/vendor/k8s.io/kubernetes/pkg/api/v1/pod/util.go
@ -0,0 +1,296 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package pod
+
+import (
+	"fmt"
+	"time"
+
+	"k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
+)
+
+// FindPort locates the container port for the given pod and portName.  If the
+// targetPort is a number, use that.  If the targetPort is a string, look that
+// string up in all named ports in all containers in the target pod.  If no
+// match is found, fail.
+func FindPort(pod *v1.Pod, svcPort *v1.ServicePort) (int, error) {
+	portName := svcPort.TargetPort
+	switch portName.Type {
+	case intstr.String:
+		name := portName.StrVal
+		for _, container := range pod.Spec.Containers {
+			for _, port := range container.Ports {
+				if port.Name == name && port.Protocol == svcPort.Protocol {
+					return int(port.ContainerPort), nil
+				}
+			}
+		}
+	case intstr.Int:
+		return portName.IntValue(), nil
+	}
+
+	return 0, fmt.Errorf("no suitable port for manifest: %s", pod.UID)
+}
+
+// Visitor is called with each object name, and returns true if visiting should continue
+type Visitor func(name string) (shouldContinue bool)
+
+// VisitPodSecretNames invokes the visitor function with the name of every secret
+// referenced by the pod spec. If visitor returns false, visiting is short-circuited.
+// Transitive references (e.g. pod -> pvc -> pv -> secret) are not visited.
+// Returns true if visiting completed, false if visiting was short-circuited.
+func VisitPodSecretNames(pod *v1.Pod, visitor Visitor) bool {
+	for _, reference := range pod.Spec.ImagePullSecrets {
+		if !visitor(reference.Name) {
+			return false
+		}
+	}
+	for i := range pod.Spec.InitContainers {
+		if !visitContainerSecretNames(&pod.Spec.InitContainers[i], visitor) {
+			return false
+		}
+	}
+	for i := range pod.Spec.Containers {
+		if !visitContainerSecretNames(&pod.Spec.Containers[i], visitor) {
+			return false
+		}
+	}
+	var source *v1.VolumeSource
+
+	for i := range pod.Spec.Volumes {
+		source = &pod.Spec.Volumes[i].VolumeSource
+		switch {
+		case source.AzureFile != nil:
+			if len(source.AzureFile.SecretName) > 0 && !visitor(source.AzureFile.SecretName) {
+				return false
+			}
+		case source.CephFS != nil:
+			if source.CephFS.SecretRef != nil && !visitor(source.CephFS.SecretRef.Name) {
+				return false
+			}
+		case source.FlexVolume != nil:
+			if source.FlexVolume.SecretRef != nil && !visitor(source.FlexVolume.SecretRef.Name) {
+				return false
+			}
+		case source.Projected != nil:
+			for j := range source.Projected.Sources {
+				if source.Projected.Sources[j].Secret != nil {
+					if !visitor(source.Projected.Sources[j].Secret.Name) {
+						return false
+					}
+				}
+			}
+		case source.RBD != nil:
+			if source.RBD.SecretRef != nil && !visitor(source.RBD.SecretRef.Name) {
+				return false
+			}
+		case source.Secret != nil:
+			if !visitor(source.Secret.SecretName) {
+				return false
+			}
+		case source.ScaleIO != nil:
+			if source.ScaleIO.SecretRef != nil && !visitor(source.ScaleIO.SecretRef.Name) {
+				return false
+			}
+		case source.ISCSI != nil:
+			if source.ISCSI.SecretRef != nil && !visitor(source.ISCSI.SecretRef.Name) {
+				return false
+			}
+		case source.StorageOS != nil:
+			if source.StorageOS.SecretRef != nil && !visitor(source.StorageOS.SecretRef.Name) {
+				return false
+			}
+		}
+	}
+	return true
+}
+
+func visitContainerSecretNames(container *v1.Container, visitor Visitor) bool {
+	for _, env := range container.EnvFrom {
+		if env.SecretRef != nil {
+			if !visitor(env.SecretRef.Name) {
+				return false
+			}
+		}
+	}
+	for _, envVar := range container.Env {
+		if envVar.ValueFrom != nil && envVar.ValueFrom.SecretKeyRef != nil {
+			if !visitor(envVar.ValueFrom.SecretKeyRef.Name) {
+				return false
+			}
+		}
+	}
+	return true
+}
+
+// VisitPodConfigmapNames invokes the visitor function with the name of every configmap
+// referenced by the pod spec. If visitor returns false, visiting is short-circuited.
+// Transitive references (e.g. pod -> pvc -> pv -> secret) are not visited.
+// Returns true if visiting completed, false if visiting was short-circuited.
+func VisitPodConfigmapNames(pod *v1.Pod, visitor Visitor) bool {
+	for i := range pod.Spec.InitContainers {
+		if !visitContainerConfigmapNames(&pod.Spec.InitContainers[i], visitor) {
+			return false
+		}
+	}
+	for i := range pod.Spec.Containers {
+		if !visitContainerConfigmapNames(&pod.Spec.Containers[i], visitor) {
+			return false
+		}
+	}
+	var source *v1.VolumeSource
+	for i := range pod.Spec.Volumes {
+		source = &pod.Spec.Volumes[i].VolumeSource
+		switch {
+		case source.Projected != nil:
+			for j := range source.Projected.Sources {
+				if source.Projected.Sources[j].ConfigMap != nil {
+					if !visitor(source.Projected.Sources[j].ConfigMap.Name) {
+						return false
+					}
+				}
+			}
+		case source.ConfigMap != nil:
+			if !visitor(source.ConfigMap.Name) {
+				return false
+			}
+		}
+	}
+	return true
+}
+
+func visitContainerConfigmapNames(container *v1.Container, visitor Visitor) bool {
+	for _, env := range container.EnvFrom {
+		if env.ConfigMapRef != nil {
+			if !visitor(env.ConfigMapRef.Name) {
+				return false
+			}
+		}
+	}
+	for _, envVar := range container.Env {
+		if envVar.ValueFrom != nil && envVar.ValueFrom.ConfigMapKeyRef != nil {
+			if !visitor(envVar.ValueFrom.ConfigMapKeyRef.Name) {
+				return false
+			}
+		}
+	}
+	return true
+}
+
+// GetContainerStatus extracts the status of container "name" from "statuses".
+// It also returns if "name" exists.
+func GetContainerStatus(statuses []v1.ContainerStatus, name string) (v1.ContainerStatus, bool) {
+	for i := range statuses {
+		if statuses[i].Name == name {
+			return statuses[i], true
+		}
+	}
+	return v1.ContainerStatus{}, false
+}
+
+// GetExistingContainerStatus extracts the status of container "name" from "statuses",
+// and returns empty status if "name" does not exist.
+func GetExistingContainerStatus(statuses []v1.ContainerStatus, name string) v1.ContainerStatus {
+	for i := range statuses {
+		if statuses[i].Name == name {
+			return statuses[i]
+		}
+	}
+	return v1.ContainerStatus{}
+}
+
+// IsPodAvailable returns true if a pod is available; false otherwise.
+// Precondition for an available pod is that it must be ready. On top
+// of that, there are two cases when a pod can be considered available:
+// 1. minReadySeconds == 0, or
+// 2. LastTransitionTime (is set) + minReadySeconds < current time
+func IsPodAvailable(pod *v1.Pod, minReadySeconds int32, now metav1.Time) bool {
+	if !IsPodReady(pod) {
+		return false
+	}
+
+	c := GetPodReadyCondition(pod.Status)
+	minReadySecondsDuration := time.Duration(minReadySeconds) * time.Second
+	if minReadySeconds == 0 || !c.LastTransitionTime.IsZero() && c.LastTransitionTime.Add(minReadySecondsDuration).Before(now.Time) {
+		return true
+	}
+	return false
+}
+
+// IsPodReady returns true if a pod is ready; false otherwise.
+func IsPodReady(pod *v1.Pod) bool {
+	return IsPodReadyConditionTrue(pod.Status)
+}
+
+// IsPodReady retruns true if a pod is ready; false otherwise.
+func IsPodReadyConditionTrue(status v1.PodStatus) bool {
+	condition := GetPodReadyCondition(status)
+	return condition != nil && condition.Status == v1.ConditionTrue
+}
+
+// Extracts the pod ready condition from the given status and returns that.
+// Returns nil if the condition is not present.
+func GetPodReadyCondition(status v1.PodStatus) *v1.PodCondition {
+	_, condition := GetPodCondition(&status, v1.PodReady)
+	return condition
+}
+
+// GetPodCondition extracts the provided condition from the given status and returns that.
+// Returns nil and -1 if the condition is not present, and the index of the located condition.
+func GetPodCondition(status *v1.PodStatus, conditionType v1.PodConditionType) (int, *v1.PodCondition) {
+	if status == nil {
+		return -1, nil
+	}
+	for i := range status.Conditions {
+		if status.Conditions[i].Type == conditionType {
+			return i, &status.Conditions[i]
+		}
+	}
+	return -1, nil
+}
+
+// Updates existing pod condition or creates a new one. Sets LastTransitionTime to now if the
+// status has changed.
+// Returns true if pod condition has changed or has been added.
+func UpdatePodCondition(status *v1.PodStatus, condition *v1.PodCondition) bool {
+	condition.LastTransitionTime = metav1.Now()
+	// Try to find this pod condition.
+	conditionIndex, oldCondition := GetPodCondition(status, condition.Type)
+
+	if oldCondition == nil {
+		// We are adding new pod condition.
+		status.Conditions = append(status.Conditions, *condition)
+		return true
+	} else {
+		// We are updating an existing condition, so we need to check if it has changed.
+		if condition.Status == oldCondition.Status {
+			condition.LastTransitionTime = oldCondition.LastTransitionTime
+		}
+
+		isEqual := condition.Status == oldCondition.Status &&
+			condition.Reason == oldCondition.Reason &&
+			condition.Message == oldCondition.Message &&
+			condition.LastProbeTime.Equal(&oldCondition.LastProbeTime) &&
+			condition.LastTransitionTime.Equal(&oldCondition.LastTransitionTime)
+
+		status.Conditions[conditionIndex] = *condition
+		// Return true if one of the fields have changed.
+		return !isEqual
+	}
+}