package progress import ( "fmt" "strconv" "testing" "github.com/docker/docker/api/types/swarm" "github.com/docker/docker/pkg/progress" "gotest.tools/v3/assert" is "gotest.tools/v3/assert/cmp" ) type mockProgress struct { p []progress.Progress } func (mp *mockProgress) WriteProgress(p progress.Progress) error { mp.p = append(mp.p, p) return nil } func (mp *mockProgress) clear() { mp.p = nil } type updaterTester struct { t *testing.T updater progressUpdater p *mockProgress service swarm.Service activeNodes map[string]struct{} rollback bool } func (u updaterTester) testUpdater(tasks []swarm.Task, expectedConvergence bool, expectedProgress []progress.Progress) { u.p.clear() converged, err := u.updater.update(u.service, tasks, u.activeNodes, u.rollback) assert.Check(u.t, err) assert.Check(u.t, is.Equal(expectedConvergence, converged)) assert.Check(u.t, is.DeepEqual(expectedProgress, u.p.p)) } func (u updaterTester) testUpdaterNoOrder(tasks []swarm.Task, expectedConvergence bool, expectedProgress []progress.Progress) { u.p.clear() converged, err := u.updater.update(u.service, tasks, u.activeNodes, u.rollback) assert.Check(u.t, err) assert.Check(u.t, is.Equal(expectedConvergence, converged)) // instead of checking that expected and actual match exactly, verify that // they are the same length, and every time from actual is in expected. assert.Check(u.t, is.Equal(len(expectedProgress), len(u.p.p))) for _, prog := range expectedProgress { assert.Check(u.t, is.Contains(u.p.p, prog)) } } func TestReplicatedProgressUpdaterOneReplica(t *testing.T) { replicas := uint64(1) service := swarm.Service{ Spec: swarm.ServiceSpec{ Mode: swarm.ServiceMode{ Replicated: &swarm.ReplicatedService{ Replicas: &replicas, }, }, }, } p := &mockProgress{} ut := updaterTester{ t: t, updater: &replicatedProgressUpdater{ progressOut: p, }, p: p, activeNodes: map[string]struct{}{"a": {}, "b": {}}, service: service, } tasks := []swarm.Task{} ut.testUpdater(tasks, false, []progress.Progress{ {ID: "overall progress", Action: "0 out of 1 tasks"}, {ID: "1/1", Action: " "}, {ID: "overall progress", Action: "0 out of 1 tasks"}, }) // Task with DesiredState beyond Running is ignored tasks = append(tasks, swarm.Task{ ID: "1", NodeID: "a", DesiredState: swarm.TaskStateShutdown, Status: swarm.TaskStatus{State: swarm.TaskStateNew}, }) ut.testUpdater(tasks, false, []progress.Progress{ {ID: "overall progress", Action: "0 out of 1 tasks"}, }) // Task with valid DesiredState and State updates progress bar tasks[0].DesiredState = swarm.TaskStateRunning ut.testUpdater(tasks, false, []progress.Progress{ {ID: "1/1", Action: "new ", Current: 1, Total: 9, HideCounts: true}, {ID: "overall progress", Action: "0 out of 1 tasks"}, }) // If the task exposes an error, we should show that instead of the // progress bar. tasks[0].Status.Err = "something is wrong" ut.testUpdater(tasks, false, []progress.Progress{ {ID: "1/1", Action: "something is wrong"}, {ID: "overall progress", Action: "0 out of 1 tasks"}, }) // When the task reaches running, update should return true tasks[0].Status.Err = "" tasks[0].Status.State = swarm.TaskStateRunning ut.testUpdater(tasks, true, []progress.Progress{ {ID: "1/1", Action: "running ", Current: 9, Total: 9, HideCounts: true}, {ID: "overall progress", Action: "1 out of 1 tasks"}, }) // If the task fails, update should return false again tasks[0].Status.Err = "task failed" tasks[0].Status.State = swarm.TaskStateFailed ut.testUpdater(tasks, false, []progress.Progress{ {ID: "1/1", Action: "task failed"}, {ID: "overall progress", Action: "0 out of 1 tasks"}, }) // If the task is restarted, progress output should be shown for the // replacement task, not the old task. tasks[0].DesiredState = swarm.TaskStateShutdown tasks = append(tasks, swarm.Task{ ID: "2", NodeID: "b", DesiredState: swarm.TaskStateRunning, Status: swarm.TaskStatus{State: swarm.TaskStateRunning}, }) ut.testUpdater(tasks, true, []progress.Progress{ {ID: "1/1", Action: "running ", Current: 9, Total: 9, HideCounts: true}, {ID: "overall progress", Action: "1 out of 1 tasks"}, }) // Add a new task while the current one is still running, to simulate // "start-then-stop" updates. tasks = append(tasks, swarm.Task{ ID: "3", NodeID: "b", DesiredState: swarm.TaskStateRunning, Status: swarm.TaskStatus{State: swarm.TaskStatePreparing}, }) ut.testUpdater(tasks, false, []progress.Progress{ {ID: "1/1", Action: "preparing", Current: 6, Total: 9, HideCounts: true}, {ID: "overall progress", Action: "0 out of 1 tasks"}, }) } func TestReplicatedProgressUpdaterManyReplicas(t *testing.T) { replicas := uint64(50) service := swarm.Service{ Spec: swarm.ServiceSpec{ Mode: swarm.ServiceMode{ Replicated: &swarm.ReplicatedService{ Replicas: &replicas, }, }, }, } p := &mockProgress{} ut := updaterTester{ t: t, updater: &replicatedProgressUpdater{ progressOut: p, }, p: p, activeNodes: map[string]struct{}{"a": {}, "b": {}}, service: service, } tasks := []swarm.Task{} // No per-task progress bars because there are too many replicas ut.testUpdater(tasks, false, []progress.Progress{ {ID: "overall progress", Action: fmt.Sprintf("0 out of %d tasks", replicas)}, {ID: "overall progress", Action: fmt.Sprintf("0 out of %d tasks", replicas)}, }) for i := 0; i != int(replicas); i++ { tasks = append(tasks, swarm.Task{ ID: strconv.Itoa(i), Slot: i + 1, NodeID: "a", DesiredState: swarm.TaskStateRunning, Status: swarm.TaskStatus{State: swarm.TaskStateNew}, }) if i%2 == 1 { tasks[i].NodeID = "b" } ut.testUpdater(tasks, false, []progress.Progress{ {ID: "overall progress", Action: fmt.Sprintf("%d out of %d tasks", i, replicas)}, }) tasks[i].Status.State = swarm.TaskStateRunning ut.testUpdater(tasks, uint64(i) == replicas-1, []progress.Progress{ {ID: "overall progress", Action: fmt.Sprintf("%d out of %d tasks", i+1, replicas)}, }) } } func TestGlobalProgressUpdaterOneNode(t *testing.T) { service := swarm.Service{ Spec: swarm.ServiceSpec{ Mode: swarm.ServiceMode{ Global: &swarm.GlobalService{}, }, }, } p := &mockProgress{} ut := updaterTester{ t: t, updater: &globalProgressUpdater{ progressOut: p, }, p: p, activeNodes: map[string]struct{}{"a": {}, "b": {}}, service: service, } tasks := []swarm.Task{} ut.testUpdater(tasks, false, []progress.Progress{ {ID: "overall progress", Action: "waiting for new tasks"}, }) // Task with DesiredState beyond Running is ignored tasks = append(tasks, swarm.Task{ ID: "1", NodeID: "a", DesiredState: swarm.TaskStateShutdown, Status: swarm.TaskStatus{State: swarm.TaskStateNew}, }) ut.testUpdater(tasks, false, []progress.Progress{ {ID: "overall progress", Action: "0 out of 1 tasks"}, {ID: "overall progress", Action: "0 out of 1 tasks"}, }) // Task with valid DesiredState and State updates progress bar tasks[0].DesiredState = swarm.TaskStateRunning ut.testUpdater(tasks, false, []progress.Progress{ {ID: "a", Action: "new ", Current: 1, Total: 9, HideCounts: true}, {ID: "overall progress", Action: "0 out of 1 tasks"}, }) // If the task exposes an error, we should show that instead of the // progress bar. tasks[0].Status.Err = "something is wrong" ut.testUpdater(tasks, false, []progress.Progress{ {ID: "a", Action: "something is wrong"}, {ID: "overall progress", Action: "0 out of 1 tasks"}, }) // When the task reaches running, update should return true tasks[0].Status.Err = "" tasks[0].Status.State = swarm.TaskStateRunning ut.testUpdater(tasks, true, []progress.Progress{ {ID: "a", Action: "running ", Current: 9, Total: 9, HideCounts: true}, {ID: "overall progress", Action: "1 out of 1 tasks"}, }) // If the task fails, update should return false again tasks[0].Status.Err = "task failed" tasks[0].Status.State = swarm.TaskStateFailed ut.testUpdater(tasks, false, []progress.Progress{ {ID: "a", Action: "task failed"}, {ID: "overall progress", Action: "0 out of 1 tasks"}, }) // If the task is restarted, progress output should be shown for the // replacement task, not the old task. tasks[0].DesiredState = swarm.TaskStateShutdown tasks = append(tasks, swarm.Task{ ID: "2", NodeID: "a", DesiredState: swarm.TaskStateRunning, Status: swarm.TaskStatus{State: swarm.TaskStateRunning}, }) ut.testUpdater(tasks, true, []progress.Progress{ {ID: "a", Action: "running ", Current: 9, Total: 9, HideCounts: true}, {ID: "overall progress", Action: "1 out of 1 tasks"}, }) // Add a new task while the current one is still running, to simulate // "start-then-stop" updates. tasks = append(tasks, swarm.Task{ ID: "3", NodeID: "a", DesiredState: swarm.TaskStateRunning, Status: swarm.TaskStatus{State: swarm.TaskStatePreparing}, }) ut.testUpdater(tasks, false, []progress.Progress{ {ID: "a", Action: "preparing", Current: 6, Total: 9, HideCounts: true}, {ID: "overall progress", Action: "0 out of 1 tasks"}, }) } func TestGlobalProgressUpdaterManyNodes(t *testing.T) { nodes := 50 service := swarm.Service{ Spec: swarm.ServiceSpec{ Mode: swarm.ServiceMode{ Global: &swarm.GlobalService{}, }, }, } p := &mockProgress{} ut := updaterTester{ t: t, updater: &globalProgressUpdater{ progressOut: p, }, p: p, activeNodes: map[string]struct{}{}, service: service, } for i := 0; i != nodes; i++ { ut.activeNodes[strconv.Itoa(i)] = struct{}{} } tasks := []swarm.Task{} ut.testUpdater(tasks, false, []progress.Progress{ {ID: "overall progress", Action: "waiting for new tasks"}, }) for i := 0; i != nodes; i++ { tasks = append(tasks, swarm.Task{ ID: "task" + strconv.Itoa(i), NodeID: strconv.Itoa(i), DesiredState: swarm.TaskStateRunning, Status: swarm.TaskStatus{State: swarm.TaskStateNew}, }) } ut.testUpdater(tasks, false, []progress.Progress{ {ID: "overall progress", Action: fmt.Sprintf("0 out of %d tasks", nodes)}, {ID: "overall progress", Action: fmt.Sprintf("0 out of %d tasks", nodes)}, }) for i := 0; i != nodes; i++ { tasks[i].Status.State = swarm.TaskStateRunning ut.testUpdater(tasks, i == nodes-1, []progress.Progress{ {ID: "overall progress", Action: fmt.Sprintf("%d out of %d tasks", i+1, nodes)}, }) } } func TestReplicatedJobProgressUpdaterSmall(t *testing.T) { concurrent := uint64(2) total := uint64(5) service := swarm.Service{ Spec: swarm.ServiceSpec{ Mode: swarm.ServiceMode{ ReplicatedJob: &swarm.ReplicatedJob{ MaxConcurrent: &concurrent, TotalCompletions: &total, }, }, }, JobStatus: &swarm.JobStatus{ JobIteration: swarm.Version{Index: 1}, }, } p := &mockProgress{} ut := updaterTester{ t: t, updater: newReplicatedJobProgressUpdater(service, p), p: p, activeNodes: map[string]struct{}{"a": {}, "b": {}}, service: service, } // create some tasks belonging to a previous iteration tasks := []swarm.Task{ { ID: "oldtask1", Slot: 0, NodeID: "", DesiredState: swarm.TaskStateComplete, Status: swarm.TaskStatus{State: swarm.TaskStateNew}, JobIteration: &swarm.Version{Index: 0}, }, { ID: "oldtask2", Slot: 1, NodeID: "", DesiredState: swarm.TaskStateComplete, Status: swarm.TaskStatus{State: swarm.TaskStateComplete}, JobIteration: &swarm.Version{Index: 0}, }, } ut.testUpdater(tasks, false, []progress.Progress{ // on the initial pass, we draw all of the progress bars at once, which // puts them in order for the rest of the operation {ID: "job progress", Action: "0 out of 5 complete", Current: 0, Total: 5, HideCounts: true}, {ID: "active tasks", Action: "0 out of 2 tasks"}, {ID: "1/5", Action: " "}, {ID: "2/5", Action: " "}, {ID: "3/5", Action: " "}, {ID: "4/5", Action: " "}, {ID: "5/5", Action: " "}, // from here on, we draw as normal. as a side effect, we will have a // second update for the job progress and active tasks. This has no // practical effect on the UI, it's just a side effect of the update // logic. {ID: "job progress", Action: "0 out of 5 complete", Current: 0, Total: 5, HideCounts: true}, {ID: "active tasks", Action: "0 out of 2 tasks"}, }) // wipe the old tasks out of the list tasks = []swarm.Task{} tasks = append(tasks, swarm.Task{ ID: "task1", Slot: 0, NodeID: "", DesiredState: swarm.TaskStateComplete, Status: swarm.TaskStatus{State: swarm.TaskStateNew}, JobIteration: &swarm.Version{Index: service.JobStatus.JobIteration.Index}, }, swarm.Task{ ID: "task2", Slot: 1, NodeID: "", DesiredState: swarm.TaskStateComplete, Status: swarm.TaskStatus{State: swarm.TaskStateNew}, JobIteration: &swarm.Version{Index: service.JobStatus.JobIteration.Index}, }, ) ut.testUpdater(tasks, false, []progress.Progress{ {ID: "1/5", Action: "new ", Current: 1, Total: 10, HideCounts: true}, {ID: "2/5", Action: "new ", Current: 1, Total: 10, HideCounts: true}, {ID: "job progress", Action: "0 out of 5 complete", Current: 0, Total: 5, HideCounts: true}, {ID: "active tasks", Action: "2 out of 2 tasks"}, }) tasks[0].Status.State = swarm.TaskStatePreparing tasks[1].Status.State = swarm.TaskStateAssigned ut.testUpdater(tasks, false, []progress.Progress{ {ID: "1/5", Action: "preparing", Current: 6, Total: 10, HideCounts: true}, {ID: "2/5", Action: "assigned ", Current: 4, Total: 10, HideCounts: true}, {ID: "job progress", Action: "0 out of 5 complete", Current: 0, Total: 5, HideCounts: true}, {ID: "active tasks", Action: "2 out of 2 tasks"}, }) tasks[0].Status.State = swarm.TaskStateRunning tasks[1].Status.State = swarm.TaskStatePreparing ut.testUpdater(tasks, false, []progress.Progress{ {ID: "1/5", Action: "running ", Current: 9, Total: 10, HideCounts: true}, {ID: "2/5", Action: "preparing", Current: 6, Total: 10, HideCounts: true}, {ID: "job progress", Action: "0 out of 5 complete", Current: 0, Total: 5, HideCounts: true}, {ID: "active tasks", Action: "2 out of 2 tasks"}, }) tasks[0].Status.State = swarm.TaskStateComplete tasks[1].Status.State = swarm.TaskStateComplete ut.testUpdater(tasks, false, []progress.Progress{ {ID: "1/5", Action: "complete ", Current: 10, Total: 10, HideCounts: true}, {ID: "2/5", Action: "complete ", Current: 10, Total: 10, HideCounts: true}, {ID: "job progress", Action: "2 out of 5 complete", Current: 2, Total: 5, HideCounts: true}, {ID: "active tasks", Action: "0 out of 2 tasks"}, }) tasks = append(tasks, swarm.Task{ ID: "task3", Slot: 2, NodeID: "", DesiredState: swarm.TaskStateComplete, Status: swarm.TaskStatus{State: swarm.TaskStateNew}, JobIteration: &swarm.Version{Index: service.JobStatus.JobIteration.Index}, }, swarm.Task{ ID: "task4", Slot: 3, NodeID: "", DesiredState: swarm.TaskStateComplete, Status: swarm.TaskStatus{State: swarm.TaskStateNew}, JobIteration: &swarm.Version{Index: service.JobStatus.JobIteration.Index}, }, ) ut.testUpdater(tasks, false, []progress.Progress{ {ID: "1/5", Action: "complete ", Current: 10, Total: 10, HideCounts: true}, {ID: "2/5", Action: "complete ", Current: 10, Total: 10, HideCounts: true}, {ID: "3/5", Action: "new ", Current: 1, Total: 10, HideCounts: true}, {ID: "4/5", Action: "new ", Current: 1, Total: 10, HideCounts: true}, {ID: "job progress", Action: "2 out of 5 complete", Current: 2, Total: 5, HideCounts: true}, {ID: "active tasks", Action: "2 out of 2 tasks"}, }) tasks[2].Status.State = swarm.TaskStateRunning tasks[3].Status.State = swarm.TaskStateRunning ut.testUpdater(tasks, false, []progress.Progress{ {ID: "1/5", Action: "complete ", Current: 10, Total: 10, HideCounts: true}, {ID: "2/5", Action: "complete ", Current: 10, Total: 10, HideCounts: true}, {ID: "3/5", Action: "running ", Current: 9, Total: 10, HideCounts: true}, {ID: "4/5", Action: "running ", Current: 9, Total: 10, HideCounts: true}, {ID: "job progress", Action: "2 out of 5 complete", Current: 2, Total: 5, HideCounts: true}, {ID: "active tasks", Action: "2 out of 2 tasks"}, }) tasks[3].Status.State = swarm.TaskStateComplete tasks = append(tasks, swarm.Task{ ID: "task5", Slot: 4, NodeID: "", DesiredState: swarm.TaskStateComplete, Status: swarm.TaskStatus{State: swarm.TaskStateRunning}, JobIteration: &swarm.Version{Index: service.JobStatus.JobIteration.Index}, }, ) ut.testUpdater(tasks, false, []progress.Progress{ {ID: "1/5", Action: "complete ", Current: 10, Total: 10, HideCounts: true}, {ID: "2/5", Action: "complete ", Current: 10, Total: 10, HideCounts: true}, {ID: "3/5", Action: "running ", Current: 9, Total: 10, HideCounts: true}, {ID: "4/5", Action: "complete ", Current: 10, Total: 10, HideCounts: true}, {ID: "5/5", Action: "running ", Current: 9, Total: 10, HideCounts: true}, {ID: "job progress", Action: "3 out of 5 complete", Current: 3, Total: 5, HideCounts: true}, {ID: "active tasks", Action: "2 out of 2 tasks"}, }) tasks[2].Status.State = swarm.TaskStateFailed tasks[2].Status.Err = "the task failed" ut.testUpdater(tasks, false, []progress.Progress{ {ID: "1/5", Action: "complete ", Current: 10, Total: 10, HideCounts: true}, {ID: "2/5", Action: "complete ", Current: 10, Total: 10, HideCounts: true}, {ID: "3/5", Action: "the task failed"}, {ID: "4/5", Action: "complete ", Current: 10, Total: 10, HideCounts: true}, {ID: "5/5", Action: "running ", Current: 9, Total: 10, HideCounts: true}, {ID: "job progress", Action: "3 out of 5 complete", Current: 3, Total: 5, HideCounts: true}, {ID: "active tasks", Action: "1 out of 2 tasks"}, }) tasks[4].Status.State = swarm.TaskStateComplete tasks = append(tasks, swarm.Task{ ID: "task6", Slot: 2, NodeID: "", DesiredState: swarm.TaskStateComplete, Status: swarm.TaskStatus{State: swarm.TaskStateRunning}, JobIteration: &swarm.Version{Index: service.JobStatus.JobIteration.Index}, }, ) ut.testUpdater(tasks, false, []progress.Progress{ {ID: "1/5", Action: "complete ", Current: 10, Total: 10, HideCounts: true}, {ID: "2/5", Action: "complete ", Current: 10, Total: 10, HideCounts: true}, {ID: "3/5", Action: "running ", Current: 9, Total: 10, HideCounts: true}, {ID: "4/5", Action: "complete ", Current: 10, Total: 10, HideCounts: true}, {ID: "5/5", Action: "complete ", Current: 10, Total: 10, HideCounts: true}, {ID: "job progress", Action: "4 out of 5 complete", Current: 4, Total: 5, HideCounts: true}, {ID: "active tasks", Action: "1 out of 1 tasks"}, }) tasks[5].Status.State = swarm.TaskStateComplete ut.testUpdater(tasks, true, []progress.Progress{ {ID: "1/5", Action: "complete ", Current: 10, Total: 10, HideCounts: true}, {ID: "2/5", Action: "complete ", Current: 10, Total: 10, HideCounts: true}, {ID: "3/5", Action: "complete ", Current: 10, Total: 10, HideCounts: true}, {ID: "4/5", Action: "complete ", Current: 10, Total: 10, HideCounts: true}, {ID: "5/5", Action: "complete ", Current: 10, Total: 10, HideCounts: true}, {ID: "job progress", Action: "5 out of 5 complete", Current: 5, Total: 5, HideCounts: true}, {ID: "active tasks", Action: "0 out of 0 tasks"}, }) } func TestReplicatedJobProgressUpdaterLarge(t *testing.T) { concurrent := uint64(10) total := uint64(50) service := swarm.Service{ Spec: swarm.ServiceSpec{ Mode: swarm.ServiceMode{ ReplicatedJob: &swarm.ReplicatedJob{ MaxConcurrent: &concurrent, TotalCompletions: &total, }, }, }, JobStatus: &swarm.JobStatus{ JobIteration: swarm.Version{Index: 0}, }, } p := &mockProgress{} ut := updaterTester{ t: t, updater: newReplicatedJobProgressUpdater(service, p), p: p, activeNodes: map[string]struct{}{"a": {}, "b": {}}, service: service, } tasks := []swarm.Task{} // see the comments in TestReplicatedJobProgressUpdaterSmall for why // we write this out twice. ut.testUpdater(tasks, false, []progress.Progress{ {ID: "job progress", Action: " 0 out of 50 complete", Current: 0, Total: 50, HideCounts: true}, {ID: "active tasks", Action: " 0 out of 10 tasks"}, // we don't write out individual status bars for a large job, only the // overall progress bar {ID: "job progress", Action: " 0 out of 50 complete", Current: 0, Total: 50, HideCounts: true}, {ID: "active tasks", Action: " 0 out of 10 tasks"}, }) // first, create the initial batch of running tasks for i := 0; i < int(concurrent); i++ { tasks = append(tasks, swarm.Task{ ID: strconv.Itoa(i), Slot: i, DesiredState: swarm.TaskStateComplete, Status: swarm.TaskStatus{State: swarm.TaskStateNew}, JobIteration: &swarm.Version{Index: 0}, }) ut.testUpdater(tasks, false, []progress.Progress{ {ID: "job progress", Action: " 0 out of 50 complete", Current: 0, Total: 50, HideCounts: true}, {ID: "active tasks", Action: fmt.Sprintf("%2d out of 10 tasks", i+1)}, }) } // now, start moving tasks to completed, and starting new tasks after them. // to do this, we'll start at 0, mark a task complete, and then append a // new one. we'll stop before we get to the end, because the end has a // steadily decreasing denominator for the active tasks // // for 10 concurrent 50 total, this means we'll stop at 50 - 10 = 40 tasks // in the completed state, 10 tasks running. the last index in use will be // 39. for i := 0; i < int(total)-int(concurrent); i++ { tasks[i].Status.State = swarm.TaskStateComplete ut.testUpdater(tasks, false, []progress.Progress{ {ID: "job progress", Action: fmt.Sprintf("%2d out of 50 complete", i+1), Current: int64(i + 1), Total: 50, HideCounts: true}, {ID: "active tasks", Action: " 9 out of 10 tasks"}, }) last := len(tasks) tasks = append(tasks, swarm.Task{ ID: strconv.Itoa(last), Slot: last, DesiredState: swarm.TaskStateComplete, Status: swarm.TaskStatus{State: swarm.TaskStateNew}, JobIteration: &swarm.Version{Index: 0}, }) ut.testUpdater(tasks, false, []progress.Progress{ {ID: "job progress", Action: fmt.Sprintf("%2d out of 50 complete", i+1), Current: int64(i + 1), Total: 50, HideCounts: true}, {ID: "active tasks", Action: "10 out of 10 tasks"}, }) } // quick check, to make sure we did the math right when we wrote this code: // we do have 50 tasks in the slice, right? assert.Check(t, is.Equal(len(tasks), int(total))) // now, we're down to our last 10 tasks, which are all running. We need to // wind these down for i := int(total) - int(concurrent) - 1; i < int(total); i++ { tasks[i].Status.State = swarm.TaskStateComplete ut.testUpdater(tasks, (i+1 == int(total)), []progress.Progress{ {ID: "job progress", Action: fmt.Sprintf("%2d out of 50 complete", i+1), Current: int64(i + 1), Total: 50, HideCounts: true}, {ID: "active tasks", Action: fmt.Sprintf("%2[1]d out of %2[1]d tasks", int(total)-(i+1))}, }) } } func TestGlobalJobProgressUpdaterSmall(t *testing.T) { service := swarm.Service{ Spec: swarm.ServiceSpec{ Mode: swarm.ServiceMode{ GlobalJob: &swarm.GlobalJob{}, }, }, JobStatus: &swarm.JobStatus{ JobIteration: swarm.Version{Index: 1}, }, } p := &mockProgress{} ut := updaterTester{ t: t, updater: &globalJobProgressUpdater{ progressOut: p, }, p: p, activeNodes: map[string]struct{}{"a": {}, "b": {}, "c": {}}, service: service, } tasks := []swarm.Task{ { ID: "oldtask1", DesiredState: swarm.TaskStateComplete, Status: swarm.TaskStatus{State: swarm.TaskStateComplete}, JobIteration: &swarm.Version{Index: 0}, NodeID: "a", }, { ID: "oldtask2", DesiredState: swarm.TaskStateComplete, Status: swarm.TaskStatus{State: swarm.TaskStateComplete}, JobIteration: &swarm.Version{Index: 0}, NodeID: "b", }, { ID: "task1", DesiredState: swarm.TaskStateComplete, Status: swarm.TaskStatus{State: swarm.TaskStateNew}, JobIteration: &swarm.Version{Index: 1}, NodeID: "a", }, { ID: "task2", DesiredState: swarm.TaskStateComplete, Status: swarm.TaskStatus{State: swarm.TaskStateNew}, JobIteration: &swarm.Version{Index: 1}, NodeID: "b", }, { ID: "task3", DesiredState: swarm.TaskStateComplete, Status: swarm.TaskStatus{State: swarm.TaskStateNew}, JobIteration: &swarm.Version{Index: 1}, NodeID: "c", }, } // we don't know how many tasks will be created until we get the initial // task list, so we should not write out any definitive answers yet. ut.testUpdater([]swarm.Task{}, false, []progress.Progress{ {ID: "job progress", Action: "waiting for tasks"}, }) ut.testUpdaterNoOrder(tasks, false, []progress.Progress{ {ID: "job progress", Action: "0 out of 3 complete", Current: 0, Total: 3, HideCounts: true}, {ID: "a", Action: "new ", Current: 1, Total: 10, HideCounts: true}, {ID: "b", Action: "new ", Current: 1, Total: 10, HideCounts: true}, {ID: "c", Action: "new ", Current: 1, Total: 10, HideCounts: true}, {ID: "job progress", Action: "0 out of 3 complete", Current: 0, Total: 3, HideCounts: true}, }) tasks[2].Status.State = swarm.TaskStatePreparing tasks[3].Status.State = swarm.TaskStateRunning tasks[4].Status.State = swarm.TaskStateAccepted ut.testUpdaterNoOrder(tasks, false, []progress.Progress{ {ID: "a", Action: "preparing", Current: 6, Total: 10, HideCounts: true}, {ID: "b", Action: "running ", Current: 9, Total: 10, HideCounts: true}, {ID: "c", Action: "accepted ", Current: 5, Total: 10, HideCounts: true}, {ID: "job progress", Action: "0 out of 3 complete", Current: 0, Total: 3, HideCounts: true}, }) tasks[2].Status.State = swarm.TaskStateRunning tasks[3].Status.State = swarm.TaskStateComplete tasks[4].Status.State = swarm.TaskStateRunning ut.testUpdaterNoOrder(tasks, false, []progress.Progress{ {ID: "a", Action: "running ", Current: 9, Total: 10, HideCounts: true}, {ID: "b", Action: "complete ", Current: 10, Total: 10, HideCounts: true}, {ID: "c", Action: "running ", Current: 9, Total: 10, HideCounts: true}, {ID: "job progress", Action: "1 out of 3 complete", Current: 1, Total: 3, HideCounts: true}, }) tasks[2].Status.State = swarm.TaskStateFailed tasks[2].Status.Err = "task failed" tasks[4].Status.State = swarm.TaskStateComplete ut.testUpdaterNoOrder(tasks, false, []progress.Progress{ {ID: "a", Action: "task failed"}, {ID: "b", Action: "complete ", Current: 10, Total: 10, HideCounts: true}, {ID: "c", Action: "complete ", Current: 10, Total: 10, HideCounts: true}, {ID: "job progress", Action: "2 out of 3 complete", Current: 2, Total: 3, HideCounts: true}, }) tasks = append(tasks, swarm.Task{ ID: "task4", DesiredState: swarm.TaskStateComplete, Status: swarm.TaskStatus{State: swarm.TaskStatePreparing}, NodeID: tasks[2].NodeID, JobIteration: &swarm.Version{Index: 1}, }) ut.testUpdaterNoOrder(tasks, false, []progress.Progress{ {ID: "a", Action: "preparing", Current: 6, Total: 10, HideCounts: true}, {ID: "b", Action: "complete ", Current: 10, Total: 10, HideCounts: true}, {ID: "c", Action: "complete ", Current: 10, Total: 10, HideCounts: true}, {ID: "job progress", Action: "2 out of 3 complete", Current: 2, Total: 3, HideCounts: true}, }) tasks[5].Status.State = swarm.TaskStateComplete ut.testUpdaterNoOrder(tasks, true, []progress.Progress{ {ID: "a", Action: "complete ", Current: 10, Total: 10, HideCounts: true}, {ID: "b", Action: "complete ", Current: 10, Total: 10, HideCounts: true}, {ID: "c", Action: "complete ", Current: 10, Total: 10, HideCounts: true}, {ID: "job progress", Action: "3 out of 3 complete", Current: 3, Total: 3, HideCounts: true}, }) } func TestGlobalJobProgressUpdaterLarge(t *testing.T) { service := swarm.Service{ Spec: swarm.ServiceSpec{ Mode: swarm.ServiceMode{ GlobalJob: &swarm.GlobalJob{}, }, }, JobStatus: &swarm.JobStatus{ JobIteration: swarm.Version{Index: 1}, }, } activeNodes := map[string]struct{}{} for i := 0; i < 50; i++ { activeNodes[fmt.Sprintf("node%v", i)] = struct{}{} } p := &mockProgress{} ut := updaterTester{ t: t, updater: &globalJobProgressUpdater{ progressOut: p, }, p: p, activeNodes: activeNodes, service: service, } tasks := []swarm.Task{} for nodeID := range activeNodes { tasks = append(tasks, swarm.Task{ ID: "task" + nodeID, NodeID: nodeID, DesiredState: swarm.TaskStateComplete, Status: swarm.TaskStatus{ State: swarm.TaskStateNew, }, JobIteration: &swarm.Version{Index: 1}, }) } // no bars, because too many tasks ut.testUpdater(tasks, false, []progress.Progress{ {ID: "job progress", Action: " 0 out of 50 complete", Current: 0, Total: 50, HideCounts: true}, {ID: "job progress", Action: " 0 out of 50 complete", Current: 0, Total: 50, HideCounts: true}, }) for i := range tasks { tasks[i].Status.State = swarm.TaskStateComplete ut.testUpdater(tasks, i+1 == len(activeNodes), []progress.Progress{ { ID: "job progress", Action: fmt.Sprintf("%2d out of 50 complete", i+1), Current: int64(i + 1), Total: 50, HideCounts: true, }, }) } }