mirror of https://github.com/docker/cli.git
904 lines
30 KiB
Go
904 lines
30 KiB
Go
package progress
|
|
|
|
import (
|
|
"fmt"
|
|
"strconv"
|
|
"testing"
|
|
|
|
"github.com/docker/docker/api/types/swarm"
|
|
"github.com/docker/docker/pkg/progress"
|
|
"gotest.tools/v3/assert"
|
|
is "gotest.tools/v3/assert/cmp"
|
|
)
|
|
|
|
type mockProgress struct {
|
|
p []progress.Progress
|
|
}
|
|
|
|
func (mp *mockProgress) WriteProgress(p progress.Progress) error {
|
|
mp.p = append(mp.p, p)
|
|
return nil
|
|
}
|
|
|
|
func (mp *mockProgress) clear() {
|
|
mp.p = nil
|
|
}
|
|
|
|
type updaterTester struct {
|
|
t *testing.T
|
|
updater progressUpdater
|
|
p *mockProgress
|
|
service swarm.Service
|
|
activeNodes map[string]struct{}
|
|
rollback bool
|
|
}
|
|
|
|
func (u updaterTester) testUpdater(tasks []swarm.Task, expectedConvergence bool, expectedProgress []progress.Progress) {
|
|
u.p.clear()
|
|
|
|
converged, err := u.updater.update(u.service, tasks, u.activeNodes, u.rollback)
|
|
assert.Check(u.t, err)
|
|
assert.Check(u.t, is.Equal(expectedConvergence, converged))
|
|
assert.Check(u.t, is.DeepEqual(expectedProgress, u.p.p))
|
|
}
|
|
|
|
func (u updaterTester) testUpdaterNoOrder(tasks []swarm.Task, expectedConvergence bool, expectedProgress []progress.Progress) {
|
|
u.p.clear()
|
|
converged, err := u.updater.update(u.service, tasks, u.activeNodes, u.rollback)
|
|
assert.Check(u.t, err)
|
|
assert.Check(u.t, is.Equal(expectedConvergence, converged))
|
|
|
|
// instead of checking that expected and actual match exactly, verify that
|
|
// they are the same length, and every time from actual is in expected.
|
|
assert.Check(u.t, is.Equal(len(expectedProgress), len(u.p.p)))
|
|
for _, prog := range expectedProgress {
|
|
assert.Check(u.t, is.Contains(u.p.p, prog))
|
|
}
|
|
}
|
|
|
|
func TestReplicatedProgressUpdaterOneReplica(t *testing.T) {
|
|
replicas := uint64(1)
|
|
|
|
service := swarm.Service{
|
|
Spec: swarm.ServiceSpec{
|
|
Mode: swarm.ServiceMode{
|
|
Replicated: &swarm.ReplicatedService{
|
|
Replicas: &replicas,
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
p := &mockProgress{}
|
|
ut := updaterTester{
|
|
t: t,
|
|
updater: &replicatedProgressUpdater{
|
|
progressOut: p,
|
|
},
|
|
p: p,
|
|
activeNodes: map[string]struct{}{"a": {}, "b": {}},
|
|
service: service,
|
|
}
|
|
|
|
tasks := []swarm.Task{}
|
|
|
|
ut.testUpdater(tasks, false,
|
|
[]progress.Progress{
|
|
{ID: "overall progress", Action: "0 out of 1 tasks"},
|
|
{ID: "1/1", Action: " "},
|
|
{ID: "overall progress", Action: "0 out of 1 tasks"},
|
|
})
|
|
|
|
// Task with DesiredState beyond Running is ignored
|
|
tasks = append(tasks,
|
|
swarm.Task{
|
|
ID: "1",
|
|
NodeID: "a",
|
|
DesiredState: swarm.TaskStateShutdown,
|
|
Status: swarm.TaskStatus{State: swarm.TaskStateNew},
|
|
})
|
|
ut.testUpdater(tasks, false,
|
|
[]progress.Progress{
|
|
{ID: "overall progress", Action: "0 out of 1 tasks"},
|
|
})
|
|
|
|
// Task with valid DesiredState and State updates progress bar
|
|
tasks[0].DesiredState = swarm.TaskStateRunning
|
|
ut.testUpdater(tasks, false,
|
|
[]progress.Progress{
|
|
{ID: "1/1", Action: "new ", Current: 1, Total: 9, HideCounts: true},
|
|
{ID: "overall progress", Action: "0 out of 1 tasks"},
|
|
})
|
|
|
|
// If the task exposes an error, we should show that instead of the
|
|
// progress bar.
|
|
tasks[0].Status.Err = "something is wrong"
|
|
ut.testUpdater(tasks, false,
|
|
[]progress.Progress{
|
|
{ID: "1/1", Action: "something is wrong"},
|
|
{ID: "overall progress", Action: "0 out of 1 tasks"},
|
|
})
|
|
|
|
// When the task reaches running, update should return true
|
|
tasks[0].Status.Err = ""
|
|
tasks[0].Status.State = swarm.TaskStateRunning
|
|
ut.testUpdater(tasks, true,
|
|
[]progress.Progress{
|
|
{ID: "1/1", Action: "running ", Current: 9, Total: 9, HideCounts: true},
|
|
{ID: "overall progress", Action: "1 out of 1 tasks"},
|
|
})
|
|
|
|
// If the task fails, update should return false again
|
|
tasks[0].Status.Err = "task failed"
|
|
tasks[0].Status.State = swarm.TaskStateFailed
|
|
ut.testUpdater(tasks, false,
|
|
[]progress.Progress{
|
|
{ID: "1/1", Action: "task failed"},
|
|
{ID: "overall progress", Action: "0 out of 1 tasks"},
|
|
})
|
|
|
|
// If the task is restarted, progress output should be shown for the
|
|
// replacement task, not the old task.
|
|
tasks[0].DesiredState = swarm.TaskStateShutdown
|
|
tasks = append(tasks,
|
|
swarm.Task{
|
|
ID: "2",
|
|
NodeID: "b",
|
|
DesiredState: swarm.TaskStateRunning,
|
|
Status: swarm.TaskStatus{State: swarm.TaskStateRunning},
|
|
})
|
|
ut.testUpdater(tasks, true,
|
|
[]progress.Progress{
|
|
{ID: "1/1", Action: "running ", Current: 9, Total: 9, HideCounts: true},
|
|
{ID: "overall progress", Action: "1 out of 1 tasks"},
|
|
})
|
|
|
|
// Add a new task while the current one is still running, to simulate
|
|
// "start-then-stop" updates.
|
|
tasks = append(tasks,
|
|
swarm.Task{
|
|
ID: "3",
|
|
NodeID: "b",
|
|
DesiredState: swarm.TaskStateRunning,
|
|
Status: swarm.TaskStatus{State: swarm.TaskStatePreparing},
|
|
})
|
|
ut.testUpdater(tasks, false,
|
|
[]progress.Progress{
|
|
{ID: "1/1", Action: "preparing", Current: 6, Total: 9, HideCounts: true},
|
|
{ID: "overall progress", Action: "0 out of 1 tasks"},
|
|
})
|
|
}
|
|
|
|
func TestReplicatedProgressUpdaterManyReplicas(t *testing.T) {
|
|
replicas := uint64(50)
|
|
|
|
service := swarm.Service{
|
|
Spec: swarm.ServiceSpec{
|
|
Mode: swarm.ServiceMode{
|
|
Replicated: &swarm.ReplicatedService{
|
|
Replicas: &replicas,
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
p := &mockProgress{}
|
|
ut := updaterTester{
|
|
t: t,
|
|
updater: &replicatedProgressUpdater{
|
|
progressOut: p,
|
|
},
|
|
p: p,
|
|
activeNodes: map[string]struct{}{"a": {}, "b": {}},
|
|
service: service,
|
|
}
|
|
|
|
tasks := []swarm.Task{}
|
|
|
|
// No per-task progress bars because there are too many replicas
|
|
ut.testUpdater(tasks, false,
|
|
[]progress.Progress{
|
|
{ID: "overall progress", Action: fmt.Sprintf("0 out of %d tasks", replicas)},
|
|
{ID: "overall progress", Action: fmt.Sprintf("0 out of %d tasks", replicas)},
|
|
})
|
|
|
|
for i := 0; i != int(replicas); i++ {
|
|
tasks = append(tasks,
|
|
swarm.Task{
|
|
ID: strconv.Itoa(i),
|
|
Slot: i + 1,
|
|
NodeID: "a",
|
|
DesiredState: swarm.TaskStateRunning,
|
|
Status: swarm.TaskStatus{State: swarm.TaskStateNew},
|
|
})
|
|
|
|
if i%2 == 1 {
|
|
tasks[i].NodeID = "b"
|
|
}
|
|
ut.testUpdater(tasks, false,
|
|
[]progress.Progress{
|
|
{ID: "overall progress", Action: fmt.Sprintf("%d out of %d tasks", i, replicas)},
|
|
})
|
|
|
|
tasks[i].Status.State = swarm.TaskStateRunning
|
|
ut.testUpdater(tasks, uint64(i) == replicas-1,
|
|
[]progress.Progress{
|
|
{ID: "overall progress", Action: fmt.Sprintf("%d out of %d tasks", i+1, replicas)},
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestGlobalProgressUpdaterOneNode(t *testing.T) {
|
|
service := swarm.Service{
|
|
Spec: swarm.ServiceSpec{
|
|
Mode: swarm.ServiceMode{
|
|
Global: &swarm.GlobalService{},
|
|
},
|
|
},
|
|
}
|
|
|
|
p := &mockProgress{}
|
|
ut := updaterTester{
|
|
t: t,
|
|
updater: &globalProgressUpdater{
|
|
progressOut: p,
|
|
},
|
|
p: p,
|
|
activeNodes: map[string]struct{}{"a": {}, "b": {}},
|
|
service: service,
|
|
}
|
|
|
|
tasks := []swarm.Task{}
|
|
|
|
ut.testUpdater(tasks, false,
|
|
[]progress.Progress{
|
|
{ID: "overall progress", Action: "waiting for new tasks"},
|
|
})
|
|
|
|
// Task with DesiredState beyond Running is ignored
|
|
tasks = append(tasks,
|
|
swarm.Task{
|
|
ID: "1",
|
|
NodeID: "a",
|
|
DesiredState: swarm.TaskStateShutdown,
|
|
Status: swarm.TaskStatus{State: swarm.TaskStateNew},
|
|
})
|
|
ut.testUpdater(tasks, false,
|
|
[]progress.Progress{
|
|
{ID: "overall progress", Action: "0 out of 1 tasks"},
|
|
{ID: "overall progress", Action: "0 out of 1 tasks"},
|
|
})
|
|
|
|
// Task with valid DesiredState and State updates progress bar
|
|
tasks[0].DesiredState = swarm.TaskStateRunning
|
|
ut.testUpdater(tasks, false,
|
|
[]progress.Progress{
|
|
{ID: "a", Action: "new ", Current: 1, Total: 9, HideCounts: true},
|
|
{ID: "overall progress", Action: "0 out of 1 tasks"},
|
|
})
|
|
|
|
// If the task exposes an error, we should show that instead of the
|
|
// progress bar.
|
|
tasks[0].Status.Err = "something is wrong"
|
|
ut.testUpdater(tasks, false,
|
|
[]progress.Progress{
|
|
{ID: "a", Action: "something is wrong"},
|
|
{ID: "overall progress", Action: "0 out of 1 tasks"},
|
|
})
|
|
|
|
// When the task reaches running, update should return true
|
|
tasks[0].Status.Err = ""
|
|
tasks[0].Status.State = swarm.TaskStateRunning
|
|
ut.testUpdater(tasks, true,
|
|
[]progress.Progress{
|
|
{ID: "a", Action: "running ", Current: 9, Total: 9, HideCounts: true},
|
|
{ID: "overall progress", Action: "1 out of 1 tasks"},
|
|
})
|
|
|
|
// If the task fails, update should return false again
|
|
tasks[0].Status.Err = "task failed"
|
|
tasks[0].Status.State = swarm.TaskStateFailed
|
|
ut.testUpdater(tasks, false,
|
|
[]progress.Progress{
|
|
{ID: "a", Action: "task failed"},
|
|
{ID: "overall progress", Action: "0 out of 1 tasks"},
|
|
})
|
|
|
|
// If the task is restarted, progress output should be shown for the
|
|
// replacement task, not the old task.
|
|
tasks[0].DesiredState = swarm.TaskStateShutdown
|
|
tasks = append(tasks,
|
|
swarm.Task{
|
|
ID: "2",
|
|
NodeID: "a",
|
|
DesiredState: swarm.TaskStateRunning,
|
|
Status: swarm.TaskStatus{State: swarm.TaskStateRunning},
|
|
})
|
|
ut.testUpdater(tasks, true,
|
|
[]progress.Progress{
|
|
{ID: "a", Action: "running ", Current: 9, Total: 9, HideCounts: true},
|
|
{ID: "overall progress", Action: "1 out of 1 tasks"},
|
|
})
|
|
|
|
// Add a new task while the current one is still running, to simulate
|
|
// "start-then-stop" updates.
|
|
tasks = append(tasks,
|
|
swarm.Task{
|
|
ID: "3",
|
|
NodeID: "a",
|
|
DesiredState: swarm.TaskStateRunning,
|
|
Status: swarm.TaskStatus{State: swarm.TaskStatePreparing},
|
|
})
|
|
ut.testUpdater(tasks, false,
|
|
[]progress.Progress{
|
|
{ID: "a", Action: "preparing", Current: 6, Total: 9, HideCounts: true},
|
|
{ID: "overall progress", Action: "0 out of 1 tasks"},
|
|
})
|
|
}
|
|
|
|
func TestGlobalProgressUpdaterManyNodes(t *testing.T) {
|
|
nodes := 50
|
|
|
|
service := swarm.Service{
|
|
Spec: swarm.ServiceSpec{
|
|
Mode: swarm.ServiceMode{
|
|
Global: &swarm.GlobalService{},
|
|
},
|
|
},
|
|
}
|
|
|
|
p := &mockProgress{}
|
|
ut := updaterTester{
|
|
t: t,
|
|
updater: &globalProgressUpdater{
|
|
progressOut: p,
|
|
},
|
|
p: p,
|
|
activeNodes: map[string]struct{}{},
|
|
service: service,
|
|
}
|
|
|
|
for i := 0; i != nodes; i++ {
|
|
ut.activeNodes[strconv.Itoa(i)] = struct{}{}
|
|
}
|
|
|
|
tasks := []swarm.Task{}
|
|
|
|
ut.testUpdater(tasks, false,
|
|
[]progress.Progress{
|
|
{ID: "overall progress", Action: "waiting for new tasks"},
|
|
})
|
|
|
|
for i := 0; i != nodes; i++ {
|
|
tasks = append(tasks,
|
|
swarm.Task{
|
|
ID: "task" + strconv.Itoa(i),
|
|
NodeID: strconv.Itoa(i),
|
|
DesiredState: swarm.TaskStateRunning,
|
|
Status: swarm.TaskStatus{State: swarm.TaskStateNew},
|
|
})
|
|
}
|
|
|
|
ut.testUpdater(tasks, false,
|
|
[]progress.Progress{
|
|
{ID: "overall progress", Action: fmt.Sprintf("0 out of %d tasks", nodes)},
|
|
{ID: "overall progress", Action: fmt.Sprintf("0 out of %d tasks", nodes)},
|
|
})
|
|
|
|
for i := 0; i != nodes; i++ {
|
|
tasks[i].Status.State = swarm.TaskStateRunning
|
|
ut.testUpdater(tasks, i == nodes-1,
|
|
[]progress.Progress{
|
|
{ID: "overall progress", Action: fmt.Sprintf("%d out of %d tasks", i+1, nodes)},
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestReplicatedJobProgressUpdaterSmall(t *testing.T) {
|
|
concurrent := uint64(2)
|
|
total := uint64(5)
|
|
|
|
service := swarm.Service{
|
|
Spec: swarm.ServiceSpec{
|
|
Mode: swarm.ServiceMode{
|
|
ReplicatedJob: &swarm.ReplicatedJob{
|
|
MaxConcurrent: &concurrent,
|
|
TotalCompletions: &total,
|
|
},
|
|
},
|
|
},
|
|
JobStatus: &swarm.JobStatus{
|
|
JobIteration: swarm.Version{Index: 1},
|
|
},
|
|
}
|
|
|
|
p := &mockProgress{}
|
|
ut := updaterTester{
|
|
t: t,
|
|
updater: newReplicatedJobProgressUpdater(service, p),
|
|
p: p,
|
|
activeNodes: map[string]struct{}{"a": {}, "b": {}},
|
|
service: service,
|
|
}
|
|
|
|
// create some tasks belonging to a previous iteration
|
|
tasks := []swarm.Task{
|
|
{
|
|
ID: "oldtask1",
|
|
Slot: 0,
|
|
NodeID: "",
|
|
DesiredState: swarm.TaskStateComplete,
|
|
Status: swarm.TaskStatus{State: swarm.TaskStateNew},
|
|
JobIteration: &swarm.Version{Index: 0},
|
|
}, {
|
|
ID: "oldtask2",
|
|
Slot: 1,
|
|
NodeID: "",
|
|
DesiredState: swarm.TaskStateComplete,
|
|
Status: swarm.TaskStatus{State: swarm.TaskStateComplete},
|
|
JobIteration: &swarm.Version{Index: 0},
|
|
},
|
|
}
|
|
|
|
ut.testUpdater(tasks, false, []progress.Progress{
|
|
// on the initial pass, we draw all of the progress bars at once, which
|
|
// puts them in order for the rest of the operation
|
|
{ID: "job progress", Action: "0 out of 5 complete", Current: 0, Total: 5, HideCounts: true},
|
|
{ID: "active tasks", Action: "0 out of 2 tasks"},
|
|
{ID: "1/5", Action: " "},
|
|
{ID: "2/5", Action: " "},
|
|
{ID: "3/5", Action: " "},
|
|
{ID: "4/5", Action: " "},
|
|
{ID: "5/5", Action: " "},
|
|
// from here on, we draw as normal. as a side effect, we will have a
|
|
// second update for the job progress and active tasks. This has no
|
|
// practical effect on the UI, it's just a side effect of the update
|
|
// logic.
|
|
{ID: "job progress", Action: "0 out of 5 complete", Current: 0, Total: 5, HideCounts: true},
|
|
{ID: "active tasks", Action: "0 out of 2 tasks"},
|
|
})
|
|
|
|
// wipe the old tasks out of the list
|
|
tasks = []swarm.Task{}
|
|
tasks = append(tasks,
|
|
swarm.Task{
|
|
ID: "task1",
|
|
Slot: 0,
|
|
NodeID: "",
|
|
DesiredState: swarm.TaskStateComplete,
|
|
Status: swarm.TaskStatus{State: swarm.TaskStateNew},
|
|
JobIteration: &swarm.Version{Index: service.JobStatus.JobIteration.Index},
|
|
},
|
|
swarm.Task{
|
|
ID: "task2",
|
|
Slot: 1,
|
|
NodeID: "",
|
|
DesiredState: swarm.TaskStateComplete,
|
|
Status: swarm.TaskStatus{State: swarm.TaskStateNew},
|
|
JobIteration: &swarm.Version{Index: service.JobStatus.JobIteration.Index},
|
|
},
|
|
)
|
|
ut.testUpdater(tasks, false, []progress.Progress{
|
|
{ID: "1/5", Action: "new ", Current: 1, Total: 10, HideCounts: true},
|
|
{ID: "2/5", Action: "new ", Current: 1, Total: 10, HideCounts: true},
|
|
{ID: "job progress", Action: "0 out of 5 complete", Current: 0, Total: 5, HideCounts: true},
|
|
{ID: "active tasks", Action: "2 out of 2 tasks"},
|
|
})
|
|
|
|
tasks[0].Status.State = swarm.TaskStatePreparing
|
|
tasks[1].Status.State = swarm.TaskStateAssigned
|
|
ut.testUpdater(tasks, false, []progress.Progress{
|
|
{ID: "1/5", Action: "preparing", Current: 6, Total: 10, HideCounts: true},
|
|
{ID: "2/5", Action: "assigned ", Current: 4, Total: 10, HideCounts: true},
|
|
{ID: "job progress", Action: "0 out of 5 complete", Current: 0, Total: 5, HideCounts: true},
|
|
{ID: "active tasks", Action: "2 out of 2 tasks"},
|
|
})
|
|
|
|
tasks[0].Status.State = swarm.TaskStateRunning
|
|
tasks[1].Status.State = swarm.TaskStatePreparing
|
|
ut.testUpdater(tasks, false, []progress.Progress{
|
|
{ID: "1/5", Action: "running ", Current: 9, Total: 10, HideCounts: true},
|
|
{ID: "2/5", Action: "preparing", Current: 6, Total: 10, HideCounts: true},
|
|
{ID: "job progress", Action: "0 out of 5 complete", Current: 0, Total: 5, HideCounts: true},
|
|
{ID: "active tasks", Action: "2 out of 2 tasks"},
|
|
})
|
|
|
|
tasks[0].Status.State = swarm.TaskStateComplete
|
|
tasks[1].Status.State = swarm.TaskStateComplete
|
|
ut.testUpdater(tasks, false, []progress.Progress{
|
|
{ID: "1/5", Action: "complete ", Current: 10, Total: 10, HideCounts: true},
|
|
{ID: "2/5", Action: "complete ", Current: 10, Total: 10, HideCounts: true},
|
|
{ID: "job progress", Action: "2 out of 5 complete", Current: 2, Total: 5, HideCounts: true},
|
|
{ID: "active tasks", Action: "0 out of 2 tasks"},
|
|
})
|
|
|
|
tasks = append(tasks,
|
|
swarm.Task{
|
|
ID: "task3",
|
|
Slot: 2,
|
|
NodeID: "",
|
|
DesiredState: swarm.TaskStateComplete,
|
|
Status: swarm.TaskStatus{State: swarm.TaskStateNew},
|
|
JobIteration: &swarm.Version{Index: service.JobStatus.JobIteration.Index},
|
|
},
|
|
swarm.Task{
|
|
ID: "task4",
|
|
Slot: 3,
|
|
NodeID: "",
|
|
DesiredState: swarm.TaskStateComplete,
|
|
Status: swarm.TaskStatus{State: swarm.TaskStateNew},
|
|
JobIteration: &swarm.Version{Index: service.JobStatus.JobIteration.Index},
|
|
},
|
|
)
|
|
|
|
ut.testUpdater(tasks, false, []progress.Progress{
|
|
{ID: "1/5", Action: "complete ", Current: 10, Total: 10, HideCounts: true},
|
|
{ID: "2/5", Action: "complete ", Current: 10, Total: 10, HideCounts: true},
|
|
{ID: "3/5", Action: "new ", Current: 1, Total: 10, HideCounts: true},
|
|
{ID: "4/5", Action: "new ", Current: 1, Total: 10, HideCounts: true},
|
|
{ID: "job progress", Action: "2 out of 5 complete", Current: 2, Total: 5, HideCounts: true},
|
|
{ID: "active tasks", Action: "2 out of 2 tasks"},
|
|
})
|
|
|
|
tasks[2].Status.State = swarm.TaskStateRunning
|
|
tasks[3].Status.State = swarm.TaskStateRunning
|
|
ut.testUpdater(tasks, false, []progress.Progress{
|
|
{ID: "1/5", Action: "complete ", Current: 10, Total: 10, HideCounts: true},
|
|
{ID: "2/5", Action: "complete ", Current: 10, Total: 10, HideCounts: true},
|
|
{ID: "3/5", Action: "running ", Current: 9, Total: 10, HideCounts: true},
|
|
{ID: "4/5", Action: "running ", Current: 9, Total: 10, HideCounts: true},
|
|
{ID: "job progress", Action: "2 out of 5 complete", Current: 2, Total: 5, HideCounts: true},
|
|
{ID: "active tasks", Action: "2 out of 2 tasks"},
|
|
})
|
|
|
|
tasks[3].Status.State = swarm.TaskStateComplete
|
|
tasks = append(tasks,
|
|
swarm.Task{
|
|
ID: "task5",
|
|
Slot: 4,
|
|
NodeID: "",
|
|
DesiredState: swarm.TaskStateComplete,
|
|
Status: swarm.TaskStatus{State: swarm.TaskStateRunning},
|
|
JobIteration: &swarm.Version{Index: service.JobStatus.JobIteration.Index},
|
|
},
|
|
)
|
|
ut.testUpdater(tasks, false, []progress.Progress{
|
|
{ID: "1/5", Action: "complete ", Current: 10, Total: 10, HideCounts: true},
|
|
{ID: "2/5", Action: "complete ", Current: 10, Total: 10, HideCounts: true},
|
|
{ID: "3/5", Action: "running ", Current: 9, Total: 10, HideCounts: true},
|
|
{ID: "4/5", Action: "complete ", Current: 10, Total: 10, HideCounts: true},
|
|
{ID: "5/5", Action: "running ", Current: 9, Total: 10, HideCounts: true},
|
|
{ID: "job progress", Action: "3 out of 5 complete", Current: 3, Total: 5, HideCounts: true},
|
|
{ID: "active tasks", Action: "2 out of 2 tasks"},
|
|
})
|
|
|
|
tasks[2].Status.State = swarm.TaskStateFailed
|
|
tasks[2].Status.Err = "the task failed"
|
|
ut.testUpdater(tasks, false, []progress.Progress{
|
|
{ID: "1/5", Action: "complete ", Current: 10, Total: 10, HideCounts: true},
|
|
{ID: "2/5", Action: "complete ", Current: 10, Total: 10, HideCounts: true},
|
|
{ID: "3/5", Action: "the task failed"},
|
|
{ID: "4/5", Action: "complete ", Current: 10, Total: 10, HideCounts: true},
|
|
{ID: "5/5", Action: "running ", Current: 9, Total: 10, HideCounts: true},
|
|
{ID: "job progress", Action: "3 out of 5 complete", Current: 3, Total: 5, HideCounts: true},
|
|
{ID: "active tasks", Action: "1 out of 2 tasks"},
|
|
})
|
|
|
|
tasks[4].Status.State = swarm.TaskStateComplete
|
|
tasks = append(tasks,
|
|
swarm.Task{
|
|
ID: "task6",
|
|
Slot: 2,
|
|
NodeID: "",
|
|
DesiredState: swarm.TaskStateComplete,
|
|
Status: swarm.TaskStatus{State: swarm.TaskStateRunning},
|
|
JobIteration: &swarm.Version{Index: service.JobStatus.JobIteration.Index},
|
|
},
|
|
)
|
|
ut.testUpdater(tasks, false, []progress.Progress{
|
|
{ID: "1/5", Action: "complete ", Current: 10, Total: 10, HideCounts: true},
|
|
{ID: "2/5", Action: "complete ", Current: 10, Total: 10, HideCounts: true},
|
|
{ID: "3/5", Action: "running ", Current: 9, Total: 10, HideCounts: true},
|
|
{ID: "4/5", Action: "complete ", Current: 10, Total: 10, HideCounts: true},
|
|
{ID: "5/5", Action: "complete ", Current: 10, Total: 10, HideCounts: true},
|
|
{ID: "job progress", Action: "4 out of 5 complete", Current: 4, Total: 5, HideCounts: true},
|
|
{ID: "active tasks", Action: "1 out of 1 tasks"},
|
|
})
|
|
|
|
tasks[5].Status.State = swarm.TaskStateComplete
|
|
ut.testUpdater(tasks, true, []progress.Progress{
|
|
{ID: "1/5", Action: "complete ", Current: 10, Total: 10, HideCounts: true},
|
|
{ID: "2/5", Action: "complete ", Current: 10, Total: 10, HideCounts: true},
|
|
{ID: "3/5", Action: "complete ", Current: 10, Total: 10, HideCounts: true},
|
|
{ID: "4/5", Action: "complete ", Current: 10, Total: 10, HideCounts: true},
|
|
{ID: "5/5", Action: "complete ", Current: 10, Total: 10, HideCounts: true},
|
|
{ID: "job progress", Action: "5 out of 5 complete", Current: 5, Total: 5, HideCounts: true},
|
|
{ID: "active tasks", Action: "0 out of 0 tasks"},
|
|
})
|
|
}
|
|
|
|
func TestReplicatedJobProgressUpdaterLarge(t *testing.T) {
|
|
concurrent := uint64(10)
|
|
total := uint64(50)
|
|
|
|
service := swarm.Service{
|
|
Spec: swarm.ServiceSpec{
|
|
Mode: swarm.ServiceMode{
|
|
ReplicatedJob: &swarm.ReplicatedJob{
|
|
MaxConcurrent: &concurrent,
|
|
TotalCompletions: &total,
|
|
},
|
|
},
|
|
},
|
|
JobStatus: &swarm.JobStatus{
|
|
JobIteration: swarm.Version{Index: 0},
|
|
},
|
|
}
|
|
|
|
p := &mockProgress{}
|
|
ut := updaterTester{
|
|
t: t,
|
|
updater: newReplicatedJobProgressUpdater(service, p),
|
|
p: p,
|
|
activeNodes: map[string]struct{}{"a": {}, "b": {}},
|
|
service: service,
|
|
}
|
|
|
|
tasks := []swarm.Task{}
|
|
|
|
// see the comments in TestReplicatedJobProgressUpdaterSmall for why
|
|
// we write this out twice.
|
|
ut.testUpdater(tasks, false, []progress.Progress{
|
|
{ID: "job progress", Action: " 0 out of 50 complete", Current: 0, Total: 50, HideCounts: true},
|
|
{ID: "active tasks", Action: " 0 out of 10 tasks"},
|
|
// we don't write out individual status bars for a large job, only the
|
|
// overall progress bar
|
|
{ID: "job progress", Action: " 0 out of 50 complete", Current: 0, Total: 50, HideCounts: true},
|
|
{ID: "active tasks", Action: " 0 out of 10 tasks"},
|
|
})
|
|
|
|
// first, create the initial batch of running tasks
|
|
for i := 0; i < int(concurrent); i++ {
|
|
tasks = append(tasks, swarm.Task{
|
|
ID: strconv.Itoa(i),
|
|
Slot: i,
|
|
DesiredState: swarm.TaskStateComplete,
|
|
Status: swarm.TaskStatus{State: swarm.TaskStateNew},
|
|
JobIteration: &swarm.Version{Index: 0},
|
|
})
|
|
|
|
ut.testUpdater(tasks, false, []progress.Progress{
|
|
{ID: "job progress", Action: " 0 out of 50 complete", Current: 0, Total: 50, HideCounts: true},
|
|
{ID: "active tasks", Action: fmt.Sprintf("%2d out of 10 tasks", i+1)},
|
|
})
|
|
}
|
|
|
|
// now, start moving tasks to completed, and starting new tasks after them.
|
|
// to do this, we'll start at 0, mark a task complete, and then append a
|
|
// new one. we'll stop before we get to the end, because the end has a
|
|
// steadily decreasing denominator for the active tasks
|
|
//
|
|
// for 10 concurrent 50 total, this means we'll stop at 50 - 10 = 40 tasks
|
|
// in the completed state, 10 tasks running. the last index in use will be
|
|
// 39.
|
|
for i := 0; i < int(total)-int(concurrent); i++ {
|
|
tasks[i].Status.State = swarm.TaskStateComplete
|
|
ut.testUpdater(tasks, false, []progress.Progress{
|
|
{ID: "job progress", Action: fmt.Sprintf("%2d out of 50 complete", i+1), Current: int64(i + 1), Total: 50, HideCounts: true},
|
|
{ID: "active tasks", Action: " 9 out of 10 tasks"},
|
|
})
|
|
|
|
last := len(tasks)
|
|
tasks = append(tasks, swarm.Task{
|
|
ID: strconv.Itoa(last),
|
|
Slot: last,
|
|
DesiredState: swarm.TaskStateComplete,
|
|
Status: swarm.TaskStatus{State: swarm.TaskStateNew},
|
|
JobIteration: &swarm.Version{Index: 0},
|
|
})
|
|
|
|
ut.testUpdater(tasks, false, []progress.Progress{
|
|
{ID: "job progress", Action: fmt.Sprintf("%2d out of 50 complete", i+1), Current: int64(i + 1), Total: 50, HideCounts: true},
|
|
{ID: "active tasks", Action: "10 out of 10 tasks"},
|
|
})
|
|
}
|
|
|
|
// quick check, to make sure we did the math right when we wrote this code:
|
|
// we do have 50 tasks in the slice, right?
|
|
assert.Check(t, is.Equal(len(tasks), int(total)))
|
|
|
|
// now, we're down to our last 10 tasks, which are all running. We need to
|
|
// wind these down
|
|
for i := int(total) - int(concurrent) - 1; i < int(total); i++ {
|
|
tasks[i].Status.State = swarm.TaskStateComplete
|
|
ut.testUpdater(tasks, (i+1 == int(total)), []progress.Progress{
|
|
{ID: "job progress", Action: fmt.Sprintf("%2d out of 50 complete", i+1), Current: int64(i + 1), Total: 50, HideCounts: true},
|
|
{ID: "active tasks", Action: fmt.Sprintf("%2[1]d out of %2[1]d tasks", int(total)-(i+1))},
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestGlobalJobProgressUpdaterSmall(t *testing.T) {
|
|
service := swarm.Service{
|
|
Spec: swarm.ServiceSpec{
|
|
Mode: swarm.ServiceMode{
|
|
GlobalJob: &swarm.GlobalJob{},
|
|
},
|
|
},
|
|
JobStatus: &swarm.JobStatus{
|
|
JobIteration: swarm.Version{Index: 1},
|
|
},
|
|
}
|
|
|
|
p := &mockProgress{}
|
|
ut := updaterTester{
|
|
t: t,
|
|
updater: &globalJobProgressUpdater{
|
|
progressOut: p,
|
|
},
|
|
p: p,
|
|
activeNodes: map[string]struct{}{"a": {}, "b": {}, "c": {}},
|
|
service: service,
|
|
}
|
|
|
|
tasks := []swarm.Task{
|
|
{
|
|
ID: "oldtask1",
|
|
DesiredState: swarm.TaskStateComplete,
|
|
Status: swarm.TaskStatus{State: swarm.TaskStateComplete},
|
|
JobIteration: &swarm.Version{Index: 0},
|
|
NodeID: "a",
|
|
}, {
|
|
ID: "oldtask2",
|
|
DesiredState: swarm.TaskStateComplete,
|
|
Status: swarm.TaskStatus{State: swarm.TaskStateComplete},
|
|
JobIteration: &swarm.Version{Index: 0},
|
|
NodeID: "b",
|
|
}, {
|
|
ID: "task1",
|
|
DesiredState: swarm.TaskStateComplete,
|
|
Status: swarm.TaskStatus{State: swarm.TaskStateNew},
|
|
JobIteration: &swarm.Version{Index: 1},
|
|
NodeID: "a",
|
|
}, {
|
|
ID: "task2",
|
|
DesiredState: swarm.TaskStateComplete,
|
|
Status: swarm.TaskStatus{State: swarm.TaskStateNew},
|
|
JobIteration: &swarm.Version{Index: 1},
|
|
NodeID: "b",
|
|
}, {
|
|
ID: "task3",
|
|
DesiredState: swarm.TaskStateComplete,
|
|
Status: swarm.TaskStatus{State: swarm.TaskStateNew},
|
|
JobIteration: &swarm.Version{Index: 1},
|
|
NodeID: "c",
|
|
},
|
|
}
|
|
|
|
// we don't know how many tasks will be created until we get the initial
|
|
// task list, so we should not write out any definitive answers yet.
|
|
ut.testUpdater([]swarm.Task{}, false, []progress.Progress{
|
|
{ID: "job progress", Action: "waiting for tasks"},
|
|
})
|
|
|
|
ut.testUpdaterNoOrder(tasks, false, []progress.Progress{
|
|
{ID: "job progress", Action: "0 out of 3 complete", Current: 0, Total: 3, HideCounts: true},
|
|
{ID: "a", Action: "new ", Current: 1, Total: 10, HideCounts: true},
|
|
{ID: "b", Action: "new ", Current: 1, Total: 10, HideCounts: true},
|
|
{ID: "c", Action: "new ", Current: 1, Total: 10, HideCounts: true},
|
|
{ID: "job progress", Action: "0 out of 3 complete", Current: 0, Total: 3, HideCounts: true},
|
|
})
|
|
|
|
tasks[2].Status.State = swarm.TaskStatePreparing
|
|
tasks[3].Status.State = swarm.TaskStateRunning
|
|
tasks[4].Status.State = swarm.TaskStateAccepted
|
|
ut.testUpdaterNoOrder(tasks, false, []progress.Progress{
|
|
{ID: "a", Action: "preparing", Current: 6, Total: 10, HideCounts: true},
|
|
{ID: "b", Action: "running ", Current: 9, Total: 10, HideCounts: true},
|
|
{ID: "c", Action: "accepted ", Current: 5, Total: 10, HideCounts: true},
|
|
{ID: "job progress", Action: "0 out of 3 complete", Current: 0, Total: 3, HideCounts: true},
|
|
})
|
|
|
|
tasks[2].Status.State = swarm.TaskStateRunning
|
|
tasks[3].Status.State = swarm.TaskStateComplete
|
|
tasks[4].Status.State = swarm.TaskStateRunning
|
|
ut.testUpdaterNoOrder(tasks, false, []progress.Progress{
|
|
{ID: "a", Action: "running ", Current: 9, Total: 10, HideCounts: true},
|
|
{ID: "b", Action: "complete ", Current: 10, Total: 10, HideCounts: true},
|
|
{ID: "c", Action: "running ", Current: 9, Total: 10, HideCounts: true},
|
|
{ID: "job progress", Action: "1 out of 3 complete", Current: 1, Total: 3, HideCounts: true},
|
|
})
|
|
|
|
tasks[2].Status.State = swarm.TaskStateFailed
|
|
tasks[2].Status.Err = "task failed"
|
|
tasks[4].Status.State = swarm.TaskStateComplete
|
|
ut.testUpdaterNoOrder(tasks, false, []progress.Progress{
|
|
{ID: "a", Action: "task failed"},
|
|
{ID: "b", Action: "complete ", Current: 10, Total: 10, HideCounts: true},
|
|
{ID: "c", Action: "complete ", Current: 10, Total: 10, HideCounts: true},
|
|
{ID: "job progress", Action: "2 out of 3 complete", Current: 2, Total: 3, HideCounts: true},
|
|
})
|
|
|
|
tasks = append(tasks, swarm.Task{
|
|
ID: "task4",
|
|
DesiredState: swarm.TaskStateComplete,
|
|
Status: swarm.TaskStatus{State: swarm.TaskStatePreparing},
|
|
NodeID: tasks[2].NodeID,
|
|
JobIteration: &swarm.Version{Index: 1},
|
|
})
|
|
|
|
ut.testUpdaterNoOrder(tasks, false, []progress.Progress{
|
|
{ID: "a", Action: "preparing", Current: 6, Total: 10, HideCounts: true},
|
|
{ID: "b", Action: "complete ", Current: 10, Total: 10, HideCounts: true},
|
|
{ID: "c", Action: "complete ", Current: 10, Total: 10, HideCounts: true},
|
|
{ID: "job progress", Action: "2 out of 3 complete", Current: 2, Total: 3, HideCounts: true},
|
|
})
|
|
|
|
tasks[5].Status.State = swarm.TaskStateComplete
|
|
ut.testUpdaterNoOrder(tasks, true, []progress.Progress{
|
|
{ID: "a", Action: "complete ", Current: 10, Total: 10, HideCounts: true},
|
|
{ID: "b", Action: "complete ", Current: 10, Total: 10, HideCounts: true},
|
|
{ID: "c", Action: "complete ", Current: 10, Total: 10, HideCounts: true},
|
|
{ID: "job progress", Action: "3 out of 3 complete", Current: 3, Total: 3, HideCounts: true},
|
|
})
|
|
}
|
|
|
|
func TestGlobalJobProgressUpdaterLarge(t *testing.T) {
|
|
service := swarm.Service{
|
|
Spec: swarm.ServiceSpec{
|
|
Mode: swarm.ServiceMode{
|
|
GlobalJob: &swarm.GlobalJob{},
|
|
},
|
|
},
|
|
JobStatus: &swarm.JobStatus{
|
|
JobIteration: swarm.Version{Index: 1},
|
|
},
|
|
}
|
|
|
|
activeNodes := map[string]struct{}{}
|
|
for i := 0; i < 50; i++ {
|
|
activeNodes[fmt.Sprintf("node%v", i)] = struct{}{}
|
|
}
|
|
|
|
p := &mockProgress{}
|
|
ut := updaterTester{
|
|
t: t,
|
|
updater: &globalJobProgressUpdater{
|
|
progressOut: p,
|
|
},
|
|
p: p,
|
|
activeNodes: activeNodes,
|
|
service: service,
|
|
}
|
|
|
|
tasks := []swarm.Task{}
|
|
for nodeID := range activeNodes {
|
|
tasks = append(tasks, swarm.Task{
|
|
ID: "task" + nodeID,
|
|
NodeID: nodeID,
|
|
DesiredState: swarm.TaskStateComplete,
|
|
Status: swarm.TaskStatus{
|
|
State: swarm.TaskStateNew,
|
|
},
|
|
JobIteration: &swarm.Version{Index: 1},
|
|
})
|
|
}
|
|
|
|
// no bars, because too many tasks
|
|
ut.testUpdater(tasks, false, []progress.Progress{
|
|
{ID: "job progress", Action: " 0 out of 50 complete", Current: 0, Total: 50, HideCounts: true},
|
|
{ID: "job progress", Action: " 0 out of 50 complete", Current: 0, Total: 50, HideCounts: true},
|
|
})
|
|
|
|
for i := range tasks {
|
|
tasks[i].Status.State = swarm.TaskStateComplete
|
|
ut.testUpdater(tasks, i+1 == len(activeNodes), []progress.Progress{
|
|
{
|
|
ID: "job progress",
|
|
Action: fmt.Sprintf("%2d out of 50 complete", i+1),
|
|
Current: int64(i + 1), Total: 50, HideCounts: true,
|
|
},
|
|
})
|
|
}
|
|
}
|