Skip to content

Commit 5141363

Browse files
mrnuggetLawnGnome
andauthored
Show current worker goroutine status below progress bar (#338)
* First try to add progress with status bars * Refactoring and renaming * Tiny step and refactoring * Get ProgressWithStatusBars working * Change API of progressWithStatusbars * Refactor StatusBar printing * Embed ProgressTTY in ProgressWithStatusBarsTTY * More merging of progressTTY and progressWithStatusbarsTTY * Add simple progress with status bars * Move demo of progress with bars to _examples * Restore overwritten methods * Restore more overwritten methods * Restore another overwritten method * WIP: Wire up status bars with executor * Print status bars below progress bar and add ASCII box characters * WIP1: Introduce campaign progress printer * WIP2: More refactoring of campaign status printing * More cleanup and refactoring * Change styling of progress bar with status * Add time to StatusBar and display right-aligned * Use a const * Add CHANGELOG entry * Ignore invisible characters for text length in progress bar * Update CHANGELOG.md Co-authored-by: Adam Harvey <[email protected]> * Erase previous content with spaces on blank line * Thin box drawing lines * Improve non-TTY output for progress indicators with status bars. Co-authored-by: Adam Harvey <[email protected]> Co-authored-by: Adam Harvey <[email protected]>
1 parent 2fdf37c commit 5141363

13 files changed

+813
-103
lines changed

CHANGELOG.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ All notable changes to `src-cli` are documented in this file.
2020
- Error reporting by `src campaign [preview|apply]` has been improved and now includes more information about which step failed in which repository. [#325](https://github.com/sourcegraph/src-cli/pull/325)
2121
- The default behaviour of `src campaigns [preview|apply]` has been changed to retain downloaded archives of repositories for better performance across re-runs of the command. To use the old behaviour and delete the archives use the `-clean-archives` flag. Repository archives are also not stored in the directory for temp data (see `-tmp` flag) anymore but in the cache directory, which can be configured with the `-cache` flag. To manually delete archives between runs, delete the `*.zip` files in the `-cache` directory (see `src campaigns -help` for its default location).
2222
- `src campaign [preview|apply]` now check whether `git` and `docker` are available before trying to execute a campaign spec's steps. [#326](https://github.com/sourcegraph/src-cli/pull/326)
23+
- The progress bar displayed by `src campaign [preview|apply]` has been extended by status bars that show which steps are currently being executed for each repository. [#338](https://github.com/sourcegraph/src-cli/pull/338)
2324

2425
### Fixed
2526

Lines changed: 188 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,188 @@
1+
package main
2+
3+
import (
4+
"fmt"
5+
"strings"
6+
7+
"github.com/sourcegraph/go-diff/diff"
8+
"github.com/sourcegraph/src-cli/internal/campaigns"
9+
"github.com/sourcegraph/src-cli/internal/output"
10+
)
11+
12+
func newCampaignProgressPrinter(out *output.Output, numParallelism int) *campaignProgressPrinter {
13+
return &campaignProgressPrinter{
14+
out: out,
15+
16+
numParallelism: numParallelism,
17+
18+
completedTasks: map[string]bool{},
19+
runningTasks: map[string]*campaigns.TaskStatus{},
20+
21+
repoStatusBar: map[string]int{},
22+
statusBarRepo: map[int]string{},
23+
}
24+
}
25+
26+
type campaignProgressPrinter struct {
27+
out *output.Output
28+
progress output.ProgressWithStatusBars
29+
30+
maxRepoName int
31+
numParallelism int
32+
33+
completedTasks map[string]bool
34+
runningTasks map[string]*campaigns.TaskStatus
35+
36+
repoStatusBar map[string]int
37+
statusBarRepo map[int]string
38+
}
39+
40+
func (p *campaignProgressPrinter) initProgressBar(statuses []*campaigns.TaskStatus) {
41+
statusBars := []*output.StatusBar{}
42+
for i := 0; i < p.numParallelism; i++ {
43+
statusBars = append(statusBars, output.NewStatusBarWithLabel("Starting worker..."))
44+
}
45+
46+
p.progress = p.out.ProgressWithStatusBars([]output.ProgressBar{{
47+
Label: fmt.Sprintf("Executing steps in %d repositories", len(statuses)),
48+
Max: float64(len(statuses)),
49+
}}, statusBars, nil)
50+
}
51+
52+
func (p *campaignProgressPrinter) Complete() {
53+
if p.progress != nil {
54+
p.progress.Complete()
55+
}
56+
}
57+
58+
func (p *campaignProgressPrinter) PrintStatuses(statuses []*campaigns.TaskStatus) {
59+
if p.progress == nil {
60+
p.initProgressBar(statuses)
61+
}
62+
63+
newlyCompleted := []*campaigns.TaskStatus{}
64+
currentlyRunning := []*campaigns.TaskStatus{}
65+
66+
for _, ts := range statuses {
67+
if len(ts.RepoName) > p.maxRepoName {
68+
p.maxRepoName = len(ts.RepoName)
69+
}
70+
71+
if ts.IsCompleted() {
72+
if !p.completedTasks[ts.RepoName] {
73+
p.completedTasks[ts.RepoName] = true
74+
newlyCompleted = append(newlyCompleted, ts)
75+
}
76+
77+
if _, ok := p.runningTasks[ts.RepoName]; ok {
78+
delete(p.runningTasks, ts.RepoName)
79+
80+
// Free slot
81+
idx := p.repoStatusBar[ts.RepoName]
82+
delete(p.statusBarRepo, idx)
83+
}
84+
}
85+
86+
if ts.IsRunning() {
87+
currentlyRunning = append(currentlyRunning, ts)
88+
}
89+
90+
}
91+
92+
p.progress.SetValue(0, float64(len(p.completedTasks)))
93+
94+
newlyStarted := map[string]*campaigns.TaskStatus{}
95+
statusBarIndex := 0
96+
for _, ts := range currentlyRunning {
97+
if _, ok := p.runningTasks[ts.RepoName]; ok {
98+
continue
99+
}
100+
101+
newlyStarted[ts.RepoName] = ts
102+
p.runningTasks[ts.RepoName] = ts
103+
104+
// Find free slot
105+
_, ok := p.statusBarRepo[statusBarIndex]
106+
for ok {
107+
statusBarIndex += 1
108+
_, ok = p.statusBarRepo[statusBarIndex]
109+
}
110+
111+
p.statusBarRepo[statusBarIndex] = ts.RepoName
112+
p.repoStatusBar[ts.RepoName] = statusBarIndex
113+
}
114+
115+
for _, ts := range newlyCompleted {
116+
statusText, err := taskStatusText(ts)
117+
if err != nil {
118+
p.progress.Verbosef("%-*s failed to display status: %s", p.maxRepoName, ts.RepoName, err)
119+
continue
120+
}
121+
122+
p.progress.Verbosef("%-*s %s", p.maxRepoName, ts.RepoName, statusText)
123+
124+
if idx, ok := p.repoStatusBar[ts.RepoName]; ok {
125+
// Log that this task completed, but only if there is no
126+
// currently executing one in this bar, to avoid flicker.
127+
if _, ok := p.statusBarRepo[idx]; !ok {
128+
p.progress.StatusBarCompletef(idx, statusText)
129+
}
130+
delete(p.repoStatusBar, ts.RepoName)
131+
}
132+
}
133+
134+
for statusBar, repo := range p.statusBarRepo {
135+
ts, ok := p.runningTasks[repo]
136+
if !ok {
137+
// This should not happen
138+
continue
139+
}
140+
141+
statusText, err := taskStatusText(ts)
142+
if err != nil {
143+
p.progress.Verbosef("%-*s failed to display status: %s", p.maxRepoName, ts.RepoName, err)
144+
continue
145+
}
146+
147+
if _, ok := newlyStarted[repo]; ok {
148+
p.progress.StatusBarResetf(statusBar, ts.RepoName, statusText)
149+
} else {
150+
p.progress.StatusBarUpdatef(statusBar, statusText)
151+
}
152+
}
153+
}
154+
155+
func taskStatusText(ts *campaigns.TaskStatus) (string, error) {
156+
var statusText string
157+
158+
if ts.IsCompleted() {
159+
if ts.ChangesetSpec == nil {
160+
statusText = "No changes"
161+
} else {
162+
fileDiffs, err := diff.ParseMultiFileDiff([]byte(ts.ChangesetSpec.Commits[0].Diff))
163+
if err != nil {
164+
return "", err
165+
}
166+
167+
statusText = diffStatDescription(fileDiffs) + " " + diffStatDiagram(sumDiffStats(fileDiffs))
168+
}
169+
170+
if ts.Cached {
171+
statusText += " (cached)"
172+
}
173+
} else if ts.IsRunning() {
174+
if ts.CurrentlyExecuting != "" {
175+
lines := strings.Split(ts.CurrentlyExecuting, "\n")
176+
escapedLine := strings.ReplaceAll(lines[0], "%", "%%")
177+
if len(lines) > 1 {
178+
statusText = fmt.Sprintf("%s ...", escapedLine)
179+
} else {
180+
statusText = fmt.Sprintf("%s", escapedLine)
181+
}
182+
} else {
183+
statusText = fmt.Sprintf("...")
184+
}
185+
}
186+
187+
return statusText, nil
188+
}

cmd/src/campaigns_common.go

Lines changed: 5 additions & 79 deletions
Original file line numberDiff line numberDiff line change
@@ -240,12 +240,12 @@ func campaignsExecute(ctx context.Context, out *output.Output, svc *campaigns.Se
240240
campaignsCompletePending(pending, "Resolved repositories")
241241
}
242242

243-
execProgress, execProgressComplete := executeCampaignSpecProgress(out)
244-
specs, err := svc.ExecuteCampaignSpec(ctx, repos, executor, campaignSpec, execProgress)
243+
p := newCampaignProgressPrinter(out, opts.Parallelism)
244+
specs, err := svc.ExecuteCampaignSpec(ctx, repos, executor, campaignSpec, p.PrintStatuses)
245245
if err != nil {
246246
return "", "", err
247247
}
248-
execProgressComplete()
248+
p.Complete()
249249

250250
if logFiles := executor.LogFiles(); len(logFiles) > 0 && flags.keepLogs {
251251
func() {
@@ -372,9 +372,10 @@ func diffStatDiagram(stat diff.Stat) string {
372372
added *= x
373373
deleted *= x
374374
}
375-
return fmt.Sprintf("%s%s%s%s",
375+
return fmt.Sprintf("%s%s%s%s%s",
376376
output.StyleLinesAdded, strings.Repeat("+", int(added)),
377377
output.StyleLinesDeleted, strings.Repeat("-", int(deleted)),
378+
output.StyleReset,
378379
)
379380
}
380381

@@ -409,78 +410,3 @@ func contextCancelOnInterrupt(parent context.Context) (context.Context, func())
409410
ctxCancel()
410411
}
411412
}
412-
413-
// executeCampaignSpecProgress returns a function that can be passed to
414-
// (*Service).ExecuteCampaignSpec as the "progress" function.
415-
//
416-
// It prints a progress bar and, if verbose mode is activated, diff stats of
417-
// the produced diffs.
418-
//
419-
// The second return value is the "complete" function that completes the
420-
// progress bar and should be called after ExecuteCampaignSpec returns
421-
// successfully.
422-
func executeCampaignSpecProgress(out *output.Output) (func(statuses []*campaigns.TaskStatus), func()) {
423-
var (
424-
progress output.Progress
425-
maxRepoName int
426-
completedTasks = map[string]bool{}
427-
)
428-
429-
complete := func() {
430-
if progress != nil {
431-
progress.Complete()
432-
}
433-
}
434-
435-
progressFunc := func(statuses []*campaigns.TaskStatus) {
436-
if progress == nil {
437-
progress = out.Progress([]output.ProgressBar{{
438-
Label: fmt.Sprintf("Executing steps in %d repositories", len(statuses)),
439-
Max: float64(len(statuses)),
440-
}}, nil)
441-
}
442-
443-
unloggedCompleted := []*campaigns.TaskStatus{}
444-
445-
for _, ts := range statuses {
446-
if len(ts.RepoName) > maxRepoName {
447-
maxRepoName = len(ts.RepoName)
448-
}
449-
450-
if ts.FinishedAt.IsZero() {
451-
continue
452-
}
453-
454-
if !completedTasks[ts.RepoName] {
455-
completedTasks[ts.RepoName] = true
456-
unloggedCompleted = append(unloggedCompleted, ts)
457-
}
458-
459-
}
460-
461-
progress.SetValue(0, float64(len(completedTasks)))
462-
463-
for _, ts := range unloggedCompleted {
464-
var statusText string
465-
466-
if ts.ChangesetSpec == nil {
467-
statusText = "No changes"
468-
} else {
469-
fileDiffs, err := diff.ParseMultiFileDiff([]byte(ts.ChangesetSpec.Commits[0].Diff))
470-
if err != nil {
471-
panic(err)
472-
}
473-
474-
statusText = diffStatDescription(fileDiffs) + " " + diffStatDiagram(sumDiffStats(fileDiffs))
475-
}
476-
477-
if ts.Cached {
478-
statusText += " (cached)"
479-
}
480-
481-
progress.Verbosef("%-*s %s", maxRepoName, ts.RepoName, statusText)
482-
}
483-
}
484-
485-
return progressFunc, complete
486-
}

internal/campaigns/executor.go

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,12 +60,21 @@ type TaskStatus struct {
6060
FinishedAt time.Time
6161

6262
// TODO: add current step and progress fields.
63+
CurrentlyExecuting string
6364

6465
// Result fields.
6566
ChangesetSpec *ChangesetSpec
6667
Err error
6768
}
6869

70+
func (ts *TaskStatus) IsRunning() bool {
71+
return !ts.StartedAt.IsZero() && ts.FinishedAt.IsZero()
72+
}
73+
74+
func (ts *TaskStatus) IsCompleted() bool {
75+
return !ts.StartedAt.IsZero() && !ts.FinishedAt.IsZero()
76+
}
77+
6978
type executor struct {
7079
ExecutorOpts
7180

@@ -156,6 +165,7 @@ func (x *executor) do(ctx context.Context, task *Task) (err error) {
156165
// Ensure that the status is updated when we're done.
157166
defer func() {
158167
status.FinishedAt = time.Now()
168+
status.CurrentlyExecuting = ""
159169
status.Err = err
160170
x.updateTaskStatus(task, status)
161171
}()
@@ -231,7 +241,10 @@ func (x *executor) do(ctx context.Context, task *Task) (err error) {
231241
defer cancel()
232242

233243
// Actually execute the steps.
234-
diff, err := runSteps(runCtx, x.creator, task.Repository, task.Steps, log, x.tempDir)
244+
diff, err := runSteps(runCtx, x.creator, task.Repository, task.Steps, log, x.tempDir, func(currentlyExecuting string) {
245+
status.CurrentlyExecuting = currentlyExecuting
246+
x.updateTaskStatus(task, status)
247+
})
235248
if err != nil {
236249
if reachedTimeout(runCtx, err) {
237250
err = &errTimeoutReached{timeout: x.Timeout}

internal/campaigns/run_steps.go

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,9 @@ import (
1717
"github.com/sourcegraph/src-cli/internal/campaigns/graphql"
1818
)
1919

20-
func runSteps(ctx context.Context, wc *WorkspaceCreator, repo *graphql.Repository, steps []Step, logger *TaskLogger, tempDir string) ([]byte, error) {
20+
func runSteps(ctx context.Context, wc *WorkspaceCreator, repo *graphql.Repository, steps []Step, logger *TaskLogger, tempDir string, reportProgress func(string)) ([]byte, error) {
21+
reportProgress("Downloading archive")
22+
2123
volumeDir, err := wc.Create(ctx, repo)
2224
if err != nil {
2325
return nil, errors.Wrap(err, "creating workspace")
@@ -34,6 +36,7 @@ func runSteps(ctx context.Context, wc *WorkspaceCreator, repo *graphql.Repositor
3436
return out, nil
3537
}
3638

39+
reportProgress("Initializing workspace")
3740
if _, err := runGitCmd("init"); err != nil {
3841
return nil, errors.Wrap(err, "git init failed")
3942
}
@@ -59,6 +62,7 @@ func runSteps(ctx context.Context, wc *WorkspaceCreator, repo *graphql.Repositor
5962

6063
for i, step := range steps {
6164
logger.Logf("[Step %d] docker run %s %q", i+1, step.Container, step.Run)
65+
reportProgress(step.Run)
6266

6367
cidFile, err := ioutil.TempFile(tempDir, repo.Slug()+"-container-id")
6468
if err != nil {
@@ -143,6 +147,7 @@ func runSteps(ctx context.Context, wc *WorkspaceCreator, repo *graphql.Repositor
143147
return nil, errors.Wrap(err, "git add failed")
144148
}
145149

150+
reportProgress("Calculating diff")
146151
// As of Sourcegraph 3.14 we only support unified diff format.
147152
// That means we need to strip away the `a/` and `/b` prefixes with `--no-prefix`.
148153
// See: https://github.com/sourcegraph/sourcegraph/blob/82d5e7e1562fef6be5c0b17f18631040fd330835/enterprise/internal/campaigns/service.go#L324-L329

0 commit comments

Comments
 (0)