Skip to content

Commit bc8ab68

Browse files
mrnuggetmrnugget
authored
For internal use only: add a -text-only flag to src batch [apply|preview] (#562)
* Start hacking on text-only output * Get stupidest text only output working * Print JSON objects per line when logging text * Remove last use of output in text-only mode * Update CHANGELOG
1 parent 621ef79 commit bc8ab68

File tree

5 files changed

+311
-28
lines changed

5 files changed

+311
-28
lines changed

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ All notable changes to `src-cli` are documented in this file.
1414
### Added
1515

1616
- Starting with Sourcegraph 3.30.0, the `published` field is optional in batch specs. If omitted, the publication state will be controlled through the Batch Changes UI. [#538](https://github.com/sourcegraph/src-cli/pull/538)
17+
- For internal use only: `-text-only` flag added to `src batch [apply|preview]`. [#562](https://github.com/sourcegraph/src-cli/pull/562)
1718

1819
### Changed
1920

‎cmd/src/batch_apply.go‎

Lines changed: 26 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -38,22 +38,35 @@ Examples:
3838
return &usageError{errors.New("additional arguments not allowed")}
3939
}
4040

41-
out := output.NewOutput(flagSet.Output(), output.OutputOpts{Verbose: *verbose})
42-
4341
ctx, cancel := contextCancelOnInterrupt(context.Background())
4442
defer cancel()
4543

46-
err := executeBatchSpec(ctx, executeBatchSpecOpts{
47-
flags: flags,
48-
out: out,
49-
client: cfg.apiClient(flags.api, flagSet.Output()),
50-
51-
applyBatchSpec: true,
52-
})
53-
if err != nil {
54-
printExecutionError(out, err)
55-
out.Write("")
56-
return &exitCodeError{nil, 1}
44+
var err error
45+
if flags.textOnly {
46+
err = textOnlyExecuteBatchSpec(ctx, executeBatchSpecOpts{
47+
flags: flags,
48+
client: cfg.apiClient(flags.api, flagSet.Output()),
49+
50+
applyBatchSpec: true,
51+
})
52+
if err != nil {
53+
fmt.Printf("ERROR: %s\n", err)
54+
return &exitCodeError{nil, 1}
55+
}
56+
} else {
57+
out := output.NewOutput(flagSet.Output(), output.OutputOpts{Verbose: *verbose})
58+
err = executeBatchSpec(ctx, executeBatchSpecOpts{
59+
flags: flags,
60+
out: out,
61+
client: cfg.apiClient(flags.api, flagSet.Output()),
62+
63+
applyBatchSpec: true,
64+
})
65+
if err != nil {
66+
printExecutionError(out, err)
67+
out.Write("")
68+
return &exitCodeError{nil, 1}
69+
}
5770
}
5871

5972
return nil

‎cmd/src/batch_common.go‎

Lines changed: 220 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,13 +47,20 @@ type batchExecuteFlags struct {
4747
workspace string
4848
cleanArchives bool
4949
skipErrors bool
50+
51+
// EXPERIMENTAL
52+
textOnly bool
5053
}
5154

5255
func newBatchExecuteFlags(flagSet *flag.FlagSet, cacheDir, tempDir string) *batchExecuteFlags {
5356
caf := &batchExecuteFlags{
5457
api: api.NewFlags(flagSet),
5558
}
5659

60+
flagSet.BoolVar(
61+
&caf.textOnly, "text-only", false,
62+
"INTERNAL USE ONLY. EXPERIMENTAL. Switches off the TUI to only print JSON lines.",
63+
)
5764
flagSet.BoolVar(
5865
&caf.allowUnsupported, "allow-unsupported", false,
5966
"Allow unsupported code hosts.",
@@ -415,6 +422,219 @@ func executeBatchSpec(ctx context.Context, opts executeBatchSpecOpts) error {
415422
return nil
416423
}
417424

425+
// TODO: This is a straight up copy of the other function
426+
func textOnlyExecuteBatchSpec(ctx context.Context, opts executeBatchSpecOpts) error {
427+
svc := service.New(&service.Opts{
428+
AllowUnsupported: opts.flags.allowUnsupported,
429+
AllowIgnored: opts.flags.allowIgnored,
430+
Client: opts.client,
431+
})
432+
433+
if err := svc.DetermineFeatureFlags(ctx); err != nil {
434+
return err
435+
}
436+
437+
if err := checkExecutable("git", "version"); err != nil {
438+
return err
439+
}
440+
441+
if err := checkExecutable("docker", "version"); err != nil {
442+
return err
443+
}
444+
445+
// Parse flags and build up our service and executor options.
446+
logOperationStart("PARSING_BATCH_SPEC", "")
447+
batchSpec, rawSpec, err := batchParseSpec(opts.out, &opts.flags.file, svc)
448+
if err != nil {
449+
return err
450+
}
451+
logOperationSuccess("PARSING_BATCH_SPEC", "")
452+
453+
logOperationStart("RESOLVING_NAMESPACE", "")
454+
namespace, err := svc.ResolveNamespace(ctx, opts.flags.namespace)
455+
if err != nil {
456+
return err
457+
}
458+
logOperationSuccess("RESOLVING_NAMESPACE", fmt.Sprintf("Namespace: %s", namespace))
459+
460+
logOperationStart("PREPARING_DOCKER_IMAGES", "")
461+
err = svc.SetDockerImages(ctx, batchSpec, func(perc float64) {
462+
logOperationProgress("PREPARING_DOCKER_IMAGES", fmt.Sprintf("%d%% done", int(perc*100)))
463+
})
464+
if err != nil {
465+
return err
466+
}
467+
logOperationSuccess("PREPARING_DOCKER_IMAGES", "")
468+
469+
logOperationStart("DETERMINING_WORKSPACE_TYPE", "")
470+
workspaceCreator := workspace.NewCreator(ctx, opts.flags.workspace, opts.flags.cacheDir, opts.flags.tempDir, batchSpec.Steps)
471+
if workspaceCreator.Type() == workspace.CreatorTypeVolume {
472+
_, err = svc.EnsureImage(ctx, workspace.DockerVolumeWorkspaceImage)
473+
if err != nil {
474+
return err
475+
}
476+
}
477+
switch workspaceCreator.Type() {
478+
case workspace.CreatorTypeVolume:
479+
logOperationSuccess("DETERMINING_WORKSPACE_TYPE", "VOLUME")
480+
case workspace.CreatorTypeBind:
481+
logOperationSuccess("DETERMINING_WORKSPACE_TYPE", "BIND")
482+
}
483+
484+
logOperationStart("RESOLVING_REPOSITORIES", "")
485+
repos, err := svc.ResolveRepositories(ctx, batchSpec)
486+
if err != nil {
487+
if repoSet, ok := err.(batches.UnsupportedRepoSet); ok {
488+
logOperationSuccess("RESOLVING_REPOSITORIES", fmt.Sprintf("%d unsupported repositories", len(repoSet)))
489+
} else if repoSet, ok := err.(batches.IgnoredRepoSet); ok {
490+
logOperationSuccess("RESOLVING_REPOSITORIES", fmt.Sprintf("%d ignored repositories", len(repoSet)))
491+
} else {
492+
return errors.Wrap(err, "resolving repositories")
493+
}
494+
} else {
495+
logOperationSuccess("RESOLVING_REPOSITORIES", fmt.Sprintf("Resolved %d repositories", len(repos)))
496+
}
497+
498+
logOperationStart("DETERMINING_WORKSPACES", "")
499+
tasks, err := svc.BuildTasks(ctx, repos, batchSpec)
500+
if err != nil {
501+
return err
502+
}
503+
logOperationSuccess("DETERMINING_WORKSPACES", fmt.Sprintf("Found %d workspaces with steps to execute", len(tasks)))
504+
505+
// EXECUTION OF TASKS
506+
coord := svc.NewCoordinator(executor.NewCoordinatorOpts{
507+
Creator: workspaceCreator,
508+
CacheDir: opts.flags.cacheDir,
509+
ClearCache: opts.flags.clearCache,
510+
SkipErrors: opts.flags.skipErrors,
511+
CleanArchives: opts.flags.cleanArchives,
512+
Parallelism: opts.flags.parallelism,
513+
Timeout: opts.flags.timeout,
514+
KeepLogs: opts.flags.keepLogs,
515+
TempDir: opts.flags.tempDir,
516+
})
517+
518+
logOperationStart("CHECKING_CACHE", "")
519+
uncachedTasks, cachedSpecs, err := coord.CheckCache(ctx, tasks)
520+
if err != nil {
521+
return err
522+
}
523+
var specsFoundMessage string
524+
if len(cachedSpecs) == 1 {
525+
specsFoundMessage = "Found 1 cached changeset spec"
526+
} else {
527+
specsFoundMessage = fmt.Sprintf("Found %d cached changeset specs", len(cachedSpecs))
528+
}
529+
switch len(uncachedTasks) {
530+
case 0:
531+
logOperationSuccess("CHECKING_CACHE", fmt.Sprintf("%s; no tasks need to be executed", specsFoundMessage))
532+
case 1:
533+
logOperationSuccess("CHECKING_CACHE", fmt.Sprintf("%s; %d task needs to be executed", specsFoundMessage, len(uncachedTasks)))
534+
default:
535+
logOperationSuccess("CHECKING_CACHE", fmt.Sprintf("%s; %d tasks need to be executed", specsFoundMessage, len(uncachedTasks)))
536+
}
537+
538+
logOperationStart("EXECUTING_TASKS", "")
539+
freshSpecs, logFiles, err := coord.Execute(ctx, uncachedTasks, batchSpec, func(statuses []*executor.TaskStatus) {
540+
541+
finishedExecution := 0
542+
finishedBuilding := 0
543+
currentlyRunning := 0
544+
errored := 0
545+
546+
for _, ts := range statuses {
547+
if ts.FinishedExecution() {
548+
if ts.Err != nil {
549+
errored += 1
550+
}
551+
552+
finishedExecution += 1
553+
}
554+
555+
if ts.FinishedBuildingSpecs() {
556+
finishedBuilding += 1
557+
}
558+
559+
if ts.IsRunning() {
560+
currentlyRunning += 1
561+
}
562+
}
563+
564+
logOperationProgress("EXECUTING_TASKS", fmt.Sprintf("running: %d, executed: %d, built: %d, errored: %d", currentlyRunning, finishedExecution, finishedBuilding, errored))
565+
})
566+
567+
if err != nil && !opts.flags.skipErrors {
568+
return err
569+
}
570+
if err != nil && opts.flags.skipErrors {
571+
logOperationFailure("EXECUTING_TASKS", fmt.Sprintf("Error: %s. Skipping errors because -skip-errors was used.", err))
572+
} else {
573+
logOperationSuccess("EXECUTING_TASKS", "")
574+
}
575+
576+
if len(logFiles) > 0 && opts.flags.keepLogs {
577+
for _, file := range logFiles {
578+
logOperationSuccess("LOG_FILE_KEPT", file)
579+
}
580+
}
581+
582+
specs := append(cachedSpecs, freshSpecs...)
583+
584+
err = svc.ValidateChangesetSpecs(repos, specs)
585+
if err != nil {
586+
return err
587+
}
588+
589+
ids := make([]graphql.ChangesetSpecID, len(specs))
590+
591+
if len(specs) > 0 {
592+
var label string
593+
if len(specs) == 1 {
594+
label = "Sending changeset spec"
595+
} else {
596+
label = fmt.Sprintf("Sending %d changeset specs", len(specs))
597+
}
598+
599+
logOperationStart("UPLOADING_CHANGESET_SPECS", label)
600+
601+
for i, spec := range specs {
602+
id, err := svc.CreateChangesetSpec(ctx, spec)
603+
if err != nil {
604+
return err
605+
}
606+
ids[i] = id
607+
logOperationProgress("UPLOADING_CHANGESET_SPECS", fmt.Sprintf("Uploaded %d out of %d", i+1, len(specs)))
608+
609+
}
610+
logOperationSuccess("UPLOADING_CHANGESET_SPECS", "")
611+
} else {
612+
if len(repos) == 0 {
613+
fmt.Println("No changeset specs created")
614+
}
615+
}
616+
617+
logOperationStart("CREATING_BATCH_SPEC", "")
618+
id, url, err := svc.CreateBatchSpec(ctx, namespace, rawSpec, ids)
619+
if err != nil {
620+
return prettyPrintBatchUnlicensedError(opts.out, err)
621+
} else {
622+
logOperationSuccess("CREATING_BATCH_SPEC", fmt.Sprintf("%s%s", cfg.Endpoint, url))
623+
}
624+
625+
if opts.applyBatchSpec {
626+
logOperationStart("APPLYING_BATCH_SPEC", "")
627+
batch, err := svc.ApplyBatchChange(ctx, id)
628+
if err != nil {
629+
return err
630+
}
631+
632+
logOperationSuccess("APPLYING_BATCH_SPEC", fmt.Sprintf("%s%s", cfg.Endpoint, batch.URL))
633+
}
634+
635+
return nil
636+
}
637+
418638
// batchParseSpec parses and validates the given batch spec. If the spec has
419639
// validation errors, the errors are output in a human readable form and an
420640
// exitCodeError is returned.

‎cmd/src/batch_preview.go‎

Lines changed: 27 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -36,24 +36,36 @@ Examples:
3636
return &usageError{errors.New("additional arguments not allowed")}
3737
}
3838

39-
out := output.NewOutput(flagSet.Output(), output.OutputOpts{Verbose: *verbose})
40-
4139
ctx, cancel := contextCancelOnInterrupt(context.Background())
4240
defer cancel()
4341

44-
err := executeBatchSpec(ctx, executeBatchSpecOpts{
45-
flags: flags,
46-
out: out,
47-
client: cfg.apiClient(flags.api, flagSet.Output()),
48-
49-
// Do not apply the uploaded batch spec
50-
applyBatchSpec: false,
51-
})
52-
53-
if err != nil {
54-
printExecutionError(out, err)
55-
out.Write("")
56-
return &exitCodeError{nil, 1}
42+
var err error
43+
if flags.textOnly {
44+
err = textOnlyExecuteBatchSpec(ctx, executeBatchSpecOpts{
45+
flags: flags,
46+
client: cfg.apiClient(flags.api, flagSet.Output()),
47+
48+
// Do not apply the uploaded batch spec
49+
applyBatchSpec: false,
50+
})
51+
if err != nil {
52+
fmt.Printf("ERROR: %s\n", err)
53+
return &exitCodeError{nil, 1}
54+
}
55+
} else {
56+
out := output.NewOutput(flagSet.Output(), output.OutputOpts{Verbose: *verbose})
57+
err = executeBatchSpec(ctx, executeBatchSpecOpts{
58+
flags: flags,
59+
out: out,
60+
client: cfg.apiClient(flags.api, flagSet.Output()),
61+
// Do not apply the uploaded batch spec
62+
applyBatchSpec: false,
63+
})
64+
if err != nil {
65+
printExecutionError(out, err)
66+
out.Write("")
67+
return &exitCodeError{nil, 1}
68+
}
5769
}
5870

5971
return nil

‎cmd/src/batch_text_logging.go‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
package main
2+
3+
import (
4+
"encoding/json"
5+
"os"
6+
"time"
7+
)
8+
9+
type batchesLogEvent struct {
10+
Operation string `json:"operation"` // "PREPARING_DOCKER_IMAGES"
11+
12+
Timestamp time.Time `json:"timestamp"`
13+
14+
Status string `json:"status"` // "STARTED", "PROGRESS", "SUCCESS", "FAILURE"
15+
Message string `json:"message,omitempty"` // "70% done"
16+
}
17+
18+
func logOperationStart(op, msg string) {
19+
logEvent(batchesLogEvent{Operation: op, Status: "STARTED", Message: msg})
20+
}
21+
22+
func logOperationSuccess(op, msg string) {
23+
logEvent(batchesLogEvent{Operation: op, Status: "SUCCESS", Message: msg})
24+
}
25+
26+
func logOperationFailure(op, msg string) {
27+
logEvent(batchesLogEvent{Operation: op, Status: "FAILURE", Message: msg})
28+
}
29+
30+
func logOperationProgress(op, msg string) {
31+
logEvent(batchesLogEvent{Operation: op, Status: "PROGRESS", Message: msg})
32+
}
33+
34+
func logEvent(e batchesLogEvent) {
35+
e.Timestamp = time.Now().UTC().Truncate(time.Millisecond)
36+
json.NewEncoder(os.Stdout).Encode(e)
37+
}

0 commit comments

Comments
 (0)