Skip to content

Commit 07012a5

Browse files
authored
async upload cassettes during pr test (#15874)
1 parent 857fd21 commit 07012a5

File tree

9 files changed

+131
-21
lines changed

9 files changed

+131
-21
lines changed

.ci/gcb-pr-downstream-generation-and-test.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -265,6 +265,7 @@ steps:
265265
- $BUILD_ID
266266
- $PROJECT_ID
267267
- "23" # Build step
268+
- "true"
268269

269270
- name: 'gcr.io/graphite-docker-images/go-plus'
270271
entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'

.ci/magician/cmd/check_cassettes.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,7 @@ var checkCassettesCmd = &cobra.Command{
7777

7878
ctlr := source.NewController(env["GOPATH"], "modular-magician", githubToken, rnr)
7979

80-
vt, err := vcr.NewTester(env, "ci-vcr-cassettes", "vcr-check-cassettes", rnr)
80+
vt, err := vcr.NewTester(env, "ci-vcr-cassettes", "vcr-check-cassettes", rnr, false)
8181
if err != nil {
8282
return fmt.Errorf("error creating VCR tester: %w", err)
8383
}

.ci/magician/cmd/test_eap_vcr.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -120,7 +120,7 @@ The following environment variables are required:
120120
if err != nil {
121121
return err
122122
}
123-
vt, err := vcr.NewTester(env, "ci-vcr-cassettes", "ci-vcr-logs", rnr)
123+
vt, err := vcr.NewTester(env, "ci-vcr-cassettes", "ci-vcr-logs", rnr, false)
124124
if err != nil {
125125
return err
126126
}

.ci/magician/cmd/test_terraform_vcr.go

Lines changed: 16 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,8 @@ It expects the following arguments:
9393
3. Build ID
9494
4. Project ID where Cloud Builds are located
9595
5. Build step number
96-
96+
6. Enable async upload cassettes
97+
9798
The following environment variables are required:
9899
` + listTTVRequiredEnvironmentVariables(),
99100
RunE: func(cmd *cobra.Command, args []string) error {
@@ -134,13 +135,17 @@ The following environment variables are required:
134135
}
135136
ctlr := source.NewController(env["GOPATH"], "modular-magician", env["GITHUB_TOKEN_DOWNSTREAMS"], rnr)
136137

137-
vt, err := vcr.NewTester(env, "ci-vcr-cassettes", "ci-vcr-logs", rnr)
138-
if err != nil {
139-
return fmt.Errorf("error creating VCR tester: %w", err)
138+
if len(args) < 5 {
139+
return fmt.Errorf("wrong number of arguments %d, expected >=5", len(args))
140+
}
141+
enableAsyncUploadCassettes := false
142+
if len(args) > 5 {
143+
enableAsyncUploadCassettes = strings.ToLower(args[5]) == "true"
140144
}
141145

142-
if len(args) != 5 {
143-
return fmt.Errorf("wrong number of arguments %d, expected 5", len(args))
146+
vt, err := vcr.NewTester(env, "ci-vcr-cassettes", "ci-vcr-logs", rnr, enableAsyncUploadCassettes)
147+
if err != nil {
148+
return fmt.Errorf("error creating VCR tester: %w", err)
144149
}
145150

146151
return execTestTerraformVCR(args[0], args[1], args[2], args[3], args[4], baseBranch, gh, rnr, ctlr, vt)
@@ -260,10 +265,11 @@ func execTestTerraformVCR(prNumber, mmCommitSha, buildID, projectID, buildStep,
260265
}
261266
if len(replayingResult.FailedTests) > 0 {
262267
recordingResult, recordingErr := vt.RunParallel(vcr.RunOptions{
263-
Mode: vcr.Recording,
264-
Version: provider.Beta,
265-
TestDirs: testDirs,
266-
Tests: replayingResult.FailedTests,
268+
Mode: vcr.Recording,
269+
Version: provider.Beta,
270+
TestDirs: testDirs,
271+
Tests: replayingResult.FailedTests,
272+
UploadBranchName: newBranch,
267273
})
268274
if recordingErr != nil {
269275
testState = "failure"

.ci/magician/cmd/vcr_cassette_update.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,7 @@ var vcrCassetteUpdateCmd = &cobra.Command{
103103
}
104104
ctlr := source.NewController(env["GOPATH"], "hashicorp", env["GITHUB_TOKEN_CLASSIC"], rnr)
105105

106-
vt, err := vcr.NewTester(env, "ci-vcr-cassettes", "", rnr)
106+
vt, err := vcr.NewTester(env, "ci-vcr-cassettes", "", rnr, false)
107107
if err != nil {
108108
return fmt.Errorf("error creating VCR tester: %w", err)
109109
}

.ci/magician/cmd/vcr_cassette_update_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -407,7 +407,7 @@ func TestExecVCRCassetteUpdate(t *testing.T) {
407407
ctlr := source.NewController("gopath", "hashicorp", "token", rnr)
408408
vt, err := vcr.NewTester(map[string]string{
409409
"SA_KEY": "sa_key",
410-
}, "ci-vcr-cassettes", "", rnr)
410+
}, "ci-vcr-cassettes", "", rnr, false)
411411
if err != nil {
412412
t.Fatalf("Failed to create new tester: %v", err)
413413
}

.ci/magician/go.mod

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ require (
1515

1616
require (
1717
cloud.google.com/go/storage v1.50.0
18+
github.com/fsnotify/fsnotify v1.9.0
1819
github.com/google/go-cmp v0.6.0
1920
github.com/google/go-github/v68 v68.0.0
2021
github.com/otiai10/copy v1.12.0

.ci/magician/go.sum

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,8 @@ github.com/envoyproxy/protoc-gen-validate v1.1.0 h1:tntQDh69XqOCOZsDz0lVJQez/2L6
5757
github.com/envoyproxy/protoc-gen-validate v1.1.0/go.mod h1:sXRDRVmzEbkM7CVcM06s9shE/m23dg3wzjl0UWqJ2q4=
5858
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
5959
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
60+
github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
61+
github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
6062
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
6163
github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
6264
github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=

.ci/magician/vcr/tester.go

Lines changed: 107 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,16 @@ import (
44
"fmt"
55
"io/fs"
66
"magician/provider"
7+
"math"
78
"path/filepath"
89
"regexp"
910
"sort"
1011
"strconv"
1112
"strings"
1213
"sync"
14+
"time"
15+
16+
"github.com/fsnotify/fsnotify"
1317
)
1418

1519
type Result struct {
@@ -60,6 +64,11 @@ type Tester struct {
6064
cassettePaths map[provider.Version]string // where cassettes are relative to baseDir by version
6165
logPaths map[logKey]string // where logs are relative to baseDir by version and mode
6266
repoPaths map[provider.Version]string // relative paths of already cloned repos by version
67+
68+
// the following are for async upload cassettes
69+
enableAsyncUploadCassettes bool
70+
watcher *fsnotify.Watcher
71+
uploadFunc func(head string, version provider.Version, fileName string) error
6372
}
6473

6574
const accTestParallelism = 32
@@ -116,15 +125,15 @@ var safeToLog = map[string]bool{
116125
} // true if shown, false if hidden (default false)
117126

118127
// Create a new tester in the current working directory and write the service account key file.
119-
func NewTester(env map[string]string, cassetteBucket, logBucket string, rnr ExecRunner) (*Tester, error) {
128+
func NewTester(env map[string]string, cassetteBucket, logBucket string, rnr ExecRunner, enableAsyncUpload bool) (*Tester, error) {
120129
var saKeyPath string
121130
if saKeyVal, ok := env["SA_KEY"]; ok {
122131
saKeyPath = "sa_key.json"
123132
if err := rnr.WriteFile(saKeyPath, saKeyVal); err != nil {
124133
return nil, err
125134
}
126135
}
127-
return &Tester{
136+
vt := &Tester{
128137
env: env,
129138
rnr: rnr,
130139
cassetteBucket: cassetteBucket,
@@ -134,7 +143,13 @@ func NewTester(env map[string]string, cassetteBucket, logBucket string, rnr Exec
134143
cassettePaths: make(map[provider.Version]string, provider.NumVersions),
135144
logPaths: make(map[logKey]string, provider.NumVersions*numModes),
136145
repoPaths: make(map[provider.Version]string, provider.NumVersions),
137-
}, nil
146+
}
147+
148+
if enableAsyncUpload {
149+
vt.enableAsyncUploadCassettes = true
150+
vt.uploadFunc = vt.uploadOneCassetteFile
151+
}
152+
return vt, nil
138153
}
139154

140155
func (vt *Tester) SetRepoPath(version provider.Version, repoPath string) {
@@ -194,10 +209,11 @@ func (vt *Tester) LogPath(mode Mode, version provider.Version) string {
194209
}
195210

196211
type RunOptions struct {
197-
Mode Mode
198-
Version provider.Version
199-
TestDirs []string
200-
Tests []string
212+
Mode Mode
213+
Version provider.Version
214+
TestDirs []string
215+
Tests []string
216+
UploadBranchName string
201217
}
202218

203219
// Run the vcr tests in the given mode and provider version and return the result.
@@ -348,6 +364,19 @@ func (vt *Tester) RunParallel(opt RunOptions) (Result, error) {
348364
return Result{}, fmt.Errorf("error creating cassette dir: %v", err)
349365
}
350366
vt.cassettePaths[opt.Version] = cassettePath
367+
368+
if vt.enableAsyncUploadCassettes {
369+
w, err := fsnotify.NewWatcher()
370+
if err != nil {
371+
return Result{}, fmt.Errorf("failed to create watcher")
372+
}
373+
defer w.Close()
374+
if err := w.Add(cassettePath); err != nil {
375+
return Result{}, fmt.Errorf("failed to add cassette path into watcher")
376+
}
377+
vt.watcher = w
378+
go vt.asyncUploadCassettes(opt.Version, opt.UploadBranchName, w)
379+
}
351380
}
352381

353382
running := make(chan struct{}, parallelJobs)
@@ -534,6 +563,77 @@ func (vt *Tester) UploadLogs(opts UploadLogsOptions) error {
534563
return nil
535564
}
536565

566+
func (vt *Tester) asyncUploadCassettes(version provider.Version, branch string, w *fsnotify.Watcher) error {
567+
var (
568+
waitFor = 100 * time.Millisecond
569+
mu sync.Mutex
570+
timers = make(map[string]*time.Timer)
571+
572+
// Callback we run.
573+
cb = func(e fsnotify.Event) {
574+
err := vt.uploadFunc(branch, version, e.Name)
575+
if err != nil {
576+
fmt.Println("upload failed: ", err)
577+
}
578+
mu.Lock()
579+
delete(timers, e.Name)
580+
mu.Unlock()
581+
}
582+
)
583+
584+
for {
585+
select {
586+
case err, ok := <-w.Errors:
587+
if !ok { // Channel was closed (i.e. Watcher.Close() was called).
588+
return nil
589+
}
590+
fmt.Println(err)
591+
case e, ok := <-w.Events:
592+
if !ok { // Channel was closed (i.e. Watcher.Close() was called).
593+
return nil
594+
}
595+
// ignore everything outside of Create and Write.
596+
if !e.Has(fsnotify.Create) && !e.Has(fsnotify.Write) {
597+
continue
598+
}
599+
600+
// Get timer.
601+
mu.Lock()
602+
t, ok := timers[e.Name]
603+
mu.Unlock()
604+
605+
// No timer yet, so create one.
606+
if !ok {
607+
t = time.AfterFunc(math.MaxInt64, func() { cb(e) })
608+
t.Stop()
609+
610+
mu.Lock()
611+
timers[e.Name] = t
612+
mu.Unlock()
613+
}
614+
615+
// Reset the timer for this path, so it will start from 100ms again.
616+
t.Reset(waitFor)
617+
}
618+
}
619+
}
620+
621+
func (vt *Tester) uploadOneCassetteFile(head string, version provider.Version, fileName string) error {
622+
uploadPath := fmt.Sprintf("gs://%s/%s/refs/heads/%s/fixtures/", vt.cassetteBucket, version, head)
623+
args := []string{
624+
"-m",
625+
"-q",
626+
"cp",
627+
fileName,
628+
uploadPath,
629+
}
630+
fmt.Printf("Uploading %s to %s: %v\n", fileName, uploadPath, "gsutil "+strings.Join(args, " "))
631+
if _, err := vt.rnr.Run("gsutil", args, nil); err != nil {
632+
return fmt.Errorf("error uploading file %s: %s", fileName, err)
633+
}
634+
return nil
635+
}
636+
537637
func (vt *Tester) UploadCassettes(head string, version provider.Version) error {
538638
cassettePath, ok := vt.cassettePaths[version]
539639
if !ok {

0 commit comments

Comments
 (0)