Skip to content
This repository was archived by the owner on Sep 30, 2024. It is now read-only.

Commit 8a3cc3c

Browse files
ggilmoreggilmore
authored
gitserver: make vcs syncer implementation request a fresh clone URL in between each remote git operation (#61312)
wip: gitserver: make vcs syncer implementation request a fresh clone URL in between each remote git operation commit-id:4575598b (cherry picked from commit cbbb9c3) Conflicts: cmd/gitserver/internal/integration_tests/clone_test.go cmd/gitserver/internal/integration_tests/resolverevisions_test.go cmd/gitserver/internal/integration_tests/test_utils.go cmd/gitserver/internal/server.go cmd/gitserver/internal/server_test.go cmd/gitserver/internal/vcssyncer/git.go
1 parent 642df1f commit 8a3cc3c

26 files changed

+824
-492
lines changed

‎CHANGELOG.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ All notable changes to Sourcegraph are documented in this file.
2626

2727
### Fixed
2828

29-
-
29+
- Fixed a bug in gitserver where it was possible to use expired Github App authorization tokens when syncing a large repository. Now, gitserver will use the latest tokens for each step of the syncing process (and refresh them if necessary). [#61179](https://github.com/sourcegraph/sourcegraph/pull/61179/)
3030

3131
### Removed
3232

‎cmd/gitserver/internal/executil/executil.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -347,6 +347,11 @@ func RunCommandWriteOutput(ctx context.Context, cmd wrexec.Cmder, writer io.Writ
347347
}
348348

349349
err = cmd.Wait()
350+
var exitErr *exec.ExitError
351+
if errors.As(err, &exitErr) {
352+
// Redact the stderr output if we have it.
353+
exitErr.Stderr = []byte(redactor(string(exitErr.Stderr)))
354+
}
350355

351356
if ps := cmd.Unwrap().ProcessState; ps != nil && ps.Sys() != nil {
352357
if ws, ok := ps.Sys().(syscall.WaitStatus); ok {

‎cmd/gitserver/internal/integration_tests/BUILD.bazel‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,9 @@ go_library(
2525
"//internal/observation",
2626
"//internal/ratelimit",
2727
"//internal/types",
28+
"//internal/vcs",
2829
"//internal/wrexec",
30+
"//lib/errors",
2931
"@com_github_sourcegraph_log//:log",
3032
"@com_github_sourcegraph_log//logtest",
3133
"@org_golang_x_sync//semaphore",

‎cmd/gitserver/internal/integration_tests/archivereader_test.go‎

Lines changed: 29 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import (
66
"context"
77
"encoding/base64"
88
"fmt"
9+
"github.com/sourcegraph/sourcegraph/internal/vcs"
910
"io"
1011
"net/http/httptest"
1112
"net/url"
@@ -96,21 +97,42 @@ func TestClient_ArchiveReader(t *testing.T) {
9697
runArchiveReaderTestfunc := func(t *testing.T, mkClient func(t *testing.T, addrs []string) gitserver.Client, name api.RepoName, test test) {
9798
t.Run(string(name), func(t *testing.T) {
9899
// Setup: Prepare the test Gitserver server + register the gRPC server
100+
101+
getRemoteURLFunc := func(_ context.Context, name api.RepoName) (string, error) {
102+
if test.remote != "" {
103+
return test.remote, nil
104+
}
105+
return "", errors.Errorf("no remote for %s", test.name)
106+
}
107+
99108
s := &server.Server{
100109
Logger: logtest.Scoped(t),
101110
ReposDir: filepath.Join(root, "repos"),
102111
DB: newMockDB(),
103112
GetBackendFunc: func(dir common.GitDir, repoName api.RepoName) git.GitBackend {
104113
return gitcli.NewBackend(logtest.Scoped(t), wrexec.NewNoOpRecordingCommandFactory(), dir, repoName)
105114
},
106-
GetRemoteURLFunc: func(_ context.Context, name api.RepoName) (string, error) {
107-
if test.remote != "" {
108-
return test.remote, nil
109-
}
110-
return "", errors.Errorf("no remote for %s", test.name)
111-
},
115+
GetRemoteURLFunc: getRemoteURLFunc,
112116
GetVCSSyncer: func(ctx context.Context, name api.RepoName) (vcssyncer.VCSSyncer, error) {
113-
return vcssyncer.NewGitRepoSyncer(logtest.Scoped(t), wrexec.NewNoOpRecordingCommandFactory()), nil
117+
getRemoteURLSource := func(ctx context.Context, name api.RepoName) (vcssyncer.RemoteURLSource, error) {
118+
source := vcssyncer.RemoteURLSourceFunc(func(ctx context.Context) (*vcs.URL, error) {
119+
raw, err := getRemoteURLFunc(ctx, name)
120+
if err != nil {
121+
return nil, errors.Wrapf(err, "failed to get remote URL for %s", name)
122+
}
123+
124+
u, err := vcs.ParseURL(raw)
125+
if err != nil {
126+
return nil, errors.Wrapf(err, "failed to parse remote URL for repository: %s, raw: %s", name, raw)
127+
}
128+
129+
return u, nil
130+
})
131+
132+
return source, nil
133+
}
134+
135+
return vcssyncer.NewGitRepoSyncer(logtest.Scoped(t), wrexec.NewNoOpRecordingCommandFactory(), getRemoteURLSource), nil
114136
},
115137
RecordingCommandFactory: wrexec.NewNoOpRecordingCommandFactory(),
116138
Locker: server.NewRepositoryLocker(),

‎cmd/gitserver/internal/integration_tests/clone_test.go‎

Lines changed: 49 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package inttests
33
import (
44
"container/list"
55
"context"
6+
"github.com/sourcegraph/sourcegraph/lib/errors"
67
"net/http/httptest"
78
"net/url"
89
"os"
@@ -51,19 +52,37 @@ func TestClone(t *testing.T) {
5152
lock := NewMockRepositoryLock()
5253
locker.TryAcquireFunc.SetDefaultReturn(lock, true)
5354

55+
getRemoteURLFunc := func(_ context.Context, name api.RepoName) (string, error) { //nolint:unparam
56+
require.Equal(t, repo, name)
57+
return remote, nil
58+
}
59+
5460
s := server.Server{
5561
Logger: logger,
5662
ReposDir: reposDir,
5763
GetBackendFunc: func(dir common.GitDir, repoName api.RepoName) git.GitBackend {
5864
return gitcli.NewBackend(logtest.Scoped(t), wrexec.NewNoOpRecordingCommandFactory(), dir, repoName)
5965
},
60-
GetRemoteURLFunc: func(_ context.Context, name api.RepoName) (string, error) {
61-
require.Equal(t, repo, name)
62-
return remote, nil
63-
},
66+
GetRemoteURLFunc: getRemoteURLFunc,
6467
GetVCSSyncer: func(ctx context.Context, name api.RepoName) (vcssyncer.VCSSyncer, error) {
68+
getRemoteURLSource := func(ctx context.Context, name api.RepoName) (vcssyncer.RemoteURLSource, error) {
69+
require.Equal(t, repo, name)
70+
return vcssyncer.RemoteURLSourceFunc(func(ctx context.Context) (*vcs.URL, error) {
71+
raw, err := getRemoteURLFunc(ctx, name)
72+
if err != nil {
73+
return nil, errors.Wrapf(err, "failed to get remote URL for %s", name)
74+
}
75+
76+
u, err := vcs.ParseURL(raw)
77+
if err != nil {
78+
return nil, errors.Wrapf(err, "failed to parse remote URL %q", raw)
79+
}
80+
return u, nil
81+
}), nil
82+
}
83+
6584
require.Equal(t, repo, name)
66-
return vcssyncer.NewGitRepoSyncer(logtest.Scoped(t), wrexec.NewNoOpRecordingCommandFactory()), nil
85+
return vcssyncer.NewGitRepoSyncer(logtest.Scoped(t), wrexec.NewNoOpRecordingCommandFactory(), getRemoteURLSource), nil
6786
},
6887
DB: db,
6988
Perforce: perforce.NewService(ctx, observation.TestContextTB(t), logger, db, list.New()),
@@ -93,10 +112,10 @@ func TestClone(t *testing.T) {
93112

94113
// Should have acquired a lock.
95114
mockassert.CalledOnce(t, locker.TryAcquireFunc)
96-
// Should have reported status. 21 lines is the output git currently produces.
115+
// Should have reported status. 24 lines is the output git currently produces.
97116
// This number might need to be adjusted over time, but before doing so please
98117
// check that the calls actually use the args you would expect them to use.
99-
mockassert.CalledN(t, lock.SetStatusFunc, 21)
118+
mockassert.CalledN(t, lock.SetStatusFunc, 24)
100119
// Should have released the lock.
101120
mockassert.CalledOnce(t, lock.ReleaseFunc)
102121

@@ -147,19 +166,36 @@ func TestClone_Fail(t *testing.T) {
147166
lock := NewMockRepositoryLock()
148167
locker.TryAcquireFunc.SetDefaultReturn(lock, true)
149168

169+
getRemoteURLFunc := func(_ context.Context, name api.RepoName) (string, error) { //nolint:unparam
170+
require.Equal(t, repo, name)
171+
return remote, nil
172+
}
173+
150174
s := server.Server{
151175
Logger: logger,
152176
ReposDir: reposDir,
153177
GetBackendFunc: func(dir common.GitDir, repoName api.RepoName) git.GitBackend {
154178
return gitcli.NewBackend(logtest.Scoped(t), wrexec.NewNoOpRecordingCommandFactory(), dir, repoName)
155179
},
156-
GetRemoteURLFunc: func(_ context.Context, name api.RepoName) (string, error) {
157-
require.Equal(t, repo, name)
158-
return remote, nil
159-
},
180+
GetRemoteURLFunc: getRemoteURLFunc,
160181
GetVCSSyncer: func(ctx context.Context, name api.RepoName) (vcssyncer.VCSSyncer, error) {
161182
require.Equal(t, repo, name)
162-
return vcssyncer.NewGitRepoSyncer(logtest.Scoped(t), wrexec.NewNoOpRecordingCommandFactory()), nil
183+
getRemoteURLSource := func(ctx context.Context, name api.RepoName) (vcssyncer.RemoteURLSource, error) {
184+
require.Equal(t, repo, name)
185+
return vcssyncer.RemoteURLSourceFunc(func(ctx context.Context) (*vcs.URL, error) {
186+
raw, err := getRemoteURLFunc(ctx, name)
187+
if err != nil {
188+
return nil, errors.Wrapf(err, "failed to get remote URL for %s", name)
189+
}
190+
191+
u, err := vcs.ParseURL(raw)
192+
if err != nil {
193+
return nil, errors.Wrapf(err, "failed to parse remote URL %q", raw)
194+
}
195+
return u, nil
196+
}), nil
197+
}
198+
return vcssyncer.NewGitRepoSyncer(logtest.Scoped(t), wrexec.NewNoOpRecordingCommandFactory(), getRemoteURLSource), nil
163199
},
164200
DB: db,
165201
Perforce: perforce.NewService(ctx, observation.TestContextTB(t), logger, db, list.New()),
@@ -210,7 +246,7 @@ func TestClone_Fail(t *testing.T) {
210246

211247
// Now, fake that the IsCloneable check passes, then Clone will be called
212248
// and is expected to fail.
213-
vcssyncer.TestGitRepoExists = func(ctx context.Context, remoteURL *vcs.URL) error {
249+
vcssyncer.TestGitRepoExists = func(ctx context.Context, name api.RepoName) error {
214250
return nil
215251
}
216252
t.Cleanup(func() {

‎cmd/gitserver/internal/integration_tests/resolverevisions_test.go‎

Lines changed: 22 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@ package inttests
33
import (
44
"container/list"
55
"context"
6+
"github.com/sourcegraph/sourcegraph/internal/vcs"
7+
"github.com/sourcegraph/sourcegraph/lib/errors"
68
"net/http/httptest"
79
"net/url"
810
"path/filepath"
@@ -69,17 +71,33 @@ func TestClient_ResolveRevisions(t *testing.T) {
6971
db := newMockDB()
7072
ctx := context.Background()
7173

74+
getRemoteURLFunc := func(_ context.Context, name api.RepoName) (string, error) { //nolint:unparam
75+
return remote, nil
76+
}
77+
7278
s := server.Server{
7379
Logger: logger,
7480
ReposDir: filepath.Join(root, "repos"),
7581
GetBackendFunc: func(dir common.GitDir, repoName api.RepoName) git.GitBackend {
7682
return gitcli.NewBackend(logtest.Scoped(t), wrexec.NewNoOpRecordingCommandFactory(), dir, repoName)
7783
},
78-
GetRemoteURLFunc: func(_ context.Context, name api.RepoName) (string, error) {
79-
return remote, nil
80-
},
84+
GetRemoteURLFunc: getRemoteURLFunc,
8185
GetVCSSyncer: func(ctx context.Context, name api.RepoName) (vcssyncer.VCSSyncer, error) {
82-
return vcssyncer.NewGitRepoSyncer(logtest.Scoped(t), wrexec.NewNoOpRecordingCommandFactory()), nil
86+
getRemoteURLSource := func(ctx context.Context, name api.RepoName) (vcssyncer.RemoteURLSource, error) {
87+
return vcssyncer.RemoteURLSourceFunc(func(ctx context.Context) (*vcs.URL, error) {
88+
raw, err := getRemoteURLFunc(ctx, name)
89+
if err != nil {
90+
return nil, errors.Wrapf(err, "failed to get remote URL for %s", name)
91+
}
92+
93+
u, err := vcs.ParseURL(raw)
94+
if err != nil {
95+
return nil, errors.Wrapf(err, "failed to parse remote URL %q", raw)
96+
}
97+
return u, nil
98+
}), nil
99+
}
100+
return vcssyncer.NewGitRepoSyncer(logtest.Scoped(t), wrexec.NewNoOpRecordingCommandFactory(), getRemoteURLSource), nil
83101
},
84102
DB: db,
85103
Perforce: perforce.NewService(ctx, observation.TestContextTB(t), logger, db, list.New()),

‎cmd/gitserver/internal/integration_tests/test_utils.go‎

Lines changed: 24 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@ package inttests
22

33
import (
44
"context"
5+
"github.com/sourcegraph/sourcegraph/internal/vcs"
6+
"github.com/sourcegraph/sourcegraph/lib/errors"
57
"net"
68
"net/http"
79
"os"
@@ -76,18 +78,36 @@ func InitGitserver() {
7678
})
7779
db.ReposFunc.SetDefaultReturn(r)
7880

81+
getRemoteURLFunc := func(_ context.Context, name api.RepoName) (string, error) { //nolint:unparam // context is unused but required by the interface, error is not used in this test
82+
return filepath.Join(root, "remotes", string(name)), nil
83+
}
84+
7985
s := server.Server{
8086
Logger: sglog.Scoped("server"),
8187
ObservationCtx: &observation.TestContext,
8288
ReposDir: filepath.Join(root, "repos"),
8389
GetBackendFunc: func(dir common.GitDir, repoName api.RepoName) git.GitBackend {
8490
return gitcli.NewBackend(logtest.Scoped(&t), wrexec.NewNoOpRecordingCommandFactory(), dir, repoName)
8591
},
86-
GetRemoteURLFunc: func(ctx context.Context, name api.RepoName) (string, error) {
87-
return filepath.Join(root, "remotes", string(name)), nil
88-
},
92+
GetRemoteURLFunc: getRemoteURLFunc,
8993
GetVCSSyncer: func(ctx context.Context, name api.RepoName) (vcssyncer.VCSSyncer, error) {
90-
return vcssyncer.NewGitRepoSyncer(logger, wrexec.NewNoOpRecordingCommandFactory()), nil
94+
getRemoteURLSource := func(ctx context.Context, name api.RepoName) (vcssyncer.RemoteURLSource, error) {
95+
return vcssyncer.RemoteURLSourceFunc(func(ctx context.Context) (*vcs.URL, error) {
96+
raw, err := getRemoteURLFunc(ctx, name)
97+
if err != nil {
98+
return nil, errors.Wrapf(err, "failed to get remote URL for %s", name)
99+
}
100+
101+
u, err := vcs.ParseURL(raw)
102+
if err != nil {
103+
return nil, errors.Wrapf(err, "failed to parse remote URL %q", raw)
104+
}
105+
106+
return u, nil
107+
}), nil
108+
}
109+
110+
return vcssyncer.NewGitRepoSyncer(logger, wrexec.NewNoOpRecordingCommandFactory(), getRemoteURLSource), nil
91111
},
92112
GlobalBatchLogSemaphore: semaphore.NewWeighted(32),
93113
DB: db,

0 commit comments

Comments
 (0)