Skip to content

Commit ff0cedd

Browse files
tonistiigitonistiigi
authored
Merge pull request moby#5441 from jedevc/git-clone-hidden-sha
git: allow cloning commit shas not referenced by branch/tag
2 parents 17896f6 + 90d2d8b commit ff0cedd

File tree

4 files changed

+94
-17
lines changed

4 files changed

+94
-17
lines changed

‎source/git/source.go

Lines changed: 9 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,6 @@ import (
3636
"google.golang.org/grpc/status"
3737
)
3838

39-
var validHex = regexp.MustCompile(`^[a-f0-9]{40}$`)
4039
var defaultBranch = regexp.MustCompile(`refs/heads/(\S+)`)
4140

4241
type Opt struct {
@@ -341,7 +340,7 @@ func (gs *gitSourceHandler) CacheKey(ctx context.Context, g session.Group, index
341340
gs.locker.Lock(remote)
342341
defer gs.locker.Unlock(remote)
343342

344-
if ref := gs.src.Ref; ref != "" && isCommitSHA(ref) {
343+
if ref := gs.src.Ref; ref != "" && gitutil.IsCommitSHA(ref) {
345344
cacheKey := gs.shaToCacheKey(ref)
346345
gs.cacheKey = cacheKey
347346
return cacheKey, ref, nil, true, nil
@@ -400,7 +399,7 @@ func (gs *gitSourceHandler) CacheKey(ctx context.Context, g session.Group, index
400399
if sha == "" {
401400
return "", "", nil, false, errors.Errorf("repository does not contain ref %s, output: %q", ref, string(buf))
402401
}
403-
if !isCommitSHA(sha) {
402+
if !gitutil.IsCommitSHA(sha) {
404403
return "", "", nil, false, errors.Errorf("invalid commit sha %q", sha)
405404
}
406405

@@ -455,7 +454,7 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context, g session.Group) (out
455454
}
456455

457456
doFetch := true
458-
if isCommitSHA(ref) {
457+
if gitutil.IsCommitSHA(ref) {
459458
// skip fetch if commit already exists
460459
if _, err := git.Run(ctx, "cat-file", "-e", ref+"^{commit}"); err == nil {
461460
doFetch = false
@@ -467,7 +466,7 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context, g session.Group) (out
467466
os.RemoveAll(filepath.Join(gitDir, "shallow.lock"))
468467

469468
args := []string{"fetch"}
470-
if !isCommitSHA(ref) { // TODO: find a branch from ls-remote?
469+
if !gitutil.IsCommitSHA(ref) { // TODO: find a branch from ls-remote?
471470
args = append(args, "--depth=1", "--no-tags")
472471
} else {
473472
args = append(args, "--tags")
@@ -476,11 +475,13 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context, g session.Group) (out
476475
}
477476
}
478477
args = append(args, "origin")
479-
if !isCommitSHA(ref) {
480-
args = append(args, "--force", ref+":tags/"+ref)
478+
if gitutil.IsCommitSHA(ref) {
479+
args = append(args, ref)
480+
} else {
481481
// local refs are needed so they would be advertised on next fetches. Force is used
482482
// in case the ref is a branch and it now points to a different commit sha
483483
// TODO: is there a better way to do this?
484+
args = append(args, "--force", ref+":tags/"+ref)
484485
}
485486
if _, err := git.Run(ctx, args...); err != nil {
486487
return nil, errors.Wrapf(err, "failed to fetch remote %s", urlutil.RedactCredentials(gs.src.Remote))
@@ -549,7 +550,7 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context, g session.Group) (out
549550
pullref := ref
550551
if isAnnotatedTag {
551552
pullref += ":refs/tags/" + pullref
552-
} else if isCommitSHA(ref) {
553+
} else if gitutil.IsCommitSHA(ref) {
553554
pullref = "refs/buildkit/" + identity.NewID()
554555
_, err = git.Run(ctx, "update-ref", pullref, ref)
555556
if err != nil {
@@ -710,10 +711,6 @@ func (gs *gitSourceHandler) gitCli(ctx context.Context, g session.Group, opts ..
710711
return gitCLI(opts...), cleanup, err
711712
}
712713

713-
func isCommitSHA(str string) bool {
714-
return validHex.MatchString(str)
715-
}
716-
717714
func tokenScope(remote string) string {
718715
// generally we can only use the token for fetching main remote but in case of github.com we do best effort
719716
// to try reuse same token for all github.com remotes. This is the same behavior actions/checkout uses

‎source/git/source_test.go

Lines changed: 42 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -218,15 +218,31 @@ func testFetchBySHA(t *testing.T, keepGitDir bool) {
218218
}
219219

220220
func TestFetchUnreferencedTagSha(t *testing.T) {
221-
testFetchUnreferencedTagSha(t, false)
221+
testFetchUnreferencedRefSha(t, "v1.2.3-special", false)
222222
}
223223

224224
func TestFetchUnreferencedTagShaKeepGitDir(t *testing.T) {
225-
testFetchUnreferencedTagSha(t, true)
225+
testFetchUnreferencedRefSha(t, "v1.2.3-special", true)
226226
}
227227

228-
// testFetchUnreferencedTagSha tests fetching a SHA that points to a tag that is not reachable from any branch.
229-
func testFetchUnreferencedTagSha(t *testing.T, keepGitDir bool) {
228+
func TestFetchUnreferencedRefSha(t *testing.T) {
229+
testFetchUnreferencedRefSha(t, "refs/special", false)
230+
}
231+
232+
func TestFetchUnreferencedRefShaKeepGitDir(t *testing.T) {
233+
testFetchUnreferencedRefSha(t, "refs/special", true)
234+
}
235+
236+
func TestFetchUnadvertisedRefSha(t *testing.T) {
237+
testFetchUnreferencedRefSha(t, "refs/special~", false)
238+
}
239+
240+
func TestFetchUnadvertisedRefShaKeepGitDir(t *testing.T) {
241+
testFetchUnreferencedRefSha(t, "refs/special~", true)
242+
}
243+
244+
// testFetchUnreferencedRefSha tests fetching a SHA that points to a ref that is not reachable from any branch.
245+
func testFetchUnreferencedRefSha(t *testing.T, ref string, keepGitDir bool) {
230246
if runtime.GOOS == "windows" {
231247
t.Skip("Depends on unimplemented containerd bind-mount support on Windows")
232248
}
@@ -239,7 +255,7 @@ func testFetchUnreferencedTagSha(t *testing.T, keepGitDir bool) {
239255

240256
repo := setupGitRepo(t)
241257

242-
cmd := exec.Command("git", "rev-parse", "v1.2.3-special")
258+
cmd := exec.Command("git", "rev-parse", ref)
243259
cmd.Dir = repo.mainPath
244260

245261
out, err := cmd.Output()
@@ -691,6 +707,8 @@ func setupGitRepo(t *testing.T) gitRepoFixture {
691707
// * (refs/heads/feature) withsub
692708
// * feature
693709
// * (HEAD -> refs/heads/master, tag: refs/tags/lightweight-tag) third
710+
// | * ref only
711+
// | * commit only
694712
// | * (tag: refs/tags/v1.2.3-special) tagonly-leaf
695713
// |/
696714
// * (tag: refs/tags/v1.2.3) second
@@ -699,35 +717,53 @@ func setupGitRepo(t *testing.T) gitRepoFixture {
699717
"git -c init.defaultBranch=master init",
700718
"git config --local user.email test",
701719
"git config --local user.name test",
720+
702721
"echo foo > abc",
703722
"git add abc",
704723
"git commit -m initial",
705724
"git tag --no-sign a/v1.2.3",
725+
706726
"echo bar > def",
707727
"mkdir subdir",
708728
"echo subcontents > subdir/subfile",
709729
"git add def subdir",
710730
"git commit -m second",
711731
"git tag -a -m \"this is an annotated tag\" v1.2.3",
732+
712733
"echo foo > bar",
713734
"git add bar",
714735
"git commit -m tagonly-leaf",
715736
"git tag --no-sign v1.2.3-special",
737+
738+
"echo foo2 > bar2",
739+
"git add bar2",
740+
"git commit -m \"commit only\"",
741+
"echo foo3 > bar3",
742+
"git add bar3",
743+
"git commit -m \"ref only\"",
744+
"git update-ref refs/special $(git rev-parse HEAD)",
745+
716746
// switch master back to v1.2.3
717747
"git checkout -B master v1.2.3",
748+
718749
"echo sbb > foo13",
719750
"git add foo13",
720751
"git commit -m third",
721752
"git tag --no-sign lightweight-tag",
753+
722754
"git checkout -B feature",
755+
723756
"echo baz > ghi",
724757
"git add ghi",
725758
"git commit -m feature",
726759
"git update-ref refs/test $(git rev-parse HEAD)",
760+
727761
"git submodule add "+fixture.subURL+" sub",
728762
"git add -A",
729763
"git commit -m withsub",
764+
730765
"git checkout master",
766+
731767
// "git log --oneline --graph --decorate=full --all",
732768
)
733769
return fixture
@@ -785,6 +821,7 @@ func runShell(t *testing.T, dir string, cmds ...string) {
785821
cmd = exec.Command("sh", "-c", args)
786822
}
787823
cmd.Dir = dir
824+
// cmd.Stdout = os.Stdout
788825
cmd.Stderr = os.Stderr
789826
require.NoErrorf(t, cmd.Run(), "error running %v", args)
790827
}

‎util/gitutil/git_cli.go

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -218,6 +218,14 @@ func (cli *GitCLI) Run(ctx context.Context, args ...string) (_ []byte, err error
218218
continue
219219
}
220220
}
221+
if strings.Contains(errbuf.String(), "not our ref") || strings.Contains(errbuf.String(), "unadvertised object") {
222+
// server-side error: https://github.com/git/git/blob/34b6ce9b30747131b6e781ff718a45328aa887d0/upload-pack.c#L811-L812
223+
// client-side error: https://github.com/git/git/blob/34b6ce9b30747131b6e781ff718a45328aa887d0/fetch-pack.c#L2250-L2253
224+
if newArgs := argsNoCommitRefspec(args); len(args) > len(newArgs) {
225+
args = newArgs
226+
continue
227+
}
228+
}
221229

222230
return buf.Bytes(), errors.Wrapf(err, "git stderr:\n%s", errbuf.String())
223231
}
@@ -244,3 +252,19 @@ func argsNoDepth(args []string) []string {
244252
}
245253
return out
246254
}
255+
256+
func argsNoCommitRefspec(args []string) []string {
257+
if len(args) <= 2 {
258+
return args
259+
}
260+
if args[0] != "fetch" {
261+
return args
262+
}
263+
264+
// assume the refspec is the last arg
265+
if IsCommitSHA(args[len(args)-1]) {
266+
return args[:len(args)-1]
267+
}
268+
269+
return args
270+
}

‎util/gitutil/git_commit.go

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
package gitutil
2+
3+
func IsCommitSHA(str string) bool {
4+
if len(str) != 40 {
5+
return false
6+
}
7+
8+
for _, ch := range str {
9+
if ch >= '0' && ch <= '9' {
10+
continue
11+
}
12+
if ch >= 'a' && ch <= 'f' {
13+
continue
14+
}
15+
return false
16+
}
17+
18+
return true
19+
}

0 commit comments

Comments
 (0)