git: add resolvercache support to git source

Make sure remote ref does not change to different commit  if
git repo changes in the middle of the build.

Signed-off-by: Tonis Tiigi <[email protected]>

Author: tonistiigi

client/client_test.go

@@ -248,6 +248,7 @@ var allTests = []func(t *testing.T, sb integration.Sandbox){
 	testHTTPPruneAfterResolveMeta,
 	testHTTPResolveMetaReuse,
 	testHTTPResolveMultiBuild,
+	testGitResolveMutatedSource,
 }
 
 func TestIntegration(t *testing.T) {
@@ -12413,6 +12414,124 @@ func testHTTPResolveMultiBuild(t *testing.T, sb integration.Sandbox) {
 	require.Equal(t, "content2", string(dt))
 }
 
+func testGitResolveMutatedSource(t *testing.T, sb integration.Sandbox) {
+	integration.SkipOnPlatform(t, "windows")
+	ctx := sb.Context()
+	c, err := New(ctx, sb.Address())
+	require.NoError(t, err)
+	defer c.Close()
+
+	gitDir := t.TempDir()
+	gitCommands := []string{
+		"git init",
+		"git config --local user.email test",
+		"git config --local user.name test",
+		"echo a > a",
+		"git add a",
+		"git commit -m a",
+		"git tag -a v0.1 -m v0.1",
+		"echo b > b",
+		"git add b",
+		"git commit -m b",
+		"git checkout -B v2",
+		"git update-server-info",
+	}
+	err = runInDir(gitDir, gitCommands...)
+	require.NoError(t, err)
+
+	// cmd := exec.Command("git", "rev-parse", "HEAD")
+	// cmd.Dir = gitDir
+	// out, err := cmd.Output()
+	// require.NoError(t, err)
+	// commitHEAD := strings.TrimSpace(string(out))
+
+	cmd := exec.Command("git", "rev-parse", "v0.1")
+	cmd.Dir = gitDir
+	out, err := cmd.Output()
+	require.NoError(t, err)
+	commitTag := strings.TrimSpace(string(out))
+
+	cmd = exec.Command("git", "rev-parse", "v0.1^{commit}")
+	cmd.Dir = gitDir
+	out, err = cmd.Output()
+	require.NoError(t, err)
+	commitTagCommit := strings.TrimSpace(string(out))
+
+	server := httptest.NewServer(http.FileServer(http.Dir(filepath.Clean(gitDir))))
+	defer server.Close()
+
+	dest := t.TempDir()
+
+	_, err = c.Build(ctx, SolveOpt{
+		Exports: []ExportEntry{
+			{
+				Type:      ExporterLocal,
+				OutputDir: dest,
+			},
+		},
+	}, "test", func(ctx context.Context, c gateway.Client) (*gateway.Result, error) {
+		id := "git://" + strings.TrimPrefix(server.URL, "http://") + "/.git#v0.1"
+		md, err := c.ResolveSourceMetadata(ctx, &pb.SourceOp{
+			Identifier: id,
+			Attrs: map[string]string{
+				"git.fullurl": server.URL + "/.git",
+			},
+		}, sourceresolver.Opt{})
+		if err != nil {
+			return nil, err
+		}
+		require.NotNil(t, md.Git)
+		require.Equal(t, "refs/tags/v0.1", md.Git.Ref)
+		require.Equal(t, commitTag, md.Git.Checksum)
+		require.Equal(t, commitTagCommit, md.Git.CommitChecksum)
+		require.Equal(t, id, md.Op.Identifier)
+		require.Equal(t, server.URL+"/.git", md.Op.Attrs["git.fullurl"])
+
+		// update the tag to point to a different commit
+		err = runInDir(gitDir, []string{
+			"git tag -f v0.1",
+			"git update-server-info",
+		}...)
+		require.NoError(t, err)
+
+		md, err = c.ResolveSourceMetadata(ctx, &pb.SourceOp{
+			Identifier: id,
+			Attrs: map[string]string{
+				"git.fullurl": server.URL + "/.git",
+			},
+		}, sourceresolver.Opt{})
+		if err != nil {
+			return nil, err
+		}
+		require.NotNil(t, md.Git)
+		require.Equal(t, "refs/tags/v0.1", md.Git.Ref)
+		require.Equal(t, commitTag, md.Git.Checksum)
+		require.Equal(t, commitTagCommit, md.Git.CommitChecksum)
+		require.Equal(t, id, md.Op.Identifier)
+		require.Equal(t, server.URL+"/.git", md.Op.Attrs["git.fullurl"])
+
+		st := llb.Git(server.URL+"/.git", "", llb.GitRef("v0.1"))
+		def, err := st.Marshal(sb.Context())
+		if err != nil {
+			return nil, err
+		}
+		return c.Solve(ctx, gateway.SolveRequest{
+			Definition: def.ToPB(),
+		})
+	}, nil)
+	require.NoError(t, err)
+
+	_, err = os.ReadFile(filepath.Join(dest, "b"))
+	require.Error(t, err)
+	require.True(t, os.IsNotExist(err), "expected file b to not exist")
+
+	dt, err := os.ReadFile(filepath.Join(dest, "a"))
+	require.NoError(t, err)
+	require.Equal(t, "a\n", string(dt))
+
+	checkAllReleasable(t, c, sb, false)
+}
+
 func runInDir(dir string, cmds ...string) error {
 	for _, args := range cmds {
 		var cmd *exec.Cmd

solver/types.go

@@ -189,8 +189,8 @@ type JobContext interface {
 type ResolverCache interface {
 	// Lock locks a key until the returned release function is called.
 	// Release function can return value that will be returned to next callers.
-	// Lock can return multiple values because two steps can be merged together after
-	// they both already completed resolve independently.
+	// Lock can return multiple values because two steps can be merged once
+	// both have independently completed their resolution.
 	Lock(key any) (values []any, release func(any) error, err error)
 }
 

source/git/source.go

@@ -384,7 +384,11 @@ func (gs *gitSourceHandler) mountKnownHosts() (string, func() error, error) {
 	return knownHosts.Name(), cleanup, nil
 }
 
-func (gs *gitSourceHandler) resolveMetadata(ctx context.Context, jobCtx solver.JobContext) (*Metadata, error) {
+func (gs *gitSourceHandler) remoteKey() string {
+	return gs.src.Remote + "#" + gs.src.Ref
+}
+
+func (gs *gitSourceHandler) resolveMetadata(ctx context.Context, jobCtx solver.JobContext) (md *Metadata, retErr error) {
 	remote := gs.src.Remote
 	gs.locker.Lock(remote)
 	defer gs.locker.Unlock(remote)
@@ -397,6 +401,9 @@ func (gs *gitSourceHandler) resolveMetadata(ctx context.Context, jobCtx solver.J
 	}
 
 	if gitutil.IsCommitSHA(gs.src.Ref) {
+		if gs.src.Checksum != "" && !strings.HasPrefix(gs.src.Ref, gs.src.Checksum) {
+			return nil, errors.Errorf("expected checksum to match %s, got %s", gs.src.Checksum, gs.src.Ref)
+		}
 		return &Metadata{
 			Ref:      gs.src.Ref,
 			Checksum: gs.src.Ref,
@@ -406,6 +413,35 @@ func (gs *gitSourceHandler) resolveMetadata(ctx context.Context, jobCtx solver.J
 	var g session.Group
 	if jobCtx != nil {
 		g = jobCtx.Session()
+
+		if rc := jobCtx.ResolverCache(); rc != nil {
+			values, release, err := rc.Lock(gs.remoteKey())
+			if err != nil {
+				return nil, err
+			}
+			saveResolved := true
+			defer func() {
+				v := md
+				if retErr != nil || !saveResolved {
+					v = nil
+				}
+				if err := release(v); err != nil {
+					bklog.G(ctx).Warnf("failed to release resolver cache lock for %s: %v", gs.remoteKey(), err)
+				}
+			}()
+			for _, v := range values {
+				v2, ok := v.(*Metadata)
+				if !ok {
+					return nil, errors.Errorf("invalid resolver cache value for %s: %T", gs.remoteKey(), v)
+				}
+				if gs.src.Checksum != "" && !strings.HasPrefix(v2.Checksum, gs.src.Checksum) {
+					continue
+				}
+				saveResolved = false
+				clone := *v2
+				return &clone, nil
+			}
+		}
 	}
 
 	gs.getAuthToken(ctx, g)
@@ -483,7 +519,7 @@ func (gs *gitSourceHandler) resolveMetadata(ctx context.Context, jobCtx solver.J
 			return nil, errors.Errorf("expected checksum to match %s, got %s", gs.src.Checksum, exp)
 		}
 	}
-	md := &Metadata{
+	md = &Metadata{
 		Ref:      usedRef,
 		Checksum: sha,
 	}