contenthash: add tests to check needsScan's correctness

Most of these tests revolve around making sure that we are scanning the
correct path during Checksum, and that we don't think a scan is required
for subpaths we have already scanned.

Signed-off-by: Aleksa Sarai <[email protected]>

Author: cyphar

cache/contenthash/checksum_test.go

@@ -65,6 +65,125 @@ func TestChecksumSymlinkNoParentScan(t *testing.T) {
 	dgst, err := cc.Checksum(context.TODO(), ref, "aa/ln/bb/cc/dd", ChecksumOpts{FollowLinks: true}, nil)
 	require.NoError(t, err)
 	require.Equal(t, dgstFileData0, dgst)
+
+	// The above checksum request should have only checksummed aa/bb/cc, and so
+	// any parent directories should need a scan but non-existent (or existent)
+	// children should not.
+	root := cc.tree.Root()
+
+	for _, path := range []string{
+		// Paths not within the scanned /aa/bb/cc/.
+		"/", "/aa", "/aa/bb", "/aa/bb/ff", "/non-exist",
+	} {
+		needs1, err := cc.needsScan(root, path, false)
+		require.NoErrorf(t, err, "needsScan(%q, followTrailing=false)", path)
+		require.Truef(t, needs1, "needsScan(%q, followTrailing=false)", path)
+
+		needs2, err := cc.needsScan(root, path, true)
+		require.NoErrorf(t, err, "needsScan(%q, followTrailing=true)", path)
+		require.Truef(t, needs2, "needsScan(%q, followTrailing=true)", path)
+	}
+
+	for _, path := range []string{
+		// Paths within the scanned /aa/bb/cc, even if they don't exist.
+		"/aa/bb/cc", "/aa/bb/cc/non-exist", "/aa/bb/cc/dd/ee/ff", "/aa/bb/cc/non-exist/xx/yy/zz",
+	} {
+		needs1, err := cc.needsScan(root, path, false)
+		require.NoErrorf(t, err, "needsScan(%q, followTrailing=false)", path)
+		require.Falsef(t, needs1, "needsScan(%q, followTrailing=false)", path)
+
+		needs2, err := cc.needsScan(root, path, true)
+		require.NoErrorf(t, err, "needsScan(%q, followTrailing=true)", path)
+		require.Falsef(t, needs2, "needsScan(%q, followTrailing=true)", path)
+	}
+
+	// /aa was not scanned, but during the walk we went through /aa/ln and so
+	// we know the contents of the link. However, if we want to scan it with
+	// followTrailing=true, we will need a scan because we didn't scan /aa.
+	path := "/aa/ln"
+	needs1, err := cc.needsScan(root, path, false)
+	require.NoErrorf(t, err, "needsScan(%q, followTrailing=false)", path)
+	require.Falsef(t, needs1, "needsScan(%q, followTrailing=false)", path)
+
+	needs2, err := cc.needsScan(root, path, true)
+	require.NoErrorf(t, err, "needsScan(%q, followTrailing=true)", path)
+	require.Truef(t, needs2, "needsScan(%q, followTrailing=true)", path)
+}
+
+// https://github.com/moby/buildkit/issues/5042
+func TestNeedScanChecksumRegression(t *testing.T) {
+	tmpdir := t.TempDir()
+
+	snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
+	require.NoError(t, err)
+	cm, cleanup := setupCacheManager(t, tmpdir, "native", snapshotter)
+	t.Cleanup(cleanup)
+
+	ch := []string{
+		"ADD aa dir",
+		"ADD aa/bb dir",
+		"ADD aa/bb/cc file data0",
+		"ADD aa/ln symlink /aa",
+		"ADD aa/root symlink /",
+		"ADD bb symlink aa/bb",
+	}
+
+	ref := createRef(t, cm, ch)
+
+	cc, err := newCacheContext(ref)
+	require.NoError(t, err)
+
+	// Checksumming /aa/bb while following links will result in /aa being scanned.
+	_, err = cc.Checksum(context.TODO(), ref, "/bb", ChecksumOpts{FollowLinks: true}, nil)
+	require.NoError(t, err)
+
+	root := cc.tree.Root()
+	for _, test := range []struct {
+		path                            string
+		followTrailing, expectNeedsScan bool
+	}{
+		// Any path under /aa will not result in a re-scan.
+		{"/aa", true, false},
+		{"/aa/ln", true, false},
+		{"/aa/ln", false, false},
+		{"/aa/non-exist", true, false},
+		{"/aa/bb/non-exist", true, false},
+		{"/aa/bb/cc", true, false},
+		{"/aa/bb/cc/non-exist", true, false},
+		// followTrailing=false on a symlink to /.
+		{"/aa/root", false, false},
+		// /bb itself was scanned during the lookup in Checksum.
+		{"/bb", true, false},
+		{"/bb", false, false},
+		// A path outside /aa will need a scan.
+		{"/non-exist", true, true},
+		{"/non-exist", false, true},
+		{"/aa/root", true, true},
+		{"/", true, true},
+	} {
+		needs, err := cc.needsScan(root, test.path, test.followTrailing)
+		require.NoErrorf(t, err, "needsScan(%q, followTrailing=%v)", test.path, test.followTrailing)
+		require.Equalf(t, test.expectNeedsScan, needs, "needsScan(%q, followTrailing=%v)", test.path, test.followTrailing)
+	}
+
+	// Looking up a non-existent path in / will checksum the whole tree. See
+	// <https://github.com/moby/buildkit/issues/5042> for more information.
+	// This means that needsScan will return true for any path.
+	_, err = cc.Checksum(context.TODO(), ref, "/non-existent", ChecksumOpts{FollowLinks: true}, nil)
+	require.Error(t, err)
+
+	root = cc.tree.Root()
+	for _, path := range []string{
+		"/", "/non-exist", "/ff", "/aa/root", "/non-exist/child", "/different-non-exist",
+	} {
+		needs1, err := cc.needsScan(root, path, false)
+		require.NoErrorf(t, err, "needsScan(%q, followTrailing=false)", path)
+		require.Falsef(t, needs1, "needsScan(%q, followTrailing=false)", path)
+
+		needs2, err := cc.needsScan(root, path, true)
+		require.NoErrorf(t, err, "needsScan(%q, followTrailing=true)", path)
+		require.Falsef(t, needs2, "needsScan(%q, followTrailing=true)", path)
+	}
 }
 
 func TestChecksumNonLexicalSymlinks(t *testing.T) {
@@ -114,8 +233,8 @@ func TestChecksumNonLexicalSymlinks(t *testing.T) {
 		"link3/target/file",
 	} {
 		dgst, err := cc.Checksum(context.TODO(), ref, path, ChecksumOpts{FollowLinks: true}, nil)
-		require.NoError(t, err)
-		require.Equal(t, dgstFileData0, dgst)
+		require.NoErrorf(t, err, "Checksum(%q)", path)
+		require.Equalf(t, dgstFileData0, dgst, "Checksum(%q)", path)
 	}
 
 	// FollowLinks only affects final component resolution, so make sure that
@@ -130,8 +249,8 @@ func TestChecksumNonLexicalSymlinks(t *testing.T) {
 		"link3/target/file",
 	} {
 		dgst, err := cc.Checksum(context.TODO(), ref, path, ChecksumOpts{FollowLinks: false}, nil)
-		require.NoError(t, err)
-		require.Equal(t, dgstFileData0, dgst)
+		require.NoErrorf(t, err, "Checksum(%q)", path)
+		require.Equalf(t, dgstFileData0, dgst, "Checksum(%q)", path)
 	}
 
 	dgstLink1TargetFile, err := cc.Checksum(context.TODO(), ref, "link1/target_file", ChecksumOpts{FollowLinks: false}, nil)
@@ -159,9 +278,9 @@ func TestChecksumNonLexicalSymlinks(t *testing.T) {
 		{"link3/target_file", dgstLink3TargetFile},
 	} {
 		dgst, err := cc.Checksum(context.TODO(), ref, test.path, ChecksumOpts{FollowLinks: false}, nil)
-		require.NoError(t, err)
-		require.NotEqual(t, dgstFileData0, dgst)
-		require.Equal(t, test.expectedDgst, dgst)
+		require.NoErrorf(t, err, "Checksum(%q)", test.path)
+		require.NotEqualf(t, dgstFileData0, dgst, "Checksum(%q)", test.path)
+		require.Equalf(t, test.expectedDgst, dgst, "Checksum(%q)", test.path)
 	}
 }