Skip to content

Commit ffc8ff6

Browse files
fmeumfmeum
authored
Add support for --incompatible_compact_repo_mapping_manifest (#4375)
**What type of PR is this?** Feature **What does this PR do? Why is it needed?** Implements support for the new format added in bazelbuild/bazel#26262. **Which issues(s) does this PR fix?** Fixes # **Other notes for review**
1 parent 01e81a2 commit ffc8ff6

File tree

5 files changed

+284
-17
lines changed

5 files changed

+284
-17
lines changed

‎go/runfiles/BUILD.bazel‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,3 +49,13 @@ filegroup(
4949
srcs = glob(["**"]),
5050
visibility = ["//visibility:public"],
5151
)
52+
53+
go_test(
54+
name = "runfiles_test",
55+
srcs = [
56+
"caller_repository_example_test.go",
57+
"example_test.go",
58+
"rlocationpath_xdefs_example_test.go",
59+
],
60+
deps = [":runfiles"],
61+
)

‎go/runfiles/fs.go‎

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ func (r *Runfiles) Open(name string) (fs.File, error) {
3939
}
4040
repo, inRepoPath, hasInRepoPath := strings.Cut(name, "/")
4141
key := repoMappingKey{r.sourceRepo, repo}
42-
targetRepoDirectory, exists := r.repoMapping[key]
42+
targetRepoDirectory, exists := r.repoMapping.Get(key)
4343
if !exists {
4444
// Either name uses a canonical repo name or refers to a root symlink.
4545
// In both cases, we can just open the file directly.
@@ -93,11 +93,9 @@ func (r *rootDirFile) initEntries() error {
9393
// visible to the main repo (plus root symlinks). We thus need to read
9494
// the real entries and then transform and filter them.
9595
canonicalToApparentName := make(map[string]string)
96-
for k, v := range r.rf.repoMapping {
97-
if k.sourceRepo == r.rf.sourceRepo {
98-
canonicalToApparentName[v] = k.targetRepoApparentName
99-
}
100-
}
96+
r.rf.repoMapping.ForEachVisible(r.rf.sourceRepo, func(targetApparentName, targetRepoDirectory string) {
97+
canonicalToApparentName[targetRepoDirectory] = targetApparentName
98+
})
10199
rootFile, err := r.rf.impl.open(".")
102100
if err != nil {
103101
return err
@@ -163,7 +161,7 @@ type renamedFileInfo struct {
163161
name string
164162
}
165163

166-
func (r renamedFileInfo) Name() string { return r.name }
164+
func (r renamedFileInfo) Name() string { return r.name }
167165
func (r renamedFileInfo) String() string { return fs.FormatFileInfo(r) }
168166

169167
type emptyFile string
@@ -180,4 +178,4 @@ func (emptyFileInfo) Mode() fs.FileMode { return 0444 }
180178
func (emptyFileInfo) ModTime() time.Time { return time.Time{} }
181179
func (emptyFileInfo) IsDir() bool { return false }
182180
func (emptyFileInfo) Sys() interface{} { return nil }
183-
func (i emptyFileInfo) String() string { return fs.FormatFileInfo(i) }
181+
func (i emptyFileInfo) String() string { return fs.FormatFileInfo(i) }

‎go/runfiles/runfiles.go‎

Lines changed: 79 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@ import (
5555
"io/fs"
5656
"os"
5757
"path/filepath"
58+
"slices"
5859
"strings"
5960
)
6061

@@ -83,7 +84,7 @@ type Runfiles struct {
8384
// immutable once created.
8485
impl runfiles
8586
env []string
86-
repoMapping map[repoMappingKey]string
87+
repoMapping *repoMapping
8788
sourceRepo string
8889
}
8990

@@ -171,7 +172,7 @@ func (r *Runfiles) Rlocation(path string) (string, error) {
171172
split := strings.SplitN(path, "/", 2)
172173
if len(split) == 2 {
173174
key := repoMappingKey{r.sourceRepo, split[0]}
174-
if targetRepoDirectory, exists := r.repoMapping[key]; exists {
175+
if targetRepoDirectory, exists := r.repoMapping.Get(key); exists {
175176
mappedPath = targetRepoDirectory + "/" + split[1]
176177
}
177178
}
@@ -200,9 +201,12 @@ func isNormalizedPath(s string) error {
200201
// This mutates the Runfiles object, but is idempotent.
201202
func (r *Runfiles) loadRepoMapping() error {
202203
repoMappingPath, err := r.impl.path(repoMappingRlocation)
203-
// If Bzlmod is disabled, the repository mapping manifest isn't created, so
204-
// it is not an error if it is missing.
204+
// The repo mapping manifest only exists with Bzlmod, so it's not an
205+
// error if it's missing. Since any repository name not contained in the
206+
// mapping is assumed to be already canonical, an empty map is
207+
// equivalent to not applying any mapping.
205208
if err != nil {
209+
r.repoMapping = &repoMapping{}
206210
return nil
207211
}
208212
r.repoMapping, err = parseRepoMapping(repoMappingPath)
@@ -290,34 +294,100 @@ type runfiles interface {
290294
// https://cs.opensource.google/bazel/bazel/+/1b073ac0a719a09c9b2d1a52680517ab22dc971e:src/main/java/com/google/devtools/build/lib/analysis/Runfiles.java;l=424
291295
const repoMappingRlocation = "_repo_mapping"
292296

297+
type prefixMapping struct {
298+
prefix string
299+
mapping map[string]string
300+
}
301+
302+
// The zero value of repoMapping is an empty mapping.
303+
type repoMapping struct {
304+
exactMappings map[repoMappingKey]string
305+
prefixMappings []prefixMapping
306+
}
307+
308+
func (rm *repoMapping) Get(key repoMappingKey) (string, bool) {
309+
if mapping, ok := rm.exactMappings[key]; ok {
310+
return mapping, true
311+
}
312+
i, _ := slices.BinarySearchFunc(rm.prefixMappings, key.sourceRepo, func(prefixMapping prefixMapping, s string) int {
313+
return strings.Compare(prefixMapping.prefix, s)
314+
})
315+
if i > 0 && strings.HasPrefix(key.sourceRepo, rm.prefixMappings[i-1].prefix) {
316+
mapping, ok := rm.prefixMappings[i-1].mapping[key.targetRepoApparentName]
317+
return mapping, ok
318+
}
319+
return "", false
320+
}
321+
322+
// ForEachVisible iterates over all target repositories that are visible to the
323+
// given source repo in an unspecified order.
324+
func (rm *repoMapping) ForEachVisible(sourceRepo string, f func(targetRepoApparentName, targetRepoDirectory string)) {
325+
for key, targetRepoDirectory := range rm.exactMappings {
326+
if key.sourceRepo == sourceRepo {
327+
f(key.targetRepoApparentName, targetRepoDirectory)
328+
}
329+
}
330+
i, _ := slices.BinarySearchFunc(rm.prefixMappings, sourceRepo, func(prefixMapping prefixMapping, s string) int {
331+
return strings.Compare(prefixMapping.prefix, s)
332+
})
333+
if i > 0 && strings.HasPrefix(sourceRepo, rm.prefixMappings[i-1].prefix) {
334+
for targetRepoApparentName, targetRepoDirectory := range rm.prefixMappings[i-1].mapping {
335+
f(targetRepoApparentName, targetRepoDirectory)
336+
}
337+
}
338+
}
339+
293340
// Parses a repository mapping manifest file emitted with Bzlmod enabled.
294-
func parseRepoMapping(path string) (map[repoMappingKey]string, error) {
341+
func parseRepoMapping(path string) (*repoMapping, error) {
295342
r, err := os.Open(path)
296343
if err != nil {
297344
// The repo mapping manifest only exists with Bzlmod, so it's not an
298345
// error if it's missing. Since any repository name not contained in the
299346
// mapping is assumed to be already canonical, an empty map is
300347
// equivalent to not applying any mapping.
301-
return nil, nil
348+
return &repoMapping{}, nil
302349
}
303350
defer r.Close()
304351

305352
// Each line of the repository mapping manifest has the form:
306353
// canonical name of source repo,apparent name of target repo,target repo runfiles directory
307354
// https://cs.opensource.google/bazel/bazel/+/1b073ac0a719a09c9b2d1a52680517ab22dc971e:src/main/java/com/google/devtools/build/lib/analysis/RepoMappingManifestAction.java;l=117
308355
s := bufio.NewScanner(r)
309-
repoMapping := make(map[repoMappingKey]string)
356+
exactMappings := make(map[repoMappingKey]string)
357+
prefixMappingsMap := make(map[string]map[string]string)
310358
for s.Scan() {
311359
fields := strings.SplitN(s.Text(), ",", 3)
312360
if len(fields) != 3 {
313361
return nil, fmt.Errorf("runfiles: bad repo mapping line %q in file %s", s.Text(), path)
314362
}
315-
repoMapping[repoMappingKey{fields[0], fields[1]}] = fields[2]
363+
if strings.HasSuffix(fields[0], "*") {
364+
prefix := strings.TrimSuffix(fields[0], "*")
365+
if _, ok := prefixMappingsMap[prefix]; !ok {
366+
prefixMappingsMap[prefix] = make(map[string]string)
367+
}
368+
prefixMappingsMap[prefix][fields[1]] = fields[2]
369+
} else {
370+
exactMappings[repoMappingKey{fields[0], fields[1]}] = fields[2]
371+
}
316372
}
317373

318374
if err = s.Err(); err != nil {
319375
return nil, fmt.Errorf("runfiles: error parsing repo mapping file %s: %w", path, err)
320376
}
321377

322-
return repoMapping, nil
378+
// No prefix can be a prefix of another prefix, so we can use binary search
379+
// on a sorted slice to find the unique prefix that may match a given source
380+
// repo.
381+
var prefixMappings []prefixMapping
382+
for prefix, mapping := range prefixMappingsMap {
383+
prefixMappings = append(prefixMappings, prefixMapping{
384+
prefix: prefix,
385+
mapping: mapping,
386+
})
387+
}
388+
slices.SortFunc(prefixMappings, func(a, b prefixMapping) int {
389+
return strings.Compare(a.prefix, b.prefix)
390+
})
391+
392+
return &repoMapping{exactMappings, prefixMappings}, nil
323393
}

‎tests/runfiles/BUILD.bazel‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,14 @@ go_test(
4343
],
4444
)
4545

46+
go_test(
47+
name = "runfiles_internal_test",
48+
srcs = [
49+
"runfiles_internal_test.go",
50+
],
51+
embed = ["//go/runfiles"],
52+
)
53+
4654
go_bazel_test(
4755
name = "runfiles_bazel_test",
4856
srcs = ["runfiles_bazel_test.go"],

‎tests/runfiles/runfiles_internal_test.go‎

Lines changed: 181 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,181 @@
1+
package runfiles
2+
3+
import (
4+
"os"
5+
"path/filepath"
6+
"slices"
7+
"testing"
8+
)
9+
10+
const testRepoMapping = `my_module++ext1+repo1,my_module,my_module+
11+
my_module++ext1+repo1,repo1,my_module++ext1+repo1
12+
my_module++ext1+repo1,repo2,my_module++ext1+repo2
13+
my_module++ext1+repo2,my_module,my_module+
14+
my_module++ext1+repo2,repo1,my_module++ext1+repo1
15+
my_module++ext1+repo2,repo2,my_module++ext1+repo2
16+
my_module++ext2+*,my_module,my_module+
17+
my_module++ext2+*,repo1,my_module++ext2+repo1
18+
my_module++ext2+*,repo2,my_module++ext2+repo2
19+
my_module++ext3+*,my_module,my_module+
20+
my_module++ext+*,my_module,my_module+
21+
`
22+
23+
func TestRepoMapping_Get(t *testing.T) {
24+
rm := getRepoMapping(t)
25+
26+
for _, tc := range []struct {
27+
sourceRepo, targetRepoApparentName, expectedTargetRepo string
28+
}{
29+
{
30+
sourceRepo: "my_module++ext1+repo1",
31+
targetRepoApparentName: "my_module",
32+
expectedTargetRepo: "my_module+",
33+
},
34+
{
35+
sourceRepo: "my_module++ext1+repo1",
36+
targetRepoApparentName: "repo1",
37+
expectedTargetRepo: "my_module++ext1+repo1",
38+
},
39+
{
40+
sourceRepo: "my_module++ext1+repo1",
41+
targetRepoApparentName: "repo2",
42+
expectedTargetRepo: "my_module++ext1+repo2",
43+
},
44+
{
45+
sourceRepo: "my_module++ext1+repo1",
46+
targetRepoApparentName: "non_existent_repo",
47+
expectedTargetRepo: "",
48+
},
49+
{
50+
sourceRepo: "my_module++ext1+repo2",
51+
targetRepoApparentName: "repo1",
52+
expectedTargetRepo: "my_module++ext1+repo1",
53+
},
54+
{
55+
sourceRepo: "my_module++ext2+repo1",
56+
targetRepoApparentName: "my_module",
57+
expectedTargetRepo: "my_module+",
58+
},
59+
{
60+
sourceRepo: "my_module++ext2+repo1",
61+
targetRepoApparentName: "repo1",
62+
expectedTargetRepo: "my_module++ext2+repo1",
63+
},
64+
{
65+
sourceRepo: "my_module++ext2+repo1",
66+
targetRepoApparentName: "repo2",
67+
expectedTargetRepo: "my_module++ext2+repo2",
68+
},
69+
{
70+
sourceRepo: "my_module++ext2+repo1",
71+
targetRepoApparentName: "non_existent_repo",
72+
expectedTargetRepo: "",
73+
},
74+
{
75+
sourceRepo: "my_module++ext2+repo2",
76+
targetRepoApparentName: "my_module",
77+
expectedTargetRepo: "my_module+",
78+
},
79+
{
80+
sourceRepo: "my_module++ext2+repo2",
81+
targetRepoApparentName: "repo1",
82+
expectedTargetRepo: "my_module++ext2+repo1",
83+
},
84+
{
85+
sourceRepo: "my_module++ext2+repo2",
86+
targetRepoApparentName: "repo2",
87+
expectedTargetRepo: "my_module++ext2+repo2",
88+
},
89+
{
90+
sourceRepo: "my_module++ext2+repo2",
91+
targetRepoApparentName: "non_existent_repo",
92+
expectedTargetRepo: "",
93+
},
94+
} {
95+
t.Run(tc.sourceRepo+"->"+tc.targetRepoApparentName, func(t *testing.T) {
96+
targetRepo, found := rm.Get(repoMappingKey{tc.sourceRepo, tc.targetRepoApparentName})
97+
if targetRepo != tc.expectedTargetRepo {
98+
t.Fatalf("targetRepo differs: %q != %q", targetRepo, tc.expectedTargetRepo)
99+
}
100+
if found != (tc.expectedTargetRepo != "") {
101+
t.Fatalf("found differs: %v != %v", found, tc.expectedTargetRepo != "")
102+
}
103+
})
104+
}
105+
}
106+
107+
func TestRepoMapping_ForEachVisible(t *testing.T) {
108+
rm := getRepoMapping(t)
109+
110+
for _, tc := range []struct {
111+
sourceRepo string
112+
expectedTargetRepoApparentNames []string
113+
expectedTargetRepoDirectories []string
114+
}{
115+
{
116+
sourceRepo: "my_module++ext1+repo1",
117+
expectedTargetRepoApparentNames: []string{"my_module", "repo1", "repo2"},
118+
expectedTargetRepoDirectories: []string{"my_module+", "my_module++ext1+repo1", "my_module++ext1+repo2"},
119+
},
120+
{
121+
sourceRepo: "my_module++ext1+repo2",
122+
expectedTargetRepoApparentNames: []string{"my_module", "repo1", "repo2"},
123+
expectedTargetRepoDirectories: []string{"my_module+", "my_module++ext1+repo1", "my_module++ext1+repo2"},
124+
},
125+
{
126+
sourceRepo: "my_module++ext2+repo1",
127+
expectedTargetRepoApparentNames: []string{"my_module", "repo1", "repo2"},
128+
expectedTargetRepoDirectories: []string{"my_module+", "my_module++ext2+repo1", "my_module++ext2+repo2"},
129+
},
130+
{
131+
sourceRepo: "my_module++ext2+repo2",
132+
expectedTargetRepoApparentNames: []string{"my_module", "repo1", "repo2"},
133+
expectedTargetRepoDirectories: []string{"my_module+", "my_module++ext2+repo1", "my_module++ext2+repo2"},
134+
},
135+
{
136+
sourceRepo: "non_existent_repo+",
137+
},
138+
} {
139+
t.Run(tc.sourceRepo, func(t *testing.T) {
140+
var targetRepoApparentNames, targetRepoDirectories []string
141+
rm.ForEachVisible(tc.sourceRepo, func(targetRepoApparentName, targetRepoDirectory string) {
142+
targetRepoApparentNames = append(targetRepoApparentNames, targetRepoApparentName)
143+
targetRepoDirectories = append(targetRepoDirectories, targetRepoDirectory)
144+
})
145+
slices.Sort(targetRepoApparentNames)
146+
slices.Sort(targetRepoDirectories)
147+
148+
if len(targetRepoApparentNames) != len(tc.expectedTargetRepoApparentNames) {
149+
t.Fatalf("expected %d target repo apparent names, got %d: %v", len(tc.expectedTargetRepoApparentNames), len(targetRepoApparentNames), targetRepoApparentNames)
150+
}
151+
if len(targetRepoDirectories) != len(tc.expectedTargetRepoDirectories) {
152+
t.Fatalf("expected %d target repo directories, got %d: %v", len(tc.expectedTargetRepoDirectories), len(targetRepoDirectories), targetRepoDirectories)
153+
}
154+
for i, expectedName := range tc.expectedTargetRepoApparentNames {
155+
if targetRepoApparentNames[i] != expectedName {
156+
t.Errorf("target repo apparent name at index %d differs: %q != %q", i, targetRepoApparentNames[i], expectedName)
157+
}
158+
}
159+
for i, expectedDir := range tc.expectedTargetRepoDirectories {
160+
if targetRepoDirectories[i] != expectedDir {
161+
t.Errorf("target repo directory at index %d differs: %q != %q", i, targetRepoDirectories[i], expectedDir)
162+
}
163+
}
164+
})
165+
}
166+
}
167+
168+
func getRepoMapping(t *testing.T) *repoMapping {
169+
t.Helper()
170+
tmp := t.TempDir()
171+
repoMappingFile := filepath.Join(tmp, "repo_mapping")
172+
err := os.WriteFile(repoMappingFile, []byte(testRepoMapping), 0755)
173+
if err != nil {
174+
t.Fatalf("failed to write repo mapping file: %s", err)
175+
}
176+
rm, err := parseRepoMapping(repoMappingFile)
177+
if err != nil {
178+
t.Fatalf("failed to parse repo mapping file: %s", err)
179+
}
180+
return rm
181+
}

0 commit comments

Comments
 (0)