Skip to content

Commit 59d3a60

Browse files
wiwaShaneDelmore
wiwa
and
ShaneDelmore
committed
Rename and add tests for util/InputIndexer and init PersistentExecutorTest
Co-authored-by: Shane Delmore <shane@delmore.io>
1 parent 65da148 commit 59d3a60

File tree

6 files changed

+246
-7
lines changed

6 files changed

+246
-7
lines changed

‎src/main/java/build/buildfarm/worker/persistent/WorkFilesContext.java‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
package build.buildfarm.worker.persistent;
22

33
import build.buildfarm.v1test.Tree;
4-
import build.buildfarm.worker.util.TreeWalker;
4+
import build.buildfarm.worker.util.InputsIndexer;
55
import com.google.common.collect.ImmutableList;
66
import com.google.common.collect.ImmutableMap;
77
import com.google.devtools.build.lib.worker.WorkerProtocol.Input;
@@ -19,7 +19,7 @@ public class WorkFilesContext {
1919

2020
public final ImmutableList<String> outputDirectories;
2121

22-
private final TreeWalker treeWalker;
22+
private final InputsIndexer inputsIndexer;
2323

2424
private ImmutableMap<Path, Input> pathInputs = null;
2525

@@ -37,14 +37,14 @@ public WorkFilesContext(
3737
this.outputFiles = outputFiles;
3838
this.outputDirectories = outputDirectories;
3939

40-
this.treeWalker = new TreeWalker(execTree);
40+
this.inputsIndexer = new InputsIndexer(execTree);
4141
}
4242

4343
// Paths are absolute paths from the opRoot; same as the Input.getPath();
4444
public ImmutableMap<Path, Input> getPathInputs() {
4545
synchronized (this) {
4646
if (pathInputs == null) {
47-
pathInputs = treeWalker.getAllInputs(opRoot);
47+
pathInputs = inputsIndexer.getAllInputs(opRoot);
4848
}
4949
}
5050
return pathInputs;
@@ -53,7 +53,7 @@ public ImmutableMap<Path, Input> getPathInputs() {
5353
public ImmutableMap<Path, Input> getToolInputs() {
5454
synchronized (this) {
5555
if (toolInputs == null) {
56-
toolInputs = treeWalker.getToolInputs(opRoot);
56+
toolInputs = inputsIndexer.getToolInputs(opRoot);
5757
}
5858
}
5959
return toolInputs;

src/main/java/build/buildfarm/worker/util/TreeWalker.java renamed to src/main/java/build/buildfarm/worker/util/InputsIndexer.java

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,10 @@
1515
/**
1616
* Organizes action Inputs into files, extracting their paths, and differentiates tool inputs (e.g.
1717
* JavaBuilder, Scalac, etc.)
18+
*
19+
* <p>Indexes (and partitions) Inputs from an action's Merkle Tree.
1820
*/
19-
public class TreeWalker {
21+
public class InputsIndexer {
2022
// See: https://github.com/bazelbuild/bazel/issues/10091
2123
public static final String BAZEL_TOOL_INPUT_MARKER = "bazel_tool_input";
2224

@@ -27,7 +29,7 @@ public class TreeWalker {
2729
ImmutableMap<Path, Input> absPathInputs = null;
2830
ImmutableMap<Path, Input> toolInputs = null;
2931

30-
public TreeWalker(Tree tree) {
32+
public InputsIndexer(Tree tree) {
3133
this.tree = tree;
3234
this.proxyDirs = new ProxyDirectoriesIndex(tree.getDirectoriesMap());
3335
}

‎src/test/java/build/buildfarm/worker/persistent/BUILD‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
java_test(
2+
name = "tests",
3+
size = "small",
4+
srcs = glob(["*.java"]),
5+
test_class = "build.buildfarm.AllTests",
6+
deps = [
7+
"//src/main/java/build/buildfarm/common",
8+
"//src/main/java/build/buildfarm/common/config",
9+
"//src/main/java/build/buildfarm/instance",
10+
"//src/main/java/build/buildfarm/worker",
11+
"//src/main/java/build/buildfarm/worker/resources",
12+
"//src/main/protobuf:build_buildfarm_v1test_buildfarm_java_proto",
13+
"//src/test/java/build/buildfarm:test_runner",
14+
"@bazel_tools//src/main/protobuf:worker_protocol_java_proto",
15+
"@googleapis//:google_rpc_code_java_proto",
16+
"@maven//:com_github_jnr_jnr_constants",
17+
"@maven//:com_github_jnr_jnr_ffi",
18+
"@maven//:com_github_serceman_jnr_fuse",
19+
"@maven//:com_google_guava_guava",
20+
"@maven//:com_google_jimfs_jimfs",
21+
"@maven//:com_google_protobuf_protobuf_java",
22+
"@maven//:com_google_truth_truth",
23+
"@maven//:io_grpc_grpc_api",
24+
"@maven//:io_grpc_grpc_context",
25+
"@maven//:io_grpc_grpc_core",
26+
"@maven//:io_grpc_grpc_protobuf",
27+
"@maven//:org_mockito_mockito_core",
28+
"@maven//:org_projectlombok_lombok",
29+
"@remote_apis//:build_bazel_remote_execution_v2_remote_execution_java_proto",
30+
],
31+
)

‎src/test/java/build/buildfarm/worker/persistent/PersistentExecutorTest.java‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
package build.buildfarm.worker.persistent;
2+
3+
import org.junit.Test;
4+
import org.junit.runner.RunWith;
5+
import org.junit.runners.JUnit4;
6+
7+
@RunWith(JUnit4.class)
8+
public class PersistentExecutorTest {
9+
@Test
10+
public void testProtoCoordinatorCreatesDirs() throws Exception {
11+
assert (true);
12+
}
13+
}

‎src/test/java/build/buildfarm/worker/util/BUILD‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
java_test(
2+
name = "tests",
3+
size = "small",
4+
srcs = glob(["*.java"]),
5+
test_class = "build.buildfarm.AllTests",
6+
deps = [
7+
"//src/main/java/build/buildfarm/cas",
8+
"//src/main/java/build/buildfarm/common",
9+
"//src/main/java/build/buildfarm/common/config",
10+
"//src/main/java/build/buildfarm/worker/util",
11+
"//src/main/protobuf:build_buildfarm_v1test_buildfarm_java_proto",
12+
"//src/test/java/build/buildfarm:test_runner",
13+
"@bazel_tools//src/main/protobuf:worker_protocol_java_proto",
14+
"@googleapis//:google_rpc_code_java_proto",
15+
"@maven//:com_github_jnr_jnr_constants",
16+
"@maven//:com_github_jnr_jnr_ffi",
17+
"@maven//:com_github_serceman_jnr_fuse",
18+
"@maven//:com_google_guava_guava",
19+
"@maven//:com_google_jimfs_jimfs",
20+
"@maven//:com_google_protobuf_protobuf_java",
21+
"@maven//:com_google_truth_truth",
22+
"@maven//:io_grpc_grpc_api",
23+
"@maven//:io_grpc_grpc_context",
24+
"@maven//:io_grpc_grpc_core",
25+
"@maven//:io_grpc_grpc_protobuf",
26+
"@maven//:org_mockito_mockito_core",
27+
"@maven//:org_projectlombok_lombok",
28+
"@remote_apis//:build_bazel_remote_execution_v2_remote_execution_java_proto",
29+
],
30+
)

‎src/test/java/build/buildfarm/worker/util/InputsIndexerTest.java‎

Lines changed: 163 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,163 @@
1+
package build.buildfarm.worker.util;
2+
3+
import static build.buildfarm.worker.util.InputsIndexer.BAZEL_TOOL_INPUT_MARKER;
4+
import static com.google.common.truth.Truth.assertThat;
5+
6+
import build.bazel.remote.execution.v2.*;
7+
import build.buildfarm.common.DigestUtil;
8+
import build.buildfarm.v1test.Tree;
9+
import com.google.common.collect.ImmutableMap;
10+
import com.google.devtools.build.lib.worker.WorkerProtocol.Input;
11+
import com.google.protobuf.ByteString;
12+
import java.nio.file.Path;
13+
import java.nio.file.Paths;
14+
import java.util.stream.Collectors;
15+
import org.junit.Test;
16+
import org.junit.runner.RunWith;
17+
import org.junit.runners.JUnit4;
18+
19+
@RunWith(JUnit4.class)
20+
public class InputsIndexerTest {
21+
private final DigestUtil DIGEST_UTIL = new DigestUtil(DigestUtil.HashFunction.SHA256);
22+
23+
@Test
24+
public void basicEmptyTree() {
25+
Tree emptyTree = Tree.newBuilder().build();
26+
InputsIndexer indexer = new InputsIndexer(emptyTree);
27+
assertThat(indexer.tree).isEqualTo(emptyTree);
28+
}
29+
30+
@Test
31+
public void canGetRootDir() {
32+
Tree.Builder treeBuilder = Tree.newBuilder();
33+
34+
Directory rootDir = Directory.getDefaultInstance();
35+
Digest rootDirDigest = addDirToTree(treeBuilder, "my_root_dir", rootDir);
36+
treeBuilder.setRootDigest(rootDirDigest);
37+
38+
InputsIndexer indexer = new InputsIndexer(treeBuilder.build());
39+
assertThat(indexer.proxyDirs.get(rootDirDigest)).isEqualTo(rootDir);
40+
41+
Path arbitraryOpRoot = Paths.get(".");
42+
assertThat(indexer.getAllInputs(arbitraryOpRoot).size()).isEqualTo(0);
43+
}
44+
45+
@Test
46+
public void rootDirWithFiles() {
47+
Tree.Builder treeBuilder = Tree.newBuilder();
48+
49+
FileNode myfile =
50+
makeFileNode("my_file", "my file contents", NodeProperties.getDefaultInstance());
51+
Directory rootDir = Directory.newBuilder().addFiles(myfile).build();
52+
Digest rootDirDigest = addDirToTree(treeBuilder, "my_root_dir", rootDir);
53+
treeBuilder.setRootDigest(rootDirDigest);
54+
55+
InputsIndexer indexer = new InputsIndexer(treeBuilder.build());
56+
assertThat(indexer.proxyDirs.get(rootDirDigest)).isEqualTo(rootDir);
57+
58+
Path arbitraryOpRoot = Paths.get("asdf");
59+
Input myfileInput = makeInput(arbitraryOpRoot, myfile);
60+
61+
ImmutableMap<Path, Input> expectedInputs =
62+
ImmutableMap.of(Paths.get(myfileInput.getPath()), myfileInput);
63+
64+
assertThat(indexer.getAllInputs(arbitraryOpRoot)).isEqualTo(expectedInputs);
65+
}
66+
67+
@Test
68+
public void canRecurseAndDistinguishToolInputs() {
69+
Tree.Builder treeBuilder = Tree.newBuilder();
70+
71+
FileNode myfile =
72+
makeFileNode("my_file", "my file contents", NodeProperties.getDefaultInstance());
73+
FileNode subdirfile =
74+
makeFileNode("subdir_file", "my subdir file contents", NodeProperties.getDefaultInstance());
75+
FileNode toolfile =
76+
makeFileNode(
77+
"tool_file",
78+
"my tool file contents",
79+
makeNodeProperties(ImmutableMap.of(BAZEL_TOOL_INPUT_MARKER, "value doesn't matter")));
80+
81+
Directory subDir = Directory.newBuilder().addFiles(subdirfile).build();
82+
String subDirName = "my_sub_dir";
83+
Digest subDirDigest = addDirToTree(treeBuilder, subDirName, subDir);
84+
85+
Directory rootDir =
86+
Directory.newBuilder()
87+
.addFiles(myfile)
88+
.addFiles(toolfile)
89+
.addDirectories(makeDirNode(subDirName, subDirDigest))
90+
.build();
91+
92+
Digest rootDirDigest = addDirToTree(treeBuilder, "my_root_dir", rootDir);
93+
treeBuilder.setRootDigest(rootDirDigest);
94+
95+
InputsIndexer indexer = new InputsIndexer(treeBuilder.build());
96+
assertThat(indexer.proxyDirs.get(rootDirDigest)).isEqualTo(rootDir);
97+
assertThat(indexer.proxyDirs.size()).isEqualTo(2);
98+
99+
Path arbitraryOpRoot = Paths.get("asdf");
100+
Input myfileInput = makeInput(arbitraryOpRoot, myfile);
101+
Input subdirfileInput = makeInput(arbitraryOpRoot.resolve(subDirName), subdirfile);
102+
Input toolfileInput = makeInput(arbitraryOpRoot, toolfile);
103+
104+
ImmutableMap<Path, Input> nonToolInputs =
105+
ImmutableMap.of(
106+
Paths.get(myfileInput.getPath()),
107+
myfileInput,
108+
Paths.get(subdirfileInput.getPath()),
109+
subdirfileInput);
110+
ImmutableMap<Path, Input> toolInputs =
111+
ImmutableMap.of(Paths.get(toolfileInput.getPath()), toolfileInput);
112+
ImmutableMap<Path, Input> allInputs =
113+
ImmutableMap.<Path, Input>builder().putAll(nonToolInputs).putAll(toolInputs).build();
114+
115+
assertThat(indexer.getAllInputs(arbitraryOpRoot)).isEqualTo(allInputs);
116+
assertThat(indexer.getAllInputs(arbitraryOpRoot).size()).isEqualTo(3);
117+
assertThat(indexer.getToolInputs(arbitraryOpRoot)).isEqualTo(toolInputs);
118+
}
119+
120+
Digest addDirToTree(Tree.Builder treeBuilder, String dirname, Directory dir) {
121+
ByteString dirnameBytes = ByteString.copyFromUtf8(dirname);
122+
Digest digest = DIGEST_UTIL.compute(dirnameBytes);
123+
String hash = digest.getHash();
124+
treeBuilder.putDirectories(hash, dir);
125+
return digest;
126+
}
127+
128+
FileNode makeFileNode(String filename, String content, NodeProperties nodeProperties) {
129+
return FileNode.newBuilder()
130+
.setName(filename)
131+
.setDigest(DIGEST_UTIL.compute(ByteString.copyFromUtf8(content)))
132+
.setIsExecutable(false)
133+
.setNodeProperties(nodeProperties)
134+
.build();
135+
}
136+
137+
DirectoryNode makeDirNode(String dirname, Digest dirDigest) {
138+
// Pretty sure we don't need the actual hash for our testing purposes
139+
return DirectoryNode.newBuilder().setName(dirname).setDigest(dirDigest).build();
140+
}
141+
142+
NodeProperties makeNodeProperties(ImmutableMap<String, String> props) {
143+
return NodeProperties.newBuilder()
144+
.addAllProperties(
145+
props.entrySet().stream()
146+
.map(
147+
kv ->
148+
NodeProperty.newBuilder()
149+
.setName(kv.getKey())
150+
.setValue(kv.getValue())
151+
.build())
152+
.collect(Collectors.toList()))
153+
.build();
154+
}
155+
156+
Input makeInput(Path fileDir, FileNode file) {
157+
Path fileNodePath = fileDir.resolve(file.getName());
158+
return Input.newBuilder()
159+
.setPath(fileNodePath.toString())
160+
.setDigest(file.getDigest().getHashBytes())
161+
.build();
162+
}
163+
}

0 commit comments

Comments
 (0)