Skip to content

Commit 0c16de2

Browse files
committed
Don't keep the journal of a PersistentMap open
This ensures that the journal file is not kept open after a build, which has been observed to cause `RewindingTest` to fail on Windows, which disallows deleting open files (in this case during test case cleanup). Since the journal is written out at most every 3 seconds for all current usages of the `PersistentMap` class, the overhead of the additional `open` is negligible. Also include small tweaks to `RewindingTest` so that it can be enabled on Windows. In particular, the order of `SymlinkAction` and `SourceManifestAction` being rewound doesn't seem to be fixed and can differ on Windows. Closes #28108. PiperOrigin-RevId: 855242645 Change-Id: Ic7434139b290c6b0e2061977f747e890c3c5ece6 (cherry picked from commit 2be693e)
1 parent bd45e3c commit 0c16de2

File tree

3 files changed

+54
-56
lines changed

3 files changed

+54
-56
lines changed

src/main/java/com/google/devtools/build/lib/util/PersistentMap.java

Lines changed: 18 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,6 @@ public abstract class PersistentMap<K, V> extends ForwardingConcurrentMap<K, V>
8383
private final Path journalFile;
8484

8585
private final LinkedBlockingQueue<K> journal;
86-
private DataOutputStream journalOut;
8786

8887
/**
8988
* 'dirty' is true when the in-memory representation of the map is more recent than the on-disk
@@ -202,22 +201,24 @@ public V remove(Object object) {
202201
*/
203202
private synchronized void writeJournal() {
204203
try {
205-
if (journalOut == null) {
206-
if (journalFile.exists()) {
207-
// The journal file was left around after the last save() because
208-
// keepJournal() was true. Append to it.
209-
journalOut =
210-
new DataOutputStream(new BufferedOutputStream(journalFile.getOutputStream(true)));
211-
} else {
212-
// Create new journal.
213-
journalOut = createMapFile(journalFile);
214-
}
204+
DataOutputStream journalOut;
205+
if (journalFile.exists()) {
206+
// The journal file was left around after the last save() because
207+
// keepJournal() was true. Append to it.
208+
journalOut =
209+
new DataOutputStream(new BufferedOutputStream(journalFile.getOutputStream(true)));
210+
} else {
211+
// Create new journal.
212+
journalOut = createMapFile(journalFile);
213+
}
214+
try {
215+
// Journal may have duplicates, we can ignore them.
216+
LinkedHashSet<K> items = Sets.newLinkedHashSetWithExpectedSize(journal.size());
217+
journal.drainTo(items);
218+
writeEntries(journalOut, items, delegate());
219+
} finally {
220+
journalOut.close();
215221
}
216-
// Journal may have duplicates, we can ignore them.
217-
LinkedHashSet<K> items = Sets.newLinkedHashSetWithExpectedSize(journal.size());
218-
journal.drainTo(items);
219-
writeEntries(journalOut, items, delegate());
220-
journalOut.flush();
221222
} catch (IOException e) {
222223
this.deferredIOFailure = e.getMessage() + " during journal append";
223224
}
@@ -303,8 +304,6 @@ private synchronized long save(boolean fullSave) throws IOException {
303304
if (dirty) {
304305
if (!fullSave && keepJournal()) {
305306
forceFlush();
306-
journalOut.close();
307-
journalOut = null;
308307
return journalSize() + cacheSize();
309308
} else {
310309
dirty = false;
@@ -343,12 +342,8 @@ protected boolean keepJournal() {
343342
return false;
344343
}
345344

346-
private synchronized void clearJournal() throws IOException {
345+
private synchronized void clearJournal() {
347346
journal.clear();
348-
if (journalOut != null) {
349-
journalOut.close();
350-
journalOut = null;
351-
}
352347
}
353348

354349
private synchronized void loadEntries(Path mapFile, boolean failFast) throws IOException {

src/test/java/com/google/devtools/build/lib/skyframe/rewinding/BUILD

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,6 @@ java_test(
7575
srcs = ["RewindingTest.java"],
7676
jvm_flags = ["-Djava.lang.Thread.allowVirtualThreads=true"],
7777
shard_count = 12,
78-
tags = ["no_windows"], # BuildIntegrationTestCase isn't fully compatible with Windows.
7978
deps = [
8079
":rewinding_tests_helper",
8180
"//src/main/java/com/google/devtools/build/lib:runtime",

src/test/java/com/google/devtools/build/lib/skyframe/rewinding/RewindingTestsHelper.java

Lines changed: 36 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -1835,21 +1835,24 @@ final void runGeneratedRunfilesRewound(ImmutableList<String> lostRunfiles, Spawn
18351835
HashSet<String> expectedRewoundGenrules =
18361836
new HashSet<>(ImmutableList.of("//middle:gen1", "//middle:gen2"));
18371837
int i = 0;
1838+
boolean sourceManifestActionSeen = false;
18381839
while (i < 5) {
18391840
assertThat(rewoundKeys.get(i)).isInstanceOf(ActionLookupData.class);
18401841
ActionLookupData actionKey = (ActionLookupData) rewoundKeys.get(i);
18411842
String actionLabel = actionKey.getLabel().getCanonicalForm();
18421843
i++;
18431844
if (actionLabel.equals("//middle:tool")) {
18441845
switch (actionKey.getActionIndex()) {
1845-
case 0: // SymlinkAction
1846-
break;
1847-
case 1: // SourceManifestAction
1848-
assertActionKey(rewoundKeys.get(i), "//middle:tool", 2);
1849-
i++;
1850-
break;
1851-
default:
1852-
fail(String.format("Unexpected action index. actionKey: %s, i: %s", actionKey, i));
1846+
// SymlinkAction
1847+
case 0 -> {}
1848+
case 1 -> sourceManifestActionSeen = true;
1849+
// SymlinkTreeAction
1850+
case 2 -> assertThat(sourceManifestActionSeen).isTrue();
1851+
default ->
1852+
fail(
1853+
String.format(
1854+
"Unexpected action index. actionKey: %s, rewoundKeys: %s",
1855+
actionKey, rewoundKeys));
18531856
}
18541857
} else {
18551858
assertThat(expectedRewoundGenrules.remove(actionLabel)).isTrue();
@@ -2012,33 +2015,31 @@ final void runDupeDirectAndRunfilesDependencyRewound(
20122015

20132016
if (buildRunfileManifests()) {
20142017
assertThat(rewoundKeys).hasSize(6);
2015-
int i = 0;
2016-
while (i < 4) {
2018+
boolean sourceManifestActionSeen = false;
2019+
for (int i = 0; i < 4; i++) {
20172020
assertThat(rewoundKeys.get(i)).isInstanceOf(ActionLookupData.class);
20182021
ActionLookupData actionKey = (ActionLookupData) rewoundKeys.get(i);
20192022
String actionLabel = actionKey.getLabel().getCanonicalForm();
2020-
i++;
20212023
if (actionLabel.equals("//test:tool")) {
20222024
switch (actionKey.getActionIndex()) {
2023-
case 0: // SymlinkAction
2024-
break;
2025-
case 1: // SourceManifestAction
2026-
assertActionKey(rewoundKeys.get(i), "//test:tool", /* index= */ 2);
2027-
i++;
2028-
break;
2029-
default:
2030-
fail(
2031-
String.format(
2032-
"Unexpected action index. actionKey: %s, rewoundKeys: %s",
2033-
actionKey, rewoundKeys));
2025+
// SymlinkAction
2026+
case 0 -> {}
2027+
case 1 -> sourceManifestActionSeen = true;
2028+
// SymlinkTreeAction
2029+
case 2 -> assertThat(sourceManifestActionSeen).isTrue();
2030+
default ->
2031+
fail(
2032+
String.format(
2033+
"Unexpected action index. actionKey: %s, rewoundKeys: %s",
2034+
actionKey, rewoundKeys));
20342035
}
20352036
} else {
20362037
assertThat(actionLabel).isEqualTo("//test:rule1");
20372038
}
20382039
}
20392040

2040-
assertActionKey(rewoundKeys.get(i++), "//test:tool", /* index= */ 3);
2041-
assertArtifactKey(rewoundKeys.get(i), "_middlemen/test_Stool-runfiles");
2041+
assertActionKey(rewoundKeys.get(4), "//test:tool", /* index= */ 3);
2042+
assertArtifactKey(rewoundKeys.get(5), "_middlemen/test_Stool-runfiles");
20422043
} else {
20432044
assertThat(rewoundKeys).hasSize(4);
20442045
int i = 0;
@@ -2168,21 +2169,24 @@ def _tree_impl(ctx):
21682169
if (buildRunfileManifests()) {
21692170
assertThat(rewoundKeys).hasSize(7);
21702171
int i = 0;
2172+
boolean sourceManifestActionSeen = false;
21712173
while (i < 5) {
21722174
assertThat(rewoundKeys.get(i)).isInstanceOf(ActionLookupData.class);
21732175
ActionLookupData actionKey = (ActionLookupData) rewoundKeys.get(i);
21742176
String actionLabel = actionKey.getLabel().getCanonicalForm();
21752177
i++;
21762178
if (actionLabel.equals("//middle:tool")) {
21772179
switch (actionKey.getActionIndex()) {
2178-
case 0: // SymlinkAction
2179-
break;
2180-
case 1: // SourceManifestAction
2181-
assertActionKey(rewoundKeys.get(i), "//middle:tool", 2);
2182-
i++;
2183-
break;
2184-
default:
2185-
fail(String.format("Unexpected action index. actionKey: %s", actionKey));
2180+
// SymlinkAction
2181+
case 0 -> {}
2182+
case 1 -> sourceManifestActionSeen = true;
2183+
// SymlinkTreeAction
2184+
case 2 -> assertThat(sourceManifestActionSeen).isTrue();
2185+
default ->
2186+
fail(
2187+
String.format(
2188+
"Unexpected action index. actionKey: %s, rewoundKeys: %s",
2189+
actionKey, rewoundKeys));
21862190
}
21872191
} else {
21882192
assertThat(actionLabel).isEqualTo("//middle:gen_tree");

0 commit comments

Comments
 (0)