Skip to content

Commit f2d1481

Browse files
committed
Don't keep the journal of a PersistentMap open
This ensures that the journal file is not kept open after a build, which has been observed to cause `RewindingTest` to fail on Windows, which disallows deleting open files (in this case during test case cleanup). Since the journal is written out at most every 3 seconds for all current usages of the `PersistentMap` class, the overhead of the additional `open` is negligible. Also include small tweaks to `RewindingTest` so that it can be enabled on Windows. In particular, the order of `SymlinkAction` and `SourceManifestAction` being rewound doesn't seem to be fixed and can differ on Windows. Closes #28108. PiperOrigin-RevId: 855242645 Change-Id: Ic7434139b290c6b0e2061977f747e890c3c5ece6 (cherry picked from commit 2be693e)
1 parent 1c4243e commit f2d1481

File tree

3 files changed

+54
-56
lines changed

3 files changed

+54
-56
lines changed

src/main/java/com/google/devtools/build/lib/util/PersistentMap.java

Lines changed: 18 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,6 @@ public abstract class PersistentMap<K, V> extends ForwardingConcurrentMap<K, V>
8383
private final Path journalFile;
8484

8585
private final LinkedBlockingQueue<K> journal;
86-
private DataOutputStream journalOut;
8786

8887
/**
8988
* 'dirty' is true when the in-memory representation of the map is more recent than the on-disk
@@ -202,22 +201,24 @@ public V remove(Object object) {
202201
*/
203202
private synchronized void writeJournal() {
204203
try {
205-
if (journalOut == null) {
206-
if (journalFile.exists()) {
207-
// The journal file was left around after the last save() because
208-
// keepJournal() was true. Append to it.
209-
journalOut =
210-
new DataOutputStream(new BufferedOutputStream(journalFile.getOutputStream(true)));
211-
} else {
212-
// Create new journal.
213-
journalOut = createMapFile(journalFile);
214-
}
204+
DataOutputStream journalOut;
205+
if (journalFile.exists()) {
206+
// The journal file was left around after the last save() because
207+
// keepJournal() was true. Append to it.
208+
journalOut =
209+
new DataOutputStream(new BufferedOutputStream(journalFile.getOutputStream(true)));
210+
} else {
211+
// Create new journal.
212+
journalOut = createMapFile(journalFile);
213+
}
214+
try {
215+
// Journal may have duplicates, we can ignore them.
216+
LinkedHashSet<K> items = Sets.newLinkedHashSetWithExpectedSize(journal.size());
217+
journal.drainTo(items);
218+
writeEntries(journalOut, items, delegate());
219+
} finally {
220+
journalOut.close();
215221
}
216-
// Journal may have duplicates, we can ignore them.
217-
LinkedHashSet<K> items = Sets.newLinkedHashSetWithExpectedSize(journal.size());
218-
journal.drainTo(items);
219-
writeEntries(journalOut, items, delegate());
220-
journalOut.flush();
221222
} catch (IOException e) {
222223
this.deferredIOFailure = e.getMessage() + " during journal append";
223224
}
@@ -303,8 +304,6 @@ private synchronized long save(boolean fullSave) throws IOException {
303304
if (dirty) {
304305
if (!fullSave && keepJournal()) {
305306
forceFlush();
306-
journalOut.close();
307-
journalOut = null;
308307
return journalSize() + cacheSize();
309308
} else {
310309
dirty = false;
@@ -343,12 +342,8 @@ protected boolean keepJournal() {
343342
return false;
344343
}
345344

346-
private synchronized void clearJournal() throws IOException {
345+
private synchronized void clearJournal() {
347346
journal.clear();
348-
if (journalOut != null) {
349-
journalOut.close();
350-
journalOut = null;
351-
}
352347
}
353348

354349
private synchronized void loadEntries(Path mapFile, boolean failFast) throws IOException {

src/test/java/com/google/devtools/build/lib/skyframe/rewinding/BUILD

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,6 @@ java_test(
7575
srcs = ["RewindingTest.java"],
7676
jvm_flags = ["-Djava.lang.Thread.allowVirtualThreads=true"],
7777
shard_count = 12,
78-
tags = ["no_windows"], # BuildIntegrationTestCase isn't fully compatible with Windows.
7978
deps = [
8079
":rewinding_tests_helper",
8180
"//src/main/java/com/google/devtools/build/lib:runtime",

src/test/java/com/google/devtools/build/lib/skyframe/rewinding/RewindingTestsHelper.java

Lines changed: 36 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -1833,21 +1833,24 @@ final void runGeneratedRunfilesRewound(ImmutableList<String> lostRunfiles, Spawn
18331833
HashSet<String> expectedRewoundGenrules =
18341834
new HashSet<>(ImmutableList.of("//middle:gen1", "//middle:gen2"));
18351835
int i = 0;
1836+
boolean sourceManifestActionSeen = false;
18361837
while (i < 5) {
18371838
assertThat(rewoundKeys.get(i)).isInstanceOf(ActionLookupData.class);
18381839
ActionLookupData actionKey = (ActionLookupData) rewoundKeys.get(i);
18391840
String actionLabel = actionKey.getLabel().getCanonicalForm();
18401841
i++;
18411842
if (actionLabel.equals("//middle:tool")) {
18421843
switch (actionKey.getActionIndex()) {
1843-
case 0: // SymlinkAction
1844-
break;
1845-
case 1: // SourceManifestAction
1846-
assertActionKey(rewoundKeys.get(i), "//middle:tool", 2);
1847-
i++;
1848-
break;
1849-
default:
1850-
fail(String.format("Unexpected action index. actionKey: %s, i: %s", actionKey, i));
1844+
// SymlinkAction
1845+
case 0 -> {}
1846+
case 1 -> sourceManifestActionSeen = true;
1847+
// SymlinkTreeAction
1848+
case 2 -> assertThat(sourceManifestActionSeen).isTrue();
1849+
default ->
1850+
fail(
1851+
String.format(
1852+
"Unexpected action index. actionKey: %s, rewoundKeys: %s",
1853+
actionKey, rewoundKeys));
18511854
}
18521855
} else {
18531856
assertThat(expectedRewoundGenrules.remove(actionLabel)).isTrue();
@@ -2010,33 +2013,31 @@ final void runDupeDirectAndRunfilesDependencyRewound(
20102013

20112014
if (buildRunfileManifests()) {
20122015
assertThat(rewoundKeys).hasSize(6);
2013-
int i = 0;
2014-
while (i < 4) {
2016+
boolean sourceManifestActionSeen = false;
2017+
for (int i = 0; i < 4; i++) {
20152018
assertThat(rewoundKeys.get(i)).isInstanceOf(ActionLookupData.class);
20162019
ActionLookupData actionKey = (ActionLookupData) rewoundKeys.get(i);
20172020
String actionLabel = actionKey.getLabel().getCanonicalForm();
2018-
i++;
20192021
if (actionLabel.equals("//test:tool")) {
20202022
switch (actionKey.getActionIndex()) {
2021-
case 0: // SymlinkAction
2022-
break;
2023-
case 1: // SourceManifestAction
2024-
assertActionKey(rewoundKeys.get(i), "//test:tool", /* index= */ 2);
2025-
i++;
2026-
break;
2027-
default:
2028-
fail(
2029-
String.format(
2030-
"Unexpected action index. actionKey: %s, rewoundKeys: %s",
2031-
actionKey, rewoundKeys));
2023+
// SymlinkAction
2024+
case 0 -> {}
2025+
case 1 -> sourceManifestActionSeen = true;
2026+
// SymlinkTreeAction
2027+
case 2 -> assertThat(sourceManifestActionSeen).isTrue();
2028+
default ->
2029+
fail(
2030+
String.format(
2031+
"Unexpected action index. actionKey: %s, rewoundKeys: %s",
2032+
actionKey, rewoundKeys));
20322033
}
20332034
} else {
20342035
assertThat(actionLabel).isEqualTo("//test:rule1");
20352036
}
20362037
}
20372038

2038-
assertActionKey(rewoundKeys.get(i++), "//test:tool", /* index= */ 3);
2039-
assertArtifactKey(rewoundKeys.get(i), "_middlemen/test_Stool-runfiles");
2039+
assertActionKey(rewoundKeys.get(4), "//test:tool", /* index= */ 3);
2040+
assertArtifactKey(rewoundKeys.get(5), "_middlemen/test_Stool-runfiles");
20402041
} else {
20412042
assertThat(rewoundKeys).hasSize(4);
20422043
int i = 0;
@@ -2166,21 +2167,24 @@ def _tree_impl(ctx):
21662167
if (buildRunfileManifests()) {
21672168
assertThat(rewoundKeys).hasSize(7);
21682169
int i = 0;
2170+
boolean sourceManifestActionSeen = false;
21692171
while (i < 5) {
21702172
assertThat(rewoundKeys.get(i)).isInstanceOf(ActionLookupData.class);
21712173
ActionLookupData actionKey = (ActionLookupData) rewoundKeys.get(i);
21722174
String actionLabel = actionKey.getLabel().getCanonicalForm();
21732175
i++;
21742176
if (actionLabel.equals("//middle:tool")) {
21752177
switch (actionKey.getActionIndex()) {
2176-
case 0: // SymlinkAction
2177-
break;
2178-
case 1: // SourceManifestAction
2179-
assertActionKey(rewoundKeys.get(i), "//middle:tool", 2);
2180-
i++;
2181-
break;
2182-
default:
2183-
fail(String.format("Unexpected action index. actionKey: %s", actionKey));
2178+
// SymlinkAction
2179+
case 0 -> {}
2180+
case 1 -> sourceManifestActionSeen = true;
2181+
// SymlinkTreeAction
2182+
case 2 -> assertThat(sourceManifestActionSeen).isTrue();
2183+
default ->
2184+
fail(
2185+
String.format(
2186+
"Unexpected action index. actionKey: %s, rewoundKeys: %s",
2187+
actionKey, rewoundKeys));
21842188
}
21852189
} else {
21862190
assertThat(actionLabel).isEqualTo("//middle:gen_tree");

0 commit comments

Comments
 (0)