Skip to content

Commit caba0d6

Browse files
committed
improve jitter config documentation
1 parent 5b49190 commit caba0d6

File tree

4 files changed

+16
-5
lines changed

4 files changed

+16
-5
lines changed

docs/layouts/shortcodes/generated/expert_rocksdb_section.html

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
<td><h5>state.backend.rocksdb.checkpoint.upload-jitter</h5></td>
1919
<td style="word-wrap: break-word;">0 ms</td>
2020
<td>Duration</td>
21-
<td>The time interval used to create jitter for each checkpoint file upload.</td>
21+
<td>The jitter time interval determines the amount of random delay introduced before uploading each checkpoint file. This jitter is applied to help stagger the upload times of state files, thereby introducing a small, randomized delay for each upload operation. The purpose of this mechanism is to avoid simultaneous upload bursts, which could otherwise overwhelm the remote storage system. By distributing the upload requests more evenly over time, jitter helps maintain system stability and prevents potential performance degradation during checkpoint completion. It is therefore recommended to configure the jitter interval at the seconds level, ensuring a balanced trade-off between randomness and upload efficiency.</td>
2222
</tr>
2323
<tr>
2424
<td><h5>state.backend.rocksdb.localdir</h5></td>

docs/layouts/shortcodes/generated/rocksdb_configuration.html

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
<td><h5>state.backend.rocksdb.checkpoint.upload-jitter</h5></td>
1919
<td style="word-wrap: break-word;">0 ms</td>
2020
<td>Duration</td>
21-
<td>The time interval used to create jitter for each checkpoint file upload.</td>
21+
<td>The jitter time interval determines the amount of random delay introduced before uploading each checkpoint file. This jitter is applied to help stagger the upload times of state files, thereby introducing a small, randomized delay for each upload operation. The purpose of this mechanism is to avoid simultaneous upload bursts, which could otherwise overwhelm the remote storage system. By distributing the upload requests more evenly over time, jitter helps maintain system stability and prevents potential performance degradation during checkpoint completion. It is therefore recommended to configure the jitter interval at the seconds level, ensuring a balanced trade-off between randomness and upload efficiency.</td>
2222
</tr>
2323
<tr>
2424
<td><h5>state.backend.rocksdb.localdir</h5></td>

flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/state/rocksdb/RocksDBOptions.java

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,13 @@ public class RocksDBOptions {
100100
.durationType()
101101
.defaultValue(Duration.ZERO)
102102
.withDescription(
103-
"The time interval used to create jitter for each checkpoint file upload.");
103+
"The jitter time interval determines the amount of random delay introduced before uploading each checkpoint file. "
104+
+ "This jitter is applied to help stagger the upload times of state files, thereby introducing a small, "
105+
+ "randomized delay for each upload operation. The purpose of this mechanism is to avoid simultaneous upload bursts, "
106+
+ "which could otherwise overwhelm the remote storage system. By distributing the upload requests more evenly over time, "
107+
+ "jitter helps maintain system stability and prevents potential performance degradation during checkpoint completion. "
108+
+ "It is therefore recommended to configure the jitter interval at the seconds level, ensuring a balanced trade-off "
109+
+ "between randomness and upload efficiency.");
104110

105111
/** The predefined settings for RocksDB DBOptions and ColumnFamilyOptions by Flink community. */
106112
@Documentation.Section(Documentation.Sections.EXPERT_ROCKSDB)

flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/state/rocksdb/RocksDBStateUploader.java

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@ public class RocksDBStateUploader implements Closeable {
5858
private final Duration uploadJitter;
5959
private final Random random;
6060
private final RocksDBStateDataTransferHelper transfer;
61+
private final JitterConsumer jitterConsumer;
6162

6263
@VisibleForTesting
6364
public RocksDBStateUploader(int numberOfSnapshottingThreads, Duration uploadJitter) {
@@ -69,6 +70,7 @@ public RocksDBStateUploader(int numberOfSnapshottingThreads, Duration uploadJitt
6970
public RocksDBStateUploader(RocksDBStateDataTransferHelper transfer, Duration uploadJitter) {
7071
this.transfer = transfer;
7172
this.uploadJitter = uploadJitter;
73+
this.jitterConsumer = new JitterConsumer();
7274
this.random = new Random();
7375
}
7476

@@ -152,7 +154,7 @@ private HandleAndLocalPath uploadLocalFileToCheckpointFs(
152154

153155
try {
154156
// add a random jitter
155-
applyJitter(new JitterConsumer());
157+
applyJitter(jitterConsumer);
156158
final byte[] buffer = new byte[READ_BUFFER_SIZE];
157159

158160
inputStream = Files.newInputStream(filePath);
@@ -211,7 +213,10 @@ public void accept(Long milliseconds) {
211213
try {
212214
Thread.sleep(milliseconds);
213215
} catch (InterruptedException e) {
214-
LOG.error("Fail to apply jitter in RocksDBStateUploader.", e);
216+
LOG.error(
217+
"Fail to apply {} milliseconds jitter in RocksDBStateUploader.",
218+
milliseconds,
219+
e);
215220
}
216221
}
217222
}

0 commit comments

Comments
 (0)