Skip to content

Commit accce04

Browse files
authored
[Kernel][Metric]Rename snapshot metric following same naming pattern (delta-io#4859)
<!-- Thanks for sending a pull request! Here are some tips for you: 1. If this is your first time, please read our contributor guidelines: https://github.com/delta-io/delta/blob/master/CONTRIBUTING.md 2. If the PR is unfinished, add '[WIP]' in your PR title, e.g., '[WIP] Your PR title ...'. 3. Be sure to keep the PR description updated to reflect all changes. 4. Please write your PR title to summarize what this PR proposes. 5. If possible, provide a concise example to reproduce the issue for a faster review. 6. If applicable, include the corresponding issue number in the PR title and link it in the body. --> #### Which Delta project/connector is this regarding? <!-- Please add the component selected below to the beginning of the pull request title For example: [Spark] Title of my pull request --> - [ ] Spark - [ ] Standalone - [ ] Flink - [x] Kernel - [ ] Other (fill in here) ## Description <!-- - Describe what this PR changes. - Describe why we need the change. If this PR resolves an issue be sure to include "Resolves #XXX" to correctly link and close the issue upon merge. --> Rename snapshot metric following same naming pattern -- Load_what_[total/listing/etc]_Duration ## How was this patch tested? <!-- If tests were added, say they were added here. Please make sure to test the changes thoroughly including negative and positive cases if possible. If the changes were tested in any way other than unit tests, please clarify how you tested step by step (ideally copy and paste-able, so that other reviewers can test and check, and descendants can verify in the future). If the changes were not tested, please explain why. --> Existing tests are updated ## Does this PR introduce _any_ user-facing changes? <!-- If yes, please clarify the previous behavior and the change this PR proposes - provide the console output, description and/or an example to show the behavior difference if possible. If possible, please also clarify if this is a user-facing change compared to the released Delta Lake versions or within the unreleased branches such as master. If no, write 'No'. -->
1 parent 911f504 commit accce04

File tree

6 files changed

+91
-87
lines changed

6 files changed

+91
-87
lines changed

kernel/kernel-api/src/main/java/io/delta/kernel/internal/metrics/SnapshotMetrics.java

Lines changed: 28 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -30,48 +30,48 @@ public class SnapshotMetrics {
3030

3131
public final Timer loadSnapshotTotalTimer = new Timer();
3232

33-
public final Timer timestampToVersionResolutionTimer = new Timer();
33+
public final Timer computeTimestampToVersionTotalDurationTimer = new Timer();
3434

35-
public final Timer loadInitialDeltaActionsTimer = new Timer();
35+
public final Timer loadProtocolMetadataTotalDurationTimer = new Timer();
3636

37-
public final Timer timeToBuildLogSegmentForVersionTimer = new Timer();
37+
public final Timer loadLogSegmentTotalDurationTimer = new Timer();
3838

39-
public final Timer durationToGetCrcInfoTimer = new Timer();
39+
public final Timer loadCrcTotalDurationTimer = new Timer();
4040

4141
public SnapshotMetricsResult captureSnapshotMetricsResult() {
4242
return new SnapshotMetricsResult() {
43+
final Optional<Long> computeTimestampToVersionTotalDurationResult =
44+
computeTimestampToVersionTotalDurationTimer.totalDurationIfRecorded();
4345
final long loadSnapshotTotalDurationResult = loadSnapshotTotalTimer.totalDurationNs();
44-
final Optional<Long> timestampToVersionResolutionDurationResult =
45-
timestampToVersionResolutionTimer.totalDurationIfRecorded();
46-
final long loadInitialDeltaActionsDurationResult =
47-
loadInitialDeltaActionsTimer.totalDurationNs();
48-
final long timeToBuildLogSegmentForVersionDurationResult =
49-
timeToBuildLogSegmentForVersionTimer.totalDurationNs();
50-
final long durationToGetCrcInfoDurationResult = durationToGetCrcInfoTimer.totalDurationNs();
46+
final long loadProtocolMetadataTotalDurationResult =
47+
loadProtocolMetadataTotalDurationTimer.totalDurationNs();
48+
final long loadLogSegmentTotalDurationResult =
49+
loadLogSegmentTotalDurationTimer.totalDurationNs();
50+
final long loadCrcTotalDurationResult = loadCrcTotalDurationTimer.totalDurationNs();
5151

5252
@Override
53-
public long getLoadSnapshotTotalDurationNs() {
54-
return loadSnapshotTotalDurationResult;
53+
public Optional<Long> getComputeTimestampToVersionTotalDurationNs() {
54+
return computeTimestampToVersionTotalDurationResult;
5555
}
5656

5757
@Override
58-
public Optional<Long> getTimestampToVersionResolutionDurationNs() {
59-
return timestampToVersionResolutionDurationResult;
58+
public long getLoadSnapshotTotalDurationNs() {
59+
return loadSnapshotTotalDurationResult;
6060
}
6161

6262
@Override
63-
public long getLoadInitialDeltaActionsDurationNs() {
64-
return loadInitialDeltaActionsDurationResult;
63+
public long getLoadProtocolMetadataTotalDurationNs() {
64+
return loadProtocolMetadataTotalDurationResult;
6565
}
6666

6767
@Override
68-
public long getTimeToBuildLogSegmentForVersionNs() {
69-
return timeToBuildLogSegmentForVersionDurationResult;
68+
public long getLoadLogSegmentTotalDurationNs() {
69+
return loadLogSegmentTotalDurationResult;
7070
}
7171

7272
@Override
73-
public long getDurationToGetCrcInfoNs() {
74-
return durationToGetCrcInfoDurationResult;
73+
public long getLoadCrcTotalDurationNs() {
74+
return loadCrcTotalDurationResult;
7575
}
7676
};
7777
}
@@ -80,15 +80,15 @@ public long getDurationToGetCrcInfoNs() {
8080
public String toString() {
8181
return String.format(
8282
"SnapshotMetrics("
83+
+ "computeTimestampToVersionTotalDurationTimer=%s, "
8384
+ "loadSnapshotTotalTimer=%s,"
84-
+ "timestampToVersionResolutionTimer=%s, "
85-
+ "loadInitialDeltaActionsTimer=%s, "
85+
+ "loadProtocolMetadataTotalDurationTimer=%s, "
8686
+ "timeToBuildLogSegmentForVersionTimer=%s, "
87-
+ "durationToGetCrcInfoTimer=%s)",
87+
+ "loadCrcTotalDurationNsTimer=%s)",
88+
computeTimestampToVersionTotalDurationTimer,
8889
loadSnapshotTotalTimer,
89-
timestampToVersionResolutionTimer,
90-
loadInitialDeltaActionsTimer,
91-
timeToBuildLogSegmentForVersionTimer,
92-
durationToGetCrcInfoTimer);
90+
loadProtocolMetadataTotalDurationTimer,
91+
loadLogSegmentTotalDurationTimer,
92+
loadCrcTotalDurationTimer);
9393
}
9494
}

kernel/kernel-api/src/main/java/io/delta/kernel/internal/replay/LogReplay.java

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -258,7 +258,7 @@ private Lazy<Tuple2<Protocol, Metadata>> createLazyProtocolAndMetadata(
258258
return new Lazy<>(
259259
() -> {
260260
final Tuple2<Protocol, Metadata> result =
261-
snapshotMetrics.loadInitialDeltaActionsTimer.time(
261+
snapshotMetrics.loadProtocolMetadataTotalDurationTimer.time(
262262
() -> {
263263
final long targetVersion = getVersion();
264264

@@ -285,7 +285,7 @@ private Lazy<Tuple2<Protocol, Metadata>> createLazyProtocolAndMetadata(
285285
logger.info(
286286
"[{}] Took {}ms to load Protocol and Metadata at version {}",
287287
dataPath.toString(),
288-
snapshotMetrics.loadInitialDeltaActionsTimer.totalDurationMs(),
288+
snapshotMetrics.loadProtocolMetadataTotalDurationTimer.totalDurationMs(),
289289
getVersion());
290290

291291
return result;
@@ -611,7 +611,7 @@ public Optional<SnapshotHint> maybeGetNewerSnapshotHintAndUpdateCache(
611611
FileNames.getFileVersion(new Path(checksum.getPath())) >= crcReadLowerBound)
612612
.flatMap(
613613
checksum ->
614-
snapshotMetrics.durationToGetCrcInfoTimer.time(
614+
snapshotMetrics.loadCrcTotalDurationTimer.time(
615615
() -> ChecksumReader.getCRCInfo(engine, checksum)));
616616

617617
if (!crcInfoOpt.isPresent()) {

kernel/kernel-api/src/main/java/io/delta/kernel/internal/snapshot/SnapshotManager.java

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,7 @@ public SnapshotImpl buildLatestSnapshot(Engine engine, SnapshotQueryContext snap
8282
final LogSegment logSegment =
8383
snapshotContext
8484
.getSnapshotMetrics()
85-
.timeToBuildLogSegmentForVersionTimer
85+
.loadLogSegmentTotalDurationTimer
8686
.time(() -> getLogSegmentForVersion(engine, Optional.empty() /* versionToLoad */));
8787
snapshotContext.setVersion(logSegment.getVersion());
8888
snapshotContext.setCheckpointVersion(logSegment.getCheckpointVersionOpt());
@@ -105,7 +105,7 @@ public SnapshotImpl getSnapshotAt(
105105
final LogSegment logSegment =
106106
snapshotContext
107107
.getSnapshotMetrics()
108-
.timeToBuildLogSegmentForVersionTimer
108+
.loadLogSegmentTotalDurationTimer
109109
.time(
110110
() -> getLogSegmentForVersion(engine, Optional.of(version) /* versionToLoadOpt */));
111111

@@ -134,7 +134,7 @@ public SnapshotImpl getSnapshotForTimestamp(
134134
long versionToRead =
135135
snapshotContext
136136
.getSnapshotMetrics()
137-
.timestampToVersionResolutionTimer
137+
.computeTimestampToVersionTotalDurationTimer
138138
.time(
139139
() ->
140140
DeltaHistoryManager.getActiveCommitAtTimestamp(
@@ -149,7 +149,10 @@ public SnapshotImpl getSnapshotForTimestamp(
149149
logger.info(
150150
"{}: Took {} ms to fetch version at timestamp {}",
151151
tablePath,
152-
snapshotContext.getSnapshotMetrics().timestampToVersionResolutionTimer.totalDurationMs(),
152+
snapshotContext
153+
.getSnapshotMetrics()
154+
.computeTimestampToVersionTotalDurationTimer
155+
.totalDurationMs(),
153156
millisSinceEpochUTC);
154157

155158
return getSnapshotAt(engine, versionToRead, snapshotContext);

kernel/kernel-api/src/main/java/io/delta/kernel/metrics/SnapshotMetricsResult.java

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -20,42 +20,42 @@
2020

2121
/** Stores the metrics results for a {@link SnapshotReport} */
2222
@JsonPropertyOrder({
23+
"computeTimestampToVersionTotalDurationNs",
2324
"loadSnapshotTotalDurationNs",
24-
"timestampToVersionResolutionDurationNs",
25-
"loadInitialDeltaActionsDurationNs",
26-
"timeToBuildLogSegmentForVersionNs",
27-
"durationToGetCrcInfoNs"
25+
"loadProtocolMetadataTotalDurationNs",
26+
"loadLogSegmentTotalDurationNs",
27+
"loadCrcTotalDurationNs"
2828
})
2929
public interface SnapshotMetricsResult {
3030

31+
/**
32+
* @return the duration (ns) to resolve the provided timestamp to a table version for timestamp
33+
* time-travel queries. Empty for time-travel by version or non-time-travel queries.
34+
*/
35+
Optional<Long> getComputeTimestampToVersionTotalDurationNs();
36+
3137
/**
3238
* @return the total duration (ns) to load the snapshot, including all steps such as resolving
3339
* timestamp to version, LISTing the _delta_log, building the log segment, and determining the
3440
* latest protocol and metadata.
3541
*/
3642
long getLoadSnapshotTotalDurationNs();
3743

38-
/**
39-
* @return the duration (ns) to resolve the provided timestamp to a table version for timestamp
40-
* time-travel queries. Empty for time-travel by version or non-time-travel queries.
41-
*/
42-
Optional<Long> getTimestampToVersionResolutionDurationNs();
43-
4444
/**
4545
* @return the duration (ns) to load the initial delta actions for the snapshot (such as the table
4646
* protocol and metadata). 0 if snapshot construction fails before log replay.
4747
*/
48-
long getLoadInitialDeltaActionsDurationNs();
48+
long getLoadProtocolMetadataTotalDurationNs();
4949

5050
/**
5151
* @return the duration (ns) to build the log segment for the specified version during snapshot
5252
* construction. 0 if snapshot construction fails before this step.
5353
*/
54-
long getTimeToBuildLogSegmentForVersionNs();
54+
long getLoadLogSegmentTotalDurationNs();
5555

5656
/**
5757
* @return the duration (ns) to get CRC information during snapshot construction. 0 if snapshot
5858
* construction fails before this step or if CRC is not read in loading snapshot.
5959
*/
60-
long getDurationToGetCrcInfoNs();
60+
long getLoadCrcTotalDurationNs();
6161
}

kernel/kernel-api/src/test/scala/io/delta/kernel/internal/metrics/MetricsReportSerializerSuite.scala

Lines changed: 18 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -40,16 +40,16 @@ class MetricsReportSerializerSuite extends AnyFunSuite {
4040
}
4141

4242
private def testSnapshotReport(snapshotReport: SnapshotReport): Unit = {
43+
val computeTimestampToVersionTotalDuration = optionToString(
44+
snapshotReport.getSnapshotMetrics().getComputeTimestampToVersionTotalDurationNs())
4345
val loadSnapshotTotalDuration =
4446
snapshotReport.getSnapshotMetrics().getLoadSnapshotTotalDurationNs()
45-
val timestampToVersionResolutionDuration = optionToString(
46-
snapshotReport.getSnapshotMetrics().getTimestampToVersionResolutionDurationNs())
4747
val loadProtocolAndMetadataDuration =
48-
snapshotReport.getSnapshotMetrics().getLoadInitialDeltaActionsDurationNs()
48+
snapshotReport.getSnapshotMetrics().getLoadProtocolMetadataTotalDurationNs()
4949
val buildLogSegmentDuration =
50-
snapshotReport.getSnapshotMetrics().getTimeToBuildLogSegmentForVersionNs()
51-
val durationToGetCrcInfo =
52-
snapshotReport.getSnapshotMetrics().getDurationToGetCrcInfoNs()
50+
snapshotReport.getSnapshotMetrics().getLoadLogSegmentTotalDurationNs()
51+
val loadCrcTotalDuration =
52+
snapshotReport.getSnapshotMetrics().getLoadCrcTotalDurationNs()
5353
val exception: Optional[String] = snapshotReport.getException().map(_.toString)
5454
val expectedJson =
5555
s"""
@@ -61,11 +61,11 @@ class MetricsReportSerializerSuite extends AnyFunSuite {
6161
|"checkpointVersion":${optionToString(snapshotReport.getCheckpointVersion())},
6262
|"providedTimestamp":${optionToString(snapshotReport.getProvidedTimestamp())},
6363
|"snapshotMetrics":{
64+
|"computeTimestampToVersionTotalDurationNs":${computeTimestampToVersionTotalDuration},
6465
|"loadSnapshotTotalDurationNs":${loadSnapshotTotalDuration},
65-
|"timestampToVersionResolutionDurationNs":${timestampToVersionResolutionDuration},
66-
|"loadInitialDeltaActionsDurationNs":${loadProtocolAndMetadataDuration},
67-
|"timeToBuildLogSegmentForVersionNs":${buildLogSegmentDuration},
68-
|"durationToGetCrcInfoNs":${durationToGetCrcInfo}
66+
|"loadProtocolMetadataTotalDurationNs":${loadProtocolAndMetadataDuration},
67+
|"loadLogSegmentTotalDurationNs":${buildLogSegmentDuration},
68+
|"loadCrcTotalDurationNs":${loadCrcTotalDuration}
6969
|}
7070
|}
7171
|""".stripMargin.replaceAll("\n", "")
@@ -74,11 +74,11 @@ class MetricsReportSerializerSuite extends AnyFunSuite {
7474

7575
test("SnapshotReport serializer") {
7676
val snapshotContext1 = SnapshotQueryContext.forTimestampSnapshot("/table/path", 0)
77+
snapshotContext1.getSnapshotMetrics.computeTimestampToVersionTotalDurationTimer.record(10)
7778
snapshotContext1.getSnapshotMetrics.loadSnapshotTotalTimer.record(2000)
78-
snapshotContext1.getSnapshotMetrics.timestampToVersionResolutionTimer.record(10)
79-
snapshotContext1.getSnapshotMetrics.loadInitialDeltaActionsTimer.record(1000)
80-
snapshotContext1.getSnapshotMetrics.timeToBuildLogSegmentForVersionTimer.record(500)
81-
snapshotContext1.getSnapshotMetrics.durationToGetCrcInfoTimer.record(250)
79+
snapshotContext1.getSnapshotMetrics.loadProtocolMetadataTotalDurationTimer.record(1000)
80+
snapshotContext1.getSnapshotMetrics.loadLogSegmentTotalDurationTimer.record(500)
81+
snapshotContext1.getSnapshotMetrics.loadCrcTotalDurationTimer.record(250)
8282
snapshotContext1.setVersion(25)
8383
snapshotContext1.setCheckpointVersion(Optional.of(20))
8484
val exception = new RuntimeException("something something failed")
@@ -98,11 +98,11 @@ class MetricsReportSerializerSuite extends AnyFunSuite {
9898
|"checkpointVersion":20,
9999
|"providedTimestamp":0,
100100
|"snapshotMetrics":{
101+
|"computeTimestampToVersionTotalDurationNs":10,
101102
|"loadSnapshotTotalDurationNs":2000,
102-
|"timestampToVersionResolutionDurationNs":10,
103-
|"loadInitialDeltaActionsDurationNs":1000,
104-
|"timeToBuildLogSegmentForVersionNs":500,
105-
|"durationToGetCrcInfoNs":250
103+
|"loadProtocolMetadataTotalDurationNs":1000,
104+
|"loadLogSegmentTotalDurationNs":500,
105+
|"loadCrcTotalDurationNs":250
106106
|}
107107
|}
108108
|""".stripMargin.replaceAll("\n", "")

kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/metrics/SnapshotReportSuite.scala

Lines changed: 22 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -129,42 +129,43 @@ class SnapshotReportSuite extends AnyFunSuite with MetricsReportTestUtils {
129129
assert(metrics.getLoadSnapshotTotalDurationNs >= 0)
130130
}
131131

132-
// ===== Metric: getTimestampToVersionResolutionDurationNs =====
132+
// ===== Metric: getComputeTimestampToVersionTotalDurationNs =====
133133
if (expectations.expectNonEmptyTimestampToVersionResolutionDuration) {
134-
assert(metrics.getTimestampToVersionResolutionDurationNs.isPresent)
135-
assert(metrics.getTimestampToVersionResolutionDurationNs.get > 0)
136-
assert(metrics.getTimestampToVersionResolutionDurationNs.get < duration)
137-
assert(metrics.getTimestampToVersionResolutionDurationNs.get <=
134+
assert(metrics.getComputeTimestampToVersionTotalDurationNs.isPresent)
135+
assert(metrics.getComputeTimestampToVersionTotalDurationNs.get > 0)
136+
assert(metrics.getComputeTimestampToVersionTotalDurationNs.get < duration)
137+
assert(metrics.getComputeTimestampToVersionTotalDurationNs.get <=
138138
metrics.getLoadSnapshotTotalDurationNs)
139139
} else {
140-
assert(!metrics.getTimestampToVersionResolutionDurationNs.isPresent)
140+
assert(!metrics.getComputeTimestampToVersionTotalDurationNs.isPresent)
141141
}
142142

143-
// ===== Metric: getLoadInitialDeltaActionsDurationNs =====
143+
// ===== Metric: getLoadProtocolMetadataTotalDurationNs =====
144144
if (expectations.expectNonZeroLoadProtocolAndMetadataDuration) {
145-
assert(metrics.getLoadInitialDeltaActionsDurationNs > 0)
146-
assert(metrics.getLoadInitialDeltaActionsDurationNs < duration)
147-
assert(metrics.getLoadInitialDeltaActionsDurationNs <= metrics.getLoadSnapshotTotalDurationNs)
145+
assert(metrics.getLoadProtocolMetadataTotalDurationNs > 0)
146+
assert(metrics.getLoadProtocolMetadataTotalDurationNs < duration)
147+
assert(
148+
metrics.getLoadProtocolMetadataTotalDurationNs <= metrics.getLoadSnapshotTotalDurationNs)
148149
} else {
149-
assert(metrics.getLoadInitialDeltaActionsDurationNs == 0)
150+
assert(metrics.getLoadProtocolMetadataTotalDurationNs == 0)
150151
}
151152

152-
// ===== Metric: getTimeToBuildLogSegmentForVersionNs =====
153+
// ===== Metric: getLoadLogSegmentTotalDurationNs =====
153154
if (expectations.expectNonZeroBuildLogSegmentDuration) {
154-
assert(metrics.getTimeToBuildLogSegmentForVersionNs > 0)
155-
assert(metrics.getTimeToBuildLogSegmentForVersionNs < duration)
156-
assert(metrics.getTimeToBuildLogSegmentForVersionNs <= metrics.getLoadSnapshotTotalDurationNs)
155+
assert(metrics.getLoadLogSegmentTotalDurationNs > 0)
156+
assert(metrics.getLoadLogSegmentTotalDurationNs < duration)
157+
assert(metrics.getLoadLogSegmentTotalDurationNs <= metrics.getLoadSnapshotTotalDurationNs)
157158
} else {
158-
assert(metrics.getTimeToBuildLogSegmentForVersionNs == 0)
159+
assert(metrics.getLoadLogSegmentTotalDurationNs == 0)
159160
}
160161

161-
// ===== Metric: getDurationToGetCrcInfoNs =====
162+
// ===== Metric: getLoadCrcTotalDurationNs =====
162163
if (expectations.expectNonZeroDurationToGetCrcInfo) {
163-
assert(metrics.getDurationToGetCrcInfoNs > 0)
164-
assert(metrics.getDurationToGetCrcInfoNs < duration)
165-
assert(metrics.getDurationToGetCrcInfoNs <= metrics.getLoadSnapshotTotalDurationNs)
164+
assert(metrics.getLoadCrcTotalDurationNs > 0)
165+
assert(metrics.getLoadCrcTotalDurationNs < duration)
166+
assert(metrics.getLoadCrcTotalDurationNs <= metrics.getLoadSnapshotTotalDurationNs)
166167
} else {
167-
assert(metrics.getDurationToGetCrcInfoNs == 0)
168+
assert(metrics.getLoadCrcTotalDurationNs == 0)
168169
}
169170
}
170171

0 commit comments

Comments
 (0)