Skip to content

Commit d67d5d3

Browse files
edsavageedsavage
committed
Account for C++ code changes
1 parent d11e442 commit d67d5d3

File tree

5 files changed

+70
-31
lines changed

5 files changed

+70
-31
lines changed

‎server/src/main/java/org/elasticsearch/TransportVersions.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -214,7 +214,7 @@ static TransportVersion def(int id) {
214214
public static final TransportVersion ESQL_REMOVE_AGGREGATE_TYPE = def(9_045_0_00);
215215
public static final TransportVersion ADD_PROJECT_ID_TO_DSL_ERROR_INFO = def(9_046_0_00);
216216
public static final TransportVersion SEMANTIC_TEXT_CHUNKING_CONFIG = def(9_047_00_0);
217-
public static final TransportVersion ML_AD_ACTUAL_MEMORY_USAGE = def(9_048_0_00);
217+
public static final TransportVersion ML_AD_SYSTEM_MEMORY_USAGE = def(9_048_0_00);
218218

219219
/*
220220
* STOP! READ THIS FIRST! No, really,

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/ModelSizeStats.java‎

Lines changed: 50 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,8 @@ public class ModelSizeStats implements ToXContentObject, Writeable {
4141
*/
4242
public static final ParseField MODEL_BYTES_FIELD = new ParseField("model_bytes");
4343
public static final ParseField PEAK_MODEL_BYTES_FIELD = new ParseField("peak_model_bytes");
44-
public static final ParseField ACTUAL_MEMORY_USAGE_BYTES = new ParseField("actual_memory_usage_bytes");
44+
public static final ParseField SYSTEM_MEMORY_BYTES = new ParseField("system_memory_bytes");
45+
public static final ParseField MAX_SYSTEM_MEMORY_BYTES = new ParseField("max_system_memory_bytes");
4546
public static final ParseField MODEL_BYTES_EXCEEDED_FIELD = new ParseField("model_bytes_exceeded");
4647
public static final ParseField MODEL_BYTES_MEMORY_LIMIT_FIELD = new ParseField("model_bytes_memory_limit");
4748
public static final ParseField TOTAL_BY_FIELD_COUNT_FIELD = new ParseField("total_by_field_count");
@@ -75,7 +76,8 @@ private static ConstructingObjectParser<Builder, Void> createParser(boolean igno
7576
parser.declareString((modelSizeStat, s) -> {}, Result.RESULT_TYPE);
7677
parser.declareLong(Builder::setModelBytes, MODEL_BYTES_FIELD);
7778
parser.declareLong(Builder::setPeakModelBytes, PEAK_MODEL_BYTES_FIELD);
78-
parser.declareLong(Builder::setActualMemoryUsageBytes, ACTUAL_MEMORY_USAGE_BYTES);
79+
parser.declareLong(Builder::setSystemMemoryBytes, SYSTEM_MEMORY_BYTES);
80+
parser.declareLong(Builder::setMaxSystemMemoryBytes, MAX_SYSTEM_MEMORY_BYTES);
7981
parser.declareLong(Builder::setModelBytesExceeded, MODEL_BYTES_EXCEEDED_FIELD);
8082
parser.declareLong(Builder::setModelBytesMemoryLimit, MODEL_BYTES_MEMORY_LIMIT_FIELD);
8183
parser.declareLong(Builder::setBucketAllocationFailuresCount, BUCKET_ALLOCATION_FAILURES_COUNT_FIELD);
@@ -154,7 +156,8 @@ public String toString() {
154156
* 1. The job's model_memory_limit
155157
* 2. The current model memory, i.e. what's reported in model_bytes of this object
156158
* 3. The peak model memory, i.e. what's reported in peak_model_bytes of this object
157-
* 4. The actual memory usage, i.e. what's reported in actual_memory_usage_bytes of this object
159+
* 4. The system memory, i.e. what's reported in system_memory_bytes of this object
160+
* 5. The max system memory, i.e. what's reported in max_system_memory_bytes of this object
158161
* The field storing this enum can also be <code>null</code>, which means the
159162
* assignment code will decide on the fly - this was the old behaviour prior
160163
* to 7.11.
@@ -163,7 +166,8 @@ public enum AssignmentMemoryBasis implements Writeable {
163166
MODEL_MEMORY_LIMIT,
164167
CURRENT_MODEL_BYTES,
165168
PEAK_MODEL_BYTES,
166-
ACTUAL_MEMORY_USAGE_BYTES;
169+
SYSTEM_MEMORY_BYTES,
170+
MAX_SYSTEM_MEMORY_BYTES,;
167171

168172
public static AssignmentMemoryBasis fromString(String statusName) {
169173
return valueOf(statusName.trim().toUpperCase(Locale.ROOT));
@@ -187,7 +191,8 @@ public String toString() {
187191
private final String jobId;
188192
private final long modelBytes;
189193
private final Long peakModelBytes;
190-
private final Long actualMemoryUsageBytes;
194+
private final Long systemMemoryUsageBytes;
195+
private final Long maxSystemMemoryUsageBytes;
191196
private final Long modelBytesExceeded;
192197
private final Long modelBytesMemoryLimit;
193198
private final long totalByFieldCount;
@@ -211,7 +216,8 @@ private ModelSizeStats(
211216
String jobId,
212217
long modelBytes,
213218
Long peakModelBytes,
214-
Long actualMemoryUsageBytes,
219+
Long systemMemoryUsageBytes,
220+
Long maxSystemMemoryUsageBytes,
215221
Long modelBytesExceeded,
216222
Long modelBytesMemoryLimit,
217223
long totalByFieldCount,
@@ -234,7 +240,8 @@ private ModelSizeStats(
234240
this.jobId = jobId;
235241
this.modelBytes = modelBytes;
236242
this.peakModelBytes = peakModelBytes;
237-
this.actualMemoryUsageBytes = actualMemoryUsageBytes;
243+
this.systemMemoryUsageBytes = systemMemoryUsageBytes;
244+
this.maxSystemMemoryUsageBytes = maxSystemMemoryUsageBytes;
238245
this.modelBytesExceeded = modelBytesExceeded;
239246
this.modelBytesMemoryLimit = modelBytesMemoryLimit;
240247
this.totalByFieldCount = totalByFieldCount;
@@ -259,10 +266,12 @@ public ModelSizeStats(StreamInput in) throws IOException {
259266
jobId = in.readString();
260267
modelBytes = in.readVLong();
261268
peakModelBytes = in.readOptionalLong();
262-
if (in.getTransportVersion().onOrAfter(TransportVersions.ML_AD_ACTUAL_MEMORY_USAGE)) {
263-
actualMemoryUsageBytes = in.readOptionalLong();
269+
if (in.getTransportVersion().onOrAfter(TransportVersions.ML_AD_SYSTEM_MEMORY_USAGE)) {
270+
systemMemoryUsageBytes = in.readOptionalLong();
271+
maxSystemMemoryUsageBytes = in.readOptionalLong();
264272
} else {
265-
actualMemoryUsageBytes = null;
273+
systemMemoryUsageBytes = null;
274+
maxSystemMemoryUsageBytes = null;
266275
}
267276
modelBytesExceeded = in.readOptionalLong();
268277
modelBytesMemoryLimit = in.readOptionalLong();
@@ -305,8 +314,9 @@ public void writeTo(StreamOutput out) throws IOException {
305314
out.writeString(jobId);
306315
out.writeVLong(modelBytes);
307316
out.writeOptionalLong(peakModelBytes);
308-
if (out.getTransportVersion().onOrAfter(TransportVersions.ML_AD_ACTUAL_MEMORY_USAGE)) {
309-
out.writeOptionalLong(actualMemoryUsageBytes);
317+
if (out.getTransportVersion().onOrAfter(TransportVersions.ML_AD_SYSTEM_MEMORY_USAGE)) {
318+
out.writeOptionalLong(systemMemoryUsageBytes);
319+
out.writeOptionalLong(maxSystemMemoryUsageBytes);
310320
}
311321
out.writeOptionalLong(modelBytesExceeded);
312322
out.writeOptionalLong(modelBytesMemoryLimit);
@@ -354,8 +364,11 @@ public XContentBuilder doXContentBody(XContentBuilder builder) throws IOExceptio
354364
if (peakModelBytes != null) {
355365
builder.field(PEAK_MODEL_BYTES_FIELD.getPreferredName(), peakModelBytes);
356366
}
357-
if (actualMemoryUsageBytes != null) {
358-
builder.field(ACTUAL_MEMORY_USAGE_BYTES.getPreferredName(), actualMemoryUsageBytes);
367+
if (systemMemoryUsageBytes != null) {
368+
builder.field(SYSTEM_MEMORY_BYTES.getPreferredName(), systemMemoryUsageBytes);
369+
}
370+
if (maxSystemMemoryUsageBytes != null) {
371+
builder.field(MAX_SYSTEM_MEMORY_BYTES.getPreferredName(), maxSystemMemoryUsageBytes);
359372
}
360373
if (modelBytesExceeded != null) {
361374
builder.field(MODEL_BYTES_EXCEEDED_FIELD.getPreferredName(), modelBytesExceeded);
@@ -409,8 +422,12 @@ public Long getPeakModelBytes() {
409422
return peakModelBytes;
410423
}
411424

412-
public Long getActualMemoryUsageBytes() {
413-
return actualMemoryUsageBytes;
425+
public Long getSystemMemoryBytes() {
426+
return systemMemoryUsageBytes;
427+
}
428+
429+
public Long getMaxSystemMemoryBytes() {
430+
return maxSystemMemoryUsageBytes;
414431
}
415432

416433
public Long getModelBytesExceeded() {
@@ -501,7 +518,8 @@ public int hashCode() {
501518
jobId,
502519
modelBytes,
503520
peakModelBytes,
504-
actualMemoryUsageBytes,
521+
systemMemoryUsageBytes,
522+
maxSystemMemoryUsageBytes,
505523
modelBytesExceeded,
506524
modelBytesMemoryLimit,
507525
totalByFieldCount,
@@ -540,7 +558,8 @@ public boolean equals(Object other) {
540558

541559
return this.modelBytes == that.modelBytes
542560
&& Objects.equals(this.peakModelBytes, that.peakModelBytes)
543-
&& this.actualMemoryUsageBytes == that.actualMemoryUsageBytes
561+
&& this.systemMemoryUsageBytes == that.systemMemoryUsageBytes
562+
&& this.maxSystemMemoryUsageBytes == that.maxSystemMemoryUsageBytes
544563
&& Objects.equals(this.modelBytesExceeded, that.modelBytesExceeded)
545564
&& Objects.equals(this.modelBytesMemoryLimit, that.modelBytesMemoryLimit)
546565
&& this.totalByFieldCount == that.totalByFieldCount
@@ -567,7 +586,8 @@ public static class Builder {
567586
private final String jobId;
568587
private long modelBytes;
569588
private Long peakModelBytes;
570-
private Long actualMemoryUsageBytes;
589+
private Long systemMemoryUsageBytes;
590+
private Long maxSystemMemoryUsageBytes;
571591
private Long modelBytesExceeded;
572592
private Long modelBytesMemoryLimit;
573593
private long totalByFieldCount;
@@ -598,7 +618,8 @@ public Builder(ModelSizeStats modelSizeStats) {
598618
this.jobId = modelSizeStats.jobId;
599619
this.modelBytes = modelSizeStats.modelBytes;
600620
this.peakModelBytes = modelSizeStats.peakModelBytes;
601-
this.actualMemoryUsageBytes = modelSizeStats.actualMemoryUsageBytes;
621+
this.systemMemoryUsageBytes = modelSizeStats.systemMemoryUsageBytes;
622+
this.maxSystemMemoryUsageBytes = modelSizeStats.maxSystemMemoryUsageBytes;
602623
this.modelBytesExceeded = modelSizeStats.modelBytesExceeded;
603624
this.modelBytesMemoryLimit = modelSizeStats.modelBytesMemoryLimit;
604625
this.totalByFieldCount = modelSizeStats.totalByFieldCount;
@@ -629,8 +650,13 @@ public Builder setPeakModelBytes(long peakModelBytes) {
629650
return this;
630651
}
631652

632-
public Builder setActualMemoryUsageBytes(long actualMemoryUsageBytes) {
633-
this.actualMemoryUsageBytes = actualMemoryUsageBytes;
653+
public Builder setSystemMemoryBytes(long systemMemoryUsageBytes) {
654+
this.systemMemoryUsageBytes = systemMemoryUsageBytes;
655+
return this;
656+
}
657+
658+
public Builder setMaxSystemMemoryBytes(long maxSystemMemoryUsageBytes) {
659+
this.maxSystemMemoryUsageBytes = maxSystemMemoryUsageBytes;
634660
return this;
635661
}
636662

@@ -731,7 +757,8 @@ public ModelSizeStats build() {
731757
jobId,
732758
modelBytes,
733759
peakModelBytes,
734-
actualMemoryUsageBytes,
760+
systemMemoryUsageBytes,
761+
maxSystemMemoryUsageBytes,
735762
modelBytesExceeded,
736763
modelBytesMemoryLimit,
737764
totalByFieldCount,

‎x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProvider.java‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1590,10 +1590,17 @@ void calculateEstablishedMemoryUsage(
15901590
handler.accept((storedPeak != null) ? storedPeak : latestModelSizeStats.getModelBytes());
15911591
return;
15921592
}
1593-
case ACTUAL_MEMORY_USAGE_BYTES -> {
1594-
Long storedActualMemoryUsageBytes = latestModelSizeStats.getActualMemoryUsageBytes();
1593+
case SYSTEM_MEMORY_BYTES -> {
1594+
Long storedSystemMemoryBytes = latestModelSizeStats.getSystemMemoryBytes();
15951595
handler.accept(
1596-
(storedActualMemoryUsageBytes != null) ? storedActualMemoryUsageBytes : latestModelSizeStats.getModelBytes()
1596+
(storedSystemMemoryBytes != null) ? storedSystemMemoryBytes : latestModelSizeStats.getModelBytes()
1597+
);
1598+
return;
1599+
}
1600+
case MAX_SYSTEM_MEMORY_BYTES -> {
1601+
Long storedMaxSystemMemoryBytes = latestModelSizeStats.getMaxSystemMemoryBytes();
1602+
handler.accept(
1603+
(storedMaxSystemMemoryBytes != null) ? storedMaxSystemMemoryBytes : latestModelSizeStats.getModelBytes()
15971604
);
15981605
return;
15991606
}

‎x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1076,7 +1076,9 @@ public ByteSizeValue getOpenProcessMemoryUsage() {
10761076
case MODEL_MEMORY_LIMIT -> Optional.ofNullable(modelSizeStats.getModelBytesMemoryLimit()).orElse(0L);
10771077
case CURRENT_MODEL_BYTES -> modelSizeStats.getModelBytes();
10781078
case PEAK_MODEL_BYTES -> Optional.ofNullable(modelSizeStats.getPeakModelBytes()).orElse(modelSizeStats.getModelBytes());
1079-
case ACTUAL_MEMORY_USAGE_BYTES -> Optional.ofNullable(modelSizeStats.getActualMemoryUsageBytes())
1079+
case SYSTEM_MEMORY_BYTES -> Optional.ofNullable(modelSizeStats.getSystemMemoryBytes())
1080+
.orElse(modelSizeStats.getModelBytes());
1081+
case MAX_SYSTEM_MEMORY_BYTES -> Optional.ofNullable(modelSizeStats.getMaxSystemMemoryBytes())
10801082
.orElse(modelSizeStats.getModelBytes());
10811083
};
10821084
memoryUsedBytes += Job.PROCESS_MEMORY_OVERHEAD.getBytes();

‎x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManagerTests.java‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -834,12 +834,14 @@ public void testGetOpenProcessMemoryUsage() {
834834
long modelMemoryLimitBytes = ByteSizeValue.ofMb(randomIntBetween(10, 1000)).getBytes();
835835
long peakModelBytes = randomLongBetween(100000, modelMemoryLimitBytes - 1);
836836
long modelBytes = randomLongBetween(1, peakModelBytes - 1);
837-
long actualMemoryUsageBytes = randomLongBetween(262144, peakModelBytes - 1);
837+
long systemMemoryUsageBytes = randomLongBetween(262144, peakModelBytes - 1);
838+
long maxSystemMemoryUsageBytes = randomLongBetween(266240, peakModelBytes - 1);
838839
AssignmentMemoryBasis assignmentMemoryBasis = randomFrom(AssignmentMemoryBasis.values());
839840
modelSizeStats = new ModelSizeStats.Builder("foo").setModelBytesMemoryLimit(modelMemoryLimitBytes)
840841
.setPeakModelBytes(peakModelBytes)
841842
.setModelBytes(modelBytes)
842-
.setActualMemoryUsageBytes(actualMemoryUsageBytes)
843+
.setSystemMemoryBytes(systemMemoryUsageBytes)
844+
.setMaxSystemMemoryBytes(maxSystemMemoryUsageBytes)
843845
.setAssignmentMemoryBasis(assignmentMemoryBasis)
844846
.build();
845847
when(autodetectCommunicator.getModelSizeStats()).thenReturn(modelSizeStats);
@@ -852,7 +854,8 @@ public void testGetOpenProcessMemoryUsage() {
852854
case MODEL_MEMORY_LIMIT -> modelMemoryLimitBytes;
853855
case CURRENT_MODEL_BYTES -> modelBytes;
854856
case PEAK_MODEL_BYTES -> peakModelBytes;
855-
case ACTUAL_MEMORY_USAGE_BYTES -> actualMemoryUsageBytes;
857+
case SYSTEM_MEMORY_BYTES -> systemMemoryUsageBytes;
858+
case MAX_SYSTEM_MEMORY_BYTES -> maxSystemMemoryUsageBytes;
856859
};
857860
assertThat(manager.getOpenProcessMemoryUsage(), equalTo(ByteSizeValue.ofBytes(expectedSizeBytes)));
858861
}

0 commit comments

Comments
 (0)