Skip to content

Commit e512582

Browse files
edsavageedsavage
committed
Remove now unneeded fields.
1 parent 7e0fd66 commit e512582

File tree

6 files changed

+2
-84
lines changed

6 files changed

+2
-84
lines changed

‎docs/changelog/126256.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
pr: 126256
2-
summary: Handle new `actual_memory_usage_bytes` field in model size stats
2+
summary: Adjust REST test code to accommodate changes to calculation of "model bytes" and "peak mode bytes" values.
33
area: Machine Learning
44
type: enhancement
55
issues: []

‎server/src/main/java/org/elasticsearch/TransportVersions.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -284,7 +284,6 @@ static TransportVersion def(int id) {
284284
public static final TransportVersion JOIN_ON_ALIASES = def(9_088_0_00);
285285
public static final TransportVersion ILM_ADD_SKIP_SETTING = def(9_089_0_00);
286286
public static final TransportVersion ML_INFERENCE_MISTRAL_CHAT_COMPLETION_ADDED = def(9_090_0_00);
287-
public static final TransportVersion ML_AD_SYSTEM_MEMORY_USAGE = def(9_091_0_00);
288287

289288
/*
290289
* STOP! READ THIS FIRST! No, really,

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/ModelSizeStats.java‎

Lines changed: 1 addition & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -41,8 +41,6 @@ public class ModelSizeStats implements ToXContentObject, Writeable {
4141
*/
4242
public static final ParseField MODEL_BYTES_FIELD = new ParseField("model_bytes");
4343
public static final ParseField PEAK_MODEL_BYTES_FIELD = new ParseField("peak_model_bytes");
44-
public static final ParseField SYSTEM_MEMORY_BYTES = new ParseField("system_memory_bytes");
45-
public static final ParseField MAX_SYSTEM_MEMORY_BYTES = new ParseField("max_system_memory_bytes");
4644
public static final ParseField MODEL_BYTES_EXCEEDED_FIELD = new ParseField("model_bytes_exceeded");
4745
public static final ParseField MODEL_BYTES_MEMORY_LIMIT_FIELD = new ParseField("model_bytes_memory_limit");
4846
public static final ParseField TOTAL_BY_FIELD_COUNT_FIELD = new ParseField("total_by_field_count");
@@ -76,8 +74,6 @@ private static ConstructingObjectParser<Builder, Void> createParser(boolean igno
7674
parser.declareString((modelSizeStat, s) -> {}, Result.RESULT_TYPE);
7775
parser.declareLong(Builder::setModelBytes, MODEL_BYTES_FIELD);
7876
parser.declareLong(Builder::setPeakModelBytes, PEAK_MODEL_BYTES_FIELD);
79-
parser.declareLong(Builder::setSystemMemoryBytes, SYSTEM_MEMORY_BYTES);
80-
parser.declareLong(Builder::setMaxSystemMemoryBytes, MAX_SYSTEM_MEMORY_BYTES);
8177
parser.declareLong(Builder::setModelBytesExceeded, MODEL_BYTES_EXCEEDED_FIELD);
8278
parser.declareLong(Builder::setModelBytesMemoryLimit, MODEL_BYTES_MEMORY_LIMIT_FIELD);
8379
parser.declareLong(Builder::setBucketAllocationFailuresCount, BUCKET_ALLOCATION_FAILURES_COUNT_FIELD);
@@ -156,18 +152,14 @@ public String toString() {
156152
* 1. The job's model_memory_limit
157153
* 2. The current model memory, i.e. what's reported in model_bytes of this object
158154
* 3. The peak model memory, i.e. what's reported in peak_model_bytes of this object
159-
* 4. The system memory, i.e. what's reported in system_memory_bytes of this object
160-
* 5. The max system memory, i.e. what's reported in max_system_memory_bytes of this object
161155
* The field storing this enum can also be <code>null</code>, which means the
162156
* assignment code will decide on the fly - this was the old behaviour prior
163157
* to 7.11.
164158
*/
165159
public enum AssignmentMemoryBasis implements Writeable {
166160
MODEL_MEMORY_LIMIT,
167161
CURRENT_MODEL_BYTES,
168-
PEAK_MODEL_BYTES,
169-
SYSTEM_MEMORY_BYTES,
170-
MAX_SYSTEM_MEMORY_BYTES,;
162+
PEAK_MODEL_BYTES;
171163

172164
public static AssignmentMemoryBasis fromString(String statusName) {
173165
return valueOf(statusName.trim().toUpperCase(Locale.ROOT));
@@ -191,8 +183,6 @@ public String toString() {
191183
private final String jobId;
192184
private final long modelBytes;
193185
private final Long peakModelBytes;
194-
private final Long systemMemoryUsageBytes;
195-
private final Long maxSystemMemoryUsageBytes;
196186
private final Long modelBytesExceeded;
197187
private final Long modelBytesMemoryLimit;
198188
private final long totalByFieldCount;
@@ -216,8 +206,6 @@ private ModelSizeStats(
216206
String jobId,
217207
long modelBytes,
218208
Long peakModelBytes,
219-
Long systemMemoryUsageBytes,
220-
Long maxSystemMemoryUsageBytes,
221209
Long modelBytesExceeded,
222210
Long modelBytesMemoryLimit,
223211
long totalByFieldCount,
@@ -240,8 +228,6 @@ private ModelSizeStats(
240228
this.jobId = jobId;
241229
this.modelBytes = modelBytes;
242230
this.peakModelBytes = peakModelBytes;
243-
this.systemMemoryUsageBytes = systemMemoryUsageBytes;
244-
this.maxSystemMemoryUsageBytes = maxSystemMemoryUsageBytes;
245231
this.modelBytesExceeded = modelBytesExceeded;
246232
this.modelBytesMemoryLimit = modelBytesMemoryLimit;
247233
this.totalByFieldCount = totalByFieldCount;
@@ -266,13 +252,6 @@ public ModelSizeStats(StreamInput in) throws IOException {
266252
jobId = in.readString();
267253
modelBytes = in.readVLong();
268254
peakModelBytes = in.readOptionalLong();
269-
if (in.getTransportVersion().onOrAfter(TransportVersions.ML_AD_SYSTEM_MEMORY_USAGE)) {
270-
systemMemoryUsageBytes = in.readOptionalLong();
271-
maxSystemMemoryUsageBytes = in.readOptionalLong();
272-
} else {
273-
systemMemoryUsageBytes = null;
274-
maxSystemMemoryUsageBytes = null;
275-
}
276255
modelBytesExceeded = in.readOptionalLong();
277256
modelBytesMemoryLimit = in.readOptionalLong();
278257
totalByFieldCount = in.readVLong();
@@ -314,10 +293,6 @@ public void writeTo(StreamOutput out) throws IOException {
314293
out.writeString(jobId);
315294
out.writeVLong(modelBytes);
316295
out.writeOptionalLong(peakModelBytes);
317-
if (out.getTransportVersion().onOrAfter(TransportVersions.ML_AD_SYSTEM_MEMORY_USAGE)) {
318-
out.writeOptionalLong(systemMemoryUsageBytes);
319-
out.writeOptionalLong(maxSystemMemoryUsageBytes);
320-
}
321296
out.writeOptionalLong(modelBytesExceeded);
322297
out.writeOptionalLong(modelBytesMemoryLimit);
323298
out.writeVLong(totalByFieldCount);
@@ -364,12 +339,6 @@ public XContentBuilder doXContentBody(XContentBuilder builder) throws IOExceptio
364339
if (peakModelBytes != null) {
365340
builder.field(PEAK_MODEL_BYTES_FIELD.getPreferredName(), peakModelBytes);
366341
}
367-
if (systemMemoryUsageBytes != null) {
368-
builder.field(SYSTEM_MEMORY_BYTES.getPreferredName(), systemMemoryUsageBytes);
369-
}
370-
if (maxSystemMemoryUsageBytes != null) {
371-
builder.field(MAX_SYSTEM_MEMORY_BYTES.getPreferredName(), maxSystemMemoryUsageBytes);
372-
}
373342
if (modelBytesExceeded != null) {
374343
builder.field(MODEL_BYTES_EXCEEDED_FIELD.getPreferredName(), modelBytesExceeded);
375344
}
@@ -422,14 +391,6 @@ public Long getPeakModelBytes() {
422391
return peakModelBytes;
423392
}
424393

425-
public Long getSystemMemoryBytes() {
426-
return systemMemoryUsageBytes;
427-
}
428-
429-
public Long getMaxSystemMemoryBytes() {
430-
return maxSystemMemoryUsageBytes;
431-
}
432-
433394
public Long getModelBytesExceeded() {
434395
return modelBytesExceeded;
435396
}
@@ -518,8 +479,6 @@ public int hashCode() {
518479
jobId,
519480
modelBytes,
520481
peakModelBytes,
521-
systemMemoryUsageBytes,
522-
maxSystemMemoryUsageBytes,
523482
modelBytesExceeded,
524483
modelBytesMemoryLimit,
525484
totalByFieldCount,
@@ -558,8 +517,6 @@ public boolean equals(Object other) {
558517

559518
return this.modelBytes == that.modelBytes
560519
&& Objects.equals(this.peakModelBytes, that.peakModelBytes)
561-
&& this.systemMemoryUsageBytes == that.systemMemoryUsageBytes
562-
&& this.maxSystemMemoryUsageBytes == that.maxSystemMemoryUsageBytes
563520
&& Objects.equals(this.modelBytesExceeded, that.modelBytesExceeded)
564521
&& Objects.equals(this.modelBytesMemoryLimit, that.modelBytesMemoryLimit)
565522
&& this.totalByFieldCount == that.totalByFieldCount
@@ -586,8 +543,6 @@ public static class Builder {
586543
private final String jobId;
587544
private long modelBytes;
588545
private Long peakModelBytes;
589-
private Long systemMemoryUsageBytes;
590-
private Long maxSystemMemoryUsageBytes;
591546
private Long modelBytesExceeded;
592547
private Long modelBytesMemoryLimit;
593548
private long totalByFieldCount;
@@ -618,8 +573,6 @@ public Builder(ModelSizeStats modelSizeStats) {
618573
this.jobId = modelSizeStats.jobId;
619574
this.modelBytes = modelSizeStats.modelBytes;
620575
this.peakModelBytes = modelSizeStats.peakModelBytes;
621-
this.systemMemoryUsageBytes = modelSizeStats.systemMemoryUsageBytes;
622-
this.maxSystemMemoryUsageBytes = modelSizeStats.maxSystemMemoryUsageBytes;
623576
this.modelBytesExceeded = modelSizeStats.modelBytesExceeded;
624577
this.modelBytesMemoryLimit = modelSizeStats.modelBytesMemoryLimit;
625578
this.totalByFieldCount = modelSizeStats.totalByFieldCount;
@@ -650,16 +603,6 @@ public Builder setPeakModelBytes(long peakModelBytes) {
650603
return this;
651604
}
652605

653-
public Builder setSystemMemoryBytes(long systemMemoryUsageBytes) {
654-
this.systemMemoryUsageBytes = systemMemoryUsageBytes;
655-
return this;
656-
}
657-
658-
public Builder setMaxSystemMemoryBytes(long maxSystemMemoryUsageBytes) {
659-
this.maxSystemMemoryUsageBytes = maxSystemMemoryUsageBytes;
660-
return this;
661-
}
662-
663606
public Builder setModelBytesExceeded(long modelBytesExceeded) {
664607
this.modelBytesExceeded = modelBytesExceeded;
665608
return this;
@@ -757,8 +700,6 @@ public ModelSizeStats build() {
757700
jobId,
758701
modelBytes,
759702
peakModelBytes,
760-
systemMemoryUsageBytes,
761-
maxSystemMemoryUsageBytes,
762703
modelBytesExceeded,
763704
modelBytesMemoryLimit,
764705
totalByFieldCount,

‎x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProvider.java‎

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1590,18 +1590,6 @@ void calculateEstablishedMemoryUsage(
15901590
handler.accept((storedPeak != null) ? storedPeak : latestModelSizeStats.getModelBytes());
15911591
return;
15921592
}
1593-
case SYSTEM_MEMORY_BYTES -> {
1594-
Long storedSystemMemoryBytes = latestModelSizeStats.getSystemMemoryBytes();
1595-
handler.accept((storedSystemMemoryBytes != null) ? storedSystemMemoryBytes : latestModelSizeStats.getModelBytes());
1596-
return;
1597-
}
1598-
case MAX_SYSTEM_MEMORY_BYTES -> {
1599-
Long storedMaxSystemMemoryBytes = latestModelSizeStats.getMaxSystemMemoryBytes();
1600-
handler.accept(
1601-
(storedMaxSystemMemoryBytes != null) ? storedMaxSystemMemoryBytes : latestModelSizeStats.getModelBytes()
1602-
);
1603-
return;
1604-
}
16051593
}
16061594
}
16071595

‎x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1076,10 +1076,6 @@ public ByteSizeValue getOpenProcessMemoryUsage() {
10761076
case MODEL_MEMORY_LIMIT -> Optional.ofNullable(modelSizeStats.getModelBytesMemoryLimit()).orElse(0L);
10771077
case CURRENT_MODEL_BYTES -> modelSizeStats.getModelBytes();
10781078
case PEAK_MODEL_BYTES -> Optional.ofNullable(modelSizeStats.getPeakModelBytes()).orElse(modelSizeStats.getModelBytes());
1079-
case SYSTEM_MEMORY_BYTES -> Optional.ofNullable(modelSizeStats.getSystemMemoryBytes())
1080-
.orElse(modelSizeStats.getModelBytes());
1081-
case MAX_SYSTEM_MEMORY_BYTES -> Optional.ofNullable(modelSizeStats.getMaxSystemMemoryBytes())
1082-
.orElse(modelSizeStats.getModelBytes());
10831079
};
10841080
memoryUsedBytes += Job.PROCESS_MEMORY_OVERHEAD.getBytes();
10851081
}

‎x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManagerTests.java‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -834,14 +834,10 @@ public void testGetOpenProcessMemoryUsage() {
834834
long modelMemoryLimitBytes = ByteSizeValue.ofMb(randomIntBetween(10, 1000)).getBytes();
835835
long peakModelBytes = randomLongBetween(100000, modelMemoryLimitBytes - 1);
836836
long modelBytes = randomLongBetween(1, peakModelBytes - 1);
837-
long systemMemoryUsageBytes = randomLongBetween(262144, peakModelBytes - 1);
838-
long maxSystemMemoryUsageBytes = randomLongBetween(266240, peakModelBytes - 1);
839837
AssignmentMemoryBasis assignmentMemoryBasis = randomFrom(AssignmentMemoryBasis.values());
840838
modelSizeStats = new ModelSizeStats.Builder("foo").setModelBytesMemoryLimit(modelMemoryLimitBytes)
841839
.setPeakModelBytes(peakModelBytes)
842840
.setModelBytes(modelBytes)
843-
.setSystemMemoryBytes(systemMemoryUsageBytes)
844-
.setMaxSystemMemoryBytes(maxSystemMemoryUsageBytes)
845841
.setAssignmentMemoryBasis(assignmentMemoryBasis)
846842
.build();
847843
when(autodetectCommunicator.getModelSizeStats()).thenReturn(modelSizeStats);
@@ -854,8 +850,6 @@ public void testGetOpenProcessMemoryUsage() {
854850
case MODEL_MEMORY_LIMIT -> modelMemoryLimitBytes;
855851
case CURRENT_MODEL_BYTES -> modelBytes;
856852
case PEAK_MODEL_BYTES -> peakModelBytes;
857-
case SYSTEM_MEMORY_BYTES -> systemMemoryUsageBytes;
858-
case MAX_SYSTEM_MEMORY_BYTES -> maxSystemMemoryUsageBytes;
859853
};
860854
assertThat(manager.getOpenProcessMemoryUsage(), equalTo(ByteSizeValue.ofBytes(expectedSizeBytes)));
861855
}

0 commit comments

Comments
 (0)