Skip to content

Commit 3ddec6e

Browse files
opensearch-trigger-bot[bot]dhrubo-os
opensearch-trigger-bot[bot]
and
dhrubo-os
authored
_execute api for mcorr POC (#845) (#854)
* _execute api for mcorr POC Signed-off-by: Dhrubo Saha <[email protected]> added unit tests and fixed the deploy failed issue. Signed-off-by: Dhrubo Saha <[email protected]> * refactoring the code Signed-off-by: Dhrubo Saha <[email protected]> * addressed comments Signed-off-by: Dhrubo Saha <[email protected]> * adressed PR comments and added more unit tests Signed-off-by: Dhrubo Saha <[email protected]> * added release url and added configuration for mcorr Signed-off-by: Dhrubo Saha <[email protected]> * adressed the comments and added more tests Signed-off-by: Dhrubo Saha <[email protected]> * fix a bug Signed-off-by: Dhrubo Saha <[email protected]> * testing Signed-off-by: Dhrubo Saha <[email protected]> * Fixed a bug + addressed the comments in PR Signed-off-by: Dhrubo Saha <[email protected]> * refactoring some minor changes Signed-off-by: Dhrubo Saha <[email protected]> * addressed comments Signed-off-by: Dhrubo Saha <[email protected]> --------- Signed-off-by: Dhrubo Saha <[email protected]> (cherry picked from commit a845a2a) Co-authored-by: Dhrubo Saha <[email protected]>
1 parent 036bda0 commit 3ddec6e

File tree

46 files changed

+2563
-102
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

46 files changed

+2563
-102
lines changed

‎common/src/main/java/org/opensearch/ml/common/FunctionName.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,8 @@ public enum FunctionName {
1616
ANOMALY_LOCALIZATION,
1717
RCF_SUMMARIZE,
1818
LOGISTIC_REGRESSION,
19-
TEXT_EMBEDDING;
19+
TEXT_EMBEDDING,
20+
METRICS_CORRELATION;
2021

2122
public static FunctionName from(String value) {
2223
try {

‎common/src/main/java/org/opensearch/ml/common/MLModel.java‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
import org.opensearch.ml.common.model.MLModelFormat;
2020
import org.opensearch.ml.common.model.MLModelState;
2121
import org.opensearch.ml.common.model.TextEmbeddingModelConfig;
22+
import org.opensearch.ml.common.model.MetricsCorrelationModelConfig;
2223

2324
import java.io.IOException;
2425
import java.time.Instant;
@@ -302,7 +303,7 @@ public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params par
302303
return builder;
303304
}
304305

305-
public static MLModel parse(XContentParser parser) throws IOException {
306+
public static MLModel parse(XContentParser parser, String algorithmName) throws IOException {
306307
String name = null;
307308
FunctionName algorithm = null;
308309
String version = null;
@@ -385,7 +386,11 @@ public static MLModel parse(XContentParser parser) throws IOException {
385386
modelContentHash = parser.text();
386387
break;
387388
case MODEL_CONFIG_FIELD:
388-
modelConfig = TextEmbeddingModelConfig.parse(parser);
389+
if (FunctionName.METRICS_CORRELATION.name().equals(algorithmName)) {
390+
modelConfig = MetricsCorrelationModelConfig.parse(parser);
391+
} else {
392+
modelConfig = TextEmbeddingModelConfig.parse(parser);
393+
}
389394
break;
390395
case PLANNING_WORKER_NODE_COUNT_FIELD:
391396
planningWorkerNodeCount = parser.intValue();

‎common/src/main/java/org/opensearch/ml/common/exception/ExecuteException.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
package org.opensearch.ml.common.exception;
2+
3+
public class ExecuteException extends MLException{
4+
public ExecuteException(String msg) { super(msg); }
5+
public ExecuteException(Throwable cause) { super(cause); }
6+
public ExecuteException(String msg, Throwable cause) { super(msg, cause); }
7+
}

‎common/src/main/java/org/opensearch/ml/common/input/execute/metricscorrelation/MetricsCorrelationInput.java‎

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package org.opensearch.ml.common.input.execute.metricscorrelation;
7+
8+
import lombok.Builder;
9+
import lombok.Data;
10+
import org.opensearch.common.io.stream.StreamInput;
11+
import org.opensearch.common.io.stream.StreamOutput;
12+
import org.opensearch.core.ParseField;
13+
import org.opensearch.core.xcontent.NamedXContentRegistry;
14+
import org.opensearch.core.xcontent.XContentBuilder;
15+
import org.opensearch.core.xcontent.XContentParser;
16+
import org.opensearch.ml.common.FunctionName;
17+
import org.opensearch.ml.common.annotation.ExecuteInput;
18+
import org.opensearch.ml.common.input.Input;
19+
20+
import java.io.IOException;
21+
import java.util.ArrayList;
22+
import java.util.List;
23+
24+
import static org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
25+
26+
@ExecuteInput(algorithms={FunctionName.METRICS_CORRELATION})
27+
@Data
28+
public class MetricsCorrelationInput implements Input {
29+
public static final String PARSE_FIELD_NAME = FunctionName.METRICS_CORRELATION.name();
30+
public static final NamedXContentRegistry.Entry XCONTENT_REGISTRY = new NamedXContentRegistry.Entry(
31+
Input.class,
32+
new ParseField(PARSE_FIELD_NAME),
33+
it -> parse(it)
34+
);
35+
36+
public static final String METRICS_FIELD = "metrics";
37+
38+
List<float[]> inputData;
39+
40+
@Builder
41+
public MetricsCorrelationInput(List<float[]> inputData) {
42+
if (inputData == null || inputData.size() == 0) {
43+
throw new IllegalArgumentException("empty input data");
44+
}
45+
int expectedLength = inputData.get(0).length;
46+
for (int i = 1; i < inputData.size(); i++) {
47+
float[] array = inputData.get(i);
48+
if (array.length != expectedLength) {
49+
// found an array with different length
50+
throw new IllegalArgumentException("All the input metrics sizes should be same");
51+
}
52+
}
53+
if (inputData.size() >= expectedLength) {
54+
throw new IllegalArgumentException("The number of metrics to correlate must be smaller than the length of each time series.");
55+
}
56+
this.inputData = inputData;
57+
}
58+
59+
public MetricsCorrelationInput(StreamInput in) throws IOException {
60+
this.inputData = in.readList(StreamInput::readFloatArray);
61+
}
62+
63+
public static MetricsCorrelationInput parse(XContentParser parser) throws IOException {
64+
List<float[]> inputData = new ArrayList<>();
65+
66+
ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser);
67+
while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
68+
String fieldName = parser.currentName();
69+
parser.nextToken();
70+
71+
switch (fieldName) {
72+
case METRICS_FIELD:
73+
ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.currentToken(), parser);
74+
while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
75+
ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.currentToken(), parser);
76+
List<Float> inputItem = new ArrayList<>();
77+
while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
78+
inputItem.add(parser.floatValue());
79+
}
80+
float[] floatArray = new float[inputItem.size()];
81+
int i = 0;
82+
83+
for (Float f : inputItem) {
84+
floatArray[i++] = (f != null ? f : Float.NaN);
85+
}
86+
inputData.add(floatArray);
87+
}
88+
break;
89+
default:
90+
parser.skipChildren();
91+
break;
92+
}
93+
}
94+
return new MetricsCorrelationInput(inputData);
95+
}
96+
97+
@Override
98+
public FunctionName getFunctionName() {
99+
return FunctionName.METRICS_CORRELATION;
100+
}
101+
102+
@Override
103+
public void writeTo(StreamOutput out) throws IOException {
104+
out.writeCollection(this.inputData, StreamOutput::writeFloatArray);
105+
}
106+
107+
@Override
108+
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
109+
builder.startObject();
110+
builder.field(METRICS_FIELD, inputData);
111+
builder.endObject();
112+
return builder;
113+
}
114+
}

‎common/src/main/java/org/opensearch/ml/common/model/MetricsCorrelationModelConfig.java‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package org.opensearch.ml.common.model;
7+
8+
import lombok.Builder;
9+
import lombok.Getter;
10+
import lombok.Setter;
11+
import org.opensearch.core.xcontent.XContentBuilder;
12+
import org.opensearch.core.xcontent.XContentParser;
13+
import org.opensearch.ml.common.FunctionName;
14+
15+
import java.io.IOException;
16+
import java.util.Locale;
17+
18+
import static org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
19+
20+
@Setter
21+
@Getter
22+
public class MetricsCorrelationModelConfig extends MLModelConfig {
23+
24+
public static final String PARSE_FIELD_NAME = FunctionName.METRICS_CORRELATION.name();
25+
26+
@Builder(toBuilder = true)
27+
public MetricsCorrelationModelConfig(String modelType, String allConfig) {
28+
super(modelType, allConfig);
29+
}
30+
31+
@Override
32+
public String getWriteableName() {
33+
return PARSE_FIELD_NAME;
34+
}
35+
36+
@Override
37+
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
38+
builder.startObject();
39+
if (modelType != null) {
40+
builder.field(MODEL_TYPE_FIELD, modelType);
41+
}
42+
if (allConfig != null) {
43+
builder.field(ALL_CONFIG_FIELD, allConfig);
44+
}
45+
builder.endObject();
46+
return builder;
47+
}
48+
49+
public static MetricsCorrelationModelConfig parse(XContentParser parser) throws IOException {
50+
String modelType = null;
51+
String allConfig = null;
52+
53+
ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser);
54+
while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
55+
String fieldName = parser.currentName();
56+
parser.nextToken();
57+
58+
switch (fieldName) {
59+
case MODEL_TYPE_FIELD:
60+
modelType = parser.text();
61+
break;
62+
case ALL_CONFIG_FIELD:
63+
allConfig = parser.text();
64+
break;
65+
default:
66+
parser.skipChildren();
67+
break;
68+
}
69+
}
70+
return new MetricsCorrelationModelConfig(modelType, allConfig);
71+
}
72+
}

‎common/src/main/java/org/opensearch/ml/common/output/MLOutputType.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,5 +9,6 @@ public enum MLOutputType {
99
TRAINING,
1010
PREDICTION,
1111
SAMPLE_ALGO,
12-
MODEL_TENSOR
12+
MODEL_TENSOR,
13+
MCORR_TENSOR
1314
}

‎common/src/main/java/org/opensearch/ml/common/output/execute/metrics_correlation/MCorrModelTensor.java‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package org.opensearch.ml.common.output.execute.metrics_correlation;
7+
8+
import lombok.Builder;
9+
import lombok.Data;
10+
import org.opensearch.common.io.stream.StreamInput;
11+
import org.opensearch.common.io.stream.StreamOutput;
12+
import org.opensearch.common.io.stream.Writeable;
13+
import org.opensearch.core.xcontent.ToXContentObject;
14+
import org.opensearch.core.xcontent.XContentBuilder;
15+
16+
import java.io.IOException;
17+
18+
@Data
19+
public class MCorrModelTensor implements Writeable, ToXContentObject {
20+
21+
public static final String EVENT_WINDOW = "event_window";
22+
public static final String EVENT_PATTERN = "event_pattern";
23+
public static final String SUSPECTED_METRICS = "suspected_metrics";
24+
25+
private float[] event_window;
26+
private float[] event_pattern;
27+
private long[] suspected_metrics;
28+
29+
@Builder
30+
public MCorrModelTensor(float[] event_window, float[] event_pattern, long[] suspected_metrics) {
31+
this.event_window = event_window;
32+
this.event_pattern = event_pattern;
33+
this.suspected_metrics = suspected_metrics;
34+
}
35+
36+
@Override
37+
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
38+
builder.startObject();
39+
if (event_window != null) {
40+
builder.field(EVENT_WINDOW, event_window);
41+
}
42+
if (event_pattern != null) {
43+
builder.field(EVENT_PATTERN, event_pattern);
44+
}
45+
if (suspected_metrics != null) {
46+
builder.field(SUSPECTED_METRICS, suspected_metrics);
47+
}
48+
builder.endObject();
49+
return builder;
50+
}
51+
52+
public MCorrModelTensor(StreamInput in) throws IOException {
53+
if (in.readBoolean()) {
54+
this.event_window = in.readFloatArray();
55+
}
56+
if (in.readBoolean()) {
57+
this.event_pattern = in.readFloatArray();
58+
}
59+
if (in.readBoolean()) {
60+
this.suspected_metrics = in.readLongArray();
61+
}
62+
}
63+
64+
@Override
65+
public void writeTo(StreamOutput out) throws IOException {
66+
if (event_window != null) {
67+
out.writeBoolean(true);
68+
out.writeFloatArray(event_window);
69+
} else {
70+
out.writeBoolean(false);
71+
}
72+
73+
if (event_pattern != null) {
74+
out.writeBoolean(true);
75+
out.writeFloatArray(event_pattern);
76+
} else {
77+
out.writeBoolean(false);
78+
}
79+
if (suspected_metrics != null) {
80+
out.writeBoolean(true);
81+
out.writeLongArray(suspected_metrics);
82+
} else {
83+
out.writeBoolean(false);
84+
}
85+
}
86+
}

0 commit comments

Comments
 (0)