[7.17] [ML] ML stats failures should not stop the usage API working (#91936)

droberts195 · web-flow · commit 67e4ec9dd700 · 2022-11-24T14:58:48.000-05:00
It is possible to meddle with internal ML state such that calls to the ML stats APIs return errors. It is justifiable for these single purpose APIs to return errors when the internal state of ML is corrupted. However, it is undesirable for these low level problems to completely prevent the overall usage API from returning, because then callers cannot find out usage information from any part of the system. This change makes errors in the ML stats APIs non-fatal to the overall response of the usage API. When an ML stats APIs returns an error, the corresponding section of the ML usage information will be blank. Backport of #91917
diff --git a/docs/changelog/91917.yaml b/docs/changelog/91917.yaml
@@ -0,0 +1,6 @@
+pr: 91917
+summary: ML stats failures should not stop the usage API working
+area: Machine Learning
+type: bug
+issues:
+ - 91893
diff --git a/x-pack/plugin/ml/qa/ml-with-security/build.gradle b/x-pack/plugin/ml/qa/ml-with-security/build.gradle
@@ -187,6 +187,7 @@ tasks.named("yamlRestTest").configure {
     'ml/jobs_get_result_overall_buckets/Test overall buckets given invalid start param',
     'ml/jobs_get_result_overall_buckets/Test overall buckets given invalid end param',
     'ml/jobs_get_result_overall_buckets/Test overall buckets given bucket_span is smaller than max job bucket_span',
+    'ml/jobs_get_stats/Test closed results index',
     'ml/jobs_get_stats/Test get job stats given missing job',
     'ml/jobs_get_stats/Test no exception on get job stats with missing index',
     'ml/job_groups/Test put job with empty group',
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearningFeatureSet.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearningFeatureSet.java
@@ -7,6 +7,7 @@
 package org.elasticsearch.xpack.ml;
 
 import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.apache.lucene.util.Constants;
 import org.apache.lucene.util.Counter;
 import org.elasticsearch.ElasticsearchException;
@@ -73,6 +74,8 @@
 
 public class MachineLearningFeatureSet implements XPackFeatureSet {
 
+    private static final Logger logger = LogManager.getLogger(MachineLearningFeatureSet.class);
+
     /**
      * List of platforms for which the native processes are available
      */
@@ -368,58 +371,88 @@ public void execute(ActionListener<Usage> listener) {
                     nodeCount
                 );
                 listener.onResponse(usage);
-            }, listener::onFailure);
+            }, e -> {
+                logger.warn("Failed to get trained models usage to include in ML usage", e);
+                MachineLearningFeatureSetUsage usage = new MachineLearningFeatureSetUsage(
+                    available,
+                    enabled,
+                    jobsUsage,
+                    datafeedsUsage,
+                    analyticsUsage,
+                    inferenceUsage,
+                    nodeCount
+                );
+                listener.onResponse(usage);
+            });
 
             // Step 5. Extract usage from ingest statistics and gather trained model config count
+            GetTrainedModelsAction.Request getModelsRequest = new GetTrainedModelsAction.Request(
+                "*",
+                Collections.emptyList(),
+                Collections.emptySet()
+            );
+            getModelsRequest.setPageParams(new PageParams(0, 10_000));
             ActionListener<NodesStatsResponse> nodesStatsListener = ActionListener.wrap(response -> {
                 addInferenceIngestUsage(response, inferenceUsage);
-                GetTrainedModelsAction.Request getModelsRequest = new GetTrainedModelsAction.Request(
-                    "*",
-                    Collections.emptyList(),
-                    Collections.emptySet()
-                );
-                getModelsRequest.setPageParams(new PageParams(0, 10_000));
                 client.execute(GetTrainedModelsAction.INSTANCE, getModelsRequest, trainedModelsListener);
-            }, listener::onFailure);
+            }, e -> {
+                logger.warn("Failed to get inference ingest usage to include in ML usage", e);
+                client.execute(GetTrainedModelsAction.INSTANCE, getModelsRequest, trainedModelsListener);
+            });
 
             // Step 4. Extract usage from data frame analytics configs and then request ingest node stats
+            String[] ingestNodes = ingestNodes(state);
+            NodesStatsRequest nodesStatsRequest = new NodesStatsRequest(ingestNodes).clear()
+                .addMetric(NodesStatsRequest.Metric.INGEST.metricName());
             ActionListener<GetDataFrameAnalyticsAction.Response> dataframeAnalyticsListener = ActionListener.wrap(response -> {
                 addDataFrameAnalyticsUsage(response, analyticsUsage);
-                String[] ingestNodes = ingestNodes(state);
-                NodesStatsRequest nodesStatsRequest = new NodesStatsRequest(ingestNodes).clear()
-                    .addMetric(NodesStatsRequest.Metric.INGEST.metricName());
                 client.execute(NodesStatsAction.INSTANCE, nodesStatsRequest, nodesStatsListener);
-            }, listener::onFailure);
+            }, e -> {
+                logger.warn("Failed to get data frame analytics configs to include in ML usage", e);
+                client.execute(NodesStatsAction.INSTANCE, nodesStatsRequest, nodesStatsListener);
+            });
 
             // Step 3. Extract usage from data frame analytics stats and then request data frame analytics configs
+            GetDataFrameAnalyticsAction.Request getDfaRequest = new GetDataFrameAnalyticsAction.Request(Metadata.ALL);
+            getDfaRequest.setPageParams(new PageParams(0, 10_000));
             ActionListener<GetDataFrameAnalyticsStatsAction.Response> dataframeAnalyticsStatsListener = ActionListener.wrap(response -> {
                 addDataFrameAnalyticsStatsUsage(response, analyticsUsage);
-                GetDataFrameAnalyticsAction.Request getDfaRequest = new GetDataFrameAnalyticsAction.Request(Metadata.ALL);
-                getDfaRequest.setPageParams(new PageParams(0, 10_000));
                 client.execute(GetDataFrameAnalyticsAction.INSTANCE, getDfaRequest, dataframeAnalyticsListener);
-            }, listener::onFailure);
+            }, e -> {
+                logger.warn("Failed to get data frame analytics stats to include in ML usage", e);
+                client.execute(GetDataFrameAnalyticsAction.INSTANCE, getDfaRequest, dataframeAnalyticsListener);
+            });
 
             // Step 2. Extract usage from datafeeds stats and return usage response
+            GetDataFrameAnalyticsStatsAction.Request dataframeAnalyticsStatsRequest = new GetDataFrameAnalyticsStatsAction.Request(
+                GetDatafeedsStatsAction.ALL
+            );
+            dataframeAnalyticsStatsRequest.setPageParams(new PageParams(0, 10_000));
             ActionListener<GetDatafeedsStatsAction.Response> datafeedStatsListener = ActionListener.wrap(response -> {
                 addDatafeedsUsage(response);
-                GetDataFrameAnalyticsStatsAction.Request dataframeAnalyticsStatsRequest = new GetDataFrameAnalyticsStatsAction.Request(
-                    GetDatafeedsStatsAction.ALL
-                );
-                dataframeAnalyticsStatsRequest.setPageParams(new PageParams(0, 10_000));
                 client.execute(GetDataFrameAnalyticsStatsAction.INSTANCE, dataframeAnalyticsStatsRequest, dataframeAnalyticsStatsListener);
-            }, listener::onFailure);
+            }, e -> {
+                logger.warn("Failed to get datafeed stats to include in ML usage", e);
+                client.execute(GetDataFrameAnalyticsStatsAction.INSTANCE, dataframeAnalyticsStatsRequest, dataframeAnalyticsStatsListener);
+            });
 
             // Step 1. Extract usage from jobs stats and then request stats for all datafeeds
-            GetJobsStatsAction.Request jobStatsRequest = new GetJobsStatsAction.Request(Metadata.ALL);
+            GetDatafeedsStatsAction.Request datafeedStatsRequest = new GetDatafeedsStatsAction.Request(GetDatafeedsStatsAction.ALL);
             ActionListener<GetJobsStatsAction.Response> jobStatsListener = ActionListener.wrap(response -> {
                 jobManagerHolder.getJobManager().expandJobs(Metadata.ALL, true, ActionListener.wrap(jobs -> {
                     addJobsUsage(response, jobs.results());
-                    GetDatafeedsStatsAction.Request datafeedStatsRequest = new GetDatafeedsStatsAction.Request(GetDatafeedsStatsAction.ALL);
                     client.execute(GetDatafeedsStatsAction.INSTANCE, datafeedStatsRequest, datafeedStatsListener);
-                }, listener::onFailure));
-            }, listener::onFailure);
+                }, e -> {
+                    logger.warn("Failed to get job configs to include in ML usage", e);
+                    client.execute(GetDatafeedsStatsAction.INSTANCE, datafeedStatsRequest, datafeedStatsListener);
+                }));
+            }, e -> {
+                logger.warn("Failed to get job stats to include in ML usage", e);
+                client.execute(GetDatafeedsStatsAction.INSTANCE, datafeedStatsRequest, datafeedStatsListener);
+            });
 
             // Step 0. Kick off the chain of callbacks by requesting jobs stats
+            GetJobsStatsAction.Request jobStatsRequest = new GetJobsStatsAction.Request(Metadata.ALL);
             client.execute(GetJobsStatsAction.INSTANCE, jobStatsRequest, jobStatsListener);
         }
 
diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/jobs_get_stats.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/jobs_get_stats.yml
@@ -413,3 +413,58 @@ setup:
   - is_false: jobs.1.timing_stats.maximum_bucket_processing_time_ms
   - is_false: jobs.1.timing_stats.average_bucket_processing_time_ms
   - is_false: jobs.1.timing_stats.exponential_average_bucket_processing_time_ms
+
+---
+"Test closed results index":
+
+  - skip:
+      features:
+        - "warnings"
+
+  - do:
+      warnings:
+        - 'Posting data directly to anomaly detection jobs is deprecated, in a future major version it will be compulsory to use a datafeed'
+      ml.post_data:
+        job_id: job-stats-test
+        body: >
+          {"airline":"AAL","responsetime":"132.2046","time":"1403481600"}
+          {"airline":"JZA","responsetime":"990.4628","time":"1403481600"}
+
+  - do:
+      ml.close_job:
+        job_id: jobs-get-stats-datafeed-job
+  - match: { closed: true }
+
+  - do:
+      ml.close_job:
+        job_id: job-stats-test
+  - match: { closed: true }
+
+  - do:
+      ml.get_job_stats: {}
+  - length: { jobs : 2 }
+
+  - do:
+      xpack.usage: {}
+  - match: { ml.available: true }
+  - match: { ml.enabled: true }
+  - match: { ml.jobs.closed.count: 2 }
+
+  - do:
+      indices.close:
+        index: .ml-anomalies-shared
+        wait_for_active_shards: index-setting
+
+  # With the index closed the low level ML API reports a problem
+  - do:
+      catch: /type=cluster_block_exception, reason=index \[.ml-anomalies-shared\] blocked by. \[FORBIDDEN\/.\/index closed\]/
+      ml.get_job_stats: {}
+
+  # But the high level X-Pack API returns what it can - we do this
+  # so that corruption to ML doesn't blind observers of the general
+  # cluster status
+  - do:
+      xpack.usage: {}
+  - match: { ml.available: true }
+  - match: { ml.enabled: true }
+  - is_false: ml.jobs.closed.count
