Add telemetry for repository usage (#112133)

Adds to the `GET _cluster/stats` endpoint information about the snapshot
repositories in use, including their types, whether they are read-only
or read-write, and for Azure repositories the kind of credentials in
use.

Author: DaveCTurner

docs/changelog/112133.yaml

@@ -0,0 +1,5 @@
+pr: 112133
+summary: Add telemetry for repository usage
+area: Snapshot/Restore
+type: enhancement
+issues: []

docs/reference/cluster/stats.asciidoc

@@ -1282,6 +1282,31 @@ They are included here for expert users, but should otherwise be ignored.
 
 =====
 
+====
+
+`repositories`::
+(object) Contains statistics about the <<snapshot-restore,snapshot>> repositories defined in the cluster, broken down
+by repository type.
++
+.Properties of `repositories`
+[%collapsible%open]
+=====
+
+`count`:::
+(integer) The number of repositories of this type in the cluster.
+
+`read_only`:::
+(integer) The number of repositories of this type in the cluster which are registered read-only.
+
+`read_write`:::
+(integer) The number of repositories of this type in the cluster which are not registered as read-only.
+
+Each repository type may also include other statistics about the repositories of that type here.
+
+=====
+
+====
+
 [[cluster-stats-api-example]]
 ==== {api-examples-title}
 
@@ -1579,6 +1604,9 @@ The API returns the following response:
    },
    "snapshots": {
      ...
+   },
+   "repositories": {
+     ...
    }
 }
 --------------------------------------------------
@@ -1589,6 +1617,7 @@ The API returns the following response:
 // TESTRESPONSE[s/"count": \{[^\}]*\}/"count": $body.$_path/]
 // TESTRESPONSE[s/"packaging_types": \[[^\]]*\]/"packaging_types": $body.$_path/]
 // TESTRESPONSE[s/"snapshots": \{[^\}]*\}/"snapshots": $body.$_path/]
+// TESTRESPONSE[s/"repositories": \{[^\}]*\}/"repositories": $body.$_path/]
 // TESTRESPONSE[s/"field_types": \[[^\]]*\]/"field_types": $body.$_path/]
 // TESTRESPONSE[s/"runtime_field_types": \[[^\]]*\]/"runtime_field_types": $body.$_path/]
 // TESTRESPONSE[s/"search": \{[^\}]*\}/"search": $body.$_path/]
@@ -1600,7 +1629,7 @@ The API returns the following response:
 //    the plugins that will be in it. And because we figure folks don't need to
 //    see an exhaustive list anyway.
 // 2. Similarly, ignore the contents of `network_types`, `discovery_types`,
-//    `packaging_types` and `snapshots`.
+//    `packaging_types`, `snapshots` and `repositories`.
 // 3. Ignore the contents of the (nodes) count object, as what's shown here
 //    depends on the license. Voting-only nodes are e.g. only shown when this
 //    test runs with a basic license.

modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java

@@ -26,6 +26,7 @@
 
 import java.util.Locale;
 import java.util.Map;
+import java.util.Set;
 import java.util.function.Function;
 
 import static org.elasticsearch.core.Strings.format;
@@ -175,4 +176,9 @@ protected ByteSizeValue chunkSize() {
     public boolean isReadOnly() {
         return readonly;
     }
+
+    @Override
+    protected Set<String> getExtraUsageFeatures() {
+        return storageService.getExtraUsageFeatures(Repository.CLIENT_NAME.get(getMetadata().settings()));
+    }
 }

modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageService.java

@@ -24,6 +24,7 @@
 import java.net.Proxy;
 import java.net.URL;
 import java.util.Map;
+import java.util.Set;
 import java.util.function.BiConsumer;
 
 import static java.util.Collections.emptyMap;
@@ -165,4 +166,15 @@ public void refreshSettings(Map<String, AzureStorageSettings> clientsSettings) {
         this.storageSettings = Map.copyOf(clientsSettings);
         // clients are built lazily by {@link client(String, LocationMode)}
     }
+
+    /**
+     * For Azure repositories, we report the different kinds of credentials in use in the telemetry.
+     */
+    public Set<String> getExtraUsageFeatures(String clientName) {
+        try {
+            return getClientSettings(clientName).credentialsUsageFeatures();
+        } catch (Exception e) {
+            return Set.of();
+        }
+    }
 }

modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageSettings.java

@@ -29,6 +29,7 @@
 import java.util.HashMap;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Set;
 
 final class AzureStorageSettings {
 
@@ -130,6 +131,7 @@ final class AzureStorageSettings {
     private final int maxRetries;
     private final Proxy proxy;
     private final boolean hasCredentials;
+    private final Set<String> credentialsUsageFeatures;
 
     private AzureStorageSettings(
         String account,
@@ -150,6 +152,12 @@ private AzureStorageSettings(
         this.endpointSuffix = endpointSuffix;
         this.timeout = timeout;
         this.maxRetries = maxRetries;
+        this.credentialsUsageFeatures = Strings.hasText(key) ? Set.of("uses_key_credentials")
+            : Strings.hasText(sasToken) ? Set.of("uses_sas_token")
+            : SocketAccess.doPrivilegedException(() -> System.getenv("AZURE_FEDERATED_TOKEN_FILE")) == null
+                ? Set.of("uses_default_credentials", "uses_managed_identity")
+            : Set.of("uses_default_credentials", "uses_workload_identity");
+
         // Register the proxy if we have any
         // Validate proxy settings
         if (proxyType.equals(Proxy.Type.DIRECT) && ((proxyPort != 0) || Strings.hasText(proxyHost))) {
@@ -366,4 +374,8 @@ private String deriveURIFromSettings(boolean isPrimary) {
             throw new IllegalArgumentException(e);
         }
     }
+
+    public Set<String> credentialsUsageFeatures() {
+        return credentialsUsageFeatures;
+    }
 }

modules/repository-azure/src/yamlRestTest/resources/rest-api-spec/test/repository_azure/20_repository.yml

@@ -235,6 +235,19 @@ setup:
         snapshot: missing
         wait_for_completion: true
 
+---
+"Usage stats":
+  - requires:
+      cluster_features:
+        - repositories.supports_usage_stats
+      reason: requires this feature
+
+  - do:
+      cluster.stats: {}
+
+  - gte: { repositories.azure.count: 1 }
+  - gte: { repositories.azure.read_write: 1 }
+
 ---
 teardown:
 

modules/repository-gcs/src/yamlRestTest/resources/rest-api-spec/test/repository_gcs/20_repository.yml

@@ -232,6 +232,19 @@ setup:
         snapshot: missing
         wait_for_completion: true
 
+---
+"Usage stats":
+  - requires:
+      cluster_features:
+        - repositories.supports_usage_stats
+      reason: requires this feature
+
+  - do:
+      cluster.stats: {}
+
+  - gte: { repositories.gcs.count: 1 }
+  - gte: { repositories.gcs.read_write: 1 }
+
 ---
 teardown:
 

modules/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/20_repository_permanent_credentials.yml

@@ -345,6 +345,19 @@ setup:
         snapshot: missing
         wait_for_completion: true
 
+---
+"Usage stats":
+  - requires:
+      cluster_features:
+        - repositories.supports_usage_stats
+      reason: requires this feature
+
+  - do:
+      cluster.stats: {}
+
+  - gte: { repositories.s3.count: 1 }
+  - gte: { repositories.s3.read_write: 1 }
+
 ---
 teardown:
 

modules/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/30_repository_temporary_credentials.yml

@@ -256,6 +256,19 @@ setup:
         snapshot: missing
         wait_for_completion: true
 
+---
+"Usage stats":
+  - requires:
+      cluster_features:
+        - repositories.supports_usage_stats
+      reason: requires this feature
+
+  - do:
+      cluster.stats: {}
+
+  - gte: { repositories.s3.count: 1 }
+  - gte: { repositories.s3.read_write: 1 }
+
 ---
 teardown:
 

modules/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/40_repository_ec2_credentials.yml

@@ -256,6 +256,19 @@ setup:
         snapshot: missing
         wait_for_completion: true
 
+---
+"Usage stats":
+  - requires:
+      cluster_features:
+        - repositories.supports_usage_stats
+      reason: requires this feature
+
+  - do:
+      cluster.stats: {}
+
+  - gte: { repositories.s3.count: 1 }
+  - gte: { repositories.s3.read_write: 1 }
+
 ---
 teardown: