Skip to content

Commit b6c0d6e

Browse files
chrisale000chrisale000
committed
Adding M365 Specific Metrics Recorder and Emitting Auth Metrics
Signed-off-by: Alexander Christensen <alchrisk@amazon.com>
1 parent 6ef4e48 commit b6c0d6e

File tree

8 files changed

+752
-136
lines changed

8 files changed

+752
-136
lines changed

‎data-prepper-plugins/saas-source-plugins/microsoft-office365-source/src/main/java/org/opensearch/dataprepper/plugins/source/microsoft_office365/Office365RestClient.java‎

Lines changed: 58 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -9,13 +9,12 @@
99

1010
package org.opensearch.dataprepper.plugins.source.microsoft_office365;
1111

12-
import io.micrometer.core.instrument.Counter;
1312
import lombok.extern.slf4j.Slf4j;
14-
import org.opensearch.dataprepper.metrics.PluginMetrics;
1513
import org.opensearch.dataprepper.plugins.source.microsoft_office365.auth.Office365AuthenticationInterface;
1614
import org.opensearch.dataprepper.plugins.source.source_crawler.exception.SaaSCrawlerException;
1715
import org.opensearch.dataprepper.plugins.source.microsoft_office365.models.AuditLogsResponse;
1816
import org.opensearch.dataprepper.plugins.source.source_crawler.metrics.VendorAPIMetricsRecorder;
17+
import org.opensearch.dataprepper.plugins.source.microsoft_office365.metrics.Office365MetricsRecorder;
1918
import org.opensearch.dataprepper.plugins.source.source_crawler.utils.retry.RetryHandler;
2019
import org.opensearch.dataprepper.plugins.source.source_crawler.utils.retry.DefaultRetryStrategy;
2120
import org.opensearch.dataprepper.plugins.source.source_crawler.utils.retry.DefaultStatusCodeHandler;
@@ -45,20 +44,18 @@
4544
@Named
4645
public class Office365RestClient {
4746
private static final String MANAGEMENT_API_BASE_URL = "https://manage.office.com/api/v1.0/";
48-
private static final String API_CALLS = "apiCalls";
49-
5047
private final RestTemplate restTemplate = new RestTemplate();
5148
private final RetryHandler retryHandler;
5249
private final Office365AuthenticationInterface authConfig;
5350
private final VendorAPIMetricsRecorder metricsRecorder;
54-
private final Counter apiCallsCounter;
51+
private final Office365MetricsRecorder office365MetricsRecorder;
5552

5653
public Office365RestClient(final Office365AuthenticationInterface authConfig,
57-
final PluginMetrics pluginMetrics,
58-
final VendorAPIMetricsRecorder metricsRecorder) {
54+
final VendorAPIMetricsRecorder metricsRecorder,
55+
final Office365MetricsRecorder office365MetricsRecorder) {
5956
this.authConfig = authConfig;
6057
this.metricsRecorder = metricsRecorder;
61-
this.apiCallsCounter = pluginMetrics.counter(API_CALLS);
58+
this.office365MetricsRecorder = office365MetricsRecorder;
6259
this.retryHandler = new RetryHandler(
6360
new DefaultRetryStrategy(),
6461
new DefaultStatusCodeHandler());
@@ -69,59 +66,66 @@ public Office365RestClient(final Office365AuthenticationInterface authConfig,
6966
*/
7067
public void startSubscriptions() {
7168
log.info("Starting Office 365 subscriptions for audit logs");
72-
try {
73-
HttpHeaders headers = new HttpHeaders();
74-
headers.setContentType(MediaType.APPLICATION_JSON);
69+
70+
office365MetricsRecorder.recordStartSubscriptionLatency(() -> {
71+
try {
72+
HttpHeaders headers = new HttpHeaders();
73+
headers.setContentType(MediaType.APPLICATION_JSON);
7574

76-
// TODO: Only start the subscriptions only if the call commented
77-
// out below doesn't return all the audit log types
78-
// Check current subscriptions
79-
// final String SUBSCRIPTION_LIST_URL = MANAGEMENT_API_BASE_URL + "%s/activity/feed/subscriptions/list";
80-
// String listUrl = String.format(SUBSCRIPTION_LIST_URL, authConfig.getTenantId());
75+
// TODO: Only start the subscriptions only if the call commented
76+
// out below doesn't return all the audit log types
77+
// Check current subscriptions
78+
// final String SUBSCRIPTION_LIST_URL = MANAGEMENT_API_BASE_URL + "%s/activity/feed/subscriptions/list";
79+
// String listUrl = String.format(SUBSCRIPTION_LIST_URL, authConfig.getTenantId());
8180
//
82-
// ResponseEntity<String> listResponse = restTemplate.exchange(
83-
// listUrl,
84-
// HttpMethod.GET,
85-
// new HttpEntity<>(headers),
86-
// String.class
87-
// );
88-
// log.debug("Current subscriptions: {}", listResponse.getBody());
81+
// ResponseEntity<String> listResponse = restTemplate.exchange(
82+
// listUrl,
83+
// HttpMethod.GET,
84+
// new HttpEntity<>(headers),
85+
// String.class
86+
// );
87+
// log.debug("Current subscriptions: {}", listResponse.getBody());
8988

90-
// Start subscriptions for each content type
91-
headers.setContentLength(0);
89+
// Start subscriptions for each content type
90+
headers.setContentLength(0);
9291

93-
for (String contentType : CONTENT_TYPES) {
94-
final String SUBSCRIPTION_START_URL = MANAGEMENT_API_BASE_URL + "%s/activity/feed/subscriptions/start?contentType=%s";
95-
String url = String.format(SUBSCRIPTION_START_URL,
96-
authConfig.getTenantId(),
97-
contentType);
92+
for (String contentType : CONTENT_TYPES) {
93+
final String SUBSCRIPTION_START_URL = MANAGEMENT_API_BASE_URL + "%s/activity/feed/subscriptions/start?contentType=%s";
94+
String url = String.format(SUBSCRIPTION_START_URL,
95+
authConfig.getTenantId(),
96+
contentType);
9897

99-
retryHandler.executeWithRetry(() -> {
100-
try {
101-
headers.setBearerAuth(authConfig.getAccessToken());
102-
apiCallsCounter.increment();
103-
ResponseEntity<String> response = restTemplate.exchange(
104-
url,
105-
HttpMethod.POST,
106-
new HttpEntity<>(headers),
107-
String.class
108-
);
109-
log.debug("Started subscription for {}: {}", contentType, response.getBody());
110-
return response;
111-
} catch (HttpClientErrorException | HttpServerErrorException e) {
112-
if (e.getResponseBodyAsString().contains("AF20024")) {
113-
log.debug("Subscription for {} is already enabled", contentType);
114-
return null;
98+
retryHandler.executeWithRetry(() -> {
99+
try {
100+
headers.setBearerAuth(authConfig.getAccessToken());
101+
office365MetricsRecorder.recordStartSubscriptionCall();
102+
103+
ResponseEntity<String> response = restTemplate.exchange(
104+
url,
105+
HttpMethod.POST,
106+
new HttpEntity<>(headers),
107+
String.class
108+
);
109+
log.debug("Started subscription for {}: {}", contentType, response.getBody());
110+
return response;
111+
} catch (HttpClientErrorException | HttpServerErrorException e) {
112+
if (e.getResponseBodyAsString().contains("AF20024")) {
113+
log.debug("Subscription for {} is already enabled", contentType);
114+
return null;
115+
}
116+
throw e;
115117
}
116-
throw e;
117-
}
118-
}, authConfig::renewCredentials);
118+
}, authConfig::renewCredentials, office365MetricsRecorder::recordStartSubscriptionFailure);
119+
}
120+
121+
office365MetricsRecorder.recordStartSubscriptionSuccess();
122+
return null;
123+
} catch (Exception e) {
124+
metricsRecorder.recordError(e);
125+
log.error(NOISY, "Failed to initialize subscriptions", e);
126+
throw new SaaSCrawlerException("Failed to initialize subscriptions: " + e.getMessage(), e, true);
119127
}
120-
} catch (Exception e) {
121-
metricsRecorder.recordError(e);
122-
log.error(NOISY, "Failed to initialize subscriptions", e);
123-
throw new SaaSCrawlerException("Failed to initialize subscriptions: " + e.getMessage(), e, true);
124-
}
128+
});
125129
}
126130

127131
/**

‎data-prepper-plugins/saas-source-plugins/microsoft-office365-source/src/main/java/org/opensearch/dataprepper/plugins/source/microsoft_office365/auth/Office365AuthenticationProvider.java‎

Lines changed: 32 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111

1212
import lombok.Getter;
1313
import org.opensearch.dataprepper.plugins.source.microsoft_office365.Office365SourceConfig;
14+
import org.opensearch.dataprepper.plugins.source.source_crawler.metrics.VendorAPIMetricsRecorder;
1415
import org.opensearch.dataprepper.plugins.source.source_crawler.utils.retry.RetryHandler;
1516
import org.opensearch.dataprepper.plugins.source.source_crawler.utils.retry.DefaultRetryStrategy;
1617
import org.opensearch.dataprepper.plugins.source.source_crawler.utils.retry.DefaultStatusCodeHandler;
@@ -45,6 +46,7 @@ public class Office365AuthenticationProvider implements Office365AuthenticationI
4546
private final RetryHandler retryHandler;
4647
private final String tenantId;
4748
private final Office365SourceConfig office365SourceConfig;
49+
private final VendorAPIMetricsRecorder metricsRecorder;
4850
private String accessToken;
4951
private final Object lock = new Object();
5052
private final Object accessTokenFetchLock = new Object();
@@ -53,12 +55,13 @@ public class Office365AuthenticationProvider implements Office365AuthenticationI
5355
@Getter
5456
private Instant expireTime = Instant.ofEpochMilli(0);
5557

56-
public Office365AuthenticationProvider(Office365SourceConfig config) {
58+
public Office365AuthenticationProvider(Office365SourceConfig config, VendorAPIMetricsRecorder metricsRecorder) {
5759
this.tenantId = config.getTenantId();
5860
this.office365SourceConfig = config;
5961
this.retryHandler = new RetryHandler(
6062
new DefaultRetryStrategy(),
6163
new DefaultStatusCodeHandler());
64+
this.metricsRecorder = metricsRecorder;
6265
}
6366

6467
@Override
@@ -82,22 +85,37 @@ public void renewCredentials() {
8285
HttpEntity<String> entity = new HttpEntity<>(payload, headers);
8386
String tokenEndpoint = String.format(TOKEN_URL, office365SourceConfig.getTenantId());
8487

85-
ResponseEntity<Map> response = retryHandler.executeWithRetry(
86-
() -> restTemplate.postForEntity(tokenEndpoint, entity, Map.class),
87-
() -> {
88-
} // No credential renewal for authentication endpoint
89-
);
88+
try {
89+
// Record authentication latency and execute the request
90+
ResponseEntity<Map> response = metricsRecorder.recordAuthLatency(() ->
91+
retryHandler.executeWithRetry(
92+
() -> restTemplate.postForEntity(tokenEndpoint, entity, Map.class),
93+
() -> {
94+
} // No credential renewal for authentication endpoint
95+
)
96+
);
9097

91-
Map<String, Object> tokenResponse = response.getBody();
98+
Map<String, Object> tokenResponse = response.getBody();
9299

93-
if (tokenResponse == null || tokenResponse.get("access_token") == null) {
94-
throw new IllegalStateException("Invalid token response: missing access_token");
95-
}
100+
if (tokenResponse == null || tokenResponse.get("access_token") == null) {
101+
throw new IllegalStateException("Invalid token response: missing access_token");
102+
}
96103

97-
this.accessToken = (String) tokenResponse.get("access_token");
98-
int expiresIn = (int) tokenResponse.get("expires_in");
99-
this.expireTime = Instant.now().plusSeconds(expiresIn);
100-
log.info("Received new access token. Expires in {} seconds", expiresIn);
104+
this.accessToken = (String) tokenResponse.get("access_token");
105+
int expiresIn = (int) tokenResponse.get("expires_in");
106+
this.expireTime = Instant.now().plusSeconds(expiresIn);
107+
108+
// Record successful authentication
109+
metricsRecorder.recordAuthSuccess();
110+
111+
log.info("Received new access token. Expires in {} seconds", expiresIn);
112+
} catch (Exception e) {
113+
// Record authentication failure and specific error details
114+
metricsRecorder.recordAuthFailure();
115+
metricsRecorder.recordError(e);
116+
log.error("Failed to renew Office 365 credentials", e);
117+
throw e;
118+
}
101119
}
102120
}
103121

‎data-prepper-plugins/saas-source-plugins/microsoft-office365-source/src/main/java/org/opensearch/dataprepper/plugins/source/microsoft_office365/configuration/Office365Configuration.java‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*
5+
* The OpenSearch Contributors require contributions made to
6+
* this file be licensed under the Apache-2.0 license or a
7+
* compatible open source license.
8+
*/
9+
10+
package org.opensearch.dataprepper.plugins.source.microsoft_office365.configuration;
11+
12+
import org.opensearch.dataprepper.metrics.PluginMetrics;
13+
import org.opensearch.dataprepper.plugins.source.microsoft_office365.Office365RestClient;
14+
import org.opensearch.dataprepper.plugins.source.microsoft_office365.Office365SourceConfig;
15+
import org.opensearch.dataprepper.plugins.source.microsoft_office365.auth.Office365AuthenticationInterface;
16+
import org.opensearch.dataprepper.plugins.source.microsoft_office365.auth.Office365AuthenticationProvider;
17+
import org.opensearch.dataprepper.plugins.source.source_crawler.metrics.VendorAPIMetricsRecorder;
18+
import org.opensearch.dataprepper.plugins.source.microsoft_office365.metrics.Office365MetricsRecorder;
19+
import org.springframework.context.annotation.Bean;
20+
import org.springframework.context.annotation.Configuration;
21+
22+
/**
23+
* Spring configuration for Microsoft Office 365 components.
24+
*
25+
* This configuration class creates all Office 365 related beans with their dependencies:
26+
* 1. VendorAPIMetricsRecorder for unified metrics across all operations
27+
* 2. Office365AuthenticationProvider with authentication metrics support
28+
* 3. Office365MetricsRecorder for M365-specific subscription metrics
29+
* 4. Office365RestClient with both metrics recorders
30+
*
31+
* This provides comprehensive metrics coverage for authentication, API calls,
32+
* and subscription management operations.
33+
*/
34+
@Configuration
35+
public class Office365Configuration {
36+
37+
/**
38+
* Creates VendorAPIMetricsRecorder with unified metrics for all operations.
39+
*
40+
* @param pluginMetrics The system plugin metrics instance
41+
* @return Configured VendorAPIMetricsRecorder
42+
*/
43+
@Bean
44+
public VendorAPIMetricsRecorder vendorAPIMetricsRecorder(PluginMetrics pluginMetrics) {
45+
return new VendorAPIMetricsRecorder(pluginMetrics);
46+
}
47+
48+
/**
49+
* Creates Office365AuthenticationProvider with metrics recording capabilities.
50+
*
51+
* @param config The Office 365 source configuration
52+
* @param metricsRecorder The metrics recorder for authentication operations
53+
* @return Configured Office365AuthenticationProvider with metrics support
54+
*/
55+
@Bean
56+
public Office365AuthenticationProvider office365AuthenticationProvider(
57+
Office365SourceConfig config,
58+
VendorAPIMetricsRecorder metricsRecorder) {
59+
return new Office365AuthenticationProvider(config, metricsRecorder);
60+
}
61+
62+
/**
63+
* Creates Office365MetricsRecorder with M365-specific subscription metrics.
64+
*
65+
* @param pluginMetrics The system plugin metrics instance
66+
* @return Configured Office365MetricsRecorder
67+
*/
68+
@Bean
69+
public Office365MetricsRecorder office365MetricsRecorder(PluginMetrics pluginMetrics) {
70+
return new Office365MetricsRecorder(pluginMetrics);
71+
}
72+
73+
/**
74+
* Creates Office365RestClient with unified metrics recorder and subscription metrics recorder.
75+
*
76+
* @param authConfig The Office 365 authentication provider
77+
* @param vendorAPIMetricsRecorder The unified metrics recorder
78+
* @param office365MetricsRecorder The M365-specific subscription metrics recorder
79+
* @return Configured Office365RestClient
80+
*/
81+
@Bean
82+
public Office365RestClient office365RestClient(
83+
Office365AuthenticationInterface authConfig,
84+
VendorAPIMetricsRecorder vendorAPIMetricsRecorder,
85+
Office365MetricsRecorder office365MetricsRecorder) {
86+
return new Office365RestClient(authConfig, vendorAPIMetricsRecorder, office365MetricsRecorder);
87+
}
88+
}

‎data-prepper-plugins/saas-source-plugins/microsoft-office365-source/src/main/java/org/opensearch/dataprepper/plugins/source/microsoft_office365/configuration/Office365RestClientConfiguration.java‎

Lines changed: 0 additions & 58 deletions
This file was deleted.

0 commit comments

Comments
 (0)