Skip to content

Commit d9ef33f

Browse files
chrisale000chrisale000
committed
Adding Subscription Metrics to Metric Recorder and Onboarding M365 to Auth Metrics from Metrics Recorder
Signed-off-by: Alexander Christensen <[email protected]>
1 parent 6ef4e48 commit d9ef33f

File tree

7 files changed

+505
-89
lines changed

7 files changed

+505
-89
lines changed

‎data-prepper-plugins/saas-source-plugins/microsoft-office365-source/src/main/java/org/opensearch/dataprepper/plugins/source/microsoft_office365/Office365RestClient.java‎

Lines changed: 54 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,7 @@
99

1010
package org.opensearch.dataprepper.plugins.source.microsoft_office365;
1111

12-
import io.micrometer.core.instrument.Counter;
1312
import lombok.extern.slf4j.Slf4j;
14-
import org.opensearch.dataprepper.metrics.PluginMetrics;
1513
import org.opensearch.dataprepper.plugins.source.microsoft_office365.auth.Office365AuthenticationInterface;
1614
import org.opensearch.dataprepper.plugins.source.source_crawler.exception.SaaSCrawlerException;
1715
import org.opensearch.dataprepper.plugins.source.microsoft_office365.models.AuditLogsResponse;
@@ -45,20 +43,15 @@
4543
@Named
4644
public class Office365RestClient {
4745
private static final String MANAGEMENT_API_BASE_URL = "https://manage.office.com/api/v1.0/";
48-
private static final String API_CALLS = "apiCalls";
49-
5046
private final RestTemplate restTemplate = new RestTemplate();
5147
private final RetryHandler retryHandler;
5248
private final Office365AuthenticationInterface authConfig;
5349
private final VendorAPIMetricsRecorder metricsRecorder;
54-
private final Counter apiCallsCounter;
5550

5651
public Office365RestClient(final Office365AuthenticationInterface authConfig,
57-
final PluginMetrics pluginMetrics,
5852
final VendorAPIMetricsRecorder metricsRecorder) {
5953
this.authConfig = authConfig;
6054
this.metricsRecorder = metricsRecorder;
61-
this.apiCallsCounter = pluginMetrics.counter(API_CALLS);
6255
this.retryHandler = new RetryHandler(
6356
new DefaultRetryStrategy(),
6457
new DefaultStatusCodeHandler());
@@ -69,59 +62,66 @@ public Office365RestClient(final Office365AuthenticationInterface authConfig,
6962
*/
7063
public void startSubscriptions() {
7164
log.info("Starting Office 365 subscriptions for audit logs");
72-
try {
73-
HttpHeaders headers = new HttpHeaders();
74-
headers.setContentType(MediaType.APPLICATION_JSON);
65+
66+
metricsRecorder.recordSubscriptionLatency(() -> {
67+
try {
68+
HttpHeaders headers = new HttpHeaders();
69+
headers.setContentType(MediaType.APPLICATION_JSON);
7570

76-
// TODO: Only start the subscriptions only if the call commented
77-
// out below doesn't return all the audit log types
78-
// Check current subscriptions
79-
// final String SUBSCRIPTION_LIST_URL = MANAGEMENT_API_BASE_URL + "%s/activity/feed/subscriptions/list";
80-
// String listUrl = String.format(SUBSCRIPTION_LIST_URL, authConfig.getTenantId());
81-
//
82-
// ResponseEntity<String> listResponse = restTemplate.exchange(
83-
// listUrl,
84-
// HttpMethod.GET,
85-
// new HttpEntity<>(headers),
86-
// String.class
87-
// );
88-
// log.debug("Current subscriptions: {}", listResponse.getBody());
71+
// TODO: Only start the subscriptions only if the call commented
72+
// out below doesn't return all the audit log types
73+
// Check current subscriptions
74+
// final String SUBSCRIPTION_LIST_URL = MANAGEMENT_API_BASE_URL + "%s/activity/feed/subscriptions/list";
75+
// String listUrl = String.format(SUBSCRIPTION_LIST_URL, authConfig.getTenantId());
76+
//
77+
// ResponseEntity<String> listResponse = restTemplate.exchange(
78+
// listUrl,
79+
// HttpMethod.GET,
80+
// new HttpEntity<>(headers),
81+
// String.class
82+
// );
83+
// log.debug("Current subscriptions: {}", listResponse.getBody());
8984

90-
// Start subscriptions for each content type
91-
headers.setContentLength(0);
85+
// Start subscriptions for each content type
86+
headers.setContentLength(0);
9287

93-
for (String contentType : CONTENT_TYPES) {
94-
final String SUBSCRIPTION_START_URL = MANAGEMENT_API_BASE_URL + "%s/activity/feed/subscriptions/start?contentType=%s";
95-
String url = String.format(SUBSCRIPTION_START_URL,
96-
authConfig.getTenantId(),
97-
contentType);
88+
for (String contentType : CONTENT_TYPES) {
89+
final String SUBSCRIPTION_START_URL = MANAGEMENT_API_BASE_URL + "%s/activity/feed/subscriptions/start?contentType=%s";
90+
String url = String.format(SUBSCRIPTION_START_URL,
91+
authConfig.getTenantId(),
92+
contentType);
9893

99-
retryHandler.executeWithRetry(() -> {
100-
try {
101-
headers.setBearerAuth(authConfig.getAccessToken());
102-
apiCallsCounter.increment();
103-
ResponseEntity<String> response = restTemplate.exchange(
104-
url,
105-
HttpMethod.POST,
106-
new HttpEntity<>(headers),
107-
String.class
108-
);
109-
log.debug("Started subscription for {}: {}", contentType, response.getBody());
110-
return response;
111-
} catch (HttpClientErrorException | HttpServerErrorException e) {
112-
if (e.getResponseBodyAsString().contains("AF20024")) {
113-
log.debug("Subscription for {} is already enabled", contentType);
114-
return null;
94+
retryHandler.executeWithRetry(() -> {
95+
try {
96+
headers.setBearerAuth(authConfig.getAccessToken());
97+
metricsRecorder.recordSubscriptionCall();
98+
99+
ResponseEntity<String> response = restTemplate.exchange(
100+
url,
101+
HttpMethod.POST,
102+
new HttpEntity<>(headers),
103+
String.class
104+
);
105+
log.debug("Started subscription for {}: {}", contentType, response.getBody());
106+
return response;
107+
} catch (HttpClientErrorException | HttpServerErrorException e) {
108+
if (e.getResponseBodyAsString().contains("AF20024")) {
109+
log.debug("Subscription for {} is already enabled", contentType);
110+
return null;
111+
}
112+
throw e;
115113
}
116-
throw e;
117-
}
118-
}, authConfig::renewCredentials);
114+
}, authConfig::renewCredentials, metricsRecorder::recordSubscriptionFailure);
115+
}
116+
117+
metricsRecorder.recordSubscriptionSuccess();
118+
return null;
119+
} catch (Exception e) {
120+
metricsRecorder.recordError(e);
121+
log.error(NOISY, "Failed to initialize subscriptions", e);
122+
throw new SaaSCrawlerException("Failed to initialize subscriptions: " + e.getMessage(), e, true);
119123
}
120-
} catch (Exception e) {
121-
metricsRecorder.recordError(e);
122-
log.error(NOISY, "Failed to initialize subscriptions", e);
123-
throw new SaaSCrawlerException("Failed to initialize subscriptions: " + e.getMessage(), e, true);
124-
}
124+
});
125125
}
126126

127127
/**

‎data-prepper-plugins/saas-source-plugins/microsoft-office365-source/src/main/java/org/opensearch/dataprepper/plugins/source/microsoft_office365/auth/Office365AuthenticationProvider.java‎

Lines changed: 32 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111

1212
import lombok.Getter;
1313
import org.opensearch.dataprepper.plugins.source.microsoft_office365.Office365SourceConfig;
14+
import org.opensearch.dataprepper.plugins.source.source_crawler.metrics.VendorAPIMetricsRecorder;
1415
import org.opensearch.dataprepper.plugins.source.source_crawler.utils.retry.RetryHandler;
1516
import org.opensearch.dataprepper.plugins.source.source_crawler.utils.retry.DefaultRetryStrategy;
1617
import org.opensearch.dataprepper.plugins.source.source_crawler.utils.retry.DefaultStatusCodeHandler;
@@ -45,6 +46,7 @@ public class Office365AuthenticationProvider implements Office365AuthenticationI
4546
private final RetryHandler retryHandler;
4647
private final String tenantId;
4748
private final Office365SourceConfig office365SourceConfig;
49+
private final VendorAPIMetricsRecorder metricsRecorder;
4850
private String accessToken;
4951
private final Object lock = new Object();
5052
private final Object accessTokenFetchLock = new Object();
@@ -53,12 +55,13 @@ public class Office365AuthenticationProvider implements Office365AuthenticationI
5355
@Getter
5456
private Instant expireTime = Instant.ofEpochMilli(0);
5557

56-
public Office365AuthenticationProvider(Office365SourceConfig config) {
58+
public Office365AuthenticationProvider(Office365SourceConfig config, VendorAPIMetricsRecorder metricsRecorder) {
5759
this.tenantId = config.getTenantId();
5860
this.office365SourceConfig = config;
5961
this.retryHandler = new RetryHandler(
6062
new DefaultRetryStrategy(),
6163
new DefaultStatusCodeHandler());
64+
this.metricsRecorder = metricsRecorder;
6265
}
6366

6467
@Override
@@ -82,22 +85,37 @@ public void renewCredentials() {
8285
HttpEntity<String> entity = new HttpEntity<>(payload, headers);
8386
String tokenEndpoint = String.format(TOKEN_URL, office365SourceConfig.getTenantId());
8487

85-
ResponseEntity<Map> response = retryHandler.executeWithRetry(
86-
() -> restTemplate.postForEntity(tokenEndpoint, entity, Map.class),
87-
() -> {
88-
} // No credential renewal for authentication endpoint
89-
);
88+
try {
89+
// Record authentication latency and execute the request
90+
ResponseEntity<Map> response = metricsRecorder.recordAuthLatency(() ->
91+
retryHandler.executeWithRetry(
92+
() -> restTemplate.postForEntity(tokenEndpoint, entity, Map.class),
93+
() -> {
94+
} // No credential renewal for authentication endpoint
95+
)
96+
);
9097

91-
Map<String, Object> tokenResponse = response.getBody();
98+
Map<String, Object> tokenResponse = response.getBody();
9299

93-
if (tokenResponse == null || tokenResponse.get("access_token") == null) {
94-
throw new IllegalStateException("Invalid token response: missing access_token");
95-
}
100+
if (tokenResponse == null || tokenResponse.get("access_token") == null) {
101+
throw new IllegalStateException("Invalid token response: missing access_token");
102+
}
96103

97-
this.accessToken = (String) tokenResponse.get("access_token");
98-
int expiresIn = (int) tokenResponse.get("expires_in");
99-
this.expireTime = Instant.now().plusSeconds(expiresIn);
100-
log.info("Received new access token. Expires in {} seconds", expiresIn);
104+
this.accessToken = (String) tokenResponse.get("access_token");
105+
int expiresIn = (int) tokenResponse.get("expires_in");
106+
this.expireTime = Instant.now().plusSeconds(expiresIn);
107+
108+
// Record successful authentication
109+
metricsRecorder.recordAuthSuccess();
110+
111+
log.info("Received new access token. Expires in {} seconds", expiresIn);
112+
} catch (Exception e) {
113+
// Record authentication failure and specific error details
114+
metricsRecorder.recordAuthFailure();
115+
metricsRecorder.recordError(e);
116+
log.error("Failed to renew Office 365 credentials", e);
117+
throw e;
118+
}
101119
}
102120
}
103121

data-prepper-plugins/saas-source-plugins/microsoft-office365-source/src/main/java/org/opensearch/dataprepper/plugins/source/microsoft_office365/configuration/Office365RestClientConfiguration.java renamed to data-prepper-plugins/saas-source-plugins/microsoft-office365-source/src/main/java/org/opensearch/dataprepper/plugins/source/microsoft_office365/configuration/Office365Configuration.java

Lines changed: 26 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -11,23 +11,26 @@
1111

1212
import org.opensearch.dataprepper.metrics.PluginMetrics;
1313
import org.opensearch.dataprepper.plugins.source.microsoft_office365.Office365RestClient;
14+
import org.opensearch.dataprepper.plugins.source.microsoft_office365.Office365SourceConfig;
1415
import org.opensearch.dataprepper.plugins.source.microsoft_office365.auth.Office365AuthenticationInterface;
16+
import org.opensearch.dataprepper.plugins.source.microsoft_office365.auth.Office365AuthenticationProvider;
1517
import org.opensearch.dataprepper.plugins.source.source_crawler.metrics.VendorAPIMetricsRecorder;
1618
import org.springframework.context.annotation.Bean;
1719
import org.springframework.context.annotation.Configuration;
1820

1921
/**
20-
* Spring configuration for Microsoft Office 365 RestClient.
22+
* Spring configuration for Microsoft Office 365 components.
2123
*
22-
* This configuration class creates the Office365RestClient with the required dependencies:
23-
* 1. Office365AuthenticationInterface for authentication
24-
* 2. PluginMetrics for unified metrics recording
24+
* This configuration class creates all Office 365 related beans with their dependencies:
25+
* 1. VendorAPIMetricsRecorder for unified metrics across all operations
26+
* 2. Office365AuthenticationProvider with authentication metrics support
27+
* 3. Office365RestClient with VendorAPIMetricsRecorder
2528
*
26-
* The Office365RestClient internally creates VendorAPIMetricsRecorder instances
27-
* for different operation types (GET, SEARCH, AUTH) from the provided PluginMetrics.
29+
* This provides comprehensive metrics coverage for authentication, API calls,
30+
* and subscription management operations.
2831
*/
2932
@Configuration
30-
public class Office365RestClientConfiguration {
33+
public class Office365Configuration {
3134

3235
/**
3336
* Creates VendorAPIMetricsRecorder with unified metrics for all operations.
@@ -40,19 +43,32 @@ public VendorAPIMetricsRecorder vendorAPIMetricsRecorder(PluginMetrics pluginMet
4043
return new VendorAPIMetricsRecorder(pluginMetrics);
4144
}
4245

46+
/**
47+
* Creates Office365AuthenticationProvider with metrics recording capabilities.
48+
*
49+
* @param config The Office 365 source configuration
50+
* @param metricsRecorder The metrics recorder for authentication operations
51+
* @return Configured Office365AuthenticationProvider with metrics support
52+
*/
53+
@Bean
54+
public Office365AuthenticationProvider office365AuthenticationProvider(
55+
Office365SourceConfig config,
56+
VendorAPIMetricsRecorder metricsRecorder) {
57+
return new Office365AuthenticationProvider(config, metricsRecorder);
58+
}
59+
60+
4361
/**
4462
* Creates Office365RestClient with unified metrics recorder.
4563
*
4664
* @param authConfig The Office 365 authentication provider
47-
* @param pluginMetrics The system plugin metrics instance
4865
* @param vendorAPIMetricsRecorder The unified metrics recorder
4966
* @return Configured Office365RestClient
5067
*/
5168
@Bean
5269
public Office365RestClient office365RestClient(
5370
Office365AuthenticationInterface authConfig,
54-
PluginMetrics pluginMetrics,
5571
VendorAPIMetricsRecorder vendorAPIMetricsRecorder) {
56-
return new Office365RestClient(authConfig, pluginMetrics, vendorAPIMetricsRecorder);
72+
return new Office365RestClient(authConfig, vendorAPIMetricsRecorder);
5773
}
5874
}

0 commit comments

Comments
 (0)