Skip to content

Commit 8af181c

Browse files
chrisale000chrisale000
committed
Adding M365 Specific Metrics Recorder and Emitting Auth Metrics
Signed-off-by: Alexander Christensen <[email protected]>
1 parent 6ef4e48 commit 8af181c

File tree

5 files changed

+591
-68
lines changed

5 files changed

+591
-68
lines changed

‎data-prepper-plugins/saas-source-plugins/microsoft-office365-source/src/main/java/org/opensearch/dataprepper/plugins/source/microsoft_office365/Office365RestClient.java‎

Lines changed: 58 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -9,13 +9,12 @@
99

1010
package org.opensearch.dataprepper.plugins.source.microsoft_office365;
1111

12-
import io.micrometer.core.instrument.Counter;
1312
import lombok.extern.slf4j.Slf4j;
14-
import org.opensearch.dataprepper.metrics.PluginMetrics;
1513
import org.opensearch.dataprepper.plugins.source.microsoft_office365.auth.Office365AuthenticationInterface;
1614
import org.opensearch.dataprepper.plugins.source.source_crawler.exception.SaaSCrawlerException;
1715
import org.opensearch.dataprepper.plugins.source.microsoft_office365.models.AuditLogsResponse;
1816
import org.opensearch.dataprepper.plugins.source.source_crawler.metrics.VendorAPIMetricsRecorder;
17+
import org.opensearch.dataprepper.plugins.source.microsoft_office365.metrics.Office365MetricsRecorder;
1918
import org.opensearch.dataprepper.plugins.source.source_crawler.utils.retry.RetryHandler;
2019
import org.opensearch.dataprepper.plugins.source.source_crawler.utils.retry.DefaultRetryStrategy;
2120
import org.opensearch.dataprepper.plugins.source.source_crawler.utils.retry.DefaultStatusCodeHandler;
@@ -45,20 +44,18 @@
4544
@Named
4645
public class Office365RestClient {
4746
private static final String MANAGEMENT_API_BASE_URL = "https://manage.office.com/api/v1.0/";
48-
private static final String API_CALLS = "apiCalls";
49-
5047
private final RestTemplate restTemplate = new RestTemplate();
5148
private final RetryHandler retryHandler;
5249
private final Office365AuthenticationInterface authConfig;
5350
private final VendorAPIMetricsRecorder metricsRecorder;
54-
private final Counter apiCallsCounter;
51+
private final Office365MetricsRecorder office365MetricsRecorder;
5552

5653
public Office365RestClient(final Office365AuthenticationInterface authConfig,
57-
final PluginMetrics pluginMetrics,
58-
final VendorAPIMetricsRecorder metricsRecorder) {
54+
final VendorAPIMetricsRecorder metricsRecorder,
55+
final Office365MetricsRecorder office365MetricsRecorder) {
5956
this.authConfig = authConfig;
6057
this.metricsRecorder = metricsRecorder;
61-
this.apiCallsCounter = pluginMetrics.counter(API_CALLS);
58+
this.office365MetricsRecorder = office365MetricsRecorder;
6259
this.retryHandler = new RetryHandler(
6360
new DefaultRetryStrategy(),
6461
new DefaultStatusCodeHandler());
@@ -69,59 +66,66 @@ public Office365RestClient(final Office365AuthenticationInterface authConfig,
6966
*/
7067
public void startSubscriptions() {
7168
log.info("Starting Office 365 subscriptions for audit logs");
72-
try {
73-
HttpHeaders headers = new HttpHeaders();
74-
headers.setContentType(MediaType.APPLICATION_JSON);
69+
70+
office365MetricsRecorder.recordStartSubscriptionLatency(() -> {
71+
try {
72+
HttpHeaders headers = new HttpHeaders();
73+
headers.setContentType(MediaType.APPLICATION_JSON);
7574

76-
// TODO: Only start the subscriptions only if the call commented
77-
// out below doesn't return all the audit log types
78-
// Check current subscriptions
79-
// final String SUBSCRIPTION_LIST_URL = MANAGEMENT_API_BASE_URL + "%s/activity/feed/subscriptions/list";
80-
// String listUrl = String.format(SUBSCRIPTION_LIST_URL, authConfig.getTenantId());
75+
// TODO: Only start the subscriptions only if the call commented
76+
// out below doesn't return all the audit log types
77+
// Check current subscriptions
78+
// final String SUBSCRIPTION_LIST_URL = MANAGEMENT_API_BASE_URL + "%s/activity/feed/subscriptions/list";
79+
// String listUrl = String.format(SUBSCRIPTION_LIST_URL, authConfig.getTenantId());
8180
//
82-
// ResponseEntity<String> listResponse = restTemplate.exchange(
83-
// listUrl,
84-
// HttpMethod.GET,
85-
// new HttpEntity<>(headers),
86-
// String.class
87-
// );
88-
// log.debug("Current subscriptions: {}", listResponse.getBody());
81+
// ResponseEntity<String> listResponse = restTemplate.exchange(
82+
// listUrl,
83+
// HttpMethod.GET,
84+
// new HttpEntity<>(headers),
85+
// String.class
86+
// );
87+
// log.debug("Current subscriptions: {}", listResponse.getBody());
8988

90-
// Start subscriptions for each content type
91-
headers.setContentLength(0);
89+
// Start subscriptions for each content type
90+
headers.setContentLength(0);
9291

93-
for (String contentType : CONTENT_TYPES) {
94-
final String SUBSCRIPTION_START_URL = MANAGEMENT_API_BASE_URL + "%s/activity/feed/subscriptions/start?contentType=%s";
95-
String url = String.format(SUBSCRIPTION_START_URL,
96-
authConfig.getTenantId(),
97-
contentType);
92+
for (String contentType : CONTENT_TYPES) {
93+
final String SUBSCRIPTION_START_URL = MANAGEMENT_API_BASE_URL + "%s/activity/feed/subscriptions/start?contentType=%s";
94+
String url = String.format(SUBSCRIPTION_START_URL,
95+
authConfig.getTenantId(),
96+
contentType);
9897

99-
retryHandler.executeWithRetry(() -> {
100-
try {
101-
headers.setBearerAuth(authConfig.getAccessToken());
102-
apiCallsCounter.increment();
103-
ResponseEntity<String> response = restTemplate.exchange(
104-
url,
105-
HttpMethod.POST,
106-
new HttpEntity<>(headers),
107-
String.class
108-
);
109-
log.debug("Started subscription for {}: {}", contentType, response.getBody());
110-
return response;
111-
} catch (HttpClientErrorException | HttpServerErrorException e) {
112-
if (e.getResponseBodyAsString().contains("AF20024")) {
113-
log.debug("Subscription for {} is already enabled", contentType);
114-
return null;
98+
retryHandler.executeWithRetry(() -> {
99+
try {
100+
headers.setBearerAuth(authConfig.getAccessToken());
101+
office365MetricsRecorder.recordStartSubscriptionCall();
102+
103+
ResponseEntity<String> response = restTemplate.exchange(
104+
url,
105+
HttpMethod.POST,
106+
new HttpEntity<>(headers),
107+
String.class
108+
);
109+
log.debug("Started subscription for {}: {}", contentType, response.getBody());
110+
return response;
111+
} catch (HttpClientErrorException | HttpServerErrorException e) {
112+
if (e.getResponseBodyAsString().contains("AF20024")) {
113+
log.debug("Subscription for {} is already enabled", contentType);
114+
return null;
115+
}
116+
throw e;
115117
}
116-
throw e;
117-
}
118-
}, authConfig::renewCredentials);
118+
}, authConfig::renewCredentials, office365MetricsRecorder::recordStartSubscriptionFailure);
119+
}
120+
121+
office365MetricsRecorder.recordStartSubscriptionSuccess();
122+
return null;
123+
} catch (Exception e) {
124+
metricsRecorder.recordError(e);
125+
log.error(NOISY, "Failed to initialize subscriptions", e);
126+
throw new SaaSCrawlerException("Failed to initialize subscriptions: " + e.getMessage(), e, true);
119127
}
120-
} catch (Exception e) {
121-
metricsRecorder.recordError(e);
122-
log.error(NOISY, "Failed to initialize subscriptions", e);
123-
throw new SaaSCrawlerException("Failed to initialize subscriptions: " + e.getMessage(), e, true);
124-
}
128+
});
125129
}
126130

127131
/**

‎data-prepper-plugins/saas-source-plugins/microsoft-office365-source/src/main/java/org/opensearch/dataprepper/plugins/source/microsoft_office365/configuration/Office365RestClientConfiguration.java‎

Lines changed: 17 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
import org.opensearch.dataprepper.plugins.source.microsoft_office365.Office365RestClient;
1414
import org.opensearch.dataprepper.plugins.source.microsoft_office365.auth.Office365AuthenticationInterface;
1515
import org.opensearch.dataprepper.plugins.source.source_crawler.metrics.VendorAPIMetricsRecorder;
16+
import org.opensearch.dataprepper.plugins.source.microsoft_office365.metrics.Office365MetricsRecorder;
1617
import org.springframework.context.annotation.Bean;
1718
import org.springframework.context.annotation.Configuration;
1819

@@ -41,18 +42,29 @@ public VendorAPIMetricsRecorder vendorAPIMetricsRecorder(PluginMetrics pluginMet
4142
}
4243

4344
/**
44-
* Creates Office365RestClient with unified metrics recorder.
45+
* Creates Office365MetricsRecorder with M365-specific subscription metrics.
4546
*
46-
* @param authConfig The Office 365 authentication provider
4747
* @param pluginMetrics The system plugin metrics instance
48+
* @return Configured Office365MetricsRecorder
49+
*/
50+
@Bean
51+
public Office365MetricsRecorder office365MetricsRecorder(PluginMetrics pluginMetrics) {
52+
return new Office365MetricsRecorder(pluginMetrics);
53+
}
54+
55+
/**
56+
* Creates Office365RestClient with unified metrics recorder and subscription metrics recorder.
57+
*
58+
* @param authConfig The Office 365 authentication provider
4859
* @param vendorAPIMetricsRecorder The unified metrics recorder
60+
* @param office365MetricsRecorder The M365-specific subscription metrics recorder
4961
* @return Configured Office365RestClient
5062
*/
5163
@Bean
5264
public Office365RestClient office365RestClient(
5365
Office365AuthenticationInterface authConfig,
54-
PluginMetrics pluginMetrics,
55-
VendorAPIMetricsRecorder vendorAPIMetricsRecorder) {
56-
return new Office365RestClient(authConfig, pluginMetrics, vendorAPIMetricsRecorder);
66+
VendorAPIMetricsRecorder vendorAPIMetricsRecorder,
67+
Office365MetricsRecorder office365MetricsRecorder) {
68+
return new Office365RestClient(authConfig, vendorAPIMetricsRecorder, office365MetricsRecorder);
5769
}
5870
}

‎data-prepper-plugins/saas-source-plugins/microsoft-office365-source/src/main/java/org/opensearch/dataprepper/plugins/source/microsoft_office365/metrics/Office365MetricsRecorder.java‎

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*
5+
* The OpenSearch Contributors require contributions made to
6+
* this file be licensed under the Apache-2.0 license or a
7+
* compatible open source license.
8+
*/
9+
10+
package org.opensearch.dataprepper.plugins.source.microsoft_office365.metrics;
11+
12+
import io.micrometer.core.instrument.Counter;
13+
import io.micrometer.core.instrument.Timer;
14+
import org.opensearch.dataprepper.metrics.PluginMetrics;
15+
16+
import java.time.Duration;
17+
import java.util.function.Supplier;
18+
19+
/**
20+
* Microsoft Office 365 specific metrics recorder for Office 365 APIs that are unique to Office 365.
21+
*
22+
* This class records metrics for Office 365 specific API operations that are not found in other vendor APIs:
23+
* - StartSubscription latency: Time taken to start/manage Office 365 audit log subscriptions
24+
* - StartSubscription success/failure rates: Track successful and failed subscription operations
25+
* - StartSubscription call counts: Track individual API calls made for subscription management
26+
*
27+
* This is separate from the shared VendorAPIMetricsRecorder to isolate Office 365-specific
28+
* API metrics that are unique to the Office 365 Management API within the M365 module.
29+
*
30+
* NOTE: Any new Office 365 specific metrics should be implemented here to maintain proper
31+
* separation of vendor-specific functionality and keep Office 365 metrics centralized.
32+
*/
33+
public class Office365MetricsRecorder {
34+
35+
// StartSubscription operation metrics
36+
private final Counter startSubscriptionSuccessCounter;
37+
private final Counter startSubscriptionFailureCounter;
38+
private final Timer startSubscriptionLatencyTimer;
39+
private final Counter startSubscriptionCallsCounter;
40+
41+
/**
42+
* Creates an Office365MetricsRecorder for startSubscription-specific metrics.
43+
*
44+
* @param pluginMetrics The plugin metrics instance
45+
*/
46+
public Office365MetricsRecorder(PluginMetrics pluginMetrics) {
47+
// Initialize startSubscription metrics without office365 prefix
48+
this.startSubscriptionSuccessCounter = pluginMetrics.counter("startSubscriptionRequestsSuccess");
49+
this.startSubscriptionFailureCounter = pluginMetrics.counter("startSubscriptionRequestsFailed");
50+
this.startSubscriptionLatencyTimer = pluginMetrics.timer("startSubscriptionRequestLatency");
51+
this.startSubscriptionCallsCounter = pluginMetrics.counter("startSubscriptionApiCalls");
52+
}
53+
54+
/**
55+
* Records a successful startSubscription operation.
56+
*/
57+
public void recordStartSubscriptionSuccess() {
58+
startSubscriptionSuccessCounter.increment();
59+
}
60+
61+
/**
62+
* Records a failed startSubscription operation.
63+
*/
64+
public void recordStartSubscriptionFailure() {
65+
startSubscriptionFailureCounter.increment();
66+
}
67+
68+
/**
69+
* Records the latency of a startSubscription operation using a Supplier.
70+
*
71+
* @param operation The operation to time
72+
* @param <T> The return type of the operation
73+
* @return The result of the operation
74+
*/
75+
public <T> T recordStartSubscriptionLatency(Supplier<T> operation) {
76+
return startSubscriptionLatencyTimer.record(operation);
77+
}
78+
79+
/**
80+
* Records the latency of a startSubscription operation using a Runnable.
81+
*
82+
* @param operation The operation to time
83+
*/
84+
public void recordStartSubscriptionLatency(Runnable operation) {
85+
startSubscriptionLatencyTimer.record(operation);
86+
}
87+
88+
/**
89+
* Records the latency of a startSubscription operation using a Duration.
90+
*
91+
* @param duration The duration to record
92+
*/
93+
public void recordStartSubscriptionLatency(Duration duration) {
94+
startSubscriptionLatencyTimer.record(duration);
95+
}
96+
97+
/**
98+
* Records an individual startSubscription API call.
99+
* This tracks the number of individual API requests made during startSubscription operations.
100+
*/
101+
public void recordStartSubscriptionCall() {
102+
startSubscriptionCallsCounter.increment();
103+
}
104+
}

0 commit comments

Comments
 (0)