initial changes, need to figure out monitoring pt parse issue (#42147)

Author: harsimar

sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulse.java

@@ -26,7 +26,7 @@ public class QuickPulse {
 
     public static QuickPulse create(HttpPipeline httpPipeline, Supplier<URL> endpointUrl,
         Supplier<String> instrumentationKey, @Nullable String roleName, @Nullable String roleInstance,
-        boolean useNormalizedValueForNonNormalizedCpuPercentage, String sdkVersion) {
+        String sdkVersion) {
 
         QuickPulse quickPulse = new QuickPulse();
 
@@ -40,8 +40,7 @@ public static QuickPulse create(HttpPipeline httpPipeline, Supplier<URL> endpoin
             } catch (InterruptedException e) {
                 Thread.currentThread().interrupt();
             }
-            quickPulse.initialize(httpPipeline, endpointUrl, instrumentationKey, roleName, roleInstance,
-                useNormalizedValueForNonNormalizedCpuPercentage, sdkVersion);
+            quickPulse.initialize(httpPipeline, endpointUrl, instrumentationKey, roleName, roleInstance, sdkVersion);
         });
         // the condition below will always be false, but by referencing the executor it ensures the
         // executor can't become unreachable in the middle of the execute() method execution above
@@ -65,8 +64,7 @@ public void add(TelemetryItem telemetryItem) {
     }
 
     private void initialize(HttpPipeline httpPipeline, Supplier<URL> endpointUrl, Supplier<String> instrumentationKey,
-        @Nullable String roleName, @Nullable String roleInstance,
-        boolean useNormalizedValueForNonNormalizedCpuPercentage, String sdkVersion) {
+        @Nullable String roleName, @Nullable String roleInstance, String sdkVersion) {
 
         String quickPulseId = UUID.randomUUID().toString().replace("-", "");
         ArrayBlockingQueue<HttpRequest> sendQueue = new ArrayBlockingQueue<>(256, true);
@@ -83,8 +81,7 @@ private void initialize(HttpPipeline httpPipeline, Supplier<URL> endpointUrl, Su
             instanceName = "Unknown host";
         }
 
-        QuickPulseDataCollector collector
-            = new QuickPulseDataCollector(useNormalizedValueForNonNormalizedCpuPercentage);
+        QuickPulseDataCollector collector = new QuickPulseDataCollector();
 
         QuickPulsePingSender quickPulsePingSender = new QuickPulsePingSender(httpPipeline, endpointUrl,
             instrumentationKey, roleName, instanceName, machineName, quickPulseId, sdkVersion);

sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollector.java

@@ -41,14 +41,12 @@ final class QuickPulseDataCollector {
     private final AtomicReference<Counters> counters = new AtomicReference<>(null);
     private final CpuPerformanceCounterCalculator cpuPerformanceCounterCalculator
         = getCpuPerformanceCounterCalculator();
-    private final boolean useNormalizedValueForNonNormalizedCpuPercentage;
 
     private volatile QuickPulseStatus quickPulseStatus = QuickPulseStatus.QP_IS_OFF;
 
     private volatile Supplier<String> instrumentationKeySupplier;
 
-    QuickPulseDataCollector(boolean useNormalizedValueForNonNormalizedCpuPercentage) {
-        this.useNormalizedValueForNonNormalizedCpuPercentage = useNormalizedValueForNonNormalizedCpuPercentage;
+    QuickPulseDataCollector() {
     }
 
     private static CpuPerformanceCounterCalculator getCpuPerformanceCounterCalculator() {
@@ -319,14 +317,14 @@ class FinalCounters {
         final long rdds;
         final double rddsDuration;
         final int unsuccessfulRdds;
-        final long memoryCommitted;
-        final double cpuUsage;
+        final long processPhysicalMemory;
+        final double processNormalizedCpuUsage;
         final List<QuickPulseDocument> documentList = new ArrayList<>();
 
         private FinalCounters(Counters currentCounters) {
 
-            memoryCommitted = getMemoryCommitted(memory);
-            cpuUsage = getNonNormalizedCpuPercentage(cpuPerformanceCounterCalculator);
+            processPhysicalMemory = getPhysicalMemory(memory);
+            processNormalizedCpuUsage = getNormalizedCpuPercentage(cpuPerformanceCounterCalculator);
             exceptions = currentCounters.exceptions.get();
 
             CountAndDuration countAndDuration
@@ -344,32 +342,28 @@ private FinalCounters(Counters currentCounters) {
             }
         }
 
-        private long getMemoryCommitted(@Nullable MemoryMXBean memory) {
+        private long getPhysicalMemory(@Nullable MemoryMXBean memory) {
             if (memory == null) {
                 return -1;
             }
             MemoryUsage heapMemoryUsage = memory.getHeapMemoryUsage();
-            if (heapMemoryUsage == null) {
+            MemoryUsage nonHeapMemoryUsage = memory.getNonHeapMemoryUsage();
+            if (heapMemoryUsage == null || nonHeapMemoryUsage == null) {
                 return -1;
             }
-            return heapMemoryUsage.getCommitted();
+            return heapMemoryUsage.getUsed() + nonHeapMemoryUsage.getUsed();
         }
 
         private double
-            getNonNormalizedCpuPercentage(@Nullable CpuPerformanceCounterCalculator cpuPerformanceCounterCalculator) {
+            getNormalizedCpuPercentage(@Nullable CpuPerformanceCounterCalculator cpuPerformanceCounterCalculator) {
             if (cpuPerformanceCounterCalculator == null) {
                 return -1;
             }
             Double cpuDatum = cpuPerformanceCounterCalculator.getCpuPercentage();
             if (cpuDatum == null) {
                 return -1;
             }
-
-            if (useNormalizedValueForNonNormalizedCpuPercentage) {
-                // normalize for backwards compatibility even though this is supposed to be non-normalized
-                cpuDatum /= operatingSystemMxBean.getAvailableProcessors();
-            }
-
+            cpuDatum /= operatingSystemMxBean.getAvailableProcessors();
             return cpuDatum;
         }
     }

sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataFetcher.java

@@ -151,8 +151,14 @@ private String buildPostEntity(QuickPulseDataCollector.FinalCounters counters) t
         metricsList.add(new QuickPulseMetrics("\\ApplicationInsights\\Dependency Calls Succeeded/Sec",
             counters.rdds - counters.unsuccessfulRdds, 1));
         metricsList.add(new QuickPulseMetrics("\\ApplicationInsights\\Exceptions/Sec", counters.exceptions, 1));
-        metricsList.add(new QuickPulseMetrics("\\Memory\\Committed Bytes", counters.memoryCommitted, 1));
-        metricsList.add(new QuickPulseMetrics("\\Processor(_Total)\\% Processor Time", counters.cpuUsage, 1));
+        // TODO: remove old memory counter name when service side makes the UI change
+        metricsList.add(new QuickPulseMetrics("\\Memory\\Committed Bytes", counters.processPhysicalMemory, 1));
+        metricsList.add(new QuickPulseMetrics("\\Process\\Physical Bytes", counters.processPhysicalMemory, 1));
+        // TODO: remove old cpu counter name when service side makes the UI change
+        metricsList
+            .add(new QuickPulseMetrics("\\Processor(_Total)\\% Processor Time", counters.processNormalizedCpuUsage, 1));
+        metricsList.add(
+            new QuickPulseMetrics("\\% Process\\Processor Time Normalized", counters.processNormalizedCpuUsage, 1));
 
         return metricsList;
     }

sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseCoordinatorTest.java

@@ -19,7 +19,7 @@ void testOnlyPings() throws InterruptedException {
         QuickPulseDataFetcher mockFetcher = mock(QuickPulseDataFetcher.class);
         QuickPulseDataSender mockSender = mock(QuickPulseDataSender.class);
         QuickPulsePingSender mockPingSender = mock(QuickPulsePingSender.class);
-        QuickPulseDataCollector collector = new QuickPulseDataCollector(true);
+        QuickPulseDataCollector collector = new QuickPulseDataCollector();
         Mockito.doReturn(new QuickPulseHeaderInfo(QuickPulseStatus.QP_IS_OFF)).when(mockPingSender).ping(null);
 
         QuickPulseCoordinatorInitData initData = new QuickPulseCoordinatorInitDataBuilder().withDataFetcher(mockFetcher)
@@ -64,7 +64,7 @@ void testOnePingAndThenOnePost() throws InterruptedException {
             .thenReturn(new QuickPulseHeaderInfo(QuickPulseStatus.QP_IS_ON),
                 new QuickPulseHeaderInfo(QuickPulseStatus.QP_IS_OFF));
 
-        QuickPulseDataCollector collector = new QuickPulseDataCollector(true);
+        QuickPulseDataCollector collector = new QuickPulseDataCollector();
         QuickPulseCoordinatorInitData initData = new QuickPulseCoordinatorInitDataBuilder().withDataFetcher(mockFetcher)
             .withDataSender(mockSender)
             .withPingSender(mockPingSender)
@@ -112,7 +112,7 @@ void testOnePingAndThenOnePostWithRedirectedLink() throws InterruptedException {
         QuickPulseCoordinatorInitData initData = new QuickPulseCoordinatorInitDataBuilder().withDataFetcher(mockFetcher)
             .withDataSender(mockSender)
             .withPingSender(mockPingSender)
-            .withCollector(new QuickPulseDataCollector(true))
+            .withCollector(new QuickPulseDataCollector())
             .withWaitBetweenPingsInMillis(10L)
             .withWaitBetweenPostsInMillis(10L)
             .withWaitOnErrorInMillis(10L)

sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollectorTests.java

@@ -23,12 +23,12 @@ class QuickPulseDataCollectorTests {
 
     @Test
     void initialStateIsDisabled() {
-        assertThat(new QuickPulseDataCollector(true).peek()).isNull();
+        assertThat(new QuickPulseDataCollector().peek()).isNull();
     }
 
     @Test
     void emptyCountsAndDurationsAfterEnable() {
-        QuickPulseDataCollector collector = new QuickPulseDataCollector(true);
+        QuickPulseDataCollector collector = new QuickPulseDataCollector();
 
         collector.enable(FAKE_CONNECTION_STRING::getInstrumentationKey);
         QuickPulseDataCollector.FinalCounters counters = collector.peek();
@@ -37,7 +37,7 @@ void emptyCountsAndDurationsAfterEnable() {
 
     @Test
     void nullCountersAfterDisable() {
-        QuickPulseDataCollector collector = new QuickPulseDataCollector(true);
+        QuickPulseDataCollector collector = new QuickPulseDataCollector();
 
         collector.enable(FAKE_CONNECTION_STRING::getInstrumentationKey);
         collector.disable();
@@ -46,7 +46,7 @@ void nullCountersAfterDisable() {
 
     @Test
     void requestTelemetryIsCounted_DurationIsSum() {
-        QuickPulseDataCollector collector = new QuickPulseDataCollector(true);
+        QuickPulseDataCollector collector = new QuickPulseDataCollector();
 
         collector.setQuickPulseStatus(QuickPulseStatus.QP_IS_ON);
         collector.enable(FAKE_CONNECTION_STRING::getInstrumentationKey);
@@ -88,7 +88,7 @@ void requestTelemetryIsCounted_DurationIsSum() {
 
     @Test
     void dependencyTelemetryIsCounted_DurationIsSum() {
-        QuickPulseDataCollector collector = new QuickPulseDataCollector(true);
+        QuickPulseDataCollector collector = new QuickPulseDataCollector();
 
         collector.setQuickPulseStatus(QuickPulseStatus.QP_IS_ON);
         collector.enable(FAKE_CONNECTION_STRING::getInstrumentationKey);
@@ -130,7 +130,7 @@ void dependencyTelemetryIsCounted_DurationIsSum() {
 
     @Test
     void exceptionTelemetryIsCounted() {
-        QuickPulseDataCollector collector = new QuickPulseDataCollector(true);
+        QuickPulseDataCollector collector = new QuickPulseDataCollector();
 
         collector.setQuickPulseStatus(QuickPulseStatus.QP_IS_ON);
         collector.enable(FAKE_CONNECTION_STRING::getInstrumentationKey);
@@ -204,7 +204,7 @@ private static void assertCountersReset(QuickPulseDataCollector.FinalCounters co
 
     @Test
     void checkDocumentsListSize() {
-        QuickPulseDataCollector collector = new QuickPulseDataCollector(true);
+        QuickPulseDataCollector collector = new QuickPulseDataCollector();
 
         collector.setQuickPulseStatus(QuickPulseStatus.QP_IS_ON);
         collector.enable(FAKE_CONNECTION_STRING::getInstrumentationKey);

sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataFetcherTests.java

@@ -24,7 +24,7 @@ class QuickPulseDataFetcherTests {
     @Test
     void testGetCurrentSdkVersion() {
         ConnectionString connectionString = ConnectionString.parse("InstrumentationKey=testing-123");
-        QuickPulseDataFetcher dataFetcher = new QuickPulseDataFetcher(new QuickPulseDataCollector(true), null,
+        QuickPulseDataFetcher dataFetcher = new QuickPulseDataFetcher(new QuickPulseDataCollector(), null,
             connectionString::getLiveEndpoint, connectionString::getInstrumentationKey, null, null, null, null);
         String sdkVersion = dataFetcher.getCurrentSdkVersion();
         assertThat(sdkVersion).isNotNull();
@@ -34,7 +34,7 @@ void testGetCurrentSdkVersion() {
     @Test
     void endpointIsFormattedCorrectlyWhenUsingConfig() throws URISyntaxException {
         ConnectionString connectionString = ConnectionString.parse("InstrumentationKey=testing-123");
-        QuickPulseDataFetcher quickPulseDataFetcher = new QuickPulseDataFetcher(new QuickPulseDataCollector(true), null,
+        QuickPulseDataFetcher quickPulseDataFetcher = new QuickPulseDataFetcher(new QuickPulseDataCollector(), null,
             connectionString::getLiveEndpoint, connectionString::getInstrumentationKey, null, null, null, null);
         String quickPulseEndpoint = quickPulseDataFetcher.getQuickPulseEndpoint();
         String endpointUrl = quickPulseDataFetcher.getEndpointUrl(quickPulseEndpoint);
@@ -47,7 +47,7 @@ void endpointIsFormattedCorrectlyWhenUsingConfig() throws URISyntaxException {
     @Test
     void endpointIsFormattedCorrectlyWhenConfigIsNull() throws URISyntaxException {
         ConnectionString connectionString = ConnectionString.parse("InstrumentationKey=testing-123");
-        QuickPulseDataFetcher quickPulseDataFetcher = new QuickPulseDataFetcher(new QuickPulseDataCollector(true), null,
+        QuickPulseDataFetcher quickPulseDataFetcher = new QuickPulseDataFetcher(new QuickPulseDataCollector(), null,
             connectionString::getLiveEndpoint, connectionString::getInstrumentationKey, null, null, null, null);
         String quickPulseEndpoint = quickPulseDataFetcher.getQuickPulseEndpoint();
         String endpointUrl = quickPulseDataFetcher.getEndpointUrl(quickPulseEndpoint);

sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseIntegrationTests.java

@@ -87,7 +87,7 @@ public void testPostRequest() throws InterruptedException {
         QuickPulseHeaderInfo quickPulseHeaderInfo = pingSender.ping(null);
         QuickPulseDataSender dataSender = new QuickPulseDataSender(
             getHttpPipeline(new ValidationPolicy(postCountDown, expectedPostRequestBody)), sendQueue);
-        QuickPulseDataCollector collector = new QuickPulseDataCollector(true);
+        QuickPulseDataCollector collector = new QuickPulseDataCollector();
         QuickPulseDataFetcher dataFetcher
             = new QuickPulseDataFetcher(collector, sendQueue, connectionString::getLiveEndpoint,
                 connectionString::getInstrumentationKey, null, "instance1", "machine1", null);