Related Metrics demo setup (#244)

*Issue #, if available:*
N/A

*Description of changes:*
Adding related metrics demo to billing service in the application
signals demo.

By submitting this pull request, I confirm that you can use, modify,
copy, and redistribute this contribution, under the terms of your
choice.

Author: leenanofal

ai-validator/libs/tests/test-billing-summary.script.md

@@ -0,0 +1,61 @@
+## Steps
+
+### 1. Navigate to CloudWatch Application Signals Services
+
+Go to CloudWatch → Application Signals (APM) → Services.
+
+### 2. Search for billing-service-python
+
+In the search field with placeholder text 'Filter services and resources by text, property or value', type 'billing-service-python' and press Enter.
+
+**Constraints:**
+- You MUST ensure you press Enter.
+
+### 3. Click on billing-service-python
+
+Click the hyperlink 'billing-service-python'.
+
+### 4. Click 'Service operation' tab
+
+Click the 'Service operation' tab. Then, ensure the `GET ^summary/$` is selected if not already.
+
+### 5. Click on a datapoint for GET /summary
+
+Click on a datapoint in the GET /summary operation graph, PASS in 1 and 3 as a PARAMETERS.
+
+**Constraints:**
+- You MUST pass in parameters 1 and 3
+
+### 6. Click 'Correlate with Other Metrics'
+
+In the diagnostic drawer on the right, click the 'Correlate with Other Metrics' button.
+
+### 7. Check you are on Related metrics tab
+
+Check that you are now on the 'Related metrics' tab.
+
+### 8. Search for BillingSummaryCacheHitCount metric
+
+In the search field with placeholder text 'Filter metrics by text, property, or value', type 'BillingSummaryCacheHitCount' and press Enter.
+
+**Constraints:**
+- You MUST ensure you press Enter.
+
+### 9. Select BillingSummaryCacheHitCount metric
+
+In the metrics table, select the 'BillingSummaryCacheHitCount' metric to add it to the graph.
+
+### 10. Search for BillingSummaryCacheMissCount metric
+
+In the search field with placeholder text 'Filter metrics by text, property, or value', type 'BillingSummaryCacheMissCount' and press Enter.
+
+**Constraints:**
+- You MUST ensure you press Enter.
+
+### 11. Select BillingSummaryCacheMissCount metric
+
+In the metrics table, select the 'BillingSummaryCacheMissCount' metric to add it to the graph.
+
+### 12. Check correlation between metrics
+
+Check that when latency spikes occur, BillingSummaryCacheHitCount decreases and BillingSummaryCacheMissCount increases at the same time periods.

data_test/test_cases/metrics_test_cases.json

@@ -60,6 +60,27 @@
         }]
       },
       "notes": "Corresponds to PDF scenario 2"
+    },
+     {
+      "test_case_id": "billing_summary_service_metric_availability_check",
+      "description": "Verify Service Latency metric availability for billing summary API (Scenario 16)",
+      "test_scenario": "Scenario 16",
+      "metric_namespace": "ApplicationSignals",
+      "metric_name": "Latency",
+      "statistic": "p99",
+      "dimensions": [
+        {"Name": "Operation", "Value": "GET ^summary/$"},
+        {"Name": "Environment", "Value": "ENVIRONMENT_NAME_PLACEHOLDER"},
+        {"Name": "Service", "Value": "billing-service-python"}
+      ],
+      "evaluation_period_minutes": 180,
+      "threshold": {
+        "comparison_operator": [{
+          "operator": "GreaterThanOrEqualToThreshold",
+          "threshold_value": 0
+        }]
+      },
+      "notes": "Corresponds to PDF scenario 16"
     },
     {
       "test_case_id": "database_insight_dependency_metric_availability_check",

pet_clinic_billing_service/billing_service/views.py

@@ -1,6 +1,8 @@
 from rest_framework import viewsets, status
 from rest_framework.response import Response
-from django.db.models import Subquery
+from django.db.models import Subquery, Count, Sum
+from django.utils import timezone
+from django.core.cache import cache
 from .models import Billing,CheckList
 from .serializers import BillingSerializer
 from opentelemetry import trace
@@ -154,6 +156,44 @@ def log(self, data):
             # Don't raise the exception to avoid disrupting the main flow
 
 
+class SummaryViewSet(viewsets.ViewSet):
+    def list(self, request, pk=None):
+        span = trace.get_current_span()
+        
+        # Always set request counter to 1
+        span.set_attribute("billing_summary_request", 1)
+
+         # Set num_summaries based on current minute
+        current_minute = timezone.now().minute
+        num_summaries = 50 if current_minute % 5 == 0 else 2
+        
+        # Use random cache key to simulate high cache miss rate
+        cache_key = f'billing_summary_last_7_days_{random.randint(1, num_summaries)}'
+        summary = cache.get(cache_key)
+        
+        if summary is None:
+            # Cache miss
+            span.set_attribute("billing_summary_cache_hit", 0)
+            
+            # Sleep to simulate high latency when there's a cache miss 
+            time.sleep(2)
+            
+            billings = Billing.objects.all()
+            
+            summary = {
+                'total_count': billings.count(),
+                'total_amount': billings.aggregate(Sum('payment'))['payment__sum'] or 0,
+                'period': 'all_time'
+            }
+            
+            cache.set(cache_key, summary, 300)  # Cache for 5 minutes
+        else:
+            # Cache hit
+            span.set_attribute("billing_summary_cache_hit", 1)
+        
+        return Response(summary)
+
+
 class HealthViewSet(viewsets.ViewSet):
     def list(self, request):
         logger.info("HealthViewSet.list() called - Health check requested")

pet_clinic_billing_service/create-metric-filters.sh

@@ -0,0 +1,92 @@
+#!/bin/bash
+
+# Set variables
+LOG_GROUP_NAME="aws/spans"
+REGION=${AWS_REGION:-us-east-1}
+NAMESPACE="CustomBilling"
+
+# TODO: [@lnnofal] convert it to CDK and hooked up with other CDK stacks (for EKS)
+echo "Creating log group if it doesn't exist..."
+aws logs create-log-group --log-group-name "$LOG_GROUP_NAME" --region "$REGION" 2>/dev/null || echo "Log group already exists or creation failed"
+
+echo "Creating CloudWatch metric filters for billing service custom metrics..."
+
+# Create metric filter for billing summary requests
+cat > metric-filter.json << EOF
+{
+"logGroupName": "aws/spans",
+"filterName": "BillingSummaryRequests",
+"filterPattern": "{ $.attributes.['billing_summary_request'] = \"1\" }",
+"metricTransformations": [
+{
+"metricName": "BillingSummaryRequestCount",
+"metricNamespace": "$NAMESPACE",
+"metricValue": "1",
+"unit": "Count",
+"dimensions": {
+"Service": "$.attributes.['aws.local.service']",
+"Environment": "$.attributes.['aws.local.environment']",
+"Operation": "$.attributes.['aws.local.operation']"
+}
+}
+]
+}
+EOF
+
+aws logs put-metric-filter --region $REGION --cli-input-json file://metric-filter.json || { rm -f metric-filter.json; exit 1; }
+
+# Create metric filter for cache hits
+cat > metric-filter.json << EOF
+{
+"logGroupName": "aws/spans",
+"filterName": "BillingSummaryCacheHits",
+"filterPattern": "{ $.attributes.['billing_summary_cache_hit'] = \"1\" }",
+"metricTransformations": [
+{
+"metricName": "BillingSummaryCacheHitCount",
+"metricNamespace": "$NAMESPACE",
+"metricValue": "1",
+"unit": "Count",
+"dimensions": {
+"Service": "$.attributes.['aws.local.service']",
+"Environment": "$.attributes.['aws.local.environment']",
+"Operation": "$.attributes.['aws.local.operation']"
+}
+}
+]
+}
+EOF
+
+aws logs put-metric-filter --region $REGION --cli-input-json file://metric-filter.json || { rm -f metric-filter.json; exit 1; }
+
+# Create metric filter for cache misses
+cat > metric-filter.json << EOF
+{
+"logGroupName": "aws/spans",
+"filterName": "BillingSummaryCacheMisses",
+"filterPattern": "{ $.attributes.['billing_summary_cache_hit'] = \"0\" }",
+"metricTransformations": [
+{
+"metricName": "BillingSummaryCacheMissCount",
+"metricNamespace": "$NAMESPACE",
+"metricValue": "1",
+"unit": "Count",
+"dimensions": {
+"Service": "$.attributes.['aws.local.service']",
+"Environment": "$.attributes.['aws.local.environment']",
+"Operation": "$.attributes.['aws.local.operation']"
+}
+}
+]
+}
+EOF
+
+aws logs put-metric-filter --region $REGION --cli-input-json file://metric-filter.json || { rm -f metric-filter.json; exit 1; }
+
+rm -f metric-filter.json
+
+echo "Metric filters created successfully!"
+echo "Metrics will appear in CloudWatch under the '$NAMESPACE' namespace:"
+echo "- BillingSummaryRequestCount"
+echo "- BillingSummaryCacheHitCount" 
+echo "- BillingSummaryCacheMissCount"

pet_clinic_billing_service/pet_clinic_billing_service/urls.py

@@ -17,10 +17,11 @@
 from django.contrib import admin
 from django.urls import include, path
 from rest_framework.routers import DefaultRouter
-from billing_service.views import HealthViewSet, BillingViewSet
+from billing_service.views import HealthViewSet, BillingViewSet, SummaryViewSet
 
 router = DefaultRouter()
 router.register('billings', BillingViewSet, basename='billings')
+router.register('summary', SummaryViewSet, basename='summary')
 router.register('health', HealthViewSet, basename='health')
 urlpatterns = [
     path("admin/", admin.site.urls),

spring-petclinic-api-gateway/src/main/java/org/springframework/samples/petclinic/api/application/BillingServiceClient.java

@@ -45,4 +45,11 @@ public Flux<BillingDetail> getBillings() {
             .bodyToFlux(BillingDetail.class);
     }
 
+    public Mono<Object> getBillingSummary() {
+        return webClientBuilder.build().get()
+            .uri("http://billing-service/summary/")
+            .retrieve()
+            .bodyToMono(Object.class);
+    }
+
 }

spring-petclinic-api-gateway/src/main/java/org/springframework/samples/petclinic/api/boundary/web/ApiController.java

@@ -122,6 +122,11 @@ public Flux<BillingDetail> getBillings() {
         return billingServiceClient.getBillings();
     }
 
+    @GetMapping(value = "billing/summary")
+    public Mono<Object> getBillingSummary() {
+        return billingServiceClient.getBillingSummary();
+    }
+
     @PostMapping(value = "insurance/pet-insurances")
     public Mono<Void> addPetInsurance(final @RequestBody PetInsurance petInsurance) {
         System.out.println(petInsurance.toString());

traffic-generator/index.js

@@ -251,3 +251,13 @@ const lowTrafficBillingTask = cron.schedule('*/2 * * * *', () => {
 }, { scheduled: false });
 
 lowTrafficBillingTask.start();
+
+const lowTrafficBillingSummaryTask = cron.schedule('* * * * * *', () => {
+    console.log('query billing summary every 1 second');
+    axios.get(`${baseUrl}/api/billing/summary`, { timeout: 10000 })
+        .catch(err => {
+            console.error(`${baseUrl}/api/billing/summary, error: ` + (err.response ? err.response.data : err.toString()));
+        }); // Catch and log errors
+}, { scheduled: false });
+
+lowTrafficBillingSummaryTask.start();