Add Grpc Readiness probe to cart microservice. Failure can be enabled through feature flag to simulate pod not ready situation (open-telemetry#2830)

* Adding readiness probe health check to cart service + feature flag added to enable it in flagd

* Clean up commented health check code

Removed commented-out health check code in Program.cs.

* Update src/cart/src/Program.cs

Co-authored-by: Cedric Ziel <mail@cedric-ziel.com>

* moving the healthcheck test to a separate file under services folder

* Fix missing newline in healthcheck.cs

Add missing newline at end of file

* Add copyright and license information to healthcheck.cs

* changelog

---------

Co-authored-by: Manojkumar Sardana <sardanam@hcl.com>
Co-authored-by: Cedric Ziel <mail@cedric-ziel.com>
Co-authored-by: Juliano Costa <julianocosta89@outlook.com>
Co-authored-by: Juliano Costa <juliano.costa@datadoghq.com>

Author: 4 people

.gitignore

@@ -53,3 +53,5 @@ test/tracetesting/tracetesting-vars.yaml
 *.apk
 
 !src/currency/build
+src/cart/src/Program.cs.bak
+src/flagd/demo.flagd.json.bak

CHANGELOG.md

@@ -30,6 +30,8 @@ the release.
   ([#2737](https://github.com/open-telemetry/opentelemetry-demo/pull/2737))
 * [collector] [dockerstats/receiver] Set API version to 1.44
   ([#2767](https://github.com/open-telemetry/opentelemetry-demo/pull/2767))
+* [cart] Add health check endpoint
+  ([#2830](https://github.com/open-telemetry/opentelemetry-demo/pull/2830))
 * [product-catalog] Use Postgres database for products
   ([#2859](https://github.com/open-telemetry/opentelemetry-demo/pull/2859))
 

src/cart/src/Program.cs

@@ -2,8 +2,16 @@
 // SPDX-License-Identifier: Apache-2.0
 using System;
 
+using Grpc.Health.V1;
+using Microsoft.AspNetCore.Diagnostics.HealthChecks;
+using System.Threading.Tasks;
+using System.Threading;
+
+using Grpc.Core;
+
 using cart.cartstore;
 using cart.services;
+using cart.healthcheck;
 
 using Microsoft.AspNetCore.Builder;
 using Microsoft.AspNetCore.Http;
@@ -79,20 +87,25 @@
         .SetExemplarFilter(ExemplarFilterType.TraceBased)
         .AddOtlpExporter());
 builder.Services.AddGrpc();
+builder.Services.AddSingleton<readinessCheck>();
 builder.Services.AddGrpcHealthChecks()
-    .AddCheck("Sample", () => HealthCheckResult.Healthy());
+    .AddCheck<readinessCheck>("oteldemo.CartService");
+
+builder.Services.AddSingleton<HealthServiceImpl>();
 
 var app = builder.Build();
 
 var ValkeyCartStore = (ValkeyCartStore)app.Services.GetRequiredService<ICartStore>();
 app.Services.GetRequiredService<StackExchangeRedisInstrumentation>().AddConnection(ValkeyCartStore.GetConnection());
 
 app.MapGrpcService<CartService>();
-app.MapGrpcHealthChecksService();
+app.MapGrpcService<HealthServiceImpl>();
 
 app.MapGet("/", async context =>
 {
     await context.Response.WriteAsync("Communication with gRPC endpoints must be made through a gRPC client. To learn how to create a client, visit: https://go.microsoft.com/fwlink/?linkid=2086909");
 });
 
 app.Run();
+
+

src/cart/src/services/HealthCheckService.cs

@@ -0,0 +1,118 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+using System;
+
+using Grpc.Core;
+using Grpc.HealthCheck;
+using Grpc.Health.V1;
+using System.Threading.Tasks;
+using System.Threading;
+
+using OpenFeature;
+using OpenFeature.Hooks;
+using OpenFeature.Contrib.Providers.Flagd;
+
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Diagnostics.HealthChecks;
+using Microsoft.Extensions.Logging;
+
+namespace cart.healthcheck
+{
+    public class readinessCheck : IHealthCheck
+    {
+        private readonly IFeatureClient _featureClient;
+
+        public readinessCheck(IFeatureClient featureClient)
+        {
+            _featureClient = featureClient;
+        }
+        public async Task<HealthCheckResult> CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default)
+        {
+
+            #pragma warning disable CA2016 // OpenFeature does not support CancellationToken
+            // Await the async call instead of blocking
+            bool isSet = await _featureClient.GetBooleanValueAsync("failedReadinessProbe", false); // Replace with actual check
+            #pragma warning restore CA2016
+            if (isSet)
+            {
+                return HealthCheckResult.Unhealthy("connection failed");
+
+            }
+
+            return HealthCheckResult.Healthy("healthy");
+        }
+    }
+
+
+    public class HealthServiceImpl : Health.HealthBase
+    {
+        private readonly ILogger<HealthServiceImpl> _logger;
+        private readonly HealthCheckService _healthCheckService;
+
+
+        public HealthServiceImpl(
+            ILogger<HealthServiceImpl> logger,
+            HealthCheckService healthCheckService)
+        {
+            _logger = logger;
+            _healthCheckService = healthCheckService;
+        }
+
+        public override async Task<HealthCheckResponse> Check(HealthCheckRequest request, ServerCallContext context)
+        {
+           if (_logger.IsEnabled(LogLevel.Information))
+           {
+            _logger.LogInformation("Received health check request for service: {Service}", request.Service);
+           }
+            var cancellationToken = context.CancellationToken;
+            // If service is empty or null, check overall health
+            if (string.IsNullOrEmpty(request.Service))
+            {
+                var health = await _healthCheckService.CheckHealthAsync(cancellationToken);
+                return new HealthCheckResponse
+                {
+                    Status = ConvertToGrpcStatus(health.Status)
+                };
+            }
+
+            // You can implement service-specific health checks here
+            // This example checks a specific service
+            var serviceHealth = await _healthCheckService.CheckHealthAsync(registration => MatchesService(registration, request.Service), cancellationToken);
+            return new HealthCheckResponse
+            {
+                Status = ConvertToGrpcStatus(serviceHealth.Entries[request.Service].Status)
+            };
+        }
+
+        private static bool MatchesService(HealthCheckRegistration registration, string service)
+        {
+            return registration.Name == service;
+        }
+
+        public override async Task Watch(HealthCheckRequest request, IServerStreamWriter<HealthCheckResponse> responseStream, ServerCallContext context)
+        {
+            if (_logger.IsEnabled(LogLevel.Information))
+            {
+            _logger.LogInformation("Received health watch request for service: {Service}", request.Service);
+            }
+            // Simple implementation to send current status once
+            var response = await Check(request, context);
+            await responseStream.WriteAsync(response);
+
+            // In a real implementation, you would periodically check health and send updates
+            // This might involve setting up a timer or listener for health changes
+        }
+
+        private static HealthCheckResponse.Types.ServingStatus ConvertToGrpcStatus(HealthStatus status)
+        {
+            return status switch
+            {
+                HealthStatus.Healthy => HealthCheckResponse.Types.ServingStatus.Serving,
+                HealthStatus.Degraded => HealthCheckResponse.Types.ServingStatus.Serving, // Or you might want to use SERVING_WITH_ISSUES if available
+                HealthStatus.Unhealthy => HealthCheckResponse.Types.ServingStatus.NotServing,
+                _ => HealthCheckResponse.Types.ServingStatus.Unknown
+            };
+        }
+    }
+}

src/flagd/demo.flagd.json

@@ -124,6 +124,16 @@
       },
       "defaultVariant": "off"
     },
+    "failedReadinessProbe": {
+      "description": "readiness probe failure for cart service",
+      "state": "ENABLED",
+      "variants": {
+        "on": true,
+        "off": false
+      },
+      "defaultVariant": "off"
+
+    },
     "emailMemoryLeak": {
       "description": "Memory leak in the email service.",
       "state": "ENABLED",