BCDA-9633: add worker health check (#1276)

## 🎫 Ticket

https://jira.cms.gov/browse/BCDA-9633

Related tickets:
[CDAP](CMSgov/cdap#354)
[bcda-ops](CMSgov/bcda-ops#1303)

## 🛠 Changes

<!-- What was added, updated, or removed in this PR? -->
- added local health checks for ssas and api, with an additional
dockerfile dependency on curl
- added local health check for worker
- refactored worker into a command-line app, with two entrypoints:
`start-worker` and `health`
- added an interface for HealthChecker and a mock implementation with
Mockery
- added a makefile target for generating mocks with Mockery that pins a
shared Mockery version and enables devs to not have to install Mockery
locally


## ℹ️ Context

<!-- Why were these changes made? Add background context suitable for a
non-technical audience. -->

Since the worker service doesn't have a target group or load balancer,
it would be useful to have a container-level health check that monitors
the health of the service. The worker does not have any api endpoints
like API and SSAS, so a command-line interface was added to enable easy
verification of the worker health. This app has two commands:
`start-worker` (which starts the worker service) and `health` (which
performs the same health check that is run on a timed loop currently in
the worker). For temporary compatibility with EC2, this health check can
be configured to still run in a timed loop, but this is skipped for ECS.
Sonarqube will complain about the test coverage -- I tested what I
thought was reasonable, but much of the gap is the CLI entrypoint that I
don't believe is relevant (and is untested in its current form on main).

<!-- If any of the following security implications apply, this PR must
not be merged without Stephen Walter's approval. Explain in this section
and add @SJWalter11 as a reviewer.
  - Adds a new software dependency or dependencies.
  - Modifies or invalidates one or more of our security controls.
  - Stores or transmits data that was not stored or transmitted before.
- Requires additional review of security implications for other reasons.
-->

## 🧪 Validation

<!-- How were the changes verified? Did you fully test the acceptance
criteria in the ticket? Provide reproducible testing instructions and
screenshots if applicable. -->

Tested in dev on ECS and EC2

Author: michaeljvaldes

.github/workflows/dev-daily-autodeploy.yml

@@ -2,6 +2,7 @@
 name: Deploy main to dev daily
 
 on:
+  workflow_dispatch:
   schedule:
     - cron: 0 12 * * 1-5 # every workday at 8am EST
 

.mockery.yml

@@ -14,6 +14,9 @@ packages:
   github.com/CMSgov/bcda-app/bcda/auth:
     interfaces:
       Provider:
+  github.com/CMSgov/bcda-app/bcda/health:
+    interfaces:
+      HealthChecker:
   github.com/CMSgov/bcda-app/bcda/models:
     interfaces:
       Repository:

Dockerfiles/Dockerfile.bcda

@@ -25,7 +25,9 @@ FROM golang:1.25.4-alpine3.22
 
 RUN addgroup -S -g 1200 bcda && adduser -S -G bcda -u 1100 bcda && \
     apk update upgrade --no-cache && \
-    apk add --no-cache aws-cli
+    apk add --no-cache \
+        aws-cli \
+        curl
 
 # install dev packages if the environment argument was set to development
 ARG ENVIRONMENT

Dockerfiles/Dockerfile.bcdaworker

@@ -55,4 +55,4 @@ USER bcda
 ENV APP_NAME="worker"
 
 ENTRYPOINT ["./entrypoint.sh"]
-CMD ["bcdaworker"]
+CMD ["bcdaworker", "start-worker"]

Dockerfiles/Dockerfile.ssas

@@ -28,9 +28,10 @@ RUN --mount=type=cache,target=/go/pkg/mod \
 
 FROM golang:1.25.4-alpine3.22
 
-RUN --mount=type=cache,target=/var/cache/apk \
-    apk update upgrade && \
-    apk add openssl
+RUN apk update upgrade && \
+    apk add --no-cache \
+        curl \
+        openssl
 
 RUN openssl genrsa -out /var/local/private.pem 2048
 RUN openssl rsa -in /var/local/private.pem -outform PEM -pubout -out /var/local/public.pem

Makefile

@@ -177,7 +177,10 @@ fhir_testing:
 	-e CLIENT_SECRET='${CLIENT_SECRET}' \
 	fhir_testing
 
-.PHONY: api-shell debug-api debug-worker docker-bootstrap docker-build lint load-fixtures load-fixtures-ssas package performance-test postman release smoke-test test unit-test worker-shell bdt fhir_testing unit-test-db unit-test-db-snapshot reset-db dbdocs
+generate-mocks:
+	docker run -v "$PWD":/src -w /src vektra/mockery:v3.6.1
+
+.PHONY: api-shell debug-api debug-worker docker-bootstrap docker-build generate-mocks lint load-fixtures load-fixtures-ssas package performance-test postman release smoke-test test unit-test worker-shell bdt fhir_testing unit-test-db unit-test-db-snapshot reset-db dbdocs
 
 credentials:
 	$(eval ACO_CMS_ID = A9994)

bcda/health/health.go

@@ -23,7 +23,15 @@ type introspectCache struct {
 	mu        sync.RWMutex
 }
 
-type HealthChecker struct {
+type HealthChecker interface {
+	IsDatabaseOK() (string, bool)
+	IsWorkerDatabaseOK() (string, bool)
+	IsBlueButtonOK() bool
+	IsSsasOK() (string, bool)
+	IsSsasIntrospectOK() (string, bool)
+}
+
+type healthCheck struct {
 	db              *sql.DB
 	introspectCache *introspectCache
 }
@@ -35,13 +43,13 @@ const (
 )
 
 func NewHealthChecker(db *sql.DB) HealthChecker {
-	return HealthChecker{
+	return healthCheck{
 		db:              db,
 		introspectCache: &introspectCache{},
 	}
 }
 
-func (h HealthChecker) IsDatabaseOK() (result string, ok bool) {
+func (h healthCheck) IsDatabaseOK() (result string, ok bool) {
 	if err := h.db.Ping(); err != nil {
 		log.API.Error("Health check: database ping error: ", err.Error())
 		return "database ping error", false
@@ -50,7 +58,7 @@ func (h HealthChecker) IsDatabaseOK() (result string, ok bool) {
 	return "ok", true
 }
 
-func (h HealthChecker) IsWorkerDatabaseOK() (result string, ok bool) {
+func (h healthCheck) IsWorkerDatabaseOK() (result string, ok bool) {
 	if err := h.db.Ping(); err != nil {
 		log.Worker.Error("Health check: database ping error: ", err.Error())
 		return "database ping error", false
@@ -59,7 +67,7 @@ func (h HealthChecker) IsWorkerDatabaseOK() (result string, ok bool) {
 	return "ok", true
 }
 
-func (h HealthChecker) IsBlueButtonOK() bool {
+func (h healthCheck) IsBlueButtonOK() bool {
 	bbc, err := client.NewBlueButtonClient(client.NewConfig("/v1/fhir"))
 	if err != nil {
 		log.Worker.Error("Health check: Blue Button client error: ", err.Error())
@@ -75,7 +83,7 @@ func (h HealthChecker) IsBlueButtonOK() bool {
 	return true
 }
 
-func (h HealthChecker) IsSsasOK() (result string, ok bool) {
+func (h healthCheck) IsSsasOK() (result string, ok bool) {
 	c, err := ssasClient.NewSSASClient()
 	if err != nil {
 		log.Auth.Errorf("no client for SSAS. no provider set; %s", err.Error())
@@ -88,7 +96,7 @@ func (h HealthChecker) IsSsasOK() (result string, ok bool) {
 	return "ok", true
 }
 
-func (h HealthChecker) IsSsasIntrospectOK() (result string, ok bool) {
+func (h healthCheck) IsSsasIntrospectOK() (result string, ok bool) {
 	// Check cache first
 	h.introspectCache.mu.RLock()
 	if h.introspectCache.timestamp.Add(introspectCacheTTL).After(time.Now()) {
@@ -185,7 +193,7 @@ func isRetryableError(err error) bool {
 }
 
 // introspectWithRetry attempts to call introspect with exponential backoff retry
-func (h HealthChecker) introspectWithRetry(c *ssasClient.SSASClient, ctx context.Context, tokenString string) ([]byte, error) {
+func (h healthCheck) introspectWithRetry(c *ssasClient.SSASClient, ctx context.Context, tokenString string) ([]byte, error) {
 	eb := backoff.NewExponentialBackOff()
 	eb.InitialInterval = introspectRetryInitial
 	eb.MaxInterval = 2 * time.Second

bcda/health/health_test.go

@@ -27,7 +27,7 @@ var (
 
 type HealthCheckerTestSuite struct {
 	suite.Suite
-	hc HealthChecker
+	hc healthCheck
 }
 
 func (s *HealthCheckerTestSuite) SetupSuite() {
@@ -39,7 +39,10 @@ func (s *HealthCheckerTestSuite) SetupSuite() {
 }
 
 func (s *HealthCheckerTestSuite) SetupTest() {
-	s.hc = NewHealthChecker(nil)
+	s.hc = healthCheck{
+		db:              nil,
+		introspectCache: &introspectCache{},
+	}
 }
 
 func (s *HealthCheckerTestSuite) TearDownTest() {