test(simulation): expose worker domain readiness errors

Signed-off-by: Diana Zawadzki <dzawa@live.de>
Co-authored-by: Cursor <cursoragent@cursor.com>

Author: zawadzkidiana

simulation/replication/replication_simulation_test.go

@@ -36,6 +36,7 @@ import (
 	"encoding/json"
 	"flag"
 	"fmt"
+	"io"
 	"net"
 	"net/http"
 	"os"
@@ -680,6 +681,7 @@ func waitUntilWorkersReady(t *testing.T) {
 	for {
 		allHealthy := true
 		lastStatus := make(map[string]int, len(workerEndpoints))
+		lastBody := make(map[string]string, len(workerEndpoints))
 		lastErr := make(map[string]error, len(workerEndpoints))
 		for _, endpoint := range workerEndpoints {
 			resp, err := client.Get(endpoint)
@@ -688,8 +690,12 @@ func waitUntilWorkersReady(t *testing.T) {
 				allHealthy = false
 				break
 			}
+			body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
 			_ = resp.Body.Close()
 			lastStatus[endpoint] = resp.StatusCode
+			if len(body) > 0 {
+				lastBody[endpoint] = strings.TrimSpace(string(body))
+			}
 			if resp.StatusCode != http.StatusOK {
 				allHealthy = false
 				break
@@ -705,7 +711,11 @@ func waitUntilWorkersReady(t *testing.T) {
 			if err := lastErr[endpoint]; err != nil {
 				simTypes.Logf(t, "Workers are not reporting healthy yet (%s error: %v). Sleep for 2s and try again", endpoint, err)
 			} else if status, ok := lastStatus[endpoint]; ok {
-				simTypes.Logf(t, "Workers are not reporting healthy yet (%s status: %d). Sleep for 2s and try again", endpoint, status)
+				if body, ok := lastBody[endpoint]; ok && body != "" {
+					simTypes.Logf(t, "Workers are not reporting healthy yet (%s status: %d body: %s). Sleep for 2s and try again", endpoint, status, body)
+				} else {
+					simTypes.Logf(t, "Workers are not reporting healthy yet (%s status: %d). Sleep for 2s and try again", endpoint, status)
+				}
 			}
 		}
 		time.Sleep(2 * time.Second)

simulation/replication/worker/cmd/main.go

@@ -22,6 +22,7 @@ package main
 
 import (
 	"context"
+	"encoding/json"
 	"flag"
 	"fmt"
 	"net/http"
@@ -54,6 +55,9 @@ var (
 	clusterName = flag.String("cluster", "", "cluster name")
 
 	ready int32
+
+	domainErrMu   sync.Mutex
+	domainLastErr = map[string]string{}
 )
 
 func main() {
@@ -101,12 +105,41 @@ func main() {
 		apiv1.NewVisibilityAPIYARPCClient(clientConfig),
 	)
 
+	type healthStatus struct {
+		Cluster      string            `json:"cluster"`
+		GRPCEndpoint string            `json:"grpcEndpoint"`
+		ReadyDomains int32             `json:"readyDomains"`
+		TotalDomains int               `json:"totalDomains"`
+		LastErrors   map[string]string `json:"lastErrors,omitempty"`
+	}
+
 	http.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) {
-		if atomic.LoadInt32(&ready) == int32(len(simCfg.Domains)) {
+		readyDomains := atomic.LoadInt32(&ready)
+		isReady := readyDomains == int32(len(simCfg.Domains))
+
+		domainErrMu.Lock()
+		lastErrors := make(map[string]string, len(domainLastErr))
+		for k, v := range domainLastErr {
+			lastErrors[k] = v
+		}
+		domainErrMu.Unlock()
+
+		w.Header().Set("Content-Type", "application/json")
+		status := healthStatus{
+			Cluster:      *clusterName,
+			GRPCEndpoint: cluster.GRPCEndpoint,
+			ReadyDomains: readyDomains,
+			TotalDomains: len(simCfg.Domains),
+			LastErrors:   lastErrors,
+		}
+		b, _ := json.Marshal(status) // best-effort diagnostics
+
+		if isReady {
 			w.WriteHeader(http.StatusOK)
 		} else {
 			w.WriteHeader(http.StatusServiceUnavailable)
 		}
+		_, _ = w.Write(b)
 	})
 	go http.ListenAndServe(":6060", nil)
 
@@ -186,6 +219,10 @@ func waitUntilDomainReady(logger *zap.Logger, client workflowserviceclient.Inter
 			return
 		}
 
+		domainErrMu.Lock()
+		domainLastErr[domainName] = err.Error()
+		domainErrMu.Unlock()
+
 		logger.Info("Domains not ready", zap.String("domain", domainName), zap.Error(err))
 		time.Sleep(2 * time.Second)
 	}