Check container runtime during healthcheck endpoint (#890)

This implements a healthcheck for both docker and k8s, and calls during the healthcheck API endpoint. Also refactor to use mockgen for mocking the runtime.

Author: dmjb

pkg/api/server.go

@@ -130,7 +130,7 @@ func Serve(
 		return fmt.Errorf("failed to create client manager: %v", err)
 	}
 	routers := map[string]http.Handler{
-		"/health":               v1.HealthcheckRouter(),
+		"/health":               v1.HealthcheckRouter(rt),
 		"/api/v1beta/version":   v1.VersionRouter(),
 		"/api/v1beta/workloads": v1.WorkloadRouter(manager, rt, debugMode),
 		"/api/v1beta/registry":  v1.RegistryRouter(),

pkg/api/v1/healtcheck_test.go

@@ -1,17 +1,72 @@
 package v1
 
 import (
+	"errors"
 	"net/http"
 	"net/http/httptest"
 	"testing"
 
-	"github.com/stretchr/testify/require"
+	"github.com/stretchr/testify/assert"
+	"go.uber.org/mock/gomock"
+
+	"github.com/stacklok/toolhive/pkg/container/runtime/mocks"
 )
 
 func TestGetHealthcheck(t *testing.T) {
 	t.Parallel()
-	resp := httptest.NewRecorder()
-	getHealthcheck(resp, nil)
-	require.Equal(t, http.StatusNoContent, resp.Code)
-	require.Empty(t, resp.Body)
+
+	// Create a new gomock controller
+	ctrl := gomock.NewController(t)
+	t.Cleanup(func() {
+		ctrl.Finish()
+	})
+
+	// Create a mock runtime
+	mockRuntime := mocks.NewMockRuntime(ctrl)
+
+	// Create healthcheck routes with the mock runtime
+	routes := &healthcheckRoutes{containerRuntime: mockRuntime}
+
+	t.Run("returns 204 when runtime is running", func(t *testing.T) {
+		t.Parallel()
+
+		// Setup mock to return nil (no error) when IsRunning is called
+		mockRuntime.EXPECT().
+			IsRunning(gomock.Any()).
+			Return(nil)
+
+		// Create a test request and response recorder
+		req := httptest.NewRequest(http.MethodGet, "/health", nil)
+		resp := httptest.NewRecorder()
+
+		// Call the handler
+		routes.getHealthcheck(resp, req)
+
+		// Assert the response
+		assert.Equal(t, http.StatusNoContent, resp.Code)
+		assert.Empty(t, resp.Body.String())
+	})
+
+	t.Run("returns 503 when runtime is not running", func(t *testing.T) {
+		t.Parallel()
+
+		// Create an error to return
+		expectedError := errors.New("container runtime is not available")
+
+		// Setup mock to return an error when IsRunning is called
+		mockRuntime.EXPECT().
+			IsRunning(gomock.Any()).
+			Return(expectedError)
+
+		// Create a test request and response recorder
+		req := httptest.NewRequest(http.MethodGet, "/health", nil)
+		resp := httptest.NewRecorder()
+
+		// Call the handler
+		routes.getHealthcheck(resp, req)
+
+		// Assert the response
+		assert.Equal(t, http.StatusServiceUnavailable, resp.Code)
+		assert.Equal(t, expectedError.Error()+"\n", resp.Body.String())
+	})
 }

pkg/api/v1/healthcheck.go

@@ -4,21 +4,34 @@ import (
 	"net/http"
 
 	"github.com/go-chi/chi/v5"
+
+	rt "github.com/stacklok/toolhive/pkg/container/runtime"
 )
 
 // HealthcheckRouter sets up healthcheck route.
-func HealthcheckRouter() http.Handler {
+func HealthcheckRouter(containerRuntime rt.Runtime) http.Handler {
+	routes := &healthcheckRoutes{containerRuntime: containerRuntime}
 	r := chi.NewRouter()
-	r.Get("/", getHealthcheck)
+	r.Get("/", routes.getHealthcheck)
 	return r
 }
 
+type healthcheckRoutes struct {
+	containerRuntime rt.Runtime
+}
+
 //	 getHealthcheck
 //		@Summary		Health check
 //		@Description	Check if the API is healthy
 //		@Tags			system
 //		@Success		204	{string}	string	"No Content"
 //		@Router			/health [get]
-func getHealthcheck(w http.ResponseWriter, _ *http.Request) {
+func (h *healthcheckRoutes) getHealthcheck(w http.ResponseWriter, r *http.Request) {
+	if err := h.containerRuntime.IsRunning(r.Context()); err != nil {
+		// If the container runtime is not running, we return a 503 Service Unavailable status.
+		http.Error(w, err.Error(), http.StatusServiceUnavailable)
+		return
+	}
+	// If the container runtime is running, we consider the API healthy.
 	w.WriteHeader(http.StatusNoContent)
 }

pkg/container/docker/client.go

@@ -850,6 +850,18 @@ func (c *Client) AttachToWorkload(ctx context.Context, workloadID string) (io.Wr
 	return resp.Conn, readCloser, nil
 }
 
+// IsRunning checks the health of the container runtime.
+// This is used to verify that the runtime is operational and can manage workloads.
+func (c *Client) IsRunning(ctx context.Context) error {
+	// Try to ping the Docker server
+	_, err := c.client.Ping(ctx)
+	if err != nil {
+		return fmt.Errorf("failed to ping Docker server: %v", err)
+	}
+
+	return nil
+}
+
 // getPermissionConfigFromProfile converts a permission profile to a container permission config
 // with transport-specific settings (internal function)
 // addReadOnlyMounts adds read-only mounts to the permission config

pkg/container/kubernetes/client.go

@@ -516,6 +516,19 @@ func (*Client) StopWorkload(_ context.Context, _ string) error {
 	return nil
 }
 
+// IsRunning checks the health of the container runtime.
+// This is used to verify that the runtime is operational and can manage workloads.
+func (c *Client) IsRunning(ctx context.Context) error {
+	// Use /readyz endpoint to check if the Kubernetes API server is ready.
+	var status int
+	result := c.client.Discovery().RESTClient().Get().AbsPath("/readyz").Do(ctx)
+	if result.StatusCode(&status); status != 200 {
+		return fmt.Errorf("kubernetes API server is not ready, status code: %d", status)
+	}
+
+	return nil
+}
+
 // waitForStatefulSetReady waits for a statefulset to be ready using the watch API
 func waitForStatefulSetReady(ctx context.Context, clientset kubernetes.Interface, namespace, name string) error {
 	// Create a field selector to watch only this specific statefulset