feat: implement agent_sandboxes point-in-time metrics

api/v1alpha1/sandbox_types.go

@@ -30,6 +30,11 @@ const (
 
 	// SandboxReasonExpired indicates expired state for Sandbox
 	SandboxReasonExpired = "SandboxExpired"
+
+	// SandboxPodNameAnnotation is the annotation used to track the pod name adopted from a warm pool.
+	SandboxPodNameAnnotation = "agents.x-k8s.io/pod-name"
+	// SandboxTemplateRefAnnotation is the annotation used to track the sandbox template ref.
+	SandboxTemplateRefAnnotation = "agents.x-k8s.io/sandbox-template-ref"
 )
 
 type PodMetadata struct {

cmd/agent-sandbox-controller/main.go

@@ -202,6 +202,9 @@ func main() {
 		os.Exit(1)
 	}
 
+	// Register the custom Sandbox metric collector globally.
+	asmetrics.RegisterSandboxCollector(mgr.GetClient(), mgr.GetLogger().WithName("sandbox-collector"))
+
 	if err = (&controllers.SandboxReconciler{
 		Client: mgr.GetClient(),
 		Scheme: mgr.GetScheme(),

controllers/sandbox_controller.go

@@ -44,7 +44,6 @@ import (
 
 const (
 	sandboxLabel                = "agents.x-k8s.io/sandbox-name-hash"
-	SandboxPodNameAnnotation    = "agents.x-k8s.io/pod-name"
 	sandboxControllerFieldOwner = "sandbox-controller"
 )
 
@@ -363,7 +362,7 @@ func (r *SandboxReconciler) reconcilePod(ctx context.Context, sandbox *sandboxv1
 	podName := sandbox.Name
 	var trackedPodName string
 	var podNameAnnotationExists bool
-	if trackedPodName, podNameAnnotationExists = sandbox.Annotations[SandboxPodNameAnnotation]; podNameAnnotationExists && trackedPodName != "" {
+	if trackedPodName, podNameAnnotationExists = sandbox.Annotations[sandboxv1alpha1.SandboxPodNameAnnotation]; podNameAnnotationExists && trackedPodName != "" {
 		podName = trackedPodName
 		log.Info("Using tracked pod name from sandbox annotation", "podName", podName)
 	}
@@ -396,11 +395,11 @@ func (r *SandboxReconciler) reconcilePod(ctx context.Context, sandbox *sandboxv1
 		}
 
 		// Remove the pod name annotation from the sandbox if it exists
-		if _, exists := sandbox.Annotations[SandboxPodNameAnnotation]; exists {
+		if _, exists := sandbox.Annotations[sandboxv1alpha1.SandboxPodNameAnnotation]; exists {
 			log.Info("Removing pod name annotation from sandbox", "Sandbox.Name", sandbox.Name)
 			// Create a patch to update only the annotations
 			patch := client.MergeFrom(sandbox.DeepCopy())
-			delete(sandbox.Annotations, SandboxPodNameAnnotation)
+			delete(sandbox.Annotations, sandboxv1alpha1.SandboxPodNameAnnotation)
 
 			if err := r.Patch(ctx, sandbox, patch); err != nil {
 				return nil, fmt.Errorf("failed to remove pod name annotation: %w", err)
@@ -642,6 +641,7 @@ func (r *SandboxReconciler) SetupWithManager(mgr ctrl.Manager, concurrentWorkers
 	if err != nil {
 		return err
 	}
+
 	return ctrl.NewControllerManagedBy(mgr).
 		For(&sandboxv1alpha1.Sandbox{}).
 		Owns(&corev1.Pod{}, builder.WithPredicates(labelSelectorPredicate)).

controllers/sandbox_controller_test.go

@@ -707,7 +707,7 @@ func TestReconcilePod(t *testing.T) {
 					Name:      sandboxName,
 					Namespace: sandboxNs,
 					Annotations: map[string]string{
-						SandboxPodNameAnnotation: "adopted-pod-name",
+						sandboxv1alpha1.SandboxPodNameAnnotation: "adopted-pod-name",
 					},
 				},
 				Spec: sandboxv1alpha1.SandboxSpec{
@@ -812,7 +812,7 @@ func TestReconcilePod(t *testing.T) {
 					Name:      sandboxName,
 					Namespace: sandboxNs,
 					Annotations: map[string]string{
-						SandboxPodNameAnnotation: "non-existent-pod",
+						sandboxv1alpha1.SandboxPodNameAnnotation: "non-existent-pod",
 					},
 				},
 				Spec: sandboxv1alpha1.SandboxSpec{
@@ -847,8 +847,8 @@ func TestReconcilePod(t *testing.T) {
 					Name:      sandboxName,
 					Namespace: sandboxNs,
 					Annotations: map[string]string{
-						SandboxPodNameAnnotation: "annotated-pod-name",
-						"other-annotation":       "other-value",
+						sandboxv1alpha1.SandboxPodNameAnnotation: "annotated-pod-name",
+						"other-annotation":                       "other-value",
 					},
 				},
 				Spec: sandboxv1alpha1.SandboxSpec{
@@ -888,7 +888,7 @@ func TestReconcilePod(t *testing.T) {
 				livePod := &corev1.Pod{}
 				podName := sandboxName
 				// Check if there's an annotation with a non-empty value
-				if annotatedPod, exists := tc.sandbox.Annotations[SandboxPodNameAnnotation]; exists && annotatedPod != "" {
+				if annotatedPod, exists := tc.sandbox.Annotations[sandboxv1alpha1.SandboxPodNameAnnotation]; exists && annotatedPod != "" {
 					podName = annotatedPod
 				}
 				err = r.Get(t.Context(), types.NamespacedName{Name: podName, Namespace: sandboxNs}, livePod)

extensions/controllers/sandboxclaim_controller.go

@@ -458,6 +458,9 @@ func (r *SandboxClaimReconciler) createSandbox(ctx context.Context, claim *exten
 		sandbox.Annotations[asmetrics.TraceContextAnnotation] = tc
 	}
 
+	// Track the sandbox template ref to be used by metrics collector
+	sandbox.Annotations[v1alpha1.SandboxTemplateRefAnnotation] = template.Name
+
 	template.Spec.PodTemplate.DeepCopyInto(&sandbox.Spec.PodTemplate)
 	// TODO: this is a workaround, remove replica assignment related issue #202
 	replicas := int32(1)
@@ -712,7 +715,7 @@ func (r *SandboxClaimReconciler) recordCreationLatencyMetric(
 	// This is unlikely to happen; here for completeness only.
 	if sandbox == nil {
 		launchType = asmetrics.LaunchTypeUnknown
-	} else if sandbox.Annotations[sandboxcontrollers.SandboxPodNameAnnotation] != "" {
+	} else if sandbox.Annotations[v1alpha1.SandboxPodNameAnnotation] != "" {
 		// Existence of the SandboxPodNameAnnotation implies the pod was adopted from a warm pool.
 		launchType = asmetrics.LaunchTypeWarm
 	}

internal/metrics/metrics.go

@@ -72,6 +72,20 @@ var (
 		},
 		[]string{"namespace", "sandbox_template", "launch_type", "warmpool_name", "pod_condition"},
 	)
+
+	// AgentSandboxesDesc describes the agent_sandboxes metric point-in-time counts.
+	// Labels:
+	// - namespace: the namespace of the sandbox
+	// - ready_condition: "true" | "false"
+	// - expired: "true" | "false"
+	// - launch_type: "warm" | "cold"
+	// - sandbox_template: sandboxTemplateRef
+	AgentSandboxesDesc = prometheus.NewDesc(
+		"agent_sandboxes",
+		"Monitor the point-in-time number of sandboxes in the cluster.",
+		[]string{"namespace", "ready_condition", "expired", "launch_type", "sandbox_template"},
+		nil,
+	)
 )
 
 // Init registers custom metrics with the global controller-runtime registry.

internal/metrics/sandbox_collector.go

@@ -0,0 +1,149 @@
+// Copyright 2026 The Kubernetes Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// nolint:revive
+package metrics
+
+import (
+	"context"
+	"time"
+
+	"github.com/go-logr/logr"
+	"github.com/prometheus/client_golang/prometheus"
+	"k8s.io/apimachinery/pkg/api/meta"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	sandboxv1alpha1 "sigs.k8s.io/agent-sandbox/api/v1alpha1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/metrics"
+)
+
+const (
+	metricsCollectTimeout = 5 * time.Second
+)
+
+// AgentSandboxesMetricKey is used to aggregate counts for identical Sandboxes metric label combinations.
+type AgentSandboxesMetricKey struct {
+	Namespace      string
+	ReadyCondition string
+	Expired        string
+	LaunchType     string
+	Template       string
+}
+
+// NewAgentSandboxesConstMetric creates a new Prometheus ConstMetric for the agent_sandboxes gauge.
+func NewAgentSandboxesConstMetric(count int, key AgentSandboxesMetricKey) prometheus.Metric {
+	return prometheus.MustNewConstMetric(
+		AgentSandboxesDesc,
+		prometheus.GaugeValue,
+		float64(count),
+		key.Namespace,
+		key.ReadyCondition,
+		key.Expired,
+		key.LaunchType,
+		key.Template,
+	)
+}
+
+// RegisterSandboxCollector registers the custom Prometheus collector for sandbox counts.
+func RegisterSandboxCollector(c client.Client, logger logr.Logger) {
+	collector := NewSandboxCollector(c, logger)
+	if err := metrics.Registry.Register(collector); err != nil {
+		if _, ok := err.(prometheus.AlreadyRegisteredError); !ok {
+			logger.Error(err, "Failed to register SandboxCollector")
+		} else {
+			logger.Info("SandboxCollector already registered, ignoring")
+		}
+	}
+}
+
+// SandboxCollector is a custom Prometheus collector that dynamically fetches sandbox counts.
+type SandboxCollector struct {
+	client             client.Client
+	logger             logr.Logger
+	agentSandboxesDesc *prometheus.Desc
+}
+
+// NewSandboxCollector initializes a SandboxCollector.
+func NewSandboxCollector(c client.Client, logger logr.Logger) *SandboxCollector {
+	return &SandboxCollector{
+		client:             c,
+		logger:             logger,
+		agentSandboxesDesc: AgentSandboxesDesc,
+	}
+}
+
+// Describe sends the metric descriptor to the channel.
+func (c *SandboxCollector) Describe(ch chan<- *prometheus.Desc) {
+	ch <- c.agentSandboxesDesc
+}
+
+// Collect fetches sandboxes, calculates labels, and sends metrics to the channel.
+// Note: Using client.List to fetch all sandboxes in the cluster on every metrics scrape
+// introduces O(N) memory allocation and CPU overhead due to deep-copying thousands of objects.
+// While updating a GaugeVec in the Reconcile loop might be slightly harder to manage,
+// it operates in O(1) memory during scrapes and is generally more performant.
+// This is a known performance trade-off to keep the Reconcile loop simpler.
+func (c *SandboxCollector) Collect(ch chan<- prometheus.Metric) {
+	var sandboxList sandboxv1alpha1.SandboxList
+	ctx, cancel := context.WithTimeout(context.Background(), metricsCollectTimeout)
+	defer cancel()
+
+	// TODO: The current O(N) List call during metrics collection poses a scalability concern.
+	// In large clusters, frequent scrapes could lead to high CPU usage or OOM.
+	// This should be replaced with a more efficient implementation.
+	if err := c.client.List(ctx, &sandboxList); err != nil {
+		c.logger.Error(err, "Failed to list sandboxes for metrics collection")
+		return
+	}
+
+	counts := make(map[AgentSandboxesMetricKey]int)
+	for _, sandbox := range sandboxList.Items {
+		readyConditionStr := "false"
+		expiredStr := "false"
+		readyCond := meta.FindStatusCondition(sandbox.Status.Conditions, string(sandboxv1alpha1.SandboxConditionReady))
+		if readyCond != nil {
+			if readyCond.Status == metav1.ConditionTrue {
+				readyConditionStr = "true"
+			}
+			if readyCond.Reason == sandboxv1alpha1.SandboxReasonExpired {
+				expiredStr = "true"
+			}
+		}
+
+		launchTypeStr := LaunchTypeCold
+		if _, ok := sandbox.Annotations[sandboxv1alpha1.SandboxPodNameAnnotation]; ok && sandbox.Annotations[sandboxv1alpha1.SandboxPodNameAnnotation] != "" {
+			launchTypeStr = LaunchTypeWarm
+		}
+
+		sandboxTemplateStr := "unknown"
+		// If a user manually creates a Sandbox without a SandboxClaim, it won't have the
+		// SandboxTemplateRefAnnotation. The collector correctly handles this by defaulting to "unknown".
+		if template, ok := sandbox.Annotations[sandboxv1alpha1.SandboxTemplateRefAnnotation]; ok && template != "" {
+			sandboxTemplateStr = template
+		}
+
+		key := AgentSandboxesMetricKey{
+			Namespace:      sandbox.Namespace,
+			ReadyCondition: readyConditionStr,
+			Expired:        expiredStr,
+			LaunchType:     launchTypeStr,
+			Template:       sandboxTemplateStr,
+		}
+		counts[key]++
+	}
+
+	for key, count := range counts {
+		ch <- NewAgentSandboxesConstMetric(count, key)
+	}
+}