feat: Add ResourceMetricsConfig for configurable core resource metrics

internal/store/builder.go

@@ -215,10 +215,29 @@ func (b *Builder) WithCustomResourceStoreFactories(fs ...customresource.Registry
 		} else {
 			gvrString = f.Name()
 		}
-		if _, ok := availableStores[gvrString]; ok {
+
+		key := gvrString
+		if _, ok := availableStores[f.Name()]; ok {
+			key = f.Name()
+			klog.InfoS("Overriding core resource store with custom factory", "resource", key)
+		} else if _, ok := availableStores[gvrString]; ok {
 			klog.InfoS("Updating store", "GVR", gvrString)
 		}
-		availableStores[gvrString] = func(b *Builder) []cache.Store {
+
+		availableStores[key] = func(b *Builder) []cache.Store {
+			klog.InfoS("Building custom resource store", "gvrString", gvrString, "clientExists", b.customResourceClients[gvrString] != nil)
+			client := b.customResourceClients[gvrString]
+			klog.InfoS("About to call buildCustomResourceStoresFunc",
+				"factoryName", f.Name(),
+				"clientIsNil", client == nil,
+				"expectedType", f.ExpectedType(),
+				"metricsCount", len(f.MetricFamilyGenerators()),
+			)
+
+			// Test ListWatch before calling buildCustomResourceStoresFunc
+			lw := f.ListWatch(client, "", "")
+			klog.InfoS("ListWatch result", "isNil", lw == nil)
+
 			return b.buildCustomResourceStoresFunc(
 				f.Name(),
 				f.MetricFamilyGenerators(),

pkg/app/server.go

@@ -61,6 +61,8 @@ import (
 	"k8s.io/kube-state-metrics/v2/pkg/options"
 	"k8s.io/kube-state-metrics/v2/pkg/util"
 	"k8s.io/kube-state-metrics/v2/pkg/util/proc"
+
+	"k8s.io/kube-state-metrics/v2/pkg/resourcestate"
 )
 
 const (
@@ -70,6 +72,19 @@ const (
 	readyzPath  = "/readyz"
 )
 
+type dropDefaultsFilter struct {
+	overridden map[string]struct{}
+}
+
+func (d dropDefaultsFilter) Test(fg generator.FamilyGenerator) bool {
+	for res := range d.overridden {
+		if strings.HasPrefix(fg.Name, "kube_"+res+"_") {
+			return false
+		}
+	}
+	return true
+}
+
 // RunKubeStateMetricsWrapper runs KSM with context cancellation.
 func RunKubeStateMetricsWrapper(ctx context.Context, opts *options.Options) error {
 	err := RunKubeStateMetrics(ctx, opts)
@@ -268,6 +283,52 @@ func RunKubeStateMetrics(ctx context.Context, opts *options.Options) error {
 		return fmt.Errorf("failed to create client: %v", err)
 	}
 	storeBuilder.WithKubeClient(kubeClient)
+	storeBuilder.WithGenerateCustomResourceStoresFunc(storeBuilder.DefaultGenerateCustomResourceStoresFunc())
+
+	if opts.ResourceMetricsConfigFile != "" {
+		cfg, err := resourcestate.LoadConfig(opts.ResourceMetricsConfigFile)
+		if err != nil {
+			return fmt.Errorf("failed to load ResourceMetricsConfig: %w", err)
+		}
+
+		factories, err := resourcestate.BuildFactoriesFromConfig(cfg)
+		if err != nil {
+			return fmt.Errorf("failed to compile ResourceMetricsConfig: %w", err)
+		}
+
+		crClients := make(map[string]interface{})
+		for _, f := range factories {
+			klog.InfoS("Processing factory", "factoryName", f.Name())
+
+			// Use only util.GVRFromType since that's what the builder expects
+			utilGVR, err := util.GVRFromType(f.Name(), f.ExpectedType())
+			if err != nil {
+				klog.ErrorS(err, "Failed to get GVR from type", "resourceName", f.Name())
+				continue
+			}
+
+			gvrString := utilGVR.String()
+			crClients[gvrString] = kubeClient
+			klog.InfoS("Registering CR client", "factoryName", f.Name(), "gvrKey", gvrString)
+		}
+
+		klog.InfoS("Total registered clients", "count", len(crClients))
+
+		storeBuilder.WithCustomResourceClients(crClients)
+		storeBuilder.WithCustomResourceStoreFactories(factories...)
+
+		if opts.DisableDefaultCoreMetrics {
+			overridden := map[string]struct{}{}
+			for _, f := range factories {
+				overridden[f.Name()] = struct{}{}
+			}
+			storeBuilder.WithFamilyGeneratorFilter(generator.NewCompositeFamilyGeneratorFilter(
+				allowDenyList,
+				optInMetricFamilyFilter,
+				dropDefaultsFilter{overridden: overridden},
+			))
+		}
+	}
 
 	storeBuilder.WithSharding(opts.Shard, opts.TotalShards)
 	if err := storeBuilder.WithAllowAnnotations(opts.AnnotationsAllowList); err != nil {

pkg/customresourcestate/custom_resource_metrics.go

@@ -33,6 +33,9 @@ import (
 	generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator"
 )
 
+// CompiledFamily is an exported alias of the internal compiled family.
+type CompiledFamily = compiledFamily
+
 // customResourceMetrics is an implementation of the customresource.RegistryFactory
 // interface which provides metrics for custom resources defined in a configuration file.
 type customResourceMetrics struct {
@@ -44,6 +47,16 @@ type customResourceMetrics struct {
 
 var _ customresource.RegistryFactory = &customResourceMetrics{}
 
+// Compile exposes the internal compile(resource) to other packages.
+func Compile(r Resource) ([]CompiledFamily, error) {
+	return compile(r)
+}
+
+// FamGen exposes the internal famGen to other packages.
+func FamGen(f CompiledFamily) generator.FamilyGenerator {
+	return famGen(f)
+}
+
 // NewCustomResourceMetrics creates a customresource.RegistryFactory from a configuration object.
 func NewCustomResourceMetrics(resource Resource) (customresource.RegistryFactory, error) {
 	compiled, err := compile(resource)

pkg/options/options.go

@@ -83,6 +83,9 @@ type Options struct {
 	UseAPIServerCache    bool  `yaml:"use_api_server_cache"`
 	ObjectLimit          int64 `yaml:"object_limit"`
 	AuthFilter           bool  `yaml:"auth_filter"`
+
+	ResourceMetricsConfigFile string
+	DisableDefaultCoreMetrics bool
 }
 
 // GetConfigFile is the getter for --config value.
@@ -159,6 +162,8 @@ func (o *Options) AddFlags(cmd *cobra.Command) {
 	o.cmd.Flags().StringVar(&o.CustomResourceConfig, "custom-resource-state-config", "", "Inline Custom Resource State Metrics config YAML (experimental)")
 	o.cmd.Flags().StringVar(&o.CustomResourceConfigFile, "custom-resource-state-config-file", "", "Path to a Custom Resource State Metrics config file (experimental)")
 	o.cmd.Flags().BoolVar(&o.ContinueWithoutCustomResourceConfigFile, "continue-without-custom-resource-state-config-file", false, "If true, Kube-state-metrics continues to run even if the config file specified by --custom-resource-state-config-file is not present. This is useful for scenarios where config file is not provided at startup but is provided later, for e.g., via configmap. Kube-state-metrics will not exit with an error if the custom-resource-state-config file is not found, instead watches and reloads when it is created.")
+	o.cmd.Flags().StringVar(&o.ResourceMetricsConfigFile, "resource-metrics-config-file", "", "Path to ResourceMetricsConfig YAML for core resources")
+	o.cmd.Flags().BoolVar(&o.DisableDefaultCoreMetrics, "disable-default-core-metrics", false, "Disable built-in core metric families; only config-defined ones will be used")
 	o.cmd.Flags().StringVar(&o.Host, "host", "::", `Host to expose metrics on.`)
 	o.cmd.Flags().StringVar(&o.Kubeconfig, "kubeconfig", "", "Absolute path to the kubeconfig file")
 	o.cmd.Flags().StringVar(&o.Namespace, "pod-namespace", "", "Name of the namespace of the pod specified by --pod. "+autoshardingNotice)

pkg/resourcestate/factory.go

@@ -0,0 +1,144 @@
+package resourcestate
+
+import (
+	"context"
+	"fmt"
+	klog "k8s.io/klog/v2"
+	"strings"
+
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/watch"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/cache"
+
+	cr "k8s.io/kube-state-metrics/v2/pkg/customresource"
+	crs "k8s.io/kube-state-metrics/v2/pkg/customresourcestate"
+	generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator"
+
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/kube-state-metrics/v2/pkg/metric"
+)
+
+type coreFactory struct {
+	name string
+	kind string
+	gvk  metav1.GroupVersionKind // Add this field
+	fams []crs.CompiledFamily
+}
+
+var _ cr.RegistryFactory = (*coreFactory)(nil)
+
+func (f *coreFactory) Name() string { return f.name }
+
+func (f *coreFactory) CreateClient(cfg *rest.Config) (interface{}, error) {
+	return kubernetes.NewForConfig(cfg)
+}
+
+func (f *coreFactory) MetricFamilyGenerators() []generator.FamilyGenerator {
+	out := make([]generator.FamilyGenerator, 0, len(f.fams))
+	for _, fam := range f.fams {
+		// Wrap the CRS generator to handle typed-to-unstructured conversion
+		crsGen := crs.FamGen(fam)
+		wrappedGen := generator.FamilyGenerator{
+			Name:              crsGen.Name,
+			Help:              crsGen.Help,
+			Type:              crsGen.Type,
+			DeprecatedVersion: crsGen.DeprecatedVersion,
+			StabilityLevel:    crsGen.StabilityLevel,
+			OptIn:             crsGen.OptIn,
+			GenerateFunc: func(obj interface{}) *metric.Family {
+				// Convert typed object to unstructured
+				unstructuredObj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(obj)
+				if err != nil {
+					klog.ErrorS(err, "Failed to convert to unstructured", "objType", fmt.Sprintf("%T", obj))
+					return &metric.Family{}
+				}
+
+				u := &unstructured.Unstructured{Object: unstructuredObj}
+				return crsGen.GenerateFunc(u)
+			},
+		}
+		out = append(out, wrappedGen)
+	}
+	return out
+}
+
+func (f *coreFactory) ExpectedType() interface{} {
+	switch f.kind {
+	case "Pod":
+		return &corev1.Pod{}
+	case "Deployment":
+		return &appsv1.Deployment{}
+	default:
+		return &corev1.Pod{}
+	}
+}
+
+func (f *coreFactory) ListWatch(client interface{}, ns, fieldSelector string) cache.ListerWatcher {
+	cs := client.(kubernetes.Interface)
+	ctx := context.Background()
+	switch f.kind {
+	case "Pod":
+		return &cache.ListWatch{
+			ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) {
+				opts.FieldSelector = fieldSelector
+				return cs.CoreV1().Pods(ns).List(ctx, opts)
+			},
+			WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) {
+				opts.FieldSelector = fieldSelector
+				return cs.CoreV1().Pods(ns).Watch(ctx, opts)
+			},
+		}
+	case "Deployment":
+		return &cache.ListWatch{
+			ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) {
+				opts.FieldSelector = fieldSelector
+				return cs.AppsV1().Deployments(ns).List(ctx, opts)
+			},
+			WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) {
+				opts.FieldSelector = fieldSelector
+				return cs.AppsV1().Deployments(ns).Watch(ctx, opts)
+			},
+		}
+	default:
+		// Return a valid but empty ListWatch instead of nil
+		return &cache.ListWatch{
+			ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) {
+				return &corev1.PodList{}, nil
+			},
+			WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) {
+				return watch.NewEmptyWatch(), nil
+			},
+		}
+	}
+}
+
+func (f *coreFactory) GVRString() string {
+	return fmt.Sprintf("%s/%s, Resource=%s", f.gvk.Group, f.gvk.Version, strings.ToLower(f.name))
+}
+
+// BuildFactoriesFromConfig compiles the config into RegistryFactories.
+func BuildFactoriesFromConfig(c *Config) ([]cr.RegistryFactory, error) {
+	var out []cr.RegistryFactory
+	for _, r := range c.Spec.Resources {
+		fams, err := crs.Compile(r)
+		if err != nil {
+			return nil, err
+		}
+		out = append(out, &coreFactory{
+			name: r.GetResourceName(),
+			kind: r.GroupVersionKind.Kind,
+			gvk: metav1.GroupVersionKind{
+				Group:   r.GroupVersionKind.Group,
+				Version: r.GroupVersionKind.Version,
+				Kind:    r.GroupVersionKind.Kind,
+			}, // Store the GVK
+			fams: fams,
+		})
+	}
+	return out, nil
+}

pkg/resourcestate/resource_metrics_config.go

@@ -0,0 +1,32 @@
+package resourcestate
+
+import (
+	"os"
+
+	"gopkg.in/yaml.v3"
+
+	crs "k8s.io/kube-state-metrics/v2/pkg/customresourcestate"
+)
+
+// Config wraps a list of customresourcestate.Resource entries for core resources.
+type Config struct {
+	Kind string `yaml:"kind"` // expect "ResourceMetricsConfig"
+	Spec struct {
+		Resources []crs.Resource `yaml:"resources"`
+	} `yaml:"spec"`
+}
+
+func LoadConfig(path string) (*Config, error) {
+	b, err := os.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+	var c Config
+	if err := yaml.Unmarshal(b, &c); err != nil {
+		return nil, err
+	}
+	if c.Kind == "" {
+		c.Kind = "ResourceMetricsConfig"
+	}
+	return &c, nil
+}

tests/manifests/resouce-metrics-config.yaml

@@ -0,0 +1,31 @@
+kind: ResourceMetricsConfig
+spec:
+  resources:
+    - groupVersionKind:
+        group: ""
+        version: "v1"
+        kind: "Pod"
+      resourceName: "pods"
+      metricNamePrefix: "kube_pod"
+      metrics:
+        - name: "phase"
+          help: "Pod phase"
+          each:
+            type: stateset  
+            stateSet:
+              labelName: "phase"
+              path: ["status","phase"]
+              list: ["Pending","Running","Succeeded","Failed","Unknown"]
+    - groupVersionKind:
+        group: "apps"
+        version: "v1"
+        kind: "Deployment"
+      resourceName: "deployments"
+      metricNamePrefix: "kube_deployment"
+      metrics:
+        - name: "spec_replicas"
+          help: "Desired replicas"
+          each:
+            type: gauge
+            gauge:
+              path: ["spec","replicas"]