allow selection of resources with wildcards

Signed-off-by: Markus Blaschke <[email protected]>

Author: mblaschke

example.yaml

@@ -8,13 +8,15 @@ ttl:
 
   resources:
     # definition of resources by group, version, kind (GVR)
-    - {group: "", version: v1, kind: configmaps}
+    # a wildcard ("*") will try to match as many possible resources
+    # from the serverside GVK list, be careful with it!
+    - {group: "", version: "v1", kind: "pods"}
     - {group: "", version: v1, kind: secrets}
-    - {group: apps, version: v1, kind: deployments}
+    - {group: apps, version: "*", kind: "*"}
 
 # static rules (fixed TTLs without using ttl definition in labels/annotations)
 rules:
-  # cleanup of compelted/failed pods which are not yet removed by the apiserver
+  # cleanup of completed/failed pods which are not yet removed by the Kubernetes control plane
   - id: CleanupCompletedPods
     resources:
       - group: ""
@@ -56,4 +58,4 @@ rules:
       matchLabels:
         kubernetes.io/metadata.name: default
 
-    ttl: 1m
+    ttl: 1h # resources expires 1 hour after creation

kube_janitor/config.go

@@ -1,6 +1,8 @@
 package kube_janitor
 
 import (
+	"bytes"
+	"encoding/gob"
 	"errors"
 	"strings"
 
@@ -15,11 +17,13 @@ type (
 	}
 
 	ConfigTtl struct {
-		Annotation string            `json:"annotation"`
-		Label      string            `json:"label"`
-		Resources  []*ConfigResource `json:"resources"`
+		Annotation string             `json:"annotation"`
+		Label      string             `json:"label"`
+		Resources  ConfigResourceList `json:"resources"`
 	}
 
+	ConfigResourceList []*ConfigResource
+
 	ConfigResource struct {
 		Group         string                `json:"group"`
 		Version       string                `json:"version"`
@@ -31,7 +35,7 @@ type (
 
 	ConfigRule struct {
 		Id                string                `json:"id"`
-		Resources         []*ConfigResource     `json:"resources"`
+		Resources         ConfigResourceList    `json:"resources"`
 		NamespaceSelector *metav1.LabelSelector `json:"namespaceSelector"`
 		Ttl               string                `json:"ttl"`
 	}
@@ -82,6 +86,20 @@ func (c *ConfigRule) Validate() error {
 	return nil
 }
 
+func (c *ConfigResource) Clone() *ConfigResource {
+	ret := ConfigResource{}
+	buf := bytes.Buffer{}
+	err := gob.NewEncoder(&buf).Encode(c)
+	if err != nil {
+		panic(err)
+	}
+	err = gob.NewDecoder(&buf).Decode(&ret)
+	if err != nil {
+		panic(err)
+	}
+	return &ret
+}
+
 func (c *ConfigResource) String() string {
 	return c.AsGVR().String()
 }

kube_janitor/init.go

@@ -5,5 +5,5 @@ import (
 )
 
 func init() {
-	yaml.RegisterCustomUnmarshalerContext(UmarshallJmesPath)
+	yaml.RegisterCustomUnmarshalerContext(UnmarshallJmesPath)
 }

kube_janitor/jmespath.go

@@ -23,7 +23,7 @@ func (path *JmesPath) IsEmpty() bool {
 	return path == nil || path.compiledPath == nil
 }
 
-func UmarshallJmesPath(ctx context.Context, path *JmesPath, data []byte) error {
+func UnmarshallJmesPath(ctx context.Context, path *JmesPath, data []byte) error {
 	var valString string
 
 	err := yaml.UnmarshalContext(ctx, data, &valString, yaml.Strict())
@@ -116,7 +116,6 @@ func (j *Janitor) checkResourceIsSkippedFromJmesPath(resource unstructured.Unstr
 
 func (j *Janitor) parseResourceTimestampFromJmesPath(resource unstructured.Unstructured, jmesPath *JmesPath) (*time.Time, error) {
 	result, err := j.fetchResourceValueByFromJmesPath(resource, jmesPath)
-	fmt.Println(jmesPath.Path, result, err)
 	if err != nil {
 		return nil, err
 	}

kube_janitor/kube.go

@@ -3,6 +3,7 @@ package kube_janitor
 import (
 	"context"
 	"fmt"
+	"slices"
 	"strings"
 	"time"
 
@@ -21,6 +22,93 @@ const (
 	KubeSelectorNone  = "<none>"
 )
 
+type (
+	KubeServerGvrList []metav1.GroupVersionKind
+)
+
+func (j *Janitor) kubeDiscoverGVKs() (KubeServerGvrList, error) {
+	cacheKey := "kube.servergroups"
+
+	// from cache
+	if val, ok := j.cache.Get(cacheKey); ok {
+		if v, ok := val.(KubeServerGvrList); ok {
+			return v, nil
+		}
+	}
+
+	ret := KubeServerGvrList{}
+
+	groupsResult, resourcesResult, err := j.kubeClient.Discovery().ServerGroupsAndResources()
+	if err != nil {
+		return nil, err
+	}
+
+	// build GVK list
+	for _, serverGroup := range groupsResult {
+		for _, resourceGroup := range resourcesResult {
+			if resourceGroup.GroupVersion == serverGroup.PreferredVersion.GroupVersion {
+				for _, resource := range resourceGroup.APIResources {
+					if slices.Contains([]string(resource.Verbs), "list") && slices.Contains([]string(resource.Verbs), "delete") {
+						ret = append(ret, metav1.GroupVersionKind{
+							Group:   serverGroup.Name,
+							Version: serverGroup.PreferredVersion.Version,
+							Kind:    resource.Name,
+						})
+					}
+				}
+			}
+		}
+	}
+
+	j.cache.SetDefault(cacheKey, ret)
+
+	return ret, nil
+}
+
+func (j *Janitor) kubeLookupGvrs(list ConfigResourceList) (ConfigResourceList, error) {
+	var (
+		gvrList KubeServerGvrList
+		err     error
+	)
+	ret := []*ConfigResource{}
+
+	for _, resource := range list {
+		if resource.Group == "*" || resource.Version == "*" || resource.Kind == "*" {
+			// lookup possible types
+			if gvrList == nil {
+				gvrList, err = j.kubeDiscoverGVKs()
+				if err != nil {
+					return nil, err
+				}
+			}
+
+			for _, row := range gvrList {
+				if resource.Group != "*" && !strings.EqualFold(resource.Group, row.Group) {
+					continue
+				}
+				if resource.Version != "*" && !strings.EqualFold(resource.Version, row.Version) {
+					continue
+				}
+				if resource.Kind != "*" && !strings.EqualFold(resource.Kind, row.Kind) {
+					continue
+				}
+
+				clone := resource.Clone()
+				clone.Group = row.Group
+				clone.Version = row.Version
+				clone.Kind = row.Kind
+
+				ret = append(ret, clone)
+			}
+		} else {
+			// no lookup needed
+			ret = append(ret, resource)
+		}
+	}
+
+	return ret, nil
+}
+
 func (j *Janitor) kubeBuildLabelSelector(selector *metav1.LabelSelector) (string, error) {
 	// no selector
 	if selector == nil {

kube_janitor/manager.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/go-logr/logr"
 	yaml "github.com/goccy/go-yaml"
+	"github.com/patrickmn/go-cache"
 	"github.com/webdevops/go-common/log/slogger"
 	"k8s.io/client-go/dynamic"
 	"k8s.io/client-go/kubernetes"
@@ -22,6 +23,8 @@ type (
 
 		config *Config
 
+		cache *cache.Cache
+
 		kubeClient *kubernetes.Clientset
 		dynClient  *dynamic.DynamicClient
 
@@ -43,6 +46,7 @@ func New() *Janitor {
 
 func (j *Janitor) init() {
 	j.setupMetrics()
+	j.cache = cache.New(1*time.Hour, 5*time.Minute)
 	j.kubePageLimit = KubeDefaultListLimit
 }
 
@@ -97,6 +101,8 @@ func (j *Janitor) GetConfigFromFile(path string) *Janitor {
 		j.config = NewConfig()
 	}
 
+	parserCtx := context.Background()
+
 	logger := j.logger.With(slog.String("path", path))
 
 	logger.Info("reading configuration from file")
@@ -108,7 +114,7 @@ func (j *Janitor) GetConfigFromFile(path string) *Janitor {
 	}
 
 	logger.Info("parsing configuration")
-	err = yaml.UnmarshalWithOptions(data, j.config, yaml.Strict(), yaml.UseJSONUnmarshaler())
+	err = yaml.UnmarshalContext(parserCtx, data, j.config, yaml.Strict(), yaml.UseJSONUnmarshaler())
 	if err != nil {
 		logger.Fatal("failed to parse config file")
 	}
@@ -135,6 +141,11 @@ func (j *Janitor) SetKubePageSize(val int64) *Janitor {
 	return j
 }
 
+func (j *Janitor) Connect() *Janitor {
+	j.connect()
+	return j
+}
+
 func (j *Janitor) Start(interval time.Duration) *Janitor {
 	go func() {
 		// wait for settle down
@@ -158,7 +169,6 @@ func (j *Janitor) Start(interval time.Duration) *Janitor {
 }
 
 func (j *Janitor) Run() error {
-	j.connect()
 	ctx := context.Background()
 
 	if j.config.Ttl.Label != "" || j.config.Ttl.Annotation != "" {

kube_janitor/task.rules.go

@@ -19,10 +19,17 @@ func (j *Janitor) runRules(ctx context.Context) error {
 		)
 		ruleLogger.Info("running rule")
 
-		err := j.kubeEachNamespace(ctx, rule.NamespaceSelector, func(namespace corev1.Namespace) error {
+		resourceList, err := j.kubeLookupGvrs(rule.Resources)
+		if err != nil {
+			return err
+		}
+		
+		// TODO: check if namespace selector is empty and if, use the cluster list instead
+
+		err = j.kubeEachNamespace(ctx, rule.NamespaceSelector, func(namespace corev1.Namespace) error {
 			namespaceLogger := ruleLogger
 
-			for _, resourceType := range rule.Resources {
+			for _, resourceType := range resourceList {
 				gvkLogger := namespaceLogger.With(slog.Any("gvk", resourceType))
 				err := j.kubeEachResource(ctx, resourceType.AsGVR(), namespace.GetName(), resourceType.Selector, func(resource unstructured.Unstructured) error {
 					resourceLogger := gvkLogger.With(

kube_janitor/task.ttl.go

@@ -13,9 +13,14 @@ import (
 func (j *Janitor) runTtlResources(ctx context.Context) error {
 	metricResourceTtl := prometheusCommon.NewMetricsList()
 
-	for _, resourceType := range j.config.Ttl.Resources {
+	resourceList, err := j.kubeLookupGvrs(j.config.Ttl.Resources)
+	if err != nil {
+		return err
+	}
+
+	for _, resourceType := range resourceList {
 		gvkLogger := j.logger.With(slog.Any("gvk", resourceType))
-		gvkLogger.Debug("checking resources")
+		gvkLogger.Info("checking resources")
 
 		err := j.kubeEachResource(ctx, resourceType.AsGVR(), KubeNoNamespace, resourceType.Selector, func(resource unstructured.Unstructured) error {
 			var ttlValue string

main.go

@@ -42,6 +42,7 @@ func main() {
 	janitor := kube_janitor.New()
 	janitor.SetKubeconfig(Opts.Kubernetes.Config).
 		SetLogger(logger).
+		Connect().
 		SetKubePageSize(Opts.Kubernetes.ItemsPerPage).
 		GetConfigFromFile(Opts.Janitor.Config).
 		SetDryRun(Opts.Janitor.DryRun)