add support for new required k8s labels: chain.link/product, chain.link/team, chain.link/component, chain.link/cost-center

Author: Tofel

lib/k8s/config/overrides.go

@@ -46,6 +46,10 @@ const (
 	EnvVarUserDescription = "Owner of an environment"
 	EnvVarUserExample     = "Satoshi"
 
+	EnvVarTeam            = "CHAINLINK_USER_TEAM"
+	EnvVarTeamDescription = "Team to, which owner of the environment belongs to"
+	EnvVarTeamExample     = "BIX, CCIP, BCM"
+
 	EnvVarCLCommitSha            = "CHAINLINK_COMMIT_SHA"
 	EnvVarCLCommitShaDescription = "The sha of the commit that you're running tests on. Mostly used for CI"
 	EnvVarCLCommitShaExample     = "${{ github.sha }}"

lib/k8s/environment/environment.go

@@ -66,6 +66,8 @@ type ConnectedChart interface {
 	GetValues() *map[string]any
 	// ExportData export deployment part data in the env
 	ExportData(e *Environment) error
+	// GetLabels returns labels for the component
+	GetLabels() map[string]string
 }
 
 // Config is an environment common configuration, labels, annotations, connection types, readiness check, etc.
@@ -80,6 +82,8 @@ type Config struct {
 	Labels []string
 	// PodLabels is a set of labels applied to every pod in the namespace
 	PodLabels map[string]string
+	// WorkloadLabels is a set of labels applied to every workload in the namespace (job, deployment, replicaset), including service and ingress
+	WorkloadLabels map[string]string
 	// PreventPodEviction if true sets a k8s annotation safe-to-evict=false to prevent pods from being evicted
 	// Note: This should only be used if your test is completely incapable of handling things like K8s rebalances without failing.
 	// If that is the case, it's worth the effort to make your test fault-tolerant soon. The alternative is expensive and infuriating.
@@ -236,6 +240,59 @@ func New(cfg *Config) *Environment {
 	return e
 }
 
+var requiredChainLinkNsLabels = []string{"chain.link/team", "chain.link/cost-center", "chain.link/product"}
+var requiredChainLinkWorkloadLabels = append([]string{}, append(requiredChainLinkNsLabels, "chain.link/component")...)
+
+func (m *Environment) validateRequiredChainLinkLabels() error {
+	if m.root.Labels() == nil {
+		return fmt.Errorf("namespace labels are nil, but it should contain at least '%s' labels. Please add them to your environment config under 'Labels' key", strings.Join(requiredChainLinkNsLabels, ", "))
+	}
+
+	var missingNsLabels []string
+	for _, l := range requiredChainLinkNsLabels {
+		if _, ok := (*m.root.Labels())[l]; !ok {
+			missingNsLabels = append(missingNsLabels, l)
+		}
+	}
+
+	children := m.root.Node().Children()
+	missingWorkloadLabels := make(map[string][]string)
+
+	if children == nil {
+		return nil
+	}
+
+	for _, child := range *children {
+		if h, ok := child.(cdk8s.Helm); ok {
+			for _, ao := range *h.ApiObjects() {
+				switch *ao.Kind() {
+				case "Deployment", "ReplicaSet", "StatefulSet", "Service", "Job", "DaemonSet", "Ingress":
+					for _, l := range requiredChainLinkWorkloadLabels {
+						maybeLabel := ao.Metadata().GetLabel(&l)
+						if maybeLabel == nil {
+							missingWorkloadLabels[*ao.Name()] = append(missingWorkloadLabels[*ao.Name()], l)
+						}
+					}
+				}
+			}
+		}
+	}
+
+	if len(missingWorkloadLabels) > 0 {
+		sb := strings.Builder{}
+		sb.WriteString("missing required labels for workloads:\n")
+		for chart, missingLabels := range missingWorkloadLabels {
+			for _, label := range missingLabels {
+				sb.WriteString(fmt.Sprintf("\t'%s': '%s'\n", chart, label))
+			}
+		}
+		sb.WriteString("Please add them to your environment configuration under 'WorkloadLabels' key. And check whether every chart has 'chain.link/component' label defined.")
+		return errors.New(sb.String())
+	}
+
+	return nil
+}
+
 func (m *Environment) initApp() error {
 	var err error
 	m.App = cdk8s.NewApp(&cdk8s.AppProps{
@@ -435,6 +492,17 @@ func addDefaultPodAnnotationsAndLabels(h cdk8s.Helm, annotations, labels map[str
 	}
 }
 
+func addRequiredChainLinkLabels(h cdk8s.Helm, labels map[string]string) {
+	for _, ao := range *h.ApiObjects() {
+		switch *ao.Kind() {
+		case "Deployment", "ReplicaSet", "StatefulSet", "Service", "Job", "DaemonSet", "Ingress":
+			for k, v := range labels {
+				ao.Metadata().AddLabel(&k, &v)
+			}
+		}
+	}
+}
+
 // UpdateHelm update a helm chart with new values. The pod will launch with an `updated=true` label if it's a Chainlink node.
 // Note: If you're modifying ConfigMap values, you'll probably need to use RollOutStatefulSets to apply the changes to the pods.
 // https://stackoverflow.com/questions/57356521/rollingupdate-for-stateful-set-doesnt-restart-pods-and-changes-from-updated-con
@@ -508,11 +576,32 @@ func (m *Environment) AddHelm(chart ConnectedChart) *Environment {
 		ReleaseName: ptr.Ptr(chart.GetName()),
 		Values:      values,
 	})
+
+	componentLabels, err := getComponentLabels(m.Cfg.WorkloadLabels, chart.GetLabels())
+	if err != nil {
+		m.err = err
+	}
+
+	addRequiredChainLinkLabels(h, componentLabels)
 	addDefaultPodAnnotationsAndLabels(h, markNotSafeToEvict(m.Cfg.PreventPodEviction, nil), m.Cfg.PodLabels)
 	m.Charts = append(m.Charts, chart)
 	return m
 }
 
+func getComponentLabels(podLabels, chartLabels map[string]string) (map[string]string, error) {
+	componentLabels := make(map[string]string)
+	err := mergo.Merge(&componentLabels, podLabels, mergo.WithOverride)
+	if err != nil {
+		return nil, err
+	}
+	err = mergo.Merge(&componentLabels, chartLabels, mergo.WithOverride)
+	if err != nil {
+		return nil, err
+	}
+
+	return componentLabels, nil
+}
+
 // PullOCIChart handles working with OCI format repositories
 // https://helm.sh/docs/topics/registries/
 // API is not compatible between helm repos and OCI repos, so we download and untar the chart
@@ -674,6 +763,12 @@ func (m *Environment) RunCustomReadyConditions(customCheck *client.ReadyCheckDat
 		m.Cfg.SkipManifestUpdate = mu
 	}
 	log.Debug().Bool("ManifestUpdate", m.Cfg.SkipManifestUpdate).Msg("Update mode")
+
+	// make sure all required chain.link labels are present in the final manifest
+	if err := m.validateRequiredChainLinkLabels(); err != nil {
+		return err
+	}
+
 	if !m.Cfg.SkipManifestUpdate || m.Cfg.JobImage != "" {
 		if err := m.DeployCustomReadyConditions(customCheck, podCount); err != nil {
 			log.Error().Err(err).Msg("Error deploying environment")
@@ -1084,3 +1179,39 @@ func markNotSafeToEvict(preventPodEviction bool, m map[string]string) map[string
 
 	return m
 }
+
+func GetRequiredChainLinkNamespaceLabels(product, testType string) ([]string, error) {
+	var nsLabels []string
+	createdLabels, err := createRequiredChainLinkLabels(product, testType)
+	if err != nil {
+		return nsLabels, err
+	}
+
+	for k, v := range createdLabels {
+		nsLabels = append(nsLabels, fmt.Sprintf("%s=%s", k, v))
+	}
+
+	return nsLabels, nil
+}
+
+func GetRequiredChainLinkWorkloadLabels(product, testType string) (map[string]string, error) {
+	createdLabels, err := createRequiredChainLinkLabels(product, testType)
+	if err != nil {
+		return nil, err
+	}
+
+	return createdLabels, nil
+}
+
+func createRequiredChainLinkLabels(product, testType string) (map[string]string, error) {
+	team := os.Getenv(config.EnvVarTeam)
+	if team == "" {
+		return nil, fmt.Errorf("missing team environment variable, please set %s to your team name or if you are seeing this in CI please either add a new input with team name or hardcode it if this jobs is only run by a single team", config.EnvVarUser)
+	}
+
+	return map[string]string{
+		"chain.link/product":     product,
+		"chain.link/team":        team,
+		"chain.link/cost-center": fmt.Sprintf("%s-%s-test", team, testType),
+	}, nil
+}

lib/k8s/environment/runner.go

@@ -50,6 +50,12 @@ func (m Chart) ExportData(e *Environment) error {
 	return nil
 }
 
+func (m Chart) GetLabels() map[string]string {
+	return map[string]string{
+		"chain.link/component": "k8s-legacy-test-runner",
+	}
+}
+
 func NewRunner(props *Props) func(root cdk8s.Chart) ConnectedChart {
 	return func(root cdk8s.Chart) ConnectedChart {
 		c := &Chart{

lib/k8s/pkg/helm/chainlink/chainlink.go

@@ -57,6 +57,12 @@ func (m Chart) GetValues() *map[string]any {
 	return m.Values
 }
 
+func (m Chart) GetLabels() map[string]string {
+	return map[string]string{
+		"chain.link/component": "chainlink",
+	}
+}
+
 func (m Chart) ExportData(e *environment.Environment) error {
 	// fetching all apps with label app=chainlink-${deploymentIndex}:${instanceIndex}
 	pods, err := e.Fwd.Client.ListPods(e.Cfg.Namespace, fmt.Sprintf("app=%s", m.Name))

lib/k8s/pkg/helm/ethereum/geth.go

@@ -57,6 +57,12 @@ func (m Chart) GetValues() *map[string]interface{} {
 	return m.HelmProps.Values
 }
 
+func (m Chart) GetLabels() map[string]string {
+	return map[string]string{
+		"chain.link/component": "geth",
+	}
+}
+
 func (m Chart) ExportData(e *environment.Environment) error {
 	if m.Props.Simulated {
 		gethLocalHttp, err := e.Fwd.FindPort("geth:0", "geth-network", "http-rpc").As(client.LocalConnection, client.HTTP)

lib/k8s/pkg/helm/foundry/foundry.go

@@ -58,6 +58,12 @@ func (m Chart) GetValues() *map[string]interface{} {
 	return m.Values
 }
 
+func (m Chart) GetLabels() map[string]string {
+	return map[string]string{
+		"chain.link/component": "anvil",
+	}
+}
+
 func (m *Chart) ExportData(e *environment.Environment) error {
 	appInstance := fmt.Sprintf("%s:0", m.Name) // uniquely identifies an instance of an anvil service running in a pod
 	var err error

lib/k8s/pkg/helm/grafana/grafana.go

@@ -40,6 +40,12 @@ func (m Chart) GetValues() *map[string]interface{} {
 	return m.Values
 }
 
+func (m Chart) GetLabels() map[string]string {
+	return map[string]string{
+		"chain.link/component": "grafana",
+	}
+}
+
 func (m Chart) ExportData(e *environment.Environment) error {
 	return nil
 }

lib/k8s/pkg/helm/influxdb/influxdb.go

@@ -40,6 +40,12 @@ func (m Chart) GetValues() *map[string]interface{} {
 	return m.Values
 }
 
+func (m Chart) GetLabels() map[string]string {
+	return map[string]string{
+		"chain.link/component": "influxdb",
+	}
+}
+
 func (m Chart) ExportData(e *environment.Environment) error {
 	return nil
 }

lib/k8s/pkg/helm/kafka-rest/kafka-rest.go

@@ -48,6 +48,12 @@ func (m Chart) GetValues() *map[string]interface{} {
 	return m.Values
 }
 
+func (m Chart) GetLabels() map[string]string {
+	return map[string]string{
+		"chain.link/component": "kafka-rest",
+	}
+}
+
 func (m Chart) ExportData(e *environment.Environment) error {
 	urls := make([]string, 0)
 	local, err := e.Fwd.FindPort("cp-kafka-rest:0", "kafka-rest", "http").As(client.LocalConnection, client.HTTP)

lib/k8s/pkg/helm/kafka/kafka.go

@@ -40,6 +40,12 @@ func (m Chart) GetValues() *map[string]interface{} {
 	return m.Values
 }
 
+func (m Chart) GetLabels() map[string]string {
+	return map[string]string{
+		"chain.link/component": "kafka",
+	}
+}
+
 func (m Chart) ExportData(e *environment.Environment) error {
 	return nil
 }