Revert "Revert previously undeployed changes (#348)"

This reverts commit 55d0831.

Author: gmdfalk

acceptance/srv-configs/clusterman-clusters/local-dev/default.kubernetes

@@ -21,3 +21,4 @@ autoscaling:
   instance_loss_threshold: 3
 
 alert_on_max_capacity: false
+pool_owner: compute_infra

acceptance/srv-configs/clusterman-clusters/local-dev/default.mesos

@@ -29,3 +29,4 @@ autoscale_signal:
       minute_range: 10
 
 alert_on_max_capacity: false
+pool_owner: compute_infra

clusterman/autoscaler/autoscaler.py

@@ -178,12 +178,9 @@ def run(self, dry_run: bool = False, timestamp: Optional[arrow.Arrow] = None) ->
         else:
             capacity_offset = get_capacity_offset(self.cluster, self.pool, self.scheduler, timestamp)
             new_target_capacity = self._compute_target_capacity(resource_request) + capacity_offset
-            self.target_capacity_gauge.set(new_target_capacity, {"dry_run": dry_run})
-            self.max_capacity_gauge.set(
-                self.pool_manager.max_capacity,
-                {"dry_run": dry_run, "alert_on_max_capacity": self.pool_manager.alert_on_max_capacity},
-            )
-            self.setpoint_gauge.set(self.autoscaling_config.setpoint, {"dry_run": dry_run})
+            self.target_capacity_gauge.set(new_target_capacity, self.add_metric_labels(dry_run))
+            self.max_capacity_gauge.set(self.pool_manager.max_capacity, self.add_metric_labels(dry_run))
+            self.setpoint_gauge.set(self.autoscaling_config.setpoint, self.add_metric_labels(dry_run))
             self._emit_requested_resource_metrics(resource_request, dry_run=dry_run)
 
         try:
@@ -202,7 +199,14 @@ def run(self, dry_run: bool = False, timestamp: Optional[arrow.Arrow] = None) ->
     def _emit_requested_resource_metrics(self, resource_request: SignalResourceRequest, dry_run: bool) -> None:
         for resource_type, resource_gauge in self.resource_request_gauges.items():
             if getattr(resource_request, resource_type) is not None:
-                resource_gauge.set(getattr(resource_request, resource_type), {"dry_run": dry_run})
+                resource_gauge.set(getattr(resource_request, resource_type), self.add_metric_labels(dry_run))
+
+    def add_metric_labels(self, dry_run):
+        return {
+            "dry_run": dry_run,
+            "alert_on_max_capacity": self.pool_manager.alert_on_max_capacity,
+            "team": self.pool_manager.pool_owner,
+        }
 
     def _get_signal_for_app(self, app: str) -> Signal:
         """Load the signal object to use for autoscaling for a particular app

clusterman/autoscaler/pool_manager.py

@@ -86,6 +86,7 @@ def __init__(
             "autoscaling.killable_nodes_prioritizing_v2", default=False
         )
         self.alert_on_max_capacity = self.pool_config.read_bool("alert_on_max_capacity", default=True)
+        self.pool_owner = self.pool_config.read_string("pool_owner", default="compute_infra")
         monitoring_info = {"cluster": cluster, "pool": pool}
         self.killable_nodes_counter = get_monitoring_client().create_counter(SFX_KILLABLE_NODES_COUNT, monitoring_info)
 

clusterman/kubernetes/kubernetes_cluster_connector.py

@@ -24,8 +24,8 @@
 import colorlog
 import kubernetes
 import staticconf
-from kubernetes.client import V1beta1Eviction
 from kubernetes.client import V1DeleteOptions
+from kubernetes.client import V1Eviction
 from kubernetes.client import V1ObjectMeta
 from kubernetes.client.models.v1_node import V1Node as KubernetesNode
 from kubernetes.client.models.v1_pod import V1Pod as KubernetesPod
@@ -356,7 +356,7 @@ def _evict_pod(self, pod: KubernetesPod):
         self._core_api.create_namespaced_pod_eviction(
             name=pod.metadata.name,
             namespace=pod.metadata.namespace,
-            body=V1beta1Eviction(
+            body=V1Eviction(
                 metadata=V1ObjectMeta(
                     name=pod.metadata.name,
                     namespace=pod.metadata.namespace,

clusterman/kubernetes/util.py

@@ -30,6 +30,7 @@
 from kubernetes.client.models.v1_node_selector_requirement import V1NodeSelectorRequirement
 from kubernetes.client.models.v1_node_selector_term import V1NodeSelectorTerm
 from kubernetes.client.models.v1_pod import V1Pod as KubernetesPod
+from kubernetes.config.config_exception import ConfigException
 
 from clusterman.util import ClustermanResources
 
@@ -72,7 +73,7 @@ def __init__(self, kubeconfig_path: str, client_class: Type) -> None:
                 kubernetes.config.load_incluster_config()
             else:
                 kubernetes.config.load_kube_config(kubeconfig_path, context=os.getenv("KUBECONTEXT"))
-        except TypeError:
+        except (TypeError, ConfigException):
             error_msg = "Could not load KUBECONFIG; is this running on Kubernetes master?"
             if "yelpcorp" in socket.getfqdn():
                 error_msg += "\nHint: try using the clusterman-k8s-<clustername> wrapper script!"

clusterman/simulator/simulated_pool_manager.py

@@ -59,6 +59,7 @@ def __init__(
             MAX_MIN_NODE_SCALEIN_UPTIME_SECONDS,
         )
         self.alert_on_max_capacity = self.pool_config.read_bool("alert_on_max_capacity", default=True)
+        self.pool_owner = self.pool_config.read_string("pool_owner", default="compute_infra")
         self.killable_nodes_prioritizing_v2 = self.pool_config.read_bool(
             "autoscaling.killable_nodes_prioritizing_v2", default=False
         )

examples/schemas/pool.json

@@ -64,7 +64,8 @@
             "additionalProperties": false
         },
         "sensu_config": {"$ref": "definitions.json#sensu_config"},
-        "alert_on_max_capacity": {"type": "boolean"}
+        "alert_on_max_capacity": {"type": "boolean"},
+        "pool_owner": {"type": "string"}
     },
     "additionalProperties": false
 }

itests/environment.py

@@ -121,6 +121,7 @@ def setup_configurations(context):
             ],
         },
         "alert_on_max_capacity": True,
+        "pool_owner": "compute_infra",
     }
     kube_pool_config = {
         "resource_groups": [
@@ -144,6 +145,7 @@ def setup_configurations(context):
             "period_minutes": 7,
         },
         "alert_on_max_capacity": True,
+        "pool_owner": "compute_infra",
     }
     with staticconf.testing.MockConfiguration(
         boto_config, namespace=CREDENTIALS_NAMESPACE

requirements.txt

@@ -16,7 +16,7 @@ idna==2.8
 jmespath==0.9.4
 jsonpickle==1.4.2
 kiwisolver==1.1.0
-kubernetes==10.0.1
+kubernetes==24.2.0
 matplotlib==3.4.2
 mypy-extensions==0.4.3
 numpy==1.21.6