Merge pull request #158 from Yelp/m6id-c6id

Update for using m6id and c6id instance type families

Author: tzhu-yelp

.pre-commit-config.yaml

@@ -44,8 +44,8 @@ repos:
           - id: pyupgrade
             args: [--py3-plus]
             language_version: python3.6
-    - repo: http://github.com/psf/black
-      rev: 19.10b0
+    - repo: https://github.com/psf/black
+      rev: 22.3.0
       hooks:
           - id: black
             language_version: python3.7

acceptance/run_instance.py

@@ -32,7 +32,9 @@
 
 vpc_response = ec2.create_vpc(CidrBlock=cidr_block)
 subnet_response = ec2.create_subnet(
-    CidrBlock=cidr_block, VpcId=vpc_response["Vpc"]["VpcId"], AvailabilityZone="us-west-2a",
+    CidrBlock=cidr_block,
+    VpcId=vpc_response["Vpc"]["VpcId"],
+    AvailabilityZone="us-west-2a",
 )
 subnet_id = subnet_response["Subnet"]["SubnetId"]
 with open("{root}/autoscaler_config.tmpl".format(root=root)) as config_template:
@@ -57,7 +59,17 @@
                 "EbsOptimized": False,
                 # note that this is not useful until we solve
                 # https://github.com/spulec/moto/issues/1644
-                "TagSpecifications": [{"ResourceType": "instance", "Tags": [{"Key": "foo", "Value": "bar",}],}],
+                "TagSpecifications": [
+                    {
+                        "ResourceType": "instance",
+                        "Tags": [
+                            {
+                                "Key": "foo",
+                                "Value": "bar",
+                            }
+                        ],
+                    }
+                ],
             },
         ],
         "IamFleetRole": "foo",
@@ -73,15 +85,22 @@
     Body=json.dumps(
         {
             "cluster_autoscaling_resources": {
-                "aws_spot_fleet_request.local-dev": {"id": sfr_response["SpotFleetRequestId"], "pool": "default",}
+                "aws_spot_fleet_request.local-dev": {
+                    "id": sfr_response["SpotFleetRequestId"],
+                    "pool": "default",
+                }
             }
         }
     ).encode(),
 )
 
 s3.create_bucket(Bucket="clusterman-signals")
 with open(
-    "{root}/{env}/clusterman_signals_acceptance.tar.gz".format(root=root, env=os.environ["DISTRIB_CODENAME"],), "rb",
+    "{root}/{env}/clusterman_signals_acceptance.tar.gz".format(
+        root=root,
+        env=os.environ["DISTRIB_CODENAME"],
+    ),
+    "rb",
 ) as f:
     s3.put_object(
         Bucket="clusterman-signals",
@@ -92,7 +111,10 @@
 try:
     dynamodb.create_table(
         TableName="clusterman_cluster_state",
-        KeySchema=[{"AttributeName": "state", "KeyType": "HASH"}, {"AttributeName": "entity", "KeyType": "SORT"},],
+        KeySchema=[
+            {"AttributeName": "state", "KeyType": "HASH"},
+            {"AttributeName": "entity", "KeyType": "SORT"},
+        ],
         AttributeDefinitions=[
             {"AttributeName": "state", "AttributeType": "S"},
             {"AttributeName": "entity", "AttributeType": "S"},

clusterman/args.py

@@ -23,7 +23,7 @@
 
 
 def subparser(command, help, entrypoint):  # pragma: no cover
-    """ Function decorator to simplify adding arguments to subcommands
+    """Function decorator to simplify adding arguments to subcommands
 
     :param command: name of the subcommand to add
     :param help: help string for the subcommand
@@ -68,7 +68,7 @@ def add_branch_or_tag_arg(parser):  # pragma: no cover
 
 
 def add_start_end_args(parser, start_help, end_help):  # pragma: no cover
-    """ Add --start-time and --end-time args to a parser
+    """Add --start-time and --end-time args to a parser
 
     :param start_help: help string for --start-time
     :param end_help: help string for --end-time
@@ -88,21 +88,25 @@ def add_start_end_args(parser, start_help, end_help):  # pragma: no cover
 
 
 def add_region_arg(parser, required=False):  # pragma: no cover
-    """ Add an --aws-region argument to a parser """
+    """Add an --aws-region argument to a parser"""
     parser.add_argument(
-        "--aws-region", required=required, help="AWS region to operate in",
+        "--aws-region",
+        required=required,
+        help="AWS region to operate in",
     )
 
 
 def add_cluster_arg(parser, required=False):  # pragma: no cover
-    """ Add --cluster argument to a parser """
+    """Add --cluster argument to a parser"""
     parser.add_argument(
-        "--cluster", default=os.environ.get("CMAN_CLUSTER", None), help="Name of Mesos cluster to operate on",
+        "--cluster",
+        default=os.environ.get("CMAN_CLUSTER", None),
+        help="Name of Mesos cluster to operate on",
     )
 
 
 def add_pool_arg(parser):  # pragma: no cover
-    """ Add --pool argument to a parser """
+    """Add --pool argument to a parser"""
     parser.add_argument(
         "--pool",
         "--role",
@@ -114,7 +118,7 @@ def add_pool_arg(parser):  # pragma: no cover
 
 
 def add_scheduler_arg(parser):  # pragma: no cover
-    """ Add --scheduler argument to a parser """
+    """Add --scheduler argument to a parser"""
     parser.add_argument(
         "--scheduler",
         default=os.environ.get("CMAN_SCHEDULER", "mesos"),
@@ -124,28 +128,36 @@ def add_scheduler_arg(parser):  # pragma: no cover
 
 
 def add_env_config_path_arg(parser):  # pragma: no cover
-    """ Add a --env-config-path argument to a parser """
+    """Add a --env-config-path argument to a parser"""
     parser.add_argument(
-        "--env-config-path", default="/nail/srv/configs/clusterman.yaml", help="Path to clusterman configuration file",
+        "--env-config-path",
+        default="/nail/srv/configs/clusterman.yaml",
+        help="Path to clusterman configuration file",
     )
 
 
 def add_cluster_config_directory_arg(parser):  # pragma: no cover
     parser.add_argument(
-        "--cluster-config-directory", metavar="directory", help="specify role configuration directory for simulation",
+        "--cluster-config-directory",
+        metavar="directory",
+        help="specify role configuration directory for simulation",
     )
 
 
 def add_disable_sensu_arg(parser):  # pragma: no cover
-    """ Add a --disable-sensu argument to a parser """
+    """Add a --disable-sensu argument to a parser"""
     parser.add_argument(
-        "--disable-sensu", action="store_true", help="Disable sensu checkins",
+        "--disable-sensu",
+        action="store_true",
+        help="Disable sensu checkins",
     )
 
 
 def add_json_arg(parser):  # pragma: no cover
     parser.add_argument(
-        "--json", action="store_true", help="Return information in JSON format",
+        "--json",
+        action="store_true",
+        help="Return information in JSON format",
     )
 
 
@@ -189,7 +201,9 @@ def get_parser(description=""):  # pragma: no cover
     root_parser = argparse.ArgumentParser(prog="clusterman", description=description, formatter_class=help_formatter)
     add_env_config_path_arg(root_parser)
     root_parser.add_argument(
-        "--log-level", default="warning", choices=["debug", "info", "warning", "error", "critical"],
+        "--log-level",
+        default="warning",
+        choices=["debug", "info", "warning", "error", "critical"],
     )
     root_parser.add_argument("-v", "--version", action="version", version="clusterman " + __version__)
 

clusterman/autoscaler/autoscaler.py

@@ -60,7 +60,7 @@ def __init__(
         metrics_client: Optional[ClustermanMetricsBotoClient] = None,
         monitoring_enabled: bool = True,
     ) -> None:
-        """ Class containing the core logic for autoscaling a cluster
+        """Class containing the core logic for autoscaling a cluster
 
         :param cluster: the name of the cluster to autoscale
         :param pool: the name of the pool to autoscale
@@ -87,7 +87,8 @@ def __init__(
         self.resource_request_gauges: Dict[str, Any] = {}
         for resource in SignalResourceRequest._fields:
             self.resource_request_gauges[resource] = monitoring_client.create_gauge(
-                RESOURCE_GAUGE_BASE_NAME.format(resource=resource), gauge_dimensions,
+                RESOURCE_GAUGE_BASE_NAME.format(resource=resource),
+                gauge_dimensions,
             )
 
         self.autoscaling_config = get_autoscaling_config(
@@ -128,7 +129,7 @@ def run_frequency(self) -> int:
         return self.signal.period_minutes * 60
 
     def run(self, dry_run: bool = False, timestamp: Optional[arrow.Arrow] = None) -> None:
-        """ Do a single check to scale the fleet up or down if necessary.
+        """Do a single check to scale the fleet up or down if necessary.
 
         :param dry_run: boolean; if True, don't modify the pool size, just print what would happen
         :param timestamp: an arrow object indicating the current time
@@ -161,7 +162,8 @@ def run(self, dry_run: bool = False, timestamp: Optional[arrow.Arrow] = None) ->
                 logger.warning(
                     "Nodes lost since last autoscaler run is {}"
                     " which is greater than the threshold ({}),".format(
-                        str(num_removed_nodes_before_last_reload), str(self.autoscaling_config.instance_loss_threshold),
+                        str(num_removed_nodes_before_last_reload),
+                        str(self.autoscaling_config.instance_loss_threshold),
                     )
                 )
                 logger.warning("Autoscaler will not kill any nodes on this run.")
@@ -226,7 +228,9 @@ def _get_signal_for_app(self, app: str) -> Signal:
                 pool_namespace,
                 self.metrics_client,
                 signal_namespace=staticconf.read_string(
-                    "autoscale_signal.namespace", default=app, namespace=pool_namespace,
+                    "autoscale_signal.namespace",
+                    default=app,
+                    namespace=pool_namespace,
                 ),
             )
         except NoSignalConfiguredException:
@@ -250,7 +254,7 @@ def _get_signal_for_app(self, app: str) -> Signal:
             return self.default_signal
 
     def _compute_target_capacity(self, resource_request: SignalResourceRequest) -> float:
-        """ Compare signal to the resources allocated and compute appropriate capacity change.
+        """Compare signal to the resources allocated and compute appropriate capacity change.
 
         :param resource_request: a resource_request object from the signal evaluation
         :returns: the new target capacity we should scale to
@@ -320,7 +324,8 @@ def _compute_target_capacity(self, resource_request: SignalResourceRequest) -> f
 
         # If we get here, everything is non-zero and we can use the "normal" logic to determine scaling
         (most_constrained_resource, usage_pct,) = self._get_most_constrained_resource_for_request(
-            resource_request, cluster_total_resources,
+            resource_request,
+            cluster_total_resources,
         )
         logger.info(
             f"Fulfilling resource request will cause {most_constrained_resource} to be the most constrained resource "
@@ -371,7 +376,9 @@ def _compute_target_capacity(self, resource_request: SignalResourceRequest) -> f
         return new_target_capacity
 
     def _get_most_constrained_resource_for_request(
-        self, resource_request: SignalResourceRequest, cluster_total_resources: ClustermanResources,
+        self,
+        resource_request: SignalResourceRequest,
+        cluster_total_resources: ClustermanResources,
     ) -> Tuple[str, float]:
         """Determine what would be the most constrained resource if were to fulfill a resource_request without scaling
         the cluster.
@@ -403,7 +410,7 @@ def _get_most_constrained_resource_for_request(
         return max(requested_resource_usage_pcts.items(), key=lambda x: x[1])
 
     def _get_historical_weighted_resource_value(self) -> ClustermanResources:
-        """ Compute the weighted value of each type of resource in the cluster
+        """Compute the weighted value of each type of resource in the cluster
 
         returns: a ClustermanResources object with the weighted resource value, or 0 if it couldn't be determined
         """
@@ -419,7 +426,9 @@ def _get_historical_weighted_resource_value(self) -> ClustermanResources:
         weighted_resource_dict: MutableMapping[str, float] = {}
         for resource in ClustermanResources._fields:
             resource_history = self._get_smoothed_non_zero_metadata(
-                f"{resource}_total", time_start=time_start, time_end=time_end,
+                f"{resource}_total",
+                time_start=time_start,
+                time_end=time_end,
             )
             if not resource_history:
                 weighted_resource_dict[resource] = 0
@@ -429,9 +438,13 @@ def _get_historical_weighted_resource_value(self) -> ClustermanResources:
         return ClustermanResources(**weighted_resource_dict)
 
     def _get_smoothed_non_zero_metadata(
-        self, metric_name: str, time_start: arrow.Arrow, time_end: arrow.Arrow, smoothing: int = 5,
+        self,
+        metric_name: str,
+        time_start: arrow.Arrow,
+        time_end: arrow.Arrow,
+        smoothing: int = 5,
     ) -> Optional[Tuple[int, int, float]]:
-        """ Compute some smoothed-out historical metrics metadata
+        """Compute some smoothed-out historical metrics metadata
 
         :param metric_name: the metadata metric to query
         :param time_start: the beginning of the historical time window to query

clusterman/autoscaler/config.py

@@ -31,7 +31,7 @@ class AutoscalingConfig(NamedTuple):
 
 
 def get_autoscaling_config(config_namespace: str) -> AutoscalingConfig:
-    """ Load autoscaling configuration values from the provided config_namespace, falling back to the
+    """Load autoscaling configuration values from the provided config_namespace, falling back to the
     values stored in the default namespace if none are specified.
 
     :param config_namespace: namespace to read from before falling back to the default namespace
@@ -46,7 +46,8 @@ def get_autoscaling_config(config_namespace: str) -> AutoscalingConfig:
         excluded_resources=reader.read_list("autoscaling.excluded_resources", default=default_excluded_resources),
         setpoint=reader.read_float("autoscaling.setpoint", default=default_setpoint),
         target_capacity_margin=reader.read_float(
-            "autoscaling.target_capacity_margin", default=default_target_capacity_margin,
+            "autoscaling.target_capacity_margin",
+            default=default_target_capacity_margin,
         ),
         prevent_scale_down_after_capacity_loss=reader.read_bool(
             "autoscaling.prevent_scale_down_after_capacity_loss", default=False