Add estimated_cpucount attribute to Pilot sections (SOFTWARE-6197)

Author: matyasselmeci

config/35-pilot.ini

@@ -23,6 +23,8 @@
 ;ram_mb = 2500
 ;; This is a whole node pilot; cpucount and ram_mb are ignored if this is true.
 ;whole_node = false
+;; This is the number of CPUs available on a whole note pilot; this is ignored if whole_node is false.
+;estimated_cpucount = 8
 ;; The number of GPUs available
 ;gpucount = 0
 ;; The maximum number of pilots of this type that can be sent

osg_configure/modules/resourcecatalog.py

@@ -29,6 +29,7 @@ class RCAttribute(namedtuple("RCAttribute", "rce_field classad_attr format_fn"))
     RCAttribute("name", "Name", utilities.classad_quote),
     RCAttribute("cpus", "CPUs", int),
     RCAttribute("memory", "Memory", int),
+    RCAttribute("estimated_cpucount", "EstimatedCPUs", int),
     RCAttribute("allowed_vos", "AllowedVOs", _to_classad_list),
     RCAttribute("max_wall_time", "MaxWallTime", int),
     RCAttribute("queue", "Queue", utilities.classad_quote),
@@ -65,6 +66,7 @@ def __init__(self, **kwargs):
         self.os = kwargs.get('os', None)
         self.send_tests = kwargs.get('send_tests', None)
         self.is_pilot = kwargs.get('is_pilot', None)
+        self.estimated_cpucount = kwargs.get('estimated_cpucount', None)
 
     def get_requirements(self, attributes):
         if self.is_pilot:

osg_configure/modules/subcluster.py

@@ -33,6 +33,7 @@
     "subclusters":         (OPTIONAL, LIST),
     "vo_tag":              (OPTIONAL, STRING),
     # added for Pilots
+    "estimated_cpucount":  (OPTIONAL, POSITIVE_INT),
     "gpucount":            (OPTIONAL, POSITIVE_INT),
     "max_pilots":          (REQUIRED_FOR_PILOT, POSITIVE_INT),
     "os":                  (OPTIONAL, STRING),
@@ -72,6 +73,7 @@
     'maxmemory': (512, 8388608),
     'cores_per_node': (1, 8192),
     'cpucount': (1, 8192),
+    'estimated_cpucount': (1, 8192),
 }
 
 CPUCOUNT_DEFAULT = 1
@@ -219,7 +221,7 @@ def rce_section_get_name(config: ConfigParser, section: str) -> Optional[str]:
     """
     m = re.search(r"(?i:subcluster|resource entry|pilot)\s+(.+)", section)
     if not m:
-        return
+        return None
     default_name = m.group(1)
     return config[section].get("name", default_name).strip()
 
@@ -235,6 +237,7 @@ def format_value(self) -> str:
 def resource_catalog_from_config(config: ConfigParser, default_allowed_vos: List[str] = None) -> ResourceCatalog:
     """
     Create a ResourceCatalog from the subcluster entries in a config
+    :param config: The config to pull the subcluster information from
     :param default_allowed_vos: The allowed_vos to use if the user specified "*"
     """
     logger = logging.getLogger(__name__)
@@ -321,9 +324,12 @@ def safegetbool(option: str, default=None) -> bool:
         if is_pilot(section):
             rcentry.max_pilots = safeget("max_pilots")
             rcentry.whole_node = safegetbool("whole_node", False)
+            rcentry.estimated_cpucount = safeget("estimated_cpucount")
             if rcentry.whole_node:
                 rcentry.cpus = None
                 rcentry.memory = None
+            else:
+                rcentry.estimated_cpucount = None
             rcentry.require_singularity = safegetbool("require_singularity", True)
             rcentry.os = safeget("os")
             rcentry.send_tests = safegetbool("send_tests", True)

tests/configs/subcluster/pilots_example.ini

@@ -13,6 +13,13 @@ max_wall_time = 1440
 allowed_vos = atlas
 max_pilots = 1000
 
+[Pilot WholeNode_with_estimated_cpucount]
+whole_node = true
+estimated_cpucount = 32
+max_wall_time = 1440
+allowed_vos = atlas
+max_pilots = 1000
+
 [Pilot default]
 ram_mb = 32768
 os = rhel6

tests/test_resourcecatalog.py

@@ -326,6 +326,17 @@ def testPilotExample(self):
     SendTests = True; \
     WholeNode = True; \
   ], \
+  [ \
+    AllowedVOs = { "atlas" }; \
+    EstimatedCPUs = 32; \
+    IsPilotEntry = True; \
+    MaxPilots = 1000; \
+    MaxWallTime = 1440; \
+    Name = "WholeNode_with_estimated_cpucount"; \
+    RequireSingularity = True; \
+    SendTests = True; \
+    WholeNode = True; \
+  ], \
   [ \
     AllowedVOs = { "osg", "cms" }; \
     CPUs = 8; \