Use ce_attributes.py in infoservices config

Author: matyasselmeci

osg_configure/configure_modules/infoservices.py

@@ -2,15 +2,15 @@
  for CE collector info services"""
 
 import re
-import configparser as ConfigParser
+from configparser import ConfigParser
 import subprocess
 import logging
 
 from osg_configure.modules import exceptions
 from osg_configure.modules import utilities
 from osg_configure.modules import configfile
 from osg_configure.modules.baseconfiguration import BaseConfiguration
-from osg_configure.modules.ce_attributes import BATCH_SYSTEMS, BATCH_SYSTEMS_CASE_MAP
+from osg_configure.modules import ce_attributes
 from osg_configure.modules import subcluster
 from osg_configure.modules import reversevomap
 
@@ -52,13 +52,10 @@ def __init__(self, *args, **kwargs):
 
         self.ce_collectors = []
         self.ce_collector_required_rpms_installed = utilities.rpm_installed('htcondor-ce')
-        self.osg_resource = ""
-        self.osg_resource_group = ""
-        self.enabled_batch_systems = []
         self.htcondor_gateway_enabled = None
-        self.resource_catalog = None
         self.authorization_method = None
-        self.subcluster_sections = None
+        self.ce_attributes_config = None
+        self.ce_attributes_str = ""
 
         self.log("InfoServicesConfiguration.__init__ completed")
 
@@ -77,7 +74,7 @@ def _parse_ce_collectors(self, val):
         else:
             return val.split(',')
 
-    def parse_configuration(self, configuration):
+    def parse_configuration(self, configuration: ConfigParser):
         """
         Try to get configuration information from ConfigParser or SafeConfigParser object given
         by configuration and write recognized settings to attributes dict
@@ -110,38 +107,12 @@ def parse_configuration(self, configuration):
 
         self.ce_collectors = self._parse_ce_collectors(self.options['ce_collectors'].value)
 
-        def csg(section, option):
-            return utilities.config_safe_get(configuration, section, option, None)
-
-        def csgbool(section, option):
-            return utilities.config_safe_getboolean(configuration, section, option, False)
-
-        # We get some values for HTCondor-CE from the Site Information section
-        self.osg_resource = csg('Site Information', 'resource')
-        self.osg_resource_group = csg('Site Information', 'resource_group')
-        # and the enabled batch systems from their respective sections
-        self.enabled_batch_systems = [bs for bs in BATCH_SYSTEMS if csgbool(bs, 'enabled')]
-
-        if csgbool("Bosco", "enabled"):
-            try:
-                bosco_batch = BATCH_SYSTEMS_CASE_MAP[csg("Bosco", "batch").lower()]
-                if bosco_batch not in self.enabled_batch_systems:
-                    self.enabled_batch_systems.append(bosco_batch)
-            except (KeyError, AttributeError):
-                # This is a warning; the bosco module raises the error.
-                self.log("Bosco.batch does not have a recognized value -- should be one of %s" %
-                         list(BATCH_SYSTEMS_CASE_MAP.keys()),
-                         level=logging.WARNING)
-
-        self.htcondor_gateway_enabled = csgbool('Gateway', 'htcondor_gateway_enabled')
-
-        self.subcluster_sections = ConfigParser.SafeConfigParser()
+        self.htcondor_gateway_enabled = configuration.get('Gateway', 'htcondor_gateway_enabled', fallback=False)
 
+        self.ce_attributes_config = ConfigParser()
         for section in configuration.sections():
-            if subcluster.is_subcluster_like(section):
-                self.subcluster_sections.add_section(section)
-                for key, value in configuration.items(section):
-                    self.subcluster_sections.set(section, key, value)
+            if subcluster.is_subcluster_like(section) or section.lower() == "site information":
+                self.ce_attributes_config[section] = configuration[section]
 
         if utilities.ce_installed() and not subcluster.check_config(configuration):
             self.log("On a CE but no valid 'Subcluster', 'Resource Entry', or 'Pilot' sections defined."
@@ -155,7 +126,7 @@ def csgbool(section, option):
         # configure(), but at this point we don't have a way of knowing what
         # default_allowed_vos should be.
         if self.ce_collector_required_rpms_installed and self.htcondor_gateway_enabled and classad is not None:
-            subcluster.resource_catalog_from_config(self.subcluster_sections, default_allowed_vos=["*"])
+            subcluster.resource_catalog_from_config(self.ce_attributes_config, default_allowed_vos=["*"])
 
         self.log('InfoServicesConfiguration.parse_configuration completed')
 
@@ -179,14 +150,12 @@ def configure(self, attributes):
             if classad is None:
                 self.log("Cannot configure HTCondor CE info services: unable to import HTCondor Python bindings."
                          "\nEnsure the 'classad' Python module is installed and accessible to Python scripts."
-                         "\nIf using HTCondor from RPMs, install the 'condor-python' RPM."
+                         "\nIf using HTCondor from RPMs, install the 'python3-condor' RPM."
                          "\nIf not, you may need to add the directory containing the Python bindings to PYTHONPATH."
                          "\nHTCondor version must be at least 8.2.0.", level=logging.WARNING)
             else:
                 try:
-                    self.resource_catalog = subcluster.resource_catalog_from_config(
-                        self.subcluster_sections,
-                        default_allowed_vos=["*"])
+                    self.ce_attributes_str = ce_attributes.get_ce_attributes_str(self.ce_attributes_config)
                 except exceptions.SettingError as err:
                     self.log("Error in info services configuration: %s" % err, level=logging.ERROR)
                     return False
@@ -238,27 +207,10 @@ def _write_ce_collector_attributes_file(self, attributes_file):
         CE-Collector to advertise
 
         """
-        schedd_attrs_list = ["$(SCHEDD_ATTRS)"]
-        attributes_file_lines = []
-
-        for name, value in [
-            ('OSG_Resource', self.osg_resource),
-            ('OSG_ResourceGroup', self.osg_resource_group),
-            ('OSG_BatchSystems', ",".join(self.enabled_batch_systems))
-        ]:
-            attributes_file_lines.append("%s = %s" % (name, utilities.classad_quote(value)))
-            schedd_attrs_list.append(name)
-
-        if self.resource_catalog:
-            attributes_file_lines.append(self.resource_catalog.compose_text())
-            schedd_attrs_list.append('OSG_ResourceCatalog')
-
         attributes_file_contents = (
             "# Do not edit - file generated by osg-configure\n"
-            + "\n".join(attributes_file_lines) + "\n"
-            + "SCHEDD_ATTRS = " + " ".join(schedd_attrs_list) + "\n"
+            + self.ce_attributes_str + "\n"
         )
-
         return utilities.atomic_write(attributes_file, attributes_file_contents)
 
     def _write_ce_collector_file(self, info_services_file):

osg_configure/modules/ce_attributes.py

@@ -95,3 +95,11 @@ def get_attributes(config: ConfigParser) -> Dict[str, str]:
         attributes["OSG_ResourceCatalog"] = resource_catalog
 
     return attributes
+
+
+def get_ce_attributes_str(
+        config: ConfigParser,
+) -> str:
+    attributes = get_attributes(config)
+    attributes["SCHEDD_ATTRS"] = "$(SCHEDD_ATTRS), " + ", ".join(attributes.keys())
+    return "\n".join(f"{key} = {value}" for key, value in attributes.items())

osg_configure/modules/subcluster.py

@@ -203,7 +203,7 @@ def check_config(config: ConfigParser) -> bool:
     """
     has_sc = False
     for section in config.sections():
-        if is_pilot(section) or is_subcluster(section) or is_resource_entry(section):
+        if is_subcluster_like(section):
             has_sc = True
             check_section(config, section)
     return has_sc
@@ -255,7 +255,7 @@ def safegetbool(option: str, default=None) -> bool:
 
     sections_without_max_wall_time = []
     for section in config.sections():
-        if not (is_subcluster(section) or is_resource_entry(section) or is_pilot(section)):
+        if not is_subcluster_like(section):
             continue
 
         check_section(config, section)
@@ -335,6 +335,6 @@ def safegetbool(option: str, default=None) -> bool:
     if sections_without_max_wall_time:
         logger.warning("No max_wall_time specified for some sections; defaulting to 1440."
                        "\nAdd 'max_wall_time=1440' to the following section(s) to clear this warning:"
-                       "\n'%s'" % "', '".join(sections_without_max_wall_time))
+                       " '%s'" % "', '".join(sections_without_max_wall_time))
 
     return rc

scripts/osg-ce-attribs-gen

@@ -18,7 +18,7 @@ if __name__ == "__main__" and __package__ is None:
     sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 # local imports here
-from osg_configure.modules.ce_attributes import BATCH_SYSTEMS, get_attributes
+from osg_configure.modules.ce_attributes import BATCH_SYSTEMS, get_ce_attributes_str
 from osg_configure.modules.exceptions import Error
 from osg_configure.version import __version__
 
@@ -96,14 +96,6 @@ def get_options(args):
     return parser.parse_args(args)
 
 
-def get_ce_attributes_str(
-        config: ConfigParser,
-) -> str:
-    attributes = get_attributes(config)
-    attributes["SCHEDD_ATTRS"] = "$(SCHEDD_ATTRS), " + ", ".join(attributes.keys())
-    return "\n".join(f"{key} = {value}" for key, value in attributes.items())
-
-
 def main(argv):
     options = get_options(argv[1:])