Merge pull request #135 from matyasselmeci/pr/SOFTWARE-4895.ceattrgen

Add osg-ce-attributes-generator (SOFTWARE-4895)

Author: matyasselmeci

README.md

@@ -475,3 +475,87 @@ This section is contained in `/etc/osg/config.d/40-siteinfo.ini` which is provid
 | **host\_name**      | String            | This should be set to be hostname of the CE that is being configured                                                                         |
 | **resource**        | String            | The resource name of this CE endpoint as registered in Topology.                                                                                  |
 | **resource\_group** | String            | The resource\_group of this CE as registered in Topology.                                                                                         |
+
+
+OSG CE Attributes Generator
+===========================
+
+The CE Attributes Generator is a standalone tool to create resource catalog attributes for a CE, that will be uploaded to the CE collector.
+It is a lightweight alternative to OSG-Configure, and it prints the attributes instead of modifying your CE's config files.
+
+It uses `*.ini` files from OSG-Configure as the source of the data.
+
+Installation
+------------
+
+The CE Attributes Generator is typically installed via RPMs from the OSG repositories.
+See [OSG documentation for how to enable the repositories](https://opensciencegrid.org/docs/common/yum/);
+once the repos are enabled, install by running
+
+    yum install osg-ce-attributes-generator
+
+Note: `osg-ce-attributes-generator` is not available in the OSG 3.5 series.
+
+
+Configuration
+-------------
+
+The CE Attributes Generator uses the same config files that OSG-Configure uses.
+See the [31-cluster.ini](#31-clusterini--subcluster-and-resource-entry-sections),
+[35-pilot.ini](#35-pilotini--pilot) for attributes.
+
+You will also need `resource` and `resource_group` from the `Site Information` section
+([40-siteinfo.ini](#40-siteinfoini--site-information-section)).
+You can also use `--resource` and `--resource-group` on the command line to specify these
+instead.
+
+In addition, you need to specify at least one batch system.
+The recognized batch systems are "Condor", "LSF", "PBS", "SGE", and "SLURM".
+You can specify available batch systems in one of three ways:
+
+1.  Have a batch system section (one of the `20-*.ini` files) with the attribute `enabled=True`, e.g.
+```ini
+[Condor]
+enabled=True
+```
+2.  If you are using BOSCO, specify your batch system in the `batch` attribute in `20-bosco.ini`.
+
+3.  Specify a comma-separated list with `--batch-systems` on the command line.
+
+
+Batch systems specified on the command line take precedence.  If you have both a BOSCO section
+and an enabled batch system section, all of them will be listed in the attributes file.
+
+
+Invocation
+----------
+
+    osg-ce-attributes-generator [<options>] [<config_location>] [<output>]
+
+- `config_location` is a file or directory to load configuration from.
+  If this is a directory, load every `*.ini` file in that directory.
+  If `-`, read from STDIN. The default is to read `/etc/osg/config.d`,
+  same as `osg-configure`.
+
+- `output` is a file to write attributes to.
+  If `-` or unspecified, write to STDOUT.
+
+
+### Options
+
+- `--resource RESOURCE_NAME`
+
+  The Resource name to use, which should match your Topology registration.
+  Equivalent to `resource` in the `Site Information` section.
+
+- `--resource-group RESOURCE_GROUP_NAME`
+
+  The Resource Group name to use, which should match your Topology registration.
+  Equivalent to `resource_group` in the `Site Information` section.
+  
+- `--batch-systems BATCH_SYSTEMS_LIST`
+
+  A comma-separated list of batch systems used by the resource.
+  Recognized batch systems are: `Condor`, `LSF`, `PBS`, `SGE`, and `SLURM`.
+  Equivalent to enabling the batch system sections in the
+  `20-*.ini` files, or, if using BOSCO, setting `batch` in the `BOSCO` section.

osg_configure/configure_modules/infoservices.py

@@ -2,14 +2,15 @@
  for CE collector info services"""
 
 import re
-import configparser as ConfigParser
+from configparser import ConfigParser
 import subprocess
 import logging
 
 from osg_configure.modules import exceptions
 from osg_configure.modules import utilities
 from osg_configure.modules import configfile
 from osg_configure.modules.baseconfiguration import BaseConfiguration
+from osg_configure.modules import ce_attributes
 from osg_configure.modules import subcluster
 from osg_configure.modules import reversevomap
 
@@ -22,10 +23,6 @@
 BAN_VOMS_MAPFILE = reversevomap.BAN_MAPFILE
 BAN_MAPFILE = '/etc/grid-security/ban-mapfile'
 
-# BATCH_SYSTEMS here is both the config sections for the batch systems
-# and the values in the OSG_BatchSystems attribute since they are
-# coincidentally the same. If they ever change, make a mapping.
-BATCH_SYSTEMS = ['Condor', 'LSF', 'PBS', 'SGE', 'SLURM']
 
 try:
     import classad
@@ -55,13 +52,10 @@ def __init__(self, *args, **kwargs):
 
         self.ce_collectors = []
         self.ce_collector_required_rpms_installed = utilities.rpm_installed('htcondor-ce')
-        self.osg_resource = ""
-        self.osg_resource_group = ""
-        self.enabled_batch_systems = []
         self.htcondor_gateway_enabled = None
-        self.resource_catalog = None
         self.authorization_method = None
-        self.subcluster_sections = None
+        self.configuration = None
+        self.ce_attributes_str = ""
 
         self.log("InfoServicesConfiguration.__init__ completed")
 
@@ -80,7 +74,7 @@ def _parse_ce_collectors(self, val):
         else:
             return val.split(',')
 
-    def parse_configuration(self, configuration):
+    def parse_configuration(self, configuration: ConfigParser):
         """
         Try to get configuration information from ConfigParser or SafeConfigParser object given
         by configuration and write recognized settings to attributes dict
@@ -113,27 +107,9 @@ def parse_configuration(self, configuration):
 
         self.ce_collectors = self._parse_ce_collectors(self.options['ce_collectors'].value)
 
-        def csg(section, option):
-            return utilities.config_safe_get(configuration, section, option, None)
+        self.htcondor_gateway_enabled = configuration.get('Gateway', 'htcondor_gateway_enabled', fallback=False)
 
-        def csgbool(section, option):
-            return utilities.config_safe_getboolean(configuration, section, option, False)
-
-        # We get some values for HTCondor-CE from the Site Information section
-        self.osg_resource = csg('Site Information', 'resource')
-        self.osg_resource_group = csg('Site Information', 'resource_group')
-        # and the enabled batch systems from their respective sections
-        self.enabled_batch_systems = [bs for bs in BATCH_SYSTEMS if csgbool(bs, 'enabled')]
-
-        self.htcondor_gateway_enabled = csgbool('Gateway', 'htcondor_gateway_enabled')
-
-        self.subcluster_sections = ConfigParser.SafeConfigParser()
-
-        for section in configuration.sections():
-            if subcluster.is_subcluster_like(section):
-                self.subcluster_sections.add_section(section)
-                for key, value in configuration.items(section):
-                    self.subcluster_sections.set(section, key, value)
+        self.configuration = configuration  # save for later: the ce_attributes module reads the whole config.
 
         if utilities.ce_installed() and not subcluster.check_config(configuration):
             self.log("On a CE but no valid 'Subcluster', 'Resource Entry', or 'Pilot' sections defined."
@@ -147,7 +123,7 @@ def csgbool(section, option):
         # configure(), but at this point we don't have a way of knowing what
         # default_allowed_vos should be.
         if self.ce_collector_required_rpms_installed and self.htcondor_gateway_enabled and classad is not None:
-            subcluster.resource_catalog_from_config(self.subcluster_sections, default_allowed_vos=["*"])
+            subcluster.resource_catalog_from_config(configuration, default_allowed_vos=["*"])
 
         self.log('InfoServicesConfiguration.parse_configuration completed')
 
@@ -171,14 +147,12 @@ def configure(self, attributes):
             if classad is None:
                 self.log("Cannot configure HTCondor CE info services: unable to import HTCondor Python bindings."
                          "\nEnsure the 'classad' Python module is installed and accessible to Python scripts."
-                         "\nIf using HTCondor from RPMs, install the 'condor-python' RPM."
+                         "\nIf using HTCondor from RPMs, install the 'python3-condor' RPM."
                          "\nIf not, you may need to add the directory containing the Python bindings to PYTHONPATH."
                          "\nHTCondor version must be at least 8.2.0.", level=logging.WARNING)
             else:
                 try:
-                    self.resource_catalog = subcluster.resource_catalog_from_config(
-                        self.subcluster_sections,
-                        default_allowed_vos=["*"])
+                    self.ce_attributes_str = ce_attributes.get_ce_attributes_str(self.configuration)
                 except exceptions.SettingError as err:
                     self.log("Error in info services configuration: %s" % err, level=logging.ERROR)
                     return False
@@ -230,27 +204,10 @@ def _write_ce_collector_attributes_file(self, attributes_file):
         CE-Collector to advertise
 
         """
-        schedd_attrs_list = ["$(SCHEDD_ATTRS)"]
-        attributes_file_lines = []
-
-        for name, value in [
-            ('OSG_Resource', self.osg_resource),
-            ('OSG_ResourceGroup', self.osg_resource_group),
-            ('OSG_BatchSystems', ",".join(self.enabled_batch_systems))
-        ]:
-            attributes_file_lines.append("%s = %s" % (name, utilities.classad_quote(value)))
-            schedd_attrs_list.append(name)
-
-        if self.resource_catalog:
-            attributes_file_lines.append(self.resource_catalog.compose_text())
-            schedd_attrs_list.append('OSG_ResourceCatalog')
-
         attributes_file_contents = (
             "# Do not edit - file generated by osg-configure\n"
-            + "\n".join(attributes_file_lines) + "\n"
-            + "SCHEDD_ATTRS = " + " ".join(schedd_attrs_list) + "\n"
+            + self.ce_attributes_str + "\n"
         )
-
         return utilities.atomic_write(attributes_file, attributes_file_contents)
 
     def _write_ce_collector_file(self, info_services_file):

osg_configure/configure_modules/siteinformation.py

@@ -40,7 +40,11 @@ def __init__(self, *args, **kwargs):
                                               mapping='OSG_SITE_NAME'),
                         'resource_group':
                             configfile.Option(name='resource_group',
-                                              required=MANDATORY_ON_CE)}
+                                              required=MANDATORY_ON_CE),
+                        'batch_systems':
+                            configfile.Option(name='batch_systems',
+                                              required=OPTIONAL,
+                                              default_value=None)}
 
         self.config_section = "Site Information"
         self.enabled = True

osg_configure/modules/ce_attributes.py

@@ -0,0 +1,105 @@
+# BATCH_SYSTEMS here is both the config sections for the batch systems
+# and the values in the OSG_BatchSystems attribute since they are
+# coincidentally the same. If they ever change, make a mapping.
+from configparser import ConfigParser
+from typing import Dict
+
+from osg_configure.modules import utilities, subcluster
+from osg_configure.modules.exceptions import SettingError
+
+BATCH_SYSTEMS_CASE_MAP = {
+    'condor': 'Condor',
+    'lsf': 'LSF',
+    'pbs': 'PBS',
+    'sge': 'SGE',
+    'slurm': 'SLURM',
+}
+BATCH_SYSTEMS = list(BATCH_SYSTEMS_CASE_MAP.values())
+
+
+def empty_if_blank(value: str) -> str:
+    return "" if utilities.blank(value) else value
+
+
+def get_resource_from_config(config: ConfigParser) -> str:
+    return utilities.classad_quote(
+        empty_if_blank(
+            config.get("Site Information", "resource", fallback="")
+        )
+    )
+
+
+def get_resource_group_from_config(config: ConfigParser) -> str:
+    return utilities.classad_quote(
+        empty_if_blank(
+            config.get("Site Information", "resource_group", fallback="")
+        )
+    )
+
+
+def get_batch_systems_from_config(config: ConfigParser) -> str:
+    batch_systems = set()
+
+    siteinfo_batch_systems = config.get("Site Information", "batch_systems", fallback=None)
+    if siteinfo_batch_systems is not None:
+        # Site Information.batch_systems specified -- this one wins
+        split_batch_systems = utilities.split_comma_separated_list(siteinfo_batch_systems)
+        for batch_system in split_batch_systems:
+            try:
+                batch_systems.add(BATCH_SYSTEMS_CASE_MAP[batch_system.lower()])
+            except KeyError:
+                raise SettingError("Unrecognized batch system %s" % batch_system)
+    else:
+        # Add each batch system that's enabled from the sections in the 20-*.ini files.
+        for batch_system in BATCH_SYSTEMS:
+            if batch_system in config:
+                if config.getboolean(section=batch_system, option="enabled", fallback=None):
+                    batch_systems.add(batch_system)
+
+        # Special case: BOSCO (see SOFTWARE-3720); use the BOSCO.batch argument.
+        if config.getboolean("BOSCO", "enabled", fallback=False):
+            bosco_batch = config.get("BOSCO", "batch", fallback=None)
+            if bosco_batch:
+                try:
+                    batch_systems.add(BATCH_SYSTEMS_CASE_MAP[bosco_batch.lower()])
+                except KeyError:
+                    raise SettingError("Unrecognized batch system %s in Bosco section" % bosco_batch)
+
+    return utilities.classad_quote(",".join(sorted(batch_systems)))
+
+
+def get_resource_catalog_from_config(config: ConfigParser) -> str:
+    return subcluster.resource_catalog_from_config(config, default_allowed_vos=[]).format_value()
+
+
+def get_attributes(config: ConfigParser) -> Dict[str, str]:
+    """Turn config from .ini files into a dict of condor settings.
+
+    """
+    attributes = {}
+
+    resource = get_resource_from_config(config)
+    if resource and resource != '""':
+        attributes["OSG_Resource"] = resource
+
+    resource_group = get_resource_group_from_config(config)
+    if resource_group and resource_group != '""':
+        attributes["OSG_ResourceGroup"] = resource_group
+
+    batch_systems = get_batch_systems_from_config(config)
+    if batch_systems and batch_systems != '""':
+        attributes["OSG_BatchSystems"] = batch_systems
+
+    resource_catalog = get_resource_catalog_from_config(config)
+    if resource_catalog and resource_catalog != "{}":
+        attributes["OSG_ResourceCatalog"] = resource_catalog
+
+    return attributes
+
+
+def get_ce_attributes_str(
+        config: ConfigParser,
+) -> str:
+    attributes = get_attributes(config)
+    attributes["SCHEDD_ATTRS"] = "$(SCHEDD_ATTRS), " + ", ".join(attributes.keys())
+    return "\n".join(f"{key} = {value}" for key, value in attributes.items())

osg_configure/modules/resourcecatalog.py

@@ -21,10 +21,6 @@ def _to_classad_list(a_list):
     return "{ " + ", ".join([utilities.classad_quote(it) for it in a_list if it]) + " }"
 
 
-def _str_to_classad_list(a_str):
-    return _to_classad_list(utilities.split_comma_separated_list(a_str))
-
-
 class RCAttribute(namedtuple("RCAttribute", "rce_field classad_attr format_fn")):
     """The mapping of an RCEntry field to a classad attribute, with a format function"""
 
@@ -140,7 +136,6 @@ def as_attributes(self):
         return attributes
 
 
-
 class ResourceCatalog(object):
     """Class for building an OSG_ResourceCatalog attribute in condor-ce configs for the ce-collector"""
 
@@ -152,7 +147,7 @@ def add_rcentry(self, rcentry):
 
         return self
 
-    def compose_text(self):
+    def format_value(self) -> str:
         """Return the OSG_ResourceCatalog classad attribute made of all the entries in this object"""
         if not self.entries:
             catalog = '{}'
@@ -168,5 +163,8 @@ def compose_text(self):
             catalog = ('{ \\\n'
                        + ', \\\n'.join(entry_texts)
                        + ' \\\n}')
+        return catalog
 
+    def compose_text(self):
+        catalog = self.format_value()
         return 'OSG_ResourceCatalog = ' + catalog