Rename osg-resource-catalog-generator to osg-ce-attribs-gen and do a lot of restructuring

Use positional arguments instead of --config and --output (as described in SOFTWARE-4895).
Create a stub ce_attributes.py; will later move code from osg-ce-attribs-gen into it.

Author: matyasselmeci

osg_configure/configure_modules/infoservices.py

@@ -9,7 +9,8 @@
 from osg_configure.modules import exceptions
 from osg_configure.modules import utilities
 from osg_configure.modules import configfile
-from osg_configure.modules.baseconfiguration import BaseConfiguration, BATCH_SYSTEMS, BOSCO_BATCH_SYSTEMS_MAP
+from osg_configure.modules.baseconfiguration import BaseConfiguration
+from osg_configure.modules.ce_attributes import BATCH_SYSTEMS, BATCH_SYSTEMS_CASE_MAP
 from osg_configure.modules import subcluster
 from osg_configure.modules import reversevomap
 
@@ -123,13 +124,13 @@ def csgbool(section, option):
 
         if csgbool("Bosco", "enabled"):
             try:
-                bosco_batch = BOSCO_BATCH_SYSTEMS_MAP[csg("Bosco", "batch").lower()]
+                bosco_batch = BATCH_SYSTEMS_CASE_MAP[csg("Bosco", "batch").lower()]
                 if bosco_batch not in self.enabled_batch_systems:
                     self.enabled_batch_systems.append(bosco_batch)
             except (KeyError, AttributeError):
                 # This is a warning; the bosco module raises the error.
                 self.log("Bosco.batch does not have a recognized value -- should be one of %s" %
-                         list(BOSCO_BATCH_SYSTEMS_MAP.keys()),
+                         list(BATCH_SYSTEMS_CASE_MAP.keys()),
                          level=logging.WARNING)
 
         self.htcondor_gateway_enabled = csgbool('Gateway', 'htcondor_gateway_enabled')

osg_configure/modules/baseconfiguration.py

@@ -10,22 +10,11 @@
 from osg_configure.modules import utilities
 from osg_configure.modules import exceptions
 
-__all__ = ['BaseConfiguration', 'BATCH_SYSTEMS']
+__all__ = ['BaseConfiguration']
 
 HOSTCERT_PATH = "/etc/grid-security/hostcert.pem"
 HOSTKEY_PATH = "/etc/grid-security/hostkey.pem"
 
-# BATCH_SYSTEMS here is both the config sections for the batch systems
-# and the values in the OSG_BatchSystems attribute since they are
-# coincidentally the same. If they ever change, make a mapping.
-BATCH_SYSTEMS = ['Condor', 'LSF', 'PBS', 'SGE', 'SLURM']
-BOSCO_BATCH_SYSTEMS_MAP = {
-    'condor': 'Condor',
-    'lsf': 'LSF',
-    'pbs': 'PBS',
-    'sge': 'SGE',
-    'slurm': 'SLURM',
-}
 
 class BaseConfiguration:
     """Base class for inheritance by configuration"""

osg_configure/modules/ce_attributes.py

@@ -0,0 +1,11 @@
+# BATCH_SYSTEMS here is both the config sections for the batch systems
+# and the values in the OSG_BatchSystems attribute since they are
+# coincidentally the same. If they ever change, make a mapping.
+BATCH_SYSTEMS_CASE_MAP = {
+    'condor': 'Condor',
+    'lsf': 'LSF',
+    'pbs': 'PBS',
+    'sge': 'SGE',
+    'slurm': 'SLURM',
+}
+BATCH_SYSTEMS = list(BATCH_SYSTEMS_CASE_MAP.values())

scripts/osg-ce-attribs-gen

@@ -0,0 +1,217 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+osg-ce-attribs-gen
+
+OSG CA Attributes File Generator
+
+Standalone script to generate a condor config file for advertising a CE resource to the CE Collector.
+"""
+
+from argparse import ArgumentParser
+from configparser import ConfigParser
+import glob
+import os
+import sys
+from typing import Dict
+
+if __name__ == "__main__" and __package__ is None:
+    sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+# local imports here
+from osg_configure.modules.ce_attributes import BATCH_SYSTEMS, BATCH_SYSTEMS_CASE_MAP
+from osg_configure.modules.exceptions import Error, SettingError
+from osg_configure.modules import subcluster
+from osg_configure.modules import utilities
+from osg_configure.version import __version__
+
+
+def warn(*args, **kwargs):
+    kwargs['file'] = sys.stderr
+    print("***", *args, **kwargs)
+
+
+def load_configs(config_location: str) -> ConfigParser:
+    config = ConfigParser()
+
+    if config_location == "-":
+        config.read_string(sys.stdin.read(), "<stdin>")
+        return config
+
+    config_file_list = []
+    if os.path.isdir(config_location):
+        config_file_list.extend(sorted(glob.glob(os.path.join(config_location, "[!.]*.ini"))))
+    else:
+        config_file_list.append(config_location)
+
+    read_files = config.read(config_file_list)
+    if not read_files:
+        raise Error(f"No valid config files found in {config_location}")
+
+    return config
+
+
+def empty_if_blank(value: str) -> str:
+    return "" if utilities.blank(value) else value
+
+
+def get_resource_from_config(config: ConfigParser) -> str:
+    return utilities.classad_quote(
+        empty_if_blank(
+            config.get("Site Information", "resource", fallback="")
+        )
+    )
+
+
+def get_resource_group_from_config(config: ConfigParser) -> str:
+    return utilities.classad_quote(
+        empty_if_blank(
+            config.get("Site Information", "resource_group", fallback="")
+        )
+    )
+
+
+def get_batch_systems_from_config(config: ConfigParser) -> str:
+    batch_systems = []
+
+    siteinfo_batch_systems = config.get("Site Information", "batch_systems", fallback=None)
+    if siteinfo_batch_systems is not None:
+        # Site Information.batch_systems specified -- this one wins
+        split_batch_systems = utilities.split_comma_separated_list(siteinfo_batch_systems)
+        for batch_system in split_batch_systems:
+            try:
+                batch_systems.append(BATCH_SYSTEMS_CASE_MAP[batch_system.lower()])
+            except KeyError:
+                raise SettingError("Unrecognized batch system %s" % batch_system)
+    else:
+        # Add each batch system that's enabled from the sections in the 20-*.ini files.
+        for batch_system in BATCH_SYSTEMS:
+            if batch_system in config:
+                if config.getboolean(section=batch_system, option="enabled", fallback=None):
+                    batch_systems.append(batch_system)
+
+        # Special case: Bosco (see SOFTWARE-3720); use the Bosco.batch argument.
+        if config.getboolean("Bosco", "enabled", fallback=False):
+            bosco_batch = config.get("Bosco", "batch", fallback=None)
+            if bosco_batch:
+                try:
+                    batch_systems.append(BATCH_SYSTEMS_CASE_MAP[bosco_batch.lower()])
+                except KeyError:
+                    raise SettingError("Unrecognized batch system %s in Bosco section" % bosco_batch)
+
+    return utilities.classad_quote(",".join(batch_systems))
+
+
+def get_resource_catalog_from_config(config: ConfigParser) -> str:
+    return subcluster.resource_catalog_from_config(config, default_allowed_vos=[]).format_value()
+
+
+def get_attributes(config: ConfigParser) -> Dict[str, str]:
+    """Turn config from .ini files into a dict of condor settings.
+
+    """
+    attributes = {}
+
+    resource = get_resource_from_config(config)
+    if resource and resource != '""':
+        attributes["OSG_Resource"] = resource
+
+    resource_group = get_resource_group_from_config(config)
+    if resource_group and resource_group != '""':
+        attributes["OSG_ResourceGroup"] = resource_group
+
+    batch_systems = get_batch_systems_from_config(config)
+    if batch_systems and batch_systems != '""':
+        attributes["OSG_BatchSystems"] = batch_systems
+
+    resource_catalog = get_resource_catalog_from_config(config)
+    if resource_catalog and resource_catalog != "{}":
+        attributes["OSG_ResourceCatalog"] = resource_catalog
+
+    return attributes
+
+
+def get_options(args):
+    """Parse, validate, and transform command-line options."""
+    parser = ArgumentParser(prog="osg-ca-attrib-gen", description=__doc__)
+    parser.add_argument("--version", action="version", version="%(prog)s " + __version__)
+    parser.add_argument(
+        "config_location",
+        nargs="?",
+        default="/etc/osg/config.d",
+        metavar="FILE_OR_DIRECTORY",
+        help="Where to load configuration from. "
+             "If this is a directory, will load every *.ini file in that directory. "
+             "Default: %(default)s. "
+             "If '-', will read from STDIN.",
+    )
+    parser.add_argument(
+        "output",
+        nargs="?",
+        default="-",
+        metavar="FILE",
+        help="Write output to this file.  If '-' or unspecified, will write to STDOUT.",
+    )
+    parser.add_argument(
+        "--resource",
+        metavar="RESOURCE_NAME",
+        default=None,
+        help="The Resource name to use, which should match your Topology registration. "
+        "Equivalent to 'resource' in the 'Site Information' section."
+    )
+    parser.add_argument(
+        "--resource-group",
+        metavar="RESOURCE_GROUP_NAME",
+        default=None,
+        help="The Resource Group name to use, which should match your Topology registration. "
+        "Equivalent to 'resource_group' in the 'Site Information' section."
+    )
+    parser.add_argument(
+        "--batch-systems",
+        metavar="BATCH_SYSTEMS_LIST",
+        default=None,
+        help="A comma-separated list of batch systems used by the resource. "
+        f'Recognized batch systems are: {", ".join(BATCH_SYSTEMS)}. '
+        "Equivalent to enabling the batch system sections in the 20-*.ini files, "
+        "or, if using Bosco, setting 'batch_system' in the 'Bosco' section."
+    )
+
+    return parser.parse_args(args)
+
+
+def get_ce_attributes_str(
+        config: ConfigParser,
+) -> str:
+    attributes = get_attributes(config)
+    attributes["SCHEDD_ATTRS"] = "$(SCHEDD_ATTRS), " + ", ".join(attributes.keys())
+    return "\n".join(f"{key} = {value}" for key, value in attributes.items())
+
+
+def main(argv):
+    options = get_options(argv[1:])
+
+    try:
+        config = load_configs(options.config_location)
+        if "Site Information" not in config:
+            config.add_section("Site Information")
+        if options.resource is not None:
+            config["Site Information"]["resource"] = options.resource
+        if options.resource_group is not None:
+            config["Site Information"]["resource_group"] = options.resource_group
+        if options.batch_systems is not None:
+            config["Site Information"]["batch_systems"] = options.batch_systems
+        output_str = get_ce_attributes_str(config)
+        if options.output and options.output != "-":
+            with open(options.output, "w") as outfh:
+                print(output_str, file=outfh)
+        else:
+            print(output_str)
+    except Error as e:
+        print(e, file=sys.stderr)
+        return 1
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv))