Merge pull request #4769 from Flamefire/multi-class-easyblocks

Ignore other classes if software specific easyblock class was found

Author: boegel

easybuild/framework/easyconfig/tools.py

@@ -58,7 +58,7 @@
 from easybuild.tools.build_log import EasyBuildError, print_error, print_msg, print_warning
 from easybuild.tools.config import build_option
 from easybuild.tools.environment import restore_env
-from easybuild.tools.filetools import find_easyconfigs, is_patch_file, locate_files
+from easybuild.tools.filetools import EASYBLOCK_CLASS_PREFIX, find_easyconfigs, is_patch_file, locate_files
 from easybuild.tools.filetools import read_file, resolve_path, which, write_file
 from easybuild.tools.github import GITHUB_EASYCONFIGS_REPO
 from easybuild.tools.github import det_pr_labels, det_pr_title, download_repo, fetch_easyconfigs_from_commit
@@ -758,7 +758,7 @@ def avail_easyblocks():
     """Return a list of all available easyblocks."""
 
     module_regexp = re.compile(r"^([^_].*)\.py$")
-    class_regex = re.compile(r"^class ([^(]*)\(", re.M)
+    class_regex = re.compile(r"^class ([^(:]*)\(", re.M)
 
     # finish initialisation of the toolchain module (ie set the TC_CONSTANT constants)
     search_toolchain('')
@@ -768,33 +768,43 @@ def avail_easyblocks():
         __import__(pkg)
 
         # determine paths for this package
-        paths = sys.modules[pkg].__path__
+        paths = [path for path in sys.modules[pkg].__path__ if os.path.exists(path)]
 
         # import all modules in these paths
         for path in paths:
-            if os.path.exists(path):
-                for fn in os.listdir(path):
-                    res = module_regexp.match(fn)
-                    if res:
-                        easyblock_mod_name = '%s.%s' % (pkg, res.group(1))
-
-                        if easyblock_mod_name not in easyblocks:
-                            __import__(easyblock_mod_name)
-                            easyblock_loc = os.path.join(path, fn)
-
-                            class_names = class_regex.findall(read_file(easyblock_loc))
-                            if len(class_names) == 1:
-                                easyblock_class = class_names[0]
-                            elif class_names:
-                                raise EasyBuildError("Found multiple class names for easyblock %s: %s",
-                                                     easyblock_loc, class_names)
-                            else:
-                                raise EasyBuildError("Failed to determine easyblock class name for %s", easyblock_loc)
-
-                            easyblocks[easyblock_mod_name] = {'class': easyblock_class, 'loc': easyblock_loc}
+            for fn in os.listdir(path):
+                res = module_regexp.match(fn)
+                if not res:
+                    continue
+                easyblock_mod_name = res.group(1)
+                easyblock_full_mod_name = '%s.%s' % (pkg, easyblock_mod_name)
+
+                if easyblock_full_mod_name in easyblocks:
+                    _log.debug("%s already imported from %s, ignoring %s",
+                               easyblock_full_mod_name, easyblocks[easyblock_full_mod_name]['loc'], path)
+                else:
+                    __import__(easyblock_full_mod_name)
+                    easyblock_loc = os.path.join(path, fn)
+
+                    class_names = class_regex.findall(read_file(easyblock_loc))
+                    if len(class_names) > 1:
+                        if pkg.endswith('.generic'):
+                            # In generic easyblocks we have e.g. ConfigureMake in configuremake.py
+                            sw_specific_class_names = [name for name in class_names
+                                                       if name.lower() == easyblock_mod_name.lower()]
                         else:
-                            _log.debug("%s already imported from %s, ignoring %s",
-                                       easyblock_mod_name, easyblocks[easyblock_mod_name]['loc'], path)
+                            # If there is exactly one software specific easyblock we use that
+                            sw_specific_class_names = [name for name in class_names
+                                                       if name.startswith(EASYBLOCK_CLASS_PREFIX)]
+                        if len(sw_specific_class_names) == 1:
+                            class_names = sw_specific_class_names
+                    if len(class_names) == 1:
+                        easyblocks[easyblock_full_mod_name] = {'class': class_names[0], 'loc': easyblock_loc}
+                    elif class_names:
+                        raise EasyBuildError("Found multiple class names for easyblock %s: %s",
+                                             easyblock_loc, class_names)
+                    else:
+                        raise EasyBuildError("Failed to determine easyblock class name for %s", easyblock_loc)
 
     return easyblocks
 

easybuild/scripts/mk_tmpl_easyblock_for.py

@@ -36,7 +36,7 @@
 import sys
 from optparse import OptionParser
 
-from easybuild.tools.filetools import encode_class_name
+from easybuild.tools.filetools import encode_class_name, EASYBLOCK_CLASS_PREFIX
 
 # parse options
 parser = OptionParser()
@@ -83,8 +83,9 @@
 # determine parent easyblock class
 parent_import = "from easybuild.framework.easyblock import EasyBlock"
 if not options.parent == "EasyBlock":
-    if options.parent.startswith('EB_'):
-        ebmod = options.parent[3:].lower()  # FIXME: here we should actually decode the encoded class name
+    if options.parent.startswith(EASYBLOCK_CLASS_PREFIX):
+        # FIXME: here we should actually decode the encoded class name
+        ebmod = options.parent[len(EASYBLOCK_CLASS_PREFIX):].lower()
     else:
         ebmod = "generic.%s" % options.parent.lower()
     parent_import = "from easybuild.easyblocks.%s import %s" % (ebmod, options.parent)

easybuild/tools/docs.py

@@ -1313,15 +1313,19 @@ def get_easyblock_classes(package_name):
     """
     Get list of all easyblock classes in specified easyblocks.* package
     """
-    easyblocks = []
+    easyblocks = set()
     modules = import_available_modules(package_name)
 
     for mod in modules:
+        easyblock_found = False
         for name, _ in inspect.getmembers(mod, inspect.isclass):
             eb_class = getattr(mod, name)
             # skip imported classes that are not easyblocks
-            if eb_class.__module__.startswith(package_name) and eb_class not in easyblocks:
-                easyblocks.append(eb_class)
+            if eb_class.__module__.startswith(package_name) and EasyBlock in inspect.getmro(eb_class):
+                easyblocks.add(eb_class)
+                easyblock_found = True
+        if not easyblock_found:
+            raise RuntimeError("No easyblocks found in module: %s", mod.__name__)
 
     return easyblocks
 

easybuild/tools/include.py

@@ -37,7 +37,7 @@
 
 from easybuild.base import fancylogger
 from easybuild.tools.build_log import EasyBuildError
-from easybuild.tools.filetools import expand_glob_paths, read_file, symlink
+from easybuild.tools.filetools import expand_glob_paths, read_file, symlink, EASYBLOCK_CLASS_PREFIX
 # these are imported just to we can reload them later
 import easybuild.tools.module_naming_scheme
 import easybuild.toolchains
@@ -157,7 +157,8 @@ def verify_imports(pymods, pypkg, from_path):
 
 def is_software_specific_easyblock(module):
     """Determine whether Python module at specified location is a software-specific easyblock."""
-    return bool(re.search(r'^class EB_.*\(.*\):\s*$', read_file(module), re.M))
+    # All software-specific easyblocks start with the prefix and derive from another class, at least EasyBlock
+    return bool(re.search(r"^class %s[^(:]+\([^)]+\):\s*$" % EASYBLOCK_CLASS_PREFIX, read_file(module), re.M))
 
 
 def include_easyblocks(tmpdir, paths):

test/framework/docs.py

@@ -25,7 +25,6 @@
 """
 Unit tests for docs.py.
 """
-import inspect
 import os
 import re
 import sys
@@ -38,7 +37,7 @@
 from easybuild.tools.docs import list_easyblocks, list_software, list_toolchains
 from easybuild.tools.docs import md_title_and_table, rst_title_and_table
 from easybuild.tools.options import EasyBuildOptions
-from easybuild.tools.utilities import import_available_modules, mk_md_table, mk_rst_table
+from easybuild.tools.utilities import mk_md_table, mk_rst_table
 from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config
 
 
@@ -520,7 +519,7 @@ def test_get_easyblock_classes(self):
     def test_gen_easyblocks_overview(self):
         """ Test gen_easyblocks_overview_* functions """
         gen_easyblocks_pkg = 'easybuild.easyblocks.generic'
-        modules = import_available_modules(gen_easyblocks_pkg)
+        names = [eb_class.__name__ for eb_class in get_easyblock_classes(gen_easyblocks_pkg)]
         common_params = {
             'ConfigureMake': ['configopts', 'buildopts', 'installopts'],
         }
@@ -564,15 +563,9 @@ def test_gen_easyblocks_overview(self):
         ])
 
         self.assertIn(check_configuremake, ebdoc)
-        names = []
 
-        for mod in modules:
-            for name, _ in inspect.getmembers(mod, inspect.isclass):
-                eb_class = getattr(mod, name)
-                # skip imported classes that are not easyblocks
-                if eb_class.__module__.startswith(gen_easyblocks_pkg):
-                    self.assertIn(name, ebdoc)
-                    names.append(name)
+        for name in names:
+            self.assertIn(name, ebdoc)
 
         toc = [":ref:`" + n + "`" for n in sorted(set(names))]
         pattern = " - ".join(toc)
@@ -610,17 +603,11 @@ def test_gen_easyblocks_overview(self):
         ])
 
         self.assertIn(check_configuremake, ebdoc)
-        names = []
 
-        for mod in modules:
-            for name, _ in inspect.getmembers(mod, inspect.isclass):
-                eb_class = getattr(mod, name)
-                # skip imported classes that are not easyblocks
-                if eb_class.__module__.startswith(gen_easyblocks_pkg):
-                    self.assertIn(name, ebdoc)
-                    names.append(name)
+        for name in names:
+            self.assertIn(name, ebdoc)
 
-        toc = ["\\[" + n + "\\]\\(#" + n.lower() + "\\)" for n in sorted(set(names))]
+        toc = ["\\[" + n + "\\]\\(#" + n.lower() + "\\)" for n in sorted(names)]
         pattern = " - ".join(toc)
         regex = re.compile(pattern)
         self.assertTrue(re.search(regex, ebdoc), "Pattern %s found in %s" % (regex.pattern, ebdoc))

test/framework/sandbox/easybuild/easyblocks/f/foofoo.py

@@ -32,6 +32,14 @@
 from easybuild.framework.easyconfig import CUSTOM, MANDATORY
 
 
+class dummy1:
+    """Only to verify that unrelated classes in software specific easyblocks are ignored"""
+
+
+class dummy2(dummy1):
+    """Same but with inheritance"""
+
+
 class EB_foofoo(EB_foo):
     """Support for building/installing foofoo."""
 

test/framework/sandbox/easybuild/easyblocks/generic/bar.py

@@ -32,6 +32,18 @@
 from easybuild.framework.easyconfig import CUSTOM, MANDATORY
 
 
+class dummy1:
+    """Only to verify that unrelated classes in software specific easyblocks are ignored"""
+
+
+class dummy2(dummy1):
+    """Same but with inheritance"""
+
+
+class dummy3:
+    """Class without inheritance before the real easyblock to verify the regex not being too greedy"""
+
+
 class bar(EasyBlock):
     """Generic support for building/installing bar."""