CI: Enable flake8-comprehension and fix issues

.github/workflows/linting.yml

@@ -22,7 +22,7 @@ jobs:
     - name: install Python packages
       run: |
         pip install --upgrade pip
-        pip install --upgrade flake8
+        pip install --upgrade flake8 flake8-comprehensions
 
     - name: Run flake8 to verify PEP8-compliance of Python code
       run: flake8

easybuild/easyblocks/c/clang.py

@@ -262,7 +262,7 @@ def find_source_dir(globpatterns, targetdir):
                                      glob_src_dirs)
             src_dirs[glob_src_dirs[0]] = targetdir
 
-        if any([x['name'].startswith('llvm-project') for x in self.src]):
+        if any(x['name'].startswith('llvm-project') for x in self.src):
             # if sources contain 'llvm-project*', we use the full tarball
             find_source_dir("../llvm-project-*", os.path.join(self.llvm_src_dir, "llvm-project-%s" % self.version))
             self.cfg.update('configopts', '-DLLVM_ENABLE_PROJECTS="%s"' % ';'.join(self.cfg['llvm_projects']))

easybuild/easyblocks/c/cplex.py

@@ -142,8 +142,8 @@ def make_module_extra(self):
             bins = []
             libs = []
 
-        txt += self.module_generator.prepend_paths('PATH', [path for path in bins])
-        txt += self.module_generator.prepend_paths('LD_LIBRARY_PATH', [path for path in bins + libs])
+        txt += self.module_generator.prepend_paths('PATH', bins)
+        txt += self.module_generator.prepend_paths('LD_LIBRARY_PATH', bins + libs)
 
         txt += self.module_generator.set_environment('CPLEX_HOME', os.path.join(self.installdir, 'cplex'))
         txt += self.module_generator.set_environment('CPLEXDIR', os.path.join(self.installdir, 'cplex'))

easybuild/easyblocks/e/easybuildmeta.py

@@ -230,7 +230,7 @@ def sanity_check_step(self):
         # order matters, e.g. setuptools before distutils
         eb_dirs = OrderedDict()
         eb_dirs['setuptools'] = []
-        eb_dirs['distutils.core'] = flatten([x for x in subdirs_by_pkg.values()])
+        eb_dirs['distutils.core'] = flatten(subdirs_by_pkg.values())
 
         # determine setup tool (setuptools or distutils)
         setup_tool = None

easybuild/easyblocks/f/flexiblas.py

@@ -69,7 +69,7 @@ def __init__(self, *args, **kwargs):
         """Easyblock constructor."""
         super().__init__(*args, **kwargs)
 
-        dep_names = [dep['name'] for dep in self.cfg.dependencies()]
+        dep_names = self.cfg.dependency_names()
         if self.cfg['backends']:
             self.blas_libs = self.cfg['backends'][:]
             # make sure that all listed backends except imkl are (build)dependencies
@@ -80,7 +80,7 @@ def __init__(self, *args, **kwargs):
                 raise EasyBuildError("One or more backends not listed as (build)dependencies: %s",
                                      ', '.join(backends_nodep))
         else:
-            build_dep_names = set(dep['name'] for dep in self.cfg.dependencies(build_only=True))
+            build_dep_names = self.cfg.dependency_names(build_only=True)
             self.blas_libs = [x for x in dep_names if x not in build_dep_names]
 
         self.obj_builddir = os.path.join(self.builddir, 'easybuild_obj')

easybuild/easyblocks/g/gamess_us.py

@@ -470,8 +470,8 @@ def test_step(self):
 
             # verify output of tests
             failed_regex = re.compile(r"^.*!!FAILED\.$", re.M)
-            failed_tests = set([exam[0:6] for exam in failed_regex.findall(res.output)])
-            done_tests = set([exam[0] for exam in target_tests])
+            failed_tests = {exam[0:6] for exam in failed_regex.findall(res.output)}
+            done_tests = {exam[0] for exam in target_tests}
             if done_tests - failed_tests == done_tests:
                 info_msg = "All target tests ran successfully!"
                 if self.cfg['ddi_comm'] == 'mpi':

easybuild/easyblocks/g/gcc.py

@@ -662,7 +662,7 @@ def configure_step(self):
                 "libc6-dev-i386",  # Debian-based
                 "gcc-c++-32bit",  # OpenSuSE, SLES
             ]
-            if not any([check_os_dependency(dep) for dep in glibc_32bit]):
+            if not any(check_os_dependency(dep) for dep in glibc_32bit):
                 raise EasyBuildError("Using multilib requires 32-bit glibc (install one of %s, depending on your OS)",
                                      ', '.join(glibc_32bit))
             self.configopts += " --enable-multilib --with-multilib-list=m32,m64"

easybuild/easyblocks/g/gromacs.py

@@ -383,7 +383,7 @@ def configure_step(self):
 
             # set regression test path
             prefix = 'regressiontests'
-            if any([src['name'].startswith(prefix) for src in self.src]):
+            if any(src['name'].startswith(prefix) for src in self.src):
                 self.cfg.update('configopts', "-DREGRESSIONTEST_PATH='%%(builddir)s/%s-%%(version)s' " % prefix)
 
             # enable OpenMP support if desired

easybuild/easyblocks/generic/cmakemake.py

@@ -326,7 +326,7 @@ def configure_step(self, srcdir=None, builddir=None):
 
         # If the cache does not exist CMake reads the environment variables
         cache_exists = os.path.exists('CMakeCache.txt')
-        env_to_options = dict()
+        env_to_options = {}
 
         # Setting compilers is not required unless we want absolute paths
         if self.cfg.get('abs_path_compilers', False) or cache_exists:

easybuild/easyblocks/generic/pythonbundle.py

@@ -116,7 +116,7 @@ def make_module_extra(self, *args, **kwargs):
         # update $EBPYTHONPREFIXES rather than $PYTHONPATH
         # if this Python package was installed for multiple Python versions, or if we prefer it
         use_ebpythonprefixes = False
-        runtime_deps = [dep['name'] for dep in self.cfg.dependencies(runtime_only=True)]
+        runtime_deps = self.cfg.dependency_names(runtime_only=True)
 
         if 'Python' in runtime_deps:
             self.log.info("Found Python runtime dependency, so considering $EBPYTHONPREFIXES...")

easybuild/easyblocks/generic/pythonpackage.py

@@ -628,7 +628,7 @@ def should_use_ebpythonprefixes(self) -> bool:
         """
 
         use_ebpythonprefixes = False
-        runtime_deps = [dep['name'] for dep in self.cfg.dependencies(runtime_only=True)]
+        runtime_deps = self.cfg.dependency_names(runtime_only=True)
 
         if 'Python' in runtime_deps:
             self.log.info("Found Python runtime dependency, so considering $EBPYTHONPREFIXES...")

easybuild/easyblocks/generic/systemmpi.py

@@ -139,7 +139,7 @@ def prepare_step(self, *args, **kwargs):
 
             # Extract any OpenMPI environment variables in the current environment and ensure they are added to the
             # final module
-            self.mpi_env_vars = dict((key, value) for key, value in os.environ.items() if key.startswith('OMPI_'))
+            self.mpi_env_vars = {key: value for key, value in os.environ.items() if key.startswith('OMPI_')}
 
             # Extract the C compiler used underneath the MPI implementation, check for the definition of OMPI_MPICC
             self.mpi_c_compiler = self.extract_ompi_setting("C compiler", output_of_ompi_info)

easybuild/easyblocks/h/hypre.py

@@ -89,7 +89,7 @@ def sanity_check_step(self):
         """Custom sanity check for Hypre."""
 
         # Add static and shared libs depending on configopts
-        hypre_libs = list()
+        hypre_libs = []
         if self.config_shared:
             shlib_ext = get_shared_lib_ext()
             hypre_libs.append(os.path.join('lib', 'libHYPRE.%s' % shlib_ext))

easybuild/easyblocks/o/openmpi.py

@@ -80,11 +80,11 @@ def config_opt_used(key, enable_opt=False):
         # No entry is interpreted as no option added at all
         # This is to make builds reproducible even when the system libraries are changed and avoids failures
         # due to e.g. finding only PMIx but not libevent on the system
-        unused_dep_value = dict()
+        unused_dep_value = {}
         # Known options since version 3.0 (no earlier ones checked)
         if LooseVersion(self.version) >= LooseVersion('3.0'):
             # Default to disable the option with "no"
-            unused_dep_value = {dep: 'no' for dep in known_dependencies}
+            unused_dep_value = dict.fromkeys(known_dependencies, 'no')
             # For these the default is to use an internal copy and not using any is not supported
             for dep in ('hwloc', 'libevent', 'PMIx'):
                 unused_dep_value[dep] = 'internal'
@@ -225,7 +225,7 @@ def sanity_check_step(self):
 
         # Add minimal test program to sanity checks
         # Run with correct MPI launcher
-        mpi_cmd_tmpl, params = get_mpi_cmd_template(toolchain.OPENMPI, dict(), mpi_version=self.version)
+        mpi_cmd_tmpl, params = get_mpi_cmd_template(toolchain.OPENMPI, {}, mpi_version=self.version)
         # Limit number of ranks to 8 to avoid it failing due to hyperthreading
         ranks = min(8, self.cfg.parallel)
         for srcdir, src, compiler in (

easybuild/easyblocks/o/orca.py

@@ -98,7 +98,7 @@ def install_step(self):
 
             # Shared builds have additional libraries
             libs_to_copy = (['liborca*'], 'lib')
-            if all([glob.glob(p) for p in libs_to_copy[0]]):
+            if all(glob.glob(p) for p in libs_to_copy[0]):
                 files_to_copy.append(libs_to_copy)
 
             self.cfg['files_to_copy'] = files_to_copy

easybuild/easyblocks/p/petsc.py

@@ -147,7 +147,7 @@ def prepare_step(self, *args, **kwargs):
         super().prepare_step(*args, **kwargs)
 
         # build with Python support if Python is loaded as a non-build (runtime) dependency
-        runtime_dep_names = [dep['name'] for dep in self.cfg.dependencies(runtime_only=True)]
+        runtime_dep_names = self.cfg.dependency_names(runtime_only=True)
         if get_software_root('Python') and 'Python' in runtime_dep_names:
             self.with_python = True
             self.module_load_environment.PYTHONPATH = self.bin_dir
@@ -256,7 +256,7 @@ def configure_step(self):
         sep_deps = ['BLACS', 'BLAS', 'CMake', 'FFTW', 'LAPACK', 'numpy', 'mpi4py',
                     'papi', 'ScaLAPACK', 'SciPy-bundle', 'SCOTCH', 'SuiteSparse']
 
-        for dep in [dep['name'] for dep in self.cfg.dependencies(runtime_only=True) if dep['name'] not in sep_deps]:
+        for dep in (name for name in self.cfg.dependencies(runtime_only=True) if name not in sep_deps):
             if isinstance(dep, str):
                 dep = (dep, dep)
             deproot = get_software_root(dep[0])

easybuild/easyblocks/p/pytorch.py

@@ -448,7 +448,7 @@ def add_enable_option(name, enabled):
             raise EasyBuildError("Did not find a supported BLAS in dependencies. Don't know which BLAS lib to use")
 
         available_dependency_options = EB_PyTorch.get_dependency_options_for_version(self.version)
-        dependency_names = {dep['name'] for dep in self.cfg.dependencies()}
+        dependency_names = self.cfg.dependency_names()
         not_used_dep_names = []
         for enable_opt, dep_name in available_dependency_options:
             if dep_name is None:

easybuild/easyblocks/r/rust.py

@@ -123,7 +123,7 @@ def configure_step(self):
         # don't use Ninja if it is not listed as a build dependency;
         # may be because Ninja requires Python, and Rust is a build dependency for cryptography
         # which may be included as an extension with Python
-        build_dep_names = set(dep['name'] for dep in self.cfg.dependencies(build_only=True))
+        build_dep_names = self.cfg.dependency_names(build_only=True)
         if 'Ninja' not in build_dep_names:
             self.cfg.update('configopts', "--set=llvm.ninja=false")
 

easybuild/easyblocks/s/slepc.py

@@ -104,7 +104,7 @@ def configure_step(self):
 
         # optional dependencies
         dep_filter = ['PETSc', 'Python']
-        deps = [dep['name'] for dep in self.cfg.dependencies(runtime_only=True) if dep['name'] not in dep_filter]
+        deps = [name for name in self.cfg.dependency_names(runtime_only=True) if name not in dep_filter]
         for dep in deps:
             deproot = get_software_root(dep)
             if deproot:

easybuild/easyblocks/t/tensorflow.py

@@ -365,7 +365,7 @@ def get_system_libs(self):
         ignored_system_deps = []
 
         # Check direct dependencies
-        dep_names = {dep['name'] for dep in self.cfg.dependencies()}
+        dep_names = self.cfg.dependency_names()
         for dep_name, tf_name in sorted(dependency_mapping.items(), key=lambda i: i[0].lower()):
             if dep_name in dep_names:
                 if tf_name in deps_with_python_pkg:

easybuild/easyblocks/t/tinker.py

@@ -131,7 +131,7 @@ def test_step(self):
                 # salt and dialinine takes too long
                 skip_tests.extend(['salt', 'dialanine'])
 
-            tests = [t for t in tests if not any([t.endswith('%s.run' % x) for x in skip_tests])]
+            tests = [t for t in tests if not any(t.endswith(f'{x}.run') for x in skip_tests)]
 
             for test in tests:
                 run_shell_cmd(test)

easybuild/easyblocks/t/torchvision.py

@@ -44,7 +44,7 @@ def __init__(self, *args, **kwargs):
         """Initialize torchvision easyblock."""
         super().__init__(*args, **kwargs)
 
-        dep_names = set(dep['name'] for dep in self.cfg.dependencies())
+        dep_names = self.cfg.dependency_names()
 
         # require that PyTorch is listed as dependency
         if 'PyTorch' not in dep_names:

easybuild/easyblocks/u/ucx_plugins.py

@@ -117,7 +117,7 @@ def make_module_extra(self, *args, **kwargs):
         """Add extra statements to generated module file specific to UCX plugins"""
         txt = super().make_module_extra(*args, **kwargs)
 
-        base_conf = dict()
+        base_conf = {}
         cmd = ['ucx_info', '-b']
         full_cmd = ' '.join(cmd)
         self.log.info("Running command '%s'" % full_cmd)

setup.cfg

@@ -5,14 +5,10 @@ requires = easybuild-config
 
 [flake8]
 max-line-length = 120
+# C4: Comprehensions
+extend-select = C4
 
-# Hound CI runs with Python 3 (no way around it),
-# so we need to specify some Python 2 builtins to avoid that it complains about them
-# cfr. https://stackoverflow.com/questions/47427916/how-to-config-hound-ci-to-support-python2-7
-builtins =
-    basestring,
-    reduce
-
-# ignore "Black would make changes" produced by flake8-black
+# BLK100: "Black would make changes"
 # see also https://github.com/houndci/hound/issues/1769
-extend-ignore = BLK100
+# G002-G004,G200: Logging statement uses '%', '+', f-string, exception
+extend-ignore = BLK100,G002,G003,G004,G200

test/easyblocks/easyblock_specific.py

@@ -399,7 +399,7 @@ def mocked_run_shell_cmd_pip(cmd, **kwargs):
             python.run_pip_check(python_cmd=sys.executable, unversioned_packages=('zero', ))
 
         with self.mocked_stdout_stderr():
-            python.run_pip_check(python_cmd=sys.executable, unversioned_packages=set(['zero']))
+            python.run_pip_check(python_cmd=sys.executable, unversioned_packages={'zero'})
 
         # inject all possible errors
         def mocked_run_shell_cmd_pip(cmd, **kwargs):