Merge pull request #3146 from boegel/fix_list_software_detailed_py3

fix --list-software=detailed when using Python 3 by leveraging sort_looseversions function from py2vs3 module

Author: migueldiascosta

easybuild/tools/docs.py

@@ -57,7 +57,7 @@
 from easybuild.tools.config import build_option
 from easybuild.tools.filetools import read_file
 from easybuild.tools.modules import modules_tool
-from easybuild.tools.py2vs3 import OrderedDict, ascii_lowercase
+from easybuild.tools.py2vs3 import OrderedDict, ascii_lowercase, sort_looseversions
 from easybuild.tools.toolchain.toolchain import DUMMY_TOOLCHAIN_NAME, SYSTEM_TOOLCHAIN_NAME, is_system_toolchain
 from easybuild.tools.toolchain.utilities import search_toolchain
 from easybuild.tools.utilities import INDENT_2SPACES, INDENT_4SPACES
@@ -635,14 +635,22 @@ def key_to_ref(name):
             table_titles = ['version', 'toolchain']
             table_values = [[], []]
 
+            # first determine unique pairs of version/versionsuffix
+            # we can't use LooseVersion yet here, since nub uses set and LooseVersion instances are not hashable
             pairs = nub((x['version'], x['versionsuffix']) for x in software[key])
 
+            # check whether any non-empty versionsuffixes are in play
             with_vsuff = any(vs for (_, vs) in pairs)
             if with_vsuff:
                 table_titles.insert(1, 'versionsuffix')
                 table_values.insert(1, [])
 
-            for ver, vsuff in sorted((LooseVersion(v), vs) for (v, vs) in pairs):
+            # sort pairs by version (and then by versionsuffix);
+            # we sort by LooseVersion to obtain chronological version ordering,
+            # but we also need to retain original string version for filtering-by-version done below
+            sorted_pairs = sort_looseversions((LooseVersion(v), vs, v) for v, vs in pairs)
+
+            for _, vsuff, ver in sorted_pairs:
                 table_values[0].append('``%s``' % ver)
                 if with_vsuff:
                     if vsuff:
@@ -690,8 +698,17 @@ def list_software_txt(software, detailed=False):
                 "homepage: %s" % software[key][-1]['homepage'],
                 '',
             ])
+
+            # first determine unique pairs of version/versionsuffix
+            # we can't use LooseVersion yet here, since nub uses set and LooseVersion instances are not hashable
             pairs = nub((x['version'], x['versionsuffix']) for x in software[key])
-            for ver, vsuff in sorted((LooseVersion(v), vs) for (v, vs) in pairs):
+
+            # sort pairs by version (and then by versionsuffix);
+            # we sort by LooseVersion to obtain chronological version ordering,
+            # but we also need to retain original string version for filtering-by-version done below
+            sorted_pairs = sort_looseversions((LooseVersion(v), vs, v) for v, vs in pairs)
+
+            for _, vsuff, ver in sorted_pairs:
                 tcs = [x['toolchain'] for x in software[key] if x['version'] == ver and x['versionsuffix'] == vsuff]
 
                 line = "  * %s v%s" % (key, ver)

easybuild/tools/py2vs3/py2.py

@@ -82,3 +82,10 @@ class WrapperBase(object):
         __wraps__ = None
 
     return WrapperBase
+
+
+def sort_looseversions(looseversions):
+    """Sort list of (values including) LooseVersion instances."""
+    # with Python 2, we can safely use 'sorted' on LooseVersion instances
+    # (but we can't in Python 3, see https://bugs.python.org/issue14894)
+    return sorted(looseversions)

easybuild/tools/py2vs3/py3.py

@@ -36,6 +36,9 @@
 import sys
 import urllib.request as std_urllib  # noqa
 from collections import OrderedDict  # noqa
+from distutils.version import LooseVersion
+from functools import cmp_to_key
+from itertools import zip_longest
 from io import StringIO  # noqa
 from string import ascii_letters, ascii_lowercase  # noqa
 from urllib.request import HTTPError, HTTPSHandler, Request, URLError, build_opener, urlopen  # noqa
@@ -86,3 +89,59 @@ class WrapperBase(object, metaclass=metaclass):
         __wraps__ = None
 
     return WrapperBase
+
+
+def safe_cmp_looseversions(v1, v2):
+    """Safe comparison function for two (values containing) LooseVersion instances."""
+
+    if type(v1) != type(v2):
+        raise TypeError("Can't compare values of different types: %s (%s) vs %s (%s)" % (v1, type(v1), v2, type(v2)))
+
+    # if we receive two iterative values, we need to recurse
+    if isinstance(v1, (list, tuple)) and isinstance(v2, (list, tuple)):
+        if len(v1) == len(v2):
+            for x1, x2 in zip(v1, v2):
+                res = safe_cmp_looseversions(x1, x2)
+                # if a difference was found, we know the result;
+                # if not, we need comparison on next item (if any), done in next iteration
+                if res != 0:
+                    return res
+            return 0  # no difference
+        else:
+            raise ValueError("Can only compare iterative values of same length: %s vs %s" % (v1, v2))
+
+    def simple_compare(x1, x2):
+        """Helper function for simple comparison using standard operators ==, <, > """
+        if x1 < x2:
+            return -1
+        elif x1 > x2:
+            return 1
+        else:
+            return 0
+
+    if isinstance(v1, LooseVersion) and isinstance(v2, LooseVersion):
+        # implementation based on '14894.patch' patch file provided in https://bugs.python.org/issue14894
+        for ver1_part, ver2_part in zip_longest(v1.version, v2.version, fillvalue=''):
+            # use string comparison if version parts have different type
+            if type(ver1_part) != type(ver2_part):
+                ver1_part = str(ver1_part)
+                ver2_part = str(ver2_part)
+
+            res = simple_compare(ver1_part, ver2_part)
+            if res == 0:
+                continue
+            else:
+                return res
+
+        # identical versions
+        return 0
+    else:
+        # for non-LooseVersion values, use simple comparison
+        return simple_compare(v1, v2)
+
+
+def sort_looseversions(looseversions):
+    """Sort list of (values including) LooseVersion instances."""
+    # with Python 2, we can safely use 'sorted' on LooseVersion instances
+    # (but we can't in Python 3, see https://bugs.python.org/issue14894)
+    return sorted(looseversions, key=cmp_to_key(safe_cmp_looseversions))

test/framework/options.py

@@ -33,6 +33,7 @@
 import shutil
 import sys
 import tempfile
+from distutils.version import LooseVersion
 from unittest import TextTestRunner
 
 import easybuild.main
@@ -55,7 +56,7 @@
 from easybuild.tools.github import URL_SEPARATOR, fetch_github_token
 from easybuild.tools.modules import Lmod
 from easybuild.tools.options import EasyBuildOptions, parse_external_modules_metadata, set_tmpdir, use_color
-from easybuild.tools.py2vs3 import URLError, reload
+from easybuild.tools.py2vs3 import URLError, reload, sort_looseversions
 from easybuild.tools.toolchain.utilities import TC_CONST_PREFIX
 from easybuild.tools.run import run_cmd
 from easybuild.tools.version import VERSION
@@ -4637,6 +4638,32 @@ def test_installdir(self):
         eb = EasyBlock(EasyConfig(toy_ec))
         self.assertTrue(eb.installdir.endswith('/software/Core/toy/0.0'))
 
+    def test_sort_looseversions(self):
+        """Test sort_looseversions funuction."""
+        ver1 = LooseVersion('1.2.3')
+        ver2 = LooseVersion('4.5.6')
+        ver3 = LooseVersion('1.2.3dev')
+        ver4 = LooseVersion('system')
+        ver5 = LooseVersion('rc3')
+        ver6 = LooseVersion('v1802')
+
+        # some versions are included multiple times on purpose,
+        # to also test comparison between equal LooseVersion instances
+        input = [ver3, ver5, ver1, ver2, ver4, ver6, ver3, ver4, ver1]
+        expected = [ver1, ver1, ver3, ver3, ver2, ver5, ver4, ver4, ver6]
+        self.assertEqual(sort_looseversions(input), expected)
+
+        # also test on list of tuples consisting of a LooseVersion instance + a string
+        # (as in the list_software_* functions)
+        suff1 = ''
+        suff2 = '-foo'
+        suff3 = '-bar'
+        input = [(ver3, suff1), (ver5, suff3), (ver1, suff2), (ver2, suff3), (ver4, suff1),
+                 (ver6, suff2), (ver3, suff3), (ver4, suff3), (ver1, suff1)]
+        expected = [(ver1, suff1), (ver1, suff2), (ver3, suff1), (ver3, suff3), (ver2, suff3),
+                    (ver5, suff3), (ver4, suff1), (ver4, suff3), (ver6, suff2)]
+        self.assertEqual(sort_looseversions(input), expected)
+
 
 def suite():
     """ returns all the testcases in this module """