Skip to content

Commit 644dcd9

Browse files
authored
Merge pull request #3609 from Flamefire/naturalSortSearch
Sort output of eb --search in natural order (respecting numbers)
2 parents d6fd180 + dbf9d28 commit 644dcd9

File tree

8 files changed

+166
-67
lines changed

8 files changed

+166
-67
lines changed

easybuild/tools/filetools.py

Lines changed: 15 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@
6060
from easybuild.tools.build_log import EasyBuildError, dry_run_msg, print_msg, print_warning
6161
from easybuild.tools.config import DEFAULT_WAIT_ON_LOCK_INTERVAL, GENERIC_EASYBLOCK_PKG, build_option, install_path
6262
from easybuild.tools.py2vs3 import HTMLParser, std_urllib, string_type
63-
from easybuild.tools.utilities import nub, remove_unwanted_chars
63+
from easybuild.tools.utilities import natural_keys, nub, remove_unwanted_chars
6464

6565
try:
6666
import requests
@@ -1000,8 +1000,11 @@ def search_file(paths, query, short=False, ignore_dirs=None, silent=False, filen
10001000
if not terse:
10011001
print_msg("Searching (case-insensitive) for '%s' in %s " % (query.pattern, path), log=_log, silent=silent)
10021002

1003-
path_index = load_index(path, ignore_dirs=ignore_dirs)
1004-
if path_index is None or build_option('ignore_index'):
1003+
if build_option('ignore_index'):
1004+
path_index = None
1005+
else:
1006+
path_index = load_index(path, ignore_dirs=ignore_dirs)
1007+
if path_index is None:
10051008
if os.path.exists(path):
10061009
_log.info("No index found for %s, creating one...", path)
10071010
path_index = create_index(path, ignore_dirs=ignore_dirs)
@@ -1021,15 +1024,17 @@ def search_file(paths, query, short=False, ignore_dirs=None, silent=False, filen
10211024
else:
10221025
path_hits.append(os.path.join(path, filepath))
10231026

1024-
path_hits = sorted(path_hits)
1027+
path_hits = sorted(path_hits, key=natural_keys)
10251028

10261029
if path_hits:
1027-
common_prefix = det_common_path_prefix(path_hits)
1028-
if not terse and short and common_prefix is not None and len(common_prefix) > len(var) * 2:
1029-
var_defs.append((var, common_prefix))
1030-
hits.extend([os.path.join('$%s' % var, fn[len(common_prefix) + 1:]) for fn in path_hits])
1031-
else:
1032-
hits.extend(path_hits)
1030+
if not terse and short:
1031+
common_prefix = det_common_path_prefix(path_hits)
1032+
if common_prefix is not None and len(common_prefix) > len(var) * 2:
1033+
var_defs.append((var, common_prefix))
1034+
var_spec = '$' + var
1035+
# Replace the common prefix by var_spec
1036+
path_hits = (var_spec + fn[len(common_prefix):] for fn in path_hits)
1037+
hits.extend(path_hits)
10331038

10341039
return var_defs, hits
10351040

easybuild/tools/utilities.py

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -316,3 +316,10 @@ def time2str(delta):
316316
res = '%d %s %d min %d sec' % (hours, hours_str, mins, secs)
317317

318318
return res
319+
320+
321+
def natural_keys(key):
322+
"""Can be used as the sort key in list.sort(key=natural_keys) to sort in natural order (i.e. respecting numbers)"""
323+
def try_to_int(key_part):
324+
return int(key_part) if key_part.isdigit() else key_part
325+
return [try_to_int(key_part) for key_part in re.split(r'(\d+)', key)]

test/framework/easyblock.py

Lines changed: 0 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,6 @@
3434
import sys
3535
import tempfile
3636
from inspect import cleandoc
37-
from datetime import datetime
3837
from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config
3938
from unittest import TextTestRunner
4039

@@ -50,7 +49,6 @@
5049
from easybuild.tools.filetools import verify_checksum, write_file
5150
from easybuild.tools.module_generator import module_generator
5251
from easybuild.tools.modules import reset_module_caches
53-
from easybuild.tools.utilities import time2str
5452
from easybuild.tools.version import get_git_revision, this_is_easybuild
5553
from easybuild.tools.py2vs3 import string_type
5654

@@ -2108,34 +2106,6 @@ def test_avail_easyblocks(self):
21082106
self.assertEqual(hpl['class'], 'EB_HPL')
21092107
self.assertTrue(hpl['loc'].endswith('sandbox/easybuild/easyblocks/h/hpl.py'))
21102108

2111-
def test_time2str(self):
2112-
"""Test time2str function."""
2113-
2114-
start = datetime(2019, 7, 30, 5, 14, 23)
2115-
2116-
test_cases = [
2117-
(start, "0 sec"),
2118-
(datetime(2019, 7, 30, 5, 14, 37), "14 sec"),
2119-
(datetime(2019, 7, 30, 5, 15, 22), "59 sec"),
2120-
(datetime(2019, 7, 30, 5, 15, 23), "1 min 0 sec"),
2121-
(datetime(2019, 7, 30, 5, 16, 22), "1 min 59 sec"),
2122-
(datetime(2019, 7, 30, 5, 37, 26), "23 min 3 sec"),
2123-
(datetime(2019, 7, 30, 6, 14, 22), "59 min 59 sec"),
2124-
(datetime(2019, 7, 30, 6, 14, 23), "1 hour 0 min 0 sec"),
2125-
(datetime(2019, 7, 30, 6, 49, 14), "1 hour 34 min 51 sec"),
2126-
(datetime(2019, 7, 30, 7, 14, 23), "2 hours 0 min 0 sec"),
2127-
(datetime(2019, 7, 30, 8, 35, 59), "3 hours 21 min 36 sec"),
2128-
(datetime(2019, 7, 30, 16, 29, 24), "11 hours 15 min 1 sec"),
2129-
(datetime(2019, 7, 31, 5, 14, 22), "23 hours 59 min 59 sec"),
2130-
(datetime(2019, 7, 31, 5, 14, 23), "24 hours 0 min 0 sec"),
2131-
(datetime(2019, 8, 5, 20, 39, 44), "159 hours 25 min 21 sec"),
2132-
]
2133-
for end, expected in test_cases:
2134-
self.assertEqual(time2str(end - start), expected)
2135-
2136-
error_pattern = "Incorrect value type provided to time2str, should be datetime.timedelta: <.* 'int'>"
2137-
self.assertErrorRegex(EasyBuildError, error_pattern, time2str, 123)
2138-
21392109
def test_sanity_check_paths_verification(self):
21402110
"""Test verification of sanity_check_paths w.r.t. keys & values."""
21412111

test/framework/filetools.py

Lines changed: 25 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -2158,11 +2158,11 @@ def test_search_file(self):
21582158
self.assertEqual(var_defs, [])
21592159
self.assertEqual(len(hits), 5)
21602160
self.assertTrue(all(os.path.exists(p) for p in hits))
2161-
self.assertTrue(hits[0].endswith('/hwloc-1.11.8-GCC-4.6.4.eb'))
2162-
self.assertTrue(hits[1].endswith('/hwloc-1.11.8-GCC-6.4.0-2.28.eb'))
2163-
self.assertTrue(hits[2].endswith('/hwloc-1.11.8-GCC-7.3.0-2.30.eb'))
2164-
self.assertTrue(hits[3].endswith('/hwloc-1.6.2-GCC-4.9.3-2.26.eb'))
2165-
self.assertTrue(hits[4].endswith('/hwloc-1.8-gcccuda-2018a.eb'))
2161+
self.assertTrue(hits[0].endswith('/hwloc-1.6.2-GCC-4.9.3-2.26.eb'))
2162+
self.assertTrue(hits[1].endswith('/hwloc-1.8-gcccuda-2018a.eb'))
2163+
self.assertTrue(hits[2].endswith('/hwloc-1.11.8-GCC-4.6.4.eb'))
2164+
self.assertTrue(hits[3].endswith('/hwloc-1.11.8-GCC-6.4.0-2.28.eb'))
2165+
self.assertTrue(hits[4].endswith('/hwloc-1.11.8-GCC-7.3.0-2.30.eb'))
21662166

21672167
# also test case-sensitive searching
21682168
var_defs, hits_bis = ft.search_file([test_ecs], 'HWLOC', silent=True, case_sensitive=True)
@@ -2176,9 +2176,12 @@ def test_search_file(self):
21762176
# check filename-only mode
21772177
var_defs, hits = ft.search_file([test_ecs], 'HWLOC', silent=True, filename_only=True)
21782178
self.assertEqual(var_defs, [])
2179-
self.assertEqual(hits, ['hwloc-1.11.8-GCC-4.6.4.eb', 'hwloc-1.11.8-GCC-6.4.0-2.28.eb',
2180-
'hwloc-1.11.8-GCC-7.3.0-2.30.eb', 'hwloc-1.6.2-GCC-4.9.3-2.26.eb',
2181-
'hwloc-1.8-gcccuda-2018a.eb'])
2179+
self.assertEqual(hits, ['hwloc-1.6.2-GCC-4.9.3-2.26.eb',
2180+
'hwloc-1.8-gcccuda-2018a.eb',
2181+
'hwloc-1.11.8-GCC-4.6.4.eb',
2182+
'hwloc-1.11.8-GCC-6.4.0-2.28.eb',
2183+
'hwloc-1.11.8-GCC-7.3.0-2.30.eb',
2184+
])
21822185

21832186
# check specifying of ignored dirs
21842187
var_defs, hits = ft.search_file([test_ecs], 'HWLOC', silent=True, ignore_dirs=['hwloc'])
@@ -2187,28 +2190,34 @@ def test_search_file(self):
21872190
# check short mode
21882191
var_defs, hits = ft.search_file([test_ecs], 'HWLOC', silent=True, short=True)
21892192
self.assertEqual(var_defs, [('CFGS1', os.path.join(test_ecs, 'h', 'hwloc'))])
2190-
self.assertEqual(hits, ['$CFGS1/hwloc-1.11.8-GCC-4.6.4.eb', '$CFGS1/hwloc-1.11.8-GCC-6.4.0-2.28.eb',
2191-
'$CFGS1/hwloc-1.11.8-GCC-7.3.0-2.30.eb', '$CFGS1/hwloc-1.6.2-GCC-4.9.3-2.26.eb',
2192-
'$CFGS1/hwloc-1.8-gcccuda-2018a.eb'])
2193+
self.assertEqual(hits, ['$CFGS1/hwloc-1.6.2-GCC-4.9.3-2.26.eb',
2194+
'$CFGS1/hwloc-1.8-gcccuda-2018a.eb',
2195+
'$CFGS1/hwloc-1.11.8-GCC-4.6.4.eb',
2196+
'$CFGS1/hwloc-1.11.8-GCC-6.4.0-2.28.eb',
2197+
'$CFGS1/hwloc-1.11.8-GCC-7.3.0-2.30.eb'
2198+
])
21932199

21942200
# check terse mode (implies 'silent', overrides 'short')
21952201
var_defs, hits = ft.search_file([test_ecs], 'HWLOC', terse=True, short=True)
21962202
self.assertEqual(var_defs, [])
21972203
expected = [
2204+
os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.6.2-GCC-4.9.3-2.26.eb'),
2205+
os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.8-gcccuda-2018a.eb'),
21982206
os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.11.8-GCC-4.6.4.eb'),
21992207
os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.11.8-GCC-6.4.0-2.28.eb'),
22002208
os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.11.8-GCC-7.3.0-2.30.eb'),
2201-
os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.6.2-GCC-4.9.3-2.26.eb'),
2202-
os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.8-gcccuda-2018a.eb'),
22032209
]
22042210
self.assertEqual(hits, expected)
22052211

22062212
# check combo of terse and filename-only
22072213
var_defs, hits = ft.search_file([test_ecs], 'HWLOC', terse=True, filename_only=True)
22082214
self.assertEqual(var_defs, [])
2209-
self.assertEqual(hits, ['hwloc-1.11.8-GCC-4.6.4.eb', 'hwloc-1.11.8-GCC-6.4.0-2.28.eb',
2210-
'hwloc-1.11.8-GCC-7.3.0-2.30.eb', 'hwloc-1.6.2-GCC-4.9.3-2.26.eb',
2211-
'hwloc-1.8-gcccuda-2018a.eb'])
2215+
self.assertEqual(hits, ['hwloc-1.6.2-GCC-4.9.3-2.26.eb',
2216+
'hwloc-1.8-gcccuda-2018a.eb',
2217+
'hwloc-1.11.8-GCC-4.6.4.eb',
2218+
'hwloc-1.11.8-GCC-6.4.0-2.28.eb',
2219+
'hwloc-1.11.8-GCC-7.3.0-2.30.eb',
2220+
])
22122221

22132222
# patterns that include special characters + (or ++) shouldn't cause trouble
22142223
# cfr. https://github.com/easybuilders/easybuild-framework/issues/2966

test/framework/options.py

Lines changed: 7 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -988,7 +988,7 @@ def test_ignore_index(self):
988988
toy_ec = os.path.join(test_ecs_dir, 'test_ecs', 't', 'toy', 'toy-0.0.eb')
989989
copy_file(toy_ec, self.test_prefix)
990990

991-
toy_ec_list = ['toy-0.0.eb', 'toy-1.2.3.eb', 'toy-4.5.6.eb']
991+
toy_ec_list = ['toy-0.0.eb', 'toy-1.2.3.eb', 'toy-4.5.6.eb', 'toy-11.5.6.eb']
992992

993993
# install index that list more files than are actually available,
994994
# so we can check whether it's used
@@ -998,27 +998,25 @@ def test_ignore_index(self):
998998
args = [
999999
'--search=toy',
10001000
'--robot-paths=%s' % self.test_prefix,
1001+
'--terse',
10011002
]
10021003
self.mock_stdout(True)
10031004
self.eb_main(args, testing=False, raise_error=True)
10041005
stdout = self.get_stdout()
10051006
self.mock_stdout(False)
10061007

1007-
for toy_ec_fn in toy_ec_list:
1008-
regex = re.compile(re.escape(os.path.join(self.test_prefix, toy_ec_fn)), re.M)
1009-
self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout))
1008+
# Also checks for ordering: 11.x comes last!
1009+
expected_output = '\n'.join(os.path.join(self.test_prefix, ec) for ec in toy_ec_list) + '\n'
1010+
self.assertEqual(stdout, expected_output)
10101011

10111012
args.append('--ignore-index')
10121013
self.mock_stdout(True)
10131014
self.eb_main(args, testing=False, raise_error=True)
10141015
stdout = self.get_stdout()
10151016
self.mock_stdout(False)
10161017

1017-
regex = re.compile(re.escape(os.path.join(self.test_prefix, 'toy-0.0.eb')), re.M)
1018-
self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout))
1019-
for toy_ec_fn in ['toy-1.2.3.eb', 'toy-4.5.6.eb']:
1020-
regex = re.compile(re.escape(os.path.join(self.test_prefix, toy_ec_fn)), re.M)
1021-
self.assertFalse(regex.search(stdout), "Pattern '%s' should not be found in: %s" % (regex.pattern, stdout))
1018+
# This should be the only EC found
1019+
self.assertEqual(stdout, os.path.join(self.test_prefix, 'toy-0.0.eb') + '\n')
10221020

10231021
def test_search_archived(self):
10241022
"Test searching for archived easyconfigs"

test/framework/robot.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1513,10 +1513,10 @@ def test_search_easyconfigs(self):
15131513

15141514
paths = search_easyconfigs('8-gcc', consider_extra_paths=False, print_result=False)
15151515
ref_paths = [
1516+
os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.8-gcccuda-2018a.eb'),
15161517
os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.11.8-GCC-4.6.4.eb'),
15171518
os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.11.8-GCC-6.4.0-2.28.eb'),
15181519
os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.11.8-GCC-7.3.0-2.30.eb'),
1519-
os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.8-gcccuda-2018a.eb'),
15201520
os.path.join(test_ecs, 'o', 'OpenBLAS', 'OpenBLAS-0.2.8-GCC-4.8.2-LAPACK-3.4.2.eb')
15211521
]
15221522
self.assertEqual(paths, ref_paths)

test/framework/suite.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,7 @@
7777
import test.framework.toy_build as t
7878
import test.framework.type_checking as et
7979
import test.framework.tweak as tw
80+
import test.framework.utilities_test as u
8081
import test.framework.variables as v
8182
import test.framework.yeb as y
8283

@@ -118,7 +119,7 @@
118119
# call suite() for each module and then run them all
119120
# note: make sure the options unit tests run first, to avoid running some of them with a readily initialized config
120121
tests = [gen, bl, o, r, ef, ev, ebco, ep, e, mg, m, mt, f, run, a, robot, b, v, g, tcv, tc, t, c, s, lic, f_c,
121-
tw, p, i, pkg, d, env, et, y, st, h, ct, lib]
122+
tw, p, i, pkg, d, env, et, y, st, h, ct, lib, u]
122123

123124
SUITE = unittest.TestSuite([x.suite() for x in tests])
124125
res = unittest.TextTestRunner().run(SUITE)

test/framework/utilities_test.py

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
##
2+
# Copyright 2012-2021 Ghent University
3+
#
4+
# This file is part of EasyBuild,
5+
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
6+
# with support of Ghent University (http://ugent.be/hpc),
7+
# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
8+
# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
9+
# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
10+
#
11+
# https://github.com/easybuilders/easybuild
12+
#
13+
# EasyBuild is free software: you can redistribute it and/or modify
14+
# it under the terms of the GNU General Public License as published by
15+
# the Free Software Foundation v2.
16+
#
17+
# EasyBuild is distributed in the hope that it will be useful,
18+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
19+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20+
# GNU General Public License for more details.
21+
#
22+
# You should have received a copy of the GNU General Public License
23+
# along with EasyBuild. If not, see <http://www.gnu.org/licenses/>.
24+
##
25+
"""
26+
Unit tests for utilities.py
27+
28+
@author: Jens Timmerman (Ghent University)
29+
@author: Kenneth Hoste (Ghent University)
30+
@author: Alexander Grund (TU Dresden)
31+
"""
32+
import os
33+
import random
34+
import sys
35+
import tempfile
36+
from datetime import datetime
37+
from unittest import TextTestRunner
38+
39+
from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered
40+
from easybuild.tools.build_log import EasyBuildError
41+
from easybuild.tools.utilities import time2str, natural_keys
42+
43+
44+
class UtilitiesTest(EnhancedTestCase):
45+
"""Class for utilities testcases """
46+
47+
def setUp(self):
48+
""" setup """
49+
super(UtilitiesTest, self).setUp()
50+
51+
self.test_tmp_logdir = tempfile.mkdtemp()
52+
os.environ['EASYBUILD_TMP_LOGDIR'] = self.test_tmp_logdir
53+
54+
def test_time2str(self):
55+
"""Test time2str function."""
56+
57+
start = datetime(2019, 7, 30, 5, 14, 23)
58+
59+
test_cases = [
60+
(start, "0 sec"),
61+
(datetime(2019, 7, 30, 5, 14, 37), "14 sec"),
62+
(datetime(2019, 7, 30, 5, 15, 22), "59 sec"),
63+
(datetime(2019, 7, 30, 5, 15, 23), "1 min 0 sec"),
64+
(datetime(2019, 7, 30, 5, 16, 22), "1 min 59 sec"),
65+
(datetime(2019, 7, 30, 5, 37, 26), "23 min 3 sec"),
66+
(datetime(2019, 7, 30, 6, 14, 22), "59 min 59 sec"),
67+
(datetime(2019, 7, 30, 6, 14, 23), "1 hour 0 min 0 sec"),
68+
(datetime(2019, 7, 30, 6, 49, 14), "1 hour 34 min 51 sec"),
69+
(datetime(2019, 7, 30, 7, 14, 23), "2 hours 0 min 0 sec"),
70+
(datetime(2019, 7, 30, 8, 35, 59), "3 hours 21 min 36 sec"),
71+
(datetime(2019, 7, 30, 16, 29, 24), "11 hours 15 min 1 sec"),
72+
(datetime(2019, 7, 31, 5, 14, 22), "23 hours 59 min 59 sec"),
73+
(datetime(2019, 7, 31, 5, 14, 23), "24 hours 0 min 0 sec"),
74+
(datetime(2019, 8, 5, 20, 39, 44), "159 hours 25 min 21 sec"),
75+
]
76+
for end, expected in test_cases:
77+
self.assertEqual(time2str(end - start), expected)
78+
79+
error_pattern = "Incorrect value type provided to time2str, should be datetime.timedelta: <.* 'int'>"
80+
self.assertErrorRegex(EasyBuildError, error_pattern, time2str, 123)
81+
82+
def test_natural_keys(self):
83+
"""Test the natural_keys function"""
84+
sorted_items = [
85+
'ACoolSw-1.0',
86+
'ACoolSw-2.1',
87+
'ACoolSw-11.0',
88+
'ACoolSw-23.0',
89+
'ACoolSw-30.0',
90+
'ACoolSw-30.1',
91+
'BigNumber-1234567890',
92+
'BigNumber-1234567891',
93+
'NoNumbers',
94+
'VeryLastEntry-10'
95+
]
96+
shuffled_items = sorted_items[:]
97+
random.shuffle(shuffled_items)
98+
shuffled_items.sort(key=natural_keys)
99+
self.assertEqual(shuffled_items, sorted_items)
100+
101+
102+
def suite():
103+
""" return all the tests in this file """
104+
return TestLoaderFiltered().loadTestsFromTestCase(UtilitiesTest, sys.argv[1:])
105+
106+
107+
if __name__ == '__main__':
108+
res = TextTestRunner(verbosity=1).run(suite())
109+
sys.exit(len(res.failures))

0 commit comments

Comments
 (0)