Merge pull request #4526 from branfosj/checksums

deprecate old checksum options (incl. md5)

Author: boegel

easybuild/framework/easyblock.py

@@ -77,7 +77,7 @@
 from easybuild.tools.config import build_option, build_path, get_log_filename, get_repository, get_repositorypath
 from easybuild.tools.config import install_path, log_path, package_path, source_paths
 from easybuild.tools.environment import restore_env, sanitize_env
-from easybuild.tools.filetools import CHECKSUM_TYPE_MD5, CHECKSUM_TYPE_SHA256
+from easybuild.tools.filetools import CHECKSUM_TYPE_SHA256
 from easybuild.tools.filetools import adjust_permissions, apply_patch, back_up_file, change_dir, check_lock
 from easybuild.tools.filetools import compute_checksum, convert_name, copy_file, create_lock, create_patch_info
 from easybuild.tools.filetools import derive_alt_pypi_url, diff_files, dir_contains_files, download_file
@@ -666,8 +666,7 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True):
                         src_path = ext_src['src']
                         src_fn = os.path.basename(src_path)
 
-                        # report both MD5 and SHA256 checksums, since both are valid default checksum types
-                        for checksum_type in (CHECKSUM_TYPE_MD5, CHECKSUM_TYPE_SHA256):
+                        for checksum_type in [CHECKSUM_TYPE_SHA256]:
                             src_checksum = compute_checksum(src_path, checksum_type=checksum_type)
                             self.log.info("%s checksum for %s: %s", checksum_type, src_path, src_checksum)
 
@@ -695,9 +694,7 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True):
                         if verify_checksums:
                             for patch in ext_patches:
                                 patch = patch['path']
-                                # report both MD5 and SHA256 checksums,
-                                # since both are valid default checksum types
-                                for checksum_type in (CHECKSUM_TYPE_MD5, CHECKSUM_TYPE_SHA256):
+                                for checksum_type in [CHECKSUM_TYPE_SHA256]:
                                     checksum = compute_checksum(patch, checksum_type=checksum_type)
                                     self.log.info("%s checksum for %s: %s", checksum_type, patch, checksum)
 
@@ -2410,8 +2407,7 @@ def fetch_step(self, skip_checksums=False):
         # compute checksums for all source and patch files
         if not (skip_checksums or self.dry_run):
             for fil in self.src + self.patches:
-                # report both MD5 and SHA256 checksums, since both are valid default checksum types
-                for checksum_type in [CHECKSUM_TYPE_MD5, CHECKSUM_TYPE_SHA256]:
+                for checksum_type in [CHECKSUM_TYPE_SHA256]:
                     fil[checksum_type] = compute_checksum(fil['path'], checksum_type=checksum_type)
                     self.log.info("%s checksum for %s: %s", checksum_type, fil['path'], fil[checksum_type])
 

easybuild/framework/easyconfig/types.py

@@ -516,7 +516,7 @@ def to_checksums(checksums):
     for checksum in checksums:
         # each list entry can be:
         # * None (indicates no checksum)
-        # * a string (MD5 or SHA256 checksum)
+        # * a string (SHA256 checksum)
         # * a tuple with 2 elements: checksum type + checksum value
         # * a list of checksums (i.e. multiple checksums for a single file)
         # * a dict (filename to checksum mapping)

easybuild/tools/filetools.py

@@ -1192,12 +1192,16 @@ def compute_checksum(path, checksum_type=DEFAULT_CHECKSUM):
     Compute checksum of specified file.
 
     :param path: Path of file to compute checksum for
-    :param checksum_type: type(s) of checksum ('adler32', 'crc32', 'md5' (default), 'sha1', 'sha256', 'sha512', 'size')
+    :param checksum_type: type(s) of checksum ('adler32', 'crc32', 'md5', 'sha1', 'sha256', 'sha512', 'size')
     """
     if checksum_type not in CHECKSUM_FUNCTIONS:
         raise EasyBuildError("Unknown checksum type (%s), supported types are: %s",
                              checksum_type, CHECKSUM_FUNCTIONS.keys())
 
+    if checksum_type in ['adler32', 'crc32', 'md5', 'sha1', 'size']:
+        _log.deprecated("Checksum type %s is deprecated. Use sha256 (default) or sha512 instead" % checksum_type,
+                        '6.0')
+
     try:
         checksum = CHECKSUM_FUNCTIONS[checksum_type](path)
     except IOError as err:
@@ -1235,7 +1239,7 @@ def verify_checksum(path, checksums):
     Verify checksum of specified file.
 
     :param path: path of file to verify checksum of
-    :param checksums: checksum values (and type, optionally, default is MD5), e.g., 'af314', ('sha', '5ec1b')
+    :param checksums: checksum values (and type, optionally, default is sha256), e.g., 'af314', ('sha', '5ec1b')
     """
 
     filename = os.path.basename(path)
@@ -1287,7 +1291,7 @@ def verify_checksum(path, checksums):
                 # no matching checksums
                 return False
         else:
-            raise EasyBuildError("Invalid checksum spec '%s': should be a string (MD5 or SHA256), "
+            raise EasyBuildError("Invalid checksum spec '%s': should be a string (SHA256), "
                                  "2-tuple (type, value), or tuple of alternative checksum specs.",
                                  checksum)
 

test/framework/easyblock.py

@@ -1536,9 +1536,8 @@ def test_fetch_sources(self):
         self.assertTrue(os.path.samefile(eb.src[0]['path'], toy_source))
         self.assertEqual(eb.src[0]['name'], 'toy-0.0.tar.gz')
         self.assertEqual(eb.src[0]['cmd'], None)
-        self.assertEqual(len(eb.src[0]['checksum']), 7)
-        self.assertEqual(eb.src[0]['checksum'][0], 'be662daa971a640e40be5c804d9d7d10')
-        self.assertEqual(eb.src[0]['checksum'][1], '44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc')
+        self.assertEqual(len(eb.src[0]['checksum']), 2)
+        self.assertEqual(eb.src[0]['checksum'][0], '44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc')
 
         # reconfigure EasyBuild so we can check 'downloaded' sources
         os.environ['EASYBUILD_SOURCEPATH'] = self.test_prefix
@@ -2573,8 +2572,8 @@ def test_checksum_step(self):
         copy_file(toy_ec, self.test_prefix)
         toy_ec = os.path.join(self.test_prefix, os.path.basename(toy_ec))
         ectxt = read_file(toy_ec)
-        # replace MD5 checksum for toy-0.0.tar.gz
-        ectxt = ectxt.replace('be662daa971a640e40be5c804d9d7d10', '00112233445566778899aabbccddeeff')
+        # replace SHA256 checksum for toy-0.0.tar.gz
+        ectxt = ectxt.replace('44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc', '76543210' * 8)
         # replace SHA256 checksums for source of bar extension
         ectxt = ectxt.replace('f3676716b610545a4e8035087f5be0a0248adee0abb3930d3edb76d498ae91e7', '01234567' * 8)
         write_file(toy_ec, ectxt)
@@ -2664,7 +2663,8 @@ def test_checksum_step(self):
         self.mock_stderr(False)
         self.mock_stdout(False)
         self.assertEqual(stdout, '')
-        self.assertEqual(stderr.strip(), "WARNING: Ignoring failing checksum verification for bar-0.0.tar.gz")
+        self.assertEqual(stderr.strip(), "WARNING: Ignoring failing checksum verification for bar-0.0.tar.gz\n\n\n"
+                                         "WARNING: Ignoring failing checksum verification for toy-0.0.tar.gz")
 
     def test_check_checksums(self):
         """Test for check_checksums_for and check_checksums methods."""

test/framework/easyconfig.py

@@ -485,7 +485,7 @@ def test_exts_list(self):
                        # SHA256 checksum for source (gzip-1.4.eb)
                        "6a5abcab719cefa95dca4af0db0d2a9d205d68f775a33b452ec0f2b75b6a3a45",
                        # SHA256 checksum for 'patch' (toy-0.0.eb)
-                       "2d964e0e8f05a7cce0dd83a3e68c9737da14b87b61b8b8b0291d58d4c8d1031c",
+                       "177b34bcdfa1abde96f30354848a01894ebc9c24913bc5145306cd30f78fc8ad",
                    ],
                }),
                # Can use templates in name and version
@@ -509,7 +509,7 @@ def test_exts_list(self):
         self.assertEqual(exts_sources[1]['version'], '2.0')
         self.assertEqual(exts_sources[1]['options'], {
             'checksums': ['6a5abcab719cefa95dca4af0db0d2a9d205d68f775a33b452ec0f2b75b6a3a45',
-                          '2d964e0e8f05a7cce0dd83a3e68c9737da14b87b61b8b8b0291d58d4c8d1031c'],
+                          '177b34bcdfa1abde96f30354848a01894ebc9c24913bc5145306cd30f78fc8ad'],
             'patches': [('toy-0.0.eb', '.')],
             'source_tmpl': 'gzip-1.4.eb',
             'source_urls': [('http://example.com', 'suffix')],

test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-deps.eb

@@ -9,13 +9,10 @@ toolchain = SYSTEM
 
 sources = [SOURCE_TAR_GZ]
 checksums = [[
-    'be662daa971a640e40be5c804d9d7d10',  # default (MD5)
     '44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc',  # default (SHA256)
-    ('adler32', '0x998410035'),
-    ('crc32', '0x1553842328'),
-    ('md5', 'be662daa971a640e40be5c804d9d7d10'),
-    ('sha1', 'f618096c52244539d0e89867405f573fdb0b55b0'),
-    ('size', 273),
+    ('sha512',
+     '3c9dc629e1f2fd01a15c68f9f2a328b5da045c2ec1a189dc72d7195642f32e0'
+     'ff59275aba5fa2a78e84417c7645d0ca5d06aff39e688a8936061ed5c4c600708'),
 ]]
 patches = ['toy-0.0_fix-silly-typo-in-printf-statement.patch']
 

test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-gompi-2018a-test.eb

@@ -10,13 +10,10 @@ toolchainopts = {'pic': True, 'opt': True, 'optarch': True}
 
 sources = [SOURCE_TAR_GZ]
 checksums = [[
-    'be662daa971a640e40be5c804d9d7d10',  # default (MD5)
     '44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc',  # default (SHA256)
-    ('adler32', '0x998410035'),
-    ('crc32', '0x1553842328'),
-    ('md5', 'be662daa971a640e40be5c804d9d7d10'),
-    ('sha1', 'f618096c52244539d0e89867405f573fdb0b55b0'),
-    ('size', 273),
+    ('sha512',
+     '3c9dc629e1f2fd01a15c68f9f2a328b5da045c2ec1a189dc72d7195642f32e0'
+     'ff59275aba5fa2a78e84417c7645d0ca5d06aff39e688a8936061ed5c4c600708'),
     {SOURCE_TAR_GZ: '44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc',
      'bar.tgz': '33ac60685a3e29538db5094259ea85c15906cbd0f74368733f4111eab6187c8f'},
 ]]

test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-gompi-2018a.eb

@@ -9,13 +9,10 @@ toolchainopts = {'pic': True, 'opt': True, 'optarch': True}
 
 sources = [SOURCE_TAR_GZ]
 checksums = [[
-    'be662daa971a640e40be5c804d9d7d10',  # default (MD5)
     '44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc',  # default (SHA256)
-    ('adler32', '0x998410035'),
-    ('crc32', '0x1553842328'),
-    ('md5', 'be662daa971a640e40be5c804d9d7d10'),
-    ('sha1', 'f618096c52244539d0e89867405f573fdb0b55b0'),
-    ('size', 273),
+    ('sha512',
+     '3c9dc629e1f2fd01a15c68f9f2a328b5da045c2ec1a189dc72d7195642f32e0'
+     'ff59275aba5fa2a78e84417c7645d0ca5d06aff39e688a8936061ed5c4c600708'),
 ]]
 patches = [
     'toy-0.0_fix-silly-typo-in-printf-statement.patch',

test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-multiple.eb

@@ -11,8 +11,8 @@ toolchain = SYSTEM
 sources = [SOURCE_TAR_GZ]
 patches = ['toy-0.0_fix-silly-typo-in-printf-statement.patch']
 checksums = [
-    ('adler32', '0x998410035'),
-    'a99f2a72cee1689a2f7e3ace0356efb1',
+    '44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc',
+    '81a3accc894592152f81814fbf133d39afad52885ab52c25018722c7bda92487',
 ]
 
 moduleclass = 'tools'

test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-test.eb

@@ -9,13 +9,10 @@ toolchain = SYSTEM
 
 sources = [SOURCE_TAR_GZ]
 checksums = [[
-    'be662daa971a640e40be5c804d9d7d10',  # default (MD5)
     '44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc',  # default (SHA256)
-    ('adler32', '0x998410035'),
-    ('crc32', '0x1553842328'),
-    ('md5', 'be662daa971a640e40be5c804d9d7d10'),
-    ('sha1', 'f618096c52244539d0e89867405f573fdb0b55b0'),
-    ('size', 273),
+    ('sha512',
+     '3c9dc629e1f2fd01a15c68f9f2a328b5da045c2ec1a189dc72d7195642f32e0'
+     'ff59275aba5fa2a78e84417c7645d0ca5d06aff39e688a8936061ed5c4c600708'),
 ]]
 patches = [
     'toy-0.0_fix-silly-typo-in-printf-statement.patch',