Fix the cloning of tags

Can't use refs/tags/xxx for git clone so clone it assuming it is a tag
and check afterwards.
Fall back to fetching the full history and checking out the tag and
submodules manually

Author: Flamefire

easybuild/tools/filetools.py

@@ -2477,7 +2477,7 @@ def get_source_tarball_from_git(filename, targetdir, git_config):
         clone_cmd.extend(['--depth', '1'])
 
     if tag:
-        clone_cmd.extend(['--branch', 'refs/tags/' + tag])
+        clone_cmd.extend(['--branch', tag])
         if recursive:
             clone_cmd.append('--recursive')
     else:
@@ -2487,22 +2487,41 @@ def get_source_tarball_from_git(filename, targetdir, git_config):
 
     tmpdir = tempfile.mkdtemp()
     cwd = change_dir(tmpdir)
-    run.run_cmd(' '.join(clone_cmd), log_all=True, log_ok=False, simple=False, regexp=False)
+    run.run_cmd(' '.join(clone_cmd), log_all=True, simple=True, regexp=False)
 
     # if a specific commit is asked for, check it out
     if commit:
         checkout_cmd = ['git', 'checkout', commit]
         if recursive:
             checkout_cmd.extend(['&&', 'git', 'submodule', 'update', '--init', '--recursive'])
 
-        run.run_cmd(' '.join(checkout_cmd), log_all=True, log_ok=False, simple=False, regexp=False, path=repo_name)
+        run.run_cmd(' '.join(checkout_cmd), log_all=True, simple=True, regexp=False, path=repo_name)
+    elif not build_option('extended_dry_run'):
+        # If we wanted to get a tag make sure we actually got a tag and not a branch with the same name
+        # This doesn't make sense in dry-run mode as we don't have anything to check
+        cmd = 'git describe --exact-match --tags HEAD'
+        # Note: Disable logging to also disable the error handling in run_cmd
+        (out, ec) = run.run_cmd(cmd, log_ok=False, log_all=False, regexp=False, path=repo_name)
+        if ec != 0 or tag not in out.splitlines():
+            cmds = []
+            if not keep_git_dir:
+                # Make the repo unshallow, same as git fetch --unshallow in git 1.8.3+
+                # The first fetch seemingly does nothing, no idea why.
+                cmds.append('git fetch --depth=2147483647 && git fetch --depth=2147483647')
+            cmds.append('git checkout refs/tags/' + tag)
+            # Clean all untracked files, e.g. from left-over submodules
+            cmds.append('git clean --force -d -x')
+            if recursive:
+                cmds.append('git submodule update --init --recursive')
+            for cmd in cmds:
+                run.run_cmd(cmd, log_all=True, simple=True, regexp=False, path=repo_name)
 
     # create an archive and delete the git repo directory
     if keep_git_dir:
         tar_cmd = ['tar', 'cfvz', targetpath, repo_name]
     else:
         tar_cmd = ['tar', 'cfvz', targetpath, '--exclude', '.git', repo_name]
-    run.run_cmd(' '.join(tar_cmd), log_all=True, log_ok=False, simple=False, regexp=False)
+    run.run_cmd(' '.join(tar_cmd), log_all=True, simple=True, regexp=False)
 
     # cleanup (repo_name dir does not exist in dry run mode)
     change_dir(cwd)

test/framework/filetools.py

@@ -2517,7 +2517,7 @@ def run_check():
         }
         git_repo = {'git_repo': '[email protected]:easybuilders/testrepository.git'}  # Just to make the below shorter
         expected = '\n'.join([
-            r'  running command "git clone --depth 1 --branch refs/tags/tag_for_tests %(git_repo)s"',
+            r'  running command "git clone --depth 1 --branch tag_for_tests %(git_repo)s"',
             r"  \(in .*/tmp.*\)",
             r'  running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"',
             r"  \(in .*/tmp.*\)",
@@ -2526,7 +2526,7 @@ def run_check():
 
         git_config['recursive'] = True
         expected = '\n'.join([
-            r'  running command "git clone --depth 1 --branch refs/tags/tag_for_tests --recursive %(git_repo)s"',
+            r'  running command "git clone --depth 1 --branch tag_for_tests --recursive %(git_repo)s"',
             r"  \(in .*/tmp.*\)",
             r'  running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"',
             r"  \(in .*/tmp.*\)",
@@ -2535,7 +2535,7 @@ def run_check():
 
         git_config['keep_git_dir'] = True
         expected = '\n'.join([
-            r'  running command "git clone --branch refs/tags/tag_for_tests --recursive %(git_repo)s"',
+            r'  running command "git clone --branch tag_for_tests --recursive %(git_repo)s"',
             r"  \(in .*/tmp.*\)",
             r'  running command "tar cfvz .*/target/test.tar.gz testrepository"',
             r"  \(in .*/tmp.*\)",
@@ -2566,12 +2566,12 @@ def run_check():
         ]) % git_repo
         run_check()
 
-        # Test with real data
+        # Test with real data.
         init_config()
         git_config = {
             'repo_name': 'testrepository',
             'url': 'https://github.com/easybuilders',
-            'tag': 'tag_for_tests',
+            'tag': 'branch_tag_for_test',
         }
 
         try:
@@ -2581,11 +2581,21 @@ def run_check():
             self.assertEqual(res, test_file)
             self.assertTrue(os.path.isfile(test_file))
             self.assertEqual(os.listdir(target_dir), ['test.tar.gz'])
+            # Check that we indeed downloaded the right tag
+            extracted_dir = tempfile.mkdtemp(prefix='extracted_dir')
+            extracted_repo_dir = ft.extract_file(test_file, extracted_dir, change_into_dir=False)
+            self.assertTrue(os.path.isfile(os.path.join(extracted_repo_dir, 'this-is-a-branch.txt')))
+            os.remove(test_file)
 
-            # Check that we indeed downloaded the tag and not a branch
+            # use a tag that clashes with a branch name and make sure this is handled correctly
+            git_config['tag'] = 'tag_for_tests'
+            res = ft.get_source_tarball_from_git('test.tar.gz', target_dir, git_config)
+            self.assertEqual(res, test_file)
+            self.assertTrue(os.path.isfile(test_file))
+            # Check that we indeed downloaded the tag and not the branch
             extracted_dir = tempfile.mkdtemp(prefix='extracted_dir')
-            target_dir = ft.extract_file(test_file, extracted_dir, change_into_dir=False)
-            self.assertTrue(os.path.isfile(os.path.join(target_dir, 'this-is-a-tag.txt')))
+            extracted_repo_dir = ft.extract_file(test_file, extracted_dir, change_into_dir=False)
+            self.assertTrue(os.path.isfile(os.path.join(extracted_repo_dir, 'this-is-a-tag.txt')))
 
             del git_config['tag']
             git_config['commit'] = '8456f86'