obtain PR/commit diff via GitHub API rather than downloading *.diff file via github.com

Author: boegel

easybuild/tools/github.py

@@ -54,7 +54,7 @@
 from easybuild.framework.easyconfig.parser import EasyConfigParser
 from easybuild.tools import LooseVersion
 from easybuild.tools.build_log import EasyBuildError, EasyBuildExit, print_msg, print_warning
-from easybuild.tools.config import DEFAULT_DOWNLOAD_MAX_ATTEMPTS, build_option
+from easybuild.tools.config import build_option
 from easybuild.tools.filetools import apply_patch, copy_dir, copy_easyblocks, copy_file, copy_framework_files
 from easybuild.tools.filetools import det_patched_files, download_file, extract_file
 from easybuild.tools.filetools import get_easyblock_class_name, mkdir, read_file, symlink, which, write_file
@@ -548,15 +548,22 @@ def fetch_files_from_pr(pr, path=None, github_user=None, github_account=None, gi
     diff_url = pr_data['diff_url']
     diff_fn = os.path.basename(diff_url)
     diff_filepath = os.path.join(path, diff_fn)
-    # max. 6 attempts + initial wait time of 10sec -> max. 10 * (2^6) = 640sec (~10min) before giving up on download
-    # see also https://github.com/easybuilders/easybuild-framework/issues/4869
-    max_attempts = DEFAULT_DOWNLOAD_MAX_ATTEMPTS
-    download_file(diff_fn, diff_url, diff_filepath, forced=True, trace=False,
-                  max_attempts=max_attempts)
-    if not os.path.exists(diff_filepath):
-        raise EasyBuildError(f"Failed to download {diff_url}, even after {max_attempts} attempts and being patient...")
-    diff_txt = read_file(diff_filepath)
-    _log.debug("Diff for PR #%s:\n%s", pr, diff_txt)
+
+    def pr_request_fn(gh):
+        return gh.repos[github_account][github_repo].pulls[pr]
+
+    # see also https://docs.github.com/en/rest/pulls/pulls#get-a-pull-request,
+    # in particular part about media types
+    accept_diff = {'Accept': 'application/vnd.github.diff'}
+    status, data = github_api_get_request(pr_request_fn, github_user=github_user, headers=accept_diff)
+    if status == HTTP_STATUS_OK:
+        # decode from bytes to text
+        diff_txt = data.decode()
+        _log.debug("Diff for PR #%s:\n%s", pr, diff_txt)
+        write_file(diff_filepath, diff_txt)
+    else:
+        error_msg = f"Failed to download diff for {github_account}/{github_repo} PR #{pr}! (HTTP status code: {status})"
+        raise EasyBuildError(error_msg)
 
     patched_files = det_patched_files(txt=diff_txt, omit_ab_prefix=True, github=True, filter_deleted=True)
     _log.debug("List of patched files for PR #%s: %s", pr, patched_files)
@@ -703,24 +710,25 @@ def fetch_files_from_commit(commit, files=None, path=None, github_account=None,
 
     # if no files are specified, determine which files are touched in commit
     if not files:
-        diff_url = os.path.join(GITHUB_URL, github_account, github_repo, 'commit', commit + '.diff')
-        diff_fn = os.path.basename(diff_url)
-        diff_filepath = os.path.join(path, diff_fn)
-        # max. 6 attempts + initial wait time of 10sec -> max. 10 * (2^6) = 640sec (~10min) before giving up on download
-        # see also https://github.com/easybuilders/easybuild-framework/issues/4869
-        max_attempts = DEFAULT_DOWNLOAD_MAX_ATTEMPTS
-        download_file(diff_fn, diff_url, diff_filepath, forced=True, trace=False,
-                      max_attempts=max_attempts)
-        if os.path.exists(diff_filepath):
-            diff_txt = read_file(diff_filepath)
+        github_user = build_option('github_user')
+
+        def pr_request_fn(gh):
+            return gh.repos[github_account][github_repo].pulls[pr]
+
+        # see also https://docs.github.com/en/rest/commits/commits#get-a-commit,
+        # in particular part about media types
+        accept_diff = {'Accept': 'application/vnd.github.diff'}
+        status, data = github_api_get_request(pr_request_fn, github_user=github_user, headers=accept_diff)
+        if status == HTTP_STATUS_OK:
+            # decode from bytes to text
+            diff_txt = data.decode()
             _log.debug("Diff for commit %s:\n%s", commit, diff_txt)
 
             files = det_patched_files(txt=diff_txt, omit_ab_prefix=True, github=True, filter_deleted=True)
             _log.debug("List of patched files for commit %s: %s", commit, files)
         else:
-            msg = f"Failed to download diff for commit {commit} of {github_account}/{github_repo} "
-            msg += " (after {max_attempts} attempts)"
-            raise EasyBuildError(msg, exit_code=EasyBuildExit.FAIL_GITHUB)
+            error_msg = f"Failed to download diff for {github_account}/{github_repo} PR #{pr}! (HTTP status code: {status})"
+            raise EasyBuildError(error_msg)
 
     # download tarball for specific commit
     repo_commit = download_repo(repo=github_repo, commit=commit, account=github_account)