From 7c9935746b6b45c8e932e0a7f089614cbbdca39f Mon Sep 17 00:00:00 2001 From: "jiyeong.seok" Date: Thu, 6 Feb 2025 18:55:14 +0900 Subject: [PATCH 1/4] Modify the git download func Signed-off-by: jiyeong.seok --- src/fosslight_util/download.py | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/src/fosslight_util/download.py b/src/fosslight_util/download.py index 4a6e1bd..afb8d43 100755 --- a/src/fosslight_util/download.py +++ b/src/fosslight_util/download.py @@ -102,6 +102,7 @@ def cli_download_and_extract(link: str, target_dir: str, log_dir: str, checkout_ msg_wget = "" oss_name = "" oss_version = "" + clone_default_branch_flag = False log_file_name = "fosslight_download_" + \ datetime.now().strftime('%Y%m%d_%H-%M-%S')+".txt" logger, log_item = init_log(os.path.join(log_dir, log_file_name)) @@ -123,8 +124,10 @@ def cli_download_and_extract(link: str, target_dir: str, log_dir: str, checkout_ is_rubygems = src_info.get("rubygems", False) # General download (git clone, wget) - success_git, msg, oss_name, oss_version = download_git_clone(link, target_dir, checkout_to, - tag, branch, ssh_key, id, git_token) + success_git, msg, oss_name, oss_version, clone_default_branch_flag = download_git_clone(link, target_dir, + checkout_to, + tag, branch, + ssh_key, id, git_token) link = change_ssh_link_to_https(link) if (not is_rubygems) and (not success_git): if os.path.isfile(target_dir): @@ -151,7 +154,7 @@ def cli_download_and_extract(link: str, target_dir: str, log_dir: str, checkout_ msg = str(error) logger.info(f"\n* FOSSLight Downloader - Result: {success} ({msg})") - return success, msg, oss_name, oss_version + return success, msg, oss_name, oss_version, clone_default_branch_flag def get_ref_to_checkout(checkout_to, ref_list): @@ -221,17 +224,21 @@ def download_git_repository(refs_to_checkout, git_url, target_dir, tag): repo = Repo.clone_from(git_url, target_dir) clone_default_branch_flag = True success = True - - if refs_to_checkout != tag or clone_default_branch_flag: - oss_version = repo.active_branch.name - else: - oss_version = repo.git.describe('--tags') - return success, oss_version + try: + if clone_default_branch_flag: + oss_version = repo.active_branch.name + else: + oss_version = repo.git.describe('--tags') + except Exception as error: + logger.debug(f"Get tag/branch error:{error}") + + return success, oss_version, clone_default_branch_flag def download_git_clone(git_url, target_dir, checkout_to="", tag="", branch="", ssh_key="", id="", git_token=""): oss_name = get_github_ossname(git_url) refs_to_checkout = decide_checkout(checkout_to, tag, branch) + clone_default_branch_flag = False msg = "" success = True @@ -253,7 +260,7 @@ def download_git_clone(git_url, target_dir, checkout_to="", tag="", branch="", s logger.info(f"Download git with ssh_key:{git_url}") git_ssh_cmd = f'ssh -i {ssh_key}' with Git().custom_environment(GIT_SSH_COMMAND=git_ssh_cmd): - success, oss_version = download_git_repository(refs_to_checkout, git_url, target_dir, tag) + success, oss_version, clone_default_branch_flag = download_git_repository(refs_to_checkout, git_url, target_dir, tag) else: if id and git_token: try: @@ -263,7 +270,7 @@ def download_git_clone(git_url, target_dir, checkout_to="", tag="", branch="", s git_url = git_url.replace(protocol, f"{protocol}{id}:{git_token}@") except Exception as error: logger.info(f"Failed to insert id, token to git url:{error}") - success, oss_version = download_git_repository(refs_to_checkout, git_url, target_dir, tag) + success, oss_version, clone_default_branch_flag = download_git_repository(refs_to_checkout, git_url, target_dir, tag) logger.info(f"git checkout: {oss_version}") refs_to_checkout = oss_version @@ -277,7 +284,7 @@ def download_git_clone(git_url, target_dir, checkout_to="", tag="", branch="", s logger.warning(f"git clone - failed: {error}") msg = str(error) - return success, msg, oss_name, refs_to_checkout + return success, msg, oss_name, refs_to_checkout, clone_default_branch_flag def download_wget(link, target_dir, compressed_only): From 481ecdf37fb4b7b4063f40137c4cbd266405c289 Mon Sep 17 00:00:00 2001 From: Soim Kim Date: Thu, 6 Feb 2025 19:51:44 +0900 Subject: [PATCH 2/4] Print branch name if checkout fails --- src/fosslight_util/download.py | 32 +++++++++++--------------------- 1 file changed, 11 insertions(+), 21 deletions(-) diff --git a/src/fosslight_util/download.py b/src/fosslight_util/download.py index afb8d43..27352cf 100755 --- a/src/fosslight_util/download.py +++ b/src/fosslight_util/download.py @@ -102,7 +102,6 @@ def cli_download_and_extract(link: str, target_dir: str, log_dir: str, checkout_ msg_wget = "" oss_name = "" oss_version = "" - clone_default_branch_flag = False log_file_name = "fosslight_download_" + \ datetime.now().strftime('%Y%m%d_%H-%M-%S')+".txt" logger, log_item = init_log(os.path.join(log_dir, log_file_name)) @@ -124,10 +123,10 @@ def cli_download_and_extract(link: str, target_dir: str, log_dir: str, checkout_ is_rubygems = src_info.get("rubygems", False) # General download (git clone, wget) - success_git, msg, oss_name, oss_version, clone_default_branch_flag = download_git_clone(link, target_dir, - checkout_to, - tag, branch, - ssh_key, id, git_token) + success_git, msg, oss_name, oss_version = download_git_clone(link, target_dir, + checkout_to, + tag, branch, + ssh_key, id, git_token) link = change_ssh_link_to_https(link) if (not is_rubygems) and (not success_git): if os.path.isfile(target_dir): @@ -154,7 +153,7 @@ def cli_download_and_extract(link: str, target_dir: str, log_dir: str, checkout_ msg = str(error) logger.info(f"\n* FOSSLight Downloader - Result: {success} ({msg})") - return success, msg, oss_name, oss_version, clone_default_branch_flag + return success, msg, oss_name, oss_version def get_ref_to_checkout(checkout_to, ref_list): @@ -208,7 +207,6 @@ def get_github_token(git_url): def download_git_repository(refs_to_checkout, git_url, target_dir, tag): success = False oss_version = "" - clone_default_branch_flag = False logger.info(f"Download git url :{git_url}") if refs_to_checkout: @@ -216,29 +214,21 @@ def download_git_repository(refs_to_checkout, git_url, target_dir, tag): # gitPython uses the branch argument the same whether you check out to a branch or a tag. repo = Repo.clone_from(git_url, target_dir, branch=refs_to_checkout) success = True + oss_version = refs_to_checkout except GitCommandError as error: logger.debug(f"Git checkout error:{error}") success = False if not success: repo = Repo.clone_from(git_url, target_dir) - clone_default_branch_flag = True success = True - try: - if clone_default_branch_flag: - oss_version = repo.active_branch.name - else: - oss_version = repo.git.describe('--tags') - except Exception as error: - logger.debug(f"Get tag/branch error:{error}") - - return success, oss_version, clone_default_branch_flag + oss_version = repo.active_branch.name + return success, oss_version def download_git_clone(git_url, target_dir, checkout_to="", tag="", branch="", ssh_key="", id="", git_token=""): oss_name = get_github_ossname(git_url) refs_to_checkout = decide_checkout(checkout_to, tag, branch) - clone_default_branch_flag = False msg = "" success = True @@ -260,7 +250,7 @@ def download_git_clone(git_url, target_dir, checkout_to="", tag="", branch="", s logger.info(f"Download git with ssh_key:{git_url}") git_ssh_cmd = f'ssh -i {ssh_key}' with Git().custom_environment(GIT_SSH_COMMAND=git_ssh_cmd): - success, oss_version, clone_default_branch_flag = download_git_repository(refs_to_checkout, git_url, target_dir, tag) + success, oss_version = download_git_repository(refs_to_checkout, git_url, target_dir, tag) else: if id and git_token: try: @@ -270,7 +260,7 @@ def download_git_clone(git_url, target_dir, checkout_to="", tag="", branch="", s git_url = git_url.replace(protocol, f"{protocol}{id}:{git_token}@") except Exception as error: logger.info(f"Failed to insert id, token to git url:{error}") - success, oss_version, clone_default_branch_flag = download_git_repository(refs_to_checkout, git_url, target_dir, tag) + success, oss_version = download_git_repository(refs_to_checkout, git_url, target_dir, tag) logger.info(f"git checkout: {oss_version}") refs_to_checkout = oss_version @@ -284,7 +274,7 @@ def download_git_clone(git_url, target_dir, checkout_to="", tag="", branch="", s logger.warning(f"git clone - failed: {error}") msg = str(error) - return success, msg, oss_name, refs_to_checkout, clone_default_branch_flag + return success, msg, oss_name, refs_to_checkout def download_wget(link, target_dir, compressed_only): From afc25ad702167717aeb40248076256c0a423aca8 Mon Sep 17 00:00:00 2001 From: Soim Kim Date: Thu, 6 Feb 2025 20:13:44 +0900 Subject: [PATCH 3/4] Update branch name for testing --- tests/test_download.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_download.py b/tests/test_download.py index 4a45979..d99c89d 100644 --- a/tests/test_download.py +++ b/tests/test_download.py @@ -23,7 +23,7 @@ def test_download_from_github(): @pytest.mark.parametrize("git_url", - ["git://git.kernel.org/pub/scm/utils/kernel/kmod/kmod.git;protocol=git;branch=ci-test", + ["git://git.kernel.org/pub/scm/utils/kernel/kmod/kmod.git;protocol=git;branch=hash-stat2", "git://git.kernel.org/pub/scm/utils/kernel/kmod/kmod.git;protocol=git;tag=v32"]) def test_download_from_github_with_branch_or_tag(git_url): # given @@ -62,7 +62,7 @@ def test_download_git_clone_with_branch(): # given git_url = "git://git.kernel.org/pub/scm/utils/kernel/kmod/kmod.git" target_dir = os.path.join(constants.TEST_RESULT_DIR, "download/example") - branch_name = "ci-test" + branch_name = "hash-stat2" # when success, _, oss_name, oss_version = download_git_clone(git_url, target_dir, "", "", branch_name) From fb33ab64853d9e1a9c673e73a1a8e19e1fcb7436 Mon Sep 17 00:00:00 2001 From: "jiyeong.seok" Date: Fri, 7 Feb 2025 15:31:10 +0900 Subject: [PATCH 4/4] Remove default checkout version Signed-off-by: jiyeong.seok --- src/fosslight_util/download.py | 5 ++--- tests/test_download.py | 12 ++++++------ 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/src/fosslight_util/download.py b/src/fosslight_util/download.py index 27352cf..228b20c 100755 --- a/src/fosslight_util/download.py +++ b/src/fosslight_util/download.py @@ -212,7 +212,7 @@ def download_git_repository(refs_to_checkout, git_url, target_dir, tag): if refs_to_checkout: try: # gitPython uses the branch argument the same whether you check out to a branch or a tag. - repo = Repo.clone_from(git_url, target_dir, branch=refs_to_checkout) + Repo.clone_from(git_url, target_dir, branch=refs_to_checkout) success = True oss_version = refs_to_checkout except GitCommandError as error: @@ -220,9 +220,8 @@ def download_git_repository(refs_to_checkout, git_url, target_dir, tag): success = False if not success: - repo = Repo.clone_from(git_url, target_dir) + Repo.clone_from(git_url, target_dir) success = True - oss_version = repo.active_branch.name return success, oss_version diff --git a/tests/test_download.py b/tests/test_download.py index d99c89d..cc9982d 100644 --- a/tests/test_download.py +++ b/tests/test_download.py @@ -94,7 +94,7 @@ def test_download_main_branch_when_any_branch_or_tag_not_entered(): # given git_url = "https://github.com/LGE-OSS/example" target_dir = os.path.join(constants.TEST_RESULT_DIR, "download/example") - expected_oss_name = "main" + expected_oss_ver = "" # when success, _, oss_name, oss_version = download_git_clone(git_url, target_dir) @@ -103,7 +103,7 @@ def test_download_main_branch_when_any_branch_or_tag_not_entered(): assert success is True assert len(os.listdir(target_dir)) > 0 assert oss_name == 'LGE-OSS-example' - assert oss_version == expected_oss_name + assert oss_version == expected_oss_ver def test_download_main_branch_when_non_existent_branch_entered(): @@ -111,7 +111,7 @@ def test_download_main_branch_when_non_existent_branch_entered(): git_url = "https://github.com/LGE-OSS/example" target_dir = os.path.join(constants.TEST_RESULT_DIR, "download/example") branch_name = "non-existent-branch" - expected_oss_name = "main" + expected_oss_ver = "" # when success, _, oss_name, oss_version = download_git_clone(git_url, target_dir, "", "", branch_name) @@ -120,7 +120,7 @@ def test_download_main_branch_when_non_existent_branch_entered(): assert success is True assert len(os.listdir(target_dir)) > 0 assert oss_name == 'LGE-OSS-example' - assert oss_version == expected_oss_name + assert oss_version == expected_oss_ver def test_download_main_branch_when_non_existent_tag_entered(): @@ -128,7 +128,7 @@ def test_download_main_branch_when_non_existent_tag_entered(): git_url = "https://github.com/LGE-OSS/example" target_dir = os.path.join(constants.TEST_RESULT_DIR, "download/example") tag_name = "non-existent-tag" - expected_oss_name = "main" + expected_oss_ver = "" # when success, _, oss_name, oss_version = download_git_clone(git_url, target_dir, "", tag_name) @@ -137,4 +137,4 @@ def test_download_main_branch_when_non_existent_tag_entered(): assert success is True assert len(os.listdir(target_dir)) > 0 assert oss_name == 'LGE-OSS-example' - assert oss_version == expected_oss_name + assert oss_version == expected_oss_ver