From 907dc778e0aa916b58be136f6b059828394af587 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EC=84=9D=EC=A7=80=EC=98=81/=EC=B1=85=EC=9E=84=EC=97=B0?= =?UTF-8?q?=EA=B5=AC=EC=9B=90/SW=EA=B3=B5=ED=95=99=28=EC=97=B0=29Open=20So?= =?UTF-8?q?urce=20TP?= Date: Tue, 8 Jul 2025 17:55:41 +0900 Subject: [PATCH] Support cargo downloadable url MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 석지영/책임연구원/SW공학(연)Open Source TP --- src/fosslight_util/_get_downloadable_url.py | 31 ++++++++++++++++++++- src/fosslight_util/constant.py | 4 ++- src/fosslight_util/download.py | 11 ++++++-- 3 files changed, 42 insertions(+), 4 deletions(-) diff --git a/src/fosslight_util/_get_downloadable_url.py b/src/fosslight_util/_get_downloadable_url.py index 263959a..d23007c 100755 --- a/src/fosslight_util/_get_downloadable_url.py +++ b/src/fosslight_util/_get_downloadable_url.py @@ -47,6 +47,9 @@ def extract_name_version_from_link(link): origin_name = origin_name[:-1] oss_name = f"go:{origin_name}" oss_version = match.group(2) + elif key == "cargo": + oss_name = f"cargo:{origin_name}" + oss_version = match.group(2) except Exception as ex: logger.info(f"extract_name_version_from_link {key}:{ex}") if oss_name and (not oss_version): @@ -110,8 +113,34 @@ def get_downloadable_url(link): ret, result_link = get_download_location_for_pub(new_link) elif pkg_type == "go": ret, result_link = get_download_location_for_go(new_link) + elif pkg_type == "cargo": + ret, result_link = get_download_location_for_cargo(new_link) + return ret, result_link, oss_name, oss_version, pkg_type + + +def get_download_location_for_cargo(link): + # get the url for downloading source file: https://crates.io/api/v1/crates///download + ret = False + new_link = '' + host = 'https://crates.io/api/v1/crates' + + try: + dn_loc_re = re.findall(r'crates.io\/crates\/([^\/]+)\/?([^\/]*)', link) + if dn_loc_re: + oss_name = dn_loc_re[0][0] + oss_version = dn_loc_re[0][1] - return ret, result_link, oss_name, oss_version + new_link = f'{host}/{oss_name}/{oss_version}/download' + res = urlopen(new_link) + if res.getcode() == 200: + ret = True + else: + logger.warning(f'Cannot find the valid link for cargo (url:{new_link}') + except Exception as error: + ret = False + logger.warning(f'Cannot find the link for cargo (url:{link}({(new_link)})): {error}') + + return ret, new_link def get_download_location_for_go(link): diff --git a/src/fosslight_util/constant.py b/src/fosslight_util/constant.py index a9764a9..bf6acae 100755 --- a/src/fosslight_util/constant.py +++ b/src/fosslight_util/constant.py @@ -35,6 +35,7 @@ # pub: https://pub.dev/packages/(package)/versions/(version) # Cocoapods : https://cocoapods.org/(package) # go : https://pkg.go.dev/(package_name_with_slash)@(version) +# cargo : https://crates.io/crates/(crate_name)/(version) PKG_PATTERN = { "pypi": r'https?:\/\/pypi\.org\/project\/([^\/]+)[\/]?([^\/]*)', "pypi2": r'https?:\/\/files\.pythonhosted\.org\/packages\/source\/[\w]\/([^\/]+)\/[\S]+-([^\-]+)\.tar\.gz', @@ -43,5 +44,6 @@ "npm2": r'https?:\/\/www\.npmjs\.com\/package\/(\@[^\/]+\/[^\/]+)(?:\/v\/)?([^\/]*)', "pub": r'https?:\/\/pub\.dev\/packages\/([^\/]+)(?:\/versions\/)?([^\/]*)', "cocoapods": r'https?:\/\/cocoapods\.org\/pods\/([^\/]+)', - "go": r'https?:\/\/pkg.go.dev\/([^\@]+)\@?v?([^\/]*)' + "go": r'https?:\/\/pkg.go.dev\/([^\@]+)\@?v?([^\/]*)', + "cargo": r'https?:\/\/crates\.io\/crates\/([^\/]+)\/?([^\/]*)', } diff --git a/src/fosslight_util/download.py b/src/fosslight_util/download.py index 9873c52..8df7a9e 100755 --- a/src/fosslight_util/download.py +++ b/src/fosslight_util/download.py @@ -314,7 +314,7 @@ def download_wget(link, target_dir, compressed_only): Path(target_dir).mkdir(parents=True, exist_ok=True) - ret, new_link, oss_name, oss_version = get_downloadable_url(link) + ret, new_link, oss_name, oss_version, pkg_type = get_downloadable_url(link) if ret and new_link: link = new_link @@ -323,6 +323,9 @@ def download_wget(link, target_dir, compressed_only): if link.endswith(ext): success = True break + if not success: + if pkg_type == 'cargo': + success = True else: success = True @@ -330,7 +333,11 @@ def download_wget(link, target_dir, compressed_only): raise Exception('Not supported compression type (link:{0})'.format(link)) logger.info(f"wget: {link}") - downloaded_file = wget.download(link, target_dir) + if pkg_type == 'cargo': + outfile = os.path.join(target_dir, f'{oss_name}.tar.gz') + downloaded_file = wget.download(link, out=outfile) + else: + downloaded_file = wget.download(link, target_dir) if platform.system() != "Windows": signal.alarm(0) else: