From 476403fa34892193e1bdd1728f3ab9eeeede4d96 Mon Sep 17 00:00:00 2001 From: zejianzhang Date: Thu, 7 May 2026 15:22:49 +0800 Subject: [PATCH] Specify the download tool via the command line or by setting environment variables to avoid download failures caused by using a proxy server. --- cget/builder.py | 4 ++-- cget/cli.py | 6 ++++-- cget/prefix.py | 20 ++++++++++++-------- cget/util.py | 40 +++++++++++++++++++++++++++++++++++++--- 4 files changed, 55 insertions(+), 15 deletions(-) diff --git a/cget/builder.py b/cget/builder.py index 32f345f..2113478 100644 --- a/cget/builder.py +++ b/cget/builder.py @@ -42,10 +42,10 @@ def targets(self): if line.startswith(six.b('... ')): yield line[4:] - def fetch(self, url, hash=None, copy=False, insecure=False): + def fetch(self, url, hash=None, copy=False, insecure=False, download_tool=None): self.prefix.log("fetch:", url) if insecure: url = url.replace('https', 'http') - f = util.retrieve_url(url, self.top_dir, copy=copy, insecure=insecure, hash=hash) + f = util.retrieve_url(url, self.top_dir, copy=copy, insecure=insecure, hash=hash, download_tool=download_tool) if os.path.isfile(f): with display.status("Extracting archive..."): util.extract_ar(archive=f, dst=self.top_dir) diff --git a/cget/cli.py b/cget/cli.py index 68c7e55..e8bf3ed 100644 --- a/cget/cli.py +++ b/cget/cli.py @@ -47,12 +47,14 @@ def get_command(self, ctx, cmd_name): @click.option('-p', '--prefix', envvar='CGET_PREFIX', help='Set prefix used to install packages') @click.option('-v', '--verbose', is_flag=True, envvar='VERBOSE', help="Enable verbose mode") @click.option('-B', '--build-path', envvar='CGET_BUILD_PATH', help='Set the path for the build directory to use when building the package') +@click.option('--download-tool', envvar='CGET_DOWNLOAD_TOOL', help='Set the download tool to use (wget or curl)') @click.pass_context -def cli(ctx, prefix, verbose, build_path): +def cli(ctx, prefix, verbose, build_path, download_tool): ctx.obj = {} if prefix: ctx.obj['PREFIX'] = prefix if verbose: ctx.obj['VERBOSE'] = verbose if build_path: ctx.obj['BUILD_PATH'] = build_path + if download_tool: ctx.obj['DOWNLOAD_TOOL'] = download_tool def use_prefix(f): @click.option('-p', '--prefix', help='Set prefix used to install packages') @@ -61,7 +63,7 @@ def use_prefix(f): @click.pass_obj @functools.wraps(f) def w(obj, prefix, verbose, build_path, *args, **kwargs): - p = CGetPrefix(prefix or obj.get('PREFIX'), verbose or obj.get('VERBOSE'), build_path or obj.get('BUILD_PATH')) + p = CGetPrefix(prefix or obj.get('PREFIX'), verbose or obj.get('VERBOSE'), build_path or obj.get('BUILD_PATH'), obj.get('DOWNLOAD_TOOL')) f(p, *args, **kwargs) return w diff --git a/cget/prefix.py b/cget/prefix.py index e9a2695..1dff382 100644 --- a/cget/prefix.py +++ b/cget/prefix.py @@ -102,10 +102,11 @@ def find_cmake(p, start): PACKAGE_SOURCE_TYPES = (six.string_types, PackageSource, PackageBuild) class CGetPrefix: - def __init__(self, prefix, verbose=False, build_path=None): + def __init__(self, prefix, verbose=False, build_path=None, download_tool=None): self.prefix = os.path.abspath(prefix or 'cget') self.verbose = verbose self.build_path_var = build_path + self.download_tool = download_tool self.cmd = util.Commander(paths=[self.get_path('bin')], env=self.get_env(), verbose=self.verbose) self.toolchain = self.write_cmake() @@ -299,22 +300,25 @@ def from_file(self, file, url=None, no_recipe=False): def write_parent(self, pb, track=True): if track and pb.parent is not None: util.mkfile(self.get_deps_directory(pb.to_fname()), pb.parent, pb.parent) - def install_deps(self, pb, d, test=False, test_all=False, generator=None, insecure=False, ignore_requirements=False): + def install_deps(self, pb, d, test=False, test_all=False, generator=None, insecure=False, ignore_requirements=False, download_tool=None): req_txt = find_requirements_file(d) if not ignore_requirements else None for dependent in self.from_file(pb.requirements or req_txt, pb.pkg_src.url): transient = dependent.test or dependent.build testing = test or test_all installable = not dependent.test or dependent.test == testing - if installable: - self.install(dependent.of(pb), test_all=test_all, generator=generator, track=not transient, insecure=insecure) + if installable: + self.install(dependent.of(pb), test_all=test_all, generator=generator, track=not transient, insecure=insecure, download_tool=download_tool) @returns(six.string_types) @params(pb=PACKAGE_SOURCE_TYPES, test=bool, test_all=bool, update=bool, track=bool) - def install(self, pb, test=False, test_all=False, generator=None, update=False, track=True, insecure=False): + def install(self, pb, test=False, test_all=False, generator=None, update=False, track=True, insecure=False, download_tool=None): pb = self.parse_pkg_build(pb) pkg_dir = self.get_package_directory(pb.to_fname()) unlink_dir = self.get_unlink_directory(pb.to_fname()) install_dir = self.get_package_directory(pb.to_fname(), 'install') + # Use download_tool from instance if not provided + if download_tool is None: + download_tool = self.download_tool # If its been unlinked, then link it in if os.path.exists(unlink_dir): if update: shutil.rmtree(unlink_dir) @@ -322,15 +326,15 @@ def install(self, pb, test=False, test_all=False, generator=None, update=False, self.link(pb) self.write_parent(pb, track=track) return "[green]\u2713[/] Linking package {}".format(display.pkg(pb.to_name())) - if os.path.exists(pkg_dir): + if os.path.exists(pkg_dir): self.write_parent(pb, track=track) if update: self.remove(pb) else: return "[yellow]![/] Package {} already installed".format(display.pkg(pb.to_name())) with self.create_builder(pb.pkg_src.get_hash(), tmp=True) as builder: # Fetch package - src_dir = builder.fetch(pb.pkg_src.url, pb.hash, (pb.cmake != None), insecure=insecure) + src_dir = builder.fetch(pb.pkg_src.url, pb.hash, (pb.cmake != None), insecure=insecure, download_tool=download_tool) # Install any dependencies first - self.install_deps(pb, src_dir, test=test, test_all=test_all, generator=generator, insecure=insecure, ignore_requirements=pb.ignore_requirements) + self.install_deps(pb, src_dir, test=test, test_all=test_all, generator=generator, insecure=insecure, ignore_requirements=pb.ignore_requirements, download_tool=download_tool) # Setup cmake file if pb.cmake: target = os.path.join(src_dir, 'CMakeLists.txt') diff --git a/cget/util.py b/cget/util.py index 0e79303..51c3154 100644 --- a/cget/util.py +++ b/cget/util.py @@ -217,10 +217,15 @@ def http_error_default(self, url, fp, errcode, errmsg, headers): raise BuildError("Download failed with error {0} for: {1}".format(errcode, url)) return request.FancyURLopener.http_error_default(self, url, fp, errcode, errmsg, headers) -def download_to(url, download_dir, insecure=False): +def download_to(url, download_dir, insecure=False, download_tool=None): name = url.split('/')[-1] file = os.path.join(download_dir, name) display.info("Downloading [bold]{}[/bold]".format(url)) + + # Use external tool if specified + if download_tool: + return download_with_tool(url, file, download_tool, insecure) + with display.create_download_progress() as progress: task = progress.add_task(name, total=None) def hook(count, block_size, total_size): @@ -237,17 +242,46 @@ def hook(count, block_size, total_size): raise BuildError("Download failed for: {0}".format(url)) return file +def download_with_tool(url, file, tool, insecure=False): + """Download using external tool (wget or curl)""" + tool = tool.lower() + cmd_args = None + + if tool == 'wget': + cmd_args = ['wget', '-O', file, url] + if insecure: + cmd_args.insert(1, '--no-check-certificate') + elif tool == 'curl': + cmd_args = ['curl', '-L', '-o', file, url] + if insecure: + cmd_args.insert(1, '-k') + else: + raise BuildError("Unsupported download tool: {}. Supported tools: wget, curl".format(tool)) + + display.info("Using {} to download".format(tool)) + try: + child = subprocess.Popen(cmd_args) + child.communicate() + if child.returncode != 0: + raise BuildError("Download failed with {} for: {}".format(tool, url)) + except FileNotFoundError: + raise BuildError("Download tool '{}' not found. Please install it first.".format(tool)) + + if not os.path.exists(file): + raise BuildError("Download failed for: {0}".format(url)) + return file + def transfer_to(f, dst, copy=False): if USE_SYMLINKS and not copy: return symlink_to(f, dst) else: return copy_to(f, dst) -def retrieve_url(url, dst, copy=False, insecure=False, hash=None): +def retrieve_url(url, dst, copy=False, insecure=False, hash=None, download_tool=None): remote = not url.startswith('file://') # Retrieve from cache if remote and hash: f = get_cache_file(hash.replace(':', '-')) if f: return f - f = download_to(url, dst, insecure=insecure) if remote else transfer_to(url[7:], dst, copy=copy) + f = download_to(url, dst, insecure=insecure, download_tool=download_tool) if remote else transfer_to(url[7:], dst, copy=copy) if os.path.isfile(f) and hash: with display.status("Computing hash..."): result = check_hash(f, hash)