Skip to content

Commit 1368016

Browse files
authored
fix wget download [cherry-pick #59957] (#60029)
* fix wget download * fix wget
1 parent e06bb2f commit 1368016

File tree

1 file changed

+24
-14
lines changed

1 file changed

+24
-14
lines changed

python/paddle/utils/download.py

Lines changed: 24 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import tarfile
2222
import time
2323
import zipfile
24+
from urllib.parse import urlparse
2425

2526
import httpx
2627

@@ -196,22 +197,31 @@ def _get_download(url, fullname):
196197
return False
197198

198199

199-
def _wget_download(url, fullname):
200-
# using wget to download url
201-
tmp_fullname = fullname + "_tmp"
202-
# –user-agent
203-
command = f'wget -O {tmp_fullname} -t {DOWNLOAD_RETRY_LIMIT} {url}'
204-
subprc = subprocess.Popen(
205-
command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE
206-
)
207-
_ = subprc.communicate()
208-
209-
if subprc.returncode != 0:
210-
raise RuntimeError(
211-
f'{command} failed. Please make sure `wget` is installed or {url} exists'
200+
def _wget_download(url: str, fullname: str):
201+
try:
202+
assert urlparse(url).scheme in (
203+
'http',
204+
'https',
205+
), 'Only support https and http url'
206+
# using wget to download url
207+
tmp_fullname = fullname + "_tmp"
208+
# –user-agent
209+
command = f'wget -O {tmp_fullname} -t {DOWNLOAD_RETRY_LIMIT} {url}'
210+
subprc = subprocess.Popen(
211+
command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE
212212
)
213+
_ = subprc.communicate()
214+
215+
if subprc.returncode != 0:
216+
raise RuntimeError(
217+
f'{command} failed. Please make sure `wget` is installed or {url} exists'
218+
)
219+
220+
shutil.move(tmp_fullname, fullname)
213221

214-
shutil.move(tmp_fullname, fullname)
222+
except Exception as e: # requests.exceptions.ConnectionError
223+
logger.info(f"Downloading {url} failed with exception {str(e)}")
224+
return False
215225

216226
return fullname
217227

0 commit comments

Comments
 (0)