|
21 | 21 | import tarfile
|
22 | 22 | import time
|
23 | 23 | import zipfile
|
| 24 | +from urllib.parse import urlparse |
24 | 25 |
|
25 | 26 | import httpx
|
26 | 27 |
|
@@ -196,22 +197,31 @@ def _get_download(url, fullname):
|
196 | 197 | return False
|
197 | 198 |
|
198 | 199 |
|
199 |
| -def _wget_download(url, fullname): |
200 |
| - # using wget to download url |
201 |
| - tmp_fullname = fullname + "_tmp" |
202 |
| - # –user-agent |
203 |
| - command = f'wget -O {tmp_fullname} -t {DOWNLOAD_RETRY_LIMIT} {url}' |
204 |
| - subprc = subprocess.Popen( |
205 |
| - command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE |
206 |
| - ) |
207 |
| - _ = subprc.communicate() |
208 |
| - |
209 |
| - if subprc.returncode != 0: |
210 |
| - raise RuntimeError( |
211 |
| - f'{command} failed. Please make sure `wget` is installed or {url} exists' |
| 200 | +def _wget_download(url: str, fullname: str): |
| 201 | + try: |
| 202 | + assert urlparse(url).scheme in ( |
| 203 | + 'http', |
| 204 | + 'https', |
| 205 | + ), 'Only support https and http url' |
| 206 | + # using wget to download url |
| 207 | + tmp_fullname = fullname + "_tmp" |
| 208 | + # –user-agent |
| 209 | + command = f'wget -O {tmp_fullname} -t {DOWNLOAD_RETRY_LIMIT} {url}' |
| 210 | + subprc = subprocess.Popen( |
| 211 | + command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE |
212 | 212 | )
|
| 213 | + _ = subprc.communicate() |
| 214 | + |
| 215 | + if subprc.returncode != 0: |
| 216 | + raise RuntimeError( |
| 217 | + f'{command} failed. Please make sure `wget` is installed or {url} exists' |
| 218 | + ) |
| 219 | + |
| 220 | + shutil.move(tmp_fullname, fullname) |
213 | 221 |
|
214 |
| - shutil.move(tmp_fullname, fullname) |
| 222 | + except Exception as e: # requests.exceptions.ConnectionError |
| 223 | + logger.info(f"Downloading {url} failed with exception {str(e)}") |
| 224 | + return False |
215 | 225 |
|
216 | 226 | return fullname
|
217 | 227 |
|
|
0 commit comments