Skip to content

Commit 2f9753a

Browse files
authored
If boto3 raises no credentials exception use urlib for downloading the dataset (#11)
1 parent db1896d commit 2f9753a

File tree

1 file changed

+8
-2
lines changed

1 file changed

+8
-2
lines changed

benchmark/dataset.py

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
from dataclasses import dataclass, field
66
from typing import Dict, List, Optional, Union
77
import boto3
8+
import botocore.exceptions
89
from benchmark import DATASETS_DIR
910
from dataset_reader.ann_compound_reader import AnnCompoundReader
1011
from dataset_reader.ann_h5_reader import AnnH5Reader
@@ -109,10 +110,15 @@ def download(self):
109110
return
110111

111112
if self.config.link:
113+
downloaded_withboto = False
112114
if is_s3_link(self.config.link):
113115
print("Use boto3 to download from S3. Faster!")
114-
self._download_from_s3(self.config.link, target_path)
115-
else:
116+
try:
117+
self._download_from_s3(self.config.link, target_path)
118+
downloaded_withboto = True
119+
except botocore.exceptions.NoCredentialsError:
120+
print("Credentials not found, downloading without boto3")
121+
if not downloaded_withboto:
116122
print(f"Downloading from URL {self.config.link}...")
117123
tmp_path, _ = urllib.request.urlretrieve(
118124
self.config.link, None, show_progress

0 commit comments

Comments
 (0)