Skip to content

Commit bcbfbdd

Browse files
wyliwyli
authored
fixes #3708 (#3814)
Signed-off-by: Wenqi Li <[email protected]>
1 parent 894e989 commit bcbfbdd

11 files changed

+151
-52
lines changed

‎tests/test_download_and_extract.py‎

Lines changed: 15 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -15,21 +15,24 @@
1515
from pathlib import Path
1616
from urllib.error import ContentTooShortError, HTTPError
1717

18+
from parameterized import parameterized
19+
1820
from monai.apps import download_and_extract, download_url, extractall
19-
from tests.utils import skip_if_downloading_fails, skip_if_quick
21+
from tests.utils import skip_if_downloading_fails, skip_if_quick, testing_data_config
2022

2123

2224
class TestDownloadAndExtract(unittest.TestCase):
2325
@skip_if_quick
2426
def test_actions(self):
2527
testing_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "testing_data")
26-
url = "https://drive.google.com/uc?id=1QsnnkvZyJPcbRoV_ArW8SnE1OTuoVbKE"
28+
config_dict = testing_data_config("images", "mednist")
29+
url = config_dict["url"]
2730
filepath = Path(testing_dir) / "MedNIST.tar.gz"
2831
output_dir = Path(testing_dir)
29-
md5_value = "0bc7306e7427e00ad1c5526a6677552d"
32+
hash_val, hash_type = config_dict["hash_val"], config_dict["hash_type"]
3033
with skip_if_downloading_fails():
31-
download_and_extract(url, filepath, output_dir, md5_value)
32-
download_and_extract(url, filepath, output_dir, md5_value)
34+
download_and_extract(url, filepath, output_dir, hash_val=hash_val, hash_type=hash_type)
35+
download_and_extract(url, filepath, output_dir, hash_val=hash_val, hash_type=hash_type)
3336

3437
wrong_md5 = "0"
3538
with self.assertLogs(logger="monai.apps", level="ERROR"):
@@ -47,22 +50,17 @@ def test_actions(self):
4750
self.assertTrue(str(e).startswith("md5 check"))
4851

4952
@skip_if_quick
50-
def test_default(self):
53+
@parameterized.expand((("icon", "tar"), ("favicon", "zip")))
54+
def test_default(self, key, file_type):
5155
with tempfile.TemporaryDirectory() as tmp_dir:
5256
with skip_if_downloading_fails():
53-
# icon.tar.gz https://drive.google.com/file/d/1HrQd-AKPbts9jkTNN4pT8vLZyhM5irVn/view?usp=sharing
54-
download_and_extract(
55-
"https://drive.google.com/uc?id=1HrQd-AKPbts9jkTNN4pT8vLZyhM5irVn",
56-
output_dir=tmp_dir,
57-
hash_val="a55d11ad26ed9eb7277905d796205531",
58-
file_type="tar",
59-
)
60-
# favicon.ico.zip https://drive.google.com/file/d/1TqBTJap621NO9arzXRrYi04lr9NTVF8H/view?usp=sharing
57+
img_spec = testing_data_config("images", key)
6158
download_and_extract(
62-
"https://drive.google.com/uc?id=1TqBTJap621NO9arzXRrYi04lr9NTVF8H",
59+
img_spec["url"],
6360
output_dir=tmp_dir,
64-
hash_val="ac6e167ee40803577d98237f2b0241e5",
65-
file_type="zip",
61+
hash_val=img_spec["hash_val"],
62+
hash_type=img_spec["hash_type"],
63+
file_type=file_type,
6664
)
6765

6866

‎tests/test_global_mutual_information_loss.py‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,11 +16,10 @@
1616

1717
from monai import transforms
1818
from monai.losses.image_dissimilarity import GlobalMutualInformationLoss
19-
from tests.utils import SkipIfBeforePyTorchVersion, download_url_or_skip_test, skip_if_quick
19+
from tests.utils import SkipIfBeforePyTorchVersion, download_url_or_skip_test, skip_if_quick, testing_data_config
2020

2121
device = "cuda" if torch.cuda.is_available() else "cpu"
2222

23-
FILE_URL = "https://drive.google.com/uc?id=17tsDLvG_GZm7a4fCVMCv-KyDx0hqq1ji"
2423
FILE_PATH = os.path.join(os.path.dirname(__file__), "testing_data", "temp_" + "mri.nii")
2524

2625
EXPECTED_VALUE = {
@@ -54,7 +53,13 @@
5453
@skip_if_quick
5554
class TestGlobalMutualInformationLoss(unittest.TestCase):
5655
def setUp(self):
57-
download_url_or_skip_test(FILE_URL, FILE_PATH)
56+
config = testing_data_config("images", "Prostate_T2W_AX_1")
57+
download_url_or_skip_test(
58+
url=config["url"],
59+
filepath=FILE_PATH,
60+
hash_val=config.get("hash_val"),
61+
hash_type=config.get("hash_type", "sha256"),
62+
)
5863

5964
@SkipIfBeforePyTorchVersion((1, 9))
6065
def test_bspline(self):

‎tests/test_integration_classification_2d.py‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -38,10 +38,8 @@
3838
)
3939
from monai.utils import set_determinism
4040
from tests.testing_data.integration_answers import test_integration_value
41-
from tests.utils import DistTestCase, TimedCall, skip_if_downloading_fails, skip_if_quick
41+
from tests.utils import DistTestCase, TimedCall, skip_if_downloading_fails, skip_if_quick, testing_data_config
4242

43-
TEST_DATA_URL = "https://drive.google.com/uc?id=1QsnnkvZyJPcbRoV_ArW8SnE1OTuoVbKE"
44-
MD5_VALUE = "0bc7306e7427e00ad1c5526a6677552d"
4543
TASK = "integration_classification_2d"
4644

4745

@@ -186,7 +184,14 @@ def setUp(self):
186184

187185
if not os.path.exists(data_dir):
188186
with skip_if_downloading_fails():
189-
download_and_extract(TEST_DATA_URL, dataset_file, self.data_dir, MD5_VALUE)
187+
data_spec = testing_data_config("images", "mednist")
188+
download_and_extract(
189+
data_spec["url"],
190+
dataset_file,
191+
self.data_dir,
192+
hash_val=data_spec["hash_val"],
193+
hash_type=data_spec["hash_type"],
194+
)
190195

191196
assert os.path.exists(data_dir)
192197

‎tests/test_masked_inference_wsi_dataset.py‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,14 +19,15 @@
1919

2020
from monai.apps.pathology.data import MaskedInferenceWSIDataset
2121
from monai.utils import optional_import
22-
from tests.utils import download_url_or_skip_test, skip_if_quick
22+
from tests.utils import download_url_or_skip_test, skip_if_quick, testing_data_config
2323

2424
_, has_cim = optional_import("cucim", name="CuImage")
2525
_, has_osl = optional_import("openslide")
2626

27-
FILE_URL = "https://drive.google.com/uc?id=1sGTKZlJBIz53pfqTxoTqiIQzIoEzHLAe"
28-
base_name, extension = FILE_URL.split("id=")[1], ".tiff"
29-
FILE_NAME = "temp_" + base_name
27+
FILE_KEY = "wsi_img"
28+
FILE_URL = testing_data_config("images", FILE_KEY, "url")
29+
base_name, extension = os.path.basename(f"{FILE_URL}"), ".tiff"
30+
FILE_NAME = f"temp_{base_name}"
3031
FILE_PATH = os.path.join(os.path.dirname(__file__), "testing_data", FILE_NAME + extension)
3132

3233
MASK1 = os.path.join(os.path.dirname(__file__), "testing_data", "temp_tissue_mask1.npy")
@@ -159,7 +160,9 @@ def prepare_data():
159160
class TestMaskedInferenceWSIDataset(unittest.TestCase):
160161
def setUp(self):
161162
prepare_data()
162-
download_url_or_skip_test(FILE_URL, FILE_PATH, "5a3cfd4fd725c50578ddb80b517b759f")
163+
hash_type = testing_data_config("images", FILE_KEY, "hash_type")
164+
hash_val = testing_data_config("images", FILE_KEY, "hash_val")
165+
download_url_or_skip_test(FILE_URL, FILE_PATH, hash_type=hash_type, hash_val=hash_val)
163166

164167
@parameterized.expand([TEST_CASE_0, TEST_CASE_1, TEST_CASE_2, TEST_CASE_3, TEST_CASE_4])
165168
@skipUnless(has_cim, "Requires CuCIM")

‎tests/test_patch_wsi_dataset.py‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -19,14 +19,15 @@
1919

2020
from monai.apps.pathology.data import PatchWSIDataset
2121
from monai.utils import optional_import
22-
from tests.utils import download_url_or_skip_test
22+
from tests.utils import download_url_or_skip_test, testing_data_config
2323

2424
_cucim, has_cim = optional_import("cucim")
2525
has_cim = has_cim and hasattr(_cucim, "CuImage")
2626
_, has_osl = optional_import("openslide")
2727

28-
FILE_URL = "https://drive.google.com/uc?id=1sGTKZlJBIz53pfqTxoTqiIQzIoEzHLAe"
29-
base_name, extension = FILE_URL.split("id=")[1], ".tiff"
28+
FILE_KEY = "wsi_img"
29+
FILE_URL = testing_data_config("images", FILE_KEY, "url")
30+
base_name, extension = os.path.basename(f"{FILE_URL}"), ".tiff"
3031
FILE_PATH = os.path.join(os.path.dirname(__file__), "testing_data", "temp_" + base_name + extension)
3132

3233
TEST_CASE_0 = [
@@ -108,7 +109,9 @@
108109

109110
class TestPatchWSIDataset(unittest.TestCase):
110111
def setUp(self):
111-
download_url_or_skip_test(FILE_URL, FILE_PATH, "5a3cfd4fd725c50578ddb80b517b759f")
112+
hash_type = testing_data_config("images", FILE_KEY, "hash_type")
113+
hash_val = testing_data_config("images", FILE_KEY, "hash_val")
114+
download_url_or_skip_test(FILE_URL, FILE_PATH, hash_type=hash_type, hash_val=hash_val)
112115

113116
@parameterized.expand([TEST_CASE_0, TEST_CASE_1, TEST_CASE_2, TEST_CASE_3])
114117
@skipUnless(has_cim, "Requires CuCIM")

‎tests/test_senet.py‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@
2121
from monai.networks import eval_mode
2222
from monai.networks.nets import SENet154, SEResNet50, SEResNet101, SEResNet152, SEResNext50, SEResNext101
2323
from monai.utils import optional_import
24-
from tests.utils import test_is_quick, test_pretrained_networks, test_script_save
24+
from tests.utils import test_is_quick, test_pretrained_networks, test_script_save, testing_data_config
2525

2626
if TYPE_CHECKING:
2727
import pretrainedmodels
@@ -69,27 +69,27 @@ def setUp(self):
6969
testing_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "testing_data")
7070
testing_data_urls = {
7171
"senet154": {
72-
"url": "https://drive.google.com/uc?id=1e10LFGVIV9L8_Q5Fhwi3X5nU6mDRrCDh",
72+
"url": testing_data_config("models", "senet154-c7b49a05", "url"),
7373
"filename": "senet154-c7b49a05.pth",
7474
},
7575
"se_resnet50": {
76-
"url": "https://drive.google.com/uc?id=1WCeveS0tvjta4Wcp1wAGRi_uyXRfXAGA",
76+
"url": testing_data_config("models", "se_resnet50-ce0d4300", "url"),
7777
"filename": "se_resnet50-ce0d4300.pth",
7878
},
7979
"se_resnet101": {
80-
"url": "https://drive.google.com/uc?id=1Bh0PmLISUltsY8FevtlTbt6vT35clzWg",
80+
"url": testing_data_config("models", "se_resnet101-7e38fcc6", "url"),
8181
"filename": "se_resnet101-7e38fcc6.pth",
8282
},
8383
"se_resnet152": {
84-
"url": "https://drive.google.com/uc?id=1fcqpP0ITOcALy_TZAcBdkyf7HcH687J-",
84+
"url": testing_data_config("models", "se_resnet152-d17c99b7", "url"),
8585
"filename": "se_resnet152-d17c99b7.pth",
8686
},
8787
"se_resnext50_32x4d": {
88-
"url": "https://drive.google.com/uc?id=1kRKW8YjGaEwYdQUyhoCIDg1H9ZAoJ-jI",
88+
"url": testing_data_config("models", "se_resnext50_32x4d-a260b3a4", "url"),
8989
"filename": "se_resnext50_32x4d-a260b3a4.pth",
9090
},
9191
"se_resnext101_32x4d": {
92-
"url": "https://drive.google.com/uc?id=1Tg6Zim1lXgmYgH7FyTXAgihbkq5Jegni",
92+
"url": testing_data_config("models", "se_resnext101_32x4d-3b2fe3d8", "url"),
9393
"filename": "se_resnext101_32x4d-3b2fe3d8.pth",
9494
},
9595
}

‎tests/test_smartcache_patch_wsi_dataset.py‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -19,13 +19,14 @@
1919

2020
from monai.apps.pathology.data import SmartCachePatchWSIDataset
2121
from monai.utils import optional_import
22-
from tests.utils import download_url_or_skip_test
22+
from tests.utils import download_url_or_skip_test, testing_data_config
2323

2424
_cucim, has_cim = optional_import("cucim")
2525
has_cim = has_cim and hasattr(_cucim, "CuImage")
2626

27-
FILE_URL = "https://drive.google.com/uc?id=1sGTKZlJBIz53pfqTxoTqiIQzIoEzHLAe"
28-
base_name, extension = FILE_URL.split("id=")[1], ".tiff"
27+
FILE_KEY = "wsi_img"
28+
FILE_URL = testing_data_config("images", FILE_KEY, "url")
29+
base_name, extension = os.path.basename(f"{FILE_URL}"), ".tiff"
2930
FILE_PATH = os.path.join(os.path.dirname(__file__), "testing_data", "temp_" + base_name + extension)
3031

3132
TEST_CASE_0 = [
@@ -134,7 +135,9 @@
134135

135136
class TestSmartCachePatchWSIDataset(unittest.TestCase):
136137
def setUp(self):
137-
download_url_or_skip_test(FILE_URL, FILE_PATH, "5a3cfd4fd725c50578ddb80b517b759f")
138+
hash_type = testing_data_config("images", FILE_KEY, "hash_type")
139+
hash_val = testing_data_config("images", FILE_KEY, "hash_val")
140+
download_url_or_skip_test(FILE_URL, FILE_PATH, hash_type=hash_type, hash_val=hash_val)
138141

139142
@parameterized.expand([TEST_CASE_0, TEST_CASE_1, TEST_CASE_2])
140143
@skipUnless(has_cim, "Requires CuCIM")

‎tests/test_warp.py‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
from monai.networks.blocks.warp import Warp
2121
from monai.transforms import LoadImaged
2222
from monai.utils import GridSampleMode, GridSamplePadMode
23-
from tests.utils import SkipIfBeforePyTorchVersion, SkipIfNoModule, download_url_or_skip_test
23+
from tests.utils import SkipIfBeforePyTorchVersion, SkipIfNoModule, download_url_or_skip_test, testing_data_config
2424

2525
LOW_POWER_TEST_CASES = [ # run with BUILD_MONAI=1 to test csrc/resample, BUILD_MONAI=0 to test native grid_sample
2626
[
@@ -98,7 +98,13 @@
9898

9999
class TestWarp(unittest.TestCase):
100100
def setUp(self):
101-
download_url_or_skip_test(FILE_URL, FILE_PATH)
101+
config = testing_data_config("images", "Prostate_T2W_AX_1")
102+
download_url_or_skip_test(
103+
url=config["url"],
104+
filepath=FILE_PATH,
105+
hash_val=config.get("hash_val"),
106+
hash_type=config.get("hash_type", "sha256"),
107+
)
102108

103109
@SkipIfNoModule("itk")
104110
def test_itk_benchmark(self):
@@ -141,7 +147,6 @@ def test_grad(self):
141147
gradcheck(warp_layer, (input_image, ddf), atol=1e-2, eps=1e-2)
142148

143149

144-
FILE_URL = "https://drive.google.com/uc?id=17tsDLvG_GZm7a4fCVMCv-KyDx0hqq1ji"
145150
FILE_PATH = os.path.join(os.path.dirname(__file__), "testing_data", "temp_" + "mri.nii")
146151

147152

‎tests/test_wsireader.py‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@
2323
from monai.transforms import Compose, LoadImaged, ToTensord
2424
from monai.utils import first, optional_import
2525
from monai.utils.enums import PostFix
26-
from tests.utils import download_url_or_skip_test
26+
from tests.utils import download_url_or_skip_test, testing_data_config
2727

2828
cucim, has_cucim = optional_import("cucim")
2929
has_cucim = has_cucim and hasattr(cucim, "CuImage")
@@ -32,8 +32,9 @@
3232
_, has_codec = optional_import("imagecodecs")
3333
has_tiff = has_tiff and has_codec
3434

35-
FILE_URL = "https://drive.google.com/uc?id=1sGTKZlJBIz53pfqTxoTqiIQzIoEzHLAe"
36-
base_name, extension = FILE_URL.split("id=")[1], ".tiff"
35+
FILE_KEY = "wsi_img"
36+
FILE_URL = testing_data_config("images", FILE_KEY, "url")
37+
base_name, extension = os.path.basename(f"{FILE_URL}"), ".tiff"
3738
FILE_PATH = os.path.join(os.path.dirname(__file__), "testing_data", "temp_" + base_name + extension)
3839

3940
HEIGHT = 32914
@@ -104,7 +105,9 @@ def save_rgba_tiff(array: np.ndarray, filename: str, mode: str):
104105

105106
@skipUnless(has_cucim or has_osl or has_tiff, "Requires cucim, openslide, or tifffile!")
106107
def setUpModule(): # noqa: N802
107-
download_url_or_skip_test(FILE_URL, FILE_PATH, "5a3cfd4fd725c50578ddb80b517b759f")
108+
hash_type = testing_data_config("images", FILE_KEY, "hash_type")
109+
hash_val = testing_data_config("images", FILE_KEY, "hash_val")
110+
download_url_or_skip_test(FILE_URL, FILE_PATH, hash_type=hash_type, hash_val=hash_val)
108111

109112

110113
class WSIReaderTests:

‎tests/testing_data/data_config.json‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
{
2+
"images": {
3+
"wsi_img": {
4+
"url": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/CMU-1.tiff",
5+
"hash_type": "sha256",
6+
"hash_val": "73a7e89bc15576587c3d68e55d9bf92f09690280166240b48ff4b48230b13bcd"
7+
},
8+
"favicon": {
9+
"url": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/favicon.ico.zip",
10+
"hash_type": "sha256",
11+
"hash_val": "3a3635c8d8adb81feebc5926b4106e8eb643a24a4be2a69a9d35f9a578acadb5"
12+
},
13+
"icon": {
14+
"url": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/icon.tar.gz",
15+
"hash_type": "sha256",
16+
"hash_val": "90f24cd8f20f3932624da95190ce384302261acf0ea15b358f7832e3b6becac0"
17+
},
18+
"mednist": {
19+
"url": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/MedNIST.tar.gz",
20+
"hash_type": "sha256",
21+
"hash_val": "f2f4881ff8799a170b10a403495f0ce0ad7486491901cde67a647e6627e7f916"
22+
},
23+
"Prostate_T2W_AX_1": {
24+
"url": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/Prostate_T2W_AX_1.nii",
25+
"hash_type": "sha256",
26+
"hash_val": "a14231f539c0f365a5f83f2a046969a9b9870e56ffd126fd8e7242364d25938a"
27+
}
28+
},
29+
"models": {
30+
"senet154-c7b49a05": {
31+
"url": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/senet154-c7b49a05.pth",
32+
"hash_type": "sha256",
33+
"hash_val": "c7b49a056b98b0bed65b0237c27acdead655e599669215573d357ad337460413"
34+
},
35+
"se_resnet101-7e38fcc6": {
36+
"url": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/se_resnet101-7e38fcc6.pth",
37+
"hash_type": "sha256",
38+
"hash_val": "7e38fcc64eff3225a3ea4e6081efeb6087e8d5a61c204d94edc2ed1aab0b9d70"
39+
},
40+
"se_resnet152-d17c99b7": {
41+
"url": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/se_resnet152-d17c99b7.pth",
42+
"hash_type": "sha256",
43+
"hash_val": "d17c99b703dcca2d2507ddfb68f72625a2f7e23ee64396eb992f1b2cf7e6bdc1"
44+
},
45+
"se_resnet50-ce0d4300": {
46+
"url": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/se_resnet50-ce0d4300.pth",
47+
"hash_type": "sha256",
48+
"hash_val": "ce0d430017d3f4aa6b5658c72209f3bfffb060207fd26a2ef0b203ce592eba01"
49+
},
50+
"se_resnext101_32x4d-3b2fe3d8": {
51+
"url": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/se_resnext101_32x4d-3b2fe3d8.pth",
52+
"hash_type": "sha256",
53+
"hash_val": "3b2fe3d8acb8de7d5976c4baf518f24a0237509272a69366e816682d3e57b989"
54+
},
55+
"se_resnext50_32x4d-a260b3a4": {
56+
"url": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/se_resnext50_32x4d-a260b3a4.pth",
57+
"hash_type": "sha256",
58+
"hash_val": "a260b3a40f82dfe37c58d26a612bcf7bef0d27c6fed096226b0e4e9fb364168e"
59+
}
60+
}
61+
}

0 commit comments

Comments
 (0)