Skip to content

Commit 7ef86fc

Browse files
hanjr92hanjr92
authored
update files/datasets download info (#65)
1 parent 0f5ec37 commit 7ef86fc

File tree

13 files changed

+68
-29
lines changed

13 files changed

+68
-29
lines changed

‎tensorlayerx/dataflow/dataloader.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -68,12 +68,12 @@ def __init__(
6868
):
6969
self.dataset = dataset
7070
assert num_workers >= 0, "num_workers should be a non_negative integer"
71-
if num_workers == 0 and prefetch_factor != 2:
72-
raise ValueError("prefetch_factor option should not be specified, when num_workers is 0.")
71+
# if num_workers == 0 and prefetch_factor != 2:
72+
# raise ValueError("prefetch_factor option should not be specified, when num_workers is 0.")
7373
if persistent_workers and num_workers == 0:
7474
raise ValueError('persistent_workers option needs num_workers > 0')
75-
self.num_workers = num_workers
76-
self.prefetch_factor = prefetch_factor
75+
self.num_workers = 0 # TODO optimizer multiprocess in multi backends
76+
self.prefetch_factor = 2
7777
self.time_out = time_out
7878
self.worker_init_fn = worker_init_fn
7979
if isinstance(dataset, IterableDataset):

‎tensorlayerx/files/dataset_loaders/celebA_dataset.py‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,9 @@
33

44
import os
55
import zipfile
6-
76
from tensorlayerx import logging
87
from tensorlayerx.files.utils import (download_file_from_google_drive, exists_or_mkdir, load_file_list)
9-
8+
logging.set_verbosity(logging.INFO)
109
__all__ = ['load_celebA_dataset']
1110

1211

@@ -21,6 +20,11 @@ def load_celebA_dataset(path='data'):
2120
The path that the data is downloaded to, defaults is ``data/celebA/``.
2221
2322
"""
23+
logging.info("The dataset is stored on google drive, if you can't download it from google drive, "
24+
"please download it from the official website manually. "
25+
"Large-scale CelebFaces Attributes (CelebA) Dataset <http://mmlab.ie.cuhk.edu.hk/projects/CelebA.html>. "
26+
"Please place dataset 'img_align_celeba.zip' under 'data/celebA/' by default.")
27+
2428
data_dir = 'celebA'
2529
filename, drive_id = "img_align_celeba.zip", "0B7EVK8r0v71pZjFTYXZWM3FlRnM"
2630
save_path = os.path.join(path, filename)

‎tensorlayerx/files/dataset_loaders/cifar10_dataset.py‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,11 @@
44
import os
55
import pickle
66
import sys
7-
87
import numpy as np
98

109
from tensorlayerx import logging
1110
from tensorlayerx.files.utils import maybe_download_and_extract
12-
11+
logging.set_verbosity(logging.INFO)
1312
__all__ = ['load_cifar10_dataset']
1413

1514

@@ -46,7 +45,10 @@ def load_cifar10_dataset(shape=(-1, 32, 32, 3), path='data', plotable=False):
4645
4746
"""
4847
path = os.path.join(path, 'cifar10')
49-
logging.info("Load or Download cifar10 > {}".format(path))
48+
logging.info("If can't download this dataset automatically, "
49+
"please download it from the official website manually."
50+
"CIFAR10 Dataset <https://www.cs.toronto.edu/~kriz/cifar.html>."
51+
"Please place dataset 'cifar-10-python.tar.gz' under 'data/cifar10/' by default.")
5052

5153
#Helper function to unpickle the data
5254
def unpickle(file):

‎tensorlayerx/files/dataset_loaders/cyclegan_dataset.py‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
from tensorlayerx import logging
99
from tensorlayerx.vision import load_images
1010
from tensorlayerx.files.utils import (del_file, folder_exists, load_file_list, maybe_download_and_extract)
11-
11+
logging.set_verbosity(logging.INFO)
1212
__all__ = ['load_cyclegan_dataset']
1313

1414

@@ -30,6 +30,11 @@ def load_cyclegan_dataset(filename='summer2winter_yosemite', path='data'):
3030
path = os.path.join(path, 'cyclegan')
3131
url = 'https://people.eecs.berkeley.edu/~taesung_park/CycleGAN/datasets/'
3232

33+
logging.info("If can't download this dataset automatically, "
34+
"please download it from the official website manually."
35+
"cyclegan Dataset <https://people.eecs.berkeley.edu/~taesung_park/CycleGAN/datasets/>."
36+
"Please place dataset under 'data/cyclegan/' by default.")
37+
3338
if folder_exists(os.path.join(path, filename)) is False:
3439
logging.info("[*] {} is nonexistent in {}".format(filename, path))
3540
maybe_download_and_extract(filename + '.zip', path, url, extract=True)

‎tensorlayerx/files/dataset_loaders/flickr_1M_dataset.py‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,8 @@
44
import os
55

66
from tensorlayerx import logging
7-
from tensorlayerx.vision import load_images
8-
7+
from tensorlayerx.utils import visualize
8+
logging.set_verbosity(logging.INFO)
99
from tensorlayerx.files.utils import (
1010
del_file, folder_exists, load_file_list, load_folder_list, maybe_download_and_extract, read_file
1111
)
@@ -30,7 +30,7 @@ def load_flickr1M_dataset(tag='sky', size=10, path="data", n_threads=50, printab
3030
size : int
3131
integer between 1 to 10. 1 means 100k images ... 5 means 500k images, 10 means all 1 million images. Default is 10.
3232
path : str
33-
The path that the data is downloaded to, defaults is ``data/flickr25k/``.
33+
The path that the data is downloaded to, defaults is ``data/flickr1M/``.
3434
n_threads : int
3535
The number of thread to read image.
3636
printable : boolean
@@ -57,7 +57,10 @@ def load_flickr1M_dataset(tag='sky', size=10, path="data", n_threads=50, printab
5757
]
5858
tag_zip = 'tags.zip'
5959
url = 'http://press.liacs.nl/mirflickr/mirflickr1m/'
60-
60+
logging.info("If can't download this dataset automatically, "
61+
"please download it from the official website manually."
62+
"flickr1M Dataset <http://press.liacs.nl/mirflickr/mirflickr1m/>."
63+
"Please place dataset under 'data/flickr1M/' by default.")
6164
# download dataset
6265
for image_zip in images_zip[0:size]:
6366
image_folder = image_zip.split(".")[0]

‎tensorlayerx/files/dataset_loaders/flickr_25k_dataset.py‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55

66
from tensorlayerx import logging
77
from tensorlayerx.vision import load_images
8-
8+
logging.set_verbosity(logging.INFO)
99
from tensorlayerx.files.utils import (
1010
del_file, folder_exists, load_file_list, maybe_download_and_extract, natural_keys, read_file
1111
)
@@ -49,7 +49,10 @@ def load_flickr25k_dataset(tag='sky', path="data", n_threads=50, printable=False
4949

5050
filename = 'mirflickr25k.zip'
5151
url = 'http://press.liacs.nl/mirflickr/mirflickr25k/'
52-
52+
logging.info("If can't download this dataset automatically, "
53+
"please download it from the official website manually."
54+
"flickr25k Dataset <http://press.liacs.nl/mirflickr/mirflickr25k/>."
55+
"Please place dataset under 'data/flickr25k/' by default.")
5356
# download dataset
5457
if folder_exists(os.path.join(path, "mirflickr")) is False:
5558
logging.info("[*] Flickr25k is nonexistent in {}".format(path))

‎tensorlayerx/files/dataset_loaders/imdb_dataset.py‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,9 @@
66

77
import numpy as np
88
import six.moves.cPickle as pickle
9-
9+
from tensorlayerx import logging
1010
from tensorlayerx.files.utils import maybe_download_and_extract
11-
11+
logging.set_verbosity(logging.INFO)
1212
__all__ = ['load_imdb_dataset']
1313

1414

@@ -52,7 +52,10 @@ def load_imdb_dataset(
5252
5353
"""
5454
path = os.path.join(path, 'imdb')
55-
55+
logging.info("If can't download this dataset automatically, "
56+
"please download it from the official website manually."
57+
"imdb Dataset <https://s3.amazonaws.com/text-datasets/imdb.pkl>."
58+
"Please place dataset under 'data/imdb/' by default.")
5659
filename = "imdb.pkl"
5760
url = 'https://s3.amazonaws.com/text-datasets/'
5861
maybe_download_and_extract(filename, path, url)

‎tensorlayerx/files/dataset_loaders/matt_mahoney_dataset.py‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66

77
from tensorlayerx import logging
88
from tensorlayerx.files.utils import maybe_download_and_extract
9-
9+
logging.set_verbosity(logging.INFO)
1010
__all__ = ['load_matt_mahoney_text8_dataset']
1111

1212

@@ -35,7 +35,10 @@ def load_matt_mahoney_text8_dataset(path='data'):
3535
3636
"""
3737
path = os.path.join(path, 'mm_test8')
38-
logging.info("Load or Download matt_mahoney_text8 Dataset> {}".format(path))
38+
logging.info("If can't download this dataset automatically, "
39+
"please download it from the official website manually."
40+
"mm_test8 Dataset <http://mattmahoney.net/dc/>."
41+
"Please place dataset under 'data/mm_test8/' by default.")
3942

4043
filename = 'text8.zip'
4144
url = 'http://mattmahoney.net/dc/'

‎tensorlayerx/files/dataset_loaders/mnist_dataset.py‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,8 @@
22
# -*- coding: utf-8 -*-
33

44
from tensorlayerx.files.utils import _load_mnist_dataset
5-
5+
from tensorlayerx import logging
6+
logging.set_verbosity(logging.INFO)
67
__all__ = ['load_mnist_dataset']
78

89

@@ -28,4 +29,8 @@ def load_mnist_dataset(shape=(-1, 784), path='data'):
2829
>>> X_train, y_train, X_val, y_val, X_test, y_test = tlx.files.load_mnist_dataset(shape=(-1,784), path='datasets')
2930
>>> X_train, y_train, X_val, y_val, X_test, y_test = tlx.files.load_mnist_dataset(shape=(-1, 28, 28, 1))
3031
"""
32+
logging.info("If can't download this dataset automatically, "
33+
"please download it from the official website manually."
34+
"mnist Dataset <http://yann.lecun.com/exdb/mnist/>."
35+
"Please place dataset under 'data/mnist/' by default.")
3136
return _load_mnist_dataset(shape, path, name='mnist', url='http://yann.lecun.com/exdb/mnist/')

‎tensorlayerx/files/dataset_loaders/mnist_fashion_dataset.py‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,8 @@
22
# -*- coding: utf-8 -*-
33

44
from tensorlayerx.files.utils import _load_mnist_dataset
5-
5+
from tensorlayerx import logging
6+
logging.set_verbosity(logging.INFO)
67
__all__ = ['load_fashion_mnist_dataset']
78

89

@@ -28,6 +29,11 @@ def load_fashion_mnist_dataset(shape=(-1, 784), path='data'):
2829
>>> X_train, y_train, X_val, y_val, X_test, y_test = tlx.files.load_fashion_mnist_dataset(shape=(-1,784), path='datasets')
2930
>>> X_train, y_train, X_val, y_val, X_test, y_test = tlx.files.load_fashion_mnist_dataset(shape=(-1, 28, 28, 1))
3031
"""
32+
logging.info("If can't download this dataset automatically, "
33+
"please download it from the official website manually."
34+
"fashion_mnist Dataset <http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/fashion_mnist>."
35+
"Please place dataset under 'data/fashion_mnist/' by default.")
36+
3137
return _load_mnist_dataset(
3238
shape, path, name='fashion_mnist', url='http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/'
3339
)

0 commit comments

Comments
 (0)