Skip to content

Commit 0e9172b

Browse files
Addressed PR suggestions:
- Fix corrupt samples by re-encoding instead of skipping them - Rename head bbox feature - Update checksums - Bump version - Minor refactoring
1 parent 1ff3092 commit 0e9172b

File tree

3 files changed

+35
-23
lines changed

3 files changed

+35
-23
lines changed
Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,5 @@
11
The Oxford-IIIT pet dataset is a 37 category pet image dataset with roughly 200
22
images for each class. The images have large variations in scale, pose and
3-
lighting. All images have an associated ground truth annotation of breed.
4-
5-
Please take note of the following:
6-
- Valid bounding boxes of the pets' heads are only provided for the training split. The test split does not contain annotations.
7-
- A few image files in the original dataset are corrupt or in a wrong format. These are being skipped.
3+
lighting. All images have an associated ground truth annotation of breed and species.
4+
Additionally, head bounding boxes are provided for the training split, allowing using this dataset for simple
5+
object detection tasks. In the test split, the bounding boxes are empty.
Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
1-
http://www.robots.ox.ac.uk/~vgg/data/pets/data/annotations.tar.gz 19173078 52425fb6de5c424942b7626b428656fcbd798db970a937df61750c0f1d358e91 annotations.tar.gz
2-
http://www.robots.ox.ac.uk/~vgg/data/pets/data/images.tar.gz 791918971 67195c5e1c01f1ab5f9b6a5d22b8c27a580d896ece458917e61d459337fa318d images.tar.gz
1+
https://thor.robots.ox.ac.uk/~vgg/data/pets/annotations.tar.gz 19173078 52425fb6de5c424942b7626b428656fcbd798db970a937df61750c0f1d358e91 annotations.tar.gz
2+
https://thor.robots.ox.ac.uk/~vgg/data/pets/images.tar.gz 791918971 67195c5e1c01f1ab5f9b6a5d22b8c27a580d896ece458917e61d459337fa318d images.tar.gz

tensorflow_datasets/datasets/oxford_iiit_pet/oxford_iiit_pet_dataset_builder.py

Lines changed: 30 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
import os
1919
import xml.etree.ElementTree
2020

21+
from etils.epath import Path
2122
from tensorflow_datasets.core.utils.lazy_imports_utils import tensorflow as tf
2223
import tensorflow_datasets.public_api as tfds
2324

@@ -64,8 +65,8 @@
6465
]
6566
_SPECIES_CLASSES = ["Cat", "Dog"]
6667

67-
# List of samples with corrupt image files
68-
_SKIP_SAMPLES = [
68+
# List of samples with corrupt image files (mostly wrong format -> we are fixing these during dataset creation)
69+
_CORRUPT_SAMPLES = [
6970
"beagle_116",
7071
"chihuahua_121",
7172
"Abyssinian_5",
@@ -80,15 +81,15 @@
8081
"Egyptian_Mau_191"
8182
]
8283

84+
_EMPTY_BBOX = tfds.features.BBox(0., 0., 0., 0.)
85+
86+
8387
def _get_head_bbox(annon_filepath):
8488
"""Read head bbox from annotation XML file."""
85-
with tf.io.gfile.GFile(annon_filepath, "r") as f:
89+
with Path(annon_filepath).open("r") as f:
8690
root = xml.etree.ElementTree.parse(f).getroot()
8791

88-
# Disable pytype to avoid attribute-error due to find returning
89-
# Optional[Element]
90-
# pytype: disable=attribute-error
91-
size = root.find("size")
92+
size = root.find("size") # pytype: disable=annotation-type-mismatch
9293
width = float(size.find("width").text)
9394
height = float(size.find("height").text)
9495

@@ -106,7 +107,10 @@ def _get_head_bbox(annon_filepath):
106107
class Builder(tfds.core.GeneratorBasedBuilder):
107108
"""Oxford-IIIT pet dataset."""
108109

109-
VERSION = tfds.core.Version("3.2.0")
110+
VERSION = tfds.core.Version("4.0.0")
111+
RELEASE_NOTES = {
112+
'4.0.0': 'Add head bounding boxes. Fix corrupt iamges. Update dataset URL.'
113+
}
110114

111115
def _info(self):
112116
return self.dataset_info_from_configs(
@@ -118,7 +122,7 @@ def _info(self):
118122
"segmentation_mask": tfds.features.Image(
119123
shape=(None, None, 1), use_colormap=True
120124
),
121-
"head": tfds.features.BBoxFeature()
125+
"head_bbox": tfds.features.BBoxFeature()
122126
}),
123127
supervised_keys=("image", "label"),
124128
homepage="http://www.robots.ox.ac.uk/~vgg/data/pets/",
@@ -162,13 +166,23 @@ def _split_generators(self, dl_manager):
162166
def _generate_examples(
163167
self, images_dir_path, annotations_dir_path, images_list_file
164168
):
165-
with tf.io.gfile.GFile(images_list_file, "r") as images_list:
169+
with Path(images_list_file).open("r") as images_list:
166170
for line in images_list:
167171
image_name, label, species, _ = line.strip().split(" ")
168172

169-
# skip corrupt samples
170-
if image_name in _SKIP_SAMPLES:
171-
continue
173+
image_path = os.path.join(images_dir_path, image_name + ".jpg")
174+
175+
if image_name in _CORRUPT_SAMPLES:
176+
# some images caused 'Corrupt JPEG data...' messages during training or any other iteration
177+
# recoding them once fixes the issue (discussion: https://github.com/tensorflow/datasets/issues/2188)
178+
with Path(image_path).open("rb") as image_file:
179+
img_data = image_file.read()
180+
img_tensor = tf.image.decode_image(img_data)
181+
if tf.shape(img_tensor)[-1] == 4: # some files have an alpha channel -> remove
182+
img_tensor = img_tensor[:, :, :-1]
183+
img_recoded = tf.io.encode_jpeg(img_tensor)
184+
with Path(image_path).open("wb") as image_file:
185+
image_file.write(img_recoded.numpy())
172186

173187
trimaps_dir_path = os.path.join(annotations_dir_path, "trimaps")
174188
xmls_dir_path = os.path.join(annotations_dir_path, "xmls")
@@ -181,16 +195,16 @@ def _generate_examples(
181195

182196
try:
183197
head_bbox = _get_head_bbox(os.path.join(xmls_dir_path, xml_name))
184-
except tf.errors.NotFoundError:
198+
except FileNotFoundError as e:
185199
# test samples do not have an annotation file
186-
head_bbox = tfds.features.BBox(0., 0., 0., 0.)
200+
head_bbox = _EMPTY_BBOX
187201

188202
record = {
189203
"image": os.path.join(images_dir_path, image_name),
190204
"label": int(label),
191205
"species": species,
192206
"file_name": image_name,
193207
"segmentation_mask": os.path.join(trimaps_dir_path, trimap_name),
194-
"head": head_bbox
208+
"head_bbox": head_bbox
195209
}
196210
yield image_name, record

0 commit comments

Comments
 (0)