Skip to content

Commit f36da11

Browse files
MILK-BIOSPillar1989
MILK-BIOS
authored and
Pillar1989
committed
Feat: Add asynchronous dataloader
1 parent 5507bdc commit f36da11

23 files changed

+4885
-344
lines changed

‎configs/swift_yolo/base_arch.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -257,11 +257,12 @@
257257
filter_cfg=dict(filter_empty_gt=False, min_size=32),
258258
pipeline=train_pipeline,
259259
),
260+
async_load=True,
260261
)
261262

262263

263264
test_pipeline = [
264-
dict(type='sscma.LoadImageFromFile', file_client_args=dict(backend='disk')),
265+
dict(type='mmengine.LoadImageFromFile', file_client_args=dict(backend='disk')),
265266
dict(type='sscma.YOLOv5KeepRatioResize', scale=imgsz),
266267
dict(type='sscma.LetterResize', scale=imgsz, allow_scale_up=False, pad_val=dict(img=114)),
267268
dict(type='sscma.LoadAnnotations', with_bbox=True, _scope_='mmdet'),
@@ -287,6 +288,7 @@
287288
pipeline=test_pipeline,
288289
batch_shapes_cfg=batch_shapes_cfg,
289290
),
291+
async_load=True,
290292
)
291293

292294
test_dataloader = val_dataloader

‎configs/swift_yolo/swift_yolo_nano_1xb16_300e_coco.py‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,10 +16,10 @@
1616

1717
# dataset link: https://universe.roboflow.com/team-roboflow/coco-128
1818
data_root = 'https://universe.roboflow.com/ds/z5UOcgxZzD?key=bwx9LQUT0t'
19-
height = 192
20-
width = 192
19+
height = 640
20+
width = 640
2121
batch = 16
22-
workers = 16
22+
workers = 8
2323
use_cached = True
2424
max_cached_images = 4096
2525
val_batch = batch
@@ -106,9 +106,11 @@
106106
*pre_transform,
107107
dict(
108108
type='sscma.Mosaic',
109+
prob=1,
109110
img_scale=imgsz,
110111
pad_val=114.0,
111112
use_cached=use_cached,
113+
pre_transform=pre_transform,
112114
max_cached_images=max_cached_images,
113115
),
114116
dict(
@@ -153,6 +155,7 @@
153155
filter_cfg=dict(filter_empty_gt=False, min_size=32),
154156
pipeline=train_pipeline,
155157
),
158+
async_load=True,
156159
)
157160

158161
test_pipeline = [
@@ -182,6 +185,7 @@
182185
pipeline=test_pipeline,
183186
batch_shapes_cfg=batch_shapes_cfg,
184187
),
188+
async_load=False,
185189
)
186190

187191
test_dataloader = val_dataloader

‎configs/swift_yolo/swift_yolo_shuff_1xb16_300e_coco.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -163,10 +163,11 @@
163163
filter_cfg=dict(filter_empty_gt=False, min_size=32),
164164
pipeline=train_pipeline,
165165
),
166+
async_load=True,
166167
)
167168

168169
test_pipeline = [
169-
dict(type='sscma.LoadImageFromFile', file_client_args=dict(backend='disk')),
170+
dict(type='mmengine.LoadImageFromFile', file_client_args=dict(backend='disk')),
170171
dict(type='sscma.YOLOv5KeepRatioResize', scale=imgsz),
171172
dict(type='sscma.LetterResize', scale=imgsz, allow_scale_up=False, pad_val=dict(img=114)),
172173
dict(type='sscma.LoadAnnotations', with_bbox=True, _scope_='mmdet'),
@@ -192,6 +193,7 @@
192193
pipeline=test_pipeline,
193194
batch_shapes_cfg=batch_shapes_cfg,
194195
),
196+
async_load=True,
195197
)
196198

197199
test_dataloader = val_dataloader

‎configs/swift_yolo/swift_yolo_small_1xb16_300e_coco.py‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -105,7 +105,7 @@
105105
train_pipeline = [
106106
*pre_transform,
107107
dict(
108-
type='sscma.Mosaic',
108+
type='sscma.',
109109
img_scale=imgsz,
110110
pad_val=114.0,
111111
use_cached=use_cached,
@@ -153,10 +153,11 @@
153153
filter_cfg=dict(filter_empty_gt=False, min_size=32),
154154
pipeline=train_pipeline,
155155
),
156+
async_load=True,
156157
)
157158

158159
test_pipeline = [
159-
dict(type='sscma.LoadImageFromFile', file_client_args=dict(backend='disk')),
160+
dict(type='mmengine.LoadImageFromFile', file_client_args=dict(backend='disk')),
160161
dict(type='sscma.YOLOv5KeepRatioResize', scale=imgsz),
161162
dict(type='sscma.LetterResize', scale=imgsz, allow_scale_up=False, pad_val=dict(img=114)),
162163
dict(type='sscma.LoadAnnotations', with_bbox=True, _scope_='mmdet'),
@@ -182,6 +183,7 @@
182183
pipeline=test_pipeline,
183184
batch_shapes_cfg=batch_shapes_cfg,
184185
),
186+
async_load=True,
185187
)
186188

187189
test_dataloader = val_dataloader

‎configs/swift_yolo/swift_yolo_tiny_1xb16_300e_coco.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -153,10 +153,11 @@
153153
filter_cfg=dict(filter_empty_gt=False, min_size=32),
154154
pipeline=train_pipeline,
155155
),
156+
async_load=True,
156157
)
157158

158159
test_pipeline = [
159-
dict(type='sscma.LoadImageFromFile', file_client_args=dict(backend='disk')),
160+
dict(type='mmengine.LoadImageFromFile', file_client_args=dict(backend='disk')),
160161
dict(type='sscma.YOLOv5KeepRatioResize', scale=imgsz),
161162
dict(type='sscma.LetterResize', scale=imgsz, allow_scale_up=False, pad_val=dict(img=114)),
162163
dict(type='sscma.LoadAnnotations', with_bbox=True, _scope_='mmdet'),
@@ -182,6 +183,7 @@
182183
pipeline=test_pipeline,
183184
batch_shapes_cfg=batch_shapes_cfg,
184185
),
186+
async_load=True,
185187
)
186188

187189
test_dataloader = val_dataloader

‎requirements/mmlab.txt‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
# use openmim to install
2-
mmcv-full<=2.1.0
3-
mmdet>=3.0.0, <3.1.0 # mmyolo currently does not support mmdet 3.1.0
2+
mmcv<=2.1.0
3+
mmdet>=3.0.0, <=3.3.0 # mmyolo currently does not support mmdet 3.1.0
44
mmengine>=0.8.2

‎sscma/datasets/sampler.py‎

Lines changed: 13 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,18 @@
11
# Copyright (c) Seeed Technology Co.,Ltd. All rights reserved.
2+
import random
23
from typing import Iterator, List, Optional, Union
34

4-
import torch
5-
import random
65
import numpy as np
7-
from torch.utils.data import ConcatDataset
6+
import torch
87
from mmengine.dataset import DefaultSampler
8+
from torch.utils.data import ConcatDataset
99

10-
from sscma.registry import DATA_SANPLERS
10+
from sscma.registry import DATA_SAMPLERS
1111

1212

13-
@DATA_SANPLERS.register_module()
13+
@DATA_SAMPLERS.register_module()
1414
class SemiSampler(DefaultSampler):
15-
"""
16-
Sampler for scaled sampling of semi-supervised data
15+
"""Sampler for scaled sampling of semi-supervised data.
1716
1817
Params:
1918
dataset (torch::ConcatDataset): Multiple merged datasets
@@ -38,7 +37,7 @@ def __init__(
3837
) -> None:
3938
assert len(sample_ratio) == len(
4039
dataset.cumulative_sizes
41-
), "Sampling rate length must be equal to the number of datasets."
40+
), 'Sampling rate length must be equal to the number of datasets.'
4241

4342
super(SemiSampler, self).__init__(dataset, shuffle=shuffle, seed=seed, round_up=round_up)
4443
if seed is not None:
@@ -65,7 +64,7 @@ def __init__(
6564
self.computer_epoch()
6665

6766
def __iter__(self) -> Iterator[int]:
68-
indexs = []
67+
indexes = []
6968
num1 = 0
7069
num2 = 0
7170
data1_len = len(self.data1)
@@ -77,21 +76,21 @@ def __iter__(self) -> Iterator[int]:
7776
for _ in range(self.total_epoch):
7877
if self.all_data:
7978
for _ in range(self.sample_size[0]):
80-
indexs.append(self.data1[num1 % data1_len])
79+
indexes.append(self.data1[num1 % data1_len])
8180
num1 += 1
8281
for _ in range(self.sample_size[1]):
83-
indexs.append(self.data2[num2 % data2_len])
82+
indexes.append(self.data2[num2 % data2_len])
8483
num2 += 1
8584
elif self.only_label:
8685
for _ in range(self.sample_size[0] + self.sample_size[1]):
87-
indexs.append(self.data1[num1 % data1_len])
86+
indexes.append(self.data1[num1 % data1_len])
8887
num1 += 1
8988
else:
9089
for _ in range(self.sample_size[0] + self.sample_size[1]):
91-
indexs.append(self.data2[num2 % data2_len])
90+
indexes.append(self.data2[num2 % data2_len])
9291
num2 += 1
9392

94-
return iter(indexs)
93+
return iter(indexes)
9594

9695
def __len__(self) -> int:
9796
return self.total_epoch * self._batch_size

‎sscma/datasets/transforms/__init__.py‎

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,10 @@
11
# Copyright (c) Seeed Technology Co.,Ltd. All rights reserved.
2+
from .affine import YOLOv5RandomAffine
23
from .auto_augment import RandomRotate
4+
from .color import YOLOv5HSVRandomAug
35
from .formatting import PackClsInputs, PackSensorInputs
46
from .loading import LoadSensorFromFile, YOLOLoadAnnotations
5-
from .utils import BatchShapePolicy, yolov5_collate
6-
from .wrappers import MutiBranchPipe
77
from .mosaic import Mosaic
8-
from .color import YOLOv5HSVRandomAug
9-
from .affine import YOLOv5RandomAffine
10-
from .resize import YOLOv5KeepRatioResize, LetterResize
118
from .processing import (
129
AlbumentationsCls,
1310
ColorJitterCls,
@@ -18,7 +15,9 @@
1815
RandomResizedCropCls,
1916
ResizeEdge,
2017
)
21-
18+
from .resize import LetterResize, YOLOv5KeepRatioResize
19+
from .utils import BatchShapePolicy, yolov5_collate
20+
from .wrappers import MutiBranchPipe
2221

2322
__all__ = [
2423
'PackSensorInputs',
@@ -29,7 +28,7 @@
2928
'LetterResize',
3029
'YOLOv5HSVRandomAug',
3130
'YOLOv5RandomAffine',
32-
'LoadAnnotations',
31+
'YOLOLoadAnnotations',
3332
'RandomRotate',
3433
'Mosaic',
3534
'BatchShapePolicy',

0 commit comments

Comments
 (0)