Skip to content

Commit e618c31

Browse files
rod409pgmpablo157321arjunsureshmrmhodakmlcommons-bot
authored
Merge automotive reference into main branch (#2047)
* Automotive reference implementation sketch * WIP: automotive reference implementation * WIP add segmentation, dataset, and util functions to reference * WIP: reference implementation with issues during post processing * WIP update dockerfile remove pdb breaks * WIP: reference implementation that runs samples * Update README.md with initial docker runs * WIP: add accuracy checker to reference * Fix: set lidar detector to evaluation mode * [Automated Commit] Format Codebase * Update README.md * Remove unnecessary mlperf conf load and implement dataset get_item_count * add cpu only version of reference * update dockerfile and display overal mAP * code cleanup * Update README.md * fix packages versions in dockerfile * handle frames if model predicts no objects * Allow accuracy check on subset of datasets * [Automated Commit] Format Codebase * Updates automotive reference implementation (#2045) * Remove unnecessary mlperf conf load and implement dataset get_item_count * add cpu only version of reference * update dockerfile and display overal mAP * code cleanup * Update README.md * fix packages versions in dockerfile * handle frames if model predicts no objects * Allow accuracy check on subset of datasets * [Automated Commit] Format Codebase --------- Co-authored-by: mlcommons-bot <[email protected]> * [Automated Commit] Format Codebase * fix whitespace merge errors * [Automated Commit] Format Codebase * [Automated Commit] Format Codebase * add removed code during merge * [Automated Commit] Format Codebase * change nms thresholds on cpu only code --------- Co-authored-by: Pablo Gonzalez <[email protected]> Co-authored-by: Arjun Suresh <[email protected]> Co-authored-by: arjunsuresh <[email protected]> Co-authored-by: Miro <[email protected]> Co-authored-by: mlcommons-bot <[email protected]>
1 parent 25bba8f commit e618c31

File tree

11 files changed

+800
-46
lines changed

11 files changed

+800
-46
lines changed
Lines changed: 18 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,27 @@
11
## Reference implementation fo automotive 3D detection benchmark
22

3-
## TODO: Instructions for dataset download after it is uploaded somewhere appropriate
4-
5-
## TODO: Instructions for checkpoints downloads after it is uploaded somewhere appropriate
3+
## Dataset and model checkpoints
4+
Contact MLCommons support for accessing the Waymo Open Dataset along with the model checkpoints for the reference implementation. You will need to accept a license agreement and will be given directions to download the data. You will need to place the kitti_format folder under a directory named waymo. There are four total checkpoints 2 for pytorch and 2 for onnx.
65

76
## Running with docker
7+
Build the container and mount the inference repo and Waymo dataset directory.
88
```
99
docker build -t auto_inference -f dockerfile.gpu .
1010
11-
docker run --gpus=all -it -v <directory to inference repo>/inference/:/inference -v <directory to waymo dataset>/waymo:/waymo --rm auto_inference
12-
11+
docker run --gpus=all -it -v <directory to inference repo>/inference/:/inference -v <path to waymo dataset>/waymo:/waymo --rm auto_inference
12+
```
13+
### Run with GPU
14+
```
1315
cd /inference/automotive/3d-object-detection
1416
python main.py --dataset waymo --dataset-path /waymo/kitti_format/ --lidar-path <checkpoint_path>/pp_ep36.pth --segmentor-path <checkpoint_path>/best_deeplabv3plus_resnet50_waymo_os16.pth --mlperf_conf /inference/mlperf.conf
17+
```
18+
19+
### Run with CPU and ONNX
20+
```
21+
python main.py --dataset waymo --dataset-path /waymo/kitti_format/ --lidar-path <checkpoint_path>/pp.onnx --segmentor-path <checkpoint_path>/deeplabv3+.onnx --mlperf_conf /inference/mlperf.conf
22+
```
23+
24+
### Run the accuracy checker
25+
```
26+
python accuracy_waymo.py --mlperf-accuracy-file <path to accuracy file>/mlperf_log_accuracy.json --waymo-dir /waymo/kitti_format/
27+
```

automotive/3d-object-detection/accuracy_waymo.py

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -95,13 +95,13 @@ def main():
9595
'bbox': [],
9696
'score': []
9797
}
98-
99-
detections[image_idx]['name'].append(LABEL2CLASSES[label])
100-
detections[image_idx]['dimensions'].append(dimension)
101-
detections[image_idx]['location'].append(location)
102-
detections[image_idx]['rotation_y'].append(rotation_y)
103-
detections[image_idx]['bbox'].append(bbox)
104-
detections[image_idx]['score'].append(score)
98+
if dimension[0] > 0:
99+
detections[image_idx]['name'].append(LABEL2CLASSES[label])
100+
detections[image_idx]['dimensions'].append(dimension)
101+
detections[image_idx]['location'].append(location)
102+
detections[image_idx]['rotation_y'].append(rotation_y)
103+
detections[image_idx]['bbox'].append(bbox)
104+
detections[image_idx]['score'].append(score)
105105
image_ids.add(image_idx)
106106

107107
with open(args.output_file, "w") as fp:
@@ -115,6 +115,7 @@ def main():
115115
val_dataset.data_infos,
116116
CLASSES,
117117
cam_sync=False)
118+
map_stats['Total'] = np.mean(list(map_stats.values()))
118119

119120
print(map_stats)
120121
if args.verbose:

automotive/3d-object-detection/backend_deploy.py

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -78,13 +78,11 @@ def load(self):
7878
return self
7979

8080
def predict(self, inputs):
81-
# TODO: implement predict
8281
dimensions, locations, rotation_y, box2d, class_labels, class_scores, ids = [
8382
], [], [], [], [], [], []
8483
with torch.inference_mode():
8584
device = torch.device(
8685
"cuda:0" if torch.cuda.is_available() else "cpu")
87-
format_results = {}
8886
model_input = inputs[0]
8987
batched_pts = model_input['pts']
9088
scores_from_cam = []
@@ -114,7 +112,7 @@ def predict(self, inputs):
114112
calib_info = model_input['calib_info']
115113
image_info = model_input['image_info']
116114
idx = model_input['image_info']['image_idx']
117-
115+
format_result['idx'] = idx
118116
calib_info = change_calib_device(calib_info, False)
119117
result_filter = keep_bbox_from_image_range(
120118
result, calib_info, 5, image_info, False)
@@ -135,9 +133,6 @@ def predict(self, inputs):
135133
format_result['location'].append(camera_bbox[:3])
136134
format_result['rotation_y'].append(camera_bbox[6].item())
137135
format_result['score'].append(score.item())
138-
format_results['idx'] = idx
139-
140-
# write_label(format_result, os.path.join(saved_submit_path, f'{idx:06d}.txt'))
141136

142137
if len(format_result['dimensions']) > 0:
143138
format_result['dimensions'] = torch.stack(
@@ -150,6 +145,5 @@ def predict(self, inputs):
150145
class_labels.append(format_result['class'])
151146
class_scores.append(format_result['score'])
152147
box2d.append(format_result['bbox'])
153-
ids.append(format_results['idx'])
154-
# return Boxes, Classes, Scores # Change to desired output
148+
ids.append(format_result['idx'])
155149
return dimensions, locations, rotation_y, box2d, class_labels, class_scores, ids
Lines changed: 168 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,168 @@
1+
from typing import Optional, List, Union
2+
import os
3+
import torch
4+
import logging
5+
import backend
6+
from collections import namedtuple
7+
from model.painter import Painter
8+
from model.pointpillars_core import PointPillarsPre, PointPillarsPos
9+
import numpy as np
10+
from tools.process import keep_bbox_from_image_range
11+
from waymo import Waymo
12+
import onnxruntime as ort
13+
14+
15+
logging.basicConfig(level=logging.INFO)
16+
log = logging.getLogger("backend-onnx")
17+
18+
19+
def change_calib_device(calib, cuda):
20+
result = {}
21+
if cuda:
22+
device = 'cuda'
23+
else:
24+
device = 'cpu'
25+
result['R0_rect'] = calib['R0_rect'].to(device=device, dtype=torch.float)
26+
for i in range(5):
27+
result['P' + str(i)] = calib['P' + str(i)
28+
].to(device=device, dtype=torch.float)
29+
result['Tr_velo_to_cam_' +
30+
str(i)] = calib['Tr_velo_to_cam_' +
31+
str(i)].to(device=device, dtype=torch.float)
32+
return result
33+
34+
35+
def to_numpy(tensor):
36+
return tensor.detach().cpu().numpy() if tensor.requires_grad else tensor.cpu().numpy()
37+
38+
39+
class BackendOnnx(backend.Backend):
40+
def __init__(
41+
self,
42+
segmentor_path,
43+
lidar_detector_path,
44+
data_path
45+
):
46+
super(BackendOnnx, self).__init__()
47+
self.segmentor_path = segmentor_path
48+
self.lidar_detector_path = lidar_detector_path
49+
# self.segmentation_classes = 18
50+
self.detection_classes = 3
51+
self.data_root = data_path
52+
CLASSES = Waymo.CLASSES
53+
self.LABEL2CLASSES = {v: k for k, v in CLASSES.items()}
54+
55+
def version(self):
56+
return torch.__version__
57+
58+
def name(self):
59+
return "python-SUT"
60+
61+
def load(self):
62+
device = torch.device("cpu")
63+
PaintArgs = namedtuple(
64+
'PaintArgs', [
65+
'training_path', 'model_path', 'cam_sync'])
66+
painting_args = PaintArgs(
67+
os.path.join(
68+
self.data_root,
69+
'training'),
70+
self.segmentor_path,
71+
False)
72+
self.painter = Painter(painting_args, onnx=True)
73+
self.segmentor = self.painter.model
74+
model_pre = PointPillarsPre()
75+
model_post = PointPillarsPos(self.detection_classes)
76+
model_pre.eval()
77+
model_post.eval()
78+
ort_sess = ort.InferenceSession(self.lidar_detector_path)
79+
self.lidar_detector = ort_sess
80+
self.model_pre = model_pre
81+
self.model_post = model_post
82+
return self
83+
84+
def predict(self, inputs):
85+
dimensions, locations, rotation_y, box2d, class_labels, class_scores, ids = [
86+
], [], [], [], [], [], []
87+
with torch.inference_mode():
88+
model_input = inputs[0]
89+
batched_pts = model_input['pts']
90+
scores_from_cam = []
91+
for i in range(len(model_input['images'])):
92+
input_image_name = self.segmentor.get_inputs()[0].name
93+
input_data = {
94+
input_image_name: to_numpy(
95+
model_input['images'][i])}
96+
segmentation_score = self.segmentor.run(None, input_data)
97+
segmentation_score = [
98+
torch.from_numpy(item) for item in segmentation_score]
99+
scores_from_cam.append(
100+
self.painter.get_score(
101+
segmentation_score[0].squeeze(0)).cpu())
102+
points = self.painter.augment_lidar_class_scores_both(
103+
scores_from_cam, batched_pts, model_input['calib_info'])
104+
pillars, coors_batch, npoints_per_pillar = self.model_pre(batched_pts=[
105+
points])
106+
input_pillars_name = self.lidar_detector.get_inputs()[0].name
107+
input_coors_batch_name = self.lidar_detector.get_inputs()[1].name
108+
input_npoints_per_pillar_name = self.lidar_detector.get_inputs()[
109+
2].name
110+
input_data = {input_pillars_name: to_numpy(pillars),
111+
input_coors_batch_name: to_numpy(coors_batch),
112+
input_npoints_per_pillar_name: to_numpy(npoints_per_pillar)}
113+
result = self.lidar_detector.run(None, input_data)
114+
result = [torch.from_numpy(item) for item in result]
115+
batch_results = self.model_post(result)
116+
for j, result in enumerate(batch_results):
117+
format_result = {
118+
'class': [],
119+
'truncated': [],
120+
'occluded': [],
121+
'alpha': [],
122+
'bbox': [],
123+
'dimensions': [],
124+
'location': [],
125+
'rotation_y': [],
126+
'score': [],
127+
'idx': -1
128+
}
129+
130+
calib_info = model_input['calib_info']
131+
image_info = model_input['image_info']
132+
idx = model_input['image_info']['image_idx']
133+
format_result['idx'] = idx
134+
calib_info = change_calib_device(calib_info, False)
135+
result_filter = keep_bbox_from_image_range(
136+
result, calib_info, 5, image_info, False)
137+
138+
lidar_bboxes = result_filter['lidar_bboxes']
139+
labels, scores = result_filter['labels'], result_filter['scores']
140+
bboxes2d, camera_bboxes = result_filter['bboxes2d'], result_filter['camera_bboxes']
141+
for lidar_bbox, label, score, bbox2d, camera_bbox in \
142+
zip(lidar_bboxes, labels, scores, bboxes2d, camera_bboxes):
143+
format_result['class'].append(label.item())
144+
format_result['truncated'].append(0.0)
145+
format_result['occluded'].append(0)
146+
alpha = camera_bbox[6] - \
147+
np.arctan2(camera_bbox[0], camera_bbox[2])
148+
format_result['alpha'].append(alpha.item())
149+
format_result['bbox'].append(bbox2d.tolist())
150+
format_result['dimensions'].append(camera_bbox[3:6])
151+
format_result['location'].append(camera_bbox[:3])
152+
format_result['rotation_y'].append(camera_bbox[6].item())
153+
format_result['score'].append(score.item())
154+
155+
if len(format_result['dimensions']) > 0:
156+
format_result['dimensions'] = torch.stack(
157+
format_result['dimensions'])
158+
format_result['location'] = torch.stack(
159+
format_result['location'])
160+
dimensions.append(format_result['dimensions'])
161+
locations.append(format_result['location'])
162+
rotation_y.append(format_result['rotation_y'])
163+
class_labels.append(format_result['class'])
164+
class_scores.append(format_result['score'])
165+
box2d.append(format_result['bbox'])
166+
ids.append(format_result['idx'])
167+
168+
return dimensions, locations, rotation_y, box2d, class_labels, class_scores, ids

automotive/3d-object-detection/dockerfile.gpu

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
ARG FROM_IMAGE_NAME=pytorch/pytorch:2.0.1-cuda11.7-cudnn8-devel
1+
ARG FROM_IMAGE_NAME=pytorch/pytorch:2.2.2-cuda11.8-cudnn8-devel
22
FROM ${FROM_IMAGE_NAME}
33

44
ENV DEBIAN_FRONTEND=noninteractive
@@ -20,12 +20,12 @@ RUN cd /tmp && \
2020
CFLAGS="-std=c++14" python setup.py install && \
2121
rm -rf mlperf
2222

23-
RUN pip install tqdm
24-
RUN pip install numba
25-
RUN pip install opencv-python
26-
RUN pip install open3d
27-
RUN pip install tensorboard
28-
RUN pip install scikit-image
29-
RUN pip install ninja
30-
RUN pip install visdom
31-
RUN pip install shapely
23+
RUN pip install tqdm==4.65.0
24+
RUN pip install numba==0.60.0
25+
RUN pip install opencv-python==4.11.0.86
26+
RUN pip install open3d==0.19.0
27+
RUN pip install scikit-image==0.25.0
28+
RUN pip install ninja==1.11.1
29+
RUN pip install shapely==2.0.6
30+
RUN pip install tensorboard==2.18.0
31+
RUN pip install onnxruntime==1.20.1

automotive/3d-object-detection/main.py

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -167,7 +167,9 @@ def get_backend(backend, **kwargs):
167167
from backend_deploy import BackendDeploy
168168

169169
backend = BackendDeploy(**kwargs)
170-
170+
elif backend == 'onnx':
171+
from backend_onnx import BackendOnnx
172+
backend = BackendOnnx(**kwargs)
171173
elif backend == "debug":
172174
from backend_debug import BackendDebug
173175

@@ -403,7 +405,6 @@ def flush_queries():
403405
log_settings.log_output = log_output_settings
404406

405407
settings = lg.TestSettings()
406-
settings.FromConfig(mlperf_conf, args.model_name, args.scenario)
407408
settings.FromConfig(user_conf, args.model_name, args.scenario)
408409
settings.scenario = scenario
409410
settings.mode = lg.TestMode.PerformanceOnly

automotive/3d-object-detection/model/painter.py

Lines changed: 19 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import onnxruntime as ort
12
import argparse
23
import model.segmentation as network
34
import os
@@ -34,24 +35,34 @@ def get_calib_from_file(calib_file):
3435
return data
3536

3637

38+
def to_numpy(tensor):
39+
return tensor.detach().cpu().numpy() if tensor.requires_grad else tensor.cpu().numpy()
40+
41+
3742
class Painter:
38-
def __init__(self, args):
43+
def __init__(self, args, onnx=False):
3944
self.root_split_path = args.training_path
4045
self.save_path = os.path.join(args.training_path, "painted_lidar/")
46+
self.onnx = onnx
4147
if not os.path.exists(self.save_path):
4248
os.mkdir(self.save_path)
4349

4450
self.seg_net_index = 0
4551
self.model = None
4652
print(f'Using Segmentation Network -- deeplabv3plus')
4753
checkpoint_file = args.model_path
48-
model = network.modeling.__dict__['deeplabv3plus_resnet50'](
49-
num_classes=19, output_stride=16)
50-
checkpoint = torch.load(checkpoint_file)
51-
model.load_state_dict(checkpoint["model_state"])
52-
model.eval()
53-
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
54-
model.to(device)
54+
if self.onnx:
55+
model = ort.InferenceSession(checkpoint_file)
56+
self.input_image_name = model.get_inputs()[0].name
57+
else:
58+
model = network.modeling.__dict__['deeplabv3plus_resnet50'](
59+
num_classes=19, output_stride=16)
60+
checkpoint = torch.load(checkpoint_file)
61+
model.load_state_dict(checkpoint["model_state"])
62+
model.eval()
63+
device = torch.device(
64+
'cuda' if torch.cuda.is_available() else 'cpu')
65+
model.to(device)
5566
self.model = model
5667
self.cam_sync = args.cam_sync
5768

automotive/3d-object-detection/model/pointpillars.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -397,7 +397,9 @@ def get_predicted_bboxes_single(
397397

398398
# 3.2 nms core
399399
keep_inds = ml3d.ops.nms(
400-
cur_bbox_pred2d, cur_bbox_cls_pred, self.nms_thr)
400+
cur_bbox_pred2d.cpu(),
401+
cur_bbox_cls_pred.cpu(),
402+
self.nms_thr)
401403

402404
cur_bbox_cls_pred = cur_bbox_cls_pred[keep_inds]
403405
cur_bbox_pred = cur_bbox_pred[keep_inds]

0 commit comments

Comments
 (0)