From 3edad007bce96d6ec1deafb4fc6fca4fb658239c Mon Sep 17 00:00:00 2001 From: Wanli Date: Thu, 28 Dec 2023 19:14:38 +0800 Subject: [PATCH 1/3] add benchmark for optical flow Raft model --- README.md | 4 ++++ .../config/optical_flow_estimation_raft.yaml | 16 +++++++++++++ benchmark/download_data.py | 4 ++++ benchmark/utils/dataloaders/__init__.py | 3 ++- benchmark/utils/dataloaders/optical_flow.py | 24 +++++++++++++++++++ benchmark/utils/metrics/__init__.py | 3 ++- benchmark/utils/metrics/optical_flow.py | 22 +++++++++++++++++ models/__init__.py | 2 ++ models/optical_flow_estimation_raft/raft.py | 6 +++++ 9 files changed, 82 insertions(+), 2 deletions(-) create mode 100644 benchmark/config/optical_flow_estimation_raft.yaml create mode 100644 benchmark/utils/dataloaders/optical_flow.py create mode 100644 benchmark/utils/metrics/optical_flow.py diff --git a/README.md b/README.md index e2af9a83..e105e474 100644 --- a/README.md +++ b/README.md @@ -115,6 +115,10 @@ Some examples are listed below. You can find more in the directory of each model ![crnn_demo](./models/text_recognition_crnn/example_outputs/CRNNCTC.gif) +### Optical Estimation with [RAFT](./models/optical_flow_estimation_raft/) + +![raft_demo](./models/optical_flow_estimation_raft/example_outputs/result.jpg) + ## License OpenCV Zoo is licensed under the [Apache 2.0 license](./LICENSE). Please refer to licenses of different models. diff --git a/benchmark/config/optical_flow_estimation_raft.yaml b/benchmark/config/optical_flow_estimation_raft.yaml new file mode 100644 index 00000000..2a73c42b --- /dev/null +++ b/benchmark/config/optical_flow_estimation_raft.yaml @@ -0,0 +1,16 @@ +Benchmark: + name: "Optical Flow Estimation Benchmark" + type: "OpticalFlow" + data: + path: "data/optical_flow_estimation" + files: [["driving0.png", "driving1.png"], ["flyingThings3D0.png", "flyingThings3D1.png"], ["monkaa0.png", "monkaa1.png"]] + sizes: # [[w1, h1], ...], Omit to run at original scale + - [360, 480] + metric: + warmup: 30 + repeat: 10 + backend: "default" + target: "cpu" + +Model: + name: "Raft" diff --git a/benchmark/download_data.py b/benchmark/download_data.py index 68033302..000cf8ec 100644 --- a/benchmark/download_data.py +++ b/benchmark/download_data.py @@ -217,6 +217,10 @@ def get_confirm_token(response): # in case of large files url='https://drive.google.com/u/0/uc?id=1RbLyetgqFUTt0IHaVmu6c_b7KeXJgKbc&export=download', sha='fbae2fb0a47fe65e316bbd0ec57ba21461967550', filename='person_detection.zip'), + optical_flow_estimation=Downloader(name='optical_flow_estimation', + url='https://drive.google.com/u/0/uc?id=1_fvN7cgc-j92MeI_wHKGkhWxbXeML_gR&export=download', + sha='96b75eaef250efdde62184b07707827d76bd336c', + filename='optical_flow_estimation.zip'), ) if __name__ == '__main__': diff --git a/benchmark/utils/dataloaders/__init__.py b/benchmark/utils/dataloaders/__init__.py index 5d0e4aed..4947362d 100644 --- a/benchmark/utils/dataloaders/__init__.py +++ b/benchmark/utils/dataloaders/__init__.py @@ -2,5 +2,6 @@ from .classification import ClassificationImageLoader from .recognition import RecognitionImageLoader from .tracking import TrackingVideoLoader +from .optical_flow import OpticalFlowImageLoader -__all__ = ['BaseImageLoader', 'BaseVideoLoader', 'ClassificationImageLoader', 'RecognitionImageLoader', 'TrackingVideoLoader'] \ No newline at end of file +__all__ = ['BaseImageLoader', 'BaseVideoLoader', 'ClassificationImageLoader', 'RecognitionImageLoader', 'TrackingVideoLoader', 'OpticalFlowImageLoader'] \ No newline at end of file diff --git a/benchmark/utils/dataloaders/optical_flow.py b/benchmark/utils/dataloaders/optical_flow.py new file mode 100644 index 00000000..fcd7362d --- /dev/null +++ b/benchmark/utils/dataloaders/optical_flow.py @@ -0,0 +1,24 @@ +import os + +import numpy as np +import cv2 as cv + +from .base_dataloader import _BaseImageLoader +from ..factory import DATALOADERS + +@DATALOADERS.register +class OpticalFlowImageLoader(_BaseImageLoader): + def __init__(self, **kwargs): + super().__init__(**kwargs) + + def __iter__(self): + for case in self._files: + image0 = cv.imread(os.path.join(self._path, case[0])) + image1 = cv.imread(os.path.join(self._path, case[1])) + if [0, 0] in self._sizes: + yield "{}, {}".format(case[0], case[1]), image0, image1 + else: + for size in self._sizes: + image0_r = cv.resize(image0, size) + image1_r = cv.resize(image1, size) + yield "{}, {}".format(case[0], case[1]), image0_r, image1_r \ No newline at end of file diff --git a/benchmark/utils/metrics/__init__.py b/benchmark/utils/metrics/__init__.py index 9f524870..df76f22d 100644 --- a/benchmark/utils/metrics/__init__.py +++ b/benchmark/utils/metrics/__init__.py @@ -2,5 +2,6 @@ from .detection import Detection from .recognition import Recognition from .tracking import Tracking +from .optical_flow import OpticalFlow -__all__ = ['Base', 'Detection', 'Recognition', 'Tracking'] \ No newline at end of file +__all__ = ['Base', 'Detection', 'Recognition', 'Tracking', 'OpticalFlow'] \ No newline at end of file diff --git a/benchmark/utils/metrics/optical_flow.py b/benchmark/utils/metrics/optical_flow.py new file mode 100644 index 00000000..ed49e457 --- /dev/null +++ b/benchmark/utils/metrics/optical_flow.py @@ -0,0 +1,22 @@ +import cv2 as cv + +from .base_metric import BaseMetric +from ..factory import METRICS + +@METRICS.register +class OpticalFlow(BaseMetric): + def __init__(self, **kwargs): + super().__init__(**kwargs) + + def forward(self, model, *args, **kwargs): + img0, img1 = args + + self._timer.reset() + for _ in range(self._warmup): + model.infer(img0, img1) + for _ in range(self._repeat): + self._timer.start() + model.infer(img0, img1) + self._timer.stop() + + return self._timer.getRecords() diff --git a/models/__init__.py b/models/__init__.py index 1af41b7f..ca5548aa 100644 --- a/models/__init__.py +++ b/models/__init__.py @@ -20,6 +20,7 @@ from .facial_expression_recognition.facial_fer_model import FacialExpressionRecog from .object_tracking_vittrack.vittrack import VitTrack from .text_detection_ppocr.ppocr_det import PPOCRDet +from .optical_flow_estimation_raft.raft import Raft class ModuleRegistery: def __init__(self, name): @@ -94,3 +95,4 @@ def register(self, item): MODELS.register(FacialExpressionRecog) MODELS.register(VitTrack) MODELS.register(PPOCRDet) +MODELS.register(Raft) diff --git a/models/optical_flow_estimation_raft/raft.py b/models/optical_flow_estimation_raft/raft.py index e0557cce..9d54fd62 100644 --- a/models/optical_flow_estimation_raft/raft.py +++ b/models/optical_flow_estimation_raft/raft.py @@ -29,6 +29,12 @@ def _preprocess(self, image): img_input = img_input.astype(np.float32) return img_input + def setBackendAndTarget(self, backendId, targetId): + self.backend_id = backendId + self.target_id = targetId + self.model.setPreferableBackend(self.backend_id) + self.model.setPreferableTarget(self.target_id) + def infer(self, image1, image2): # Preprocess From ed15dc733a105b16389b12934aea06e90ba0fcc4 Mon Sep 17 00:00:00 2001 From: Wanli Date: Sun, 31 Dec 2023 00:18:22 +0800 Subject: [PATCH 2/3] add benchmark result --- benchmark/README.md | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/benchmark/README.md b/benchmark/README.md index 52a23557..200cada1 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -114,6 +114,7 @@ mean median min input size model 9.68 9.21 7.74 [1280, 720] CRNN with ['text_recognition_CRNN_CH_2022oct_int8.onnx'] 9.85 10.63 7.74 [1280, 720] CRNN with ['text_recognition_CRNN_CN_2021nov_int8.onnx'] 9.63 9.28 7.74 [1280, 720] CRNN with ['text_recognition_CRNN_EN_2022oct_int8.onnx'] +1023.71 1024.90 1016.75 [360, 480] Raft with ['optical_flow_estimation_raft_2023aug.onnx'] ``` ### Raspberry Pi 4B @@ -171,6 +172,7 @@ mean median min input size model 223.58 219.82 200.44 [1280, 720] CRNN with ['text_recognition_CRNN_CH_2022oct_int8.onnx'] 225.60 243.89 200.44 [1280, 720] CRNN with ['text_recognition_CRNN_CN_2021nov_int8.onnx'] 220.97 223.16 193.91 [1280, 720] CRNN with ['text_recognition_CRNN_EN_2022oct_int8.onnx'] +16176.34 16377.71 15127.76 [360, 480] Raft with ['optical_flow_estimation_raft_2023aug.onnx'] ``` ### Jetson Nano B01 @@ -229,12 +231,13 @@ mean median min input size model 243.46 238.98 219.06 [1280, 720] CRNN with ['text_recognition_CRNN_CH_2022oct_int8.onnx'] 246.87 256.05 219.06 [1280, 720] CRNN with ['text_recognition_CRNN_CN_2021nov_int8.onnx'] 243.37 238.90 219.06 [1280, 720] CRNN with ['text_recognition_CRNN_EN_2022oct_int8.onnx'] +12659.89 12689.15 12543.48 [360, 480] Raft with ['optical_flow_estimation_raft_2023aug.onnx'] ``` GPU (CUDA-FP32): ``` -$ python3 benchmark.py --all --fp32 --cfg_exclude wechat --cfg_overwrite_backend_target 1 +$ python3 benchmark.py --all --fp32 --cfg_exclude wechat:raft --cfg_overwrite_backend_target 1 Benchmarking ... backend=cv.dnn.DNN_BACKEND_CUDA target=cv.dnn.DNN_TARGET_CUDA @@ -265,7 +268,7 @@ mean median min input size model GPU (CUDA-FP16): ``` -$ python3 benchmark.py --all --fp32 --cfg_exclude wechat --cfg_overwrite_backend_target 2 +$ python3 benchmark.py --all --fp32 --cfg_exclude wechat:raft --cfg_overwrite_backend_target 2 Benchmarking ... backend=cv.dnn.DNN_BACKEND_CUDA target=cv.dnn.DNN_TARGET_CUDA_FP16 @@ -348,6 +351,7 @@ mean median min input size model 182.90 178.97 161.37 [1280, 720] CRNN with ['text_recognition_CRNN_CH_2022oct_int8.onnx'] 184.26 194.43 161.37 [1280, 720] CRNN with ['text_recognition_CRNN_CN_2021nov_int8.onnx'] 180.65 180.59 155.36 [1280, 720] CRNN with ['text_recognition_CRNN_EN_2022oct_int8.onnx'] +9608.99 9618.12 9544.66 [360, 480] Raft with ['optical_flow_estimation_raft_2023aug.onnx'] ``` NPU (TIMVX): @@ -433,6 +437,7 @@ mean median min input size model 277.84 262.99 243.87 [1280, 720] CRNN with ['text_recognition_CRNN_CH_2022oct_int8.onnx'] 283.02 280.77 243.87 [1280, 720] CRNN with ['text_recognition_CRNN_CN_2021nov_int8.onnx'] 279.21 262.55 243.87 [1280, 720] CRNN with ['text_recognition_CRNN_EN_2022oct_int8.onnx'] +22969.43 22973.42 22954.32 [360, 480] Raft with ['optical_flow_estimation_raft_2023aug.onnx'] ``` NPU (CANN): @@ -440,7 +445,7 @@ NPU (CANN): ``` -$ python3 benchmark.py --all --fp32 --cfg_exclude wechat:crnn:vittrack --model_exclude pose_estimation_mediapipe_2023mar.onnx --cfg_overwrite_backend_target 4 +$ python3 benchmark.py --all --fp32 --cfg_exclude wechat:crnn:vittrack:raft --model_exclude pose_estimation_mediapipe_2023mar.onnx --cfg_overwrite_backend_target 4 Benchmarking ... backend=cv.dnn.DNN_BACKEND_CANN target=cv.dnn.DNN_TARGET_NPU @@ -516,6 +521,7 @@ mean median min input size model 2035.98 2185.05 1268.94 [1280, 720] CRNN with ['text_recognition_CRNN_CH_2022oct_int8.onnx'] 1927.93 2178.84 1268.94 [1280, 720] CRNN with ['text_recognition_CRNN_CN_2021nov_int8.onnx'] 1822.23 2213.30 1183.93 [1280, 720] CRNN with ['text_recognition_CRNN_EN_2022oct_int8.onnx'] +316818.19 316973.10 316458.29 [360, 480] Raft with ['optical_flow_estimation_raft_2023aug.onnx'] ``` ### Khadas Edge2 (with RK3588) @@ -574,6 +580,7 @@ mean median min input size model 67.36 65.65 61.13 [1280, 720] CRNN with ['text_recognition_CRNN_CH_2022oct_int8.onnx'] 68.52 69.93 61.13 [1280, 720] CRNN with ['text_recognition_CRNN_CN_2021nov_int8.onnx'] 68.36 65.65 61.13 [1280, 720] CRNN with ['text_recognition_CRNN_EN_2022oct_int8.onnx'] +4643.54 4649.53 4575.53 [360, 480] Raft with ['optical_flow_estimation_raft_2023aug.onnx'] ``` ### Horizon Sunrise X3 PI @@ -632,6 +639,7 @@ mean median min input size model 425.24 426.69 380.35 [1280, 720] CRNN with ['text_recognition_CRNN_CH_2022oct_int8.onnx'] 431.14 447.85 380.35 [1280, 720] CRNN with ['text_recognition_CRNN_CN_2021nov_int8.onnx'] 424.77 417.01 380.35 [1280, 720] CRNN with ['text_recognition_CRNN_EN_2022oct_int8.onnx'] +25455.99 25444.32 25274.29 [360, 480] Raft with ['optical_flow_estimation_raft_2023aug.onnx'] ``` ### MAIX-III AX-PI @@ -690,6 +698,7 @@ mean median min input size model 3065.33 3217.99 2348.42 [1280, 720] CRNN with ['text_recognition_CRNN_CH_2022oct_int8.onnx'] 2976.24 3244.75 2348.42 [1280, 720] CRNN with ['text_recognition_CRNN_CN_2021nov_int8.onnx'] 2864.72 3219.46 2208.44 [1280, 720] CRNN with ['text_recognition_CRNN_EN_2022oct_int8.onnx'] +433898.12 433937.47 433829.11 [360, 480] Raft with ['optical_flow_estimation_raft_2023aug.onnx'] ``` ### StarFive VisionFive 2 @@ -747,6 +756,7 @@ mean median min input size model 1313.68 1427.46 808.70 [1280, 720] CRNN with ['text_recognition_CRNN_CH_2022oct_int8.onnx'] 1242.07 1408.93 808.70 [1280, 720] CRNN with ['text_recognition_CRNN_CN_2021nov_int8.onnx'] 1174.32 1426.07 774.78 [1280, 720] CRNN with ['text_recognition_CRNN_EN_2022oct_int8.onnx'] +221318.94 221288.53 221240.32 [360, 480] Raft with ['optical_flow_estimation_raft_2023aug.onnx'] ``` ### Khadas VIM4 @@ -807,6 +817,7 @@ mean median min input size model 127.63 124.81 113.82 [1280, 720] CRNN with ['text_recognition_CRNN_CH_2022oct_int8.onnx'] 129.24 134.50 113.82 [1280, 720] CRNN with ['text_recognition_CRNN_CN_2021nov_int8.onnx'] 126.64 125.09 110.45 [1280, 720] CRNN with ['text_recognition_CRNN_EN_2022oct_int8.onnx'] +7945.31 7945.83 7917.72 [360, 480] Raft with ['optical_flow_estimation_raft_2023aug.onnx'] ``` ### Jetson Nano Orin @@ -865,12 +876,13 @@ mean median min input size model 135.17 130.23 109.24 [1280, 720] CRNN with ['text_recognition_CRNN_CH_2022oct_int8.onnx'] 138.38 143.25 109.24 [1280, 720] CRNN with ['text_recognition_CRNN_CN_2021nov_int8.onnx'] 137.08 134.22 109.24 [1280, 720] CRNN with ['text_recognition_CRNN_EN_2022oct_int8.onnx'] +4404.12 4435.06 4329.78 [360, 480] Raft with ['optical_flow_estimation_raft_2023aug.onnx'] ``` GPU (CUDA-FP32): ``` -$ python3 benchmark.py --all --fp32 --cfg_exclude wechat --cfg_overwrite_backend_target 1 +$ python3 benchmark.py --all --fp32 --cfg_exclude wechat:raft --cfg_overwrite_backend_target 1 Benchmarking ... backend=cv.dnn.DNN_BACKEND_CUDA target=cv.dnn.DNN_TARGET_CUDA @@ -901,7 +913,7 @@ mean median min input size model GPU (CUDA-FP16): ``` -$ python3 benchmark.py --all --fp32 --cfg_exclude wechat --cfg_overwrite_backend_target 2 +$ python3 benchmark.py --all --fp32 --cfg_exclude wechat:raft --cfg_overwrite_backend_target 2 Benchmarking ... backend=cv.dnn.DNN_BACKEND_CUDA target=cv.dnn.DNN_TARGET_CUDA_FP16 @@ -984,4 +996,5 @@ mean median min input size model 163.43 152.16 135.52 [1280, 720] CRNN with ['text_recognition_CRNN_CH_2022oct_int8.onnx'] 173.46 162.85 135.52 [1280, 720] CRNN with ['text_recognition_CRNN_CN_2021nov_int8.onnx'] 175.28 145.22 135.52 [1280, 720] CRNN with ['text_recognition_CRNN_EN_2022oct_int8.onnx'] +11978.21 11971.35 11135.99 [360, 480] Raft with ['optical_flow_estimation_raft_2023aug.onnx'] ``` From 7bfd3fc8a98d5c10414bd22a351469ee5079e579 Mon Sep 17 00:00:00 2001 From: Wanli Date: Sun, 31 Dec 2023 00:33:24 +0800 Subject: [PATCH 3/3] update benchmark table --- benchmark/color_table.svg | 3972 ++++++++++--------- benchmark/table_config.yaml | 7 + models/optical_flow_estimation_raft/demo.py | 2 +- 3 files changed, 2106 insertions(+), 1875 deletions(-) diff --git a/benchmark/color_table.svg b/benchmark/color_table.svg index 480584c0..38433d2d 100644 --- a/benchmark/color_table.svg +++ b/benchmark/color_table.svg @@ -1,7 +1,7 @@ - + @@ -15,23 +15,23 @@ - - + +iVBORw0KGgoAAAANSUhEUgAAAwgAAAATCAYAAAA0/CrSAAABSklEQVR4nO3XS27DMAwFQMk5VK/U++8jdVOwNGC2ChIbLTqzIgiKVn4IXm/vb7N96r2FvvXX1/2Enam++nlb2n9L/Vu60r5f1Fsr+ifPP3P/Yn71TP7Y8vl9/4T54t71/hfNF5/B468r19thv7XWtvbz3MquXf1XdqaZ3lKdz1b94uyuv7DnimecsX9njq96jOP+Sn3l2Tx/8V3nKPaMWfR/28wJ+7+Zm7tdqb7PNDMO+/uZ437eOe/H95vF/JV3eHTPuB+/h7Paszg3iv4sXsPSzoWZsl88d2lnUZdf2WfqhZ/E1c97pp5pf/HPAAAA/EcCAgAAEAQEAAAgCAgAAEAQEAAAgCAgAAAAQUAAAACCgAAAAAQBAQAACAICAAAQBAQAACAICAAAQBAQAACAICAAAABBQAAAAIKAAAAABAEBAAAIAgIAABA+AEq1I/9cZQrJAAAAAElFTkSuQmCC" id="imaged38e91eac8" transform="scale(1 -1) translate(0 -13.68)" x="901.672258" y="0.635294" width="558.72" height="13.68"/> - Faster + Faster - Slower + Slower @@ -84,8 +84,8 @@ z @@ -93,349 +93,349 @@ z - Intel + Intel - 12700K + 12700K - CPU + CPU - - Atlas 200I DK A2 + Atlas 200I DK A2 - Ascend 310B + Ascend 310B - CPU + CPU - - Atlas 200 DK + Atlas 200 DK - Ascend 310 + Ascend 310 - CPU + CPU - - Khadas VIM3 + Khadas VIM3 - A311D + A311D - CPU + CPU - - Khadas VIM4 + Khadas VIM4 - A311D2 + A311D2 - CPU + CPU - - Khadas Edge2 + Khadas Edge2 - RK3588S + RK3588S - CPU + CPU - - Jetson Nano + Jetson Nano - B01 + B01 - CPU + CPU - - Jetson Nano + Jetson Nano - Orin + Orin - CPU + CPU - - Raspberry Pi 4B + Raspberry Pi 4B - BCM2711 + BCM2711 - CPU + CPU - - Horizon Sunrise Pi + Horizon Sunrise Pi - X3 + X3 - CPU + CPU - - MAIX-III AX-Pi + MAIX-III AX-Pi - AX620A + AX620A - CPU + CPU - - Toybrick + Toybrick - RV1126 + RV1126 - CPU + CPU - - StarFive VisionFive 2 + StarFive VisionFive 2 - StarFive JH7110 + StarFive JH7110 - CPU + CPU - - Jetson Nano + Jetson Nano - B01 + B01 - GPU + GPU - - Jetson Nano + Jetson Nano - Orin + Orin - GPU + GPU - - Khadas VIM3 + Khadas VIM3 - A311D + A311D - NPU + NPU - - Atlas 200 DK + Atlas 200 DK - Ascend 310 + Ascend 310 - NPU + NPU @@ -477,190 +477,190 @@ z - 0.69 + 0.69 - - 6.67 + 6.67 - - 7.82 + 7.82 - - 4.62 + 4.62 - - 4.27 + 4.27 - - 2.30 + 2.30 - - 5.62 + 5.62 - - 2.59 + 2.59 - - 6.23 + 6.23 - - 10.56 + 10.56 - - 83.95 + 83.95 - - 56.78 + 56.78 - - 41.13 + 41.13 - - 10.99 + 10.99 - - 5.23 + 5.23 - - 5.24 + 5.24 - - 2.24 + 2.24 @@ -701,190 +701,190 @@ z - 5.09 + 5.09 - - 78.90 + 78.90 - - 92.21 + 92.21 - - 55.04 + 55.04 - - 39.94 + 39.94 - - 28.94 + 28.94 - - 64.80 + 64.80 - - 20.05 + 20.05 - - 68.82 + 68.82 - - 124.80 + 124.80 - - 2326.96 + 2326.96 - - 1737.74 + 1737.74 - - 1169.96 + 1169.96 - - 25.25 + 25.25 - - 7.59 + 7.59 - - 45.96 + 45.96 - - 2.66 + 2.66 @@ -925,190 +925,190 @@ z - 1.79 + 1.79 - - 36.94 + 36.94 - - 42.93 + 42.93 - - 29.50 + 29.50 - - 17.28 + 17.28 - - 12.44 + 12.44 - - 26.54 + 26.54 - - 9.15 + 9.15 - - 27.81 + 27.81 - - 55.12 + 55.12 - - 823.42 + 823.42 - - 609.51 + 609.51 - - 423.91 + 423.91 - - 13.97 + 13.97 - - 8.48 + 8.48 - - 30.25 + 30.25 - - 2.19 + 2.19 @@ -1149,190 +1149,190 @@ z - 5.68 + 5.68 - - 155.73 + 155.73 - - 201.99 + 201.99 - - 128.58 + 128.58 - - 81.46 + 81.46 - - 53.83 + 53.83 - - 134.36 + 134.36 - - 39.91 + 39.91 - - 153.87 + 153.87 - - 252.05 + 252.05 - - 4371.75 + 4371.75 - - 3260.22 + 3260.22 - - 2267.96 + 2267.96 - - 54.88 + 54.88 - - 16.33 + 16.33 - - 32.11 + 32.11 - - 7.63 + 7.63 @@ -1373,190 +1373,190 @@ z - 78.77 + 78.77 - - 1407.75 + 1407.75 - - 1875.33 + 1875.33 - - 1108.12 + 1108.12 - - 805.54 + 805.54 - - 554.30 + 554.30 - - 1209.12 + 1209.12 - - 400.91 + 400.91 - - 1614.13 + 1614.13 - - 2516.91 + 2516.91 - - 50633.38 + 50633.38 - - 37600.81 + 37600.81 - - 26185.41 + 26185.41 - - 371.32 + 371.32 - - 103.28 + 103.28 - - 408.18 + 408.18 - - 28.59 + 28.59 @@ -1597,190 +1597,190 @@ z - 41.02 + 41.02 - - 253.05 + 253.05 - - 313.66 + 313.66 - - 179.93 + 179.93 - - 136.14 + 136.14 - - 98.03 + 98.03 - - 215.67 + 215.67 - - 125.31 + 125.31 - - 214.59 + 214.59 - - 399.35 + 399.35 - - 3421.19 + 3421.19 - - 2335.89 + 2335.89 - - 1731.61 + 1731.61 - - 63.86 + 63.86 - - 24.46 + 24.46 - - 116.32 + 116.32 - - 20.62 + 20.62 @@ -1821,190 +1821,190 @@ z - 18.76 + 18.76 - - 219.28 + 219.28 - - 360.26 + 360.26 - - 211.02 + 211.02 - - 148.82 + 148.82 - - 103.42 + 103.42 - - 209.80 + 209.80 - - 78.10 + 78.10 - - 325.02 + 325.02 - - 475.98 + 475.98 - - 5899.23 + 5899.23 - - 4267.03 + 4267.03 - - 2988.22 + 2988.22 - - 112.68 + 112.68 - - 27.49 + 27.49 - - 160.70 + 160.70 - - --- + --- @@ -2045,190 +2045,190 @@ z - 18.59 + 18.59 - - 217.18 + 217.18 - - 361.22 + 361.22 - - 210.19 + 210.19 - - 148.91 + 148.91 - - 103.41 + 103.41 - - 209.60 + 209.60 - - 78.03 + 78.03 - - 323.54 + 323.54 - - 475.90 + 475.90 - - 5889.39 + 5889.39 - - 4265.58 + 4265.58 - - 2981.84 + 2981.84 - - 112.48 + 112.48 - - 27.53 + 27.53 - - 160.47 + 160.47 - - --- + --- @@ -2269,190 +2269,190 @@ z - 9.85 + 9.85 - - 158.82 + 158.82 - - 289.82 + 289.82 - - 185.45 + 185.45 - - 127.49 + 127.49 - - 67.15 + 67.15 - - 245.18 + 245.18 - - 134.47 + 134.47 - - 226.09 + 226.09 - - 427.47 + 427.47 - - 3247.56 + 3247.56 - - 2217.15 + 2217.15 - - 1432.21 + 1432.21 - - 36.77 + 36.77 - - 13.58 + 13.58 - - 196.69 + 196.69 - - --- + --- @@ -2493,190 +2493,190 @@ z - 11.03 + 11.03 - - 169.22 + 169.22 - - 318.96 + 318.96 - - 197.16 + 197.16 - - 135.27 + 135.27 - - 70.63 + 70.63 - - 259.28 + 259.28 - - 144.53 + 144.53 - - 240.95 + 240.95 - - 453.60 + 453.60 - - 3281.31 + 3281.31 - - 2217.08 + 2217.08 - - 1425.42 + 1425.42 - - 44.97 + 44.97 - - 15.91 + 15.91 - - 197.61 + 197.61 - - --- + --- @@ -2717,190 +2717,190 @@ z - 19.47 + 19.47 - - 417.31 + 417.31 - - 499.55 + 499.55 - - 335.75 + 335.75 - - 219.81 + 219.81 - - 151.10 + 151.10 - - 346.78 + 346.78 - - 102.30 + 102.30 - - 420.93 + 420.93 - - 653.39 + 653.39 - - 15841.89 + 15841.89 - - 11855.64 + 11855.64 - - 8116.21 + 8116.21 - - 98.80 + 98.80 - - 32.58 + 32.58 - - 73.19 + 73.19 - - 6.99 + 6.99 @@ -2941,190 +2941,190 @@ z - 3.13 + 3.13 - - 70.20 + 70.20 - - 92.66 + 92.66 - - 59.27 + 59.27 - - 38.73 + 38.73 - - 27.45 + 27.45 - - 65.72 + 65.72 - - 18.47 + 18.47 - - 64.20 + 64.20 - - 119.71 + 119.71 - - 2123.08 + 2123.08 - - 1567.09 + 1567.09 - - 1079.69 + 1079.69 - - 28.96 + 28.96 - - 8.41 + 8.41 - - 148.80\* + 148.80\* - - 5.15 + 5.15 @@ -3165,190 +3165,190 @@ z - 3.04 + 3.04 - - 61.72 + 61.72 - - 79.39 + 79.39 - - 52.17 + 52.17 - - 33.68 + 33.68 - - 22.95 + 22.95 - - 56.66 + 56.66 - - 17.08 + 17.08 - - 57.91 + 57.91 - - 102.57 + 102.57 - - 1619.08 + 1619.08 - - 1188.83 + 1188.83 - - 820.15 + 820.15 - - 28.61 + 28.61 - - 9.36 + 9.36 - - 143.17\* + 143.17\* - - 5.41 + 5.41 @@ -3389,190 +3389,190 @@ z - 5.59 + 5.59 - - 78.01 + 78.01 - - 102.49 + 102.49 - - 71.92 + 71.92 - - 47.68 + 47.68 - - 29.63 + 29.63 - - 64.95 + 64.95 - - 24.86 + 24.86 - - 71.35 + 71.35 - - 132.95 + 132.95 - - 2175.34 + 2175.34 - - 1109.61 + 1109.61 - - 1127.61 + 1127.61 - - 67.25 + 67.25 - - 12.91 + 12.91 - - 28.75 + 28.75 - - 6.94 + 6.94 @@ -3613,190 +3613,190 @@ z - 1.19 + 1.19 - - --- + --- - - --- + --- - - --- + --- - - --- + --- - - --- + --- - - 5.56 + 5.56 - - 3.53 + 3.53 - - 5.90 + 5.90 - - 9.93 + 9.93 - - --- + --- - - --- + --- - - --- + --- - - --- + --- - - --- + --- - - --- + --- - - --- + --- @@ -3837,190 +3837,190 @@ z - 15.56 + 15.56 - - 285.75 + 285.75 - - 521.46 + 521.46 - - 327.07 + 327.07 - - 218.22 + 218.22 - - 148.01 + 148.01 - - 356.78 + 356.78 - - 93.58 + 93.58 - - 478.89 + 478.89 - - 678.74 + 678.74 - - 14895.99 + 14895.99 - - 11143.23 + 11143.23 - - 7613.64 + 7613.64 - - 90.84 + 90.84 - - 23.53 + 23.53 - - 41.82 + 41.82 - - 5.58 + 5.58 @@ -4061,190 +4061,190 @@ z - 5.35 + 5.35 - - 92.56 + 92.56 - - 159.80 + 159.80 - - 78.26 + 78.26 - - 54.45 + 54.45 - - 45.03 + 45.03 - - 75.30 + 75.30 - - 27.73 + 27.73 - - 97.05 + 97.05 - - 168.54 + 168.54 - - 1285.54 + 1285.54 - - 905.77 + 905.77 - - 628.64 + 628.64 - - 37.61 + 37.61 - - 10.84 + 10.84 - - 37.34 + 37.34 - - 5.17 + 5.17 @@ -4285,190 +4285,190 @@ z - 2.40 + 2.40 - - 56.00 + 56.00 - - 67.85 + 67.85 - - 35.80 + 35.80 - - 25.83 + 25.83 - - 20.22 + 20.22 - - 38.45 + 38.45 - - 15.00 + 15.00 - - 42.58 + 42.58 - - 77.44 + 77.44 - - 664.73 + 664.73 - - 465.12 + 465.12 - - 319.69 + 319.69 - - 24.47 + 24.47 - - 12.29 + 12.29 - - 19.75 + 19.75 - - 6.27 + 6.27 @@ -4509,190 +4509,190 @@ z - 7.65 + 7.65 - - 90.13 + 90.13 - - 145.83 + 145.83 - - 83.22 + 83.22 - - 57.22 + 57.22 - - 42.90 + 42.90 - - 87.65 + 87.65 - - 33.05 + 33.05 - - 105.60 + 105.60 - - 172.55 + 172.55 - - 1849.87 + 1849.87 - - 1315.48 + 1315.48 - - 910.38 + 910.38 - - 37.39 + 37.39 - - 14.50 + 14.50 - - --- + --- - - 16.45 + 16.45 @@ -4733,190 +4733,190 @@ z - 6.33 + 6.33 - - 83.16 + 83.16 - - 134.02 + 134.02 - - 75.38 + 75.38 - - 53.06 + 53.06 - - 37.06 + 37.06 - - 75.20 + 75.20 - - 31.51 + 31.51 - - 116.15 + 116.15 - - 162.87 + 162.87 - - 1045.98 + 1045.98 - - 736.02 + 736.02 - - 524.52 + 524.52 - - 76.44 + 76.44 - - 26.54 + 26.54 - - --- + --- - - --- + --- @@ -4957,205 +4957,429 @@ z - 4.01 + 4.01 - - 37.02 + 37.02 - - 143.62 + 143.62 - - 32.20 + 32.20 - - 29.47 + 29.47 - - 14.02 + 14.02 - - 48.39 + 48.39 - - 19.16 + 19.16 - - 48.55 + 48.55 - - 84.15 + 84.15 - - 548.36 + 548.36 - - 411.41 + 411.41 - - 288.95 + 288.95 - - 47.26 + 47.26 - - 19.75 + 19.75 - - --- + --- - - --- + --- + + + + + + + + + RAFT + + + + + + + Optical Flow Estimation + + + + + + 360x480 + + + + + + 1023.71 + + + + + + 11978.21 + + + + + + 22969.43 + + + + + + 9608.99 + + + + + + 7945.31 + + + + + + 4643.54 + + + + + + 12659.89 + + + + + + 4404.12 + + + + + + 16176.34 + + + + + + 25455.99 + + + + + + 433898.12 + + + + + + 316818.19 + + + + + + 221318.94 + + + + + + --- + + + + + + --- + + + + + + --- + + + + + + --- - - Units: All data in milliseconds (ms). + + Units: All data in milliseconds (ms). - - \*: Models are quantized in per-channel mode, which run slower than per-tensor quantized models on NPU. + + \*: Models are quantized in per-channel mode, which run slower than per-tensor quantized models on NPU. - - + + diff --git a/benchmark/table_config.yaml b/benchmark/table_config.yaml index c84f62f9..29760370 100644 --- a/benchmark/table_config.yaml +++ b/benchmark/table_config.yaml @@ -164,6 +164,13 @@ Models: acceptable_time: 1000 keyword: "object_tracking_vittrack" + - name: "RAFT" + task: "Optical Flow Estimation" + input_size: "360x480" + folder: "optical_flow_estimation_raft" + acceptable_time: 30000 + keyword: "optical_flow_estimation_raft" + Devices: - name: "Intel 12700K" diff --git a/models/optical_flow_estimation_raft/demo.py b/models/optical_flow_estimation_raft/demo.py index 67a63c76..f6a67218 100644 --- a/models/optical_flow_estimation_raft/demo.py +++ b/models/optical_flow_estimation_raft/demo.py @@ -6,7 +6,7 @@ from raft import Raft # Check OpenCV version -assert cv.__version__ > "4.9.0", \ +assert cv.__version__ >= "4.9.0", \ "Please install latest opencv-python to try this demo: python3 -m pip install --upgrade opencv-python" parser = argparse.ArgumentParser(description='RAFT (https://github.com/princeton-vl/RAFT)')