Merge pull request #194 from lpylpy0514:main

Add VIT track model and demo #194

GSOC Realtime tracking model
opencv repo PR link is [here](opencv/opencv#24201)

Author: lpylpy0514

.gitattributes

@@ -20,3 +20,4 @@
 *.gif filter=lfs diff=lfs merge=lfs -text
 *.png filter=lfs diff=lfs merge=lfs -text
 *.webp filter=lfs diff=lfs merge=lfs -text
+

models/object_tracking_vittrack/README.md

@@ -0,0 +1,36 @@
+# VIT tracker
+
+VIT tracker(vision transformer tracker) is a much better model for  real-time object tracking. VIT tracker can achieve speeds exceeding  nanotrack by 20% in single-threaded mode with ARM chip, and the  advantage becomes even more pronounced in multi-threaded mode. In  addition, on the dataset, vit tracker demonstrates better performance  compared to nanotrack. Moreover, vit trackerprovides confidence values  during the tracking process, which can be used to determine if the  tracking is currently lost.
+
+video demo: https://youtu.be/MJiPnu1ZQRI
+ In target tracking tasks, the score is an important indicator that can  indicate whether the current target is lost. In the video, vit tracker  can track the target and display the current score in the upper left  corner of the video. When the target is lost, the score drops  significantly. While nanotrack will only return 0.9 score in any  situation, so that we cannot determine whether the target is lost.
+
+**NOTE: OpenCV > 4.8.0**
+
+
+# speed test
+
+NOTE: The speed below is tested by **onnxruntime** because opencv has poor support for the transformer architecture for now.
+
+ONNX speed test on ARM platform(apple M2)(ms):
+
+| thread nums | 1    | 2    | 3    | 4             |
+| ----------- | ---- | ---- | ---- | ------------- |
+| nanotrack   | 5.25 | 4.86 | 4.72 | 4.49          |
+| vit tracker | 4.18 | 2.41 | 1.97 | **1.46 (3X)** |
+
+ONNX speed test on x86 platform(intel i3 10105)(ms):
+
+| thread nums | 1    | 2    | 3    | 4    |
+| ----------- | ---- | ---- | ---- | ---- |
+| nanotrack   | 3.20 | 2.75 | 2.46 | 2.55 |
+| vit tracker | 3.84 | 2.37 | 2.10 | 2.01 |
+
+# performance test
+
+preformance test on lasot dataset(AUC is the most important data. Higher AUC means better tracker):
+
+| LASOT       | AUC  | P    | Pnorm |
+| ----------- | ---- | ---- | ----- |
+| nanotrack   | 46.8 | 45.0 | 43.3  |
+| vit tracker | 48.6 | 44.8 | 54.7  |

models/object_tracking_vittrack/demo.py

@@ -0,0 +1,79 @@
+import cv2 as cv
+import argparse
+
+# Check OpenCV version
+assert cv.__version__ > "4.8.0", \
+       "Please install latest opencv-python to try this demo: python3 -m pip install --upgrade opencv-python"
+
+parser = argparse.ArgumentParser(
+    description="VIT track opencv API")
+parser.add_argument('--input', '-i', type=str,
+                    help='Usage: Set path to the input video. Omit for using default camera.')
+parser.add_argument('--model_path', type=str, default='vitTracker.onnx',
+                    help='Usage: Set model path, defaults to vitTracker.onnx.')
+args = parser.parse_args()
+
+def visualize(image, bbox, score, isLocated, fps=None, box_color=(0, 255, 0),text_color=(0, 255, 0), fontScale = 1, fontSize = 1):
+    output = image.copy()
+    h, w, _ = output.shape
+
+    if fps is not None:
+        cv.putText(output, 'FPS: {:.2f}'.format(fps), (0, 30), cv.FONT_HERSHEY_DUPLEX, fontScale, text_color, fontSize)
+
+    if isLocated and score >= 0.3:
+        # bbox: Tuple of length 4
+        x, y, w, h = bbox
+        cv.rectangle(output, (x, y), (x+w, y+h), box_color, 2)
+        cv.putText(output, '{:.2f}'.format(score), (x, y+20), cv.FONT_HERSHEY_DUPLEX, fontScale, text_color, fontSize)
+    else:
+        text_size, baseline = cv.getTextSize('Target lost!', cv.FONT_HERSHEY_DUPLEX, fontScale, fontSize)
+        text_x = int((w - text_size[0]) / 2)
+        text_y = int((h - text_size[1]) / 2)
+        cv.putText(output, 'Target lost!', (text_x, text_y), cv.FONT_HERSHEY_DUPLEX, fontScale, (0, 0, 255), fontSize)
+
+    return output
+
+if __name__ == '__main__':
+
+    params = cv.TrackerVit_Params()
+    params.net = args.model_path
+    model = cv.TrackerVit_create(params)
+
+    # Read from args.input
+    _input = args.input
+    if args.input is None:
+        device_id = 0
+        _input = device_id
+    video = cv.VideoCapture(_input)
+
+    # Select an object
+    has_frame, first_frame = video.read()
+    if not has_frame:
+        print('No frames grabbed!')
+        exit()
+    first_frame_copy = first_frame.copy()
+    cv.putText(first_frame_copy, "1. Drag a bounding box to track.", (0, 15), cv.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0))
+    cv.putText(first_frame_copy, "2. Press ENTER to confirm", (0, 35), cv.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0))
+    roi = cv.selectROI('vitTrack Demo', first_frame_copy)
+    print("Selected ROI: {}".format(roi))
+
+    # Init tracker with ROI
+    model.init(first_frame, roi)
+
+    # Track frame by frame
+    tm = cv.TickMeter()
+    while cv.waitKey(1) < 0:
+        has_frame, frame = video.read()
+        if not has_frame:
+            print('End of video')
+            break
+        # Inference
+        tm.start()
+        isLocated, bbox = model.update(frame)
+        score = model.getTrackingScore()
+        tm.stop()
+        # Visualize
+        frame = visualize(frame, bbox, score, isLocated, fps=tm.getFPS())
+        cv.imshow('vittrack Demo', frame)
+        tm.reset()
+

models/object_tracking_vittrack/object_tracking_vittrack_2023sep.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b4e32f6eec82e1aecf90077b2ea2589c4dfd30e8533e9f7c5da4125f52588af
+size 714726