CC-3DT++ for CR-3DT. (#142)

* feat: Update CC-3DT++.

* feat: Support CC-3DT++ for CR-3DT

* feat: Update CC-3DT++ and config refactor.

* fix: Fix test.

Author: RoyYang0714

README.md

@@ -2,7 +2,7 @@
     <!-- pypi-strip -->
     <picture>
     <!-- /pypi-strip -->
-    <img alt="vis4d" src="https://dl.cv.ethz.ch/vis4d/vis4d_logo.svg" width="400">
+    <img alt="vis4d" src="docs/source/_static/vis4d_logo.svg" width="400">
     <!-- pypi-strip -->
     </picture>
     <!-- /pypi-strip -->
@@ -12,7 +12,7 @@
 
 ## Quickstart
 
-You can checkout our [documentation](https://docs.vis.xyz/4d/index.html).
+You can checkout our [documentation](https://vis4d.readthedocs.io).
 
 You can use the [template](https://github.com/SysCV/vis4d-template) here to start your own project with Vis4D.
 
@@ -24,7 +24,7 @@ Installation is as easy as
 python3 -m pip install vis4d
 ```
 
-[For more detailed information, check out our installation guide](docs/source/user_guide/install.rst)
+[For more detailed information, check out our installation guide](https://vis4d.readthedocs.io/en/latest/user_guide/install.html)
 
 ## Basic CLI usage
 
@@ -33,19 +33,13 @@ python3 -m pip install vis4d
 ```bash
 # vis4d.engine
 vis4d fit --config vis4d/zoo/faster_rcnn/faster_rcnn_coco.py --gpus 1
-
-# vis4d.pl
-vis4d-pl fit --config vis4d/zoo/faster_rcnn/faster_rcnn_coco.py --gpus 1
 ```
 
 - To test a model
 
 ```bash
 # vis4d.engine
 vis4d test --config vis4d/zoo/faster_rcnn/faster_rcnn_coco.py --gpus 1
-
-# vis4d.pl
-vis4d-pl test --config vis4d/zoo/faster_rcnn/faster_rcnn_coco.py --gpus 1
 ```
 
 ## DDP
@@ -55,43 +49,27 @@ vis4d-pl test --config vis4d/zoo/faster_rcnn/faster_rcnn_coco.py --gpus 1
 - Local machine / SLURM interactivate job (`job-name=bash`)
 
 ```bash
-# vis4d.engine
-./scripts/dist_train.sh <config-file> <num-gpus>
-
-# vis4d.pl
-vis4d-pl fit --config <config-file> --gpus <num-gpus>
+vis4d fit --config <config-file> --gpus <num-gpus>
 ```
 
 - SLURM
 
 ```bash
-# vis4d.engine
-srun vis4d fit --config <config-file> --gpus <num-gpus> --slurm True
-
-# vis4d.pl
-srun vis4d-pl fit --config <config-file> --gpus <num-gpus>
+srun vis4d fit --config <config-file> --gpus <num-gpus>
 ```
 
 ### Testing
 
 - Local machine / SLURM interactivate job (`job-name=bash`)
 
 ```bash
-# vis4d.engine
-./scripts/dist_test.sh <config-file> <num-gpus>
-
-# vis4d.pl
-vis4d-pl test --config <config-file> --gpus <num-gpus>
+vis4d test --config <config-file> --gpus <num-gpus>
 ```
 
 - SLURM
 
 ```bash
-# vis4d.engine
-srun vis4d test --config <config-file> --gpus <num-gpus> --slurm True
-
-# vis4d.pl
-srun vis4d-pl test --config <config-file> --gpus <num-gpus>
+srun vis4d test --config <config-file> --gpus <num-gpus>
 ```
 
 ## Acknowledgement

docs/source/user_guide/visualization.ipynb

@@ -270,7 +270,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": null,
    "metadata": {},
    "outputs": [
     {
@@ -286,7 +286,7 @@
    ],
    "source": [
     "img = draw_masks(images[1], masks[1], class_ids = class_ids[1])\n",
-    "img = draw_bboxes(img, boxes[0], class_ids = classes[0], scores = scores[0], track_ids = tracks[0], class_id_mapping = {1 : \"Human\", 51: \"Pan\"})\n",
+    "img = draw_bboxes(img, boxes[0], class_ids = classes[0], scores = scores[0], track_ids = tracks[0], class_id_mapping = {1 : \"Human\", 51: \"Pan\"}).as_numpy_image()\n",
     "imshow(img)"
    ]
   },

pyproject.toml

@@ -136,11 +136,10 @@ dependencies = {file = ["requirements/install.txt"]}
 include = ["vis4d*"]
 
 [project.urls]
-"Documentation" = "https://docs.vis.xyz/4d"
+"Documentation" = "https://vis4d.readthedocs.io"
 "Source" = "https://github.com/syscv/vis4d"
-"Tracker" = "https://github.com/syscv/"
+"Tracker" = "https://github.com/syscv/vis4d/issues"
 
 [project.scripts]
 vis4d = "vis4d.engine.run:entrypoint"
-vis4d-pl = "vis4d.pl.run:entrypoint"
 vis4d-zoo = "vis4d.zoo.run:entrypoint"

scripts/eval_nusc/README.md

@@ -0,0 +1,27 @@
+# nuScenes 3D Detection and Tracking Evaluation
+
+This folder contains the code and python environment to run nuScenes 3D detection and tracking evaluation locally.
+
+### Installation
+- Python: 3.6
+
+```bash
+pip install -r nusc.txt
+```
+
+### Run
+```bash
+# Detection
+python run.py \
+--input $FOLDER_OF_PREDICTION \
+--version $VERSION \
+--dataroot $NUSC_DATA_ROOT \
+--mode detection
+
+# Tracking
+python run.py \
+--input $FOLDER_OF_PREDICTION \
+--version $VERSION \
+--dataroot $NUSC_DATA_ROOT \
+--mode tracking
+```

scripts/eval_nusc/nusc.txt

@@ -0,0 +1,84 @@
+argon2-cffi==21.3.0
+argon2-cffi-bindings==21.2.0
+async-generator==1.10
+attrs==22.2.0
+backcall==0.2.0
+bleach==4.1.0
+cachetools==4.2.4
+certifi==2021.5.30
+cffi==1.15.1
+comm==0.1.4
+cycler==0.11.0
+dataclasses==0.8
+decorator==5.1.1
+defusedxml==0.7.1
+descartes==1.1.0
+entrypoints==0.4
+fire==0.5.0
+importlib-metadata==4.8.3
+importlib-resources==5.4.0
+ipykernel==5.5.6
+ipython==7.16.3
+ipython-genutils==0.2.0
+ipywidgets==7.8.1
+jedi==0.17.2
+Jinja2==3.0.3
+joblib==1.1.1
+jsonschema==3.2.0
+jupyter==1.0.0
+jupyter-client==7.1.2
+jupyter-console==6.4.3
+jupyter-core==4.9.2
+jupyterlab-pygments==0.1.2
+jupyterlab-widgets==1.1.7
+kiwisolver==1.3.1
+MarkupSafe==2.0.1
+matplotlib==3.3.4
+mistune==0.8.4
+motmetrics==1.1.3
+nbclient==0.5.9
+nbconvert==6.0.7
+nbformat==5.1.3
+nest-asyncio==1.5.8
+notebook==6.4.10
+numpy==1.19.5
+nuscenes-devkit==1.1.10
+opencv-python==4.5.4.58
+packaging==21.3
+pandas==1.1.5
+pandocfilters==1.5.0
+parso==0.7.1
+pexpect==4.9.0
+pickleshare==0.7.5
+Pillow==8.4.0
+prometheus-client==0.17.1
+prompt-toolkit==3.0.36
+ptyprocess==0.7.0
+pycocotools==2.0.7
+pycparser==2.21
+Pygments==2.14.0
+pyparsing==3.1.1
+pyquaternion==0.9.9
+pyrsistent==0.18.0
+python-dateutil==2.8.2
+pytz==2023.3.post1
+pyzmq==25.1.2
+qtconsole==5.2.2
+QtPy==2.0.1
+scikit-learn==0.24.2
+scipy==1.5.4
+Send2Trash==1.8.2
+Shapely==1.8.5
+six==1.16.0
+termcolor==1.1.0
+terminado==0.12.1
+testpath==0.6.0
+threadpoolctl==3.1.0
+tornado==6.1
+tqdm==4.64.1
+traitlets==4.3.3
+typing_extensions==4.1.1
+wcwidth==0.2.13
+webencodings==0.5.1
+widgetsnbextension==3.6.6
+zipp==3.6.0

scripts/eval_nusc/run.py

@@ -0,0 +1,130 @@
+"""nuScenes evaluation pipeline for Vis4D."""
+
+import argparse
+import os
+import json
+
+from nuscenes import NuScenes
+from nuscenes.eval.detection.evaluate import NuScenesEval
+from nuscenes.eval.detection.config import config_factory
+
+from nuscenes.eval.tracking.evaluate import TrackingEval as track_eval
+from nuscenes.eval.tracking.utils import print_final_metrics
+from nuscenes.eval.tracking.data_classes import TrackingConfig, TrackingMetrics
+from nuscenes.eval.common.config import config_factory as track_configs
+
+
+def eval_detection(
+    version: str,
+    dataroot: str,
+    output_dir: str,
+    result_path: str,
+    eval_set: str,
+) -> None:
+    """Evaluate detection results."""
+    nusc = NuScenes(version=version, dataroot=dataroot, verbose=True)
+
+    nusc_eval = NuScenesEval(
+        nusc,
+        config=config_factory("detection_cvpr_2019"),
+        result_path=result_path,
+        eval_set=eval_set,
+        output_dir=output_dir,
+        verbose=True,
+    )
+    _ = nusc_eval.main(render_curves=False)
+
+
+def eval_tracking(
+    version: str, output_dir: str, result_path: str, root: str, eval_set: str
+) -> None:
+    """Evaluate tracking results."""
+    nusc_eval = track_eval(
+        config=track_configs("tracking_nips_2019"),
+        result_path=result_path,
+        eval_set=eval_set,
+        output_dir=output_dir,
+        verbose=True,
+        nusc_version=version,
+        nusc_dataroot=root,
+    )
+    _ = nusc_eval.main()
+
+
+def evaluate(
+    version: str,
+    dataroot: str,
+    mode: str,
+    output_dir: str,
+    result_path: str,
+    root: str,
+) -> None:
+    """nuScenes evaluation."""
+    if "mini" in version:
+        eval_set = "mini_val"
+    else:
+        eval_set = "val"
+
+    if mode == "tracking":
+        eval_tracking(version, output_dir, result_path, root, eval_set)
+    else:
+        eval_detection(version, dataroot, output_dir, result_path, eval_set)
+
+
+def print_metric_summary(metric_summary_path: str) -> None:
+    """Print metric summary."""
+    with open(metric_summary_path, "r") as f:
+        metrics = json.load(f)
+
+    cfg = TrackingConfig.deserialize(metrics["cfg"])
+    tm = TrackingMetrics(cfg=cfg)
+    tm.add_runtime(metrics["eval_time"])
+    tm.label_metrics = metrics["label_metrics"]
+
+    print_final_metrics(metrics)
+
+
+if __name__ == "__main__":
+    """Main."""
+    parser = argparse.ArgumentParser(description="NuScenes eval for Vis4D.")
+    parser.add_argument(
+        "--input",
+        "-i",
+        help=(
+            "Folder path to the nuScenes format detection / tracking results."
+        ),
+    )
+    parser.add_argument(
+        "--version",
+        "-v",
+        choices=["trainval", "mini"],
+        help="NuScenes dataset version to convert.",
+    )
+    parser.add_argument(
+        "--dataroot",
+        "-d",
+        help="NuScenes dataset root.",
+        default="./data/nuscenes",
+    )
+    parser.add_argument(
+        "-m",
+        "--mode",
+        default="tracking",
+        choices=["tracking", "detection"],
+        help="Conversion mode: detection or tracking.",
+    )
+    args = parser.parse_args()
+
+    if args.mode == "detection":
+        metric = "detect_3d"
+    else:
+        metric = "track_3d"
+
+    evaluate(
+        f"v1.0-{args.version}",
+        args.dataroot,
+        args.mode,
+        args.input,
+        os.path.join(args.input, f"{metric}_predictions.json"),
+        args.dataroot,
+    )