Skip to content

Commit 7c83cc8

Browse files
jianwensongjianwensong
authored
[fix] missing evaluation scripts for pandaset (#19)
1 parent 0fa372f commit 7c83cc8

File tree

3 files changed

+310
-0
lines changed

3 files changed

+310
-0
lines changed

‎scripts/evaluation/pandaset/eval_on_pandaset_hm.sh‎

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
#!/usr/bin/env bash
2+
set -eu
3+
4+
FCM_TESTDATA=""
5+
INNER_CODEC_PATH=""
6+
OUTPUT_DIR=""
7+
EXPERIMENT="test"
8+
DEVICE="cpu"
9+
QP=42
10+
SEQ=""
11+
PIPELINE_PARAMS=""
12+
PIPELINE="split"
13+
14+
#parse args
15+
while [[ $# -gt 0 ]]
16+
do
17+
key="$1"
18+
case $key in
19+
-h|--help)
20+
cat << _EOF_
21+
Runs an evaluation pipeline (split or remote) using HM as codec
22+
23+
RUN OPTIONS:
24+
[-t|--testdata) path to the test dataset, e.g. path/to/fcm_testdata/, default=""]
25+
[-p|--pipeline) type of pipeline, split/remote, default="split"]
26+
[-i|--inner_codec) path to core codec, e.g. /path/to/HM_repo (that contains subfolders bin/ cfg/ ...), default=""]
27+
[-o|--output_dir) root of output folder for logs and artifacts, default=""]
28+
[-e|--exp_name) name of the experiments to locate/label outputs, default="test"]
29+
[-d|--device) cuda or cpu, default="cpu"]
30+
[-q|--qp) quality level, depends on the inner codec, default=42]
31+
[-s|--seq_name) sequence name as used in testdata root folder. E.g., "Traffic_2560x1600_30_val" in sfu_hw_obj, default="42"]
32+
[-x|--extra_params) additional parameters to override default configs (pipeline/codec/evaluation...), default=""]
33+
EXAMPLE [bash eval_on_pandaset_hm.sh -t /path/to/testdata -p split -i /path/to/HM_repo -d cpu -q 32 -s Traffic_2560x1600_30_val]
34+
_EOF_
35+
exit;
36+
;;
37+
-t|--testdata) shift; FCM_TESTDATA="$1"; shift; ;;
38+
-p|--pipeline) shift; PIPELINE="$1"; shift; ;;
39+
-i|--inner_codec) shift; INNER_CODEC_PATH="$1"; shift; ;;
40+
-o|--output_dir) shift; OUTPUT_DIR="$1"; shift; ;;
41+
-e|--exp_name) shift; EXPERIMENT="$1"; shift; ;;
42+
-d|--device) shift; DEVICE="$1"; shift; ;;
43+
-q|--qp) shift; QP="$1"; shift; ;;
44+
-s|--seq_name) shift; SEQ="$1"; shift; ;;
45+
-x|--extra_params) shift; declare -a PIPELINE_PARAMS="($1)"; shift; ;;
46+
*) echo "[ERROR] Unknown parameter $1"; exit; ;;
47+
esac;
48+
done;
49+
50+
export DNNL_MAX_CPU_ISA=AVX2
51+
export DEVICE=${DEVICE}
52+
53+
DATASET_SRC="${FCM_TESTDATA}/PandaSet"
54+
55+
CONF_NAME="eval_split_inference_example.yaml"
56+
if [[ ${PIPELINE} == "remote" ]]; then
57+
CONF_NAME="eval_remote_inference_example.yaml"
58+
fi
59+
60+
61+
62+
INTRA_PERIOD=32
63+
FRAME_RATE=30
64+
65+
echo "============================== RUNNING COMPRESSAI-VISION EVAL== =================================="
66+
echo "Pipeline Type: " ${PIPELINE} " Video"
67+
echo "Datatset location: " ${FCM_TESTDATA}
68+
echo "Output directory: " ${OUTPUT_DIR}
69+
echo "Experiment folder: " "hm"${EXPERIMENT}
70+
echo "Running Device: " ${DEVICE}
71+
echo "Input sequence: " ${SEQ}
72+
echo "Seq. Framerate: " ${FRAME_RATE}
73+
echo "QP for Inner Codec: " ${QP}
74+
echo "Intra Period for Inner Codec: "${INTRA_PERIOD}
75+
echo "Other Parameters: " ${PIPELINE_PARAMS[@]}
76+
echo "=================================================================================================="
77+
78+
compressai-${PIPELINE}-inference --config-name=${CONF_NAME} \
79+
++pipeline.type=video \
80+
++paths._run_root=${OUTPUT_DIR} \
81+
++vision_model.arch=panoptic_rcnn_R_101_FPN_3x \
82+
++dataset.type=Detectron2Dataset \
83+
++dataset.datacatalog=PANDASET \
84+
++dataset.config.imgs_folder='camera/front_camera' \
85+
++dataset.config.ext=jpg \
86+
++dataset.config.root=${DATASET_SRC}/${SEQ} \
87+
++dataset.config.annotation_file=annotations/${SEQ}.npz \
88+
++dataset.config.dataset_name=pandaset-${SEQ} \
89+
++evaluator.type=SEMANTICSEG-EVAL \
90+
++codec.experiment=${EXPERIMENT} \
91+
codec=hm.yaml \
92+
++codec.encoder_config.intra_period=${INTRA_PERIOD} \
93+
++codec.encoder_config.parallel_encoding=True \
94+
++codec.encoder_config.qp=${QP} \
95+
++codec.codec_paths.encoder_exe=${INNER_CODEC_PATH}'/bin/TAppEncoderStatic' \
96+
++codec.codec_paths.decoder_exe=${INNER_CODEC_PATH}'/bin/TAppDecoderStatic' \
97+
++codec.codec_paths.parcat_exe=${INNER_CODEC_PATH}'/bin/parcatStatic' \
98+
++codec.codec_paths.cfg_file=${INNER_CODEC_PATH}'/cfg/encoder_lowdelay_main10.cfg' \
99+
++codec.eval_encode='bitrate' \
100+
++codec.verbosity=0 \
101+
++codec.device=${DEVICE} \
102+
++misc.device.nn_parts=${DEVICE} \
103+
"${PIPELINE_PARAMS[@]}" \
104+

‎scripts/evaluation/pandaset/eval_on_pandaset_vcmrs.sh‎

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
#!/usr/bin/env bash
2+
set -eu
3+
4+
FCM_TESTDATA=""
5+
INNER_CODEC_PATH=""
6+
OUTPUT_DIR=""
7+
EXPERIMENT="test"
8+
DEVICE="cpu"
9+
QP=42
10+
SEQ=""
11+
PIPELINE_PARAMS=""
12+
PIPELINE="split"
13+
14+
#parse args
15+
while [[ $# -gt 0 ]]
16+
do
17+
key="$1"
18+
case $key in
19+
-h|--help)
20+
cat << _EOF_
21+
Runs an evaluation pipeline (split or remote) using VTM as codec
22+
23+
RUN OPTIONS:
24+
[-t|--testdata) path to the test dataset, e.g. path/to/fcm_testdata/, default=""]
25+
[-p|--pipeline) type of pipeline, split/remote, default="split"]
26+
[-i|--inner_codec) path to core codec, e.g. /path/to/VTM_repo (that contains subfolders bin/ cfg/ ...), default=""]
27+
[-o|--output_dir) root of output folder for logs and artifacts, default=""]
28+
[-e|--exp_name) name of the experiments to locate/label outputs, default="test"]
29+
[-d|--device) cuda or cpu, default="cpu"]
30+
[-q|--qp) quality level, depends on the inner codec, default=42]
31+
[-s|--seq_name) sequence name as used in testdata root folder. E.g., "Traffic_2560x1600_30_val" in sfu_hw_obj, default="42"]
32+
[-x|--extra_params) additional parameters to override default configs (pipeline/codec/evaluation...), default=""]
33+
EXAMPLE [bash eval_on_pandaset_vcmrs.sh -t /path/to/testdata -p split -i /path/to/VTM_repo -d cpu -q 32 -s Traffic_2560x1600_30_val]
34+
_EOF_
35+
exit;
36+
;;
37+
-t|--testdata) shift; FCM_TESTDATA="$1"; shift; ;;
38+
-p|--pipeline) shift; PIPELINE="$1"; shift; ;;
39+
-i|--inner_codec) shift; INNER_CODEC_PATH="$1"; shift; ;;
40+
-o|--output_dir) shift; OUTPUT_DIR="$1"; shift; ;;
41+
-e|--exp_name) shift; EXPERIMENT="$1"; shift; ;;
42+
-d|--device) shift; DEVICE="$1"; shift; ;;
43+
-q|--qp) shift; QP="$1"; shift; ;;
44+
-s|--seq_name) shift; SEQ="$1"; shift; ;;
45+
-x|--extra_params) shift; PIPELINE_PARAMS="$1"; shift; ;;
46+
*) echo "[ERROR] Unknown parameter $1"; exit; ;;
47+
esac;
48+
done;
49+
50+
export DNNL_MAX_CPU_ISA=AVX2
51+
export DEVICE=${DEVICE}
52+
53+
DATASET_SRC="${FCM_TESTDATA}/PandaSet"
54+
55+
CONF_NAME="eval_split_inference_example.yaml"
56+
if [[ ${PIPELINE} == "remote" ]]; then
57+
CONF_NAME="eval_remote_inference_example.yaml"
58+
fi
59+
60+
declare -A roi_descriptor
61+
declare -A spatial_descriptor
62+
63+
INTRA_PERIOD=32
64+
FRAME_RATE=30
65+
66+
echo "============================== RUNNING COMPRESSAI-VISION EVAL== =================================="
67+
echo "Pipeline Type: " ${PIPELINE} " Video"
68+
echo "Datatset location: " ${FCM_TESTDATA}
69+
echo "Output directory: " ${OUTPUT_DIR}
70+
echo "Experiment folder: " "vcmrs"${EXPERIMENT}
71+
echo "Running Device: " ${DEVICE}
72+
echo "Input sequence: " ${SEQ}
73+
echo "Seq. Framerate: " ${FRAME_RATE}
74+
echo "QP for Inner Codec: " ${QP}
75+
echo "Intra Period for Inner Codec: "${INTRA_PERIOD}
76+
echo "Other Parameters: " ${PIPELINE_PARAMS}
77+
echo "=================================================================================================="
78+
79+
compressai-${PIPELINE}-inference --config-name=${CONF_NAME} \
80+
++pipeline.type=video \
81+
++pipeline.codec.vcm_mode=True \
82+
++paths._run_root=${OUTPUT_DIR} \
83+
++vision_model.arch=panoptic_rcnn_R_101_FPN_3x \
84+
++dataset.type=Detectron2Dataset \
85+
++dataset.datacatalog=PANDASET \
86+
++dataset.config.imgs_folder='camera/front_camera' \
87+
++dataset.config.ext=jpg \
88+
++dataset.config.root=${DATASET_SRC}/${SEQ} \
89+
++dataset.config.annotation_file=annotations/${SEQ}.npz \
90+
++dataset.config.dataset_name=pandaset-${SEQ} \
91+
++evaluator.type=SEMANTICSEG-EVAL \
92+
++codec.experiment=${EXPERIMENT} \
93+
codec=vcmrs.yaml \
94+
++codec.encoder_config.intra_period=${INTRA_PERIOD} \
95+
++codec.encoder_config.parallel_encoding=True \
96+
++codec.encoder_config.qp=${QP} \
97+
++codec.encoder_config.input_bitdepth=8 \
98+
++codec.eval_encode='bitrate' \
99+
++codec.verbosity=0 \
100+
++codec.device=${DEVICE} \
101+
++misc.device.nn_parts=${DEVICE} \
102+
${PIPELINE_PARAMS} \

‎scripts/evaluation/pandaset/eval_on_pandaset_vtm.sh‎

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
#!/usr/bin/env bash
2+
set -eu
3+
4+
FCM_TESTDATA=""
5+
INNER_CODEC_PATH=""
6+
OUTPUT_DIR=""
7+
EXPERIMENT="test"
8+
DEVICE="cpu"
9+
QP=42
10+
SEQ=""
11+
PIPELINE_PARAMS=""
12+
PIPELINE="split"
13+
14+
#parse args
15+
while [[ $# -gt 0 ]]
16+
do
17+
key="$1"
18+
case $key in
19+
-h|--help)
20+
cat << _EOF_
21+
Runs an evaluation pipeline (split or remote) using VTM as codec
22+
23+
RUN OPTIONS:
24+
[-t|--testdata) path to the test dataset, e.g. path/to/fcm_testdata/, default=""]
25+
[-p|--pipeline) type of pipeline, split/remote, default="split"]
26+
[-i|--inner_codec) path to core codec, e.g. /path/to/VTM_repo (that contains subfolders bin/ cfg/ ...), default=""]
27+
[-o|--output_dir) root of output folder for logs and artifacts, default=""]
28+
[-e|--exp_name) name of the experiments to locate/label outputs, default="test"]
29+
[-d|--device) cuda or cpu, default="cpu"]
30+
[-q|--qp) quality level, depends on the inner codec, default=42]
31+
[-s|--seq_name) sequence name as used in testdata root folder. E.g., "Traffic_2560x1600_30_val" in sfu_hw_obj, default="42"]
32+
[-x|--extra_params) additional parameters to override default configs (pipeline/codec/evaluation...), default=""]
33+
EXAMPLE [bash eval_on_pandaset_vtm.sh -t /path/to/testdata -p split -i /path/to/VTM_repo -d cpu -q 32 -s Traffic_2560x1600_30_val]
34+
_EOF_
35+
exit;
36+
;;
37+
-t|--testdata) shift; FCM_TESTDATA="$1"; shift; ;;
38+
-p|--pipeline) shift; PIPELINE="$1"; shift; ;;
39+
-i|--inner_codec) shift; INNER_CODEC_PATH="$1"; shift; ;;
40+
-o|--output_dir) shift; OUTPUT_DIR="$1"; shift; ;;
41+
-e|--exp_name) shift; EXPERIMENT="$1"; shift; ;;
42+
-d|--device) shift; DEVICE="$1"; shift; ;;
43+
-q|--qp) shift; QP="$1"; shift; ;;
44+
-s|--seq_name) shift; SEQ="$1"; shift; ;;
45+
-x|--extra_params) shift; declare -a PIPELINE_PARAMS="($1)"; shift; ;;
46+
*) echo "[ERROR] Unknown parameter $1"; exit; ;;
47+
esac;
48+
done;
49+
50+
export DNNL_MAX_CPU_ISA=AVX2
51+
export DEVICE=${DEVICE}
52+
53+
DATASET_SRC="${FCM_TESTDATA}/PandaSet"
54+
55+
CONF_NAME="eval_split_inference_example.yaml"
56+
if [[ ${PIPELINE} == "remote" ]]; then
57+
CONF_NAME="eval_remote_inference_example.yaml"
58+
fi
59+
60+
61+
62+
INTRA_PERIOD=32
63+
FRAME_RATE=30
64+
65+
echo "============================== RUNNING COMPRESSAI-VISION EVAL== =================================="
66+
echo "Pipeline Type: " ${PIPELINE} " Video"
67+
echo "Datatset location: " ${FCM_TESTDATA}
68+
echo "Output directory: " ${OUTPUT_DIR}
69+
echo "Experiment folder: " "vtm"${EXPERIMENT}
70+
echo "Running Device: " ${DEVICE}
71+
echo "Input sequence: " ${SEQ}
72+
echo "Seq. Framerate: " ${FRAME_RATE}
73+
echo "QP for Inner Codec: " ${QP}
74+
echo "Intra Period for Inner Codec: "${INTRA_PERIOD}
75+
echo "Other Parameters: " ${PIPELINE_PARAMS[@]}
76+
echo "=================================================================================================="
77+
78+
compressai-${PIPELINE}-inference --config-name=${CONF_NAME} \
79+
++pipeline.type=video \
80+
++paths._run_root=${OUTPUT_DIR} \
81+
++vision_model.arch=panoptic_rcnn_R_101_FPN_3x \
82+
++dataset.type=Detectron2Dataset \
83+
++dataset.datacatalog=PANDASET \
84+
++dataset.config.imgs_folder='camera/front_camera' \
85+
++dataset.config.ext=jpg \
86+
++dataset.config.root=${DATASET_SRC}/${SEQ} \
87+
++dataset.config.annotation_file=annotations/${SEQ}.npz \
88+
++dataset.config.dataset_name=pandaset-${SEQ} \
89+
++evaluator.type=SEMANTICSEG-EVAL \
90+
++codec.experiment=${EXPERIMENT} \
91+
codec=vtm.yaml \
92+
++codec.encoder_config.intra_period=${INTRA_PERIOD} \
93+
++codec.encoder_config.parallel_encoding=True \
94+
++codec.encoder_config.qp=${QP} \
95+
++codec.codec_paths.encoder_exe=${INNER_CODEC_PATH}'/bin/EncoderAppStatic' \
96+
++codec.codec_paths.decoder_exe=${INNER_CODEC_PATH}'/bin/DecoderAppStatic' \
97+
++codec.codec_paths.parcat_exe=${INNER_CODEC_PATH}'/bin/parcatStatic' \
98+
++codec.codec_paths.cfg_file=${INNER_CODEC_PATH}'/cfg/encoder_lowdelay_vtm.cfg' \
99+
++codec.eval_encode='bitrate' \
100+
++codec.verbosity=0 \
101+
++codec.device=${DEVICE} \
102+
++misc.device.nn_parts=${DEVICE} \
103+
"${PIPELINE_PARAMS[@]}" \
104+

0 commit comments

Comments
 (0)