[fix] default performance + image based split inference

Author: chyomin06

compressai_vision/codecs/base.py

@@ -75,17 +75,25 @@ def encode(
         del bitstream_name  # used in other codecs that write bitstream files
         del codec_output_dir  # used in other codecs that write log files
 
+        mac_calculations = None  # no NN-related complexity calculation
+
         if remote_inference is True:
             org_fH = input["org_input_size"]["height"]
             org_fW = input["org_input_size"]["width"]
 
             num_elements = org_fH * org_fW
             num_frames = len(input["file_names"])
 
-            return {
-                "bytes": [num_elements] * num_frames,
-                "bitstream": input,
-            }
+            enc_time = 0
+
+            return (
+                {
+                    "bytes": [num_elements] * num_frames,
+                    "bitstream": input,
+                },
+                enc_time,
+                mac_calculations,
+            )
 
         # for n-bit quantization error experiments
         max_lvl = ((2**self.nbit_quant) - 1) if self.nbit_quant != -1 else None
@@ -117,8 +125,6 @@ def encode(
             "bypass": time_measure() - start_time,
         }
 
-        mac_calculations = None  # no NN-related complexity calculation
-
         return (
             {
                 "bytes": total_bytes,
@@ -140,14 +146,12 @@ def decode(
         del file_prefix  # used in other codecs that write log files
         del codec_output_dir  # used in other codecs that write log files
 
-        if remote_inference:
-            assert "file_names" in input
-            return input
-
         dec_time = {"bypass": 0}
-
         mac_calculations = None  # no NN-related complexity calculation
 
+        if remote_inference:
+            assert "file_names" in input
+
         return input, dec_time, mac_calculations
 
 

compressai_vision/pipelines/remote_inference/image_remote_inference.py

@@ -102,7 +102,7 @@ def __call__(
                     "org_input_size": org_img_size,
                 }
 
-                res = self._compress(
+                res, enc_time_details, _ = self._compress(
                     codec,
                     frame,
                     self.codec_output_dir,
@@ -147,7 +147,8 @@ def __call__(
             timing["decode"].append((end - start))
 
             start = time_measure()
-            dec_d = {"file_name": dec_seq["file_names"][0]}
+            # dec_d = {"file_name": dec_seq["file_names"][0]}
+            dec_d = {"file_name": dec_seq[0]["file_names"][0]}
 
             pred = vision_model.forward(org_map_func(dec_d))
             end = time_measure()

scripts/evaluation/default_vision_performances.sh

@@ -7,6 +7,7 @@ set -eu
 
 ENTRY_CMD=$1
 TESTDATA_DIR=$2
+DEVICE=$3
 
 # List of entry cmds 
 CMD_OPTS=("compressai-split-inference", "compressai-remote-inference")
@@ -21,7 +22,7 @@ fi
 
 declare -A configs
 
-configs["compressai-split-inference"]="eval_example"
+configs["compressai-split-inference"]="eval_split_inference_example"
 configs["compressai-remote-inference"]="eval_remote_inference_example"
 
 CONF_NAME=${configs[${ENTRY_CMD}]}
@@ -54,7 +55,8 @@ ${ENTRY_CMD} --config-name=${CONF_NAME}.yaml \
              ++evaluator.type=OIC-EVAL \
              ++pipeline.nn_task_part1.load_features=False \
              ++pipeline.nn_task_part1.dump_features=False \
-             ++pipeline.nn_task_part2.dump_features=False
+             ++pipeline.nn_task_part2.dump_features=False \
+             ++misc.device.nn_parts=${DEVICE}
 
 # MPEGOIV6 - Segmentation with Mask RCNN
 ${ENTRY_CMD} --config-name=${CONF_NAME}.yaml \
@@ -70,7 +72,8 @@ ${ENTRY_CMD} --config-name=${CONF_NAME}.yaml \
              ++evaluator.type=OIC-EVAL \
              ++pipeline.nn_task_part1.load_features=False \
              ++pipeline.nn_task_part1.dump_features=False \
-             ++pipeline.nn_task_part2.dump_features=False
+             ++pipeline.nn_task_part2.dump_features=False \
+             ++misc.device.nn_parts=${DEVICE}
 
 # SFU - Detection with Faster RCNN
 for SEQ in \
@@ -102,7 +105,8 @@ do
                  ++evaluator.type=COCO-EVAL \
                  ++pipeline.nn_task_part1.load_features=False \
                  ++pipeline.nn_task_part1.dump_features=False \
-                 ++pipeline.nn_task_part2.dump_features=False
+                 ++pipeline.nn_task_part2.dump_features=False \
+                 ++misc.device.nn_parts=${DEVICE}
 done
 
 # TVD - Object Tracking with JDE
@@ -127,7 +131,8 @@ do
                  ++evaluator.type=MOT-TVD-EVAL \
                  ++pipeline.nn_task_part1.load_features=False \
                  ++pipeline.nn_task_part1.dump_features=False \
-                 ++pipeline.nn_task_part2.dump_features=False
+                 ++pipeline.nn_task_part2.dump_features=False \
+                 ++misc.device.nn_parts=${DEVICE}
 done
 
 # HIEVE - Object Tracking with JDE
@@ -154,5 +159,6 @@ do
                  ++evaluator.type=MOT-HIEVE-EVAL \
                  ++pipeline.nn_task_part1.load_features=False \
                  ++pipeline.nn_task_part1.dump_features=False \
-                 ++pipeline.nn_task_part2.dump_features=False
+                 ++pipeline.nn_task_part2.dump_features=False \
+                 ++misc.device.nn_parts=${DEVICE}
 done