refactor: metrics

Author: fracape

scripts/metrics/compute_overall_map.py scripts/metrics/compute_per_class_map.pyscripts/metrics/compute_overall_map.py renamed to scripts/metrics/compute_per_class_map.py

@@ -36,47 +36,19 @@
 
 from __future__ import annotations
 
-import argparse
-import csv
 import json
 import os
 
 import numpy as np
 import pandas as pd
 import utils
 
-
 # from detectron2.evaluation import COCOEvaluator
 from pycocotools.coco import COCO
 from pycocotools.cocoeval import COCOeval
 
-from compressai_vision.evaluators.evaluators import BaseEvaluator
-
 CLASSES = ["CLASS-AB", "CLASS-C", "CLASS-D", "CLASS-AB*"]
 
-SEQS_BY_CLASS = {
-    CLASSES[0]: [
-        "Traffic_2560x1600_30",
-        "Kimono_1920x1080_24",
-        "ParkScene_1920x1080_24",
-        "Cactus_1920x1080_50",
-        "BasketballDrive_1920x1080_50",
-        "BQTerrace_1920x1080_60",
-    ],
-    CLASSES[1]: [
-        "BasketballDrill_832x480_50",
-        "BQMall_832x480_60",
-        "PartyScene_832x480_50",
-        "RaceHorses_832x480_832x480_30",
-    ],
-    CLASSES[2]: [
-        "BasketballPass_416x240_50",
-        "BQSquare_416x240_60",
-        "BlowingBubbles_416x240_50",
-        "RaceHorses_416x240_30",
-    ],
-    CLASSES[3]: ["ns_Traffic_2560x1600_30", "ns_BQTerrace_1920x1080_60"],
-}
 
 SEQUENCE_TO_OFFSET = {
     "Traffic_2560x1600_30": 10000,
@@ -101,7 +73,7 @@
 NS_SEQ_PREFIX = "ns_"  # Prefix of non-scaled sequences
 
 
-def compute_overall_mAP(seq_root_names, items):
+def compute_per_class_mAP(seq_root_names, items):
     classwise_instances_results = []
     classwise_anchor_images = []
     classwise_annotation = []
@@ -158,10 +130,6 @@ def compute_overall_mAP(seq_root_names, items):
     os.remove(TMP_EVAL_FILE)
     os.remove(TMP_ANCH_FILE)
 
-    # print("\n")
-    # print(summary)
-    # print("\n")
-
     return summary
 
 
@@ -179,80 +147,10 @@ def coco_evaluation(ann_file, detections):
     coco_eval.accumulate()
     coco_eval.summarize()
 
-    import logging
 
     headers = ["AP", "AP50", "AP75", "APS", "APM", "APL"]
     npstat = np.array(coco_eval.stats[:6])
     npstat = npstat * 100  # Percent
-    # npstat = np.around(npstat, 2)
     data_frame = pd.DataFrame([npstat], columns=headers)
 
-    return data_frame
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-
-    parser.add_argument(
-        "-r",
-        "--result_path",
-        required=True,
-        help="For example, '.../logs/runs/[pipeline]/[codec]/[datacatalog]/' ",
-    )
-    parser.add_argument(
-        "-q",
-        "--quality_index",
-        required=False,
-        default=-1,
-        type=int,
-        help="Provide index of quality folders under the `result_path'. quality_index is only meant to point the orderd folders by qp names because there might be different range of qps are used for different sequences",
-    )
-    parser.add_argument(
-        "-a",
-        "--all_qualities",
-        action="store_true",
-        help="run all 6 rate points in MPEG CTCs",
-    )
-    parser.add_argument(
-        "-d",
-        "--dataset_path",
-        required=True,
-        help="For example, '.../vcm_testdata/[dataset]' ",
-    )
-    parser.add_argument(
-        "-c",
-        "--class_to_compute",
-        type=str,
-        choices=CLASSES,
-        required=True,
-    )
-
-    args = parser.parse_args()
-    if args.all_qualities:
-        qualities = range(0, 6)
-    else:
-        qualities = [args.quality_index]
-
-    with open(
-        f"{args.result_path}/{args.class_to_compute}.csv", "w", newline=""
-    ) as file:
-        writer = csv.writer(file)
-        for q in qualities:
-            items = utils.search_items(
-                args.result_path,
-                args.dataset_path,
-                q,
-                SEQS_BY_CLASS[args.class_to_compute],
-                BaseEvaluator.get_coco_eval_info_name,
-            )
-
-            assert (
-                len(items) > 0
-            ), "Nothing relevant information found from given directories..."
-
-            summary = compute_overall_mAP(SEQS_BY_CLASS[args.class_to_compute], items)
-
-            writer.writerow([f"{q}", f"{summary['AP'][0]:.4f}"])
-            print(f"{'=' * 10} FINAL OVERALL mAP SUMMARY {'=' * 10}")
-            print(f"{'-' * 32} AP : {summary['AP'][0]:.4f}")
-            print("\n\n")
+    return data_frame

scripts/metrics/compute_overall_miou.py scripts/metrics/compute_per_class_miou.pyscripts/metrics/compute_overall_miou.py renamed to scripts/metrics/compute_per_class_miou.py

@@ -38,7 +38,7 @@
 
 import json
 
-def compute_overall_mIoU(class_name, items):
+def compute_per_class_mIoU(items):
     miou_acc = 0.0
     for item in items:
         with open(item["eval_info"], "r") as f:

scripts/metrics/compute_overall_mot.py scripts/metrics/compute_per_class_mota.pyscripts/metrics/compute_overall_mot.py renamed to scripts/metrics/compute_per_class_mota.py

@@ -44,6 +44,14 @@
 
 from compressai_vision.evaluators.evaluators import MOT_JDE_Eval
 
+CLASSES = ["TVD", "HIEVE-1080P", "HIEVE-720P"]
+
+SEQS_BY_CLASS = {
+    CLASSES[0]: ["TVD-01", "TVD-02", "TVD-03"],
+    CLASSES[1]: ["HIEVE-13", "HIEVE-16"],
+    CLASSES[2]: ["HIEVE-2", "HIEVE-17", "HIEVE-18"],
+}
+
 
 def get_accumulator_res_for_tvd(item: Dict):
     _gt_pd = MOT_JDE_Eval._load_gt_in_motchallenge(item[utils.GT_INFO_KEY])
@@ -67,7 +75,7 @@ def get_accumulator_res_for_hieve(item: Dict):
     return acc, None, item[utils.SEQ_NAME_KEY]
 
 
-def compute_overall_mota(class_name, items):
+def compute_per_class_mota(class_name, items):
     get_accumulator_res = {
         CLASSES[0]: get_accumulator_res_for_tvd,
         CLASSES[1]: get_accumulator_res_for_hieve,

scripts/metrics/gen_mpeg_cttc_csv.py

@@ -44,9 +44,9 @@
 import pandas as pd
 import utils
 
-from compute_overall_map import compute_overall_mAP
-from compute_overall_miou import compute_overall_mIoU
-from compute_overall_mot import compute_overall_mota
+from compute_per_class_mAP import compute_per_class_mAP
+from compute_per_class_mIoU import compute_per_class_mIoU
+from compute_per_class_mota import compute_per_class_mota
 from curve_fitting import convert_to_monotonic_points_SFU
 
 from compressai_vision.datasets import get_seq_info
@@ -102,18 +102,18 @@ def df_append(df1, df2):
     return out
 
 
-def generate_classwise_df(result_df, classes: dict):
-    classwise = pd.DataFrame(columns=result_df.columns)
-    classwise.drop(columns=["fps", "num_of_coded_frame"], inplace=True)
+def generate_class_df(result_df, classes: dict):
+    class_data = pd.DataFrame(columns=result_df.columns)
+    class_data.drop(columns=["fps", "num_of_coded_frame"], inplace=True)
 
     for tag, item in classes.items():
-        output = compute_class_wise_results(result_df, tag, item)
-        classwise_df = df_append(classwise, output)
+        output = compute_per_class_results(result_df, tag, item)
+        classwise_df = df_append(class_data, output)
 
     return classwise_df
 
 
-def compute_class_wise_results(result_df, name, sequences):
+def compute_per_class_results(result_df, name, sequences):
     samples = None
     num_points = prev_num_points = -1
     output = pd.DataFrame(columns=result_df.columns)
@@ -168,7 +168,7 @@ def compute_class_wise_results(result_df, name, sequences):
 def generate_csv_classwise_video_map(
     result_path,
     dataset_path,
-    dict_of_classwise_seq,
+    dict_of_class_seq,
     metric="AP",
     gt_folder="annotations",
     nb_operation_points: int = 4,
@@ -177,7 +177,7 @@ def generate_csv_classwise_video_map(
     dataset_prefix: str = None,
 ):
     seq_list = []
-    [seq_list.extend(sequences) for sequences in dict_of_classwise_seq.values()]
+    [seq_list.extend(sequences) for sequences in dict_of_class_seq.values()]
 
     opts_metrics = {"AP": 0, "AP50": 1, "AP75": 2, "APS": 3, "APM": 4, "APL": 5}
     results_df = read_df_rec(result_path, dataset_prefix, seq_list, nb_operation_points)
@@ -192,14 +192,14 @@ def generate_csv_classwise_video_map(
     ## drop columns
     output_df.drop(columns=["fps", "num_of_coded_frame"], inplace=True)
 
-    for classwise_name, classwise_seqs in dict_of_classwise_seq.items():
+    for class_name, class_seqs in dict_of_class_seq.items():
         class_wise_maps = []
         for q in range(nb_operation_points):
             items = utils.search_items(
                 result_path,
                 dataset_path,
                 q,
-                classwise_seqs,
+                class_seqs,
                 BaseEvaluator.get_coco_eval_info_name,
                 by_name=True,
                 gt_folder=gt_folder,
@@ -212,15 +212,15 @@ def generate_csv_classwise_video_map(
             ), "No evaluation information found in provided result directories..."
 
             if not skip_classwise:
-                summary = compute_overall_mAP(
-                    dict_of_classwise_seq[classwise_name], items
+                summary = compute_per_class_mAP(
+                    dict_of_class_seq[class_name], items
                 )
                 maps = summary.values[0][opts_metrics[metric]]
                 class_wise_maps.append(maps)
 
         if not skip_classwise and nb_operation_points > 0:
-            class_wise_results_df = generate_classwise_df(
-                results_df, {classwise_name: classwise_seqs}
+            class_wise_results_df = generate_class_df(
+                results_df, {class_name: class_seqs}
             )
             class_wise_results_df["end_accuracy"] = class_wise_maps
 
@@ -232,12 +232,12 @@ def generate_csv_classwise_video_map(
 def generate_csv_classwise_video_mota(
     result_path,
     dataset_path,
-    dict_of_classwise_seq,
+    dict_of_class_seq,
     nb_operation_points: int = 4,
     dataset_prefix: str = None,
 ):
     seq_list = []
-    [seq_list.extend(sequences) for sequences in dict_of_classwise_seq.values()]
+    [seq_list.extend(sequences) for sequences in dict_of_class_seq.values()]
 
     results_df = read_df_rec(result_path, dataset_prefix, seq_list, nb_operation_points)
     results_df = results_df.sort_values(by=["Dataset", "qp"], ascending=[True, True])
@@ -249,29 +249,29 @@ def generate_csv_classwise_video_mota(
     ## drop columns
     output_df.drop(columns=["fps", "num_of_coded_frame"], inplace=True)
 
-    for classwise_name, classwise_seqs in dict_of_classwise_seq.items():
+    for class_name, class_seqs in dict_of_class_seq.items():
         class_wise_motas = []
         for q in range(nb_operation_points):
             items = utils.search_items(
                 result_path,
                 dataset_path,
                 q,
-                classwise_seqs,
+                class_seqs,
                 BaseEvaluator.get_jde_eval_info_name,
             )
 
             assert (
                 len(items) > 0
             ), "Nothing relevant information found from given directories..."
 
-            summary, _ = compute_overall_mota(classwise_name, items)
+            summary, _ = compute_per_class_mota(class_name, items)
 
             mota = summary.values[-1][13] * 100.0
             class_wise_motas.append(mota)
 
         if nb_operation_points > 0:
-            class_wise_results_df = generate_classwise_df(
-                results_df, {classwise_name: classwise_seqs}
+            class_wise_results_df = generate_class_df(
+                results_df, {class_name: class_seqs}
             )
 
             class_wise_results_df["end_accuracy"] = class_wise_motas
@@ -284,12 +284,11 @@ def generate_csv_classwise_video_mota(
 def generate_csv_classwise_video_miou(
     result_path,
     dataset_path,
-    dict_of_classwise_seq,
+    dict_of_class_seq,
     nb_operation_points: int = 4,
-    dataset_prefix: str = None,
 ):
     seq_list = []
-    [seq_list.extend(sequences) for sequences in dict_of_classwise_seq.values()]
+    [seq_list.extend(sequences) for sequences in dict_of_class_seq.values()]
 
     results_df = read_df_rec(result_path, "", seq_list, nb_operation_points)
 
@@ -303,14 +302,14 @@ def generate_csv_classwise_video_miou(
     ## drop columns
     output_df.drop(columns=["fps", "num_of_coded_frame"], inplace=True)
 
-    for classwise_name, classwise_seqs in dict_of_classwise_seq.items():
+    for class_name, class_seqs in dict_of_class_seq.items():
         class_wise_mious = []
         for q in range(nb_operation_points):
             items = utils.search_items(
                 result_path,
                 dataset_path,
                 q,
-                classwise_seqs,
+                class_seqs,
                 BaseEvaluator.get_miou_eval_info_name,
                 by_name=True,
                 pandaset_flag=True,
@@ -320,16 +319,16 @@ def generate_csv_classwise_video_miou(
                 len(items) > 0
             ), "Nothing relevant information found from given directories..."
 
-            miou = compute_overall_mIoU(classwise_name, items)
+            miou = compute_per_class_mIoU(class_name, items)
             class_wise_mious.append(miou)
 
         matched_seq_names = []
         for seq_info in items:
             name, _, _ = get_seq_info(seq_info[utils.SEQ_INFO_KEY])
             matched_seq_names.append(name)
 
-        class_wise_results_df = generate_classwise_df(
-            results_df, {classwise_name: classwise_seqs}
+        class_wise_results_df = generate_class_df(
+            results_df, {class_name: class_seqs}
         )
 
         class_wise_results_df["end_accuracy"] = class_wise_mious

scripts/metrics/utils.py

@@ -40,12 +40,9 @@
 import re
 
 from pathlib import Path
-from typing import Dict, Optional
 
 __all__ = [
     "get_seq_number",
-    # "get_eval_info_path",
-    # "get_seq_info_path",
 ]
 
 SEQ_NAME_KEY = "seq_name"