diff --git a/Readme.md b/Readme.md index 4de0b9f..43f9a08 100644 --- a/Readme.md +++ b/Readme.md @@ -71,6 +71,9 @@ print(mh.list_metrics_markdown()) | pred_frequencies | `pd.Series` Total number of occurrences of individual predictions over all frames. | | track_ratios | `pd.Series` Ratio of assigned to total appearance count per unique object id. | | id_global_assignment | `dict` ID measures: Global min-cost assignment for ID measures. | +| deta_alpha | HOTA: Detection Accuracy (DetA) for a given threshold. | +| assa_alpha | HOTA: Association Accuracy (AssA) for a given threshold. | +| hota_alpha | HOTA: Higher Order Tracking Accuracy (HOTA) for a given threshold. | @@ -362,6 +365,54 @@ OVERALL 80.0% 80.0% 80.0% 80.0% 80.0% 4 2 2 0 2 2 1 1 50.0% 0.275 """ ``` +#### [Underdeveloped] Computing HOTA metrics + +Computing HOTA metrics is also possible. However, it cannot be used with the `Accumulator` class directly, as HOTA requires to computing a reweighting matrix from all the frames at the beginning. Here is an example of how to use it: + +```python +import os +import numpy as np +import motmetrics as mm + + +def compute_motchallenge(dir_name): + # `gt.txt` and `test.txt` should be prepared in MOT15 format + df_gt = mm.io.loadtxt(os.path.join(dir_name, "gt.txt")) + df_test = mm.io.loadtxt(os.path.join(dir_name, "test.txt")) + # Require different thresholds for matching + th_list = np.arange(0.05, 0.99, 0.05) + res_list = mm.utils.compare_to_groundtruth_reweighting(df_gt, df_test, "iou", distth=th_list) + return res_list + +# `data_dir` is the directory containing the gt.txt and test.txt files +acc = compute_motchallenge("data_dir") +mh = mm.metrics.create() + +summary = mh.compute_many( + acc, + metrics=[ + "deta_alpha", + "assa_alpha", + "hota_alpha", + ], + generate_overall=True, # `Overall` is the average we need only +) +strsummary = mm.io.render_summary( + summary.iloc[[-1], :], # Use list to preserve `DataFrame` type + formatters=mh.formatters, + namemap={"hota_alpha": "HOTA", "assa_alpha": "ASSA", "deta_alpha": "DETA"}, +) +print(strsummary) +""" +# data_dir=motmetrics/data/TUD-Campus + DETA ASSA HOTA +OVERALL 41.8% 36.9% 39.1% +# data_dir=motmetrics/data/TUD-Stadtmitte + DETA ASSA HOTA +OVERALL 39.2% 40.9% 39.8% +""" +``` + ### Computing distances Up until this point we assumed the pairwise object/hypothesis distances to be known. Usually this is not the case. You are mostly given either rectangles or points (centroids) of related objects. To compute a distance matrix from them you can use `motmetrics.distance` module as shown below. diff --git a/motmetrics/distances.py b/motmetrics/distances.py index 93dfe1e..8ad7113 100644 --- a/motmetrics/distances.py +++ b/motmetrics/distances.py @@ -80,7 +80,7 @@ def boxiou(a, b): math_util.quiet_divide(i_vol, u_vol)) -def iou_matrix(objs, hyps, max_iou=1.): +def iou_matrix(objs, hyps, max_iou=1., return_dist=True): """Computes 'intersection over union (IoU)' distance matrix between object and hypothesis rectangles. The IoU is computed as @@ -104,11 +104,14 @@ def iou_matrix(objs, hyps, max_iou=1.): Maximum tolerable overlap distance. Object / hypothesis points with larger distance are set to np.nan signalling do-not-pair. Defaults to 0.5 + return_dist : bool + If true, return distance matrix. If false, return similarity (IoU) matrix. Returns ------- C : NxK array Distance matrix containing pairwise distances or np.nan. + if `return_dist` is False, then the matrix contains the pairwise IoU. """ if np.size(objs) == 0 or np.size(hyps) == 0: @@ -119,5 +122,7 @@ def iou_matrix(objs, hyps, max_iou=1.): assert objs.shape[1] == 4 assert hyps.shape[1] == 4 iou = boxiou(objs[:, None], hyps[None, :]) - dist = 1 - iou - return np.where(dist > max_iou, np.nan, dist) + if return_dist: + dist = 1 - iou + return np.where(dist > max_iou, np.nan, dist) + return iou diff --git a/motmetrics/metrics.py b/motmetrics/metrics.py index e16f16c..eeaef77 100644 --- a/motmetrics/metrics.py +++ b/motmetrics/metrics.py @@ -9,14 +9,12 @@ # pylint: disable=redefined-outer-name -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function -from collections import OrderedDict import inspect import logging import time +from collections import OrderedDict import numpy as np import pandas as pd @@ -495,6 +493,22 @@ def num_predictions(df, pred_frequencies): simple_add_func.append(num_predictions) +def num_gt_ids(df): + """Number of unique gt ids.""" + return df.full["OId"].dropna().unique().shape[0] + + +simple_add_func.append(num_gt_ids) + + +def num_dt_ids(df): + """Number of unique dt ids.""" + return df.full["HId"].dropna().unique().shape[0] + + +simple_add_func.append(num_dt_ids) + + def track_ratios(df, obj_frequencies): """Ratio of assigned to total appearance count per unique object id.""" tracked = df.noraw[df.noraw.Type != "MISS"]["OId"].value_counts() @@ -597,6 +611,64 @@ def recall_m(partials, num_detections, num_objects): return math_util.quiet_divide(num_detections, num_objects) +def deta_alpha(df, num_detections, num_objects, num_false_positives): + r"""DeTA under specific threshold $\alpha$ + Source: https://jonathonluiten.medium.com/how-to-evaluate-tracking-with-the-hota-metrics-754036d183e1 + """ + del df # unused + return math_util.quiet_divide(num_detections, max(1, num_objects + num_false_positives)) + + +def deta_alpha_m(partials): + res = 0 + for v in partials: + res += v["deta_alpha"] + return math_util.quiet_divide(res, len(partials)) + + +def assa_alpha(df, num_detections, num_gt_ids, num_dt_ids): + r"""AssA under specific threshold $\alpha$ + Source: https://github.com/JonathonLuiten/TrackEval/blob/12c8791b303e0a0b50f753af204249e622d0281a/trackeval/metrics/hota.py#L107-L108 + """ + max_gt_ids = int(df.noraw.OId.max()) + max_dt_ids = int(df.noraw.HId.max()) + + match_count_array = np.zeros((max_gt_ids, max_dt_ids)) + gt_id_counts = np.zeros((max_gt_ids, 1)) + tracker_id_counts = np.zeros((1, max_dt_ids)) + for idx in range(len(df.noraw)): + oid, hid = df.noraw.iloc[idx, 1], df.noraw.iloc[idx, 2] + if df.noraw.iloc[idx, 0] in ["SWITCH", "MATCH"]: + match_count_array[int(oid) - 1, int(hid) - 1] += 1 + if oid == oid: # check non nan + gt_id_counts[int(oid) - 1] += 1 + if hid == hid: + tracker_id_counts[0, int(hid) - 1] += 1 + + ass_a = match_count_array / np.maximum(1, gt_id_counts + tracker_id_counts - match_count_array) + return math_util.quiet_divide((ass_a * match_count_array).sum(), max(1, num_detections)) + + +def assa_alpha_m(partials): + res = 0 + for v in partials: + res += v["assa_alpha"] + return math_util.quiet_divide(res, len(partials)) + + +def hota_alpha(df, deta_alpha, assa_alpha): + r"""HOTA under specific threshold $\alpha$""" + del df + return (deta_alpha * assa_alpha) ** 0.5 + + +def hota_alpha_m(partials): + res = 0 + for v in partials: + res += v["hota_alpha"] + return math_util.quiet_divide(res, len(partials)) + + class DataFrameMap: # pylint: disable=too-few-public-methods def __init__(self, full, raw, noraw, extra): self.full = full @@ -783,6 +855,8 @@ def create(): m.register(num_detections, formatter="{:d}".format) m.register(num_objects, formatter="{:d}".format) m.register(num_predictions, formatter="{:d}".format) + m.register(num_gt_ids, formatter="{:d}".format) + m.register(num_dt_ids, formatter="{:d}".format) m.register(num_unique_objects, formatter="{:d}".format) m.register(track_ratios) m.register(mostly_tracked, formatter="{:d}".format) @@ -802,6 +876,10 @@ def create(): m.register(idr, formatter="{:.1%}".format) m.register(idf1, formatter="{:.1%}".format) + m.register(deta_alpha, formatter="{:.1%}".format) + m.register(assa_alpha, formatter="{:.1%}".format) + m.register(hota_alpha, formatter="{:.1%}".format) + return m diff --git a/motmetrics/mot.py b/motmetrics/mot.py index ec7ae59..39771f9 100644 --- a/motmetrics/mot.py +++ b/motmetrics/mot.py @@ -134,7 +134,7 @@ def _append_to_events(self, typestr, oid, hid, distance): self._events['HId'].append(hid) self._events['D'].append(distance) - def update(self, oids, hids, dists, frameid=None, vf=''): + def update(self, oids, hids, dists, frameid=None, vf='', similartiy_matrix=None, th=None): """Updates the accumulator with frame specific objects/detections. This method generates events based on the following algorithm [1]: @@ -202,6 +202,12 @@ def update(self, oids, hids, dists, frameid=None, vf=''): self._append_to_indices(frameid, next(eid)) self._append_to_events('RAW', np.nan, np.nan, np.nan) + # Postcompute the distance matrix if necessary. (e.g., HOTA) + cost_for_matching = dists.copy() + if similartiy_matrix is not None and th is not None: + dists = 1 - similartiy_matrix + dists = np.where(similartiy_matrix < th - np.finfo("float").eps, np.nan, dists) + # There must be at least one RAW event per object and hypothesis. # Record all finite distances as RAW events. valid_i, valid_j = np.where(np.isfinite(dists)) @@ -224,34 +230,36 @@ def update(self, oids, hids, dists, frameid=None, vf=''): if oids.size * hids.size > 0: # 1. Try to re-establish tracks from correspondences in last update - for i in range(oids.shape[0]): - # No need to check oids_masked[i] here. - if not (oids[i] in self.m and self.last_match[oids[i]] == self.last_update_frameid): - continue - - hprev = self.m[oids[i]] - j, = np.where(~hids_masked & (hids == hprev)) - if j.shape[0] == 0: - continue - j = j[0] + # ignore this if post processing is performed (e.g., HOTA) + if similartiy_matrix is None or th is None: + for i in range(oids.shape[0]): + # No need to check oids_masked[i] here. + if not (oids[i] in self.m and self.last_match[oids[i]] == self.last_update_frameid): + continue + + hprev = self.m[oids[i]] + j, = np.where(~hids_masked & (hids == hprev)) + if j.shape[0] == 0: + continue + j = j[0] + + if np.isfinite(dists[i, j]): + o = oids[i] + h = hids[j] + oids_masked[i] = True + hids_masked[j] = True + self.m[oids[i]] = hids[j] - if np.isfinite(dists[i, j]): - o = oids[i] - h = hids[j] - oids_masked[i] = True - hids_masked[j] = True - self.m[oids[i]] = hids[j] - - self._append_to_indices(frameid, next(eid)) - self._append_to_events('MATCH', oids[i], hids[j], dists[i, j]) - self.last_match[o] = frameid - self.hypHistory[h] = frameid + self._append_to_indices(frameid, next(eid)) + self._append_to_events('MATCH', oids[i], hids[j], dists[i, j]) + self.last_match[o] = frameid + self.hypHistory[h] = frameid # 2. Try to remaining objects/hypotheses dists[oids_masked, :] = np.nan dists[:, hids_masked] = np.nan - rids, cids = linear_sum_assignment(dists) + rids, cids = linear_sum_assignment(cost_for_matching) for i, j in zip(rids, cids): if not np.isfinite(dists[i, j]): @@ -265,10 +273,10 @@ def update(self, oids, hids, dists, frameid=None, vf=''): # self.m[o] != h and # abs(frameid - self.last_occurrence[o]) <= self.max_switch_time) switch_condition = ( - o in self.m and - self.m[o] != h and - o in self.last_occurrence and # Ensure the object ID 'o' is initialized in last_occurrence - abs(frameid - self.last_occurrence[o]) <= self.max_switch_time + o in self.m and + self.m[o] != h and + o in self.last_occurrence and # Ensure the object ID 'o' is initialized in last_occurrence + abs(frameid - self.last_occurrence[o]) <= self.max_switch_time ) is_switch = switch_condition ###################################################################### @@ -470,7 +478,7 @@ def merge_event_dataframes(dfs, update_frame_indices=True, update_oids=True, upd copy['HId'] = copy['HId'].map(lambda x: hid_map[x], na_action='ignore') infos['hid_map'] = hid_map - r = pd.concat([r,copy]) + r = pd.concat([r, copy]) mapping_infos.append(infos) if return_mappings: diff --git a/motmetrics/tests/test_metrics.py b/motmetrics/tests/test_metrics.py index 504fd28..891154e 100644 --- a/motmetrics/tests/test_metrics.py +++ b/motmetrics/tests/test_metrics.py @@ -537,3 +537,49 @@ def my_motp(df: mm.metrics.DataFrameMap): ) print(summary) + + +def test_hota(): + TUD_golden_ans = { # From TrackEval + "TUD-Campus": {"hota": 0.3913974378451139, "deta": 0.418047030142763, "assa": 0.36912068120832836}, + "TUD-Stadtmitte": {"hota": 0.3978490169927877, "deta": 0.3922675723693166, "assa": 0.4088407518112996} + } + + DATA_DIR = "motmetrics/data" + + def compute_motchallenge(dname): + df_gt = mm.io.loadtxt(os.path.join(dname, "gt.txt")) + df_test = mm.io.loadtxt(os.path.join(dname, "test.txt")) + th_list = np.arange(0.05, 0.99, 0.05) + res_list = mm.utils.compare_to_groundtruth_reweighting(df_gt, df_test, "iou", distth=th_list) + return res_list + + accs = [compute_motchallenge(os.path.join(DATA_DIR, d)) for d in TUD_golden_ans.keys()] + mh = mm.metrics.create() + + for dataset_idx, dname in enumerate(TUD_golden_ans.keys()): + deta = [] + assa = [] + hota = [] + for alpha_idx in range(len(accs[dataset_idx])): + summary = mh.compute_many( + [accs[dataset_idx][alpha_idx]], + metrics=[ + "deta_alpha", + "assa_alpha", + "hota_alpha", + ], + names=[dname], + generate_overall=False, + ) + deta.append(float(summary["deta_alpha"].iloc[0])) + assa.append(float(summary["assa_alpha"].iloc[0])) + hota.append(float(summary["hota_alpha"].iloc[0])) + + deta = sum(deta) / len(deta) + assa = sum(assa) / len(assa) + hota = sum(hota) / len(hota) + + assert deta == approx(TUD_golden_ans[dname]["deta"]) + assert assa == approx(TUD_golden_ans[dname]["assa"]) + assert hota == approx(TUD_golden_ans[dname]["hota"]) diff --git a/motmetrics/utils.py b/motmetrics/utils.py index 520573a..74ff37d 100644 --- a/motmetrics/utils.py +++ b/motmetrics/utils.py @@ -7,9 +7,7 @@ """Functions for populating event accumulators.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import numpy as np @@ -18,6 +16,147 @@ from motmetrics.preprocess import preprocessResult +def compute_global_aligment_score( + allframeids, fid_to_fgt, fid_to_fdt, num_gt_id, num_det_id, dist_func +): + """Taken from https://github.com/JonathonLuiten/TrackEval/blob/12c8791b303e0a0b50f753af204249e622d0281a/trackeval/metrics/hota.py""" + potential_matches_count = np.zeros((num_gt_id, num_det_id)) + gt_id_count = np.zeros((num_gt_id, 1)) + tracker_id_count = np.zeros((1, num_det_id)) + + for fid in allframeids: + oids = np.empty(0) + hids = np.empty(0) + if fid in fid_to_fgt: + fgt = fid_to_fgt[fid] + oids = fgt.index.get_level_values("Id") + if fid in fid_to_fdt: + fdt = fid_to_fdt[fid] + hids = fdt.index.get_level_values("Id") + if len(oids) > 0 and len(hids) > 0: + gt_ids = np.array(oids.values) - 1 + dt_ids = np.array(hids.values) - 1 + similarity = dist_func(fgt.values, fdt.values, return_dist=False) + + sim_iou_denom = ( + similarity.sum(0)[np.newaxis, :] + similarity.sum(1)[:, np.newaxis] - similarity + ) + sim_iou = np.zeros_like(similarity) + sim_iou_mask = sim_iou_denom > 0 + np.finfo("float").eps + sim_iou[sim_iou_mask] = similarity[sim_iou_mask] / sim_iou_denom[sim_iou_mask] + potential_matches_count[gt_ids[:, np.newaxis], dt_ids[np.newaxis, :]] += sim_iou + + # Calculate the total number of dets for each gt_id and tracker_id. + gt_id_count[gt_ids] += 1 + tracker_id_count[0, dt_ids] += 1 + global_alignment_score = potential_matches_count / ( + np.maximum(1, gt_id_count + tracker_id_count - potential_matches_count) + ) + return global_alignment_score + + +def compare_to_groundtruth_reweighting(gt, dt, dist="iou", distfields=None, distth=(0.5)): + """Compare groundtruth and detector results with global alignment score. + + This method assumes both results are given in terms of DataFrames with at least the following fields + - `FrameId` First level index used for matching ground-truth and test frames. + - `Id` Secondary level index marking available object / hypothesis ids + + Depending on the distance to be used relevant distfields need to be specified. + + Params + ------ + gt : pd.DataFrame + Dataframe for ground-truth + test : pd.DataFrame + Dataframe for detector results + + Kwargs + ------ + dist : str, optional + String identifying distance to be used. Defaults to intersection over union ('iou'). Euclidean + distance ('euclidean') and squared euclidean distance ('seuc') are also supported. + distfields: array, optional + Fields relevant for extracting distance information. Defaults to ['X', 'Y', 'Width', 'Height'] + distth: Union(float, array_like), optional + Maximum tolerable distance. Pairs exceeding this threshold are marked 'do-not-pair'. + If a list of thresholds is given, multiple accumulators are returned. + """ + # pylint: disable=too-many-locals + if distfields is None: + distfields = ["X", "Y", "Width", "Height"] + + def compute_iou(a, b, return_dist): + return iou_matrix(a, b, max_iou=distth, return_dist=return_dist) + + def compute_euc(a, b, *args, **kwargs): + return np.sqrt(norm2squared_matrix(a, b, max_d2=distth**2)) + + def compute_seuc(a, b, *args, **kwargs): + return norm2squared_matrix(a, b, max_d2=distth) + + if dist.upper() == "IOU": + compute_dist = compute_iou + elif dist.upper() == "EUC": + compute_dist = compute_euc + import warnings + + warnings.warn( + f"'euc' flag changed its behavior. The euclidean distance is now used instead of the squared euclidean distance. Make sure the used threshold (distth={distth}) is not squared. Use 'euclidean' flag to avoid this warning." + ) + elif dist.upper() == "EUCLIDEAN": + compute_dist = compute_euc + elif dist.upper() == "SEUC": + compute_dist = compute_seuc + else: + raise f'Unknown distance metric {dist}. Use "IOU", "EUCLIDEAN", or "SEUC"' + + return_single = False + if isinstance(distth, float): + distth = [distth] + return_single = True + + acc_list = [MOTAccumulator() for _ in range(len(distth))] + + num_gt_id = gt.index.get_level_values("Id").max() + num_det_id = dt.index.get_level_values("Id").max() + + # We need to account for all frames reported either by ground truth or + # detector. In case a frame is missing in GT this will lead to FPs, in + # case a frame is missing in detector results this will lead to FNs. + allframeids = gt.index.union(dt.index).levels[0] + + gt = gt[distfields] + dt = dt[distfields] + fid_to_fgt = dict(iter(gt.groupby("FrameId"))) + fid_to_fdt = dict(iter(dt.groupby("FrameId"))) + + global_alignment_score = compute_global_aligment_score( + allframeids, fid_to_fgt, fid_to_fdt, num_gt_id, num_det_id, compute_dist + ) + + for fid in allframeids: + oids = np.empty(0) + hids = np.empty(0) + weighted_dists = np.empty((0, 0)) + if fid in fid_to_fgt: + fgt = fid_to_fgt[fid] + oids = fgt.index.get_level_values("Id") + if fid in fid_to_fdt: + fdt = fid_to_fdt[fid] + hids = fdt.index.get_level_values("Id") + if len(oids) > 0 and len(hids) > 0: + gt_ids = np.array(oids.values) - 1 + dt_ids = np.array(hids.values) - 1 + dists = compute_dist(fgt.values, fdt.values, return_dist=False) + weighted_dists = ( + dists * global_alignment_score[gt_ids[:, np.newaxis], dt_ids[np.newaxis, :]] + ) + for acc, th in zip(acc_list, distth): + acc.update(oids, hids, 1 - weighted_dists, frameid=fid, similartiy_matrix=dists, th=th) + return acc_list[0] if return_single else acc_list + + def compare_to_groundtruth(gt, dt, dist='iou', distfields=None, distth=0.5): """Compare groundtruth and detector results.