Add mot metrics GitHub action (#114)

* Added basic motmetrics test * Update tests to new norfair api * Run our 4 tests in parallel in github actions * Remove empty test * Move metrics.txt file Co-authored-by: Leonardo Agis <[email protected]>
tryolabs · May 30, 2022 · bbc2752 · bbc2752
1 parent 58acb84
commit bbc2752
Show file tree

Hide file tree

Showing 7 changed files with 152 additions and 12 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -17,7 +17,7 @@ jobs:
   build-test:
     runs-on: ubuntu-latest
     strategy:
-      max-parallel: 3
+      max-parallel: 4
       matrix:
         python-version: [3.6, 3.7, 3.8, 3.9]
     steps:
@@ -36,8 +36,16 @@ jobs:
           python -m pip install --upgrade pip
           pip install tox tox-gh-actions
 
-      - name: Test with tox
-        run: tox
+      - name: Run tests
+        run: |
+          python3 -m venv ./venv
+          source venv/bin/activate
+          pip install --upgrade pip
+          python --version
+          poetry install -E metrics
+          curl -O https://motchallenge.net/data/MOT17Labels.zip
+          unzip MOT17Labels.zip
+          pytest -s tests/
 
   #
   # Build & upload release to PyPI.

diff --git a/demos/motmetrics4norfair/motmetrics4norfair.py b/demos/motmetrics4norfair/motmetrics4norfair.py
@@ -155,4 +155,4 @@ def keypoints_distance(detected_pose, tracked_pose):
 accumulator.print_metrics()
 
 if args.save_metrics:
-    accumulator.save_metrics(save_path=output_path)
+    accumulator.save_metrics(save_path=output_path)
diff --git a/norfair/video.py b/norfair/video.py
@@ -12,6 +12,8 @@
 from rich import print
 from rich.progress import BarColumn, Progress, ProgressColumn, TimeRemainingColumn
 
+from norfair import metrics
+
 from .utils import get_terminal_size
 
 

diff --git a/tests/metrics.txt b/tests/metrics.txt
@@ -0,0 +1,23 @@
+                IDF1   IDP   IDR  Rcll  Prcn   GT  MT  PT  ML    FP     FN  IDs    FM  MOTA  MOTP  IDt  IDa IDm
+MOT17-04-FRCNN 55.7% 73.9% 44.6% 56.3% 93.2%   83  18  43  22  1962  20778   90   104 52.0% 0.107    5   88   3
+MOT17-11-FRCNN 54.6% 68.6% 45.3% 61.5% 93.1%   75  19  33  23   431   3631   64    61 56.3% 0.101   15   52   5
+MOT17-05-SDP   58.5% 67.6% 51.5% 66.7% 87.6%  133  32  81  20   653   2301  134   140 55.4% 0.165   48   99  15
+MOT17-10-FRCNN 39.9% 44.7% 36.0% 61.0% 75.8%   57  16  35   6  2504   5013  319   313 39.0% 0.173  125  193  10
+MOT17-05-DPM   36.1% 57.1% 26.4% 38.0% 82.2%  133  10  58  65   570   4291   96    96 28.3% 0.242   33   72  12
+MOT17-11-DPM   38.8% 49.7% 31.8% 54.2% 84.5%   75  12  24  39   935   4321   88    64 43.4% 0.217   20   69   4
+MOT17-04-DPM   29.0% 43.1% 21.9% 42.5% 83.6%   83   7  44  32  3965  27336  401   420 33.3% 0.210   43  352   1
+MOT17-10-SDP   47.4% 51.8% 43.6% 74.2% 88.1%   57  30  24   3  1292   3316  308   289 61.7% 0.198  139  165   6
+MOT17-11-SDP   55.4% 61.0% 50.8% 75.8% 91.1%   75  34  30  11   697   2285  103   101 67.3% 0.140   43   69  13
+MOT17-13-SDP   54.4% 66.4% 46.1% 57.0% 82.1%  110  45  28  37  1444   5008  229   209 42.6% 0.201   73  166  18
+MOT17-09-DPM   33.4% 37.7% 30.0% 59.9% 75.4%   26   4  18   4  1042   2137  119   113 38.1% 0.262   41   81   3
+MOT17-13-DPM   21.3% 59.9% 12.9% 18.0% 83.5%  110   5  28  77   416   9543  120   125 13.4% 0.268   35   94   9
+MOT17-05-FRCNN 54.7% 72.0% 44.1% 54.7% 89.2%  133  24  68  41   457   3136   95    96 46.7% 0.181   44   68  17
+MOT17-02-FRCNN 33.9% 53.8% 24.7% 36.6% 79.7%   62   7  26  29  1736  11783  119   131 26.6% 0.134   31   95   8
+MOT17-09-SDP   53.2% 63.8% 45.6% 67.6% 94.6%   26  12  14   0   204   1726   52    55 62.8% 0.130   21   37   6
+MOT17-09-FRCNN 54.0% 72.4% 43.1% 58.6% 98.5%   26   7  17   2    49   2207   40    39 56.9% 0.095   14   30   4
+MOT17-02-SDP   34.4% 42.9% 28.7% 51.0% 76.1%   62  11  39  12  2979   9103  268   284 33.5% 0.182   82  190   8
+MOT17-02-DPM   16.2% 41.0% 10.1% 20.2% 81.6%   62   5  14  43   843  14834  111   112 15.0% 0.246   19   91   1
+MOT17-13-FRCNN 47.0% 52.9% 42.3% 58.8% 73.6%  110  29  57  24  2455   4802  371   366 34.5% 0.185  117  264  20
+MOT17-10-DPM   28.0% 45.8% 20.2% 37.3% 84.6%   57   6  20  31   871   8051  127   154 29.5% 0.248   15  118   6
+MOT17-04-SDP   66.2% 74.7% 59.5% 77.6% 97.4%   83  48  26   9  1001  10672  225   254 75.0% 0.132   89  136   5
+OVERALL        45.3% 59.5% 36.6% 53.6% 87.2% 1638 381 727 530 26506 156274 3479  3526 44.7% 0.163 1052 2529 174
diff --git a/tests/test_mot_metrics.py b/tests/test_mot_metrics.py
@@ -0,0 +1,110 @@
+import os.path
+
+import numpy as np
+import pandas as pd
+
+from norfair import Tracker, metrics
+
+DATASET_PATH = "train"
+MOTA_ERROR_THRESHOLD = 0.0
+
+FRAME_SKIP_PERIOD = 1
+DETECTION_THRESHOLD = 0.01
+DISTANCE_THRESHOLD = 0.9
+DIAGONAL_PROPORTION_THRESHOLD = 1 / 18
+POINTWISE_HIT_COUNTER_MAX = 3
+HIT_COUNTER_MAX = 2
+
+def keypoints_distance(detected_pose, tracked_pose):
+    norm_orders = [1, 2, np.inf]
+    distances = 0
+    diagonal = 0
+
+    hor_min_pt = min(detected_pose.points[:, 0])
+    hor_max_pt = max(detected_pose.points[:, 0])
+    ver_min_pt = min(detected_pose.points[:, 1])
+    ver_max_pt = max(detected_pose.points[:, 1])
+
+    # Set keypoint_dist_threshold based on object size, and calculate
+    # distance between detections and tracker estimations
+    for p in norm_orders:
+        distances += np.linalg.norm(
+            detected_pose.points - tracked_pose.estimate, ord=p, axis=1
+        )
+        diagonal += np.linalg.norm(
+            [hor_max_pt - hor_min_pt, ver_max_pt - ver_min_pt], ord=p
+        )
+
+    distances = distances / len(norm_orders)
+
+    keypoint_dist_threshold = diagonal * DIAGONAL_PROPORTION_THRESHOLD
+
+    match_num = np.count_nonzero(
+        (distances < keypoint_dist_threshold)
+        * (detected_pose.scores > DETECTION_THRESHOLD)
+        * (tracked_pose.last_detection.scores > DETECTION_THRESHOLD)
+    )
+    return 1 / (1 + match_num)
+
+def test_mot_metrics():
+    """Tests that Norfair's MOT metrics didn't get worse
+
+     Configurable so that it allows some margin on how much worse metrics could get before
+     the test fails. Margin configured through MOTA_ERROR_THRESHOLD.
+
+     Raises:
+         If the previous metrics file its not found.
+     """
+    # Load previous metrics
+    try:
+        previous_metrics = pd.read_fwf('tests/metrics.txt')
+        previous_metrics.columns = [column_name.lower() for column_name in previous_metrics.columns]
+        previous_metrics = previous_metrics.set_index(previous_metrics.columns[0])
+    except FileNotFoundError as e:
+        raise e
+
+    accumulator = metrics.Accumulators()
+    sequences_paths = [element.path for element in os.scandir(DATASET_PATH) if element.is_dir()]
+    for input_path in sequences_paths:
+        # Search vertical resolution in seqinfo.ini
+        seqinfo_path = os.path.join(input_path, "seqinfo.ini")
+        info_file = metrics.InformationFile(file_path=seqinfo_path)
+
+        all_detections = metrics.DetectionFileParser(
+            input_path=input_path, information_file=info_file
+        )
+
+        tracker = Tracker(
+            distance_function=keypoints_distance,
+            distance_threshold=DISTANCE_THRESHOLD,
+            detection_threshold=DETECTION_THRESHOLD,
+            pointwise_hit_counter_max=POINTWISE_HIT_COUNTER_MAX,
+            hit_counter_max=HIT_COUNTER_MAX,
+        )
+
+        # Initialize accumulator for this video
+        accumulator.create_accumulator(input_path=input_path, information_file=info_file)
+
+        for frame_number, detections in enumerate(all_detections):
+            if frame_number % FRAME_SKIP_PERIOD == 0:
+                tracked_objects = tracker.update(
+                    detections=detections, period=FRAME_SKIP_PERIOD
+                )
+            else:
+                detections = []
+                tracked_objects = tracker.update()
+
+            accumulator.update(predictions=tracked_objects)
+
+    accumulator.compute_metrics()
+    new_metrics = accumulator.summary_dataframe
+    new_metrics.columns = [column_name.lower() for column_name in new_metrics.columns]
+
+    # Unify the scores to be able to compare them. new metrics is the percentage
+    # expressed between 0 and 1, the previous metrics have the percentage as a string
+    # with the % character at the end
+    new_overall_mota = new_metrics.loc["OVERALL", "mota"] * 100
+    previous_overall_mota = float(previous_metrics.loc["OVERALL", "mota"][:-1])
+
+    accumulator.print_metrics()
+    assert new_overall_mota >= previous_overall_mota * (1 - MOTA_ERROR_THRESHOLD), f"New overall MOTA score: {new_overall_mota} is too low, previous overall MOTA score: {previous_overall_mota}"
diff --git a/tests/test_norfair.py b/tests/test_norfair.py
diff --git a/tox.ini b/tox.ini
@@ -13,5 +13,8 @@ python =
 allowlist_externals = poetry
 commands =
     ; Need this so dev dependencies (like pytest) are installed
-    poetry install --no-root -v
-    poetry run pytest tests/
+    poetry install --no-root -v -E metrics
+    ; Download the needed files to perform the MOT metrics test
+    curl -O https://motchallenge.net/data/MOT17Labels.zip
+    unzip MOT17Labels.zip
+    poetry run pytest -s tests/