add get_incorrect_models (#375)

* 1119

* 1120

* 1120.2

* model_path

* remove unnecessary files and pre-committed

* remove unnecessary files and pre-committed

* 1121 remove unnecessary files

* modify rev version

* modify rev version

* modify rev version

* accuracy issues targeted

* test script and modify feature

* return set[str]

* add logfile for test

Author: roll-away

.pre-commit-config.yaml

@@ -3,4 +3,4 @@ repos:
   rev: 23.1.0
   hooks:
   - id: black
-    language_version: python3
+    language_version: python3

graph_net/analysis_util.py

@@ -121,6 +121,7 @@ def parse_logs_to_data(log_file: str) -> list:
             current_run_key = processing_match.group(1).strip()
             # Initialize a nested dictionary structure for this new run
             all_runs_data[current_run_key] = {
+                "model_path": line.split()[-1],
                 "configuration": {},
                 "correctness": {},
                 "performance": {
@@ -617,3 +618,26 @@ def check_sample_correctness(sample: dict, t_key: int) -> tuple[bool, str]:
     )
 
     return is_correct, None if is_correct else "accuracy"
+
+
+def get_incorrect_models(tolerance, log_file_path) -> set[str]:
+    """
+    Filters and returns models with accuracy issues based on given tolerance threshold.
+
+    Parses model data from log file and checks each model's accuracy against the specified
+    tolerance threshold. Returns paths of all models that fail to meet the accuracy requirements.
+
+    Args:
+        tolerance (float): Accuracy tolerance threshold for model validation
+        log_file_path (str): Path to the log file containing model test results
+
+    Returns:
+        failed_models(str): names of models failing accuracy check, empty set if none found
+    """
+    failed_models = set()
+    datalist = parse_logs_to_data(log_file_path)
+    for i in datalist:
+        iscorrect, err = check_sample_correctness(i, tolerance)
+        if not iscorrect:
+            failed_models.add(i.get("model_path"))
+    return failed_models

graph_net/test/get_incorrect_models_test.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+
+SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+GRAPH_NET_DIR=$(dirname "$SCRIPT_DIR")
+PROJECT_ROOT=$(dirname "$GRAPH_NET_DIR")
+
+# 将项目根目录加入Python路径
+export PYTHONPATH="$PROJECT_ROOT:$PYTHONPATH"
+
+TOLERANCE_LIST=(-2 -1 0 1 2)
+LOG_FILE_PATH="log_file_for_test.txt"
+
+python3 - <<END
+from graph_net import analysis_util
+
+result = analysis_util.get_incorrect_models($TOLERANCE_LIST, '$LOG_FILE_PATH')
+
+for item in result:
+    print(item)
+END