Merge branch 'develop' of https://github.com/PaddlePaddle/GraphNet into develop

Author: JewelRoam

.pre-commit-config.yaml

@@ -3,4 +3,25 @@ repos:
   rev: 23.1.0
   hooks:
   - id: black
-    language_version: python3
+    language_version: python3
+
+- repo: https://github.com/astral-sh/ruff-pre-commit
+  rev: v0.14.4
+  hooks:
+    - id: ruff-check
+      args: [--fix, --exit-non-zero-on-fix, --no-cache]
+
+- repo: https://github.com/PFCCLab/typos-pre-commit-mirror.git
+  rev: v1.39.2
+  hooks:
+    - id: typos
+      args: [--force-exclude]
+
+- repo: https://github.com/Lucas-C/pre-commit-hooks.git
+  rev: v1.5.1
+  hooks:
+    - id: remove-crlf
+    - id: remove-tabs
+      name: Tabs remver (Python)
+      files: (.*\.(py|bzl)|BUILD|.*\.BUILD|WORKSPACE)$
+      args: [--whitespaces-count, '4']

graph_net/__init__.py

@@ -1,21 +1,21 @@
-__all__ = ["torch", "paddle"]
-
-from importlib import import_module
-from typing import TYPE_CHECKING, Any, List
-
-
-def __getattr__(name: str) -> Any:
-    if name in __all__:
-        module = import_module(f"{__name__}.{name}")
-        globals()[name] = module
-        return module
-    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
-
-
-def __dir__() -> List[str]:
-    return sorted(list(globals().keys()) + __all__)
-
-
-if TYPE_CHECKING:
-    from . import torch as torch  # type: ignore
-    from . import paddle as paddle  # type: ignore
+__all__ = ["torch", "paddle"]
+
+from importlib import import_module
+from typing import TYPE_CHECKING, Any, List
+
+
+def __getattr__(name: str) -> Any:
+    if name in __all__:
+        module = import_module(f"{__name__}.{name}")
+        globals()[name] = module
+        return module
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+
+
+def __dir__() -> List[str]:
+    return sorted(list(globals().keys()) + __all__)
+
+
+if TYPE_CHECKING:
+    from . import torch as torch  # type: ignore
+    from . import paddle as paddle  # type: ignore

graph_net/analysis_util.py

@@ -1,6 +1,6 @@
 import os
 import re
-import numpy as np
+import sys
 from scipy.stats import gmean
 from graph_net.config.datatype_tolerance_config import get_precision
 
@@ -213,7 +213,7 @@ def scan_all_folders(benchmark_path: str) -> dict:
         print(f"Detected log file: '{benchmark_path}'")
         samples = parse_logs_to_data(benchmark_path)
         if not samples:
-            print(f"  - No valid data found in log file.")
+            print("  - No valid data found in log file.")
             return {}
 
         folder_name = (

graph_net/constraint_util.py

@@ -90,12 +90,12 @@ def __call__(self, model_path):
 
         tensor_metas = self._get_tensor_metas(model_path)
         tensor_meta_attrs_list = [asdict(tensor_meta) for tensor_meta in tensor_metas]
-        logging.warning(f"before create_inputs_by_metas")
+        logging.warning("before create_inputs_by_metas")
         inputs = self.get_dimension_generalizer().create_inputs_by_metas(
             module=self.get_model(model_path),
             tensor_meta_attrs_list=tensor_meta_attrs_list,
         )
-        logging.warning(f"after create_inputs_by_metas")
+        logging.warning("after create_inputs_by_metas")
         dyn_dim_cstr = make_dyn_dim_cstr_from_tensor_metas(tensor_metas)
 
         def data_input_predicator(input_var_name):
@@ -157,23 +157,23 @@ def get_model(self, model_path):
 
     @contextmanager
     def _try_dimension_generalization(self, dim_axes_pairs, model_path, inputs):
-        logging.warning(f"enter _try_dimension_generalization")
+        logging.warning("enter _try_dimension_generalization")
         if self.config["dimension_generalizer_filepath"] is None:
             yield model_path, ()
             return
         model = self.get_model(model_path)
         dim_generalizer = self.get_dimension_generalizer()
         dim_gen_pass = dim_generalizer(model, dim_axes_pairs)
-        logging.warning(f"before need_rewrite")
+        logging.warning("before need_rewrite")
         need_rewrite = dim_gen_pass.need_rewrite(inputs)
-        logging.warning(f"after need_rewrite")
+        logging.warning("after need_rewrite")
         if not need_rewrite:
             yield model_path, ()
             return
 
-        logging.warning(f"before rewrite")
+        logging.warning("before rewrite")
         graph_module = dim_gen_pass.rewrite(inputs)
-        logging.warning(f"after rewrite")
+        logging.warning("after rewrite")
         with tempfile.TemporaryDirectory() as tmp_dir:
             shutil.copytree(Path(model_path), Path(tmp_dir), dirs_exist_ok=True)
             dim_gen_pass.save_graph_module(graph_module, tmp_dir)
@@ -293,20 +293,20 @@ def symbolize_data_input_dims(
     Returns new DynamicDimConstraints if success.
     Returns None if no symbolicable dim .
     """
-    unqiue_dims = []
+    unique_dims = []
     dim2axes = {}
 
     def dumpy_filter_fn(input_name, input_idx, axis, dim):
         if is_data_input(input_name):
             print("data_input", input_name, input_idx, axis, dim)
-            if dim not in unqiue_dims:
-                unqiue_dims.append(dim)
+            if dim not in unique_dims:
+                unique_dims.append(dim)
                 dim2axes[dim] = []
             dim2axes[dim].append(axis)
         # No symbolization by returning False
         return False
 
-    # Collect input dimensions into `unqiue_dims`
+    # Collect input dimensions into `unique_dims`
     assert dyn_dim_cstr.symbolize(dumpy_filter_fn) is None
     total_dim_gen_pass_names = ()
 
@@ -323,7 +323,7 @@ def append_dim_gen_pass_names(dim_gen_pass_names):
             ]
         )
 
-    for i, picked_dim in enumerate(unqiue_dims):
+    for i, picked_dim in enumerate(unique_dims):
         logging.warning(f"{i=} {picked_dim=}")
         cur_dyn_dim_cstr = copy.deepcopy(dyn_dim_cstr)
 
@@ -341,20 +341,20 @@ def filter_fn(input_name, input_idx, axis, dim):
         if not cur_dyn_dim_cstr.check_delta_symbol2example_value(sym2example_value):
             continue
         dim_axes_pairs = tuple(
-            (dim, axes) for dim in unqiue_dims[: i + 1] for axes in [dim2axes[dim]]
+            (dim, axes) for dim in unique_dims[: i + 1] for axes in [dim2axes[dim]]
         )
         ctx_mgr = dyn_dim_cstr_feasibility_ctx_mgr
-        logging.warning(f"before dyn_dim_cstr_feasibility_ctx_mgr")
+        logging.warning("before dyn_dim_cstr_feasibility_ctx_mgr")
         with ctx_mgr(dim_axes_pairs) as dyn_dim_cstr_feasibility:
-            logging.warning(f"enter dyn_dim_cstr_feasibility_ctx_mgr")
+            logging.warning("enter dyn_dim_cstr_feasibility_ctx_mgr")
             tmp_dyn_dim_cstr = copy.deepcopy(cur_dyn_dim_cstr)
             tmp_dyn_dim_cstr.update_symbol2example_value(sym2example_value)
-            logging.warning(f"before dyn_dim_cstr_feasibility")
+            logging.warning("before dyn_dim_cstr_feasibility")
             is_dyn_dim_cstr_feasible = dyn_dim_cstr_feasibility(tmp_dyn_dim_cstr)
-            logging.warning(f"after dyn_dim_cstr_feasibility")
+            logging.warning("after dyn_dim_cstr_feasibility")
             if not is_dyn_dim_cstr_feasible:
                 continue
             dyn_dim_cstr = cur_dyn_dim_cstr
             append_dim_gen_pass_names(dyn_dim_cstr_feasibility.dim_gen_pass_names)
-            logging.warning(f"leave dyn_dim_cstr_feasibility_ctx_mgr")
+            logging.warning("leave dyn_dim_cstr_feasibility_ctx_mgr")
     return dyn_dim_cstr, total_dim_gen_pass_names

graph_net/dynamic_dim_constraints.py

@@ -1,8 +1,6 @@
-import sys
 import sympy
 import importlib.util as imp
 from dataclasses import dataclass
-import copy
 from typing import Callable
 from collections import namedtuple
 
@@ -22,7 +20,7 @@ class DynamicDimConstraints:
     kRelations = "dynamic_dim_constraint_relations"
 
     # len(input_shapes) equals number of Model.forward arguments
-    input_shapes: list[(tuple[sympy.Expr | int], "var-name")]
+    input_shapes: list[(tuple[sympy.Expr | int], str)]
     kInputShapes = "dynamic_dim_constraint_input_shapes"
 
     @classmethod
@@ -44,7 +42,6 @@ def symbolize(
         ]
         Returns created symbol.
         """
-        import logging
 
         InputDim = namedtuple("InputDim", ["input_idx", "axis", "dim"])
         input_dims = [
@@ -157,7 +154,7 @@ def get_module_attr(cls, module, attr, default):
         return getattr(module, attr) if hasattr(module, attr) else default
 
     @classmethod
-    def load_module(cls, path, name="unamed"):
+    def load_module(cls, path, name="unnamed"):
         spec = imp.spec_from_file_location(name, path)
         module = imp.module_from_spec(spec)
         spec.loader.exec_module(module)

graph_net/imp_util.py

@@ -1,7 +1,7 @@
 import importlib.util as imp
 
 
-def load_module(path, name="unamed"):
+def load_module(path, name="unnamed"):
     spec = imp.spec_from_file_location(name, path)
     module = imp.module_from_spec(spec)
     module.__file__ = path

graph_net/model_path_handler.py

@@ -1,16 +1,10 @@
 import traceback
 import argparse
-import importlib.util
 from graph_net.imp_util import load_module
-import inspect
 import logging
-from pathlib import Path
-from typing import Type, Any
 import sys
 import json
 import base64
-from contextlib import contextmanager
-import logging
 
 logging.basicConfig(
     level=logging.WARNING, format="%(asctime)s [%(levelname)s] %(message)s"

graph_net/plot_ESt.py

@@ -88,7 +88,7 @@ def print_verification_result(
         )
     else:
         print(
-            f"t={tolerance:3d}: MISMATCH - Microscopic: {microscopic_es:.6f}, Aggregated: {aggregated_es:.6f}, Diff: {diff:.2e} ({relative_diff*100:.4f}%)"
+            f"t={tolerance:3d}: MISMATCH - Microscopic: {microscopic_es:.6f}, Aggregated: {aggregated_es:.6f}, Diff: {diff:.2e} ({relative_diff * 100:.4f}%)"
         )
 
 
@@ -110,9 +110,9 @@ def get_verified_aggregated_es_values(es_scores: dict, folder_name: str) -> dict
     verified_es_values = {}
     mismatches = []
 
-    print(f"\n{'='*80}")
+    print(f"\n{'=' * 80}")
     print(f"Verifying Aggregated/Microscopic Consistency for '{folder_name}'")
-    print(f"{'='*80}")
+    print(f"{'=' * 80}")
 
     for tolerance, microscopic_es in es_scores.items():
         aggregated_es = aggregated_results.get(tolerance)
@@ -136,29 +136,29 @@ def get_verified_aggregated_es_values(es_scores: dict, folder_name: str) -> dict
         elif not is_matched:
             mismatches.append(
                 f"t={tolerance}: Mismatch - Microscopic={microscopic_es:.6f}, "
-                f"Aggregated={aggregated_es:.6f}, Diff={diff:.2e} ({relative_diff*100:.4f}%)"
+                f"Aggregated={aggregated_es:.6f}, Diff={diff:.2e} ({relative_diff * 100:.4f}%)"
             )
         else:
             verified_es_values[tolerance] = microscopic_es
 
     if mismatches:
         error_msg = (
-            f"\n{'='*80}\n"
+            f"\n{'=' * 80}\n"
             f"ERROR: Aggregated and microscopic results do not match for '{folder_name}'!\n"
-            f"{'='*80}\n"
+            f"{'=' * 80}\n"
             f"Mismatches:\n"
             + "\n".join(f"  - {mismatch}" for mismatch in mismatches)
             + f"\n\nCalculation validation failed. Please verify the calculation logic "
             f"using verify_aggregated_params.py\n"
-            f"{'='*80}\n"
+            f"{'=' * 80}\n"
         )
         print(error_msg)
         raise AssertionError(error_msg)
 
     print(
         f"\nSUCCESS: All aggregated and microscopic results match for '{folder_name}'."
     )
-    print(f"{'='*80}\n")
+    print(f"{'=' * 80}\n")
     return verified_es_values
 
 

graph_net/status.py

@@ -28,7 +28,7 @@ def main():
         prog="python -m graph_net.status",
         description="List contents of the $GRAPH_NET_EXTRACT_WORKSPACE directory (like ls)",
     )
-    args = parser.parse_args()
+    args = parser.parse_args()  # noqa: F841
 
     ws = os.environ.get("GRAPH_NET_EXTRACT_WORKSPACE")
     if not ws:

graph_net/tensor_meta.py

@@ -48,7 +48,7 @@ def _convert_cls_to_attrs(cls, tensor_meta_cls):
         return attrs
 
     @classmethod
-    def _get_classes(cls, file_path, name="unamed"):
+    def _get_classes(cls, file_path, name="unnamed"):
         spec = imp.spec_from_file_location("unnamed", file_path)
         unnamed = imp.module_from_spec(spec)
         spec.loader.exec_module(unnamed)