Merge branch 'develop' of https://github.com/PaddlePaddle/GraphNet into subgraph_dataset

Author: JewelRoam

graph_net/config/decomposition_error_tmp_torch_samples_list.txt

@@ -0,0 +1 @@
+samples/transformers-auto-model/dbmdz_electra-large-discriminator-finetuned-conll03-english

graph_net/config/empty_cstr_torch_samples_list.txt

@@ -0,0 +1 @@
+samples/transformers-auto-model/microsoft_xclip-base-patch32-16-frames

graph_net/config/tmp_torch_samples_list.txt

@@ -11,6 +11,9 @@
 from pathlib import Path
 from dataclasses import asdict
 import graph_net.graph_net_json_file_util as gn_json
+from collections import OrderedDict
+import copy
+from graph_net.hash_util import get_sha256_hash
 
 
 class ApplyDimGenPasses:
@@ -49,7 +52,12 @@ def __call__(self, rel_model_path):
         output_dir = Path(self.config["output_dir"])
         output_dir.mkdir(parents=True, exist_ok=True)
         generalized_model_path = output_dir / rel_model_path
-        if self.config["resume"] and (generalized_model_path / "model.py").exists():
+        if (
+            self.config["resume"]
+            and generalized_model_path.exists()
+            and generalized_model_path.is_dir()
+            and len(list(generalized_model_path.iterdir())) > 0
+        ):
             return
         tensor_metas = self._get_tensor_metas(model_path)
         tensor_meta_attrs_list = [asdict(tensor_meta) for tensor_meta in tensor_metas]
@@ -64,6 +72,7 @@ def __call__(self, rel_model_path):
         )
         dim_axes_pairs = self._get_dim_axes_pairs(dyn_dim_cstrs)
         if len(dim_axes_pairs) == 0:
+            print("No symbolic dims found. {model_path=}")
             return
 
         def get_generalized():
@@ -74,19 +83,80 @@ def get_generalized():
                 inputs=inputs,
             )
 
-        with get_generalized() as generalized_model_py_path:
-            self._save_generalized_model_path(rel_model_path, generalized_model_py_path)
+        with get_generalized() as tmp_model_py_path:
+            from_model_path = Path(self.config["model_path_prefix"]) / rel_model_path
+            triples = self._get_reified_tensor_metas(from_model_path, dyn_dim_cstrs)
+            for symbol2example_value, cur_tensor_metas, cur_dyn_dim_cstrs in triples:
+                to_model_path = self._get_to_model_path(
+                    rel_model_path, symbol2example_value
+                )
+                print(f"{str(to_model_path)=}")
+                self._copy_sample_model_path(from_model_path, to_model_path)
+                self._save_generalized_model_path(to_model_path, tmp_model_py_path)
+                self._save_tensor_metas_as_weight_meta(to_model_path, cur_tensor_metas)
+                self._save_dyn_dim_cstrs(to_model_path, cur_dyn_dim_cstrs)
 
         self._check_num_handled_models()
 
-    def _save_generalized_model_path(self, rel_model_path, generalized_model_py_path):
-        from_model_path = Path(self.config["model_path_prefix"]) / rel_model_path
-        to_model_path = Path(self.config["output_dir"]) / rel_model_path
-        print(f"{str(to_model_path)=}")
+    def _get_reified_tensor_metas(self, from_model_path, dyn_dim_cstrs):
+        tensor_metas = self._get_tensor_metas(str(from_model_path))
+        symbols, reified_dims = self._get_symbols_and_reified_dims(
+            from_model_path, dyn_dim_cstrs
+        )
+        for dims in reified_dims:
+            symbol2example_value = OrderedDict(list(zip(symbols, dims)))
+            cur_dyn_dim_cstrs = copy.deepcopy(dyn_dim_cstrs)
+            cur_tensor_metas = copy.deepcopy(tensor_metas)
+            cur_dyn_dim_cstrs.update_symbol2example_value(symbol2example_value)
+            update_tensor_metas_by_dyn_dim_cstr(cur_tensor_metas, cur_dyn_dim_cstrs)
+            yield symbol2example_value, cur_tensor_metas, cur_dyn_dim_cstrs
+
+    def _get_symbols_and_reified_dims(self, from_model_path, dyn_dim_cstrs):
+        json_value = gn_json.read_json(str(from_model_path))
+        reifier_name = json_value[gn_json.kSymbolicDimensionReifier]
+        from graph_net.torch.sym_dim_reifiers.reifier_mgr import get_reifier
+
+        reifier_class = get_reifier(reifier_name)
+        reifier_instance = reifier_class(str(from_model_path))
+        assert reifier_instance.match
+        symbols2reified_dims = reifier_instance.reify()
+        assert len(symbols2reified_dims) == 1
+        symbols, reified_dims = next(iter(symbols2reified_dims.items()))
+        assert tuple(symbols) == tuple(dyn_dim_cstrs.symbols)
+        assert all(len(symbols) == len(dims) for dims in reified_dims)
+        return symbols, reified_dims
+
+    def _save_dyn_dim_cstrs(self, to_model_path, dyn_dim_cstrs):
+        cstr_code = dyn_dim_cstrs.serialize_to_py_str()
+        (to_model_path / "input_tensor_constraints.py").write_text(cstr_code)
+
+    def _save_tensor_metas_as_weight_meta(self, to_model_path, tensor_metas):
+        weight_meta_code = "\n".join(
+            tensor_meta.serialize_to_py_str() for tensor_meta in tensor_metas
+        )
+        (to_model_path / "weight_meta.py").write_text(weight_meta_code)
+
+    def _get_to_model_path(self, rel_model_path, symbol2example_value):
+        sym_dim_str = "_".join(
+            f"{sym_name}_{dim}"
+            for symbol, dim in symbol2example_value.items()
+            for sym_name in [symbol.name]
+        )
+        sub_module_name = f"{os.path.basename(rel_model_path)}__{sym_dim_str}"
+        to_model_path = (
+            Path(self.config["output_dir"]) / rel_model_path / sub_module_name
+        )
+        return to_model_path
+
+    def _copy_sample_model_path(self, from_model_path, to_model_path):
         to_model_path.mkdir(parents=True, exist_ok=True)
         shutil.copytree(Path(from_model_path), Path(to_model_path), dirs_exist_ok=True)
-        generalized_model_py_code = Path(generalized_model_py_path).read_text()
+
+    def _save_generalized_model_path(self, to_model_path, tmp_model_py_path):
+        generalized_model_py_code = Path(tmp_model_py_path).read_text()
         (to_model_path / "model.py").write_text(generalized_model_py_code)
+        file_hash = get_sha256_hash(generalized_model_py_code)
+        (to_model_path / "graph_hash.txt").write_text(file_hash)
 
     def _get_dim_axes_pairs(self, dyn_dim_cstrs):
         sym_input_shapes = dyn_dim_cstrs.get_sorted_symbolic_input_shapes()

graph_net/dimension_generalizer.py

@@ -0,0 +1,7 @@
+import hashlib
+
+
+def get_sha256_hash(content):
+    m = hashlib.sha256()
+    m.update(content.encode())
+    return m.hexdigest()

graph_net/hash_util.py

@@ -1,10 +1,10 @@
-import traceback
 import argparse
 from graph_net.imp_util import load_module
 import logging
 import sys
 import json
 import base64
+import subprocess
 
 logging.basicConfig(
     level=logging.WARNING, format="%(asctime)s [%(levelname)s] %(message)s"
@@ -37,33 +37,49 @@ def _get_handler(args):
 
 def main(args):
     handler = _get_handler(args)
-    for model_path in _get_model_paths(args):
-        print(f"{model_path=}")
+    if args.model_path is not None:
+        handle_model_path(handler, args.model_path)
+    elif args.use_subprocess:
+        handle_model_path_list_in_subprocess(args)
+    else:
+        handle_model_path_list_in_current_process(handler, args)
+
+
+def handle_model_path_list_in_current_process(handler, args):
+    for model_path in _get_model_path_list(args):
         try:
-            handler(model_path)
+            handle_model_path(handler, model_path)
         except KeyboardInterrupt:
-            sys.exit(-1)
-        except Exception as e:
-            print("--- Concise Error Message ---")
-            print(e)
+            print("KeyboardInterrupt")
+            return
 
-            print("\n--- Full Traceback ---")
-            traceback.print_exc()
 
+def handle_model_path_list_in_subprocess(args):
+    for model_path in _get_model_path_list(args):
+        cmd = f"{sys.executable} -m graph_net.model_path_handler --model-path {model_path} --handler-config {args.handler_config}"
+        try:
+            subprocess.Popen(cmd, shell=True).wait()
+        except KeyboardInterrupt:
+            print("KeyboardInterrupt")
+            return
 
-def _get_model_paths(args):
-    assert args.model_path is not None or args.model_path_list is not None
-    if args.model_path is not None:
-        yield args.model_path
-    if args.model_path_list is not None:
-        with open(args.model_path_list) as f:
-            yield from (
-                clean_line
-                for line in f
-                for clean_line in [line.strip()]
-                if len(clean_line) > 0
-                if not clean_line.startswith("#")
-            )
+
+def handle_model_path(handler, model_path):
+    print(f"{model_path=}", flush=True)
+    handler(model_path)
+
+
+def _get_model_path_list(args):
+    assert args.model_path is None
+    assert args.model_path_list is not None
+    with open(args.model_path_list) as f:
+        yield from (
+            clean_line
+            for line in f
+            for clean_line in [line.strip()]
+            if len(clean_line) > 0
+            if not clean_line.startswith("#")
+        )
 
 
 if __name__ == "__main__":
@@ -89,5 +105,11 @@ def _get_model_paths(args):
         default=None,
         help="handler configuration string",
     )
+    parser.add_argument(
+        "--use-subprocess",
+        action="store_true",
+        default=False,
+        help="use subprocess",
+    )
     args = parser.parse_args()
     main(args=args)

graph_net/model_path_handler.py

@@ -48,4 +48,4 @@ echo "Results saved in: $FILE_PATH/ES_result.png"
 echo ""
 echo "IMPORTANT: Please verify if the curve in ES_result.png is a straight line"
 echo "If the curve is NOT a straight line, please check the log file: $FILE_PATH/log.log"
-echo "=================================================="
+echo "=================================================="

graph_net/test/decomposer_validator_test.sh

@@ -11,15 +11,16 @@ config_json_str=$(cat <<EOF
     "handler_path": "$GRAPH_NET_ROOT/torch/naive_graph_decomposer.py",
     "handler_class_name": "NaiveDecomposerExtractor",
     "handler_config": {
+        "model_path_prefix": "$GRAPH_NET_ROOT/../",
         "output_dir": "/tmp/naive_decompose_workspace",
         "split_positions": [8, 16, 32],
-        "group_head_and_tail": true,
-        "filter_path":"$GRAPH_NET_ROOT/torch/naive_subgraph_filter.py",
-        "filter_config": {}
+        "chain_style": true,
+        "group_head_and_tail": true
     }
 }
 EOF
 )
 CONFIG=$(echo $config_json_str | base64 -w 0)
 
-python3 -m graph_net.model_path_handler --model-path $GRAPH_NET_ROOT/../samples/$MODEL_PATH_IN_SAMPLES --handler-config=$CONFIG
+# python3 -m graph_net.model_path_handler --model-path $GRAPH_NET_ROOT/../samples/$MODEL_PATH_IN_SAMPLES --handler-config=$CONFIG
+python3 -m graph_net.model_path_handler --model-path-list $GRAPH_NET_ROOT/config/decomposition_error_tmp_torch_samples_list.txt --handler-config=$CONFIG

graph_net/test/naive_graph_decomposer_test.sh

@@ -1,3 +1,4 @@
+import sympy
 from pathlib import Path
 from graph_net.dynamic_dim_constraints import DynamicDimConstraints
 import graph_net.graph_net_json_file_util as gn_json
@@ -27,6 +28,10 @@ def __call__(self, model_path):
         dyn_dim_cstrs = DynamicDimConstraints.unserialize_from_py_file(
             str(input_tensor_cstr_filepath)
         )
+        for shape, name in dyn_dim_cstrs.input_shapes:
+            if not any(isinstance(dim, sympy.Expr) for dim in shape):
+                continue
+            print(f"{shape=} {name=}")
         input_shapes_str = str(dyn_dim_cstrs.serialize_symbolic_input_shapes_to_str())
         print(f"get-in-tensor-symbolic-shapes {input_shapes_str} {model_path}")
 

graph_net/tools/_get_in_tensor_symbolic_shapes.py

@@ -11,12 +11,12 @@ config_json_str=$(cat <<EOF
     "handler_path": "$GRAPH_NET_ROOT/dimension_generalizer.py",
     "handler_class_name": "ApplyDimGenPasses",
     "handler_config": {
-        "resume": true,
+        "resume": false,
         "output_dir": "/tmp/dimension_generalized_samples",
         "model_path_prefix": "$GRAPH_NET_ROOT/../",
         "dimension_generalizer_filepath": "$GRAPH_NET_ROOT/torch/static_to_dynamic.py",
         "dimension_generalizer_class_name": "StaticToDynamic",
-        "limits_handled_models": 9999999,
+        "limits_handled_models": 10,
         "last_model_log_file": "/tmp/a.py"
     }
 }