statisticize in tensor symbolic shapes (#404)

* support checking model redundancy

* revert change of vision_model_test

* reformat python code.

* reformat bert_model_test.py and utils.py

* minor fix

* fix failed check by comparing directories after os.path.realpath()

* fix bugs in check_validate.sh

* set dynamic=False in single_device_runner.py

* reset graph hash

* add robustness code for generating input tensor constraints

* Introduce input_tensor_constraints.py using shape propagation logic.

* support dimension generalization for torch.Tensor.view and torch.Tensor.reshape

* 1) support dimension generalization for torch.Tensor.expand(); 2) fix bugs in generalization for torch.Tensor.view and torch.Tensor.reshape

* dimension_generalization_passes

* Refactored DimensionGeneralizationPass.__init__ to accept argument dim_axes_pairs, enabling targeted configuration for specific use cases

* save dimension generalization pass names into graph_net.json

* Generalize sequence dimension

* more dimension generalization passes for token dimension

* refactor parse_sole_graph_module

* Enhance the performance of the input_tensor_constraints.py file generation process.

* minor fix

* more dimension generalization pass

* fix some hint bugs

* generate input_tensor_constraints.py

* statisticize in tensor symbolic shapes

Author: lixinqi

graph_net/tools/_get_in_tensor_symbolic_shapes.py

@@ -0,0 +1,34 @@
+from pathlib import Path
+from graph_net.dynamic_dim_constraints import DynamicDimConstraints
+import sympy
+
+
+class GetInTensorSymbolicShapes:
+    def __init__(self, config):
+        self.config = self.make_config(**config)
+
+    def make_config(self, model_path_prefix):
+        return {
+            "model_path_prefix": model_path_prefix,
+        }
+
+    def __call__(self, model_path):
+        original_model_path = Path(self.config["model_path_prefix"]) / model_path
+        input_tensor_cstr_filepath = original_model_path / "input_tensor_constraints.py"
+        if not input_tensor_cstr_filepath.exists():
+            print(f"get-in-tensor-symbolic-shapes None {model_path}")
+            return
+        dyn_dim_cstrs = DynamicDimConstraints.unserialize_from_py_file(
+            str(input_tensor_cstr_filepath)
+        )
+        dyn_dim_cstrs.symbol2example_value = {}
+        dyn_dim_cstrs.input_shapes = sorted(
+            [
+                tuple(shape)
+                for shape, name in dyn_dim_cstrs.input_shapes
+                if any(isinstance(dim, sympy.Expr) for dim in shape)
+            ],
+            key=str,
+        )
+        input_shapes_str = str(dyn_dim_cstrs.input_shapes).replace(" ", "")
+        print(f"get-in-tensor-symbolic-shapes {input_shapes_str} {model_path}")

graph_net/tools/get_in_tensor_symbolic_shapes.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+GRAPH_NET_ROOT=$(python3 -c "import graph_net; import os; print(
+os.path.dirname(graph_net.__file__))")
+
+# input model path
+# model_runnable_predicator=ShapePropagatablePredicator
+model_runnable_predicator=ModelRunnablePredicator
+config_json_str=$(cat <<EOF
+{
+    "handler_path": "$GRAPH_NET_ROOT/tools/_get_in_tensor_symbolic_shapes.py",
+    "handler_class_name": "GetInTensorSymbolicShapes",
+    "handler_config": {
+        "model_path_prefix": "$GRAPH_NET_ROOT/../"
+    }
+}
+EOF
+)
+CONFIG=$(echo $config_json_str | base64 -w 0)
+
+python3 -m graph_net.model_path_handler --model-path-list $GRAPH_NET_ROOT/config/torch_samples_list.txt --handler-config=$CONFIG

graph_net/tools/statisticize_in_tensor_symbolic_shapes.sh

@@ -0,0 +1,2 @@
+#/bin/bash
+bash get_in_tensor_symbolic_shapes.sh | grep get-in-tensor-symbolic-shapes | awk '{print $2}' | sort | uniq -c | sort -nk1