check_graph_module_parsable

Author: lixinqi

graph_net/test/dev_model_list/graph_module_parse_error_torch_sample_list.txt

@@ -0,0 +1,30 @@
+samples/timm/aimv2_huge_patch14_224.apple_pt
+samples/timm/vit_so400m_patch14_siglip_gap_224.pali2_3b_pt
+samples/timm/eca_resnet33ts.ra2_in1k
+samples/timm/poolformer_s12.sail_in1k
+samples/timm/eva_giant_patch14_224.clip_ft_in1k
+samples/timm/eva02_base_patch14_224.mim_in22k
+samples/timm/eva02_small_patch14_224.mim_in22k
+samples/timm/poolformer_m36.sail_in1k
+samples/timm/poolformerv2_m36.sail_in1k
+samples/timm/resnest50d_1s4x24d.in1k
+samples/timm/aimv2_1b_patch14_224.apple_pt
+samples/timm/poolformer_m48.sail_in1k
+samples/timm/poolformer_s36.sail_in1k
+samples/timm/vit_base_patch16_siglip_gap_224.v2_webli
+samples/timm/vit_giant_patch16_gap_224.in22k_ijepa
+samples/timm/ecaresnet101d.miil_in1k
+samples/timm/vit_huge_patch14_gap_224.in1k_ijepa
+samples/timm/aimv2_3b_patch14_224.apple_pt
+samples/timm/skresnet18.ra_in1k
+samples/timm/ecaresnetlight.miil_in1k
+samples/timm/eca_resnext26ts.ch_in1k
+samples/timm/skresnext50_32x4d.ra_in1k
+samples/timm/poolformerv2_m48.sail_in1k
+samples/timm/poolformer_s24.sail_in1k
+samples/timm/ecaresnet26t.ra2_in1k
+samples/timm/eva02_large_patch14_224.mim_in22k
+samples/timm/ecaresnet50d.miil_in1k
+samples/timm/poolformerv2_s36.sail_in1k
+samples/timm/aimv2_large_patch14_224.apple_pt
+samples/timm/skresnet34.ra_in1k

graph_net/tools/check_graph_module_parsable.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+GRAPH_NET_ROOT=$(python3 -c "import graph_net; import os; print(
+os.path.dirname(graph_net.__file__))")
+
+# input model path
+# model_runnable_predicator=ShapePropagatablePredicator
+model_runnable_predicator=ModelRunnablePredicator
+config_json_str=$(cat <<EOF
+{
+    "handler_path": "$GRAPH_NET_ROOT/torch/check_graph_module_parsable.py",
+    "handler_class_name": "CheckGraphModuleParsable",
+    "handler_config": {
+        "model_path_prefix": "$GRAPH_NET_ROOT/../",
+        "resume": true,
+        "limits_handled_models": 999999,
+        "output_dir": "/tmp/check_graph_module_parsable"
+    }
+}
+EOF
+)
+CONFIG=$(echo $config_json_str | base64 -w 0)
+
+python3 -m graph_net.model_path_handler --model-path-list $GRAPH_NET_ROOT/test/dev_model_list/graph_module_parse_error_torch_sample_list.txt --handler-config=$CONFIG --use-subprocess

graph_net/torch/check_graph_module_parsable.py

@@ -0,0 +1,51 @@
+from pathlib import Path
+from graph_net.torch.fx_graph_cache_util import (
+    parse_immutable_model_path_into_sole_graph_module,
+)
+import os
+import sys
+
+
+class CheckGraphModuleParsable:
+    def __init__(self, config=None):
+        if config is None:
+            config = {}
+        self.config = self._make_config(**config)
+        self.num_handled_models = 0
+
+    def _make_config(
+        self,
+        model_path_prefix,
+        output_dir,
+        resume=False,
+        limits_handled_models=None,
+    ):
+        return {
+            "model_path_prefix": model_path_prefix,
+            "output_dir": output_dir,
+            "resume": resume,
+            "limits_handled_models": limits_handled_models,
+        }
+
+    def __call__(self, rel_model_path):
+        model_path = os.path.join(self.config["model_path_prefix"], rel_model_path)
+        if self.config["resume"] and self._is_model_path_handled(rel_model_path):
+            return
+        parse_immutable_model_path_into_sole_graph_module(model_path)
+        output_dir = Path(self.config["output_dir"]) / rel_model_path
+        output_dir.mkdir(parents=True, exist_ok=True)
+        self._inc_num_handled_models()
+
+    def _is_model_path_handled(self, rel_model_path):
+        return (Path(self.config["output_dir"]) / rel_model_path).exists()
+
+    def _inc_num_handled_models(self):
+        self.num_handled_models += 1
+        limits = self.config["limits_handled_models"]
+        if limits is not None:
+            if self.num_handled_models >= limits:
+                print(
+                    "`num_handled_models` exceeds config `limits_handled_models`",
+                    file=sys.stderr,
+                )
+                sys.exit(0)