Merge count_sample into check_and_count_samples.

Author: Xreki

.github/workflows/Codestyle-Check.yml

@@ -54,18 +54,4 @@ jobs:
         run: |
           set +e
           bash -x tools/codestyle/pre_commit.sh;EXCODE=$?
-          exit $EXCODE
-
-      - name: Check samples
-      if: steps.check-bypass.outputs.can-skip != 'true'
-        run: |
-          set +e
-          python3.10 tools/check_samples.py;EXCODE=$?
-          exit $EXCODE
-
-      - name: Count samples
-      if: steps.check-bypass.outputs.can-skip != 'true'
-        run: |
-          set +e
-          python3.10 tools/count_sample.py;EXCODE=$?
           exit $EXCODE

tools/check_samples.py tools/check_and_count_samples.pytools/check_samples.py renamed to tools/check_and_count_samples.py

@@ -1,4 +1,5 @@
 import os
+import json
 
 
 def check_completeness(samples_dir):
@@ -31,7 +32,7 @@ def check_completeness(samples_dir):
     )
     for model_path in samples_missing_meta:
         print(f"  - {model_path}")
-    print()
+
     return (
         len(samples_missing_hash) == 0
         and len(samples_missing_json) == 0
@@ -52,37 +53,77 @@ def check_redandancy(samples_dir):
                 graph_hash2model_paths[graph_hash].append(model_path)
 
     has_duplicates = False
-    print(f"Totally {len(graph_hash2model_paths)} unique samples under {samples_dir}.")
+    print(f"Totally {len(graph_hash2model_paths)} unique graphs under {samples_dir}.")
     for graph_hash, model_paths in graph_hash2model_paths.items():
         graph_hash2model_paths[graph_hash] = sorted(model_paths)
         if len(model_paths) > 1:
             has_duplicates = True
             print(f"Redundant models detected for grap_hash {graph_hash}:")
             for model_path in model_paths:
                 print(f"    {model_path}")
-
     return has_duplicates, graph_hash2model_paths
 
 
+def count_samples(samples_dir, framework):
+    model_sources = os.listdir(samples_dir)
+
+    graph_net_count = 0
+    graph_net_dict = {}
+    model_names_set = set()
+    for source in model_sources:
+        source_dir = os.path.join(samples_dir, source)
+        if os.path.isdir(source_dir):
+            graph_net_dict[source] = 0
+            for root, dirs, files in os.walk(source_dir):
+                if "graph_net.json" in files:
+                    with open(os.path.join(root, "graph_net.json"), "r") as f:
+                        data = json.load(f)
+                        model_name = data.get("model_name", None)
+                    if model_name is not None and model_name != "NO_VALID_MATCH_FOUND":
+                        if model_name not in model_names_set:
+                            model_names_set.add(model_name)
+                            graph_net_count += 1
+                            graph_net_dict[source] += 1
+                    else:
+                        graph_net_count += 1
+                        graph_net_dict[source] += 1
+
+    print(f"Number of {framework} samples: {graph_net_count}")
+    for name, number in graph_net_dict.items():
+        print(f"- {name:24}: {number}")
+    print()
+
+
 def main():
     filename = os.path.abspath(__file__)
     root_dir = os.path.dirname(os.path.dirname(filename))
 
+    framework2dirname = {
+        "torch": "samples",
+        "paddle": "paddle_samples",
+    }
+
     all_samples_completed = True
-    for samples_dirname in ["samples", "paddle_samples"]:
+    for samples_dirname in framework2dirname.values():
         samples_dir = os.path.join(root_dir, samples_dirname)
         all_samples_completed = all_samples_completed and check_completeness(
             samples_dir
         )
+    print()
     assert all_samples_completed, "Please fix the incompleted samples!"
 
     all_samples_has_duplicates = False
-    for samples_dirname in ["samples", "paddle_samples"]:
+    for samples_dirname in framework2dirname.values():
         samples_dir = os.path.join(root_dir, samples_dirname)
         has_duplicates, graph_hash2model_paths = check_redandancy(samples_dir)
         all_samples_has_duplicates = all_samples_has_duplicates or has_duplicates
+    print()
     assert not all_samples_has_duplicates, "Please remove the redundant samples!"
 
+    for framework in framework2dirname.keys():
+        samples_dir = os.path.join(root_dir, framework2dirname[framework])
+        count_samples(samples_dir, framework)
+
 
 if __name__ == "__main__":
     main()

tools/ci/check_validate.sh

@@ -163,7 +163,7 @@ function main() {
   check_validation_info=$(check_paddle_validation)
   check_validation_code=$?
   summary_problems $check_validation_code "$check_validation_info"
-  python ${GRAPH_NET_EXTRACT_WORKSPACE}/tools/count_sample.py
+  python ${GRAPH_NET_EXTRACT_WORKSPACE}/tools/check_and_count_samples.py
   LOG "[INFO] check_validation run success and no error!"
 }