Merge branch 'develop' of https://github.com/PaddlePaddle/GraphNet into develop

Author: JewelRoam

graph_net/paddle/test_compiler.py

@@ -73,12 +73,24 @@ def get_input_spec(args):
     inputs_params_list = utils.load_converted_list_from_text(f"{args.model_path}")
     input_spec = [None] * len(inputs_params_list)
     for i, v in enumerate(inputs_params_list):
+        name = v["name"]
         dtype = v["info"]["dtype"]
         shape = v["info"]["shape"]
+        # print(f"-- i: {i}, v: name={name}, shape={shape}, dtype={dtype}")
         input_spec[i] = paddle.static.InputSpec(shape, dtype)
     return input_spec
 
 
+def regular_item(item):
+    if isinstance(item, paddle.Tensor) and (item.dtype == paddle.bfloat16):
+        item = np.array(item.astype("float32"))
+    else:
+        item = np.array(item)
+    if item.dtype == np.bool_:
+        item = item.astype("float32")
+    return item
+
+
 def test_single_model(args):
     synchronizer_func = get_synchronizer_func(args)
     input_dict = get_input_dict(args)
@@ -88,21 +100,18 @@ def test_single_model(args):
     build_strategy.build_cinn_pass = False
 
     # eager
-    model = paddle.jit.to_static(
-        model_dy,
-        full_graph=False,
-    )
-    model.eval()
+    print("-- Run with eager mode")
+    model_dy.eval()
     for _ in range(args.warmup if args.warmup > 0 else 0):
-        model(**input_dict)
+        model_dy(**input_dict)
     eager_duration_box = DurationBox(-1)
     with naive_timer(eager_duration_box, synchronizer_func):
-        expected_out = model(**input_dict)
+        expected_out = model_dy(**input_dict)
 
     # compiled
+    print("-- Run with compiled mode")
     build_strategy = paddle.static.BuildStrategy()
-    build_strategy.build_cinn_pass = True
-    compilation_start_time = time.time()
+    # build_strategy.build_cinn_pass = True
     compiled_model = paddle.jit.to_static(
         model_dy,
         input_spec=input_spec,
@@ -118,15 +127,25 @@ def test_single_model(args):
     with naive_timer(compiled_duration_box, synchronizer_func):
         compiled_out = compiled_model(**input_dict)
 
-    expected_out_list = (
-        expected_out if isinstance(expected_out, (list, tuple)) else [expected_out]
-    )
-    compiled_out_list = (
-        compiled_out if isinstance(compiled_out, (list, tuple)) else [compiled_out]
-    )
-
-    processed_expected_out = [t.numpy() for t in expected_out_list]
-    processed_compiled_out = [t.numpy() for t in compiled_out_list]
+    if isinstance(expected_out, paddle.Tensor):
+        expected_out = [expected_out]
+        compiled_out = [compiled_out]
+    if isinstance(expected_out, list) or isinstance(expected_out, tuple):
+        for a, b in zip(expected_out, compiled_out):
+            if (a is None and b is not None) or (a is not None and b is None):
+                raise ValueError("Both expected_out and compiled_out must be not None.")
+        expected_out = [
+            regular_item(item)
+            for item in expected_out
+            if item is not None and np.array(item).size != 0
+        ]
+        compiled_out = [
+            regular_item(item)
+            for item in compiled_out
+            if item is not None and np.array(item).size != 0
+        ]
+    else:
+        raise ValueError("Illegal return value.")
 
     def print_cmp(key, func, **kwargs):
         cmp_ret = func(processed_expected_out, processed_compiled_out, **kwargs)

graph_net/paddle/utils.py

@@ -6,6 +6,7 @@
 import argparse
 import importlib
 import inspect
+import ast
 import paddle
 
 
@@ -115,8 +116,7 @@ def load_converted_list_from_text(file_path):
     weight_info = [
         data for data in convert_meta_classes_to_tensors(f"{file_path}/weight_meta.py")
     ]
-
-    return [*input_info, *weight_info]
+    return [*weight_info, *input_info]
 
 
 def convert_meta_classes_to_tensors(file_path):
@@ -127,32 +127,42 @@ def convert_meta_classes_to_tensors(file_path):
             if not k.startswith("__") and not callable(v)
         }
         data_value = None
-        data_type = getattr(paddle, attrs.get("dtype", "paddle.float").split(".")[-1])
+        data_type = getattr(paddle, attrs.get("dtype", "float32"))
         if attrs.get("data") is not None:
             if isinstance(attrs.get("data"), str):
                 raise ValueError("Unimplemented")
             else:
-                data_value = paddle.to_tensor(
-                    attrs.get("data"), dtype=data_type
-                ).reshape(attrs.get("shape"), [])
+                data_value = paddle.reshape(
+                    paddle.to_tensor(attrs.get("data"), dtype=data_type),
+                    attrs.get("shape", []),
+                )
         yield {
             "info": {
                 "shape": attrs.get("shape", []),
                 "dtype": data_type,
                 "device": attrs.get("device", "gpu"),
                 "mean": attrs.get("mean", 0.0),
                 "std": attrs.get("std", 1.0),
+                "low": attrs.get("low", 0),
+                "high": attrs.get("high", 2),
             },
             "data": data_value,
             "name": attrs.get("name"),
         }
 
 
 def _get_classes(file_path):
+    with open(file_path, "r", encoding="utf-8") as f:
+        tree = ast.parse(f.read(), filename=file_path)
+
+    class_names = [node.name for node in tree.body if isinstance(node, ast.ClassDef)]
+
     spec = importlib.util.spec_from_file_location("unnamed", file_path)
     unnamed = importlib.util.module_from_spec(spec)
     spec.loader.exec_module(unnamed)
-    yield from inspect.getmembers(unnamed, inspect.isclass)
+
+    classes = [(name, getattr(unnamed, name)) for name in class_names]
+    return classes
 
 
 def extract_dynamic_shapes(example_inputs):
@@ -163,11 +173,28 @@ def replay_tensor(info):
     device = info["info"]["device"]
     dtype = info["info"]["dtype"]
     shape = info["info"]["shape"]
+    min_value = info["info"]["low"] if "low" in info["info"] else 0
+    max_value = info["info"]["high"] if "high" in info["info"] else 0.5
     if None in shape:
         shape = list(map(lambda i: i if i is not None else 1, shape))
-    mean = info["info"]["mean"]
-    std = info["info"]["std"]
     if "data" in info and info["data"] is not None:
-        return info["data"].to(device)
-
-    return (paddle.randn(shape).cast(dtype).to(device) * std * 1e-3 + 1e-2).cast(dtype)
+        return paddle.reshape(info["data"], shape).to(dtype).to(device)
+    elif dtype == paddle.int32 or dtype == paddle.int64:
+        # for some ops(binary_cross_entropy), label data can only be set 0 or 1.
+        return paddle.cast(
+            paddle.randint(low=0, high=2, shape=shape, dtype="int64"),
+            dtype,
+        ).to(device)
+    elif dtype == paddle.bool:
+        return paddle.cast(
+            paddle.randint(low=0, high=2, shape=shape, dtype="int32"),
+            paddle.bool,
+        ).to(device)
+    else:
+        std = info["info"]["std"]
+        # return paddle.randn(shape).to(dtype).to(device) * std * 1e-3 + 1e-2
+        return (
+            paddle.uniform(shape, dtype="float32", min=min_value, max=max_value)
+            .to(dtype)
+            .to(device)
+        )

graph_net/paddle/validate.py

@@ -66,16 +66,13 @@ def main(args):
     params.update(inputs)
     state_dict = {k: utils.replay_tensor(v) for k, v in params.items()}
 
-    y = model(**state_dict)[0]
+    y = model(**state_dict)
 
-    print(np.argmin(y), np.argmax(y))
+    # print(np.argmin(y), np.argmax(y))
     if isinstance(y, paddle.Tensor):
         print(y.shape)
-    elif (isinstance(y, list) or isinstance(y, tuple)) and all(
-        isinstance(obj, paddle.Tensor) for obj in y
-    ):
-        # list of paddle.Tensor
-        print(y[0].shape)
+    elif isinstance(y, list) or isinstance(y, tuple):
+        print(y[0].shape if isinstance(y[0], paddle.Tensor) else y[0])
     else:
         raise ValueError("Illegal return value.")
 

graph_net/test/rp_expr_parser_test.py

@@ -0,0 +1,118 @@
+import unittest
+from graph_net.torch.rp_expr import Tokenize
+from graph_net.torch.rp_expr.rp_expr_passes import (
+    FlattenTokenListPass,
+    FoldTokensPass,
+    RecursiveFoldTokensPass,
+    FoldIfTokenIdGreatEqualPass,
+)
+from graph_net.torch.rp_expr.nested_range import Range, Tree
+from graph_net.torch.rp_expr.rp_expr_parser import RpExprParser
+from graph_net.torch.rp_expr.rp_expr_util import (
+    MakeNestedIndexRangeFromLetsListTokenRpExpr,
+)
+
+
+class TestTokenize(unittest.TestCase):
+    """Tests tokenization of primitive ID lists into symbolic token sequences."""
+
+    def test_simple(self):
+        primitive_id_lists = [list(range(10 + i)) for i in range(5)]
+        token_list, id_allocator, _ = Tokenize(primitive_id_lists)
+        self.assertEqual(len(token_list.tensors), len(primitive_id_lists))
+
+
+class TestFlattenTokenListPass(unittest.TestCase):
+    """Tests flattening of nested token structures into linear sequences."""
+
+    def test_simple(self):
+        base = 10
+        size = 5
+        primitive_id_lists = [list(range(base + i)) for i in range(size)]
+        token_list, id_allocator, _ = Tokenize(primitive_id_lists)
+        rp_expr_pass = FlattenTokenListPass(id_allocator)
+        success, flattened_rp_expr_pass = rp_expr_pass(token_list)
+        self.assertTrue(success)
+        self.assertEqual(id_allocator.NextTokenId(), base + 2 * size - 1)
+
+
+class TestFoldTokensPass(unittest.TestCase):
+    """Tests folding of the most frequent contiguous token pattern into a single symbol."""
+
+    def test_simple(self):
+        base = 3
+        size = 3
+        primitive_id_lists = [list(range(base + i)) for i in range(size)]
+        token_list, id_allocator, _ = Tokenize(primitive_id_lists)
+        flatten_pass = FlattenTokenListPass(id_allocator)
+        _, flattened_rp_expr = flatten_pass(token_list)
+        fold_pass = FoldTokensPass(id_allocator)
+        success, fold_rp_expr = fold_pass(flattened_rp_expr.flattened_tensor)
+        self.assertTrue(success)
+        input = flattened_rp_expr.flattened_tensor.tensor.numpy().tolist()
+        pattern = fold_rp_expr.symbol_token_tensors[0].numpy().tolist()
+        replacement = fold_rp_expr.symbol_token_ids[0]
+        output = fold_rp_expr.body_rp_expr.tensor.numpy().tolist()
+        self.assertEqual(input, [3, 4, 5, 1, 3, 4, 5, 6, 2, 3, 4, 5, 6, 7])
+        self.assertEqual(pattern, [3, 4, 5])
+        self.assertEqual(replacement, 8)
+        self.assertEqual(output, [8, 1, 8, 6, 2, 8, 6, 7])
+
+
+class TestRecursiveFoldTokensPass(unittest.TestCase):
+    """Tests recursive folding of repeated patterns at multiple levels of nesting."""
+
+    def test_simple(self):
+        base = 3
+        size = 3
+        primitive_id_lists = [list(range(base + i)) for i in range(size)]
+        token_list, id_allocator, _ = Tokenize(primitive_id_lists)
+        flatten_pass = FlattenTokenListPass(id_allocator)
+        _, flattened_rp_expr = flatten_pass(token_list)
+        fold_pass = RecursiveFoldTokensPass(id_allocator)
+        success, fold_rp_expr = fold_pass(flattened_rp_expr.flattened_tensor)
+        self.assertTrue(success)
+        input = flattened_rp_expr.flattened_tensor.tensor.numpy().tolist()
+        pattern = [x.numpy().tolist() for x in fold_rp_expr.symbol_token_tensors]
+        replacement = fold_rp_expr.symbol_token_ids
+        output = fold_rp_expr.body_rp_expr.tensor.numpy().tolist()
+        self.assertEqual(input, [3, 4, 5, 1, 3, 4, 5, 6, 2, 3, 4, 5, 6, 7])
+        self.assertEqual(pattern, [[3, 4, 5], [8, 6]])
+        self.assertEqual(replacement, [8, 9])
+        self.assertEqual(output, [8, 1, 9, 2, 9, 7])
+
+
+class TestFoldIfTokenIdGreatEqualPass(unittest.TestCase):
+    """Tests conditional folding only for tokens with ID greater than or equal to a threshold."""
+
+    def test_simple(self):
+        base = 3
+        size = 3
+        primitive_id_lists = [list(range(base + i)) for i in range(size)]
+        token_list, id_allocator, _ = Tokenize(primitive_id_lists)
+        flatten_pass = FlattenTokenListPass(id_allocator)
+        _, flattened_rp_expr = flatten_pass(token_list)
+        fold_pass = RecursiveFoldTokensPass(id_allocator)
+        success, fold_rp_expr = fold_pass(flattened_rp_expr.flattened_tensor)
+        self.assertTrue(success)
+        threshold_fold_pass = FoldIfTokenIdGreatEqualPass(
+            id_allocator=id_allocator,
+            threshold_start_token_id=len(primitive_id_lists),
+        )
+        success, threshold_fold_rp_expr = threshold_fold_pass(fold_rp_expr.body_rp_expr)
+        self.assertTrue(success)
+        input = fold_rp_expr.body_rp_expr.tensor.numpy().tolist()
+        pattern = [
+            x.numpy().tolist() for x in threshold_fold_rp_expr.symbol_token_tensors
+        ]
+        replacement = threshold_fold_rp_expr.symbol_token_ids
+        self.assertEqual(len(threshold_fold_rp_expr.body_rp_expr), 3)
+        output = [x.numpy().tolist() for x in threshold_fold_rp_expr.body_rp_expr]
+        self.assertEqual(input, [8, 1, 9, 2, 9, 7])
+        self.assertEqual(pattern, [[9, 7]])
+        self.assertEqual(replacement, [10])
+        self.assertEqual(output, [[8], [9], [10]])
+
+
+if __name__ == "__main__":
+    unittest.main()

graph_net/torch/rp_expr/__init__.py

@@ -0,0 +1,22 @@
+from .rp_expr import Tokenize
+from .rp_expr_parser import RpExprParser
+from .nested_range import Range, Tree
+from .rp_expr_util import MakeNestedIndexRangeFromLetsListTokenRpExpr
+from .rp_expr_passes import (
+    FlattenTokenListPass,
+    FoldTokensPass,
+    RecursiveFoldTokensPass,
+    FoldIfTokenIdGreatEqualPass,
+)
+
+__all__ = [
+    "Tokenize",
+    "RpExprParser",
+    "Range",
+    "Tree",
+    "MakeNestedIndexRangeFromLetsListTokenRpExpr",
+    "FlattenTokenListPass",
+    "FoldTokensPass",
+    "RecursiveFoldTokensPass",
+    "FoldIfTokenIdGreatEqualPass",
+]