add test case

Author: akaitsuki-ii

tests/test_tokenizers/test_qwen2.py

@@ -115,6 +115,153 @@ def test_call(self):
                 "shape": (2, 12),
                 "kwargs": {"padding_side": "left"},
             },
+            {
+                "texts": """
+DiffSynth-Engine is a high-performance engine geared towards buidling efficient inference pipelines for diffusion models.
+Key Features:
+Thoughtfully-Designed Implementation: We carefully re-implemented key components in Diffusion pipelines, such as sampler and scheduler, without introducing external dependencies on libraries like k-diffusion, ldm, or sgm.
+Extensive Model Support: Compatible with popular formats (e.g., CivitAI) of base models and LoRA models , catering to diverse use cases.
+Versatile Resource Management: Comprehensive support for varous model quantization (e.g., FP8, INT8) and offloading strategies, enabling loading of larger diffusion models (e.g., Flux.1 Dev) on limited hardware budget of GPU memory.
+Optimized Performance: Carefully-crafted inference pipeline to achieve fast generation across various hardware environments.
+Cross-Platform Support: Runnable on Windows, macOS (Apple Silicon), and Linux, ensuring a smooth experience across different operating systems.
+""",
+                "expected": torch.tensor(
+                    [
+                        [
+                            198,
+                            21751,
+                            37134,
+                            339,
+                            12,
+                            4571,
+                            374,
+                            264,
+                            1550,
+                            57474,
+                            4712,
+                            58447,
+                            6974,
+                            1031,
+                            307,
+                            2718,
+                            11050,
+                            44378,
+                            57673,
+                            369,
+                            57330,
+                            4119,
+                            624,
+                            1592,
+                            19710,
+                            510,
+                            84169,
+                            3641,
+                            12,
+                            77133,
+                            30813,
+                            25,
+                            1205,
+                            15516,
+                            312,
+                            36925,
+                            14231,
+                            1376,
+                            6813,
+                            304,
+                            28369,
+                            7560,
+                            57673,
+                            11,
+                            1741,
+                            438,
+                            41799,
+                            323,
+                            28809,
+                            11,
+                            2041,
+                            31918,
+                            9250,
+                            19543,
+                            389,
+                            20186,
+                            1075,
+                            595,
+                            1737,
+                            3092,
+                            7560,
+                            11,
+                            326,
+                            13849,
+                            11,
+                            476,
+                            274,
+                            26186,
+                            624,
+                            6756,
+                            4025,
+                            4903,
+                            9186,
+                            25,
+                            66265,
+                            448,
+                            5411,
+                            19856,
+                            320,
+                            68,
+                            1302,
+                            2572,
+                            79135,
+                            275,
+                            15469,
+                            8,
+                            315,
+                            2331,
+                            4119,
+                            323,
+                            6485,
+                            5609,
+                            4119,
+                            1154,
+                            53829,
+                            311,
+                            16807,
+                            990,
+                            5048,
+                            624,
+                            83956,
+                            9010,
+                            11765,
+                            9551,
+                            25,
+                            66863,
+                            1824,
+                            369,
+                            762,
+                            782,
+                            1614,
+                            10272,
+                            2022,
+                            320,
+                            68,
+                            1302,
+                            2572,
+                            33551,
+                            23,
+                            11,
+                            9221,
+                            23,
+                            8,
+                            323,
+                            1007,
+                            10628,
+                            14830,
+                            151645,
+                        ]
+                    ]
+                ),
+                "shape": (1, 128),
+                "kwargs": {"max_length": 128, "padding_side": "left"},
+            },
         ]
 
         for case in cases: