experiment with grouping testcases and using dynamic testcase generation

Author: Allen Shih

src/test-apps/Makefile.am

@@ -946,96 +946,25 @@ endif # WEAVE_RUN_LONG_TESTS
 if WEAVE_RUN_HAPPY_WDM
 
 check_SCRIPTS                                 +=                \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_one_way_subscribe_01.py    \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_one_way_subscribe_02.py    \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_one_way_subscribe_03.py    \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_one_way_subscribe_04.py    \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_one_way_subscribe_05.py    \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_one_way_subscribe_06.py    \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_one_way_subscribe_07.py    \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_one_way_subscribe_08.py    \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_one_way_subscribe_09.py    \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_one_way_subscribe_10.py    \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_one_way_subscribe_11.py    \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_one_way_subscribe_12.py    \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_one_way_subscribe_13.py    \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_one_way_subscribe_14.py    \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_one_way_subscribe_15.py    \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_one_way_subscribe_16.py    \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_01.py     \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_02.py     \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_03.py     \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_04.py     \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_05.py     \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_06.py     \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_07.py     \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_08.py     \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_09.py     \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_10.py     \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_11.py     \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_12.py     \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_13.py     \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_14.py     \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_15.py     \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_16.py     \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_17.py     \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_18.py     \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_19.py     \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_20.py     \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_21.py     \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_22.py     \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_23.py     \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_24.py     \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_25.py     \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_26.py     \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_27.py     \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_28.py     \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_29.py     \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_30.py     \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_31.py     \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_32.py     \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_33.py     \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_34.py     \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_35.py     \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_36.py     \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_37.py     \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_38.py     \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_39.py     \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_40.py     \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_41.py     \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_42.py     \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_43.py     \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_44.py     \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_45.py     \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_46.py     \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_47.py     \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_48.py     \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_49.py     \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_50.py     \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_51.py     \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_52.py     \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_53.py     \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_54.py     \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_55.py     \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_56.py     \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_57.py     \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_58.py     \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_59.py     \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_60.py     \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_61.py     \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_62.py     \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_63.py     \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_64.py     \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_65.py     \
+    happy/tests/standalone/wdmNext/test_weave_wdm_next_one_way_subscribe_group_01.py    \
+    happy/tests/standalone/wdmNext/test_weave_wdm_next_one_way_subscribe_group_02.py    \
+    happy/tests/standalone/wdmNext/test_weave_wdm_next_one_way_subscribe_group_03.py    \
+    happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_group_01.py    \
+    happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_group_02.py    \
+    happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_group_03.py    \
+    happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_group_04.py    \
+    happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_group_05.py    \
+    happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_group_06.py    \
+    happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_group_07.py    \
+    happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_group_08.py    \
+    happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_group_09.py    \
+    happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_group_10.py    \
+    happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_group_11.py    \
+    happy/tests/standalone/wdmNext/test_weave_wdm_next_application_key_group.py    \
+    happy/tests/standalone/wdmNext/test_weave_wdm_next_resub_group.py    \
+    happy/tests/standalone/wdmNext/test_weave_wdm_next_update_group.py    \
     happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_faults_happy.sh       \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_application_key_01.py      \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_application_key_02.py      \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_oneway_resub.py            \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_resub.py            \
     happy/tests/standalone/wdmNext/test_weave_wdm_next_subless_notify_01.py       \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_update_01.py       \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_update_02.py       \
-    happy/tests/standalone/wdmNext/test_weave_wdm_next_update_03.py       \
     $(NULL)
 
 endif # WEAVE_RUN_HAPPY_WDM

src/test-apps/happy/tests/standalone/wdmNext/test_weave_wdm_next_application_key_group.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python
+
+"""
+Tests wdm application key
+
+1. B01: Stress Mutual Subscribe: Application key: Key distribution
+2. B02: Stress Mutual Subscribe: Application key: Group key
+"""
+
+from weave_wdm_next_test_base import weave_wdm_next_test_base
+import WeaveUtilities
+
+
+class test_weave_wdm_next_application_key_group(weave_wdm_next_test_base):
+    pass
+
+if __name__ == "__main__":
+    for test_name in [
+            "test_weave_wdm_next_application_key_01",
+            "test_weave_wdm_next_application_key_02"]:
+
+        test = weave_wdm_next_test_base.generate_test(
+            weave_wdm_next_test_base.get_test_param_json(test_name))
+        setattr(test_weave_wdm_next_application_key_group, test_name, test)
+
+    WeaveUtilities.run_unittest()

src/test-apps/happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_group_01.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python
+
+"""
+Tests wdm mutual subscribe root path null version, initiator action.
+
+1. M01: Stress Mutual Subscribe: Root path. Null Version. Client in initiator cancels
+2. M02: Stress Mutual Subscribe: Root path. Null Version. Publisher in initiator cancels
+3. M03: Stress Mutual Subscribe: Root path. Null Version. Client in initiator aborts
+4. M04: Stress Mutual Subscribe: Root path. Null Version. Publisher in initiator aborts
+5. M05: Stress Mutual Subscribe: Root path. Null Version. Idle, Client in initiator cancels
+6. M06: Stress Mutual Subscribe: Root path. Null Version. Idle. Publisher in initiator cancels
+7. M07: Stress Mutual Susbscribe: Root path. Null Version. Idle. Client in initiator aborts
+8. M08: Stress Mutual Subscribe: Root path. Null Version. Idle. Publisher in initiator aborts
+"""
+
+from weave_wdm_next_test_base import weave_wdm_next_test_base
+import WeaveUtilities
+
+
+class test_weave_wdm_next_one_way_subscribe_group_01(weave_wdm_next_test_base):
+    pass
+
+if __name__ == "__main__":
+    for test_name in [
+            "test_weave_wdm_next_mutual_subscribe_01",
+            "test_weave_wdm_next_mutual_subscribe_02",
+            "test_weave_wdm_next_mutual_subscribe_03",
+            "test_weave_wdm_next_mutual_subscribe_04",
+            "test_weave_wdm_next_mutual_subscribe_05",
+            "test_weave_wdm_next_mutual_subscribe_06",
+            "test_weave_wdm_next_mutual_subscribe_07",
+            "test_weave_wdm_next_mutual_subscribe_08"]:
+
+        test = weave_wdm_next_test_base.generate_test(
+            weave_wdm_next_test_base.get_test_param_json(test_name))
+        setattr(test_weave_wdm_next_one_way_subscribe_group_01, test_name, test)
+
+    WeaveUtilities.run_unittest()

src/test-apps/happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_group_02.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python
+
+"""
+Tests wdm mutual subscribe root path null version mutate data in initiator or responder,
+initator action.
+
+9. M09: Stress Mutual Subscribe: Root path. Null Version. Mutate data in initiator. Client in initiator cancels
+10. M10: Stress Mutual Subscribe: Root path. Null Version. Mutate data in initiator. Publisher in initiator cancels
+11. M11: Stress Mutual Subscribe: Root path. Null Version. Mutate data in initiator. Client in initiator aborts
+12. M12: Stress Mutual Subscribe: Root path. Null Version. Mutate data in initiator.Publisher in initiator aborts
+13. M13: Stress Mutual Subscribe: Root path. Null Version. Mutate data in responder. Client in initiator cancels
+14. M14: Stress Mutual Subscribe: Root path, Null Version, Notification in responder, Publisher in initiator Cancel
+15. M15: Stress Mutual Subscribe: Root path. Null Version. Notification in responder. Client in initiator aborts
+16. M16: Stress Mutual Subscribe: Root path. Null Version. Mutate data in responder. Publisher in initiator aborts
+"""
+
+from weave_wdm_next_test_base import weave_wdm_next_test_base
+import WeaveUtilities
+
+
+class test_weave_wdm_next_one_way_subscribe_group_02(weave_wdm_next_test_base):
+    pass
+
+if __name__ == "__main__":
+    for test_name in [
+            "test_weave_wdm_next_mutual_subscribe_09",
+            "test_weave_wdm_next_mutual_subscribe_10",
+            "test_weave_wdm_next_mutual_subscribe_11",
+            "test_weave_wdm_next_mutual_subscribe_12",
+            "test_weave_wdm_next_mutual_subscribe_13",
+            "test_weave_wdm_next_mutual_subscribe_14",
+            "test_weave_wdm_next_mutual_subscribe_15",
+            "test_weave_wdm_next_mutual_subscribe_16"]:
+
+        test = weave_wdm_next_test_base.generate_test(
+            weave_wdm_next_test_base.get_test_param_json(test_name))
+        setattr(test_weave_wdm_next_one_way_subscribe_group_02, test_name, test)
+
+    WeaveUtilities.run_unittest()

src/test-apps/happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_group_03.py

@@ -0,0 +1,31 @@
+#!/usr/bin/env python
+
+"""
+Tests wdm mutual subscribe root path null version mutate data in initiator AND responder,
+initator action.
+
+17. M17: Stress Mutual Subscribe: Root path. Null Version. Mutate data in initiator and responder. Client in initiator cancels
+18. M18: Stress Mutual Subscribe: Root path. Null Version. Mutate data in initiator and responder. Publisher in initiator cancels
+19. M19: Stress Mutual Subscribe: Root path. Null Version. Mutate data in initiator and responder. Client in initiator aborts
+20. M20: Stress Mutual Subscribe: Root path. Null Version. Mutate data in initiator and responder. Publisher in initiator aborts
+"""
+
+from weave_wdm_next_test_base import weave_wdm_next_test_base
+import WeaveUtilities
+
+
+class test_weave_wdm_next_one_way_subscribe_group_03(weave_wdm_next_test_base):
+    pass
+
+if __name__ == "__main__":
+    for test_name in [
+            "test_weave_wdm_next_mutual_subscribe_17",
+            "test_weave_wdm_next_mutual_subscribe_18",
+            "test_weave_wdm_next_mutual_subscribe_19",
+            "test_weave_wdm_next_mutual_subscribe_20"]:
+
+        test = weave_wdm_next_test_base.generate_test(
+            weave_wdm_next_test_base.get_test_param_json(test_name))
+        setattr(test_weave_wdm_next_one_way_subscribe_group_03, test_name, test)
+
+    WeaveUtilities.run_unittest()

src/test-apps/happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_group_04.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python
+
+"""
+Tests wdm mutual subscribe root path null version, responder action.
+
+21. E05: Mutual Subscribe: Root path. Null Version. Client in responder cancels
+22. E06: Mutual Subscribe: Root path. Null Version. Publisher in responder cancels
+23. E07: Mutual Subscribe: Root path. Null Version. Client in responder aborts
+24. E08: Mutual Subscribe: Root path. Null Version. Publisher in responder aborts
+25. F17: Mutual Subscribe: Root path. Null Version. Idle. Client in responder cancels
+26. F18: Mutual Subscribe: Root path. Null Version. Idle. Publisher in responder cancels
+27. F19: Mutual Subscribe: Root path. Null Version. Idle. Client in responder aborts
+28. F20: Mutual Subscribe: Root path. Null Version. Idle. Publisher in responder aborts
+"""
+
+from weave_wdm_next_test_base import weave_wdm_next_test_base
+import WeaveUtilities
+
+
+class test_weave_wdm_next_one_way_subscribe_group_04(weave_wdm_next_test_base):
+    pass
+
+if __name__ == "__main__":
+    for test_name in [
+            "test_weave_wdm_next_mutual_subscribe_21",
+            "test_weave_wdm_next_mutual_subscribe_22",
+            "test_weave_wdm_next_mutual_subscribe_23",
+            "test_weave_wdm_next_mutual_subscribe_24",
+            "test_weave_wdm_next_mutual_subscribe_25",
+            "test_weave_wdm_next_mutual_subscribe_26",
+            "test_weave_wdm_next_mutual_subscribe_27",
+            "test_weave_wdm_next_mutual_subscribe_28"]:
+
+        test = weave_wdm_next_test_base.generate_test(
+            weave_wdm_next_test_base.get_test_param_json(test_name))
+        setattr(test_weave_wdm_next_one_way_subscribe_group_04, test_name, test)
+
+    WeaveUtilities.run_unittest()

src/test-apps/happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_group_05.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python
+
+"""
+Tests wdm mutual subscribe root path null version mutate data in initiator or responder,
+responder action.
+
+29. F21: Mutual Subscribe: Root path. Null Version. Mutate data in initiator. Client in responder cancels"
+30. F22: Mutual Subscribe: Root path. Null Version. Mutate data in initiator. Publisher in responder cancels
+31. F23: Mutual Subscribe: Root path. Null Version. Mutate data in initiator. Client in responder aborts
+32. "F24: Mutual Subscribe: Root path. Null Version. Mutate data in initiator. Publisher in responder Abort
+33. F25: Mutual Subscribe: Root path. Null Version. Mutate data in responder. Client in responder cancels
+34. F26: Mutual Subscribe: Root path. Null Version. Mutate data in responder. Publisher in responder cancels"
+35. F27: Mutual Subscribe: Root path. Null Version. Mutate data in responder. Client in responder aborts"
+36. F28: Mutual Subscribe: Root path. Null Version. Mutate data in responder. Publisher in responder aborts"
+"""
+
+from weave_wdm_next_test_base import weave_wdm_next_test_base
+import WeaveUtilities
+
+
+class test_weave_wdm_next_one_way_subscribe_group_05(weave_wdm_next_test_base):
+    pass
+
+if __name__ == "__main__":
+    for test_name in [
+            "test_weave_wdm_next_mutual_subscribe_29",
+            "test_weave_wdm_next_mutual_subscribe_30",
+            "test_weave_wdm_next_mutual_subscribe_31",
+            "test_weave_wdm_next_mutual_subscribe_32",
+            "test_weave_wdm_next_mutual_subscribe_33",
+            "test_weave_wdm_next_mutual_subscribe_34",
+            "test_weave_wdm_next_mutual_subscribe_35",
+            "test_weave_wdm_next_mutual_subscribe_36"]:
+
+        test = weave_wdm_next_test_base.generate_test(
+            weave_wdm_next_test_base.get_test_param_json(test_name))
+        setattr(test_weave_wdm_next_one_way_subscribe_group_05, test_name, test)
+
+    WeaveUtilities.run_unittest()

src/test-apps/happy/tests/standalone/wdmNext/test_weave_wdm_next_mutual_subscribe_group_06.py

@@ -0,0 +1,31 @@
+#!/usr/bin/env python
+
+"""
+Tests wdm mutual subscribe root path null version mutate data in initiator AND responder,
+responder action.
+
+37. F29: Mutual Subscribe: Root path. Null Version. Mutate data in initiator and responder. Client in responder cancels
+38. F30: Mutual Subscribe: Root path. Null Version. Mutate data in initiator and responder. Publisher in responder cancels
+39. F31: Mutual Subscribe: Root path. Null Version. Mutate data in initiator and responder. Client in responder aborts
+40. F32: Mutual Subscribe: Root path. Null Version. Mutate data in initiator and responder. Publisher in responder aborts
+"""
+
+from weave_wdm_next_test_base import weave_wdm_next_test_base
+import WeaveUtilities
+
+
+class test_weave_wdm_next_one_way_subscribe_group_06(weave_wdm_next_test_base):
+    pass
+
+if __name__ == "__main__":
+    for test_name in [
+            "test_weave_wdm_next_mutual_subscribe_37",
+            "test_weave_wdm_next_mutual_subscribe_38",
+            "test_weave_wdm_next_mutual_subscribe_39",
+            "test_weave_wdm_next_mutual_subscribe_40"]:
+
+        test = weave_wdm_next_test_base.generate_test(
+            weave_wdm_next_test_base.get_test_param_json(test_name))
+        setattr(test_weave_wdm_next_one_way_subscribe_group_06, test_name, test)
+
+    WeaveUtilities.run_unittest()