Add an optional teardown action block to pages

Add an optional teardown action block that is executed only after all
playback iterations are complete.

Change-Id: If3f3e3795df9e05f1ed193fb58a61f15d1179e93
Reviewed-on: https://chromium-review.googlesource.com/c/crossbench/+/6678071
Reviewed-by: Charles Dick <[email protected]>
Commit-Queue: Kameron Lutes <[email protected]>

Author: kalutes

crossbench/action_runner/base.py

@@ -81,8 +81,22 @@ def info_stack(self) -> exception.TInfoStack:
   def bond(self) -> BondActionRunner:
     return BondActionRunner()
 
-  def teardown(self, run: Run) -> None:
-    del run
+  def teardown(self,
+               run: Run,
+               page: InteractivePage,
+               teardown: Optional[ActionBlock] = None) -> None:
+    if teardown:
+      try:
+        with exception.annotate("teardown"):
+          self._info_stack = ("teardown",)
+          teardown.run_with(self, run, page)
+      except Exception:
+        page.create_failure_artifacts(run, "failure")
+        raise
+
+    self._teardown_impl()
+
+  def _teardown_impl(self) -> None:
     pass
 
   def run_blocks(self, run: Run, page: InteractivePage,

crossbench/action_runner/default_action_runner.py

@@ -128,8 +128,7 @@ def bond(self) -> BondActionRunner:
     return self._bond
 
   @override
-  def teardown(self, run: Run) -> None:
-    del run
+  def _teardown_impl(self) -> None:
     if self._bond:
       self._bond.teardown()
 

crossbench/benchmarks/loading/config/page.py

@@ -35,6 +35,7 @@ class PageConfig(ConfigObject):
   secrets: Secrets = Secrets()
   login: LoginBlock | None = None
   setup: ActionBlock | None = None
+  teardown: Optional[ActionBlock] = None
   blocks: tuple[ActionBlock, ...] = tuple()
 
   @classmethod
@@ -99,6 +100,7 @@ def config_parser(cls) -> ConfigParser[Self]:
     parser.add_argument("secrets", type=Secrets, default=Secrets())
     parser.add_argument("login", type=LoginBlock)
     parser.add_argument("setup", type=ActionBlock)
+    parser.add_argument("teardown", type=ActionBlock)
     parser.add_argument(
         "blocks",
         aliases=("actions", "url", "urls"),

crossbench/benchmarks/loading/loading_benchmark.py

@@ -230,10 +230,18 @@ def _story_from_config(cls, args: argparse.Namespace, config: PageConfig,
     if not config.blocks:
       return LivePage(label, config.first_url, duration, playback, tabs,
                       args.about_blank_duration)
-    return InteractivePage(label, config.blocks, config.setup, config.login,
-                           config.secrets, playback, tabs,
-                           args.about_blank_duration, args.run_login,
-                           args.run_setup)
+    return InteractivePage(
+        name=label,
+        blocks=config.blocks,
+        setup=config.setup,
+        teardown=config.teardown,
+        login=config.login,
+        secrets=config.secrets,
+        playback=playback,
+        tabs=tabs,
+        about_blank_duration=args.about_blank_duration,
+        run_login=args.run_login,
+        run_setup=args.run_setup)
 
   @override
   def create_stories(self, separate: bool) -> Sequence[Page]:

crossbench/benchmarks/loading/page/interactive.py

@@ -32,6 +32,7 @@ def __init__(self,
                name: str,
                blocks: tuple[ActionBlock, ...],
                setup: Optional[ActionBlock] = None,
+               teardown: Optional[ActionBlock] = None,
                login: Optional[LoginBlock] = None,
                secrets: Optional[Secrets] = None,
                playback: PlaybackController = PlaybackController.default(),
@@ -47,6 +48,7 @@ def __init__(self,
     assert not any(block.is_login for block in blocks), (
         "No login blocks allowed as normal action block")
     self._setup_block = setup
+    self._teardown_block: Optional[ActionBlock] = teardown
     self._login_block = login
     self._run_login = run_login
     self._run_setup = run_setup
@@ -62,6 +64,10 @@ def login_block(self) -> Optional[ActionBlock]:
   def setup_block(self) -> Optional[ActionBlock]:
     return self._setup_block
 
+  @property
+  def teardown_block(self) -> Optional[ActionBlock]:
+    return self._teardown_block
+
   @property
   def blocks(self) -> tuple[ActionBlock, ...]:
     return self._blocks
@@ -102,7 +108,9 @@ def setup(self, run: Run) -> None:
   @override
   def teardown(self, run: Run) -> None:
     action_runner = get_action_runner(run)
-    action_runner.teardown(run)
+    run.browser.performance_mark("teardown-start", self._name)
+    action_runner.teardown(run, self, self.teardown_block)
+    run.browser.performance_mark("teardown-end", self._name)
 
   def run_once(self, run: Run) -> None:
     action_runner = get_action_runner(run)

tests/crossbench/benchmarks/loading/config/test_page.py

@@ -148,6 +148,7 @@ def test_parse_action_sequence(self):
     self.assertEqual(config.first_url, "http://test.com/click")
     self.assertIsNone(config.login)
     self.assertIsNone(config.setup)
+    self.assertIsNone(config.teardown)
     self.assertEqual(len(tuple(config.actions())), 2)
 
   def test_parse_actions_dict(self):
@@ -163,6 +164,7 @@ def test_parse_actions_dict(self):
     config_1 = PageConfig.parse(config_data)
     self.assertIsNone(config_1.login)
     self.assertIsNone(config_1.setup)
+    self.assertIsNone(config_1.teardown)
     self.assertEqual(config_1.first_url, "http://test.com/click")
     self.assertEqual(len(tuple(config_1.actions())), 2)
 
@@ -185,6 +187,7 @@ def test_parse_login_block(self):
     login = config.login
     self.assertTrue(login.is_login)
     self.assertIsNone(config.setup)
+    self.assertIsNone(config.teardown)
     self.assertFalse(config.blocks[0].is_login)
     self.assertEqual(config.first_url, "http://test.com/charts")
     self.assertEqual(len(config.blocks), 1)
@@ -207,6 +210,7 @@ def test_parse_setup_block(self):
     config = PageConfig.parse(config_data)
     self.assertEqual(len(config.login), 1)
     self.assertEqual(len(config.setup), 2)
+    self.assertIsNone(config.teardown)
     self.assertEqual(len(config.blocks), 1)
     self.assertEqual(config.login.first_url, "http://test.com/login")
     self.assertEqual(config.setup.first_url, "http://test.com/setup")
@@ -224,6 +228,25 @@ def test_parse_login_block_preset(self):
     self.assertEqual(len(config.blocks), 1)
     self.assertEqual(len(tuple(config.actions())), 1)
 
+  def test_parse_teardown_block(self):
+    config_data = {
+        "actions": ["http://test.com/charts",],
+        "teardown": [{
+            "action": "get",
+            "url": "http://test.com/teardown"
+        }, {
+            "action": "click",
+            "selector": "#foo"
+        }],
+    }
+
+    config = PageConfig.parse(config_data)
+    self.assertIsNone(config.setup)
+    self.assertEqual(len(config.blocks), 1)
+    self.assertEqual(config.blocks[0].first_url, "http://test.com/charts")
+    self.assertEqual(len(config.teardown), 2)
+    self.assertEqual(config.teardown.first_url, "http://test.com/teardown")
+
 
 if __name__ == "__main__":
   test_helper.run_pytest(__file__)

tests/crossbench/benchmarks/loading/test_loading.py

@@ -221,10 +221,10 @@ def test_iteration_performance_marks_single_run(self):
 
     for browser in self.browsers:
       # one mark for iteration start, one for iteration end
-      self.assertEqual(len(browser.performance_marks), 2)
-      self.assertEqual(browser.performance_marks[0],
-                       "crossbench-iteration-start")
-      self.assertEqual(browser.performance_marks[1], "crossbench-iteration-end")
+      self.assertListEqual(
+          browser.performance_marks,
+          ["crossbench-iteration-start", "crossbench-iteration-end"] +
+          ["crossbench-teardown-start", "crossbench-teardown-end"] * 2)
 
   def test_iteration_performance_marks_repeat_run(self):
     repeats: int = 3
@@ -237,13 +237,10 @@ def test_iteration_performance_marks_repeat_run(self):
     self._test_run(stories)
 
     for browser in self.browsers:
-      # one mark for iteration start, one for iteration end
-      self.assertEqual(len(browser.performance_marks), 2 * repeats)
-      for i in range(repeats):
-        self.assertEqual(browser.performance_marks[i * 2],
-                         "crossbench-iteration-start")
-        self.assertEqual(browser.performance_marks[(i * 2) + 1],
-                         "crossbench-iteration-end")
+      self.assertListEqual(
+          browser.performance_marks,
+          (["crossbench-iteration-start", "crossbench-iteration-end"] * repeats)
+          + ["crossbench-teardown-start", "crossbench-teardown-end"] * 2)
 
   def test_run_repeat_separate(self):
     url1 = "https://www.example.com/test1"
@@ -446,7 +443,7 @@ def test_actions_config(self):
         self.assertListEqual([url_1, url_2],
                              browser.url_list[self.SPLASH_URLS_LEN:])
 
-  def multiple_pages_with_setup_blocks_config(self):
+  def multiple_pages_with_setup_and_teardown_blocks_config(self):
     config = {
         "pages": {
             "first_page": {
@@ -457,7 +454,11 @@ def multiple_pages_with_setup_blocks_config(self):
                 "actions": [{
                     "action": "wait",
                     "duration": "1s"
-                }]
+                }],
+                "teardown": [{
+                    "action": "js",
+                    "script": "TEARDOWN ONE",
+                }],
             },
             "second_page": {
                 "setup": [{
@@ -467,18 +468,24 @@ def multiple_pages_with_setup_blocks_config(self):
                 "actions": [{
                     "action": "wait",
                     "duration": "1s"
-                }]
+                }],
+                "teardown": [{
+                    "action": "js",
+                    "script": "TEARDOWN TWO",
+                }],
             }
         }
     }
     return config
 
-  def test_pages_with_multiple_setup_blocks(self):
+  def test_pages_with_multiple_setup_and_teardown_blocks(self):
     for browser in self.browsers:
       browser.expect_js(JsInvocation(None, "SETUP ONE"))
       browser.expect_js(JsInvocation(None, "SETUP TWO"))
+      browser.expect_js(JsInvocation(None, "TEARDOWN ONE"))
+      browser.expect_js(JsInvocation(None, "TEARDOWN TWO"))
 
-    config = self.multiple_pages_with_setup_blocks_config()
+    config = self.multiple_pages_with_setup_and_teardown_blocks_config()
     config_file = pathlib.Path("test/page_config.json")
     self.fs.create_file(config_file, contents=json.dumps(config))
     with self._patch_get_browser():
@@ -489,9 +496,11 @@ def test_pages_with_multiple_setup_blocks(self):
       self.assertEqual(
           browser.performance_marks,
           ["crossbench-setup-start", "crossbench-setup-end"] * 2 +  # 2 pages
-          ["crossbench-iteration-start", "crossbench-iteration-end"])
+          ["crossbench-iteration-start", "crossbench-iteration-end"] +
+          ["crossbench-teardown-start", "crossbench-teardown-end"] * 2)
       self.assertEqual(browser.performance_marks_details,
-                       ["first_page"] * 2 + ["second_page"] * 2 + [None, None])
+                       ["first_page"] * 2 + ["second_page"] * 2 + [None, None] +
+                       ["first_page"] * 2 + ["second_page"] * 2)
 
   def setup_expected_google_login_js(self):
     expected_scripts: list[JsInvocation] = [