Skip to content

Commit edaf2bd

Browse files
khokhlov962V8 LUCI CQ
authored andcommitted
[LoadLine] Support --story command-line flag
Make it possible to run a subset of stories in LoadLine 1 and 2, for debugging/research purposes. Change-Id: I6ec2f4a8b9dc2cb242d01c65e1c6eadd012bff58 Reviewed-on: https://chromium-review.googlesource.com/c/crossbench/+/6651481 Reviewed-by: Camillo Bruni <[email protected]> Commit-Queue: Mikhail Khokhlov <[email protected]>
1 parent 8154545 commit edaf2bd

File tree

7 files changed

+149
-70
lines changed

7 files changed

+149
-70
lines changed
Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,40 +1,42 @@
11
{
22
pages: {
3-
loadline_phone_workload: [
4-
{action: "get", url: "about:blank"},
5-
3+
amazon_product: [
64
{action: "inject_new_document_script", script_path: "amazon_instrumentation.js", timeout: "5s"},
75
{action: "js", script: '''
86
performance.mark('LoadLine2/phone/amazon_product_start');
97
window.location.href = 'https://www.amazon.co.uk/NIVEA-Suncream-Spray-Protect-Moisture/dp/B001B0OJXM';
108
'''},
119
{action: "wait_for_ready_state", ready_state: "complete", timeout: "10s"},
1210
{action: "js", script: "performance.mark('LoadLine2/phone/amazon_product_finish')"},
13-
11+
],
12+
cnn_article: [
1413
{action: "inject_new_document_script", script_path: "cnn_instrumentation.js", timeout: "5s"},
1514
{action: "js", script: '''
1615
performance.mark('LoadLine2/phone/cnn_article_start');
1716
window.location.href = 'https://edition.cnn.com/2024/04/21/china/china-spy-agency-public-profile-intl-hnk/index.html';
1817
'''},
1918
{action: "wait_for_ready_state", ready_state: "complete", timeout: "10s"},
2019
{action: "js", script: "performance.mark('LoadLine2/phone/cnn_article_finish')"},
21-
20+
],
21+
wikipedia_article: [
2222
{action: "inject_new_document_script", script_path: "wiki_instrumentation.js", timeout: "5s"},
2323
{action: "js", script: '''
2424
performance.mark('LoadLine2/phone/wikipedia_article_start');
2525
window.location.href = 'https://en.m.wikipedia.org/wiki/Taylor_Swift';
2626
'''},
2727
{action: "wait_for_ready_state", ready_state: "complete", timeout: "10s"},
2828
{action: "js", script: "performance.mark('LoadLine2/phone/wikipedia_article_finish')"},
29-
29+
],
30+
globo_homepage: [
3031
{action: "inject_new_document_script", script_path: "globo_instrumentation.js", timeout: "5s"},
3132
{action: "js", script: '''
3233
performance.mark('LoadLine2/phone/globo_homepage_start');
3334
window.location.href = 'https://www.globo.com/';
3435
'''},
3536
{action: "wait_for_ready_state", ready_state: "complete", timeout: "10s"},
3637
{action: "js", script: "performance.mark('LoadLine2/phone/globo_homepage_finish')"},
37-
38+
],
39+
google_search_result: [
3840
// TODO(crbug.com/372457479): Consider using existing markers emitted by page such as trigger:SearchAFTEnd
3941
// instead of custom ones.
4042
{action: "inject_new_document_script", script_path: "google_search_phone_instrumentation.js", timeout: "5s"},
@@ -44,8 +46,6 @@
4446
'''},
4547
{action: "wait_for_ready_state", ready_state: "complete", timeout: "10s"},
4648
{action: "js", script: "performance.mark('LoadLine2/phone/google_search_result_finish')"},
47-
48-
{action: "get", url: "about:blank"},
4949
],
5050
},
5151
}
Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,51 +1,51 @@
11
{
22
pages: {
3-
loadline_tablet_workload: [
4-
{action: "get", url: "about:blank"},
5-
3+
amazon_product: [
64
{action: "inject_new_document_script", script_path: "amazon_instrumentation.js", timeout: "5s"},
75
{action: "js", script: '''
86
performance.mark('LoadLine2/tablet/amazon_product_start');
9-
window.location.href = "https://www.amazon.co.uk/NIVEA-Suncream-Spray-Protect-Moisture/dp/B001B0OJXM";
7+
window.location.href = 'https://www.amazon.co.uk/NIVEA-Suncream-Spray-Protect-Moisture/dp/B001B0OJXM';
108
'''},
119
{action: "wait_for_ready_state", ready_state: "complete", timeout: "20s"},
1210
{action: "js", script: "performance.mark('LoadLine2/tablet/amazon_product_finish')"},
13-
11+
],
12+
google_doc: [
1413
{action: "inject_new_document_script", script_path: "google_docs_instrumentation.js", timeout: "5s"},
1514
{action: "js", script: '''
1615
performance.mark('LoadLine2/tablet/google_doc_start');
17-
window.location.href = "https://docs.google.com/document/d/13AWeOGqtSkfpPK7meqE_X-GQQggwx4JJ1vc0YGvKg34/edit#heading=h.gjdgxs";
16+
window.location.href = 'https://docs.google.com/document/d/13AWeOGqtSkfpPK7meqE_X-GQQggwx4JJ1vc0YGvKg34/edit#heading=h.gjdgxs';
1817
'''},
1918
{action: "wait_for_ready_state", ready_state: "complete", timeout: "20s"},
2019
{action: "js", script: "performance.mark('LoadLine2/tablet/google_doc_finish')"},
21-
20+
],
21+
cnn_article: [
2222
{action: "inject_new_document_script", script_path: "cnn_instrumentation.js", timeout: "5s"},
2323
{action: "js", script: '''
2424
performance.mark('LoadLine2/tablet/cnn_article_start');
25-
window.location.href = "https://edition.cnn.com/2024/04/21/china/china-spy-agency-public-profile-intl-hnk/index.html";
25+
window.location.href = 'https://edition.cnn.com/2024/04/21/china/china-spy-agency-public-profile-intl-hnk/index.html';
2626
'''},
2727
{action: "wait_for_ready_state", ready_state: "complete", timeout: "20s"},
2828
{action: "js", script: "performance.mark('LoadLine2/tablet/cnn_article_finish')"},
29-
29+
],
30+
google_search_result: [
3031
// TODO(crbug.com/372457479): Consider using existing markers emitted by page such as trigger:SearchAFTEnd
3132
// instead of custom ones.
3233
{action: "inject_new_document_script", script_path: "google_search_tablet_instrumentation.js", timeout: "5s"},
3334
{action: "js", script: '''
3435
performance.mark('LoadLine2/tablet/google_search_result_start');
35-
window.location.href = "https://www.google.com/search?q=cats";
36+
window.location.href = 'https://www.google.com/search?q=cats';
3637
'''},
3738
{action: "wait_for_ready_state", ready_state: "complete", timeout: "20s"},
3839
{action: "js", script: "performance.mark('LoadLine2/tablet/google_search_result_finish')"},
39-
40+
],
41+
youtube_video: [
4042
{action: "inject_new_document_script", script_path: "youtube_instrumentation.js", timeout: "5s"},
4143
{action: "js", script: '''
4244
performance.mark('LoadLine2/tablet/youtube_video_start');
43-
window.location.href = "https://www.youtube.com/watch?v=WuS9kPNAXHw&themeRefresh=1";
45+
window.location.href = 'https://www.youtube.com/watch?v=WuS9kPNAXHw&themeRefresh=1';
4446
'''},
4547
{action: "wait_for_condition", condition: "return !!localStorage.getItem('youtube_complete')", timeout: "20s"},
4648
{action: "js", script: "performance.mark('LoadLine2/tablet/youtube_video_finish');"},
47-
48-
{action: "get", url: "about:blank"},
4949
],
5050
},
5151
}

crossbench/benchmarks/base.py

Lines changed: 57 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -275,53 +275,25 @@ def all_story_names(cls) -> Sequence[str]:
275275
"PressBenchmarkStoryT", bound=PressBenchmarkStory)
276276

277277

278-
class PressBenchmarkStoryFilter(StoryFilter[PressBenchmarkStoryT],
279-
Generic[PressBenchmarkStoryT]):
280-
"""
281-
Filter stories by name or regexp.
278+
class RegexFilter():
282279

283-
Syntax:
284-
"all" Include all stories (defaults to story_names).
285-
"name" Include story with the given name.
286-
"-name" Exclude story with the given name'
287-
"foo.*" Include stories whose name matches the regexp.
288-
"-foo.*" Exclude stories whose name matches the regexp.
289-
290-
These patterns can be combined:
291-
[".*", "-foo", "-bar"] Includes all except the "foo" and "bar" story
292-
"""
293-
294-
@classmethod
295-
@override
296-
def kwargs_from_cli(cls, args: argparse.Namespace) -> dict[str, Any]:
297-
kwargs = super().kwargs_from_cli(args)
298-
kwargs["separate"] = args.separate
299-
kwargs["url"] = args.custom_benchmark_url
300-
return kwargs
301-
302-
def __init__(self,
303-
story_cls: Type[PressBenchmarkStoryT],
304-
patterns: Sequence[str],
305-
args: Optional[argparse.Namespace] = None,
306-
separate: bool = False,
307-
url: Optional[str] = None) -> None:
308-
self.url: str | None = url
280+
def __init__(self, all_names: Sequence[str], default_names: Sequence[str]):
281+
self._all_names: dict[str, None] = dict.fromkeys(all_names)
282+
self._default_names: dict[str, None] = dict.fromkeys(default_names)
309283
self._selected_names: OrderedSet[str] = OrderedSet()
310-
super().__init__(story_cls, patterns, args, separate)
311-
assert issubclass(self.story_cls, PressBenchmarkStory)
312-
for name in self._known_names:
284+
for name in self._all_names:
313285
assert name, "Invalid empty story name"
314286
assert not name.startswith("-"), (
315287
f"Known story names cannot start with '-', but got '{name}'.")
316288
assert not name == "all", "Known story name cannot match 'all'."
317289

318-
@override
319-
def process_all(self, patterns: Sequence[str]) -> None:
290+
def process_all(self, patterns: Sequence[str]) -> OrderedSet[str]:
320291
if not isinstance(patterns, (list, tuple)):
321292
raise ValueError("Expected Sequence of story name or patterns "
322293
f"but got '{type(patterns)}'.")
323294
for pattern in patterns:
324295
self.process_pattern(pattern)
296+
return self._selected_names
325297

326298
def process_pattern(self, pattern: str) -> None:
327299
if pattern.startswith("-"):
@@ -343,12 +315,11 @@ def _pattern_to_regexp(self, pattern: str) -> re.Pattern:
343315
if pattern == "all":
344316
return re.compile(".*")
345317
if pattern == "default":
346-
default_story_names = self.story_cls.default_story_names()
347-
if default_story_names == self.story_cls.all_story_names():
318+
if self._default_names == self._all_names:
348319
return re.compile(".*")
349-
joined_names = "|".join(re.escape(name) for name in default_story_names)
320+
joined_names = "|".join(re.escape(name) for name in self._default_names)
350321
return re.compile(f"^({joined_names})$")
351-
if pattern in self._known_names:
322+
if pattern in self._all_names:
352323
return re.compile(re.escape(pattern))
353324
return re.compile(pattern)
354325

@@ -378,13 +349,13 @@ def _remove_matching(self, regexp: re.Pattern, original_pattern: str) -> None:
378349
def _regexp_match(self, regexp: re.Pattern,
379350
original_pattern: str) -> list[str]:
380351
substories = [
381-
substory for substory in self._known_names if regexp.fullmatch(substory)
352+
substory for substory in self._all_names if regexp.fullmatch(substory)
382353
]
383354
if not substories:
384355
substories = self._regexp_match_ignorecase(regexp)
385356
if not substories:
386357
return self._handle_no_match(original_pattern)
387-
if len(substories) == len(self._known_names) and self._selected_names:
358+
if len(substories) == len(self._all_names) and self._selected_names:
388359
raise ValueError(f"'{original_pattern}' matched all and overrode all"
389360
"previously filtered story names.")
390361
return substories
@@ -394,20 +365,62 @@ def _regexp_match_ignorecase(self, regexp: re.Pattern) -> list[str]:
394365
"No matching stories, using case-insensitive fallback regexp.")
395366
iregexp: re.Pattern = re.compile(regexp.pattern, flags=re.IGNORECASE)
396367
return [
397-
substory for substory in self._known_names
398-
if iregexp.fullmatch(substory)
368+
substory for substory in self._all_names if iregexp.fullmatch(substory)
399369
]
400370

401371
def _handle_no_match(self, original_pattern: str) -> list[str]:
402372
choices_ms, alternative = close_matches_message(original_pattern,
403-
self._known_names)
373+
self._all_names)
404374
error_message: str = f"'{original_pattern}' didn't match any stories."
405375
error_message += choices_ms
406376
if alternative:
407377
logging.error(error_message)
408378
return [alternative]
409379
raise ValueError(error_message)
410380

381+
382+
class PressBenchmarkStoryFilter(StoryFilter[PressBenchmarkStoryT],
383+
Generic[PressBenchmarkStoryT]):
384+
"""
385+
Filter stories by name or regexp.
386+
387+
Syntax:
388+
"all" Include all stories (defaults to story_names).
389+
"name" Include story with the given name.
390+
"-name" Exclude story with the given name'
391+
"foo.*" Include stories whose name matches the regexp.
392+
"-foo.*" Exclude stories whose name matches the regexp.
393+
394+
These patterns can be combined:
395+
[".*", "-foo", "-bar"] Includes all except the "foo" and "bar" story
396+
"""
397+
398+
@classmethod
399+
@override
400+
def kwargs_from_cli(cls, args: argparse.Namespace) -> dict[str, Any]:
401+
kwargs = super().kwargs_from_cli(args)
402+
kwargs["separate"] = args.separate
403+
kwargs["url"] = args.custom_benchmark_url
404+
return kwargs
405+
406+
def __init__(self,
407+
story_cls: Type[PressBenchmarkStoryT],
408+
patterns: Sequence[str],
409+
args: Optional[argparse.Namespace] = None,
410+
separate: bool = False,
411+
url: Optional[str] = None) -> None:
412+
self.url: str | None = url
413+
self._selected_names: OrderedSet[str] = OrderedSet()
414+
super().__init__(story_cls, patterns, args, separate)
415+
assert issubclass(self.story_cls, PressBenchmarkStory)
416+
417+
@override
418+
def process_all(self, patterns: Sequence[str]) -> None:
419+
regex_filter = RegexFilter(
420+
all_names=self.story_cls.all_story_names(),
421+
default_names=self.story_cls.default_story_names())
422+
self._selected_names = regex_filter.process_all(patterns)
423+
411424
@override
412425
def create_stories(self, separate: bool) -> Sequence[PressBenchmarkStoryT]:
413426
names = list(self._selected_names)

crossbench/benchmarks/loading/config/page.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66

77
import dataclasses
88
import datetime as dt
9+
import logging
910
from typing import TYPE_CHECKING, Any, Iterator, Optional, Self, Sequence, cast
1011

1112
from typing_extensions import override
@@ -142,4 +143,5 @@ def first_url(self) -> str:
142143
for action in self.actions():
143144
if action.TYPE == ActionType.GET:
144145
return cast(GetAction, action).url
145-
raise RuntimeError("No GET action with an URL found.")
146+
logging.debug("PageConfig: No GET action with an URL found.")
147+
return ""

crossbench/benchmarks/loadline/loadline.py

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
from typing_extensions import override
1515

1616
from crossbench import path as pth
17+
from crossbench.benchmarks.base import RegexFilter
1718
from crossbench.benchmarks.benchmark_probe import BenchmarkProbeMixin
1819
from crossbench.benchmarks.loading.config.pages import PagesConfig
1920
from crossbench.benchmarks.loading.loading_benchmark import (LoadingBenchmark,
@@ -138,6 +139,25 @@ def get_pages_config(
138139
"--config is not supported with loadline.")
139140
return args.pages_config
140141

142+
@classmethod
143+
@override
144+
def stories_from_cli_args(cls, args: argparse.Namespace) -> Sequence[Page]:
145+
config = cls.get_pages_config(args)
146+
assert cls._page_config is not None
147+
148+
if args.stories:
149+
all_page_labels = [str(page.label) for page in config.pages]
150+
regex_filter = RegexFilter(
151+
all_names=all_page_labels, default_names=all_page_labels)
152+
filtered_page_labels = regex_filter.process_all(args.stories.split(","))
153+
filtered_pages = tuple(
154+
page for page in config.pages if page.label in filtered_page_labels
155+
)
156+
config = PagesConfig(
157+
pages=filtered_pages, secrets=cls._page_config.secrets)
158+
159+
return cls.STORY_FILTER_CLS.stories_from_config(args, config)
160+
141161
@classmethod
142162
@override
143163
def describe_stories(cls) -> Mapping[str, str]:

crossbench/benchmarks/loadline/loadline_2.py

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,14 +4,15 @@
44

55
from __future__ import annotations
66

7-
from typing import TYPE_CHECKING, Type
7+
from typing import TYPE_CHECKING, Sequence, Type
88

99
import numpy as np
1010
import pandas as pd
1111
from typing_extensions import override
1212

1313
from crossbench import config
1414
from crossbench import path as pth
15+
from crossbench.benchmarks.loading.page.combined import CombinedPage
1516
from crossbench.benchmarks.loadline.loadline import (LoadLineBenchmark,
1617
LoadLineProbe)
1718
from crossbench.flags.base import Flags
@@ -20,10 +21,13 @@
2021
from crossbench.probes.probe import ProbeContext
2122

2223
if TYPE_CHECKING:
24+
import argparse
25+
from crossbench.benchmarks.loading.page.base import Page
2326
from crossbench.browsers.attributes import BrowserAttributes
2427
from crossbench.probes.results import ProbeResult
2528
from crossbench.runner.groups.browsers import BrowsersRunGroup
2629

30+
2731
# We should increase the minor version number every time there are any changes
2832
# that might affect the benchmark score.
2933
VERSION_STRING = "experimental"
@@ -83,6 +87,12 @@ class LoadLine2Benchmark(LoadLineBenchmark):
8387
def _base_dir(cls) -> pth.LocalPath:
8488
return config.config_dir() / "benchmark" / "loadline2"
8589

90+
@classmethod
91+
@override
92+
def stories_from_cli_args(cls, args: argparse.Namespace) -> Sequence[Page]:
93+
pages = super().stories_from_cli_args(args)
94+
return (CombinedPage(pages),)
95+
8696

8797
class LoadLine2PhoneBenchmark(LoadLine2Benchmark):
8898
"""LoadLine 2 benchmark for phones.

0 commit comments

Comments
 (0)