Skip to content

Commit 2ac682f

Browse files
meteorcloudymeteorcloudy
authored
A few improvements to bcr_presubmit.py (#2077)
Stacked on #2076 - Removed `MODULE_NAME` and `MODULE_VERSION` support - Renamed `runner` to `anonymous_module_runner` to better match [documentation](https://github.com/bazelbuild/bazel-central-registry/blob/main/docs/README.md#anonymous-module-test) - Support overriding Bazel version in BCR task configs. - Support specifying concurrency for jobs (default to no concurrency limit for BCR presubmit) .
1 parent 36f8065 commit 2ac682f

File tree

1 file changed

+45
-50
lines changed

1 file changed

+45
-50
lines changed

‎buildkite/bazel-central-registry/bcr_presubmit.py

Lines changed: 45 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -40,11 +40,9 @@
4040

4141
BUILDKITE_ORG = os.environ["BUILDKITE_ORGANIZATION_SLUG"]
4242

43-
SCRIPT_URL = {
44-
"bazel-testing": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/testing/buildkite/bazel-central-registry/bcr_presubmit.py",
45-
"bazel-trusted": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/bazel-central-registry/bcr_presubmit.py",
46-
"bazel": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/bazel-central-registry/bcr_presubmit.py",
47-
}[BUILDKITE_ORG] + "?{}".format(int(time.time()))
43+
SCRIPT_URL = "https://raw.githubusercontent.com/bazelbuild/continuous-integration/{}/buildkite/bazel-central-registry/bcr_presubmit.py?{}".format(
44+
bazelci.GITHUB_BRANCH, int(time.time())
45+
)
4846

4947

5048
def fetch_bcr_presubmit_py_command():
@@ -62,32 +60,20 @@ def error(msg):
6260

6361
def get_target_modules():
6462
"""
65-
If the `MODULE_NAME` and `MODULE_VERSION(S)` are specified, calculate the target modules from those env vars.
66-
Otherwise, calculate target modules based on changed files from the main branch.
63+
Calculate target modules based on changed files from the main branch.
6764
"""
68-
modules = []
69-
if "MODULE_NAME" in os.environ:
70-
name = os.environ["MODULE_NAME"]
71-
if "MODULE_VERSION" in os.environ:
72-
modules.append((name, os.environ["MODULE_VERSION"]))
73-
elif "MODULE_VERSIONS" in os.environ:
74-
for version in os.environ["MODULE_VERSIONS"].split(","):
75-
modules.append((name, version))
76-
77-
if modules:
78-
return list(set(modules))
79-
8065
# Get the list of changed files compared to the main branch
8166
output = subprocess.check_output(
8267
["git", "diff", "main...HEAD", "--name-only", "--pretty=format:"]
8368
)
69+
modules = set()
8470
# Matching modules/<name>/<version>/
8571
for line in output.decode("utf-8").split():
8672
s = re.match(r"modules\/([^\/]+)\/([^\/]+)\/", line)
8773
if s:
88-
modules.append(s.groups())
74+
modules.add(s.groups())
8975

90-
return list(set(modules))
76+
return sorted(modules)
9177

9278

9379
def get_metadata_json(module_name):
@@ -112,44 +98,52 @@ def get_patch_file(module_name, module_version, patch):
11298
def get_overlay_file(module_name, module_version, filename):
11399
return BCR_REPO_DIR.joinpath("modules/%s/%s/overlay/%s" % (module_name, module_version, filename))
114100

115-
def get_task_config(module_name, module_version):
101+
def get_anonymous_module_task_config(module_name, module_version, bazel_version=None):
116102
return bazelci.load_config(http_url=None,
117103
file_config=get_presubmit_yml(module_name, module_version),
118-
allow_imports=False)
119-
104+
allow_imports=False,
105+
bazel_version=bazel_version)
120106

121-
def get_test_module_task_config(module_name, module_version):
107+
def get_test_module_task_config(module_name, module_version, bazel_version=None):
122108
orig_presubmit = yaml.safe_load(open(get_presubmit_yml(module_name, module_version), "r"))
123109
if "bcr_test_module" in orig_presubmit:
124110
config = orig_presubmit["bcr_test_module"]
111+
bazelci.maybe_overwrite_bazel_version(bazel_version, config)
125112
bazelci.expand_task_config(config)
126113
return config
127114
return {}
128115

129116

130-
def add_presubmit_jobs(module_name, module_version, task_configs, pipeline_steps, is_test_module=False):
117+
def add_presubmit_jobs(module_name, module_version, task_configs, pipeline_steps, is_test_module=False, overwrite_bazel_version=None, calc_concurrency=None):
131118
for task_name, task_config in task_configs.items():
132119
platform_name = bazelci.get_platform_for_task(task_name, task_config)
133-
label = bazelci.PLATFORMS[platform_name]["emoji-name"] + " {0}@{1} {2}".format(
134-
module_name, module_version, task_config["name"] if "name" in task_config else ""
135-
)
120+
platform_label = bazelci.PLATFORMS[platform_name]["emoji-name"]
121+
task_name = task_config.get("name", "")
122+
label = f"{module_name}@{module_version} - {platform_label} - {task_name}"
136123
# The bazel version should always be set in the task config due to https://github.com/bazelbuild/bazel-central-registry/pull/1387
137124
# But fall back to empty string for more robustness.
138125
bazel_version = task_config.get("bazel", "")
139-
if bazel_version:
140-
label = ":bazel:{} - ".format(bazel_version) + label
126+
if bazel_version and not overwrite_bazel_version:
127+
label = f":bazel:{bazel_version} - {label}"
141128
command = (
142-
'%s bcr_presubmit.py %s --module_name="%s" --module_version="%s" --task=%s'
129+
'%s bcr_presubmit.py %s --module_name="%s" --module_version="%s" --task=%s %s'
143130
% (
144131
bazelci.PLATFORMS[platform_name]["python"],
145-
"test_module_runner" if is_test_module else "runner",
132+
"test_module_runner" if is_test_module else "anonymous_module_runner",
146133
module_name,
147134
module_version,
148135
task_name,
136+
"--overwrite_bazel_version=%s" % overwrite_bazel_version if overwrite_bazel_version else ""
149137
)
150138
)
151139
commands = [bazelci.fetch_bazelcipy_command(), fetch_bcr_presubmit_py_command(), command]
152-
pipeline_steps.append(bazelci.create_step(label, commands, platform_name))
140+
if calc_concurrency is None:
141+
concurrency = concurrency_group = None
142+
else:
143+
queue = bazelci.PLATFORMS[platform_name].get("queue", "default")
144+
concurrency = calc_concurrency(queue)
145+
concurrency_group = f"bcr-presubmit-test-queue-{queue}"
146+
pipeline_steps.append(bazelci.create_step(label, commands, platform_name, concurrency=concurrency, concurrency_group=concurrency_group))
153147

154148

155149
def scratch_file(root, relative_path, lines=None, mode="w"):
@@ -165,14 +159,11 @@ def scratch_file(root, relative_path, lines=None, mode="w"):
165159
return abspath
166160

167161

168-
def create_simple_repo(module_name, module_version):
169-
"""Create a simple Bazel module repo which depends on the target module."""
162+
def create_anonymous_repo(module_name, module_version):
163+
"""Create an anonymous Bazel module which depends on the target module."""
170164
root = pathlib.Path(bazelci.get_repositories_root())
171165
scratch_file(root, "WORKSPACE")
172166
scratch_file(root, "BUILD")
173-
# TODO(pcloudy): Should we test this module as the root module? Maybe we do if we support dev dependency.
174-
# Because if the module is not root module, dev dependencies are ignored, which can break test targets.
175-
# Another work around is that we can copy the dev dependencies to the generated MODULE.bazel.
176167
scratch_file(root, "MODULE.bazel", ["bazel_dep(name = '%s', version = '%s')" % (module_name, module_version)])
177168
scratch_file(root, ".bazelrc", [
178169
"build --experimental_enable_bzlmod",
@@ -289,15 +280,15 @@ def prepare_test_module_repo(module_name, module_version):
289280
return test_module_root, test_module_presubmit
290281

291282

292-
def run_test(repo_location, task_config_file, task):
283+
def run_test(repo_location, task_config_file, task, overwrite_bazel_version=None):
293284
try:
294285
return bazelci.main(
295286
[
296287
"runner",
297288
"--task=" + task,
298289
"--file_config=%s" % task_config_file,
299290
"--repo_location=%s" % repo_location,
300-
]
291+
] + (["--overwrite_bazel_version=%s" % overwrite_bazel_version] if overwrite_bazel_version else [])
301292
)
302293
except subprocess.CalledProcessError as e:
303294
bazelci.eprint(str(e))
@@ -476,26 +467,30 @@ def main(argv=None):
476467

477468
subparsers.add_parser("bcr_presubmit")
478469

479-
runner = subparsers.add_parser("runner")
480-
runner.add_argument("--module_name", type=str)
481-
runner.add_argument("--module_version", type=str)
482-
runner.add_argument("--task", type=str)
470+
anonymous_module_runner = subparsers.add_parser("anonymous_module_runner")
471+
anonymous_module_runner.add_argument("--module_name", type=str)
472+
anonymous_module_runner.add_argument("--module_version", type=str)
473+
anonymous_module_runner.add_argument("--overwrite_bazel_version", type=str)
474+
anonymous_module_runner.add_argument("--task", type=str)
483475

484476
test_module_runner = subparsers.add_parser("test_module_runner")
485477
test_module_runner.add_argument("--module_name", type=str)
486478
test_module_runner.add_argument("--module_version", type=str)
479+
test_module_runner.add_argument("--overwrite_bazel_version", type=str)
487480
test_module_runner.add_argument("--task", type=str)
488481

489482
args = parser.parse_args(argv)
483+
490484
if args.subparsers_name == "bcr_presubmit":
491485
modules = get_target_modules()
492486
if not modules:
493487
bazelci.eprint("No target module versions detected in this branch!")
488+
494489
pipeline_steps = []
495490
for module_name, module_version in modules:
496491
previous_size = len(pipeline_steps)
497492

498-
configs = get_task_config(module_name, module_version)
493+
configs = get_anonymous_module_task_config(module_name, module_version)
499494
add_presubmit_jobs(module_name, module_version, configs.get("tasks", {}), pipeline_steps)
500495
configs = get_test_module_task_config(module_name, module_version)
501496
add_presubmit_jobs(module_name, module_version, configs.get("tasks", {}), pipeline_steps, is_test_module=True)
@@ -507,13 +502,13 @@ def main(argv=None):
507502
pipeline_steps = [{"block": "Wait on BCR maintainer review", "blocked_state": "running"}] + pipeline_steps
508503

509504
upload_jobs_to_pipeline(pipeline_steps)
510-
elif args.subparsers_name == "runner":
511-
repo_location = create_simple_repo(args.module_name, args.module_version)
505+
elif args.subparsers_name == "anonymous_module_runner":
506+
repo_location = create_anonymous_repo(args.module_name, args.module_version)
512507
config_file = get_presubmit_yml(args.module_name, args.module_version)
513-
return run_test(repo_location, config_file, args.task)
508+
return run_test(repo_location, config_file, args.task, args.overwrite_bazel_version)
514509
elif args.subparsers_name == "test_module_runner":
515510
repo_location, config_file = prepare_test_module_repo(args.module_name, args.module_version)
516-
return run_test(repo_location, config_file, args.task)
511+
return run_test(repo_location, config_file, args.task, args.overwrite_bazel_version)
517512
else:
518513
parser.print_help()
519514
return 2

0 commit comments

Comments
 (0)