|
| 1 | +#!/usr/bin/env python3 |
| 2 | +# |
| 3 | +# Copyright 2024 The Bazel Authors. All rights reserved. |
| 4 | +# |
| 5 | +# Licensed under the Apache License, Version 2.0 (the "License"); |
| 6 | +# you may not use this file except in compliance with the License. |
| 7 | +# You may obtain a copy of the License at |
| 8 | +# |
| 9 | +# http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | +# |
| 11 | +# Unless required by applicable law or agreed to in writing, software |
| 12 | +# distributed under the License is distributed on an "AS IS" BASIS, |
| 13 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 14 | +# See the License for the specific language governing permissions and |
| 15 | +# limitations under the License. |
| 16 | +# pylint: disable=line-too-long |
| 17 | +# pylint: disable=missing-function-docstring |
| 18 | +# pylint: disable=unspecified-encoding |
| 19 | +# pylint: disable=invalid-name |
| 20 | +"""The CI script for BCR Bazel Compatibility Test pipeline.""" |
| 21 | + |
| 22 | + |
| 23 | +import os |
| 24 | +import sys |
| 25 | +import subprocess |
| 26 | + |
| 27 | +import bazelci |
| 28 | +import bcr_presubmit |
| 29 | + |
| 30 | +CI_MACHINE_NUM = { |
| 31 | + "bazel": { |
| 32 | + "default": 100, |
| 33 | + "windows": 30, |
| 34 | + "macos_arm64": 45, |
| 35 | + "macos": 110, |
| 36 | + "arm64": 1, |
| 37 | + }, |
| 38 | + "bazel-testing": { |
| 39 | + "default": 30, |
| 40 | + "windows": 4, |
| 41 | + "macos_arm64": 1, |
| 42 | + "macos": 10, |
| 43 | + "arm64": 1, |
| 44 | + }, |
| 45 | +}[bazelci.BUILDKITE_ORG] |
| 46 | + |
| 47 | +# Default to use only 30% of CI resources for each type of machines. |
| 48 | +CI_RESOURCE_PERCENTAGE = int(os.environ.get('CI_RESOURCE_PERCENTAGE', 30)) |
| 49 | + |
| 50 | + |
| 51 | +def select_modules_from_env_vars(): |
| 52 | + """ |
| 53 | + Parses MODULE_SELECTIONS and SMOKE_TEST_PERCENTAGE environment variables |
| 54 | + and returns a list of selected module versions. |
| 55 | + """ |
| 56 | + MODULE_SELECTIONS = os.environ.get('MODULE_SELECTIONS', '') |
| 57 | + SMOKE_TEST_PERCENTAGE = os.environ.get('SMOKE_TEST_PERCENTAGE', None) |
| 58 | + |
| 59 | + if not MODULE_SELECTIONS: |
| 60 | + return [] |
| 61 | + |
| 62 | + selections = [s.strip() for s in MODULE_SELECTIONS.split(',') if s.strip()] |
| 63 | + args = [f"--select={s}" for s in selections] |
| 64 | + if SMOKE_TEST_PERCENTAGE: |
| 65 | + args += [f"--random-percentage={SMOKE_TEST_PERCENTAGE}"] |
| 66 | + output = subprocess.check_output( |
| 67 | + ["python3", "./tools/module_selector.py"] + args, |
| 68 | + ) |
| 69 | + modules = [] |
| 70 | + for line in output.decode("utf-8").split(): |
| 71 | + name, version = line.strip().split("@") |
| 72 | + modules.append((name, version)) |
| 73 | + return modules |
| 74 | + |
| 75 | + |
| 76 | +def get_target_modules(): |
| 77 | + """ |
| 78 | + If the `MODULE_SELECTIONS` and `SMOKE_TEST_PERCENTAGE(S)` are specified, calculate the target modules from those env vars. |
| 79 | + Otherwise, calculate target modules based on changed files from the main branch. |
| 80 | + """ |
| 81 | + if "MODULE_SELECTIONS" not in os.environ: |
| 82 | + raise ValueError("Please set MODULE_SELECTIONS env var to select modules for testing!") |
| 83 | + |
| 84 | + modules = select_modules_from_env_vars() |
| 85 | + if modules: |
| 86 | + bazelci.print_expanded_group("The following modules are selected:\n\n%s" % "\n".join([f"{name}@{version}" for name, version in modules])) |
| 87 | + return sorted(list(set(modules))) |
| 88 | + else: |
| 89 | + raise ValueError("MODULE_SELECTIONS env var didn't select any modules!") |
| 90 | + |
| 91 | + |
| 92 | +def create_step_for_report_flags_results(): |
| 93 | + parts = [ |
| 94 | + bazelci.PLATFORMS[bazelci.DEFAULT_PLATFORM]["python"], |
| 95 | + "aggregate_incompatible_flags_test_result.py", |
| 96 | + "--build_number=%s" % os.getenv("BUILDKITE_BUILD_NUMBER"), |
| 97 | + ] |
| 98 | + return [ |
| 99 | + {"wait": "~", "continue_on_failure": "true"}, |
| 100 | + bazelci.create_step( |
| 101 | + label="Aggregate incompatible flags test result", |
| 102 | + commands=[ |
| 103 | + bazelci.fetch_bazelcipy_command(), |
| 104 | + bazelci.fetch_aggregate_incompatible_flags_test_result_command(), |
| 105 | + " ".join(parts), |
| 106 | + ], |
| 107 | + platform=bazelci.DEFAULT_PLATFORM, |
| 108 | + ), |
| 109 | + ] |
| 110 | + |
| 111 | +def main(): |
| 112 | + modules = get_target_modules() |
| 113 | + pipeline_steps = [] |
| 114 | + # A function to calculate concurrency number for each BuildKite queue |
| 115 | + calc_concurrency = lambda queue : max(1, (CI_RESOURCE_PERCENTAGE * CI_MACHINE_NUM[queue]) // 100) |
| 116 | + # Respect USE_BAZEL_VERSION to override bazel version in presubmit.yml files. |
| 117 | + bazel_version = os.environ.get("USE_BAZEL_VERSION") |
| 118 | + for module_name, module_version in modules: |
| 119 | + previous_size = len(pipeline_steps) |
| 120 | + |
| 121 | + configs = bcr_presubmit.get_anonymous_module_task_config(module_name, module_version, bazel_version) |
| 122 | + bcr_presubmit.add_presubmit_jobs(module_name, module_version, configs.get("tasks", {}), pipeline_steps, overwrite_bazel_version=bazel_version, calc_concurrency=calc_concurrency) |
| 123 | + configs = bcr_presubmit.get_test_module_task_config(module_name, module_version, bazel_version) |
| 124 | + bcr_presubmit.add_presubmit_jobs(module_name, module_version, configs.get("tasks", {}), pipeline_steps, is_test_module=True, overwrite_bazel_version=bazel_version, calc_concurrency=calc_concurrency) |
| 125 | + |
| 126 | + if len(pipeline_steps) == previous_size: |
| 127 | + bcr_presubmit.error("No pipeline steps generated for %s@%s. Please check the configuration." % (module_name, module_version)) |
| 128 | + |
| 129 | + if pipeline_steps: |
| 130 | + # Always wait for approval to proceed |
| 131 | + pipeline_steps = [{"block": "Please review generated jobs before proceeding", "blocked_state": "running"}] + pipeline_steps |
| 132 | + if bazelci.use_bazelisk_migrate(): |
| 133 | + pipeline_steps += create_step_for_report_flags_results() |
| 134 | + |
| 135 | + bcr_presubmit.upload_jobs_to_pipeline(pipeline_steps) |
| 136 | + |
| 137 | + |
| 138 | +if __name__ == "__main__": |
| 139 | + sys.exit(main()) |
0 commit comments