Skip to content

Commit a33441b

Browse files
philnik777philnik777
committed
Revert "[CI] Cleanup buildkite test report script"
This reverts commit 99cdc26.
1 parent 7dfcced commit a33441b

File tree

5 files changed

+204
-6
lines changed

5 files changed

+204
-6
lines changed

‎.ci/generate-buildkite-pipeline-premerge‎

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
#!/usr/bin/env bash
2+
#===----------------------------------------------------------------------===##
3+
#
4+
# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
5+
# See https://llvm.org/LICENSE.txt for license information.
6+
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7+
#
8+
#===----------------------------------------------------------------------===##
9+
10+
#
11+
# This file generates a Buildkite pipeline that triggers the various CI jobs for
12+
# the LLVM project during pre-commit CI.
13+
#
14+
# See https://buildkite.com/docs/agent/v3/cli-pipeline#pipeline-format.
15+
#
16+
# As this outputs a yaml file, it's possible to log messages to stderr or
17+
# prefix with "#".
18+
19+
20+
set -eu
21+
set -o pipefail
22+
23+
# Environment variables script works with:
24+
25+
# Set by buildkite
26+
: ${BUILDKITE_PULL_REQUEST_BASE_BRANCH:=}
27+
: ${BUILDKITE_COMMIT:=}
28+
: ${BUILDKITE_BRANCH:=}
29+
# Fetch origin to have an up to date merge base for the diff.
30+
git fetch origin
31+
# List of files affected by this commit
32+
: ${MODIFIED_FILES:=$(git diff --name-only origin/${BUILDKITE_PULL_REQUEST_BASE_BRANCH}...HEAD)}
33+
# Filter rules for generic windows tests
34+
: ${WINDOWS_AGENTS:='{"queue": "windows"}'}
35+
# Filter rules for generic linux tests
36+
: ${LINUX_AGENTS:='{"queue": "linux"}'}
37+
38+
reviewID="$(git log --format=%B -n 1 | sed -nE 's/^Review-ID:[[:space:]]*(.+)$/\1/p')"
39+
if [[ "${reviewID}" != "" ]]; then
40+
buildMessage="https://llvm.org/${reviewID}"
41+
else
42+
buildMessage="Push to branch ${BUILDKITE_BRANCH}"
43+
fi
44+
45+
cat <<EOF
46+
steps:
47+
EOF
48+
49+
echo "Files modified:" >&2
50+
echo "$MODIFIED_FILES" >&2
51+
modified_dirs=$(echo "$MODIFIED_FILES" | cut -d'/' -f1 | sort -u)
52+
echo "Directories modified:" >&2
53+
echo "$modified_dirs" >&2
54+
55+
# Project specific pipelines.
56+
57+
# If libc++ or one of the runtimes directories changed.
58+
if echo "$modified_dirs" | grep -q -E "^(libcxx|libcxxabi|libunwind|runtimes|cmake)$"; then
59+
cat <<EOF
60+
- trigger: "libcxx-ci"
61+
build:
62+
message: "${buildMessage}"
63+
commit: "${BUILDKITE_COMMIT}"
64+
branch: "${BUILDKITE_BRANCH}"
65+
EOF
66+
fi
67+
68+
# Generic pipeline for projects that have not defined custom steps.
69+
#
70+
# Individual projects should instead define the pre-commit CI tests that suits their
71+
# needs while letting them run on the infrastructure provided by LLVM.
72+
73+
# Figure out which projects need to be built on each platform
74+
source <(git diff --name-only origin/${BUILDKITE_PULL_REQUEST_BASE_BRANCH}...HEAD | python3 .ci/compute_projects.py Linux)
75+
linux_projects=${projects_to_build}
76+
linux_check_targets=${project_check_targets}
77+
linux_runtimes=${runtimes_to_build}
78+
linux_runtime_check_targets=${runtimes_check_targets}
79+
80+
source <(git diff --name-only origin/${BUILDKITE_PULL_REQUEST_BASE_BRANCH}...HEAD | python3 .ci/compute_projects.py Windows)
81+
windows_projects=${projects_to_build}
82+
windows_check_targets=${project_check_targets}
83+
84+
85+
# Generate the appropriate pipeline
86+
if [[ "${linux_projects}" != "" ]]; then
87+
cat <<EOF
88+
- label: ':linux: Linux x64'
89+
artifact_paths:
90+
- 'artifacts/**/*'
91+
- '*_result.json'
92+
- 'build/test-results.*.xml'
93+
agents: ${LINUX_AGENTS}
94+
retry:
95+
automatic:
96+
- exit_status: -1 # Agent was lost
97+
limit: 2
98+
- exit_status: 255 # Forced agent shutdown
99+
limit: 2
100+
timeout_in_minutes: 120
101+
env:
102+
CC: 'clang'
103+
CXX: 'clang++'
104+
commands:
105+
- './.ci/monolithic-linux.sh "$(echo ${linux_projects} | tr ' ' ';')" "$(echo ${linux_check_targets})" "$(echo ${linux_runtimes} | tr ' ' ';')" "$(echo ${linux_runtime_check_targets})"'
106+
EOF
107+
fi
108+
109+
if [[ "${windows_projects}" != "" ]]; then
110+
cat <<EOF
111+
- label: ':windows: Windows x64'
112+
artifact_paths:
113+
- 'artifacts/**/*'
114+
- '*_result.json'
115+
- 'build/test-results.*.xml'
116+
agents: ${WINDOWS_AGENTS}
117+
retry:
118+
automatic:
119+
- exit_status: -1 # Agent was lost
120+
limit: 2
121+
- exit_status: 255 # Forced agent shutdown
122+
limit: 2
123+
timeout_in_minutes: 150
124+
env:
125+
MAX_PARALLEL_COMPILE_JOBS: '16'
126+
MAX_PARALLEL_LINK_JOBS: '4'
127+
commands:
128+
- 'C:\\BuildTools\\Common7\\Tools\\VsDevCmd.bat -arch=amd64 -host_arch=amd64'
129+
- 'bash .ci/monolithic-windows.sh "$(echo ${windows_projects} | tr ' ' ';')" "$(echo ${windows_check_targets})"'
130+
EOF
131+
fi

‎.ci/generate_test_report_buildkite.py‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
2+
# See https://llvm.org/LICENSE.txt for license information.
3+
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
4+
"""Script to generate a build report for buildkite."""
5+
6+
import argparse
7+
import os
8+
import subprocess
9+
10+
import generate_test_report_lib
11+
12+
13+
if __name__ == "__main__":
14+
parser = argparse.ArgumentParser()
15+
parser.add_argument(
16+
"title", help="Title of the test report, without Markdown formatting."
17+
)
18+
parser.add_argument("context", help="Annotation context to write to.")
19+
parser.add_argument("return_code", help="The build's return code.", type=int)
20+
parser.add_argument("junit_files", help="Paths to JUnit report files.", nargs="*")
21+
args = parser.parse_args()
22+
23+
# All of these are required to build a link to download the log file.
24+
env_var_names = [
25+
"BUILDKITE_ORGANIZATION_SLUG",
26+
"BUILDKITE_PIPELINE_SLUG",
27+
"BUILDKITE_BUILD_NUMBER",
28+
"BUILDKITE_JOB_ID",
29+
]
30+
buildkite_info = {k: v for k, v in os.environ.items() if k in env_var_names}
31+
if len(buildkite_info) != len(env_var_names):
32+
buildkite_info = None
33+
34+
report, style = generate_test_report_lib.generate_report_from_files(
35+
args.title, args.return_code, args.junit_files, buildkite_info
36+
)
37+
38+
if report:
39+
p = subprocess.Popen(
40+
[
41+
"buildkite-agent",
42+
"annotate",
43+
"--context",
44+
args.context,
45+
"--style",
46+
style,
47+
],
48+
stdin=subprocess.PIPE,
49+
stderr=subprocess.PIPE,
50+
universal_newlines=True,
51+
)
52+
53+
# The report can be larger than the buffer for command arguments so we send
54+
# it over stdin instead.
55+
_, err = p.communicate(input=report)
56+
if p.returncode:
57+
raise RuntimeError(f"Failed to send report to buildkite-agent:\n{err}")

‎.ci/monolithic-linux.sh‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -42,9 +42,14 @@ function at-exit {
4242

4343
# If building fails there will be no results files.
4444
shopt -s nullglob
45-
46-
python3 "${MONOREPO_ROOT}"/.ci/generate_test_report_github.py ":penguin: Linux x64 Test Results" \
47-
$retcode "${BUILD_DIR}"/test-results.*.xml >> $GITHUB_STEP_SUMMARY
45+
if command -v buildkite-agent 2>&1 >/dev/null
46+
then
47+
python3 "${MONOREPO_ROOT}"/.ci/generate_test_report_buildkite.py ":linux: Linux x64 Test Results" \
48+
"linux-x64-test-results" $retcode "${BUILD_DIR}"/test-results.*.xml
49+
else
50+
python3 "${MONOREPO_ROOT}"/.ci/generate_test_report_github.py ":penguin: Linux x64 Test Results" \
51+
$retcode "${BUILD_DIR}"/test-results.*.xml >> $GITHUB_STEP_SUMMARY
52+
fi
4853
}
4954
trap at-exit EXIT
5055

‎.ci/monolithic-windows.sh‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -37,9 +37,14 @@ function at-exit {
3737

3838
# If building fails there will be no results files.
3939
shopt -s nullglob
40-
41-
python "${MONOREPO_ROOT}"/.ci/generate_test_report_github.py ":window: Windows x64 Test Results" \
42-
$retcode "${BUILD_DIR}"/test-results.*.xml >> $GITHUB_STEP_SUMMARY
40+
if command -v buildkite-agent 2>&1 >/dev/null
41+
then
42+
python "${MONOREPO_ROOT}"/.ci/generate_test_report_buildkite.py ":windows: Windows x64 Test Results" \
43+
"windows-x64-test-results" $retcode "${BUILD_DIR}"/test-results.*.xml
44+
else
45+
python "${MONOREPO_ROOT}"/.ci/generate_test_report_github.py ":window: Windows x64 Test Results" \
46+
$retcode "${BUILD_DIR}"/test-results.*.xml >> $GITHUB_STEP_SUMMARY
47+
fi
4348
}
4449
trap at-exit EXIT
4550

‎libcxx/trigger-ci‎

Whitespace-only changes.

0 commit comments

Comments
 (0)