Skip to content

Commit 8702689

Browse files
licy183licy183
committed
ci: reuse pr artifacts
1 parent c32e52b commit 8702689

File tree

2 files changed

+124
-23
lines changed

2 files changed

+124
-23
lines changed

‎.github/workflows/packages.yml‎

Lines changed: 55 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -59,30 +59,49 @@ jobs:
5959
run: ./setup-environment.sh
6060

6161
- name: Gather build summary
62+
env:
63+
MANUAL_INPUT_PACKAGES: ${{ github.event.inputs.packages }}
6264
run: |
65+
# We are intentionally not using .commits[0].id and .commits[-1].id as github seems to
66+
# only send 20 commits in the payload for github action runs instead of the 2048 documented
67+
# limit. Perhaps 2048 is the limit just for webhooks, where they haven't documented
68+
# properly that the limit for github actions is only 20:
69+
#
70+
# https://docs.github.com/en/webhooks/webhook-events-and-payloads#push
71+
OLD_COMMIT="${{ github.event.before }}"
72+
HEAD_COMMIT="${{ github.event.after }}"
6373
if [ "${{ github.event_name }}" != "workflow_dispatch" ]; then
64-
BASE_COMMIT=$(jq --raw-output .pull_request.base.sha "$GITHUB_EVENT_PATH")
65-
OLD_COMMIT=$(jq --raw-output .commits[0].id "$GITHUB_EVENT_PATH")
66-
HEAD_COMMIT=$(jq --raw-output .commits[-1].id "$GITHUB_EVENT_PATH")
67-
if [ "$BASE_COMMIT" = "null" ]; then
68-
if [ "$OLD_COMMIT" = "$HEAD_COMMIT" ]; then
69-
# Single-commit push.
70-
echo "Processing commit: ${HEAD_COMMIT}"
71-
CHANGED_FILES=$(git diff-tree --no-commit-id --name-only -r "${HEAD_COMMIT}")
72-
else
73-
# Multi-commit push.
74-
OLD_COMMIT="${OLD_COMMIT}~1"
75-
echo "Processing commit range: ${OLD_COMMIT}..${HEAD_COMMIT}"
76-
CHANGED_FILES=$(git diff-tree --no-commit-id --name-only -r "${OLD_COMMIT}" "${HEAD_COMMIT}")
74+
BASE_REF="${{ github.event.pull_request.base.ref }}"
75+
git fetch origin "${BASE_REF:-master}" 2>/dev/null
76+
BASE_COMMIT="$(git merge-base "origin/${BASE_REF:-master}" "HEAD")"
77+
if [ -z "${{ github.event.pull_request.base.sha }}" ]; then
78+
if ! git log "$OLD_COMMIT" > /dev/null; then
79+
if [ "$(git branch --show-current)" = "master" ]; then
80+
echo "Force push detected on master branch. Unable to proceed."
81+
exit 1
82+
else
83+
OLD_COMMIT=$(git fetch origin master >&2; git merge-base origin/master $HEAD_COMMIT)
84+
fi
7785
fi
86+
echo "Processing commit range: ${OLD_COMMIT}..${HEAD_COMMIT}"
87+
CHANGED_FILES=$(git diff-tree --no-commit-id --name-only -r "${OLD_COMMIT}" "${HEAD_COMMIT}")
7888
else
7989
# Pull requests.
80-
echo "Processing pull request #$(jq --raw-output .pull_request.number "$GITHUB_EVENT_PATH"): ${BASE_COMMIT}..HEAD"
90+
echo "Processing pull request #${{ github.event.pull_request.number }}: ${BASE_COMMIT}..HEAD"
8191
CHANGED_FILES=$(git diff-tree --no-commit-id --name-only -r "${BASE_COMMIT}" "HEAD")
8292
fi
8393
fi
8494
mkdir -p ./artifacts ./debs
8595
touch ./debs/.placeholder
96+
97+
if [[ "${{ github.event_name }}" == "pull_request" && -n "$(git rev-list --merges "$(git fetch origin master >&2; git merge-base origin/master $HEAD_COMMIT)..$HEAD_COMMIT")" ]]; then
98+
# Github does not allow multiline errors, but it will interpret the escape sequence %0A as a line break.
99+
echo "::error ::Merge commits are not allowed in pull requests.%0AYou should rebase your commits or squash them.%0Ahttps://docs.github.com/en/get-started/using-git/using-git-rebase-on-the-command-line"
100+
exit 1
101+
fi
102+
103+
GITHUB_OUTPUT="$GITHUB_OUTPUT" OLD_COMMIT="$OLD_COMMIT" HEAD_COMMIT="$HEAD_COMMIT" GITHUB_TOKEN="${{ secrets.GH_TOKEN }}" scripts/utils/termux_reuse_pr_build_artifacts.sh "${{ github.event_name }}" "${{ matrix.target_arch }}" || true
104+
86105
if [ "${{ github.event_name }}" != "workflow_dispatch" ]; then
87106
# Process tag '%ci:no-build' that may be added as line to commit message.
88107
# Forces CI to cancel current build with status 'passed'
@@ -134,7 +153,10 @@ jobs:
134153
done<<<${CHANGED_FILES}
135154
done
136155
else
137-
for pkg in ${{ github.event.inputs.packages }}; do
156+
# Ensure MANUAL_INPUT_PACKAGES is newline free, and put it
157+
# into an array
158+
read -a PACKAGES <<< "${MANUAL_INPUT_PACKAGES//$'\n'/ }"
159+
for pkg in "${PACKAGES[@]}"; do
138160
repo_paths=$(jq --raw-output 'keys | .[]' repo.json)
139161
found=false
140162
for repo_path in $repo_paths; do
@@ -168,14 +190,6 @@ jobs:
168190
mv ./deleted_${repo}_packages.txt.tmp ./deleted_${repo}_packages.txt
169191
fi
170192
done
171-
- name: Free additional disk space (if necessary)
172-
run: |
173-
if grep -Eq "^($(paste -s -d '|' ./big-pkgs.list))$" ./built_tur_packages.txt; then
174-
sudo apt purge -yq $(dpkg -l | grep '^ii' | awk '{ print $2 }' | grep -P '(aspnetcore|cabal-|dotnet-|ghc-|libmono|mongodb-|mysql-|php)') \
175-
firefox google-chrome-stable microsoft-edge-stable mono-devel mono-runtime-common monodoc-manual ruby
176-
sudo apt autoremove -yq
177-
sudo rm -rf /opt/hostedtoolcache /usr/local /usr/share/dotnet /usr/share/swift
178-
fi
179193
- name: Lint packages
180194
run: |
181195
declare -a package_recipes
@@ -188,8 +202,26 @@ jobs:
188202
if [ ! -z "$package_recipes" ]; then
189203
./scripts/lint-packages.sh $package_recipes
190204
fi
205+
- name: Free additional disk space (if necessary)
206+
env:
207+
SKIP_BUILDING: ${{ steps.build-info.outputs.skip-building }}
208+
run: |
209+
if [ $SKIP_BUILDING == "true" ]; then
210+
exit 0
211+
fi
212+
if grep -Eq "^($(paste -s -d '|' ./big-pkgs.list))$" ./built_tur_packages.txt; then
213+
sudo apt purge -yq $(dpkg -l | grep '^ii' | awk '{ print $2 }' | grep -P '(aspnetcore|cabal-|dotnet-|ghc-|libmono|mongodb-|mysql-|php)') \
214+
firefox google-chrome-stable microsoft-edge-stable mono-devel mono-runtime-common monodoc-manual ruby
215+
sudo apt autoremove -yq
216+
sudo rm -rf /opt/hostedtoolcache /usr/local /usr/share/dotnet /usr/share/swift
217+
fi
191218
- name: Build packages
219+
env:
220+
SKIP_BUILDING: ${{ steps.build-info.outputs.skip-building }}
192221
run: |
222+
if [ $SKIP_BUILDING == "true" ]; then
223+
exit 0
224+
fi
193225
declare -a packages
194226
for repo_path in $(jq --raw-output 'keys | .[]' repo.json); do
195227
repo=$(jq --raw-output '.["'${repo_path}'"].name' repo.json)

‎common-files/building-system-patches/0008-allow-reuse-pr.patch‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
--- a/scripts/utils/termux_reuse_pr_build_artifacts.sh
2+
+++ b/scripts/utils/termux_reuse_pr_build_artifacts.sh
3+
@@ -38,7 +38,7 @@
4+
curl --silent \
5+
-H "Authorization: token ${GITHUB_TOKEN}" \
6+
-H "Accept: application/vnd.github.v3+json" \
7+
- "https://api.github.com/repos/termux/termux-packages/actions/runs/${1}/artifacts" \
8+
+ "https://api.github.com/repos/termux-user-repository/tur/actions/runs/${1}/artifacts" \
9+
| jq -r '[.artifacts[]? | select(.name | startswith("debs-'"${TARGET_ARCH}"'")) | .archive_download_url][0] // error' \
10+
|| return $?
11+
}
12+
@@ -92,7 +92,7 @@
13+
# We should obtain data about all commits in this push to check that they are from the same PR if any
14+
RELATED_PRS_QUERY="
15+
query {
16+
- repository(owner: \"termux\", name: \"termux-packages\") {
17+
+ repository(owner: \"termux-user-repository\", name: \"tur\") {
18+
$(
19+
for commit in "${COMMITS[@]}"; do
20+
# Add a query for this commit with the shorthash as the label
21+
@@ -124,7 +124,7 @@
22+
.edges[0].node.body)' <<< "$RESPONSE" || :)
23+
[[ -n "${PR_HEAD_COMMIT:-}" ]] || infoexit "failed to read associated PR head commit, not performing CI fast path"
24+
25+
- echo "::group::Detected PR #${PRS[0]}: ${PR_COMMIT_TITLE} — https://github.com/termux/termux-packages/pull/${PRS[0]}"
26+
+ echo "::group::Detected PR #${PRS[0]}: ${PR_COMMIT_TITLE} — https://github.com/termux-user-repository/tur/pull/${PRS[0]}"
27+
echo "${PR_COMMIT_BODY}"
28+
echo "::endgroup::"
29+
30+
@@ -206,7 +206,7 @@
31+
# The same commit can be used in more than one PR or even push
32+
WORKFLOW_PR_QUERY="
33+
query {
34+
- repository(owner: \"termux\", name: \"termux-packages\") {
35+
+ repository(owner: \"termux-user-repository\", name: \"tur\") {
36+
object(oid: \"$PR_HEAD_COMMIT\") { ...workflowRun }
37+
}
38+
}
39+
@@ -228,10 +228,10 @@
40+
| .workflowRun.databaseId][0] // empty' <<< "$RESPONSE" || :
41+
)"
42+
if [[ -n "${WORKFLOW_ID}" ]]; then
43+
- echo "We can safely reuse CI artifacts from https://github.com/termux/termux-packages/actions/runs/${WORKFLOW_ID}"
44+
+ echo "We can safely reuse CI artifacts from https://github.com/termux-user-repository/tur/actions/runs/${WORKFLOW_ID}"
45+
if download_ci_artifacts "${WORKFLOW_ID}"; then
46+
# Notify CI about skipping packages building because we reuse PR artifact.
47+
- echo "skip-building=true" >> "${GITHUB_OUTPUT:-/dev/null}"
48+
+ echo "skip-building=true" >> "${GITHUB_OUTPUT}"
49+
echo "::notice::Reusing PR #${PRS[0]} (workflow run ${WORKFLOW_ID}) build artifacts, building is skipped."
50+
else
51+
infoexit "Failed to download and unpack PR artifacts"
52+
@@ -253,7 +253,7 @@
53+
# to allow CI reuse artifacts of other PRs in the case if current PR diverged from another PR branch.
54+
WORKFLOW_COMMITS_QUERY="
55+
query {
56+
- repository(owner: \"termux\", name: \"termux-packages\") {
57+
+ repository(owner: \"termux-user-repository\", name: \"tur\") {
58+
$(
59+
for commit in "${COMMITS[@]}"; do
60+
# Add a query for this commit with the shorthash as the label
61+
@@ -285,7 +285,7 @@
62+
[[ -z "${WORKFLOW_ID:-}" ]] || break
63+
done
64+
if [[ -n "${WORKFLOW_ID}" ]]; then
65+
- echo "We can safely reuse CI artifacts from https://github.com/termux/termux-packages/actions/runs/${WORKFLOW_ID}"
66+
+ echo "We can safely reuse CI artifacts from https://github.com/termux-user-repository/tur/actions/runs/${WORKFLOW_ID}"
67+
echo "CI artifact URL is $(ci_artifact_url "${WORKFLOW_ID}" || infoexit "Failed to get CI artifact URL")"
68+
else
69+
echo "We can not reuse CI artifacts since no relevant CI runs were found"

0 commit comments

Comments
 (0)