Skip to content

Commit 5fc9f37

Browse files
XargonWanXargonWan
committed
feat(cicd): Enhance build workflow with reference version handling and reuse logic
1 parent a41507d commit 5fc9f37

File tree

7 files changed

+440
-30
lines changed

7 files changed

+440
-30
lines changed

‎.github/workflows/build_components.yml‎

Lines changed: 170 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -23,9 +23,16 @@ jobs:
2323
outputs:
2424
heavy-matrix: ${{ steps.set-matrix.outputs.heavy }}
2525
light-matrix: ${{ steps.set-matrix.outputs.light }}
26+
reference-tag: ${{ steps.reference-release.outputs.reference_tag }}
2627
steps:
2728
- uses: actions/checkout@v4
2829

30+
- name: Init reference versions (for reuse)
31+
run: |
32+
mkdir -p reference
33+
: > reference/reference_tag.txt
34+
: > reference/components_version_list.md
35+
2936
- name: Assignign recipes to different runners
3037
id: set-matrix
3138
run: |
@@ -71,6 +78,39 @@ jobs:
7178
echo "heavy=$heavy_json" >> $GITHUB_OUTPUT
7279
echo "light=$light_json" >> $GITHUB_OUTPUT
7380
81+
- name: Fetch newest cooker reference versions (cooker push only)
82+
id: reference-release
83+
if: github.ref == 'refs/heads/cooker' && github.event_name != 'pull_request' && github.event_name != 'pull_request_target'
84+
env:
85+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
86+
GITHUB_REPOSITORY: ${{ github.repository }}
87+
run: |
88+
set -e
89+
tag="$(bash automation-tools/find_newest_cooker_release.sh || true)"
90+
echo "reference_tag=$tag" >> "$GITHUB_OUTPUT"
91+
echo "$tag" > reference/reference_tag.txt
92+
if [[ -n "$tag" ]]; then
93+
url="https://github.com/${GITHUB_REPOSITORY}/releases/download/${tag}/components_version_list.md"
94+
if curl -fsSL "$url" -o reference/components_version_list.md; then
95+
echo "[INFO] Downloaded reference components_version_list.md from $tag"
96+
else
97+
echo "[WARN] Could not download reference components_version_list.md from $tag; disabling reuse"
98+
: > reference/components_version_list.md
99+
: > reference/reference_tag.txt
100+
echo "reference_tag=" >> "$GITHUB_OUTPUT"
101+
fi
102+
else
103+
echo "[INFO] No prior cooker release found; disabling reuse"
104+
: > reference/components_version_list.md
105+
fi
106+
107+
- name: Upload reference versions (for reuse)
108+
if: always()
109+
uses: actions/upload-artifact@v4
110+
with:
111+
name: reference-versions
112+
path: reference/*
113+
74114
build-light:
75115
name: "Build ${{ matrix.recipe.component }}"
76116
needs: setup-recipes
@@ -84,16 +124,6 @@ jobs:
84124
steps:
85125
- uses: actions/checkout@v4
86126

87-
# Remove Stuck Mounts
88-
- name: Remove stuck mounts
89-
run: |
90-
if [ -d "/home/ubuntu/actions-runner/_work/RetroDECK/RetroDECK/.flatpak-builder/rofiles" ]; then sudo umount -f /home/ubuntu/actions-runner/_work/RetroDECK/RetroDECK/.flatpak-builder/rofiles/*; fi
91-
if [ -d "$HOME/actions-run/_work/RetroDECK/RetroDECK/.flatpak-builder/rofiles" ]; then sudo umount -f $HOME/actions-run/_work/RetroDECK/RetroDECK/.flatpak-builder/rofiles/*; fi
92-
93-
# Install Dependencies
94-
- name: Install dependencies
95-
run: curl "https://raw.githubusercontent.com/RetroDECK/components-template/main/automation_tools/install_dependencies.sh" | bash
96-
97127
# Clone Repository (pull_request_target)
98128
- name: Clone Target Branch
99129
if: github.event_name == 'pull_request_target'
@@ -121,8 +151,77 @@ jobs:
121151
}
122152
git log -1 --oneline
123153
154+
- name: Download reference versions (for reuse)
155+
if: always()
156+
uses: actions/download-artifact@v4
157+
with:
158+
name: reference-versions
159+
path: reference
160+
161+
- name: Decide reuse vs rebuild (cooker only)
162+
id: reuse-check
163+
env:
164+
GITHUB_REPOSITORY: ${{ github.repository }}
165+
run: |
166+
set -e
167+
echo "artifact_name=${{ matrix.recipe.component }}" >> $GITHUB_ENV
168+
169+
reuse=false
170+
171+
if [[ "$GITHUB_REF" == "refs/heads/cooker" ]] && [[ "$GITHUB_EVENT_NAME" != "pull_request" ]] && [[ "$GITHUB_EVENT_NAME" != "pull_request_target" ]]; then
172+
ref_tag="$(cat reference/reference_tag.txt 2>/dev/null || true)"
173+
ref_file="reference/components_version_list.md"
174+
175+
if [[ -n "$ref_tag" && -s "$ref_file" ]]; then
176+
ref_ver="$(bash automation-tools/get_component_version_from_components_version_list.sh -f "$ref_file" -c "${{ matrix.recipe.component }}" 2>/dev/null || true)"
177+
next_ver="$(bash automation-tools/resolve_component_version.sh -r "${{ matrix.recipe.recipe }}" -v automation-tools/alchemist/desired_versions.sh 2>/dev/null || true)"
178+
179+
if [[ -n "$ref_ver" && -n "$next_ver" && "$ref_ver" == "$next_ver" ]]; then
180+
reuse=true
181+
echo "REF_TAG=$ref_tag" >> $GITHUB_ENV
182+
echo "[INFO] Reusing ${{ matrix.recipe.component }} from $ref_tag (version $ref_ver)"
183+
else
184+
echo "[INFO] Rebuild ${{ matrix.recipe.component }} (ref='$ref_ver' next='$next_ver')"
185+
fi
186+
else
187+
echo "[INFO] No usable reference; rebuilding"
188+
fi
189+
else
190+
echo "[INFO] Reuse disabled for this ref/event; rebuilding"
191+
fi
192+
193+
echo "reuse=$reuse" >> $GITHUB_OUTPUT
194+
195+
- name: Reuse component artifacts from newest cooker release
196+
if: steps.reuse-check.outputs.reuse == 'true'
197+
env:
198+
GITHUB_REPOSITORY: ${{ github.repository }}
199+
run: |
200+
set -e
201+
component="${{ matrix.recipe.component }}"
202+
mkdir -p "$component/artifacts"
203+
base="https://github.com/${GITHUB_REPOSITORY}/releases/download/${REF_TAG}"
204+
curl -fsSL "$base/${component}.tar.gz" -o "$component/artifacts/${component}.tar.gz"
205+
# sha may not always exist; best-effort
206+
curl -fsSL "$base/${component}.tar.gz.sha" -o "$component/artifacts/${component}.tar.gz.sha" || true
207+
echo "$REF_TAG" > "$component/artifacts/reused_from_release.txt"
208+
echo "$REF_TAG" > "$component/artifacts/reused_from_release.txt"
209+
210+
# Remove Stuck Mounts
211+
- name: Remove stuck mounts
212+
if: steps.reuse-check.outputs.reuse != 'true'
213+
run: |
214+
if [ -d "/home/ubuntu/actions-runner/_work/RetroDECK/RetroDECK/.flatpak-builder/rofiles" ]; then sudo umount -f /home/ubuntu/actions-runner/_work/RetroDECK/RetroDECK/.flatpak-builder/rofiles/*; fi
215+
if [ -d "$HOME/actions-run/_work/RetroDECK/RetroDECK/.flatpak-builder/rofiles" ]; then sudo umount -f $HOME/actions-run/_work/RetroDECK/RetroDECK/.flatpak-builder/rofiles/*; fi
216+
217+
# Install Dependencies
218+
- name: Install dependencies
219+
if: steps.reuse-check.outputs.reuse != 'true'
220+
run: curl "https://raw.githubusercontent.com/RetroDECK/components-template/main/automation_tools/install_dependencies.sh" | bash
221+
124222
# Run Build
125223
- name: 'Alchemist: Run Build Artifacts'
224+
if: steps.reuse-check.outputs.reuse != 'true'
126225
run: |
127226
# Use the component name provided in the matrix entry
128227
echo "artifact_name=${{ matrix.recipe.component }}" >> $GITHUB_ENV
@@ -148,12 +247,6 @@ jobs:
148247

149248
- uses: actions/checkout@v4
150249

151-
# Remove Stuck Mounts
152-
- name: Remove stuck mounts
153-
run: |
154-
if [ -d "/home/ubuntu/actions-runner/_work/RetroDECK/RetroDECK/.flatpak-builder/rofiles" ]; then sudo umount -f /home/ubuntu/actions-runner/_work/RetroDECK/RetroDECK/.flatpak-builder/rofiles/*; fi
155-
if [ -d "$HOME/actions-run/_work/RetroDECK/RetroDECK/.flatpak-builder/rofiles" ]; then sudo umount -f $HOME/actions-run/_work/RetroDECK/RetroDECK/.flatpak-builder/rofiles/*; fi
156-
157250
# Clone Repository (pull_request_target)
158251
- name: Clone Target Branch
159252
if: github.event_name == 'pull_request_target'
@@ -181,8 +274,69 @@ jobs:
181274
}
182275
git log -1 --oneline
183276
277+
- name: Download reference versions (for reuse)
278+
if: always()
279+
uses: actions/download-artifact@v4
280+
with:
281+
name: reference-versions
282+
path: reference
283+
284+
- name: Decide reuse vs rebuild (cooker only)
285+
id: reuse-check
286+
env:
287+
GITHUB_REPOSITORY: ${{ github.repository }}
288+
run: |
289+
set -e
290+
echo "artifact_name=${{ matrix.recipe.component }}" >> $GITHUB_ENV
291+
292+
reuse=false
293+
294+
if [[ "$GITHUB_REF" == "refs/heads/cooker" ]] && [[ "$GITHUB_EVENT_NAME" != "pull_request" ]] && [[ "$GITHUB_EVENT_NAME" != "pull_request_target" ]]; then
295+
ref_tag="$(cat reference/reference_tag.txt 2>/dev/null || true)"
296+
ref_file="reference/components_version_list.md"
297+
298+
if [[ -n "$ref_tag" && -s "$ref_file" ]]; then
299+
ref_ver="$(bash automation-tools/get_component_version_from_components_version_list.sh -f "$ref_file" -c "${{ matrix.recipe.component }}" 2>/dev/null || true)"
300+
next_ver="$(bash automation-tools/resolve_component_version.sh -r "${{ matrix.recipe.recipe }}" -v automation-tools/alchemist/desired_versions.sh 2>/dev/null || true)"
301+
302+
if [[ -n "$ref_ver" && -n "$next_ver" && "$ref_ver" == "$next_ver" ]]; then
303+
reuse=true
304+
echo "REF_TAG=$ref_tag" >> $GITHUB_ENV
305+
echo "[INFO] Reusing ${{ matrix.recipe.component }} from $ref_tag (version $ref_ver)"
306+
else
307+
echo "[INFO] Rebuild ${{ matrix.recipe.component }} (ref='$ref_ver' next='$next_ver')"
308+
fi
309+
else
310+
echo "[INFO] No usable reference; rebuilding"
311+
fi
312+
else
313+
echo "[INFO] Reuse disabled for this ref/event; rebuilding"
314+
fi
315+
316+
echo "reuse=$reuse" >> $GITHUB_OUTPUT
317+
318+
- name: Reuse component artifacts from newest cooker release
319+
if: steps.reuse-check.outputs.reuse == 'true'
320+
env:
321+
GITHUB_REPOSITORY: ${{ github.repository }}
322+
run: |
323+
set -e
324+
component="${{ matrix.recipe.component }}"
325+
mkdir -p "$component/artifacts"
326+
base="https://github.com/${GITHUB_REPOSITORY}/releases/download/${REF_TAG}"
327+
curl -fsSL "$base/${component}.tar.gz" -o "$component/artifacts/${component}.tar.gz"
328+
curl -fsSL "$base/${component}.tar.gz.sha" -o "$component/artifacts/${component}.tar.gz.sha" || true
329+
330+
# Remove Stuck Mounts
331+
- name: Remove stuck mounts
332+
if: steps.reuse-check.outputs.reuse != 'true'
333+
run: |
334+
if [ -d "/home/ubuntu/actions-runner/_work/RetroDECK/RetroDECK/.flatpak-builder/rofiles" ]; then sudo umount -f /home/ubuntu/actions-runner/_work/RetroDECK/RetroDECK/.flatpak-builder/rofiles/*; fi
335+
if [ -d "$HOME/actions-run/_work/RetroDECK/RetroDECK/.flatpak-builder/rofiles" ]; then sudo umount -f $HOME/actions-run/_work/RetroDECK/RetroDECK/.flatpak-builder/rofiles/*; fi
336+
184337
# Run Build
185338
- name: 'Alchemist: Run Build Artifacts'
339+
if: steps.reuse-check.outputs.reuse != 'true'
186340
run: |
187341
# Use the component name provided in the matrix entry
188342
echo "artifact_name=${{ matrix.recipe.component }}" >> $GITHUB_ENV

‎automation-tools/alchemist/alchemist.sh‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -63,8 +63,18 @@ transmute() {
6363

6464
# Download stage for this object
6565
download_result=$(process_download -t "$source_type" -u "$source_url" -d "$source_dest" -v "$SOURCE_VERSION")
66-
export DOWNLOADED_FILE=$(echo "$download_result" | grep "^DOWNLOADED_FILE=" | cut -d= -f2)
67-
export SOURCE_VERSION=$(echo "$download_result" | grep "^DOWNLOADED_VERSION=" | cut -d= -f2)
66+
downloaded_file=$(echo "$download_result" | grep "^DOWNLOADED_FILE=" | cut -d= -f2)
67+
downloaded_version=$(echo "$download_result" | grep "^DOWNLOADED_VERSION=" | cut -d= -f2)
68+
69+
if [[ -n "${downloaded_file:-}" ]]; then
70+
export DOWNLOADED_FILE="$downloaded_file"
71+
fi
72+
73+
# Some downloaders (e.g. local/http) don't emit DOWNLOADED_VERSION.
74+
# In that case, keep the recipe-provided SOURCE_VERSION instead of clobbering it.
75+
if [[ -n "${downloaded_version:-}" ]]; then
76+
export SOURCE_VERSION="$downloaded_version"
77+
fi
6878

6979
# Extraction stage for this object
7080
extraction_result=$(process_extract -f "$DOWNLOADED_FILE" -d "$source_dest" -t "$extraction_type")

‎automation-tools/fallback_release_artifacts.sh‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -153,6 +153,9 @@ main() {
153153
download_asset "$url_sha" "$REPO_ROOT/$component/artifacts/$local_sha_name"
154154
fi
155155

156+
# Mark this component as reused so reporting can show built vs reused.
157+
echo "$found_tag" >"$REPO_ROOT/$component/artifacts/reused_from_release.txt"
158+
156159
echo "${component}|${found_tag}" >>"$FALLBACK_OUT_FILE"
157160
done
158161

‎automation-tools/find_newest_cooker_release.sh‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
#!/usr/bin/env bash
2+
set -euo pipefail
3+
4+
# Finds the newest (most recently published) non-draft release whose tag starts with "cooker-"
5+
# and (optionally) contains a given asset name.
6+
#
7+
# Outputs the tag to stdout.
8+
9+
ASSET_NAME="${ASSET_NAME:-components_version_list.md}"
10+
REPO="${GITHUB_REPOSITORY:-}"
11+
TOKEN="${GITHUB_TOKEN:-}"
12+
13+
if [[ -z "$REPO" ]]; then
14+
echo "[ERROR] GITHUB_REPOSITORY is required (e.g. RetroDECK/components)" >&2
15+
exit 1
16+
fi
17+
18+
api_url="https://api.github.com/repos/$REPO/releases?per_page=100"
19+
20+
headers=("-H" "Accept: application/vnd.github+json")
21+
if [[ -n "$TOKEN" ]]; then
22+
headers+=("-H" "Authorization: token $TOKEN")
23+
fi
24+
25+
json=$(curl -fsSL "${headers[@]}" "$api_url")
26+
27+
tag=$(echo "$json" | jq -r --arg asset "$ASSET_NAME" '
28+
map(select(.draft == false))
29+
| map(select(.tag_name | startswith("cooker-")))
30+
| map(select((.assets // []) | any(.name == $asset)))
31+
| sort_by(.published_at // .created_at)
32+
| reverse
33+
| .[0].tag_name // empty
34+
')
35+
36+
if [[ -z "$tag" ]]; then
37+
# If we didn't find the asset, still try to return newest cooker tag (best-effort)
38+
tag=$(echo "$json" | jq -r '
39+
map(select(.draft == false))
40+
| map(select(.tag_name | startswith("cooker-")))
41+
| sort_by(.published_at // .created_at)
42+
| reverse
43+
| .[0].tag_name // empty
44+
')
45+
fi
46+
47+
echo "$tag"

‎automation-tools/get_component_version_from_components_version_list.sh‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
#!/usr/bin/env bash
2+
set -euo pipefail
3+
4+
usage() {
5+
echo "Usage: $0 -f <components_version_list.md> -c <component_name>" >&2
6+
}
7+
8+
file=""
9+
component=""
10+
11+
while [[ $# -gt 0 ]]; do
12+
case "$1" in
13+
-f|--file)
14+
file="$2"; shift 2;;
15+
-c|--component)
16+
component="$2"; shift 2;;
17+
*)
18+
usage; exit 2;;
19+
esac
20+
done
21+
22+
if [[ -z "${file}" || -z "${component}" ]]; then
23+
usage
24+
exit 2
25+
fi
26+
27+
if [[ ! -f "$file" ]]; then
28+
echo "[ERROR] File not found: $file" >&2
29+
exit 1
30+
fi
31+
32+
# Parse markdown table rows of the form:
33+
# | Component | Version | Built at (UTC) |
34+
# | name | version | timestamp |
35+
# Component name may be prefixed with "⚠️ ".
36+
37+
awk -v want="$component" -F'|' '
38+
function trim(s){ gsub(/^[ \t]+|[ \t]+$/, "", s); return s }
39+
/^[|]/ {
40+
name = trim($2)
41+
ver = trim($3)
42+
if (name ~ /^⚠️[[:space:]]+/) sub(/^⚠️[[:space:]]+/, "", name)
43+
if (name == want) { print ver; found=1; exit }
44+
}
45+
END { if (!found) exit 3 }
46+
' "$file"

0 commit comments

Comments
 (0)