CI: Generate RPM diff and auto-publish GitHub Release #63
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change | ||||
|---|---|---|---|---|---|---|
|
|
@@ -87,7 +87,6 @@ env: | |||||
| registries_production: 'docker.io/almalinux, quay.io/almalinuxorg, ghcr.io/almalinux' | ||||||
| registries_testing: 'quay.io/almalinuxautobot' | ||||||
|
|
||||||
| jobs: | ||||||
| init-data: | ||||||
| name: Set matrix, prepare variables | ||||||
|
|
@@ -348,6 +347,254 @@ jobs: | |||||
| && ( test "${{ matrix.image_types }}" != "micro" && rpm -q gpg-pubkey) || true " | ||||||
| done | ||||||
|
|
||||||
| - | ||||||
| name: Detect previous released tag for this image type | ||||||
| if: env.check_update != 0 | ||||||
| id: detect-prev | ||||||
| env: | ||||||
| VMJ: ${{ matrix.version_major }} | ||||||
| run: | | ||||||
| set -euo pipefail | ||||||
| IMAGE_REPO="${IMAGE_NAMES%%,*}" | ||||||
| PREV_TAG="${VMJ}" | ||||||
|
|
||||||
| echo "Checking ${IMAGE_REPO}:${PREV_TAG}" | ||||||
| if skopeo inspect docker://"${IMAGE_REPO}:${PREV_TAG}" >/dev/null 2>&1; then | ||||||
| echo "prev_image=${IMAGE_REPO}" >> "$GITHUB_ENV" | ||||||
| echo "prev_tag=${PREV_TAG}" >> "$GITHUB_ENV" | ||||||
| echo "prev_found=true" >> "$GITHUB_OUTPUT" | ||||||
| echo "Previous: ${IMAGE_REPO}:${PREV_TAG}" | ||||||
| else | ||||||
| echo "No previous tag found for ${IMAGE_REPO}:${PREV_TAG} — treating as first release." | ||||||
| echo "prev_image=" >> "$GITHUB_ENV" | ||||||
| echo "prev_tag=" >> "$GITHUB_ENV" | ||||||
| echo "prev_found=false" >> "$GITHUB_OUTPUT" | ||||||
| fi | ||||||
|
|
||||||
| - | ||||||
| name: Generate RPM lists (new vs previous) per platform | ||||||
| if: env.check_update != 0 | ||||||
| id: rpm-lists | ||||||
| env: | ||||||
| IMG_NEW_DIGEST: ${{ steps.build-images.outputs.digest }} | ||||||
| IMG_PREV: ${{ env.prev_image }} | ||||||
| TAG_PREV: ${{ env.prev_tag }} | ||||||
| PLATFORMS: ${{ env.platforms }} | ||||||
| VMJ: ${{ matrix.version_major }} | ||||||
| IMT: ${{ matrix.image_types }} | ||||||
| run: | | ||||||
| set -euo pipefail | ||||||
| mkdir -p rpm_lists/${VMJ}/${IMT} | ||||||
|
|
||||||
| list_rpms() { | ||||||
| local ref="$1" plat="$2" outfile="$3" | ||||||
| # Save both package name and full version | ||||||
| docker run --platform="${plat}" --rm "${ref}" /bin/bash -c \ | ||||||
| "rpm -qa --queryformat '%{NAME}\t%{VERSION}-%{RELEASE}\n' | sort" \ | ||||||
| > "${outfile}" | ||||||
| # Also create a names-only list | ||||||
| cut -f1 "${outfile}" | sort -u > "${outfile%.txt}.names" | ||||||
| } | ||||||
|
|
||||||
| for p in ${PLATFORMS//,/ }; do | ||||||
| plat=$(echo "$p" | xargs) | ||||||
|
|
||||||
| # New image: refer by digest | ||||||
| list_rpms "${IMG_NEW_DIGEST}" "${plat}" "rpm_lists/${VMJ}/${IMT}/new_${plat//\//_}.txt" | ||||||
|
|
||||||
| # Old image (if present): <image>:<tag> | ||||||
| if [ -n "${IMG_PREV:-}" ] && [ -n "${TAG_PREV:-}" ]; then | ||||||
| docker pull --platform "${plat}" "${IMG_PREV}:${TAG_PREV}" >/dev/null 2>&1 || true | ||||||
| list_rpms "${IMG_PREV}:${TAG_PREV}" "${plat}" "rpm_lists/${VMJ}/${IMT}/old_${plat//\//_}.txt" || \ | ||||||
| : # Continue even if platform not provided previously | ||||||
| fi | ||||||
| done | ||||||
|
|
||||||
| - | ||||||
| name: Diff RPM sets and build Markdown changelog | ||||||
| if: env.check_update != 0 | ||||||
| id: rpm-diff | ||||||
| env: | ||||||
| PLATFORMS: ${{ env.platforms }} | ||||||
| VMJ: ${{ matrix.version_major }} | ||||||
| IMT: ${{ matrix.image_types }} | ||||||
| PREV_TAG: ${{ env.prev_tag }} | ||||||
| DTS: ${{ needs.init-data.outputs.date_time_stamp }} | ||||||
| run: | | ||||||
| set -euo pipefail | ||||||
| out="rpm_diff_${VMJ}_${IMT}.md" | ||||||
| echo "**AlmaLinux ${VMJ} ${IMT}** RPM changelog (build \`${DTS}\`)" > "${out}" | ||||||
| if [ -n "${PREV_TAG}" ]; then | ||||||
| echo "" >> "${out}" | ||||||
| echo "- Compared to previous: \`${PREV_TAG}\`" >> "${out}" | ||||||
| else | ||||||
| echo "" >> "${out}" | ||||||
| echo "- No previous dated tag found. Treating all as **Added**." >> "${out}" | ||||||
| fi | ||||||
| echo "" >> "${out}" | ||||||
|
|
||||||
| any_section=false | ||||||
| for p in ${PLATFORMS//,/ }; do | ||||||
| plat=$(echo "$p" | xargs) | ||||||
| key="${plat//\//_}" | ||||||
| new="rpm_lists/${VMJ}/${IMT}/new_${key}.txt" | ||||||
| old="rpm_lists/${VMJ}/${IMT}/old_${key}.txt" | ||||||
|
|
||||||
| [ ! -f "${new}" ] && continue | ||||||
|
|
||||||
|
|
||||||
| # Collapse each platform's RPM diff into a <details> section | ||||||
| printf "\n<details>\n<summary>Platform: \`%s\`</summary>\n\n" "$plat" >> "$ | ||||||
|
|
||||||
| if [ -f "${old}" ]; then | ||||||
| # Added / Removed sets based on names | ||||||
| comm -13 "${old%.txt}.names" "${new%.txt}.names" > added.tmp || true | ||||||
| comm -23 "${old%.txt}.names" "${new%.txt}.names" > removed.tmp || true | ||||||
|
|
||||||
| # Updated: same name but different versions | ||||||
| join -j1 -t $'\t' -o 0,1.2,2.2 \ | ||||||
| <(join -t $'\t' -1 1 -2 1 <(cut -f1,2 "${old}" | sort) <(cut -f1,2 "${new}" | sort -k1,1) | sort -k1,1) \ | ||||||
| <(cut -f1,2 "${new}" | sort -k1,1) >/dev/null 2>&1 || true | ||||||
|
|
||||||
| # Recompute diffs and extract version changes | ||||||
| updated_tmp=$(mktemp) | ||||||
| awk -F'\t' 'NR==FNR{old[$1]=$2;next}{ if($1 in old && old[$1]!=$2){ print $1"\t"old[$1]" → "$2 } }' \ | ||||||
| "${old}" "${new}" > "${updated_tmp}" | ||||||
|
|
||||||
| addc=$(wc -l < added.tmp | tr -d ' ') | ||||||
| updc=$(wc -l < "${updated_tmp}" | tr -d ' ') | ||||||
| delc=$(wc -l < removed.tmp | tr -d ' ') | ||||||
|
|
||||||
| printed=false | ||||||
|
|
||||||
| # Added | ||||||
| if [ "$addc" -gt 0 ]; then | ||||||
| { | ||||||
| echo "" | ||||||
| echo "**Added (${addc})**" | ||||||
| echo "" | ||||||
| awk '{print "- " $0}' added.tmp | ||||||
| echo "" | ||||||
| } >> "${out}" | ||||||
| printed=true | ||||||
| fi | ||||||
|
|
||||||
| # Updated | ||||||
| if [ "$updc" -gt 0 ]; then | ||||||
| { | ||||||
| echo "" | ||||||
| echo "**Updated (${updc})**" | ||||||
| echo "" | ||||||
| awk -F'\t' '{printf "- %s: %s\n",$1,$2}' "${updated_tmp}" | ||||||
| echo "" | ||||||
| } >> "${out}" | ||||||
| printed=true | ||||||
| fi | ||||||
|
|
||||||
| # Removed | ||||||
| if [ "$delc" -gt 0 ]; then | ||||||
| { | ||||||
| echo "" | ||||||
| echo "**Removed (${delc})**" | ||||||
| echo "" | ||||||
| awk '{print "- " $0}' removed.tmp | ||||||
| echo "" | ||||||
| } >> "${out}" | ||||||
| printed=true | ||||||
| fi | ||||||
|
|
||||||
| [ "$printed" = true ] && any_section=true | ||||||
|
|
||||||
| rm -f added.tmp removed.tmp "${updated_tmp}" | ||||||
| else | ||||||
| # No previous list: treat all as Added | ||||||
| echo "" >> "${out}" | ||||||
| echo "**Added**" >> "${out}" | ||||||
| echo "" >> "${out}" | ||||||
| cut -f1 "${new}" | awk '{print "- " $0}' >> "${out}" | ||||||
| echo "" >> "${out}" | ||||||
| any_section=true | ||||||
|
|
||||||
| fi | ||||||
| # Close the <details> section for this platform | ||||||
| printf "\n</details>\n" >> "${out}" | ||||||
| done | ||||||
|
|
||||||
| $any_section || echo "_No RPM changes detected (no comparable platforms found)._" >> "${out}" | ||||||
|
|
||||||
| # Move artifacts into a readable path | ||||||
| mkdir -p rpm_diff_artifacts/${VMJ} | ||||||
| mv "${out}" "rpm_diff_artifacts/${VMJ}/rpm_diff_${VMJ}_${IMT}.md" | ||||||
|
|
||||||
| - | ||||||
| name: Build rechunk-style label payload (gzip+base64) | ||||||
| if: env.check_update != 0 | ||||||
| id: build-rechunk-label | ||||||
| env: | ||||||
| PLATFORMS: ${{ env.platforms }} | ||||||
| VMJ: ${{ matrix.version_major }} | ||||||
| IMT: ${{ matrix.image_types }} | ||||||
| PREV_TAG: ${{ env.prev_tag }} | ||||||
| DTS: ${{ needs.init-data.outputs.date_time_stamp }} | ||||||
| LABEL_KEY: dev.hhd.rechunk.info | ||||||
|
||||||
| run: | | ||||||
| set -euo pipefail | ||||||
| base="rpm_lists/${VMJ}/${IMT}" | ||||||
|
|
||||||
| # Create a base JSON | ||||||
| jq -n \ | ||||||
| --arg vmj "${VMJ}" \ | ||||||
| --arg imt "${IMT}" \ | ||||||
| --arg dts "${DTS}" \ | ||||||
| --arg prev "${PREV_TAG:-}" \ | ||||||
| '{ | ||||||
| version_major: $vmj, | ||||||
| image_type: $imt, | ||||||
| built_at: $dts, | ||||||
| prev_tag: ( $prev | select(length>0) // null ), | ||||||
| platforms: {} | ||||||
| }' > rechunk_${VMJ}_${IMT}.json | ||||||
|
|
||||||
| # Add per-platform package lists and SHA256 | ||||||
| for p in ${PLATFORMS//,/ }; do | ||||||
| plat=$(echo "$p" | xargs) | ||||||
| key="${plat//\//_}" | ||||||
| new="${base}/new_${key}.txt" | ||||||
| [ -f "${new}" ] || continue | ||||||
|
|
||||||
| sha=$(sha256sum "${new}" | awk '{print $1}') | ||||||
| # Convert TSV "name<TAB>version-release" into JSON array | ||||||
| jq -Rs ' | ||||||
| split("\n") | ||||||
| | map(select(length>0) | split("\t") | {n: .[0], v: .[1]}) | ||||||
| ' "${new}" > pkgs_${key}.json | ||||||
|
|
||||||
| tmp=$(mktemp) | ||||||
| jq --arg plat "${plat}" --arg sha "${sha}" --slurpfile pkgs pkgs_${key}.json \ | ||||||
| '.platforms[$plat] = { pkgs_sha256: $sha, pkgs: $pkgs[0] }' \ | ||||||
| "rechunk_${VMJ}_${IMT}.json" > "${tmp}" | ||||||
| mv "${tmp}" "rechunk_${VMJ}_${IMT}.json" | ||||||
| rm -f pkgs_${key}.json | ||||||
| done | ||||||
|
|
||||||
| # Compress and encode JSON into a single-line lable payload | ||||||
|
||||||
| # Compress and encode JSON into a single-line lable payload | |
| # Compress and encode JSON into a single-line label payload |
Outdated
Member
There was a problem hiding this comment.
I understand wanting to reduce the size of this data, but making it binary makes it a bit less useful. It adds an extra couple of steps to be able to read and use it. There's no limit to the value size (as far as I can tell), so why not just store the json directly?
Author
There was a problem hiding this comment.
I’ll change it to store the JSON directly in the label instead of encoding it.
Outdated
Member
There was a problem hiding this comment.
Suggested change
| ${{ env.LABLE_KEY }}=${{ env.RCHUNK_INFO_LABEL }} | |
| ${{ env.LABEL_KEY }}=${{ env.RCHUNK_INFO_LABEL }} |
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
You don't need to run the previous image and extract the list of packages again, now that it's attached as a label you could just get it from there. If you use
crane, you can even get the label without having to pull the image!There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I'll switch to using crane.
It seems that crane isn’t included in the default ubuntu-latest runner image, so I’ll add installation steps for it as well.
https://github.com/actions/runner-images/blob/main/images/ubuntu/Ubuntu2404-Readme.md