From 513b8bbe03520e5703eb78eb08d1b8a44ce041e1 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 24 Jan 2025 16:45:56 +0100 Subject: [PATCH 01/12] git-artifacts: stop special-casing v2.48.0-rc1 In 6224fe2 (git-artifacts: force-rebuilding the v2.48.0-rc1 package, 2024-12-31), I had to special-case the v2.48.0-rc1 release. This release has been published long ago, therefore we can drop that ugly hack. Signed-off-by: Johannes Schindelin --- .github/workflows/git-artifacts.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/git-artifacts.yml b/.github/workflows/git-artifacts.yml index 95d84371..d3368c80 100644 --- a/.github/workflows/git-artifacts.yml +++ b/.github/workflows/git-artifacts.yml @@ -172,7 +172,7 @@ jobs: test "$ARCHITECTURE" != x86_64 || ARTIFACTS_TO_BUILD="$ARTIFACTS_TO_BUILD nuget" } echo "ARTIFACTS_TO_BUILD=$ARTIFACTS_TO_BUILD" >> $GITHUB_ENV - echo "PKG_CACHE_KEY=pkg-$GIT_VERSION$(test v2.48.0-rc1.windows.1 != "$GIT_VERSION" || echo "-try2")-$ARCHITECTURE-$TAG_GIT_WORKFLOW_RUN_ID" >> $GITHUB_ENV + echo "PKG_CACHE_KEY=pkg-$GIT_VERSION-$ARCHITECTURE-$TAG_GIT_WORKFLOW_RUN_ID" >> $GITHUB_ENV - name: Configure user run: USER_NAME="${{github.actor}}" && From 6b2db73d0fea24692cf9e9294cc760f21c0d1930 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 2 Dec 2024 17:58:04 +0100 Subject: [PATCH 02/12] Scope some workflows to the main repository It makes little sense to run these GitHub workflows anywhere but in the `git-for-windows` org; In fact, it may cause mayhem if attempting to run them elsewhere, so let's prevent that footgun. Signed-off-by: Johannes Schindelin --- .github/workflows/break-pacman-upload-lease.yml | 1 + .github/workflows/build-and-deploy.yml | 1 + .github/workflows/open-pr.yml | 1 + .github/workflows/release-git.yml | 1 + .github/workflows/updpkgsums.yml | 1 + 5 files changed, 5 insertions(+) diff --git a/.github/workflows/break-pacman-upload-lease.yml b/.github/workflows/break-pacman-upload-lease.yml index 8b8ba15f..ab816540 100644 --- a/.github/workflows/break-pacman-upload-lease.yml +++ b/.github/workflows/break-pacman-upload-lease.yml @@ -6,6 +6,7 @@ on: jobs: break-lease: + if: github.event.repository.owner.login == 'git-for-windows' runs-on: ubuntu-latest steps: - name: Clone build-extra diff --git a/.github/workflows/build-and-deploy.yml b/.github/workflows/build-and-deploy.yml index 221789bb..1fee68c5 100644 --- a/.github/workflows/build-and-deploy.yml +++ b/.github/workflows/build-and-deploy.yml @@ -36,6 +36,7 @@ env: jobs: build: + if: github.event.repository.owner.login == 'git-for-windows' runs-on: ${{ github.event.inputs.architecture == 'aarch64' && fromJSON('["Windows", "ARM64"]') || 'windows-latest' }} steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/open-pr.yml b/.github/workflows/open-pr.yml index 8d8b4c4c..3aff19e5 100644 --- a/.github/workflows/open-pr.yml +++ b/.github/workflows/open-pr.yml @@ -24,6 +24,7 @@ env: jobs: open-pr: + if: github.event.repository.owner.login == 'git-for-windows' runs-on: windows-latest steps: - name: Determine REPO diff --git a/.github/workflows/release-git.yml b/.github/workflows/release-git.yml index 5b695646..3d8814f1 100644 --- a/.github/workflows/release-git.yml +++ b/.github/workflows/release-git.yml @@ -25,6 +25,7 @@ env: jobs: setup: runs-on: ubuntu-latest + if: github.event.repository.owner.login == 'git-for-windows' outputs: display-version: ${{ steps.bundle-artifacts.outputs.display-version }} tag-name: ${{ steps.bundle-artifacts.outputs.tag-name }} diff --git a/.github/workflows/updpkgsums.yml b/.github/workflows/updpkgsums.yml index 06ba70e2..2f7c060b 100644 --- a/.github/workflows/updpkgsums.yml +++ b/.github/workflows/updpkgsums.yml @@ -24,6 +24,7 @@ env: jobs: updpkgsums: + if: github.event.repository.owner.login == 'git-for-windows' runs-on: windows-latest steps: - uses: actions/checkout@v4 From 60ac2307470b513e152a115b0794666e06833d73 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sun, 28 Apr 2024 17:17:19 +0200 Subject: [PATCH 03/12] downloadAndUnZip: refactor out three separate functions Logically, `downloadAndUnZip()` does three things: - generate a temporary file path - download a file to that path - unzip the downloaded file and delete it Let's refactor this function to make this separation explicit, and individually reusable. Signed-off-by: Johannes Schindelin --- github-release.js | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/github-release.js b/github-release.js index 25e3e0e8..b37e102d 100644 --- a/github-release.js +++ b/github-release.js @@ -58,16 +58,28 @@ const getWorkflowRunArtifactsURLs = async (context, token, owner, repo, workflow }, {}) } -const downloadAndUnZip = async (token, url, name) => { +const download = async (token, url, outputFile) => { const { spawnSync } = require('child_process') const auth = token ? ['-H', `Authorization: Bearer ${token}`] : [] - const tmpFile = `${process.env.RUNNER_TEMP || process.env.TEMP || '/tmp'}/${name}.zip` - const curl = spawnSync('curl', [...auth, '-Lo', tmpFile, url]) + const curl = spawnSync('curl', [...auth, '-Lo', outputFile, url]) if (curl.error) throw curl.error - const { mkdirSync, rmSync } = require('fs') - await mkdirSync(name, { recursive: true }) - const unzip = spawnSync('unzip', ['-d', name, tmpFile]) +} + +const unzip = async (zipFile, outputDirectory) => { + const { mkdirSync } = require('fs') + await mkdirSync(outputDirectory, { recursive: true }) + const { spawnSync } = require('child_process') + const unzip = spawnSync('unzip', ['-d', outputDirectory, zipFile]) if (unzip.error) throw unzip.error +} + +const getTempFile = (name) => `${process.env.RUNNER_TEMP || process.env.TEMP || '/tmp'}/${name}` + +const downloadAndUnZip = async (token, url, name) => { + const tmpFile = getTempFile(`${name}.zip`) + await download(token, url, tmpFile) + await unzip(tmpFile, name) + const { rmSync } = require('fs') rmSync(tmpFile) } @@ -292,6 +304,9 @@ module.exports = { updateRelease, uploadReleaseAsset, getWorkflowRunArtifactsURLs, + download, + unzip, + getTempFile, downloadAndUnZip, downloadBundleArtifacts, getGitArtifacts, From 59cbfd58e631e32a020ea2a1bb7f56e93489a68b Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 4 Dec 2024 14:26:09 +0100 Subject: [PATCH 04/12] github-release(download): ensure to request a binary download Some GitHub REST API calls will strangely think that you prefer a JSON when you want to get, say, a release asset... unless you specifically ask for a binary download. Tsk, tsk. Signed-off-by: Johannes Schindelin --- github-release.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/github-release.js b/github-release.js index b37e102d..3b4840a0 100644 --- a/github-release.js +++ b/github-release.js @@ -60,8 +60,12 @@ const getWorkflowRunArtifactsURLs = async (context, token, owner, repo, workflow const download = async (token, url, outputFile) => { const { spawnSync } = require('child_process') - const auth = token ? ['-H', `Authorization: Bearer ${token}`] : [] - const curl = spawnSync('curl', [...auth, '-Lo', outputFile, url]) + const headers = token ? ['-H', `Authorization: Bearer ${token}`] : [] + if (url.match(/^https:\/\/github.com\/[^/]+\/[^/]+\/releases\/assets\/\d+$/) + || url.match(/^https:\/\/api\.github.com\/repos\/[^/]+\/[^/]+\/releases\/assets\/\d+$/)) { + headers.push('-H', 'Accept: application/octet-stream') + } + const curl = spawnSync('curl', [...headers, '-fLo', outputFile, url]) if (curl.error) throw curl.error } From d7c47eb5f76c733f7acff3f52c0055676f24eef0 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 4 Dec 2024 14:27:50 +0100 Subject: [PATCH 05/12] github-release: add a function to download assets from a GitHub Release It requires an amazingly unintuitive process to do something as commonplace as downloading release assets. That's why I originally preferred to let the GitHub CLI deal with it an be done with it. However, I want to avoid the necessity to have that GitHub CLI installed on self-hosted runners, so let's reimplement this using the plain (or actually, not so plain, as indicated above) REST API. Note that I still punt on replacing the `curl` call; It strikes me as not so bad, given that a `curl` version that seems good enough to handle my request is installed on all-but-ancient Windows versions by default (see https://curl.se/windows/microsoft.html for details). Signed-off-by: Johannes Schindelin --- github-release.js | 34 ++++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/github-release.js b/github-release.js index 3b4840a0..ad7c1989 100644 --- a/github-release.js +++ b/github-release.js @@ -303,6 +303,34 @@ const pushGitTag = (context, setSecret, token, owner, repo, tagName, bundlePath) context.log('Done pushing tag') } +const downloadReleaseAssets = async (context, setSecret, appId, privateKey, owner, repo, tagName, filenameMatcher) => { + const { getAccessTokenForRepo } = require('./repository-updates.js') + const token = await getAccessTokenForRepo(context, setSecret, appId, privateKey, owner, repo) + + const githubApiRequest = require('./github-api-request.js') + const release = await githubApiRequest( + context, + token, + 'GET', + `https://api.github.com/repos/${owner}/${repo}/releases/tags/${tagName}` + ) + + for (const asset of release.assets) { + if (!filenameMatcher || filenameMatcher(asset.name)) { + context.log(`Downloading ${asset.name}`) + await download(token, asset.url, asset.name) + } + } +} + +const downloadReleaseAssetsFromURL = async (context, setSecret, appId, privateKey, releaseURL, filenameMatcher) => { + const [, owner, repo, tagName] = releaseURL.match( + /^https:\/\/github.com\/([^/]+)\/([^/]+)\/releases\/tag\/([^/]+)$/ + ) + if (!owner || !repo || !tagName) throw new Error(`Invalid release URL: ${releaseURL}`) + return await downloadReleaseAssets(context, setSecret, appId, privateKey, owner, repo, tagName, filenameMatcher) +} + module.exports = { createRelease, updateRelease, @@ -318,5 +346,7 @@ module.exports = { calculateSHA256ForFile, checkSHA256Sums, uploadGitArtifacts, - pushGitTag -} \ No newline at end of file + pushGitTag, + downloadReleaseAssets, + downloadReleaseAssetsFromURL, +} From f3a2525685a570698f7e01118547bd019747bba0 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sat, 23 Mar 2024 23:01:10 +0100 Subject: [PATCH 06/12] Add a GitHub workflow to prepare the branches for embargoed Git for Windows versions When embargoed Git for Windows versions (or backport versions) need to be prepared, a couple of branches need to be created (internally nicknamed "time-traveling branches"), to pick back up at the precise revisions from where the preceding version was built. That way, the embargoed version will only differ in the fixed Git artifacts, and avoid shipping with updates in other components that might cause risk of regressions. This new workflow prepares these branches. Signed-off-by: Johannes Schindelin --- .../workflows/prepare-embargoed-branches.yml | 60 +++++++++++ prepare-embargoed-branches.sh | 99 +++++++++++++++++++ repository-updates.js | 3 +- 3 files changed, 161 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/prepare-embargoed-branches.yml create mode 100755 prepare-embargoed-branches.sh diff --git a/.github/workflows/prepare-embargoed-branches.yml b/.github/workflows/prepare-embargoed-branches.yml new file mode 100644 index 00000000..3c807db5 --- /dev/null +++ b/.github/workflows/prepare-embargoed-branches.yml @@ -0,0 +1,60 @@ +name: prepare-embargoed-branches +run-name: Prepare branches for embargoed Git ${{ inputs.version }} + +on: + workflow_dispatch: + inputs: + version: + description: 'The Git version for which to prepare the branches' + required: true + +jobs: + prepare-embargoed-branches: + runs-on: ubuntu-latest + steps: + - name: sanity check + if: ${{ github.repository_owner == 'git-for-windows' }} + run: echo "This action is not meant to be run on the Git for Windows repository" >&2 && exit 1 + - run: actions/checkout@v4 + - name: identify actor + id: actor + uses: actions/github-script@v7 + with: + script: | + const githubApiRequest = require('./github-api-request') + const answer = await githubApiRequest( + console, + '${{ secrets.GITHUB_TOKEN }}', + 'GET', + '/users/${{ github.triggering_actor }}' + ) + core.setOutput('name', answer.name) + core.setOutput('email', answer.email || '${{ github.triggering_actor }}@users.noreply.github.com') + - name: configure + run: | + USER_NAME="${{ steps.actor.outputs.name }}" && + USER_EMAIL="${{ steps.actor.outputs.email }}" && + git config --global user.name "$USER_NAME" && + git config --global user.email "$USER_EMAIL" && + git config --global url.https://github.com/${{ github.repository_owner }}.insteadOf \ + https://github.com/embargoed-git-for-windows-builds && + git config --global credential.helper '' && + git config --global --add credential.helper cache + - name: configure push token + uses: actions/github-script@v7 + with: + script: | + const { callGit, getPushAuthorizationHeader } = require('./repository-updates.js') + for (const repo of ['build-extra', 'git', 'git-sdk-32', 'git-sdk-64', 'MINGW-packages']) { + const header = await getPushAuthorizationHeader( + console, + core.setSecret, + ${{ secrets.GH_APP_ID }}, + ${{ toJSON(secrets.GH_APP_PRIVATE_KEY) }}, + context.repo.owner, + repo + ) + console.log(callGit(['config', '--global', `http.https://github.com/${context.repo.owner}/${repo}.extraHeader`, header])) + } + - name: Prepare embargoed branches + run: sh -x ./prepare-embargoed-branches.sh "${{ inputs.version }}" \ No newline at end of file diff --git a/prepare-embargoed-branches.sh b/prepare-embargoed-branches.sh new file mode 100755 index 00000000..6e89851a --- /dev/null +++ b/prepare-embargoed-branches.sh @@ -0,0 +1,99 @@ +#!/bin/sh + +die () { + echo "$*" >&2 + exit 1 +} + +dry_run= +mingit= +while case "$1" in +--dry-run|-n) dry_run=1;; +--mingit) mingit=1;; +-*) die "Unknown option: $1";; +*) break;; +esac; do shift; done + +test $# = 1 || +die "Usage: $0 [--dry-run] [--mingit] # e.g. 2.39.1" + +version=${1#v} +if test -z "$mingit" +then + case "$version" in + *[^0-9.]*|*..*|.*|*.) die "Invalid version: '$version'";; + *.*.*) ;; # okay + *) die "Invalid version: '$version'";; + esac + previous_version_prefix=${version%.*}.$((${version##*.}-1)) + branch_name=git-$version +else + previous_version_prefix="$(expr "$version" : '\([0-9]\+\.[0-9]\+\)\.\{0,1\}[0-9]*$')" + test -n "$previous_version_prefix" || die "Invalid version: '$version'" + branch_name=mingit-$previous_version_prefix.x-releases +fi +grep_version_regex="$(echo "$previous_version_prefix" | sed 's/\./\\\\&/g')" + +handle_repo () { + name="$1" + path="$2" + args="$3" + + echo "### Handling $name ###" && + + if test -e "$path/.git" + then + git_dir="$path/.git" + main_refspec="refs/remotes/origin/main:refs/heads/main" + else + # To allow for running this script on Linux/macOS, fall back to cloning to pwd + git_dir=${path##*/}.git && + if test ! -d "$git_dir" + then + # We only need a partial clone + git clone --bare --filter=blob:none \ + https://github.com/git-for-windows/$name "$git_dir" + fi + main_refspec="refs/heads/main:refs/heads/main" + fi && + + # ensure that the `embargoed-git-for-windows-builds` remote is set + remote_url=https://github.com/embargoed-git-for-windows-builds/$name && + case "$(git --git-dir "$git_dir" remote show -n embargoed-git-for-windows-builds)" in + *"Fetch URL: $remote_url"*"Push URL: $remote_url"*) ;; # okay + *) git --git-dir "$git_dir" remote add embargoed-git-for-windows-builds $remote_url;; + esac && + + # if `embargoed-git-for-windows-builds` already has the branch, everything's fine already + revision=$(git --git-dir "$git_dir" ls-remote embargoed-git-for-windows-builds refs/heads/$branch_name | cut -f 1) && + if test -n "$revision" + then + echo "$name already has $branch_name @$revision" + else + git --git-dir "$git_dir" fetch origin main && + revision="$(eval git --git-dir "\"$git_dir\"" rev-list -1 FETCH_HEAD $args)" && + if test -z "$revision" + then + die "No matching revision for $args in $name" + fi && + echo "Creating $branch_name in $name @$revision" && + push_ref_spec="$revision:refs/heads/$branch_name $main_refspec" && + if test -n "$dry_run" + then + git --git-dir "$git_dir" show -s "$revision" && + echo "Would call 'git push embargoed-git-for-windows-builds $push_ref_spec'" + else + echo "git push embargoed-git-for-windows-builds $push_ref_spec" && + git --git-dir "$git_dir" push embargoed-git-for-windows-builds $push_ref_spec + fi + fi +} + +handle_repo git-sdk-32 /c/git-sdk-32 \ + "\"--grep=mingw-w64-i686-git \".*\" -> $grep_version_regex\" -- cmd/git.exe" && +handle_repo git-sdk-64 /c/git-sdk-64 \ + "\"--grep=mingw-w64-x86_64-git \".*\" -> $grep_version_regex\" -- cmd/git.exe" && +handle_repo build-extra /usr/src/build-extra \ + "-- versions/package-versions-$previous_version_prefix\\*-MinGit.txt" && +handle_repo MINGW-packages /usr/src/MINGW-packages \ + "\"--grep=mingw-w64-git: new version .v$grep_version_regex\" -- mingw-w64-git/PKGBUILD" diff --git a/repository-updates.js b/repository-updates.js index f8d953c2..853d7ddd 100644 --- a/repository-updates.js +++ b/repository-updates.js @@ -158,5 +158,6 @@ module.exports = { callGit, getWorkflowRunArtifact, pushRepositoryUpdate, - pushGitBranch + pushGitBranch, + getPushAuthorizationHeader } From f7f6a0b5948d3e2d6bbf090c739994594beda0b8 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 3 Dec 2024 16:27:15 +0100 Subject: [PATCH 07/12] prepare-embargoed-branches: add git-sdk-arm64 Since v2.47.1, Git for Windows supports Windows/ARM64. Signed-off-by: Johannes Schindelin --- .github/workflows/prepare-embargoed-branches.yml | 2 +- prepare-embargoed-branches.sh | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/prepare-embargoed-branches.yml b/.github/workflows/prepare-embargoed-branches.yml index 3c807db5..f5916e9c 100644 --- a/.github/workflows/prepare-embargoed-branches.yml +++ b/.github/workflows/prepare-embargoed-branches.yml @@ -45,7 +45,7 @@ jobs: with: script: | const { callGit, getPushAuthorizationHeader } = require('./repository-updates.js') - for (const repo of ['build-extra', 'git', 'git-sdk-32', 'git-sdk-64', 'MINGW-packages']) { + for (const repo of ['build-extra', 'git', 'git-sdk-32', 'git-sdk-64', 'git-sdk-arm64', 'MINGW-packages']) { const header = await getPushAuthorizationHeader( console, core.setSecret, diff --git a/prepare-embargoed-branches.sh b/prepare-embargoed-branches.sh index 6e89851a..1a812ca2 100755 --- a/prepare-embargoed-branches.sh +++ b/prepare-embargoed-branches.sh @@ -93,6 +93,8 @@ handle_repo git-sdk-32 /c/git-sdk-32 \ "\"--grep=mingw-w64-i686-git \".*\" -> $grep_version_regex\" -- cmd/git.exe" && handle_repo git-sdk-64 /c/git-sdk-64 \ "\"--grep=mingw-w64-x86_64-git \".*\" -> $grep_version_regex\" -- cmd/git.exe" && +handle_repo git-sdk-arm64 /c/git-sdk-arm64 \ + "\"--grep=mingw-w64-clang-aarch64-git \".*\" -> $grep_version_regex\" -- cmd/git.exe" && handle_repo build-extra /usr/src/build-extra \ "-- versions/package-versions-$previous_version_prefix\\*-MinGit.txt" && handle_repo MINGW-packages /usr/src/MINGW-packages \ From 24982d9e78791759b1ef08cc57dff03b496326e5 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 3 Dec 2024 22:50:18 +0100 Subject: [PATCH 08/12] prepare-embargoed-branches: support A.B.C.D versions Sometimes, Git for Windows has to go alone, and release several versions based on the same upstream version. In these instances, a version format like v2.47.0(2) is used. Let's support those, too. Signed-off-by: Johannes Schindelin --- prepare-embargoed-branches.sh | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/prepare-embargoed-branches.sh b/prepare-embargoed-branches.sh index 1a812ca2..d4cf6ca0 100755 --- a/prepare-embargoed-branches.sh +++ b/prepare-embargoed-branches.sh @@ -21,11 +21,26 @@ version=${1#v} if test -z "$mingit" then case "$version" in + *.*.*.windows.*) + # major.minor.patch.windows.extra + previous_version_prefix=${version%.windows.*} + version="${version%.windows.*}.${version##*.windows.}" + ;; + *.*.*\(*) + # major.minor.patch(extra) + previous_version_prefix=${version%(*} + version="${version%(*}.${version##*(}" + version=${version%)} + ;; *[^0-9.]*|*..*|.*|*.) die "Invalid version: '$version'";; - *.*.*) ;; # okay + *.*.*.*) + # major.minor.patch.extra + v0="${version#*.*.*.}" + previous_version_prefix=${version%.$v0} + ;; + *.*.*) previous_version_prefix=${version%.*}.$((${version##*.}-1));; # major.minor.patch *) die "Invalid version: '$version'";; esac - previous_version_prefix=${version%.*}.$((${version##*.}-1)) branch_name=git-$version else previous_version_prefix="$(expr "$version" : '\([0-9]\+\.[0-9]\+\)\.\{0,1\}[0-9]*$')" From dd1874e081df0892c94919a9e2539d7fa447e360 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sun, 24 Mar 2024 20:58:51 +0100 Subject: [PATCH 09/12] prepare-embargoed-branches: optionally prepare for MinGit backports MinGit backports are similar to embargoed Git for Windows versions, but more limited in scope, and quite often, not even embargoed: Git for Windows is only ever released for the latest major Git version and its bugfix versions, whereas MinGit caters to 3rd party applications that often choose to stay behind a major version or three. Signed-off-by: Johannes Schindelin --- .github/workflows/prepare-embargoed-branches.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/prepare-embargoed-branches.yml b/.github/workflows/prepare-embargoed-branches.yml index f5916e9c..d61d1182 100644 --- a/.github/workflows/prepare-embargoed-branches.yml +++ b/.github/workflows/prepare-embargoed-branches.yml @@ -1,5 +1,5 @@ name: prepare-embargoed-branches -run-name: Prepare branches for embargoed Git ${{ inputs.version }} +run-name: Prepare branches for embargoed ${{ inputs.mingit-only && 'Min' || '' }}Git ${{ inputs.version }} on: workflow_dispatch: @@ -7,6 +7,10 @@ on: version: description: 'The Git version for which to prepare the branches' required: true + mingit-only: + description: 'Only prepare the MinGit branches' + default: false + type: boolean jobs: prepare-embargoed-branches: @@ -15,7 +19,7 @@ jobs: - name: sanity check if: ${{ github.repository_owner == 'git-for-windows' }} run: echo "This action is not meant to be run on the Git for Windows repository" >&2 && exit 1 - - run: actions/checkout@v4 + - uses: actions/checkout@v4 - name: identify actor id: actor uses: actions/github-script@v7 @@ -57,4 +61,4 @@ jobs: console.log(callGit(['config', '--global', `http.https://github.com/${context.repo.owner}/${repo}.extraHeader`, header])) } - name: Prepare embargoed branches - run: sh -x ./prepare-embargoed-branches.sh "${{ inputs.version }}" \ No newline at end of file + run: sh -x ./prepare-embargoed-branches.sh ${{ inputs.mingit-only && '--mingit ' || ''}}"${{ inputs.version }}" \ No newline at end of file From fe95907cf7fdb9babfc7ea719f5753bd01daf6ce Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sat, 25 Jan 2025 22:59:19 +0100 Subject: [PATCH 10/12] git-artifacts(snapshots): work around a bug in `git fetch ` As part of the snapshot builds via the `git-artifacts` workflow, we try to import the tag from a bundle. This uncovered a bug that is over 17 years old, and which I am trying to fix via https://github.com/gitgitgadget/git/pull/1857. The symptom is that just after fetching the refs, when looking through the fetched revisions whether a recursive fetch is needed, Git fails to `mmap()` the newly-imported packfile. The symptom looks like this: From [...]/git.bundle * tag -> FETCH_HEAD * [new tag] -> fatal: mmap: could not determine filesize Curiously, this only happens on 32-bit Windows, not on 64-bit Windows, nor on Linux or macOS. The explanation is most likely to be found in how quickly file descriptor values are used again after closing them, which would appear to be _really_ quickly on i686 Windows. Be that as it may, to work around this issue, we simply avoid any operation that would need to access the just-imported packfile in `git fetch `. Signed-off-by: Johannes Schindelin --- .github/workflows/git-artifacts.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/git-artifacts.yml b/.github/workflows/git-artifacts.yml index d3368c80..ab31008d 100644 --- a/.github/workflows/git-artifacts.yml +++ b/.github/workflows/git-artifacts.yml @@ -246,7 +246,8 @@ jobs: then git fetch origin "refs/tags/$EXISTING_GIT_TAG:refs/tags/$EXISTING_GIT_TAG" else - git fetch --tags "$GITHUB_WORKSPACE"/bundle-artifacts/git.bundle \ + git -c fetch.writeCommitGraph=false fetch --tags --no-recurse-submodules \ + "$GITHUB_WORKSPACE"/bundle-artifacts/git.bundle \ $(cat "$GITHUB_WORKSPACE"/bundle-artifacts/next_version) fi && git reset --hard $(cat "$GITHUB_WORKSPACE"/bundle-artifacts/next_version) From 600d642205bf8c7f79be0cd18430011cec4cc786 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sat, 25 Jan 2025 23:27:56 +0100 Subject: [PATCH 11/12] git-artifacts: trace commands that prepare the git clone Signed-off-by: Johannes Schindelin --- .github/workflows/git-artifacts.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/git-artifacts.yml b/.github/workflows/git-artifacts.yml index ab31008d..49266ca1 100644 --- a/.github/workflows/git-artifacts.yml +++ b/.github/workflows/git-artifacts.yml @@ -226,6 +226,7 @@ jobs: - name: Prepare git-for-windows/git clone with the tag if: steps.restore-cached-git-pkg.outputs.cache-hit != 'true' run: | + set -x if test ! -d /usr/src/MINGW-packages then git clone --depth 1 --single-branch -b main \ From 115a75f217822d8c50d372da14f9373b4e74dc3d Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sun, 26 Jan 2025 22:37:58 +0100 Subject: [PATCH 12/12] Add a workflow definition to upload Git for Windows' snapshots This is another step in the migration from the Azure Pipeline that produced Git for Windows' snapshot builds and the Azure Release Pipeline that then uploaded them to Azure Blobs. With this new GitHub workflow, all we need are the successful `git-artifacts` runs (one per CPU architecture), and the workflow will upload them to the new snapshot location reachable via https://gitforwindows.org/git-snapshots. Signed-off-by: Johannes Schindelin --- .github/workflows/upload-snapshot.yml | 286 ++++++++++++++++++++++++++ github-release.js | 1 + 2 files changed, 287 insertions(+) create mode 100644 .github/workflows/upload-snapshot.yml diff --git a/.github/workflows/upload-snapshot.yml b/.github/workflows/upload-snapshot.yml new file mode 100644 index 00000000..8b799a53 --- /dev/null +++ b/.github/workflows/upload-snapshot.yml @@ -0,0 +1,286 @@ +name: upload-snapshot +run-name: "Upload Git for Windows snapshot" + +on: + workflow_dispatch: + inputs: + git_artifacts_i686_workflow_run_id: + description: 'ID of the git-artifacts (i686) workflow run' + required: true + git_artifacts_x86_64_workflow_run_id: + description: 'ID of the git-artifacts (x86_64) workflow run' + required: true + git_artifacts_aarch64_workflow_run_id: + description: 'ID of the git-artifacts (aarch64) workflow run' + required: true + +env: + OWNER: "${{ github.repository_owner }}" + REPO: git + SNAPSHOTS_REPO: git-snapshots + ARTIFACTS_REPO: git-for-windows-automation + I686_WORKFLOW_RUN_ID: "${{ github.event.inputs.git_artifacts_i686_workflow_run_id }}" + X86_64_WORKFLOW_RUN_ID: "${{ github.event.inputs.git_artifacts_x86_64_workflow_run_id }}" + AARCH64_WORKFLOW_RUN_ID: "${{ github.event.inputs.git_artifacts_aarch64_workflow_run_id }}" + CREATE_CHECK_RUN: "true" + NODEJS_VERSION: 16 + +jobs: + upload-snapshot: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: download `bundle-artifacts` + id: bundle-artifacts + uses: actions/github-script@v7 + with: + script: | + const { + getWorkflowRunArtifactsURLs, + downloadAndUnZip, + } = require('./github-release') + + const token = ${{ toJSON(secrets.GITHUB_TOKEN) }} + const workflowRunId = process.env.X86_64_WORKFLOW_RUN_ID + const urls = await getWorkflowRunArtifactsURLs( + console, + token, + process.env.OWNER, + process.env.ARTIFACTS_REPO, + workflowRunId + ) + core.setOutput('x86_64-urls', urls) + + const dir = 'bundle-artifacts-x86_64' + await downloadAndUnZip(token, urls['bundle-artifacts'], dir) + + const fs = require('fs') + const sha = fs.readFileSync(`${dir}/git-commit-oid`, 'utf-8').trim() + core.notice(`git-commit-oid: ${sha}`) + + const githubApiRequest = require('./github-api-request') + const { commit: { committer: { date } } } = await githubApiRequest( + console, + token, + 'GET', + `/repos/${process.env.OWNER}/${process.env.REPO}/commits/${sha}` + ) + + // emulate Git's date/time format + core.setOutput('date', new Date(date).toLocaleString('en-US', { + weekday: 'short', + month: 'short', + day: 'numeric', + year: 'numeric', + hour: '2-digit', + minute: '2-digit', + second: '2-digit', + timeZoneName: 'longOffset', + }).replace(/^(.*,.*),(.*),(.* )((PM|AM) GMT)([-+]\d\d):(\d\d)$/, '$1$2$3$6$7')) + core.setOutput('git-commit-oid', sha) + core.setOutput('ver', fs.readFileSync(`${dir}/ver`, 'utf-8').trim()) + - name: Mirror Check Run to ${{ env.OWNER }}/${{ env.REPO }} + if: env.CREATE_CHECK_RUN != 'false' + uses: ./.github/actions/check-run-action + with: + app-id: ${{ secrets.GH_APP_ID }} + private-key: ${{ secrets.GH_APP_PRIVATE_KEY }} + owner: ${{ env.OWNER }} + repo: ${{ env.REPO }} + rev: ${{ steps.bundle-artifacts.outputs.git-commit-oid }} + check-run-name: "upload-snapshot" + title: "Upload snapshot ${{ steps.bundle-artifacts.outputs.ver }}" + summary: "Upload snapshot ${{ steps.bundle-artifacts.outputs.ver }}" + text: "For details, see [this run](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id}})." + details-url: "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id}}" + - name: download remaining artifacts + id: download-artifacts + uses: actions/github-script@v7 + with: + script: | + const { + getWorkflowRunArtifactsURLs, + downloadAndUnZip, + architectures + } = require('./github-release') + + const token = ${{ toJSON(secrets.GITHUB_TOKEN) }} + const directories = ['bundle-artifacts-x86_64'] + for (const arch of architectures) { + const architecture = arch.name + + const urls = architecture === 'x86_64' + ? ${{ steps.bundle-artifacts.outputs.x86_64-urls }} + : await getWorkflowRunArtifactsURLs( + console, + token, + process.env.OWNER, + process.env.ARTIFACTS_REPO, + process.env[`${architecture.toUpperCase()}_WORKFLOW_RUN_ID`] + ) + for (const name of Object.keys(urls)) { + if (name === 'bundle-artifacts' && architecture === 'x86_64') continue // already got it + if (!name.match(/^(installer|portable|mingit|bundle)/)) continue + const outputDirectory = name.endsWith(`-${architecture}`) ? name : `${name}-${architecture}` + console.log(`Downloading ${name} and extracting to ${outputDirectory}/`) + await downloadAndUnZip(token, urls[name], outputDirectory) + directories.push(outputDirectory) + } + } + + const fs = require('fs') + const assetsToUpload = directories + .map(directory => fs + .readdirSync(directory) + .filter(file => file.match(/^(Min|Portable)Git-.*\.(exe|zip)$/)) + .map(file => `${directory}/${file}`)) + .flat() + if (assetsToUpload.length === 0) throw new Error(`No assets to upload!`) + console.log(JSON.stringify(assetsToUpload, null, 2)) + core.setOutput('paths', assetsToUpload.join(' ')) + return assetsToUpload + - name: update check-run + if: env.CREATE_CHECK_RUN != 'false' + uses: ./.github/actions/check-run-action + with: + app-id: ${{ secrets.GH_APP_ID }} + private-key: ${{ secrets.GH_APP_PRIVATE_KEY }} + append-text: 'Downloaded all artifacts' + - name: validate + id: validate + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs') + + const { architectures } = require('./github-release') + for (const arch of architectures) { + const ver = fs.readFileSync(`bundle-artifacts-${arch.name}/ver`, 'utf-8').trim() + if (${{ toJSON(steps.bundle-artifacts.outputs.ver) }} !== ver) { + core.error(`Mismatched version between x86_64 and ${arch.name}: ${{ toJSON(steps.bundle-artifacts.outputs.ver) }} != "${ver}"`) + process.exit(1) + } + } + + const githubApiRequest = require('./github-api-request') + const { ahead_by } = await githubApiRequest( + console, + ${{ toJSON(secrets.GITHUB_TOKEN) }}, + 'GET', + `/repos/${process.env.OWNER}/${process.env.REPO}/compare/HEAD...${{ steps.bundle-artifacts.outputs.git-commit-oid }}` + ) + if (ahead_by !== 0) { + core.error(`The snapshots are built from a commit that is not reachable from git-for-windows/git's default branch!`) + process.exit(1) + } + - name: configure token + id: snapshots-token + uses: actions/github-script@v7 + with: + result-encoding: string + script: | + const { callGit, getPushAuthorizationHeader } = require('./repository-updates') + const header = await getPushAuthorizationHeader( + console, + core.setSecret, + ${{ secrets.GH_APP_ID }}, + ${{ toJSON(secrets.GH_APP_PRIVATE_KEY) }}, + process.env.OWNER, + process.env.SNAPSHOTS_REPO + ) + console.log(callGit([ + 'config', + '--global', + `http.${{ github.server_url }}/${process.env.OWNER}/${process.env.SNAPSHOTS_REPO}.extraHeader`, + header + ])) + return Buffer.from(header.replace(/^Authorization: Basic /, ''), 'base64').toString('utf-8').replace(/^PAT:/, '') + - name: figure out if we need to push commits + uses: actions/github-script@v7 + with: + script: | + // Since `git-snapshots` is a fork, and forks share the same object store, we can + // assume that `git-commit-oid` is accessible in the `git-snapshots` repository even + // if it might not yet be reachable. + const githubApiRequest = require('./github-api-request') + const token = ${{ toJSON(steps.snapshots-token.outputs.result) }} + const sha = ${{ toJSON(steps.bundle-artifacts.outputs.git-commit-oid) }} + const { ahead_by, behind_by } = await githubApiRequest( + console, + token, + 'GET', + `/repos/${process.env.OWNER}/${process.env.SNAPSHOTS_REPO}/compare/${sha}...HEAD` + ) + if (ahead_by > 0) throw new Error(`The snapshots repository is ahead of ${sha}!`) + if (behind_by > 0) { + await githubApiRequest( + console, + token, + 'PATCH', + `/repos/${process.env.OWNER}/${process.env.SNAPSHOTS_REPO}/git/refs/heads/main`, { + sha, + force: false // require fast-forward + } + ) + } + - name: upload snapshots to ${{ env.SNAPSHOTS_REPO }} + env: + GH_TOKEN: ${{ steps.snapshots-token.outputs.result }} + run: | + gh release create \ + -R "$OWNER/$SNAPSHOTS_REPO" \ + --target "${{ steps.bundle-artifacts.outputs.git-commit-oid }}" \ + --title "${{ steps.bundle-artifacts.outputs.date }}" \ + ${{ steps.bundle-artifacts.outputs.ver }} \ + ${{ steps.download-artifacts.outputs.paths }} && + echo "::notice::Uploaded snapshot artifacts to ${{ github.server_url }}/${{ env.OWNER }}/${{ env.SNAPSHOTS_REPO }}/releases/tag/${{ steps.bundle-artifacts.outputs.ver }}" + - name: update check-run + if: env.CREATE_CHECK_RUN != 'false' + uses: ./.github/actions/check-run-action + with: + app-id: ${{ secrets.GH_APP_ID }} + private-key: ${{ secrets.GH_APP_PRIVATE_KEY }} + append-text: 'Created release at ${{ github.server_url }}/${{ env.OWNER }}/${{ env.SNAPSHOTS_REPO }}/releases/tag/${{ steps.bundle-artifacts.outputs.ver }}' + - name: clone gh-pages + uses: actions/checkout@v4 + with: + repository: ${{ env.OWNER }}/${{ env.SNAPSHOTS_REPO }} + ref: gh-pages + path: gh-pages + token: ${{ steps.snapshots-token.outputs.result }} + - name: update index.html + uses: actions/github-script@v7 + with: + script: | + const urlPrefix = `${{ github.server_url }}/${{ env.OWNER }}/${{ env.SNAPSHOTS_REPO }}/releases/download/${{ steps.bundle-artifacts.outputs.ver }}/` + process.chdir('gh-pages') + const main = require('./add-entry') + await main( + '--date=${{ steps.bundle-artifacts.outputs.date }}', + '--commit=${{ steps.bundle-artifacts.outputs.git-commit-oid }}', + ...${{ steps.download-artifacts.outputs.result }} + .map(path => `${urlPrefix}${path.replace(/.*\//, '')}`) + ) + - name: push gh-pages + run: | + git -C gh-pages \ + -c user.name="${{ github.actor }}" \ + -c user.email="${{ github.actor }}@noreply.github.com" \ + commit -sm "Add snapshot: ${{ steps.bundle-artifacts.outputs.ver }}" index.html && + git -C gh-pages push && + echo "::notice::Updated https://gitforwindows.org/git-snapshots (pending GitHub Pages deployment)" + - name: update check-run + if: env.CREATE_CHECK_RUN != 'false' + uses: ./.github/actions/check-run-action + with: + app-id: ${{ secrets.GH_APP_ID }} + private-key: ${{ secrets.GH_APP_PRIVATE_KEY }} + append-text: 'Updated https://gitforwindows.org/git-snapshots (pending GitHub Pages deployment)' + - name: mark check run as completed + if: env.CREATE_CHECK_RUN != 'false' && always() + uses: ./.github/actions/check-run-action + with: + app-id: ${{ secrets.GH_APP_ID }} + private-key: ${{ secrets.GH_APP_PRIVATE_KEY }} + append-text: "${{ job.status == 'success' && 'Done!' || format('Completed: {0}', job.status) }}." + conclusion: ${{ job.status }} diff --git a/github-release.js b/github-release.js index ad7c1989..41bf2662 100644 --- a/github-release.js +++ b/github-release.js @@ -349,4 +349,5 @@ module.exports = { pushGitTag, downloadReleaseAssets, downloadReleaseAssetsFromURL, + architectures, }