Skip to content

Commit d1a55d3

Browse files
dschodscho
committed
sdk-installer: switch to cloning the Git for Windows SDK
Previously, we went through an elaborate setup, designed to work around MSYS2's many assumptions that Git for Windows breaks. Things became really tricky when we switched to GPG-signing the Pacman packages, and MSYS2 really wanted to make sure that any custom GPG keys installed into Pacman's keyrings are blown away at every possible turn. And finally, everything really broke down when BinTray "rate limited" us (which in reality meant that we were blocked; nobody could download *anything* from our Pacman repository). Even switching to Azure Blobs caused problems because the current SDK installer still tries to access BinTray. So we started the work recently to make the Git mirror of the Git for Windows SDK (which is used regularly by the automated builds of Git for Windows) more widely usable. The idea was that the Git for Windows SDK can be cloned, and that is that. To save bandwidth and disk space, we can do it with a shallow clone. And to make it very convenient for the contributors, we can even change the SDK installer to do exactly that. Signed-off-by: Johannes Schindelin <[email protected]>
1 parent aeca4fa commit d1a55d3

File tree

2 files changed

+58
-225
lines changed

2 files changed

+58
-225
lines changed

‎sdk-installer/release.sh‎

Lines changed: 34 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
# Recreate git-sdk-$VERSION.exe
44

55
test -z "$1" && {
6-
echo "Usage: $0 <version> [<gitbranch>]"
6+
echo "Usage: $0 <version>"
77
exit 1
88
}
99

@@ -25,75 +25,58 @@ x86_64)
2525
;;
2626
esac
2727

28-
GIT_BRANCH="${2:-master}"
29-
GIT_CLONE_URL=https://github.com/git-for-windows/git
28+
GIT_SDK_URL=https://github.com/git-for-windows/git-sdk-$BITNESS
3029

3130
FAKEROOTDIR="$(cd "$(dirname "$0")" && pwd)/root"
3231
TARGET="$HOME"/git-sdk-installer-"$1"-$BITNESS.7z.exe
3332
OPTS7="-m0=lzma -mx=9 -md=64M"
3433
TMPPACK=/tmp.7z
3534
SCRIPT_PATH="$(cd "$(dirname "$0")" && pwd)"
35+
BIN_DIR=/mingw$BITNESS/bin
3636

37-
mkdir -p "$FAKEROOTDIR/usr/bin" "$FAKEROOTDIR/etc" ||
38-
die "Could not create fake root directory"
37+
echo "Enumerating required files..." >&2
38+
# First, enumerate the .dll files needed by the .exe files, then, enumerate all
39+
# the .dll files in bin/, then filter out the duplicates (which are the .dll
40+
# files in bin/ which are needed by the .exe files).
41+
exes_and_dlls=
42+
todo="git.exe ../libexec/git-core/git-remote-https.exe "
43+
# Add DLLs' transitive dependencies
44+
while test -n "$todo"
45+
do
46+
file=${todo%% *}
47+
todo=${todo#* }
48+
exes_and_dlls="$exes_and_dlls$file "
49+
50+
for dll in $(objdump -p "$BIN_DIR/$file" |
51+
sed -n "s|^\tDLL Name: ||p")
52+
do
53+
case " $exes_and_dlls $todo " in
54+
*" $dll "*) ;; # already found/queued
55+
*) test ! -f "$BIN_DIR/$dll" || todo="$todo$dll ";;
56+
esac
57+
done
58+
done
59+
60+
echo "Copying and compressing files..." >&2
61+
rm -rf "$FAKEROOTDIR" &&
62+
mkdir -p "$FAKEROOTDIR/mini$BIN_DIR" ||
63+
die "Could not create $FAKEROOTDIR$BIN_DIR directory"
3964

4065
sed -e "s|@@ARCH@@|$ARCH|g" \
4166
-e "s|@@BITNESS@@|$BITNESS|g" \
42-
-e "s|@@GIT_BRANCH@@|$GIT_BRANCH|g" \
43-
-e "s|@@GIT_CLONE_URL@@|$GIT_CLONE_URL|g" \
67+
-e "s|@@GIT_SDK_URL@@|$GIT_SDK_URL|g" \
4468
<"$SCRIPT_PATH"/setup-git-sdk.bat >"$FAKEROOTDIR"/setup-git-sdk.bat ||
4569
die "Could not generate setup script"
4670

47-
cp /usr/bin/dash.exe "$FAKEROOTDIR/usr/bin/sh.exe" &&
48-
sed -e 's/^#\(XferCommand.*curl\).*/\1 --anyauth -C - -L -f %u >%o/' \
49-
</etc/pacman.conf >"$FAKEROOTDIR/etc/pacman.conf.proxy" ||
50-
die "Could not copy extra files into fake root"
51-
52-
dlls_for_exes () {
53-
# Add DLLs' transitive dependencies
54-
dlls=
55-
todo="$* "
56-
while test -n "$todo"
57-
do
58-
path=${todo%% *}
59-
todo=${todo#* }
60-
case "$path" in ''|' ') continue;; esac
61-
for dll in $(objdump -p "$path" |
62-
sed -n 's/^\tDLL Name: msys-/usr\/bin\/msys-/p')
63-
do
64-
case "$dlls" in
65-
*"$dll"*) ;; # already found
66-
*) dlls="$dlls $dll"; todo="$todo /$dll ";;
67-
esac
68-
done
69-
done
70-
echo "$dlls"
71-
}
72-
73-
fileList="etc/nsswitch.conf \
74-
etc/pacman.conf \
75-
etc/pacman.d \
76-
usr/bin/pacman-key \
77-
usr/bin/tput.exe \
78-
usr/bin/pacman.exe \
79-
usr/bin/curl.exe \
80-
usr/bin/gpg.exe \
81-
usr/bin/chmod.exe \
82-
usr/bin/wc.exe \
83-
usr/bin/find.exe \
84-
$(dlls_for_exes /usr/bin/gpg.exe /usr/bin/curl.exe /usr/bin/chmod.exe \
85-
/usr/bin/wc.exe /usr/bin/find.exe) \
86-
usr/ssl/certs/ca-bundle.crt \
87-
var/lib/pacman \
88-
usr/share/pacman/keyrings \
89-
$FAKEROOTDIR/setup-git-sdk.bat $FAKEROOTDIR/etc $FAKEROOTDIR/usr"
71+
(cd $BIN_DIR && cp $exes_and_dlls "$FAKEROOTDIR/mini$BIN_DIR") ||
72+
die "Could not copy .exe and .dll files into fake root"
9073

9174
type 7za ||
9275
pacman -Sy --noconfirm p7zip ||
9376
die "Could not install 7-Zip"
9477

9578
echo "Creating archive" &&
96-
(cd / && 7za -x'!var/lib/pacman/*' a $OPTS7 "$TMPPACK" $fileList) &&
79+
(cd "$FAKEROOTDIR" && 7za -x'!var/lib/pacman/*' a $OPTS7 "$TMPPACK" *) &&
9780
(cat "$SCRIPT_PATH/../7-Zip/7zSD.sfx" &&
9881
echo ';!@Install@!UTF-8!' &&
9982
echo 'Title="Git for Windows '$BITNESS'-bit SDK"' &&

‎sdk-installer/setup-git-sdk.bat‎

Lines changed: 24 additions & 174 deletions
Original file line numberDiff line numberDiff line change
@@ -8,190 +8,40 @@
88
@FOR /F "delims=" %%D in ("%~dp0") do @set cwd=%%~fD
99

1010
@REM set PATH
11-
@set PATH=%cwd%\usr\bin;%PATH%
12-
13-
@REM set MSYSTEM so that MSYS2 starts up in the correct mode
14-
@set MSYSTEM=MINGW@@BITNESS@@
15-
16-
@REM need to rebase just to make sure that it still works even with Windows 10
17-
@SET initialrebase=false
18-
@FOR /F "tokens=4 delims=.[XP " %%i IN ('ver') DO @SET ver=%%i
19-
@IF 10 LEQ %ver% @(
20-
@SET initialrebase=true
21-
)
22-
23-
@IF %initialrebase% == true @(
24-
@REM we copy `rebase.exe` because it links to `msys-2.0.dll`.
25-
@IF NOT EXIST "%cwd%"\bin\rebase.exe @(
26-
@IF NOT EXIST "%cwd%"\bin @MKDIR "%cwd%"\bin
27-
@COPY "%cwd%"\usr\bin\rebase.exe "%cwd%"\bin\rebase.exe
28-
)
29-
@IF NOT EXIST bin\msys-2.0.dll @(
30-
@COPY usr\bin\msys-2.0.dll bin\msys-2.0.dll
31-
)
32-
@"%cwd%"\bin\rebase.exe -b 0x63000000 "%cwd%"\usr\bin\msys-2.0.dll
33-
)
34-
35-
@SET /A counter=0
36-
:INSTALL_RUNTIME
37-
@SET /A counter+=1
38-
@IF %counter% GEQ 5 @(
39-
@ECHO Could not install msys2-runtime
40-
@PAUSE
41-
@EXIT 1
42-
)
43-
44-
@REM Maybe we need a proxy?
45-
@IF %counter% GEQ 2 @(
46-
@ECHO.
47-
@ECHO There was a problem accessing the MSYS2 repositories
48-
@ECHO If your setup requires an HTTP proxy to access the web,
49-
@ECHO please specify it here, otherwise leave it empty.
50-
@ECHO.
51-
@SET /p proxy= "HTTP proxy: "
52-
)
53-
@REM Check the proxy variable here because of delayed expansion
54-
@IF NOT "%proxy%" == "" @(
55-
@SET http_proxy=%proxy%
56-
@SET https_proxy=%proxy%
57-
@IF %counter% EQU 2 @(
58-
@COPY "%cwd%"\etc\pacman.conf.proxy "%cwd%"\etc\pacman.conf
59-
)
60-
)
61-
62-
@REM update the Pacman package indices first, then force-install msys2-runtime
63-
@REM (we ship with a stripped-down msys2-runtime, gpg and pacman), so that
64-
@REM pacman's post-install scripts run without complaining about heap problems
65-
@"%cwd%"\usr\bin\pacman -Sy --needed --force --noconfirm msys2-runtime
66-
67-
@REM need to rebase just to make sure that it still works even with Windows 10
68-
@IF %initialrebase% == true @(
69-
@"%cwd%"\bin\rebase.exe -b 0x63000000 "%cwd%"\usr\bin\msys-2.0.dll
70-
)
71-
72-
@IF ERRORLEVEL 1 GOTO INSTALL_RUNTIME
73-
74-
@SET /A counter=0
75-
:INSTALL_BASH
76-
@SET /A counter+=1
77-
@IF %counter% GEQ 5 @(
78-
@ECHO Could not install bash
79-
@PAUSE
80-
@EXIT 1
81-
)
82-
83-
@REM next, force update bash
84-
@"%cwd%"\usr\bin\pacman -S --needed --force --noconfirm bash
85-
86-
@SET /A counter=0
87-
:INSTALL_INFO
88-
@SET /A counter+=1
89-
@IF %counter% GEQ 5 @(
90-
@ECHO Could not install info
91-
@PAUSE
92-
@EXIT 1
93-
)
94-
95-
@REM we need a /tmp directory, just for the time being
96-
@MKDIR "%cwd%"\tmp
97-
98-
@REM next, initialize pacman's keyring
99-
@"%cwd%"\usr\bin\bash.exe -l -c '/usr/bin/bash /usr/bin/pacman-key --init'
100-
@IF ERRORLEVEL 1 PAUSE
101-
102-
@REM next, force update info
103-
@"%cwd%"\usr\bin\pacman -S --needed --force --noconfirm info
104-
105-
@IF ERRORLEVEL 1 GOTO INSTALL_INFO
106-
107-
@SET /A counter=0
108-
:INSTALL_MSYS2_KEYRING
109-
@SET /A counter+=1
110-
@IF %counter% GEQ 5 @(
111-
@ECHO Could not install msys2-keyring
112-
@PAUSE
113-
@EXIT 1
114-
)
115-
116-
@REM next, force update msys2-keyring
117-
@"%cwd%"\usr\bin\pacman -S --needed --force --noconfirm gnupg msys2-keyring
118-
119-
@IF ERRORLEVEL 1 GOTO INSTALL_MSYS2_KEYRING
120-
121-
@REM now, add Git for Windows' keyring
122-
@"%cwd%"\usr\bin\bash.exe -l -c ^
123-
'/usr/bin/bash /usr/bin/pacman-key --populate git-for-windows'
124-
@IF ERRORLEVEL 1 PAUSE
125-
126-
@SET /A counter=0
127-
:INSTALL_PACMAN
128-
@SET /A counter+=1
129-
@IF %counter% GEQ 5 @(
130-
@ECHO Could not install pacman
131-
@PAUSE
132-
@EXIT 1
133-
)
134-
135-
@REM next, force update pacman
136-
@"%cwd%"\usr\bin\pacman -S --needed --force --noconfirm pacman
137-
138-
@IF ERRORLEVEL 1 GOTO INSTALL_PACMAN
139-
140-
@SET /A counter=0
141-
:INSTALL_REST
142-
@SET /A counter+=1
143-
@IF %counter% GEQ 5 @(
144-
@ECHO Could not install the remaining packages
145-
@PAUSE
146-
@EXIT 1
147-
)
148-
149-
@REM now update the rest
150-
@"%cwd%"\usr\bin\pacman -S --needed --force --noconfirm ^
151-
base python less openssh patch make tar diffutils ca-certificates ^
152-
git perl-Error perl perl-Authen-SASL perl-libwww perl-MIME-tools ^
153-
perl-Net-SMTP-SSL perl-TermReadKey dos2unix asciidoc xmlto ^
154-
subversion mintty vim git-extra p7zip markdown winpty ^
155-
mingw-w64-@@ARCH@@-git-doc-html ^
156-
mingw-w64-@@ARCH@@-git mingw-w64-@@ARCH@@-toolchain ^
157-
mingw-w64-@@ARCH@@-curl mingw-w64-@@ARCH@@-expat ^
158-
mingw-w64-@@ARCH@@-openssl mingw-w64-@@ARCH@@-tcl ^
159-
mingw-w64-@@ARCH@@-pcre mingw-w64-@@ARCH@@-connect ^
160-
git-flow ssh-pageant tig
161-
162-
@IF ERRORLEVEL 1 GOTO INSTALL_REST
11+
@set PATH=%cwd%\mini\mingw@@BITNESS@@\bin;%PATH%
12+
13+
@ECHO Cloning the Git for Windows SDK...
14+
@git init
15+
@IF ERRORLEVEL 1 GOTO DIE
16+
@git config http.sslbackend schannel
17+
@IF ERRORLEVEL 1 GOTO DIE
18+
@git remote add origin @@GIT_SDK_URL@@
19+
@IF ERRORLEVEL 1 GOTO DIE
20+
@git fetch --depth 1 origin
21+
@IF ERRORLEVEL 1 GOTO DIE
22+
@git checkout -t origin/master
23+
@IF ERRORLEVEL 1 GOTO DIE
24+
25+
@REM Cleaning up temporary git.exe
26+
@RMDIR /Q /S %cwd%\mini
27+
@IF ERRORLEVEL 1 GOTO DIE
16328

16429
@REM Avoid overlapping address ranges
165-
@IF MINGW32 == %MSYSTEM% @(
30+
@IF 32 == @@BITNESS@@ @(
16631
ECHO Auto-rebasing .dll files
16732
CALL "%cwd%"\autorebase.bat
16833
)
16934

170-
@REM If an HTTP proxy is required, configure it for Git Bash sessions,
171-
@REM but only if the environment variable was not already set globally
172-
@IF DEFINED proxy @(
173-
@ECHO http_proxy=%proxy% > etc\profile.d\proxy.sh
174-
@ECHO https_proxy=%proxy% >> etc\profile.d\proxy.sh
175-
@ECHO export http_proxy https_proxy >> etc\profile.d\proxy.sh
176-
@ECHO.
177-
@ECHO Installed /etc/profile.d/proxy.sh to set proxy in Git Bash
178-
)
179-
18035
@REM Before running a shell, let's prevent complaints about "permission denied"
18136
@REM from MSYS2's /etc/post-install/01-devices.post
18237
@MKDIR "%cwd%"\dev\shm 2> NUL
18338
@MKDIR "%cwd%"\dev\mqueue 2> NUL
18439

185-
@IF NOT DEFINED JENKINS_URL @(
186-
@REM Install shortcut on the desktop
187-
@ECHO.
188-
@ECHO Installing the 'Git SDK @@BITNESS@@-bit' shortcut on the Desktop
189-
@bash --login -c 'SHORTCUT="$HOME/Desktop/Git SDK @@BITNESS@@-bit.lnk"; test -f "$SHORTCUT" ^|^| create-shortcut.exe --icon-file /msys2.ico --work-dir / /git-bash.exe "$SHORTCUT"'
40+
@START /B %cwd%\git-bash.exe
41+
@EXIT /B 0
19042

191-
@REM now clone the Git sources, build it, and start an interactive shell
192-
@bash --login -c "mkdir -p /usr/src && cd /usr/src && for project in MINGW-packages MSYS2-packages build-extra; do test ! -d $project && mkdir -p $project && (cd $project && git init && git config core.autocrlf false && git remote add origin https://github.com/git-for-windows/$project); done; if test ! -d git; then git clone -b @@GIT_BRANCH@@ -c core.autocrlf=false --depth 1 https://github.com/git-for-windows/git; fi && cd git && make install"
43+
:DIE
44+
@ECHO Installation of Git for Windows' SDK failed!
45+
@PAUSE
46+
@EXIT /B 1
19347

194-
@IF ERRORLEVEL 1 PAUSE
195-
196-
@start mintty -i /msys2.ico -t "Git SDK @@BITNESS@@-bit" bash --login -i
197-
)

0 commit comments

Comments
 (0)