Replace .zip with a self-extracting 7-zip (#87)

Cuts the distribution size by around 60%, and unpacks around 100x faster
than Windows built-in .zip support. However, 7-zip is the worst packaged
project included so far. The source release is a tarbomb with a URL that
contains two variations of the release version. It is not designed for
cross-compilation, and so requires minor source corrections and a custom
build script. Still, considering the circumstances, it's surprising how
easily it compiles.

Author: skeeto

Dockerfile

@@ -2,6 +2,7 @@ FROM debian:bookworm-slim
 
 ARG VERSION=1.23.0
 ARG PREFIX=/w64devkit
+ARG Z7_VERSION=2301
 ARG BINUTILS_VERSION=2.42
 ARG BUSYBOX_VERSION=FRP-5398-g89ae34445
 ARG CTAGS_VERSION=6.0.0
@@ -18,11 +19,12 @@ ARG PDCURSES_VERSION=3.9
 ARG VIM_VERSION=9.0
 
 RUN apt-get update && apt-get install --yes --no-install-recommends \
-  build-essential curl libgmp-dev libmpc-dev libmpfr-dev m4 zip
+  build-essential curl libgmp-dev libmpc-dev libmpfr-dev m4 p7zip-full
 
 # Download, verify, and unpack
 
 RUN curl --insecure --location --remote-name-all --remote-header-name \
+    https://downloads.sourceforge.net/project/sevenzip/7-Zip/23.01/7z$Z7_VERSION-src.tar.xz \
     https://ftp.gnu.org/gnu/binutils/binutils-$BINUTILS_VERSION.tar.xz \
     https://ftp.gnu.org/gnu/gcc/gcc-$GCC_VERSION/gcc-$GCC_VERSION.tar.xz \
     https://ftp.gnu.org/gnu/gdb/gdb-$GDB_VERSION.tar.xz \
@@ -39,6 +41,7 @@ RUN curl --insecure --location --remote-name-all --remote-header-name \
     https://downloads.sourceforge.net/project/pdcurses/pdcurses/$PDCURSES_VERSION/PDCurses-$PDCURSES_VERSION.tar.gz
 COPY src/SHA256SUMS $PREFIX/src/
 RUN sha256sum -c $PREFIX/src/SHA256SUMS \
+ && tar xJf 7z$Z7_VERSION-src.tar.xz --xform 's%^%7z/%' \
  && tar xJf binutils-$BINUTILS_VERSION.tar.xz \
  && tar xzf busybox-w32-$BUSYBOX_VERSION.tgz \
  && tar xzf ctags-$CTAGS_VERSION.tar.gz \
@@ -446,6 +449,13 @@ RUN sed -i /RT_MANIFEST/d win32/ctags.rc \
         OPT= CFLAGS=-Os LDFLAGS=-s \
  && cp ctags.exe $PREFIX/bin/
 
+WORKDIR /7z
+COPY src/7z.mak $PREFIX/src/
+RUN sed -i s/CommCtrl/commctrl/ $(grep -Rl CommCtrl CPP/) \
+ && sed -i s%7z\\.ico%$PREFIX/src/w64devkit.ico% \
+           CPP/7zip/Bundles/SFXWin/resource.rc \
+ && make -f $PREFIX/src/7z.mak -j$(nproc) CROSS=$ARCH-
+
 # Pack up a release
 
 WORKDIR /
@@ -486,6 +496,7 @@ RUN printf "id ICON \"$PREFIX/src/w64devkit.ico\"" >w64devkit.rc \
         >>$PREFIX/COPYING.MinGW-w64-runtime.txt . \
  && cat /mingw-w64-v$MINGW_VERSION/mingw-w64-libraries/winpthreads/COPYING \
         >>$PREFIX/COPYING.MinGW-w64-runtime.txt \
- && echo $VERSION >$PREFIX/VERSION.txt
+ && echo $VERSION >$PREFIX/VERSION.txt \
+ && 7z a -mx=9 $PREFIX.7z $PREFIX
 ENV PREFIX=${PREFIX}
-CMD zip -q9Xr - $PREFIX
+CMD cat /7z/7z.sfx $PREFIX.7z

README.md

@@ -20,23 +20,23 @@ the kit itself.
 
 ## Build
 
-First build the image, then run it to produce a distribution .zip file:
+Build the image, then run it to produce a self-extracting 7z archive:
 
     docker build -t w64devkit .
-    docker run --rm w64devkit >w64devkit.zip
+    docker run --rm w64devkit >w64devkit-x64.exe
 
-This takes about half an hour on modern systems. You will need an internet
+This takes about 15 minutes on modern systems. You will need an internet
 connection during the first few minutes of the build. **Note:** Do not use
 PowerShell because it lacks file redirection.
 
 ## Usage
 
-The final .zip file contains tools in a typical unix-like configuration.
-Unzip the contents anywhere. Inside is `w64devkit.exe`, which launches a
-console window with the environment configured and ready to go. It is the
-easiest way to enter the development environment, and requires no system
-changes. It also sets two extra environment variables: `W64DEVKIT_HOME` to
-the installation root and `W64DEVKIT` to the version.
+The self-extracting 7z archive contains tools in a typical unix-like
+configuration. Extract wherever is convenient. Inside is `w64devkit.exe`,
+which launches a console window with the environment configured and ready
+to go. It is the easiest way to enter the development environment, and
+requires no system changes. It also sets two extra environment variables:
+`W64DEVKIT_HOME` to the installation root and `W64DEVKIT` to the version.
 
 Alternatively, add the `bin/` directory to your path. For example, inside
 a `cmd.exe` console or batch script:
@@ -176,12 +176,6 @@ ported to Mingw-w64][san] ([also][san2]), but Undefined Behavior Sanitizer
 `-fsanitize-trap`, GDB will [break precisely][break] on undefined
 behavior, and it does not require linking with libsanitizer.
 
-Since the build environment is so stable and predicable, it would be
-great for the .zip to be reproducible, i.e. builds by different people
-are bit-for-bit identical. There are multiple reasons why this is not
-currently the case, the least of which are [timestamps in the .zip
-file][zip].
-
 ## Licenses
 
 When distributing binaries built using w64devkit, your .exe will include
@@ -225,4 +219,3 @@ binaries.
 [vim]: https://www.vim.org/
 [w64]: http://mingw-w64.org/
 [w64devkit]: https://github.com/skeeto/w64devkit
-[zip]: https://tanzu.vmware.com/content/blog/barriers-to-deterministic-reproducible-zip-files

multibuild.sh

@@ -9,16 +9,15 @@
 
 set -e
 arch=""
-compact=no
 dryrun=
 flavors=""
 suffix="$(git describe --exact-match 2>/dev/null | tr v - || true)"
 
 usage() {
     cat <<EOF
 usage: multibuild.sh [-48abfhnOs] [-s SUFFIX]
-  -4         Enable i686 build (default: no)
-  -8         Enable x86_64 build (default: auto)
+  -4         Enable x86 build (default: no)
+  -8         Enable x64 build (default: auto)
   -a         All: Enable all builds
   -h         Print this help message
   -n         Dry run, print commands but do nothing
@@ -29,12 +28,11 @@ EOF
 
 while getopts 48abfhmnOs: opt; do
     case $opt in
-        4) arch="$arch w64devkit-i686";;
-        8) arch="$arch w64devkit";;
-        a) flavors="X"; arch="w64devkit w64devkit-i686";;
+        4) arch="$arch w64devkit-x86";;
+        8) arch="$arch w64devkit-x64";;
+        a) flavors="X"; arch="w64devkit-x64 w64devkit-x86";;
         h) usage; exit 0;;
         n) dryrun=echo;;
-        O) compact=yes;;
         s) suffix="$OPTARG";;
         ?) usage >&2; exit 1;;
     esac
@@ -48,7 +46,7 @@ if [ $# -gt 0 ]; then
 fi
 
 if [ -z "$arch" ]; then
-    arch="w64devkit"
+    arch="w64devkit-x64"
 fi
 if [ -z "$flavors" ]; then
     flavors="X"
@@ -76,20 +74,17 @@ for build in $builds; do
         IFS=-
         set $build; shift
         for flavor in "$@"; do
-            $dryrun patch -p1 -i src/variant-$flavor.patch
+            if [ -e src/variant-$flavor.patch ]; then
+                $dryrun patch -p1 -i src/variant-$flavor.patch
+            fi
         done
     )
     $dryrun docker build -t $target .
     if [ -n "$dryrun" ]; then
-        $dryrun docker run --rm $target ">$build$suffix.zip"
+        $dryrun docker run --rm $target ">$build$suffix.exe"
     else
-        docker run --rm $target >$build$suffix.zip
+        docker run --rm $target >$build$suffix.exe
     fi
 done
 
-if [ $compact = yes ]; then
-    printf "%s$suffix.zip\n" $builds \
-        | xargs -I{} -P$(nproc) $dryrun advzip -z4 {}
-fi
-
 cleanup

src/7z.mak

@@ -0,0 +1,132 @@
+CROSS   = x86_64-w64-mingw32-
+CC      = $(CROSS)gcc
+CXX     = $(CROSS)g++
+WINDRES = $(CROSS)windres
+LDFLAGS = -mwindows -s -Wl,--gc-sections
+LDLIBS  = -lcomdlg32 -lole32 -loleaut32 -luuid
+CFLAGS  = -fno-ident -Oz \
+  -DZ7_SFX \
+  -DZ7_EXTRACT_ONLY \
+  -DZ7_NO_CRYPTO \
+  -DZ7_NO_REGISTRY \
+  -DZ7_NO_READ_FROM_CODER \
+
+obj = \
+  CPP/7zip/Bundles/SFXWin/resource.o \
+  CPP/7zip/Bundles/SFXWin/SfxWin.o \
+  CPP/7zip/UI/GUI/ExtractDialog.o \
+  CPP/7zip/UI/GUI/ExtractGUI.o \
+  CPP/Common/CRC.o \
+  CPP/Common/CommandLineParser.o \
+  CPP/Common/IntToString.o \
+  CPP/Common/NewHandler.o \
+  CPP/Common/MyString.o \
+  CPP/Common/StringConvert.o \
+  CPP/Common/MyVector.o \
+  CPP/Common/Wildcard.o \
+  CPP/Windows/Clipboard.o \
+  CPP/Windows/CommonDialog.o \
+  CPP/Windows/DLL.o \
+  CPP/Windows/ErrorMsg.o \
+  CPP/Windows/FileDir.o \
+  CPP/Windows/FileFind.o \
+  CPP/Windows/FileIO.o \
+  CPP/Windows/FileName.o \
+  CPP/Windows/MemoryGlobal.o \
+  CPP/Windows/PropVariant.o \
+  CPP/Windows/PropVariantConv.o \
+  CPP/Windows/ResourceString.o \
+  CPP/Windows/Shell.o \
+  CPP/Windows/Synchronization.o \
+  CPP/Windows/System.o \
+  CPP/Windows/Window.o \
+  CPP/Windows/Control/ComboBox.o \
+  CPP/Windows/Control/Dialog.o \
+  CPP/Windows/Control/ListView.o \
+  CPP/7zip/Common/CreateCoder.o \
+  CPP/7zip/Common/CWrappers.o \
+  CPP/7zip/Common/FilePathAutoRename.o \
+  CPP/7zip/Common/FileStreams.o \
+  CPP/7zip/Common/InBuffer.o \
+  CPP/7zip/Common/FilterCoder.o \
+  CPP/7zip/Common/LimitedStreams.o \
+  CPP/7zip/Common/OutBuffer.o \
+  CPP/7zip/Common/ProgressUtils.o \
+  CPP/7zip/Common/PropId.o \
+  CPP/7zip/Common/StreamBinder.o \
+  CPP/7zip/Common/StreamObjects.o \
+  CPP/7zip/Common/StreamUtils.o \
+  CPP/7zip/Common/VirtThread.o \
+  CPP/7zip/UI/Common/ArchiveExtractCallback.o \
+  CPP/7zip/UI/Common/ArchiveOpenCallback.o \
+  CPP/7zip/UI/Common/DefaultName.o \
+  CPP/7zip/UI/Common/Extract.o \
+  CPP/7zip/UI/Common/ExtractingFilePath.o \
+  CPP/7zip/UI/Common/LoadCodecs.o \
+  CPP/7zip/UI/Common/OpenArchive.o \
+  CPP/7zip/UI/Explorer/MyMessages.o \
+  CPP/7zip/UI/FileManager/BrowseDialog.o \
+  CPP/7zip/UI/FileManager/ComboDialog.o \
+  CPP/7zip/UI/FileManager/ExtractCallback.o \
+  CPP/7zip/UI/FileManager/FormatUtils.o \
+  CPP/7zip/UI/FileManager/OverwriteDialog.o \
+  CPP/7zip/UI/FileManager/PasswordDialog.o \
+  CPP/7zip/UI/FileManager/ProgressDialog2.o \
+  CPP/7zip/UI/FileManager/PropertyName.o \
+  CPP/7zip/UI/FileManager/SysIconUtils.o \
+  CPP/7zip/Archive/SplitHandler.o \
+  CPP/7zip/Archive/Common/CoderMixer2.o \
+  CPP/7zip/Archive/Common/ItemNameUtils.o \
+  CPP/7zip/Archive/Common/MultiStream.o \
+  CPP/7zip/Archive/Common/OutStreamWithCRC.o \
+  CPP/7zip/Archive/7z/7zDecode.o \
+  CPP/7zip/Archive/7z/7zExtract.o \
+  CPP/7zip/Archive/7z/7zHandler.o \
+  CPP/7zip/Archive/7z/7zIn.o \
+  CPP/7zip/Archive/7z/7zRegister.o \
+  CPP/7zip/Compress/Bcj2Coder.o \
+  CPP/7zip/Compress/Bcj2Register.o \
+  CPP/7zip/Compress/BcjCoder.o \
+  CPP/7zip/Compress/BcjRegister.o \
+  CPP/7zip/Compress/BranchMisc.o \
+  CPP/7zip/Compress/BranchRegister.o \
+  CPP/7zip/Compress/CopyCoder.o \
+  CPP/7zip/Compress/CopyRegister.o \
+  CPP/7zip/Compress/DeltaFilter.o \
+  CPP/7zip/Compress/Lzma2Decoder.o \
+  CPP/7zip/Compress/Lzma2Register.o \
+  CPP/7zip/Compress/LzmaDecoder.o \
+  CPP/7zip/Compress/LzmaRegister.o \
+  CPP/7zip/Compress/PpmdDecoder.o \
+  CPP/7zip/Compress/PpmdRegister.o \
+  C/7zCrc.o \
+  C/7zCrcOpt.o \
+  C/7zStream.o \
+  C/Alloc.o \
+  C/Bcj2.o \
+  C/Bra.o \
+  C/Bra86.o \
+  C/BraIA64.o \
+  C/CpuArch.o \
+  C/Delta.o \
+  C/DllSecur.o \
+  C/Lzma2Dec.o \
+  C/Lzma2DecMt.o \
+  C/LzmaDec.o \
+  C/MtDec.o \
+  C/Ppmd7.o \
+  C/Ppmd7Dec.o \
+  C/Sha256.o \
+  C/Sha256Opt.o \
+  C/Threads.o \
+
+7z.sfx: $(obj)
+	$(CXX) $(LDFLAGS) -o $@ $(obj) $(LDLIBS)
+clean:
+	rm -f 7z.sfx $(obj)
+%.o: %.cpp
+	$(CXX) -c $(CFLAGS) -o $@ $^
+%.o: %.c
+	$(CC) -c $(CFLAGS) -o $@ $^
+%.o: %.rc
+	$(WINDRES) -o $@ $^

src/SHA256SUMS

@@ -1,3 +1,4 @@
+356071007360e5a1824d9904993e8b2480b51b570e8c9faf7c0f58ebe4bf9f74  7z2301-src.tar.xz
 f6e4d41fd5fc778b06b7891457b3620da5ecea1006c6a4a41ae998109f85a800  binutils-2.42.tar.xz
 e366f887a56bf6f64adff15b7b08ba487fea14c121045078437abe7217cf060d  busybox-w32-FRP-5398-g89ae34445.tgz
 71229a73f25529c9e3dabb2cb7310c55405d31caee8e8a9ab5c71b2406d4005a  ctags-6.0.0.tar.gz