gitian: Post-process .a libraries for win to be deterministic

Author: laanwj

contrib/gitian-descriptors/boost-win.yml

@@ -20,6 +20,7 @@ script: |
   export FAKETIME=$REFERENCE_DATETIME
   export TZ=UTC
   INDIR=$HOME/build
+  TEMPDIR=$HOME/tmp
   # Input Integrity Check
   echo "fff00023dd79486d444c8e29922f4072e1d451fc5a4d2b6075852ead7f2b7b52  boost_1_55_0.tar.bz2" | shasum -c
   echo "d2b7f6a1d7051faef3c9cf41a92fa3671d905ef1e1da920d07651a43299f6268  boost-mingw-gas-cross-compile-2013-03-03.patch" | shasum -c
@@ -70,8 +71,16 @@ script: |
     # http://statmt.org/~s0565741/software/boost_1_52_0/libs/context/doc/html/context/requirements.html
     # "For cross-compiling the lib you must specify certain additional properties at bjam command line: target-os, abi, binary-format, architecture and address-model."
     ./bjam toolset=gcc binary-format=pe target-os=windows threadapi=win32 address-model=$BITS threading=multi variant=release link=static runtime-link=static --user-config=user-config.jam --without-mpi --without-python -sNO_BZIP2=1 -sNO_ZLIB=1 --layout=tagged --build-type=complete --prefix="$INSTALLPREFIX" $MAKEOPTS install
+    # post-process all generated libraries to be deterministic
+    # extract them to a temporary directory then re-build them deterministically
+    for LIB in $(find $INSTALLPREFIX -name \*.a); do
+        rm -rf $TEMPDIR && mkdir $TEMPDIR && cd $TEMPDIR
+        $HOST-ar xv $LIB | cut -b5- > /tmp/list.txt
+        rm $LIB
+        $HOST-ar crsD $LIB $(cat /tmp/list.txt)
+    done
     #
     cd "$INSTALLPREFIX"
-    find | sort | zip -@ $OUTDIR/boost-win$BITS-1.55.0-gitian-r6.zip
+    find | sort | zip -X@ $OUTDIR/boost-win$BITS-1.55.0-gitian-r6.zip
   done # for BITS in
 

contrib/gitian-descriptors/deps-win.yml

@@ -26,6 +26,7 @@ script: |
   export FAKETIME=$REFERENCE_DATETIME
   export TZ=UTC
   INDIR=$HOME/build
+  TEMPDIR=$HOME/tmp
   # Input Integrity Check
   echo "f74f15e8c8ff11aa3d5bb5f276d202ec18d7246e95f961db76054199c69c1ae3  openssl-1.0.1e.tar.gz"  | sha256sum -c
   echo "12edc0df75bf9abd7f82f821795bcee50f42cb2e5f76a6a281b85732798364ef  db-4.8.30.NC.tar.gz"    | sha256sum -c
@@ -113,7 +114,15 @@ script: |
     make
     make install
     cd ..
+    # post-process all generated libraries to be deterministic
+    # extract them to a temporary directory then re-build them deterministically
+    for LIB in $(find $INSTALLPREFIX -name \*.a); do
+        rm -rf $TEMPDIR && mkdir $TEMPDIR && cd $TEMPDIR
+        $HOST-ar xv $LIB | cut -b5- > /tmp/list.txt
+        rm $LIB
+        $HOST-ar crsD $LIB $(cat /tmp/list.txt)
+    done
     #
     cd $INSTALLPREFIX
-    find include lib | sort | zip -@ $OUTDIR/bitcoin-deps-win$BITS-gitian-r10.zip
+    find include lib | sort | zip -X@ $OUTDIR/bitcoin-deps-win$BITS-gitian-r10.zip
   done # for BITS in

contrib/gitian-descriptors/protobuf-win.yml

@@ -17,6 +17,7 @@ script: |
   #
   export TZ=UTC
   INDIR=$HOME/build
+  TEMPDIR=$HOME/tmp
   OPTFLAGS="-O2"
   # Integrity Check
   echo "13bfc5ae543cf3aa180ac2485c0bc89495e3ae711fc6fab4f8ffe90dfb4bb677  protobuf-2.5.0.tar.bz2" | sha256sum -c
@@ -48,9 +49,17 @@ script: |
     export FAKETIME=$REFERENCE_DATETIME
     make
     make install
+    # post-process all generated libraries to be deterministic
+    # extract them to a temporary directory then re-build them deterministically
+    for LIB in $(find $INSTALLPREFIX -name \*.a); do
+        rm -rf $TEMPDIR && mkdir $TEMPDIR && cd $TEMPDIR
+        $HOST-ar xv $LIB | cut -b5- > /tmp/list.txt
+        rm $LIB
+        $HOST-ar crsD $LIB $(cat /tmp/list.txt)
+    done
     #
     cd $INSTALLPREFIX
-    find include lib host | sort | zip -@ $OUTDIR/protobuf-win$BITS-2.5.0-gitian-r4.zip
+    find include lib host | sort | zip -X@ $OUTDIR/protobuf-win$BITS-2.5.0-gitian-r4.zip
     unset LD_PRELOAD
     unset FAKETIME
   done # for BITS in

contrib/gitian-descriptors/qt-win.yml

@@ -21,6 +21,7 @@ script: |
   # Defines
   export TZ=UTC
   INDIR=$HOME/build
+  TEMPDIR=$HOME/tmp
   # Qt: workaround for determinism in resource ordering
   #  Qt5's rcc uses a QHash to store the files for the resource.
   #  A security fix in QHash makes the ordering of keys to be different on every run
@@ -58,21 +59,34 @@ script: |
     sed 's/qt_instdate=`date +%Y-%m-%d`/qt_instdate=2011-01-30/' -i qtbase/configure
     sed --posix "s|QMAKE_CFLAGS            = -pipe -fno-keep-inline-dllexport|QMAKE_CFLAGS\t\t= -pipe -fno-keep-inline-dllexport -isystem /usr/$HOST/include/ -frandom-seed=qtbuild -I$DEPSDIR/include|" -i ${SPECFILE}
     sed --posix "s|QMAKE_LFLAGS            =|QMAKE_LFLAGS\t\t= -L$DEPSDIR/lib|" -i ${SPECFILE}
+    # Before we tried to pass arguments to ar (static linking) in using QMAKE_LIB, however
+    #  qt removes the arguments for ar and provides a script which makes it impossible to pass the determinism flag -
+    #  so rather than try to replace ar, post-process all libraries and plugins at the end.
+    #
     # Don't load faketime while compiling Qt, qmake will get stuck in nearly infinite loops
     #export LD_PRELOAD=/usr/lib/faketime/libfaketime.so.1
-    export FAKETIME=$REFERENCE_DATETIME
+    #export FAKETIME=$REFERENCE_DATETIME
     #
     # Compile static libraries, and use statically linked openssl (-openssl-linked):
     OPENSSL_LIBS="-L$DEPSDIR/lib -lssl -lcrypto -lgdi32" ./configure -prefix $INSTALLPREFIX -bindir $INSTALLPREFIX/host/bin -confirm-license -release -opensource -static -xplatform $SPECNAME -device-option CROSS_COMPILE="$HOST-" -no-audio-backend -no-javascript-jit -no-sql-sqlite -no-sql-odbc -no-nis -no-cups -no-iconv -no-dbus -no-gif -no-opengl -no-compile-examples -no-feature-style-windowsce -no-feature-style-windowsmobile -no-qml-debug -openssl-linked -skip qtsvg -skip qtwebkit -skip qtwebkit-examples -skip qtserialport -skip qtdeclarative -skip qtmultimedia -skip qtimageformats -skip qtlocation -skip qtsensors -skip qtquick1 -skip qtquickcontrols -skip qtactiveqt -skip qtconnectivity -skip qtwinextras -skip qtxmlpatterns -skip qtscript -skip qtdoc -system-libpng -system-zlib
     make $MAKEOPTS install
+    # post-process all generated libraries and plugins to be deterministic
+    # extract them to a temporary directory then re-build them deterministically
+    for LIB in $(find $INSTALLPREFIX -name *.a); do
+        rm -rf $TEMPDIR && mkdir $TEMPDIR && cd $TEMPDIR
+        $HOST-ar xv $LIB | cut -b5- > /tmp/list.txt
+        rm $LIB
+        $HOST-ar crsD $LIB $(cat /tmp/list.txt)
+    done
     #
     cd $INSTALLPREFIX
     # Remove unused non-deterministic stuff
     rm host/bin/qtpaths.exe lib/libQt5Bootstrap.a lib/libQt5Bootstrap.la
     # as zip stores file timestamps, use faketime to intercept stat calls to set dates for all files to reference date
     export LD_PRELOAD=/usr/lib/faketime/libfaketime.so.1
+    export FAKETIME=$REFERENCE_DATETIME
     find -print0 | xargs -r0 touch # fix up timestamps before packaging
-    find | sort | zip -@ $OUTDIR/qt-win${BITS}-5.2.0-gitian-r2.zip
+    find | sort | zip -X@ $OUTDIR/qt-win${BITS}-5.2.0-gitian-r2.zip
     unset LD_PRELOAD
     unset FAKETIME
   done # for BITS in