Skip to content

Commit 3e3c25d

Browse files
committed
Merge pull request #3625
4ce9106 gitian: sort generated source distribution archive for windows (Wladimir J. van der Laan) 6b55e6b gitian: Post-process .a libraries for win to be deterministic (Wladimir J. van der Laan) c43d1e5 gitian: Make windows dependencies outputs fully deterministic (Wladimir J. van der Laan)
2 parents b19bcb7 + 4ce9106 commit 3e3c25d

File tree

5 files changed

+70
-31
lines changed

5 files changed

+70
-31
lines changed

contrib/gitian-descriptors/boost-win.yml

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,11 @@ files:
1616
- "boost-mingw-gas-cross-compile-2013-03-03.patch"
1717
script: |
1818
# Defines
19+
export LD_PRELOAD=/usr/lib/faketime/libfaketime.so.1
20+
export FAKETIME=$REFERENCE_DATETIME
1921
export TZ=UTC
2022
INDIR=$HOME/build
23+
TEMPDIR=$HOME/tmp
2124
# Input Integrity Check
2225
echo "fff00023dd79486d444c8e29922f4072e1d451fc5a4d2b6075852ead7f2b7b52 boost_1_55_0.tar.bz2" | shasum -c
2326
echo "d2b7f6a1d7051faef3c9cf41a92fa3671d905ef1e1da920d07651a43299f6268 boost-mingw-gas-cross-compile-2013-03-03.patch" | shasum -c
@@ -35,7 +38,7 @@ script: |
3538
mkdir -p $INSTALLPREFIX $BUILDDIR
3639
cd $BUILDDIR
3740
#
38-
tar xjf $INDIR/boost_1_55_0.tar.bz2
41+
tar --warning=no-timestamp -xjf $INDIR/boost_1_55_0.tar.bz2
3942
cd boost_1_55_0
4043
GCCVERSION=$($HOST-g++ -E -dM $(mktemp --suffix=.h) | grep __VERSION__ | cut -d ' ' -f 3 | cut -d '"' -f 2)
4144
echo "using gcc : $GCCVERSION : $HOST-g++
@@ -68,12 +71,16 @@ script: |
6871
# http://statmt.org/~s0565741/software/boost_1_52_0/libs/context/doc/html/context/requirements.html
6972
# "For cross-compiling the lib you must specify certain additional properties at bjam command line: target-os, abi, binary-format, architecture and address-model."
7073
./bjam toolset=gcc binary-format=pe target-os=windows threadapi=win32 address-model=$BITS threading=multi variant=release link=static runtime-link=static --user-config=user-config.jam --without-mpi --without-python -sNO_BZIP2=1 -sNO_ZLIB=1 --layout=tagged --build-type=complete --prefix="$INSTALLPREFIX" $MAKEOPTS install
74+
# post-process all generated libraries to be deterministic
75+
# extract them to a temporary directory then re-build them deterministically
76+
for LIB in $(find $INSTALLPREFIX -name \*.a); do
77+
rm -rf $TEMPDIR && mkdir $TEMPDIR && cd $TEMPDIR
78+
$HOST-ar xv $LIB | cut -b5- > /tmp/list.txt
79+
rm $LIB
80+
$HOST-ar crsD $LIB $(cat /tmp/list.txt)
81+
done
7182
#
7283
cd "$INSTALLPREFIX"
73-
export LD_PRELOAD=/usr/lib/faketime/libfaketime.so.1
74-
export FAKETIME=$REFERENCE_DATETIME
75-
zip -r $OUTDIR/boost-win$BITS-1.55.0-gitian-r6.zip *
76-
unset LD_PRELOAD
77-
unset FAKETIME
84+
find | sort | zip -X@ $OUTDIR/boost-win$BITS-1.55.0-gitian-r6.zip
7885
done # for BITS in
7986

contrib/gitian-descriptors/deps-win.yml

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ script: |
2626
export FAKETIME=$REFERENCE_DATETIME
2727
export TZ=UTC
2828
INDIR=$HOME/build
29+
TEMPDIR=$HOME/tmp
2930
# Input Integrity Check
3031
echo "f74f15e8c8ff11aa3d5bb5f276d202ec18d7246e95f961db76054199c69c1ae3 openssl-1.0.1e.tar.gz" | sha256sum -c
3132
echo "12edc0df75bf9abd7f82f821795bcee50f42cb2e5f76a6a281b85732798364ef db-4.8.30.NC.tar.gz" | sha256sum -c
@@ -107,11 +108,21 @@ script: |
107108
#
108109
tar xjf $INDIR/qrencode-3.4.3.tar.bz2
109110
cd qrencode-3.4.3
110-
png_CFLAGS="-I$INSTALLPREFIX/include" png_LIBS="-L$INSTALLPREFIX/lib -lpng" ./configure --prefix=$INSTALLPREFIX --host=$HOST --enable-static --disable-shared --without-tools --disable-maintainer-mode --disable-dependency-tracking
111+
png_CFLAGS="-I$INSTALLPREFIX/include" png_LIBS="-L$INSTALLPREFIX/lib -lpng" ./configure --prefix=$INSTALLPREFIX --host=$HOST --enable-static --disable-shared --without-tools --disable-dependency-tracking
112+
# Workaround to prevent re-configuring by make (resulting in missing m4 error); make all files have a date in the past
113+
find . -print0 | xargs -r0 touch -t 200001010000
111114
make
112115
make install
113116
cd ..
117+
# post-process all generated libraries to be deterministic
118+
# extract them to a temporary directory then re-build them deterministically
119+
for LIB in $(find $INSTALLPREFIX -name \*.a); do
120+
rm -rf $TEMPDIR && mkdir $TEMPDIR && cd $TEMPDIR
121+
$HOST-ar xv $LIB | cut -b5- > /tmp/list.txt
122+
rm $LIB
123+
$HOST-ar crsD $LIB $(cat /tmp/list.txt)
124+
done
114125
#
115126
cd $INSTALLPREFIX
116-
zip -r $OUTDIR/bitcoin-deps-win$BITS-gitian-r10.zip include lib
127+
find include lib | sort | zip -X@ $OUTDIR/bitcoin-deps-win$BITS-gitian-r10.zip
117128
done # for BITS in

contrib/gitian-descriptors/gitian-win.yml

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ script: |
3535
export TZ=UTC
3636
INDIR=$HOME/build
3737
OPTFLAGS='-O2'
38+
TEMPDIR="$HOME/tempdir"
3839
NEEDDIST=1
3940
# Qt: workaround for determinism in resource ordering
4041
# Qt5's rcc uses a QHash to store the files for the resource.
@@ -54,6 +55,7 @@ script: |
5455
else
5556
HOST=x86_64-w64-mingw32
5657
fi
58+
export PATH=$STAGING/host/bin:$PATH
5759
mkdir -p $STAGING $BUILDDIR $BINDIR
5860
#
5961
cd $STAGING
@@ -63,22 +65,18 @@ script: |
6365
unzip $INDIR/protobuf-win${BITS}-2.5.0-gitian-r4.zip
6466
if [ "$NEEDDIST" == "1" ]; then
6567
# Make source code archive which is architecture independent so it only needs to be done once
66-
cd $HOME/build/
67-
cd bitcoin
68-
export PATH=$STAGING/host/bin:$PATH
69-
export TAR_OPTIONS=--mtime=`echo $REFERENCE_DATETIME | awk '{ print $1 }'`
68+
cd $HOME/build/bitcoin
7069
./autogen.sh
7170
./configure --bindir=$OUTDIR --prefix=$STAGING --host=$HOST --with-qt-plugindir=$STAGING/plugins --with-qt-incdir=$STAGING/include --with-qt-bindir=$STAGING/host/bin --with-boost=$STAGING --disable-maintainer-mode --with-protoc-bindir=$STAGING/host/bin --disable-dependency-tracking CPPFLAGS="-I$STAGING/include ${OPTFLAGS}" LDFLAGS="-L$STAGING/lib ${OPTFLAGS}" CXXFLAGS="-frandom-seed=bitcoin ${OPTFLAGS}"
7271
make dist
73-
mkdir -p $OUTDIR/src
74-
cp -f bitcoin-*.tar.* $OUTDIR/src
72+
DISTNAME=`echo bitcoin-*.tar.gz`
7573
NEEDDIST=0
7674
fi
7775
# Build platform-dependent executables from source archive
7876
cd $BUILDDIR
7977
mkdir -p distsrc
8078
cd distsrc
81-
tar --strip-components=1 -xf $HOME/build/bitcoin/bitcoin-*.tar.*
79+
tar --strip-components=1 -xf $HOME/build/bitcoin/$DISTNAME
8280
./configure --bindir=$BINDIR --prefix=$STAGING --host=$HOST --with-qt-plugindir=$STAGING/plugins --with-qt-incdir=$STAGING/include --with-qt-bindir=$STAGING/host/bin --with-boost=$STAGING --disable-maintainer-mode --with-protoc-bindir=$STAGING/host/bin --disable-dependency-tracking CPPFLAGS="-I$STAGING/include ${OPTFLAGS}" LDFLAGS="-L$STAGING/lib ${OPTFLAGS}" CXXFLAGS="-frandom-seed=bitcoin ${OPTFLAGS}"
8381
export LD_PRELOAD=/usr/lib/faketime/libfaketime.so.1
8482
export FAKETIME=$REFERENCE_DATETIME
@@ -90,3 +88,10 @@ script: |
9088
unset FAKETIME
9189
done # for BITS in
9290
91+
# sort distribution tar file and normalize user/group/mtime information for deterministic output
92+
mkdir -p $OUTDIR/src
93+
rm -rf $TEMPDIR
94+
mkdir -p $TEMPDIR
95+
cd $TEMPDIR
96+
tar -xvf $HOME/build/bitcoin/$DISTNAME | sort | tar --no-recursion -cT /dev/stdin --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 --mtime="$REFERENCE_DATETIME" | gzip -n > $OUTDIR/src/$DISTNAME
97+

contrib/gitian-descriptors/protobuf-win.yml

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ script: |
1717
#
1818
export TZ=UTC
1919
INDIR=$HOME/build
20+
TEMPDIR=$HOME/tmp
2021
OPTFLAGS="-O2"
2122
# Integrity Check
2223
echo "13bfc5ae543cf3aa180ac2485c0bc89495e3ae711fc6fab4f8ffe90dfb4bb677 protobuf-2.5.0.tar.bz2" | sha256sum -c
@@ -44,13 +45,21 @@ script: |
4445
# Now recompile with the mingw cross-compiler:
4546
make distclean
4647
./configure --prefix=$INSTALLPREFIX --enable-shared=no --disable-dependency-tracking --with-protoc=$INSTALLPREFIX/host/bin/protoc --host=$HOST CXXFLAGS="-frandom-seed=11 ${OPTFLAGS}"
48+
export LD_PRELOAD=/usr/lib/faketime/libfaketime.so.1
49+
export FAKETIME=$REFERENCE_DATETIME
4750
make
4851
make install
52+
# post-process all generated libraries to be deterministic
53+
# extract them to a temporary directory then re-build them deterministically
54+
for LIB in $(find $INSTALLPREFIX -name \*.a); do
55+
rm -rf $TEMPDIR && mkdir $TEMPDIR && cd $TEMPDIR
56+
$HOST-ar xv $LIB | cut -b5- > /tmp/list.txt
57+
rm $LIB
58+
$HOST-ar crsD $LIB $(cat /tmp/list.txt)
59+
done
4960
#
5061
cd $INSTALLPREFIX
51-
export LD_PRELOAD=/usr/lib/faketime/libfaketime.so.1
52-
export FAKETIME=$REFERENCE_DATETIME
53-
zip -r $OUTDIR/protobuf-win$BITS-2.5.0-gitian-r4.zip include lib host
62+
find include lib host | sort | zip -X@ $OUTDIR/protobuf-win$BITS-2.5.0-gitian-r4.zip
5463
unset LD_PRELOAD
5564
unset FAKETIME
5665
done # for BITS in

contrib/gitian-descriptors/qt-win.yml

Lines changed: 20 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ script: |
2121
# Defines
2222
export TZ=UTC
2323
INDIR=$HOME/build
24+
TEMPDIR=$HOME/tmp
2425
# Qt: workaround for determinism in resource ordering
2526
# Qt5's rcc uses a QHash to store the files for the resource.
2627
# A security fix in QHash makes the ordering of keys to be different on every run
@@ -55,31 +56,37 @@ script: |
5556
cd qt-everywhere-opensource-src-5.2.0
5657
SPECNAME="win32-g++"
5758
SPECFILE="qtbase/mkspecs/${SPECNAME}/qmake.conf"
58-
sed 's/$TODAY/2011-01-30/' -i configure
59+
sed 's/qt_instdate=`date +%Y-%m-%d`/qt_instdate=2011-01-30/' -i qtbase/configure
5960
sed --posix "s|QMAKE_CFLAGS = -pipe -fno-keep-inline-dllexport|QMAKE_CFLAGS\t\t= -pipe -fno-keep-inline-dllexport -isystem /usr/$HOST/include/ -frandom-seed=qtbuild -I$DEPSDIR/include|" -i ${SPECFILE}
6061
sed --posix "s|QMAKE_LFLAGS =|QMAKE_LFLAGS\t\t= -L$DEPSDIR/lib|" -i ${SPECFILE}
61-
# ar adds timestamps to every object file included in the static library
62-
# providing -D as ar argument is supposed to solve it, but doesn't work as qmake strips off the arguments and adds -M to pass a script...
63-
# which somehow cannot be combined with other flags.
64-
# use faketime only for ar, as it confuses make/qmake into hanging sometimes
65-
sed --posix "s|QMAKE_LIB = \\\$\\\${CROSS_COMPILE}ar -ru|QMAKE_LIB\t\t= $HOME/ar -Dr|" -i ${SPECFILE}
66-
echo '#!/bin/bash' > $HOME/ar
67-
echo 'export LD_PRELOAD=/usr/lib/faketime/libfaketime.so.1' >> $HOME/ar
68-
echo "$HOST-ar \"\$@\"" >> $HOME/ar
69-
chmod +x $HOME/ar
62+
# Before we tried to pass arguments to ar (static linking) in using QMAKE_LIB, however
63+
# qt removes the arguments for ar and provides a script which makes it impossible to pass the determinism flag -
64+
# so rather than try to replace ar, post-process all libraries and plugins at the end.
65+
#
7066
# Don't load faketime while compiling Qt, qmake will get stuck in nearly infinite loops
7167
#export LD_PRELOAD=/usr/lib/faketime/libfaketime.so.1
72-
export FAKETIME=$REFERENCE_DATETIME
68+
#export FAKETIME=$REFERENCE_DATETIME
7369
#
7470
# Compile static libraries, and use statically linked openssl (-openssl-linked):
7571
OPENSSL_LIBS="-L$DEPSDIR/lib -lssl -lcrypto -lgdi32" ./configure -prefix $INSTALLPREFIX -bindir $INSTALLPREFIX/host/bin -confirm-license -release -opensource -static -xplatform $SPECNAME -device-option CROSS_COMPILE="$HOST-" -no-audio-backend -no-javascript-jit -no-sql-sqlite -no-sql-odbc -no-nis -no-cups -no-iconv -no-dbus -no-gif -no-opengl -no-compile-examples -no-feature-style-windowsce -no-feature-style-windowsmobile -no-qml-debug -openssl-linked -skip qtsvg -skip qtwebkit -skip qtwebkit-examples -skip qtserialport -skip qtdeclarative -skip qtmultimedia -skip qtimageformats -skip qtlocation -skip qtsensors -skip qtquick1 -skip qtquickcontrols -skip qtactiveqt -skip qtconnectivity -skip qtwinextras -skip qtxmlpatterns -skip qtscript -skip qtdoc -system-libpng -system-zlib
7672
make $MAKEOPTS install
73+
# post-process all generated libraries and plugins to be deterministic
74+
# extract them to a temporary directory then re-build them deterministically
75+
for LIB in $(find $INSTALLPREFIX -name *.a); do
76+
rm -rf $TEMPDIR && mkdir $TEMPDIR && cd $TEMPDIR
77+
$HOST-ar xv $LIB | cut -b5- > /tmp/list.txt
78+
rm $LIB
79+
$HOST-ar crsD $LIB $(cat /tmp/list.txt)
80+
done
7781
#
7882
cd $INSTALLPREFIX
79-
83+
# Remove unused non-deterministic stuff
84+
rm host/bin/qtpaths.exe lib/libQt5Bootstrap.a lib/libQt5Bootstrap.la
8085
# as zip stores file timestamps, use faketime to intercept stat calls to set dates for all files to reference date
8186
export LD_PRELOAD=/usr/lib/faketime/libfaketime.so.1
82-
zip -r $OUTDIR/qt-win${BITS}-5.2.0-gitian-r2.zip *
87+
export FAKETIME=$REFERENCE_DATETIME
88+
find -print0 | xargs -r0 touch # fix up timestamps before packaging
89+
find | sort | zip -X@ $OUTDIR/qt-win${BITS}-5.2.0-gitian-r2.zip
8390
unset LD_PRELOAD
8491
unset FAKETIME
8592
done # for BITS in

0 commit comments

Comments
 (0)