Merge pull request #5363

7a9cf80 docs: add/update docs for osx dmg signing (Cory Fields)
914868a build: add a deterministic dmg signer (Cory Fields)
d69ed2b build: Clean up the dmg layout (Cory Fields)
2f327a3 build: add the deploydir target for gitian (Cory Fields)

Author: laanwj

Makefile.am

@@ -26,7 +26,9 @@ WINDOWS_PACKAGING = $(top_srcdir)/share/pixmaps/bitcoin.ico \
 
 OSX_PACKAGING = $(OSX_DEPLOY_SCRIPT) $(OSX_FANCY_PLIST) $(OSX_INSTALLER_ICONS) \
   $(top_srcdir)/contrib/macdeploy/background.png \
-  $(top_srcdir)/contrib/macdeploy/DS_Store
+  $(top_srcdir)/contrib/macdeploy/DS_Store \
+  $(top_srcdir)/contrib/macdeploy/detached-sig-apply.sh \
+  $(top_srcdir)/contrib/macdeploy/detached-sig-create.sh
 
 COVERAGE_INFO = baseline_filtered_combined.info baseline.info block_test.info \
   leveldb_baseline.info test_bitcoin_filtered.info total_coverage.info \
@@ -85,14 +87,30 @@ if BUILD_DARWIN
 $(OSX_DMG): $(OSX_APP_BUILT) $(OSX_PACKAGING)
 	$(OSX_DEPLOY_SCRIPT) $(OSX_APP) -add-qt-tr $(OSX_QT_TRANSLATIONS) -translations-dir=$(QT_TRANSLATION_DIR) -dmg -fancy $(OSX_FANCY_PLIST) -verbose 2
 
+deploydir: $(OSX_DMG)
 else
-$(OSX_DMG): $(OSX_APP_BUILT) $(OSX_PACKAGING)
-	INSTALLNAMETOOL=$(INSTALLNAMETOOL)  OTOOL=$(OTOOL) STRIP=$(STRIP) $(OSX_DEPLOY_SCRIPT) $(OSX_APP) -add-qt-tr $(OSX_QT_TRANSLATIONS) -translations-dir=$(QT_TRANSLATION_DIR) -verbose 2
-	$(MKDIR_P) dist/.background
-	$(INSTALL) contrib/macdeploy/background.png dist/.background
-	$(INSTALL) contrib/macdeploy/DS_Store dist/.DS_Store
-	cd dist; $(LN_S) /Applications Applications
-	$(GENISOIMAGE) -no-cache-inodes -l -probe -V "Bitcoin-Qt" -no-pad -r -apple -o $@ dist
+APP_DIST_DIR=$(top_builddir)/dist
+APP_DIST_EXTRAS=$(APP_DIST_DIR)/.background/background.png $(APP_DIST_DIR)/.DS_Store $(APP_DIST_DIR)/Applications
+
+$(APP_DIST_DIR)/Applications:
+	@rm -f $@
+	@cd $(@D); $(LN_S) /Applications $(@F)
+
+$(APP_DIST_EXTRAS): $(APP_DIST_DIR)/$(OSX_APP)/Contents/MacOS/Bitcoin-Qt
+
+$(OSX_DMG): $(APP_DIST_EXTRAS)
+	$(GENISOIMAGE) -no-cache-inodes -D -l -probe -V "Bitcoin-Qt" -no-pad -r -apple -o $@ dist
+
+$(APP_DIST_DIR)/.background/background.png:
+	$(MKDIR_P) $(@D)
+	$(INSTALL) $(top_srcdir)/contrib/macdeploy/background.png $@
+$(APP_DIST_DIR)/.DS_Store:
+	$(INSTALL) $(top_srcdir)/contrib/macdeploy/DS_Store $@
+
+$(APP_DIST_DIR)/$(OSX_APP)/Contents/MacOS/Bitcoin-Qt: $(OSX_APP_BUILT) $(OSX_PACKAGING)
+	INSTALLNAMETOOL=$(INSTALLNAMETOOL)  OTOOL=$(OTOOL) STRIP=$(STRIP) $(OSX_DEPLOY_SCRIPT) $(OSX_APP) -translations-dir=$(QT_TRANSLATION_DIR) -add-qt-tr $(OSX_QT_TRANSLATIONS) -verbose 2
+
+deploydir: $(APP_DIST_EXTRAS)
 endif
 
 if TARGET_DARWIN

contrib/gitian-descriptors/gitian-osx-signer.yml

@@ -0,0 +1,37 @@
+---
+name: "bitcoin-dmg-signer"
+suites:
+- "precise"
+architectures:
+- "amd64"
+packages:
+- "libc6:i386"
+- "faketime"
+reference_datetime: "2013-06-01 00:00:00"
+remotes: []
+files:
+- "bitcoin-0.9.99-osx-unsigned.tar.gz"
+- "signature.tar.gz"
+script: |
+  WRAP_DIR=$HOME/wrapped
+  mkdir -p ${WRAP_DIR}
+  export PATH=`pwd`:$PATH
+  FAKETIME_PROGS="dmg genisoimage"
+
+  # Create global faketime wrappers
+  for prog in ${FAKETIME_PROGS}; do
+    echo '#!/bin/bash' > ${WRAP_DIR}/${prog}
+    echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog}
+    echo 'export LD_PRELOAD=/usr/lib/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${prog}
+    echo "export FAKETIME=\"${REFERENCE_DATETIME}\"" >> ${WRAP_DIR}/${prog}
+    echo "\$REAL \$@" >> $WRAP_DIR/${prog}
+    chmod +x ${WRAP_DIR}/${prog}
+  done
+
+  UNSIGNED=`echo bitcoin-*.tar.gz`
+  SIGNED=`echo ${UNSIGNED} | sed 's/.tar.*//' | sed 's/-unsigned//'`.dmg
+
+  tar -xf ${UNSIGNED}
+  ./detached-sig-apply.sh ${UNSIGNED} signature.tar.gz
+  ${WRAP_DIR}/genisoimage -no-cache-inodes -D -l -probe -V "Bitcoin-Qt" -no-pad -r -apple -o uncompressed.dmg signed-app
+  ${WRAP_DIR}/dmg dmg uncompressed.dmg ${OUTDIR}/${SIGNED}

contrib/gitian-descriptors/gitian-osx.yml

@@ -106,8 +106,21 @@ script: |
     ./configure --prefix=${BASEPREFIX}/${i} --bindir=${INSTALLPATH}/bin --includedir=${INSTALLPATH}/include --libdir=${INSTALLPATH}/lib --disable-ccache --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS}
     make ${MAKEOPTS}
     make install-strip
+
+    make deploydir
+    mkdir -p unsigned-app-${i}
+    cp contrib/macdeploy/detached-sig-apply.sh unsigned-app-${i}
+    cp contrib/macdeploy/detached-sig-create.sh unsigned-app-${i}
+    cp ${BASEPREFIX}/${i}/native/bin/dmg ${BASEPREFIX}/${i}/native/bin/genisoimage unsigned-app-${i}
+    cp ${BASEPREFIX}/${i}/native/bin/${i}-codesign_allocate unsigned-app-${i}/codesign_allocate
+    cp ${BASEPREFIX}/${i}/native/bin/${i}-pagestuff unsigned-app-${i}/pagestuff
+    mv dist unsigned-app-${i}
+    pushd unsigned-app-${i}
+    find . | sort | tar --no-recursion -czf ${OUTDIR}/${DISTNAME}-osx-unsigned.tar.gz -T -
+    popd
+
     make deploy
-    ${WRAP_DIR}/dmg dmg Bitcoin-Qt.dmg ${OUTDIR}/${DISTNAME}-osx.dmg
+    ${WRAP_DIR}/dmg dmg Bitcoin-Qt.dmg ${OUTDIR}/${DISTNAME}-osx-unsigned.dmg
 
     cd installed
     find . -name "lib*.la" -delete

contrib/macdeploy/DS_Store

@@ -0,0 +1,53 @@
+#!/bin/sh
+set -e
+
+UNSIGNED=$1
+SIGNATURE=$2
+ARCH=x86_64
+ROOTDIR=dist
+BUNDLE=${ROOTDIR}/Bitcoin-Qt.app
+TEMPDIR=signed.temp
+OUTDIR=signed-app
+
+if [ -z "$UNSIGNED" ]; then
+  echo "usage: $0 <unsigned app> <signature>"
+  exit 1
+fi
+
+if [ -z "$SIGNATURE" ]; then
+  echo "usage: $0 <unsigned app> <signature>"
+  exit 1
+fi
+
+rm -rf ${TEMPDIR} && mkdir -p ${TEMPDIR}
+tar -C ${TEMPDIR} -xf ${UNSIGNED}
+tar -C ${TEMPDIR} -xf ${SIGNATURE}
+
+if [ -z "${PAGESTUFF}" ]; then
+  PAGESTUFF=${TEMPDIR}/pagestuff
+fi
+
+if [ -z "${CODESIGN_ALLOCATE}" ]; then
+  CODESIGN_ALLOCATE=${TEMPDIR}/codesign_allocate
+fi
+
+for i in `find ${TEMPDIR} -name "*.sign"`; do
+  SIZE=`stat -c %s ${i}`
+  TARGET_FILE=`echo ${i} | sed 's/\.sign$//'`
+
+  echo "Allocating space for the signature of size ${SIZE} in ${TARGET_FILE}"
+  ${CODESIGN_ALLOCATE} -i ${TARGET_FILE} -a ${ARCH} ${SIZE} -o ${i}.tmp
+
+  OFFSET=`${PAGESTUFF} ${i}.tmp -p | tail -2 | grep offset | sed 's/[^0-9]*//g'`
+  if [ -z ${QUIET} ]; then
+    echo "Attaching signature at offset ${OFFSET}"
+  fi
+
+  dd if=$i of=${i}.tmp bs=1 seek=${OFFSET} count=${SIZE} 2>/dev/null
+  mv ${i}.tmp ${TARGET_FILE}
+  rm ${i}
+  echo "Success."
+done
+mv ${TEMPDIR}/${ROOTDIR} ${OUTDIR}
+rm -rf ${TEMPDIR}
+echo "Signed: ${OUTDIR}"

contrib/macdeploy/detached-sig-apply.sh

@@ -0,0 +1,46 @@
+#!/bin/sh
+set -e
+
+ROOTDIR=dist
+BUNDLE=${ROOTDIR}/Bitcoin-Qt.app
+CODESIGN=codesign
+TEMPDIR=sign.temp
+TEMPLIST=${TEMPDIR}/signatures.txt
+OUT=signature.tar.gz
+
+if [ ! -n "$1" ]; then
+  echo "usage: $0 <codesign args>"
+  echo "example: $0 -s MyIdentity"
+  exit 1
+fi
+
+rm -rf ${TEMPDIR} ${TEMPLIST}
+mkdir -p ${TEMPDIR}
+
+${CODESIGN} -f --file-list ${TEMPLIST} "$@" "${BUNDLE}"
+
+for i in `grep -v CodeResources ${TEMPLIST}`; do
+  TARGETFILE="${BUNDLE}/`echo ${i} | sed "s|.*${BUNDLE}/||"`"
+  SIZE=`pagestuff $i -p | tail -2 | grep size | sed 's/[^0-9]*//g'`
+  OFFSET=`pagestuff $i -p | tail -2 | grep offset | sed 's/[^0-9]*//g'`
+  SIGNFILE="${TEMPDIR}/${TARGETFILE}.sign"
+  DIRNAME="`dirname ${SIGNFILE}`"
+  mkdir -p "${DIRNAME}"
+  echo "Adding detached signature for: ${TARGETFILE}. Size: ${SIZE}. Offset: ${OFFSET}"
+  dd if=$i of=${SIGNFILE} bs=1 skip=${OFFSET} count=${SIZE} 2>/dev/null
+done
+
+for i in `grep CodeResources ${TEMPLIST}`; do
+  TARGETFILE="${BUNDLE}/`echo ${i} | sed "s|.*${BUNDLE}/||"`"
+  RESOURCE="${TEMPDIR}/${TARGETFILE}"
+  DIRNAME="`dirname "${RESOURCE}"`"
+  mkdir -p "${DIRNAME}"
+  echo "Adding resource for: "${TARGETFILE}""
+  cp "${i}" "${RESOURCE}"
+done
+
+rm ${TEMPLIST}
+
+tar -C ${TEMPDIR} -czf ${OUT} .
+rm -rf ${TEMPDIR}
+echo "Created ${OUT}"

contrib/macdeploy/detached-sig-create.sh

@@ -65,3 +65,18 @@ Background images and other features can be added to DMG files by inserting a
 .DS_Store before creation. The easiest way to create this file is to build a
 DMG without one, move it to a device running OSX, customize the layout, then
 grab the .DS_Store file for later use. That is the approach taken here.
+
+As of OSX Mavericks (10.9), using an Apple-blessed key to sign binaries is a
+requirement in order to satisfy the new Gatekeeper requirements. Because this
+private key cannot be shared, we'll have to be a bit creative in order for the
+build process to remain somewhat deterministic. Here's how it works:
+
+- Builders use gitian to create an unsigned release. This outputs an unsigned
+  dmg which users may choose to bless and run. It also outputs an unsigned app
+  structure in the form of a tarball, which also contains all of the tools
+  that have been previously (deterministically) built in order to create a
+  final dmg.
+- The Apple keyholder uses this unsigned app to create a detached signature,
+  using the script that is also included there.
+- Builders feed the unsigned app + detached signature back into gitian. It
+  uses the pre-built tools to recombine the pieces into a deterministic dmg.

doc/README_osx.txt

@@ -59,17 +59,18 @@ Release Process
 	./bin/gsign --signer $SIGNER --release ${VERSION}-win --destination ../gitian.sigs/ ../bitcoin/contrib/gitian-descriptors/gitian-win.yml
 	mv build/out/bitcoin-*.zip build/out/bitcoin-*.exe ../
 	./bin/gbuild --commit bitcoin=v${VERSION} ../bitcoin/contrib/gitian-descriptors/gitian-osx.yml
-	./bin/gsign --signer $SIGNER --release ${VERSION}-osx --destination ../gitian.sigs/ ../bitcoin/contrib/gitian-descriptors/gitian-osx.yml
+	./bin/gsign --signer $SIGNER --release ${VERSION}-osx-unsigned --destination ../gitian.sigs/ ../bitcoin/contrib/gitian-descriptors/gitian-osx.yml
+	mv build/out/bitcoin-*-unsigned.tar.gz inputs
 	mv build/out/bitcoin-*.tar.gz build/out/bitcoin-*.dmg ../
 	popd
-
+bitcoin-0.9.99-osx-unsigned.tar.gz
   Build output expected:
 
   1. source tarball (bitcoin-${VERSION}.tar.gz)
   2. linux 32-bit and 64-bit binaries dist tarballs (bitcoin-${VERSION}-linux[32|64].tar.gz)
   3. windows 32-bit and 64-bit installers and dist zips (bitcoin-${VERSION}-win[32|64]-setup.exe, bitcoin-${VERSION}-win[32|64].zip)
-  4. OSX installer (bitcoin-${VERSION}-osx.dmg)
-  5. Gitian signatures (in gitian.sigs/${VERSION}-<linux|win|osx>/(your gitian key)/
+  4. OSX unsigned installer (bitcoin-${VERSION}-osx-unsigned.dmg)
+  5. Gitian signatures (in gitian.sigs/${VERSION}-<linux|win|osx-unsigned>/(your gitian key)/
 
 ###Next steps:
 
@@ -78,7 +79,28 @@ Commit your signature to gitian.sigs:
 	pushd gitian.sigs
 	git add ${VERSION}-linux/${SIGNER}
 	git add ${VERSION}-win/${SIGNER}
-	git add ${VERSION}-osx/${SIGNER}
+	git add ${VERSION}-osx-unsigned/${SIGNER}
+	git commit -a
+	git push  # Assuming you can push to the gitian.sigs tree
+	popd
+
+Wait for OSX detached signature:
+	Once the OSX build has 3 matching signatures, Gavin will sign it with the apple App-Store key.
+	He will then upload a detached signature to be combined with the unsigned app to create a signed binary.
+
+Create the signed OSX binary:
+	pushd ./gitian-builder
+	# Fetch the signature as instructed by Gavin
+	cp signature.tar.gz inputs/
+	./bin/gbuild -i ../bitcoin/contrib/gitian-descriptors/gitian-osx-signer.yml
+	./bin/gsign --signer $SIGNER --release ${VERSION}-osx-signed --destination ../gitian.sigs/ ../bitcoin/contrib/gitian-descriptors/gitian-osx-signer.yml
+	mv build/out/bitcoin-${VERSION}-osx.dmg ../
+	popd
+
+Commit your signature for the signed OSX binary:
+
+	pushd gitian.sigs
+	git add ${VERSION}-osx-signed/${SIGNER}
 	git commit -a
 	git push  # Assuming you can push to the gitian.sigs tree
 	popd
@@ -91,8 +113,6 @@ Commit your signature to gitian.sigs:
 
     - Code-sign Windows -setup.exe (in a Windows virtual machine using signtool)
 
-    - Code-sign MacOSX .dmg
-
   Note: only Gavin has the code-signing keys currently.
 
 - Create `SHA256SUMS.asc` for the builds, and GPG-sign it: