Add data partition (#2)

* Create script to add FAT32 data partition to Raspbian img file

* Add package dosfstools to Dockerfile for use with add partition script

* Add function to convert FAT32 partiton names to uppercase

* Add comments to annotate script main vars

* Add use of add partion script and mount/unmount of data partiton to build script

* Update test script and build docker script to trigger creation of a data partition and check partiton alignment on chroot

* Add execute permission to add partition script

* Use source to call add pariton script

* Add copy of add partition script to Dockerfile

* Add install of parted to Dockerfile

* Remove aligment check, replace with lsblk to show partitions

* Add alignment check for new partition

* Update readme file with info about data partiton creation

* Add function to remove init function from cmdline.txt

Author: William Squires

Dockerfile

@@ -2,10 +2,11 @@ FROM debian:stable
 
 RUN apt-get -y update && \
 	apt-get -y install \
-	wget unzip kpartx qemu qemu-user-static binfmt-support
+	wget unzip kpartx qemu qemu-user-static binfmt-support parted dosfstools
 
 WORKDIR /build/
 
 COPY build.sh /build/
+COPY add-partition.sh /build/
 
 ENTRYPOINT /build/build.sh

README.md

@@ -18,10 +18,14 @@ To specify a script to run in the Raspbian environment use the `SCRIPT` environm
 
 The source image file can also be specified with `SOURCE_IMAGE`. This should be the path to an `.img` file, also found in your mounted volume.
 
+A FAT32 format data partition can be added to the end of the image if `ADD_DATA_PART` is set to `true`. This partiton can be helpful when writing the image to an SD card in macOS and Windows as data files can be placed there. The partition will be `64MiB`, labeled `DATA`, and mounted at `/data`. The `/etc/fstab` and `/boot/config.txt` files will be updated with correct `PARTUUID`s.
+
 ```
 docker run --privileged \
+-e MOUNT=/customisations \
 -e SOURCE_IMAGE=/customisations/raspbian-lite.img \
 -e SCRIPT=/customisations/customise.sh \
+-e ADD_DATA_PART=true \
 --mount source=customisations,destination=/customisations \
 edwardotme/raspbian-customiser:latest
 ```

add-partition.sh

@@ -0,0 +1,131 @@
+#!/bin/bash
+
+# Add a FAT32 data partion to a Rasbian .img file
+
+set -e
+
+IMG_FILE=$1
+
+# These could be set by args
+# Size should be in MiB
+PART_SIZE=64
+# Name will be converted to uppercase
+PART_NAME=DATA
+# Mount is relative to root of image
+PART_MOUNT=/data
+
+# Currently only FAT32 is supported
+PART_TYPE=fat32
+
+# Use parted to print the partition table
+PARTED_OUT=$(parted -s "${IMG_FILE}" unit b print)
+echo "PARTED OUTPUT: $PARTED_OUT"
+
+# Find the end of the root partition
+# This assumes there are two partitons
+ROOT_END=$(echo "$PARTED_OUT" | grep -e '^ 2'| xargs echo -n \
+| cut -d" " -f 3 | tr -d B)
+
+# Determine where to start the new partition so that it is properly
+# aligned to 4MiB erase blocks
+BLOCK=$((4 * 1024 * 1024))
+PART_START=$((($ROOT_END + 1) / $BLOCK))
+PART_START=$((($PART_START + 1) * $BLOCK))
+
+# Work out how much to expand the image to fit the new partition
+PART_EXPAND=
+# Convert PART_SIZE from MiB to B
+PART_EXPAND=$(($PART_SIZE * 1024 * 1024))
+# Add the free space gap required between the end of the previous
+# partiton and the new one
+PART_EXPAND=$(($PART_EXPAND + ($PART_START - $ROOT_END)))
+# Ensure the expansion aligns with 512B sector
+PART_EXPAND=$(($PART_EXPAND / 512))
+PART_EXPAND=$((($PART_EXPAND + 1) * 512))
+# Convert to MiB for quicker dd
+PART_EXPAND=$(($PART_EXPAND / (1024 * 1024)))
+
+# Print values
+echo "ROOT_END: $ROOT_END"
+echo "PART_START: $PART_START"
+echo "PART_EXPAND: $PART_EXPAND"
+
+# Expand the image with zeros using dd
+dd if=/dev/zero bs=1M count=$PART_EXPAND >> $IMG_FILE
+
+# Use parted to print the partition table
+parted -s "${IMG_FILE}" unit b print free
+
+# Create the partition in the new space
+parted -a none -s "${IMG_FILE}" mkpart primary "${PART_TYPE}" "${PART_START}B" 100%
+
+# Use parted to print the partition table
+parted -s "${IMG_FILE}" unit b print free
+
+# Check the partiton is optimally aligned
+parted -s "${IMG_FILE}" align-check opt 3
+
+# Mount the image as a loop device
+LOOP_DEV=$(losetup --show --find --partscan $IMG_FILE)
+# Wait a second or mount may fail
+sleep 1
+# Get list of partitions, drop the first line, as this is our
+# LOOP_DEV itself, we only what the child partitions
+PARTITIONS=$(lsblk --raw --output "MAJ:MIN" --noheadings ${LOOP_DEV} | tail -n +2)
+# Manually use mknod to create nodes for partitons on loop device
+# Testing indicates this is required when running in a container,
+# even if the containter is run --pivileged
+COUNTER=1
+for i in $PARTITIONS; do
+    MAJ=$(echo $i | cut -d: -f1)
+    MIN=$(echo $i | cut -d: -f2)
+    if [ ! -e "${LOOP_DEV}p${COUNTER}" ]; then mknod ${LOOP_DEV}p${COUNTER} b $MAJ $MIN; fi
+    COUNTER=$((COUNTER + 1))
+done
+# Make mount point and mount image
+ROOTFS_DIR=/mnt/raspbian
+mkdir -p $ROOTFS_DIR
+mount -o rw ${LOOP_DEV}p2 $ROOTFS_DIR
+mount -o rw ${LOOP_DEV}p1 ${ROOTFS_DIR}/boot
+
+# Format the partiton
+PART_NAME=$(echo $PART_NAME | tr a-z A-Z )
+mkdosfs -n "${PART_NAME}" -F 32 -v "${LOOP_DEV}p3" > /dev/null
+
+# Use parted to print the partition table
+parted -s "${IMG_FILE}" unit b print free
+
+# Create the mount point
+mkdir /mnt/raspbian/$PART_MOUNT
+mount -o rw ${LOOP_DEV}p2 ${ROOTFS_DIR}/${PART_MOUNT}
+
+# List the contents of mount point to verify mount was successful
+echo "CONTENTS OF /:"
+ls ${ROOTFS_DIR}
+echo "CONTENTS OF /boot:"
+ls ${ROOTFS_DIR}/boot
+
+# Add the partition mount to /etc/fstab
+IMG_ID="$(dd if="${IMG_FILE}" skip=440 bs=1 count=4 2>/dev/null | xxd -e | cut -f 2 -d' ')"
+echo "IMG_ID: $IMG_ID"
+BOOT_PARTUUID="${IMG_ID}-01"
+ROOT_PARTUUID="${IMG_ID}-02"
+DATA_PARTUUID="${IMG_ID}-03"
+sed -i "s/PARTUUID=[a-z0-9]*-01/PARTUUID=${BOOT_PARTUUID}/" "${ROOTFS_DIR}/etc/fstab"
+sed -i "s/PARTUUID=[a-z0-9]*-02/PARTUUID=${ROOT_PARTUUID}/" "${ROOTFS_DIR}/etc/fstab"
+echo "PARTUUID=${DATA_PARTUUID} ${PART_MOUNT} vfat defaults 0 1" >> "${ROOTFS_DIR}/etc/fstab"
+echo "FSTAB:"
+cat "${ROOTFS_DIR}/etc/fstab"
+
+# Update cmdline.txt to use the new PARTUUID
+sed -i "s/PARTUUID=[a-z0-9]*-02/PARTUUID=${ROOT_PARTUUID}/" "${ROOTFS_DIR}/boot/cmdline.txt"
+# Remove the reference to the expand rootfs script as it can no longer be
+# expanded with the new partition in the way
+sed -i "s/init=.*//" "${ROOTFS_DIR}/boot/cmdline.txt"
+echo "CMDLINE.TXT:"
+cat "${ROOTFS_DIR}/boot/cmdline.txt"
+
+# Unmount partitions and reset loop device
+umount /mnt/raspbian/{${PART_MOUNT},boot,}
+losetup -d $LOOP_DEV
+echo "SUCCESSFULLY ADDED PARTITION AND UNMOUNTED IMG"

build-docker.sh

@@ -17,6 +17,7 @@ docker run --privileged --rm \
   -e MOUNT=/test \
   -e SOURCE_IMAGE=/test/${IMAGE_ZIP%.zip}.img \
   -e SCRIPT=/test/test.sh \
+  -e ADD_DATA_PART=true \
   --mount type=bind,source="$(pwd)"/test,destination=/test \
   edwardotme/raspbian-customiser:$TRAVIS_BRANCH
 if [ $? -ne 0 ]; then

build.sh

@@ -1,8 +1,17 @@
 #!/bin/bash
 
+set -e
+
+ADD_DATA_PART=${ADD_DATA_PART:-false}
+
 echo "MOUNT: $MOUNT"
 echo "SOURCE_IMAGE: $SOURCE_IMAGE"
 echo "SCRIPT: $SCRIPT"
+echo "ADD_DATA_PART: $ADD_DATA_PART"
+
+if [ $ADD_DATA_PART != false ]; then
+	source ./add-partition.sh $SOURCE_IMAGE
+fi
 
 # Create loop device map from image partition table
 LOOP_DEV=$(losetup --show --find --partscan $SOURCE_IMAGE)
@@ -35,6 +44,10 @@ mkdir -p /mnt/raspbian
 mount -o rw ${LOOP_DEV}p2 /mnt/raspbian
 mount -o rw ${LOOP_DEV}p1 /mnt/raspbian/boot
 
+if [ $ADD_DATA_PART != false ]; then
+	mount -o rw ${LOOP_DEV}p3 /mnt/raspbian/data
+fi
+
 # Create bind mounts for system directories
 mount --bind /dev /mnt/raspbian/dev/
 mount --bind /sys /mnt/raspbian/sys/
@@ -60,4 +73,8 @@ chroot /mnt/raspbian $SCRIPT
 sed -i 's/^#CHROOT //g' /mnt/raspbian/etc/ld.so.preload
 
 # Unmount everything
+if [ $ADD_DATA_PART != false ]; then
+	umount /mnt/raspbian/data
+fi
 umount /mnt/raspbian/{dev/pts,dev,sys,proc,boot,${MOUNT},}
+losetup -d $LOOP_DEV

test/test.sh

@@ -3,9 +3,14 @@
 # Some basic commands to run on the Raspbian system to confirm chroot is working
 # and generate some information output
 
+set -e
+
 uname -a
 
 ls /
 ls /boot
+ls /data
+
+lsblk
 
 apt-get update