Merge pi-gen changes.

.gitignore

@@ -7,6 +7,7 @@ node_modules
 .DS_Store
 deploy/*
 work/*
+config
 postrun.sh
 SKIP
 SKIP_IMAGES

Dockerfile

@@ -1,4 +1,5 @@
-FROM debian:buster
+ARG BASE_IMAGE=debian:buster
+FROM ${BASE_IMAGE}
 
 ENV DEBIAN_FRONTEND noninteractive
 
@@ -7,7 +8,7 @@ RUN apt-get -y update && \
         git vim parted \
         quilt coreutils qemu-user-static debootstrap zerofree zip dosfstools \
         bsdtar libcap2-bin rsync grep udev xz-utils curl xxd file kmod bc\
-        binfmt-support ca-certificates \
+        binfmt-support ca-certificates qemu-utils kpartx \
     && rm -rf /var/lib/apt/lists/*
 
 COPY . /pi-gen/

build-docker.sh

@@ -1,4 +1,5 @@
 #!/bin/bash -eu
+
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 
 BUILD_OPTS="$*"
@@ -73,10 +74,23 @@ fi
 # Modify original build-options to allow config file to be mounted in the docker container
 BUILD_OPTS="$(echo "${BUILD_OPTS:-}" | sed -E 's@\-c\s?([^ ]+)@-c /config@')"
 
-${DOCKER} build -t pi-gen "${DIR}"
+# Check the arch of the machine we're running on. If it's 64-bit, use a 32-bit base image instead
+case "$(uname -m)" in
+  x86_64|aarch64)
+    BASE_IMAGE=i386/debian:buster
+    ;;
+  *)
+    BASE_IMAGE=debian:buster
+    ;;
+esac
+${DOCKER} build --build-arg BASE_IMAGE=${BASE_IMAGE} -t pi-gen "${DIR}"
+
 if [ "${CONTAINER_EXISTS}" != "" ]; then
 	trap 'echo "got CTRL+C... please wait 5s" && ${DOCKER} stop -t 5 ${CONTAINER_NAME}_cont' SIGINT SIGTERM
 	time ${DOCKER} run --rm --privileged \
+		--cap-add=ALL \
+		-v /dev:/dev \
+		-v /lib/modules:/lib/modules \
 		--volume "${CONFIG_FILE}":/config:ro \
 		-e "GIT_HASH=${GIT_HASH}" \
 		--volumes-from="${CONTAINER_NAME}" --name "${CONTAINER_NAME}_cont" \
@@ -88,6 +102,9 @@ if [ "${CONTAINER_EXISTS}" != "" ]; then
 else
 	trap 'echo "got CTRL+C... please wait 5s" && ${DOCKER} stop -t 5 ${CONTAINER_NAME}' SIGINT SIGTERM
 	time ${DOCKER} run --name "${CONTAINER_NAME}" --privileged \
+		--cap-add=ALL \
+		-v /dev:/dev \
+		-v /lib/modules:/lib/modules \
 		--volume "${CONFIG_FILE}":/config:ro \
 		-e "GIT_HASH=${GIT_HASH}" \
 		pi-gen \
@@ -96,6 +113,7 @@ else
 	rsync -av work/*/build.log deploy/" &
 	wait "$!"
 fi
+
 echo "copying results from deploy/"
 ${DOCKER} cp "${CONTAINER_NAME}":/pi-gen/deploy .
 ls -lah deploy

build.sh

@@ -1,4 +1,5 @@
 #!/bin/bash -e
+
 # shellcheck disable=SC2119
 run_sub_stage()
 {
@@ -28,6 +29,11 @@ do
     sleep 15
 done
 EOF
+				if [ "${USE_QCOW2}" = "1" ]; then
+					on_chroot << EOF
+apt-get clean
+EOF
+				fi
 			fi
 			log "End ${SUB_STAGE_DIR}/${i}-packages-nr"
 		fi
@@ -44,6 +50,11 @@ do
     sleep 15
 done
 EOF
+				if [ "${USE_QCOW2}" = "1" ]; then
+					on_chroot << EOF
+apt-get clean
+EOF
+				fi
 			fi
 			log "End ${SUB_STAGE_DIR}/${i}-packages"
 		fi
@@ -94,17 +105,30 @@ EOF
 run_stage(){
 	log "Begin ${STAGE_DIR}"
 	STAGE="$(basename "${STAGE_DIR}")"
+
 	pushd "${STAGE_DIR}" > /dev/null
-	unmount "${WORK_DIR}/${STAGE}"
+
 	STAGE_WORK_DIR="${WORK_DIR}/${STAGE}"
 	ROOTFS_DIR="${STAGE_WORK_DIR}"/rootfs
+
+	if [ "${USE_QCOW2}" = "1" ]; then 
+		if [ ! -f SKIP ]; then
+			load_qimage
+		fi
+	else
+		# make sure we are not umounting during export-image stage
+		if [ "${USE_QCOW2}" = "0" ] && [ "${NO_PRERUN_QCOW2}" = "0" ]; then
+			unmount "${WORK_DIR}/${STAGE}"
+		fi
+	fi
+	
 	if [ ! -f SKIP_IMAGES ]; then
 		if [ -f "${STAGE_DIR}/EXPORT_IMAGE" ]; then
 			EXPORT_DIRS="${EXPORT_DIRS} ${STAGE_DIR}"
 		fi
 	fi
 	if [ ! -f SKIP ]; then
-		if [ "${CLEAN}" = "1" ]; then
+		if [ "${CLEAN}" = "1" ] && [ "${USE_QCOW2}" = "0" ] ; then
 			if [ -d "${ROOTFS_DIR}" ]; then
 				rm -rf "${ROOTFS_DIR}"
 			fi
@@ -115,13 +139,21 @@ run_stage(){
 			log "End ${STAGE_DIR}/prerun.sh"
 		fi
 		for SUB_STAGE_DIR in "${STAGE_DIR}"/*; do
-			if [ -d "${SUB_STAGE_DIR}" ] &&
-			   [ ! -f "${SUB_STAGE_DIR}/SKIP" ]; then
+			if [ -d "${SUB_STAGE_DIR}" ] && [ ! -f "${SUB_STAGE_DIR}/SKIP" ]; then
 				run_sub_stage
 			fi
 		done
 	fi
-	unmount "${WORK_DIR}/${STAGE}"
+
+	if [ "${USE_QCOW2}" = "1" ]; then 
+		unload_qimage
+	else
+		# make sure we are not umounting during export-image stage
+		if [ "${USE_QCOW2}" = "0" ] && [ "${NO_PRERUN_QCOW2}" = "0" ]; then
+			unmount "${WORK_DIR}/${STAGE}"
+		fi
+	fi
+
 	PREV_STAGE="${STAGE}"
 	PREV_STAGE_DIR="${STAGE_DIR}"
 	PREV_ROOTFS_DIR="${ROOTFS_DIR}"
@@ -155,6 +187,15 @@ do
 	esac
 done
 
+term() {
+	if [ "${USE_QCOW2}" = "1" ]; then
+		log "Unloading image"
+		unload_qimage
+	fi
+}
+
+trap term EXIT INT TERM
+
 export PI_GEN=${PI_GEN:-pi-gen}
 export PI_GEN_REPO=${PI_GEN_REPO:-https://github.com/RPi-Distro/pi-gen}
 
@@ -183,6 +224,7 @@ export WPA_ESSID
 export WPA_PASSWORD
 export WPA_COUNTRY
 export ENABLE_SSH="${ENABLE_SSH:-0}"
+export PUBKEY_ONLY_SSH="${PUBKEY_ONLY_SSH:-0}"
 
 export LOCALE_DEFAULT="${LOCALE_DEFAULT:-en_GB.UTF-8}"
 
@@ -193,6 +235,8 @@ export TIMEZONE_DEFAULT="${TIMEZONE_DEFAULT:-Europe/London}"
 
 export GIT_HASH=${GIT_HASH:-"$(git rev-parse HEAD)"}
 
+export PUBKEY_SSH_FIRST_USER
+
 export CLEAN
 export IMG_NAME
 export APT_PROXY
@@ -220,6 +264,18 @@ source "${SCRIPT_DIR}/common"
 # shellcheck source=scripts/dependencies_check
 source "${SCRIPT_DIR}/dependencies_check"
 
+export NO_PRERUN_QCOW2="${NO_PRERUN_QCOW2:-1}"
+export USE_QCOW2="${USE_QCOW2:-1}"
+export BASE_QCOW2_SIZE=${BASE_QCOW2_SIZE:-12G}
+source "${SCRIPT_DIR}/qcow2_handling"
+if [ "${USE_QCOW2}" = "1" ]; then
+	NO_PRERUN_QCOW2=1
+else
+	NO_PRERUN_QCOW2=0
+fi
+
+export NO_PRERUN_QCOW2="${NO_PRERUN_QCOW2:-1}"
+
 dependencies_check "${BASE_DIR}/depends"
 
 #check username is valid
@@ -238,6 +294,11 @@ if [[ -n "${WPA_PASSWORD}" && ${#WPA_PASSWORD} -lt 8 || ${#WPA_PASSWORD} -gt 63
 	exit 1
 fi
 
+if [[ "${PUBKEY_ONLY_SSH}" = "1" && -z "${PUBKEY_SSH_FIRST_USER}" ]]; then
+	echo "Must set 'PUBKEY_SSH_FIRST_USER' to a valid SSH public key if using PUBKEY_ONLY_SSH"
+	exit 1
+fi
+
 mkdir -p "${WORK_DIR}"
 log "Begin ${BASE_DIR}"
 
@@ -254,22 +315,98 @@ for EXPORT_DIR in ${EXPORT_DIRS}; do
 	# shellcheck source=/dev/null
 	source "${EXPORT_DIR}/EXPORT_IMAGE"
 	EXPORT_ROOTFS_DIR=${WORK_DIR}/$(basename "${EXPORT_DIR}")/rootfs
-	run_stage
+	if [ "${USE_QCOW2}" = "1" ]; then
+		USE_QCOW2=0
+		EXPORT_NAME="${IMG_FILENAME}${IMG_SUFFIX}"
+		echo "------------------------------------------------------------------------"
+		echo "Running export stage for ${EXPORT_NAME}"
+		rm -f "${WORK_DIR}/export-image/${EXPORT_NAME}.img" || true
+		rm -f "${WORK_DIR}/export-image/${EXPORT_NAME}.qcow2" || true
+		rm -f "${WORK_DIR}/${EXPORT_NAME}.img" || true
+		rm -f "${WORK_DIR}/${EXPORT_NAME}.qcow2" || true
+		EXPORT_STAGE=$(basename "${EXPORT_DIR}")
+		for s in $STAGE_LIST; do
+			TMP_LIST=${TMP_LIST:+$TMP_LIST }$(basename "${s}")
+		done
+		FIRST_STAGE=${TMP_LIST%% *}
+		FIRST_IMAGE="image-${FIRST_STAGE}.qcow2"
+
+		pushd "${WORK_DIR}" > /dev/null
+		echo "Creating new base "${EXPORT_NAME}.qcow2" from ${FIRST_IMAGE}"
+		cp "./${FIRST_IMAGE}" "${EXPORT_NAME}.qcow2"
+
+		ARR=($TMP_LIST)
+		# rebase stage images to new export base
+		for CURR_STAGE in "${ARR[@]}"; do
+			if [ "${CURR_STAGE}" = "${FIRST_STAGE}" ]; then
+				PREV_IMG="${EXPORT_NAME}"
+				continue
+			fi
+		echo "Rebasing image-${CURR_STAGE}.qcow2 onto ${PREV_IMG}.qcow2"
+			qemu-img rebase -f qcow2 -u -b ${PREV_IMG}.qcow2 image-${CURR_STAGE}.qcow2
+			if [ "${CURR_STAGE}" = "${EXPORT_STAGE}" ]; then
+				break
+			fi
+			PREV_IMG="image-${CURR_STAGE}"
+		done
+
+		# commit current export stage into base export image
+		echo "Committing image-${EXPORT_STAGE}.qcow2 to ${EXPORT_NAME}.qcow2"
+		qemu-img commit -f qcow2 -p -b "${EXPORT_NAME}.qcow2" image-${EXPORT_STAGE}.qcow2
+
+		# rebase stage images back to original first stage for easy re-run
+		for CURR_STAGE in "${ARR[@]}"; do
+			if [ "${CURR_STAGE}" = "${FIRST_STAGE}" ]; then
+				PREV_IMG="image-${CURR_STAGE}"
+				continue
+			fi
+		echo "Rebasing back image-${CURR_STAGE}.qcow2 onto ${PREV_IMG}.qcow2"
+			qemu-img rebase -f qcow2 -u -b ${PREV_IMG}.qcow2 image-${CURR_STAGE}.qcow2
+			if [ "${CURR_STAGE}" = "${EXPORT_STAGE}" ]; then
+				break
+			fi
+			PREV_IMG="image-${CURR_STAGE}"
+		done
+		popd > /dev/null
+
+		mkdir -p "${WORK_DIR}/export-image/rootfs"
+		mv "${WORK_DIR}/${EXPORT_NAME}.qcow2" "${WORK_DIR}/export-image/"
+		echo "Mounting image ${WORK_DIR}/export-image/${EXPORT_NAME}.qcow2 to rootfs ${WORK_DIR}/export-image/rootfs"
+		mount_qimage "${WORK_DIR}/export-image/${EXPORT_NAME}.qcow2" "${WORK_DIR}/export-image/rootfs"
+
+		CLEAN=0
+		run_stage
+		CLEAN=1
+		USE_QCOW2=1
+
+	else
+		run_stage
+	fi 
 	if [ "${USE_QEMU}" != "1" ]; then
 		if [ -e "${EXPORT_DIR}/EXPORT_NOOBS" ]; then
 			# shellcheck source=/dev/null
 			source "${EXPORT_DIR}/EXPORT_NOOBS"
 			STAGE_DIR="${BASE_DIR}/export-noobs"
-			run_stage
+			if [ "${USE_QCOW2}" = "1" ]; then
+				USE_QCOW2=0
+				run_stage
+				USE_QCOW2=1
+			else
+				run_stage
+			fi
 		fi
 	fi
 done
 
-if [ -x ${BASE_DIR}/postrun.sh ]; then
+if [ -x postrun.sh ]; then
 	log "Begin postrun.sh"
 	cd "${BASE_DIR}"
 	./postrun.sh
 	log "End postrun.sh"
 fi
 
+if [ "${USE_QCOW2}" = "1" ]; then
+	unload_qimage
+fi
+
 log "End ${BASE_DIR}"

export-image/01-set-sources/01-run.sh

@@ -1,5 +1,7 @@
 #!/bin/bash -e
 
+rm -f "${ROOTFS_DIR}/etc/apt/apt.conf.d/51cache"
+find "${ROOTFS_DIR}/var/lib/apt/lists/" -type f -delete
 on_chroot << EOF
 apt-get update
 apt-get -y dist-upgrade

export-image/03-set-partuuid/00-run.sh

@@ -1,13 +1,18 @@
 #!/bin/bash -e
 
-IMG_FILE="${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img"
+if [ "${NO_PRERUN_QCOW2}" = "0" ]; then
 
-IMGID="$(dd if="${IMG_FILE}" skip=440 bs=1 count=4 2>/dev/null | xxd -e | cut -f 2 -d' ')"
+	IMG_FILE="${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img"
 
-BOOT_PARTUUID="${IMGID}-01"
-ROOT_PARTUUID="${IMGID}-02"
+	IMGID="$(dd if="${IMG_FILE}" skip=440 bs=1 count=4 2>/dev/null | xxd -e | cut -f 2 -d' ')"
 
-sed -i "s/BOOTDEV/PARTUUID=${BOOT_PARTUUID}/" "${ROOTFS_DIR}/etc/fstab"
-sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${ROOTFS_DIR}/etc/fstab"
+	BOOT_PARTUUID="${IMGID}-01"
+	ROOT_PARTUUID="${IMGID}-02"
+
+	sed -i "s/BOOTDEV/PARTUUID=${BOOT_PARTUUID}/" "${ROOTFS_DIR}/etc/fstab"
+	sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${ROOTFS_DIR}/etc/fstab"
+
+	sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${ROOTFS_DIR}/boot/cmdline.txt"
+
+fi
 
-sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${ROOTFS_DIR}/boot/cmdline.txt"

export-image/04-finalise/01-run.sh

@@ -16,7 +16,6 @@ if [ -d "${ROOTFS_DIR}/home/${FIRST_USER_NAME}/.config" ]; then
 	chmod 700 "${ROOTFS_DIR}/home/${FIRST_USER_NAME}/.config"
 fi
 
-rm -f "${ROOTFS_DIR}/etc/apt/apt.conf.d/51cache"
 rm -f "${ROOTFS_DIR}/usr/bin/qemu-arm-static"
 
 if [ "${USE_QEMU}" != "1" ]; then
@@ -77,25 +76,30 @@ cp "$ROOTFS_DIR/etc/rpi-issue" "$INFO_FILE"
 	dpkg -l --root "$ROOTFS_DIR"
 } >> "$INFO_FILE"
 
-ROOT_DEV="$(mount | grep "${ROOTFS_DIR} " | cut -f1 -d' ')"
+mkdir -p "${DEPLOY_DIR}"
 
-unmount "${ROOTFS_DIR}"
-zerofree "${ROOT_DEV}"
+rm -f "${DEPLOY_DIR}/${ZIP_FILENAME}${IMG_SUFFIX}.zip"
+rm -f "${DEPLOY_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img"
 
-unmount_image "${IMG_FILE}"
+mv "$INFO_FILE" "$DEPLOY_DIR/"
 
-mkdir -p "${DEPLOY_DIR}"
+if [ "${USE_QCOW2}" = "0" ] && [ "${NO_PRERUN_QCOW2}" = "0" ]; then
+	ROOT_DEV="$(mount | grep "${ROOTFS_DIR} " | cut -f1 -d' ')"
 
-rm -f "${DEPLOY_DIR}/${ZIP_FILENAME}${IMG_SUFFIX}.img.xz"
-rm -f "${DEPLOY_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img"
+	unmount "${ROOTFS_DIR}"
+	zerofree "${ROOT_DEV}"
+
+	unmount_image "${IMG_FILE}"
+else
+	unload_qimage
+	make_bootable_image "${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.qcow2" "$IMG_FILE"
+fi
 
 if [ "${DEPLOY_ZIP}" == "1" ]; then
 	pushd "${STAGE_WORK_DIR}" > /dev/null
-	xz -c "$(basename "${IMG_FILE}")" > "${DEPLOY_DIR}/${ZIP_FILENAME}${IMG_SUFFIX}.img.xz"
+	zip "${DEPLOY_DIR}/${ZIP_FILENAME}${IMG_SUFFIX}.zip" \
+		"$(basename "${IMG_FILE}")"
 	popd > /dev/null
-	rm -f "${DEPLOY_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img"
 else
-	cp "$IMG_FILE" "$DEPLOY_DIR"
+	mv "$IMG_FILE" "$DEPLOY_DIR/"
 fi
-
-cp "$INFO_FILE" "$DEPLOY_DIR"

export-image/prerun.sh

@@ -1,61 +1,87 @@
 #!/bin/bash -e
 
-IMG_FILE="${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img"
+if [ "${NO_PRERUN_QCOW2}" = "0" ]; then
+	IMG_FILE="${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img"
 
-unmount_image "${IMG_FILE}"
+	unmount_image "${IMG_FILE}"
 
-rm -f "${IMG_FILE}"
+	rm -f "${IMG_FILE}"
 
-rm -rf "${ROOTFS_DIR}"
-mkdir -p "${ROOTFS_DIR}"
+	rm -rf "${ROOTFS_DIR}"
+	mkdir -p "${ROOTFS_DIR}"
 
-BOOT_SIZE="$((256 * 1024 * 1024))"
-ROOT_SIZE=$(du --apparent-size -s "${EXPORT_ROOTFS_DIR}" --exclude var/cache/apt/archives --exclude boot --block-size=1 | cut -f 1)
+	BOOT_SIZE="$((256 * 1024 * 1024))"
+	ROOT_SIZE=$(du --apparent-size -s "${EXPORT_ROOTFS_DIR}" --exclude var/cache/apt/archives --exclude boot --block-size=1 | cut -f 1)
 
-# All partition sizes and starts will be aligned to this size
-ALIGN="$((4 * 1024 * 1024))"
-# Add this much space to the calculated file size. This allows for
-# some overhead (since actual space usage is usually rounded up to the
-# filesystem block size) and gives some free space on the resulting
-# image.
-ROOT_MARGIN="$(echo "($ROOT_SIZE * 0.2 + 200 * 1024 * 1024) / 1" | bc)"
+	# All partition sizes and starts will be aligned to this size
+	ALIGN="$((4 * 1024 * 1024))"
+	# Add this much space to the calculated file size. This allows for
+	# some overhead (since actual space usage is usually rounded up to the
+	# filesystem block size) and gives some free space on the resulting
+	# image.
+	ROOT_MARGIN="$(echo "($ROOT_SIZE * 0.2 + 200 * 1024 * 1024) / 1" | bc)"
 
-BOOT_PART_START=$((ALIGN))
-BOOT_PART_SIZE=$(((BOOT_SIZE + ALIGN - 1) / ALIGN * ALIGN))
-ROOT_PART_START=$((BOOT_PART_START + BOOT_PART_SIZE))
-ROOT_PART_SIZE=$(((ROOT_SIZE + ROOT_MARGIN + ALIGN  - 1) / ALIGN * ALIGN))
-IMG_SIZE=$((BOOT_PART_START + BOOT_PART_SIZE + ROOT_PART_SIZE))
+	BOOT_PART_START=$((ALIGN))
+	BOOT_PART_SIZE=$(((BOOT_SIZE + ALIGN - 1) / ALIGN * ALIGN))
+	ROOT_PART_START=$((BOOT_PART_START + BOOT_PART_SIZE))
+	ROOT_PART_SIZE=$(((ROOT_SIZE + ROOT_MARGIN + ALIGN  - 1) / ALIGN * ALIGN))
+	IMG_SIZE=$((BOOT_PART_START + BOOT_PART_SIZE + ROOT_PART_SIZE))
 
-truncate -s "${IMG_SIZE}" "${IMG_FILE}"
+	truncate -s "${IMG_SIZE}" "${IMG_FILE}"
 
-parted --script "${IMG_FILE}" mklabel msdos
-parted --script "${IMG_FILE}" unit B mkpart primary fat32 "${BOOT_PART_START}" "$((BOOT_PART_START + BOOT_PART_SIZE - 1))"
-parted --script "${IMG_FILE}" unit B mkpart primary ext4 "${ROOT_PART_START}" "$((ROOT_PART_START + ROOT_PART_SIZE - 1))"
+	parted --script "${IMG_FILE}" mklabel msdos
+	parted --script "${IMG_FILE}" unit B mkpart primary fat32 "${BOOT_PART_START}" "$((BOOT_PART_START + BOOT_PART_SIZE - 1))"
+	parted --script "${IMG_FILE}" unit B mkpart primary ext4 "${ROOT_PART_START}" "$((ROOT_PART_START + ROOT_PART_SIZE - 1))"
 
-PARTED_OUT=$(parted -sm "${IMG_FILE}" unit b print)
-BOOT_OFFSET=$(echo "$PARTED_OUT" | grep -e '^1:' | cut -d':' -f 2 | tr -d B)
-BOOT_LENGTH=$(echo "$PARTED_OUT" | grep -e '^1:' | cut -d':' -f 4 | tr -d B)
+	PARTED_OUT=$(parted -sm "${IMG_FILE}" unit b print)
+	BOOT_OFFSET=$(echo "$PARTED_OUT" | grep -e '^1:' | cut -d':' -f 2 | tr -d B)
+	BOOT_LENGTH=$(echo "$PARTED_OUT" | grep -e '^1:' | cut -d':' -f 4 | tr -d B)
 
-ROOT_OFFSET=$(echo "$PARTED_OUT" | grep -e '^2:' | cut -d':' -f 2 | tr -d B)
-ROOT_LENGTH=$(echo "$PARTED_OUT" | grep -e '^2:' | cut -d':' -f 4 | tr -d B)
+	ROOT_OFFSET=$(echo "$PARTED_OUT" | grep -e '^2:' | cut -d':' -f 2 | tr -d B)
+	ROOT_LENGTH=$(echo "$PARTED_OUT" | grep -e '^2:' | cut -d':' -f 4 | tr -d B)
 
-BOOT_DEV=$(losetup --show -f -o "${BOOT_OFFSET}" --sizelimit "${BOOT_LENGTH}" "${IMG_FILE}")
-ROOT_DEV=$(losetup --show -f -o "${ROOT_OFFSET}" --sizelimit "${ROOT_LENGTH}" "${IMG_FILE}")
-echo "/boot: offset $BOOT_OFFSET, length $BOOT_LENGTH"
-echo "/:     offset $ROOT_OFFSET, length $ROOT_LENGTH"
+	echo "Mounting BOOT_DEV..."
+	cnt=0
+	until BOOT_DEV=$(losetup --show -f -o "${BOOT_OFFSET}" --sizelimit "${BOOT_LENGTH}" "${IMG_FILE}"); do
+		if [ $cnt -lt 5 ]; then
+			cnt=$((cnt + 1))
+			echo "Error in losetup for BOOT_DEV.  Retrying..."
+			sleep 5
+		else
+			echo "ERROR: losetup for BOOT_DEV failed; exiting"
+			exit 1
+		fi
+	done
 
-ROOT_FEATURES="^huge_file"
-for FEATURE in metadata_csum 64bit; do
+	echo "Mounting ROOT_DEV..."
+	cnt=0
+	until ROOT_DEV=$(losetup --show -f -o "${ROOT_OFFSET}" --sizelimit "${ROOT_LENGTH}" "${IMG_FILE}"); do
+		if [ $cnt -lt 5 ]; then
+			cnt=$((cnt + 1))
+			echo "Error in losetup for ROOT_DEV.  Retrying..."
+			sleep 5
+		else
+			echo "ERROR: losetup for ROOT_DEV failed; exiting"
+			exit 1
+		fi
+	done
+
+	echo "/boot: offset $BOOT_OFFSET, length $BOOT_LENGTH"
+	echo "/:     offset $ROOT_OFFSET, length $ROOT_LENGTH"
+
+	ROOT_FEATURES="^huge_file"
+	for FEATURE in metadata_csum 64bit; do
 	if grep -q "$FEATURE" /etc/mke2fs.conf; then
-	    ROOT_FEATURES="^$FEATURE,$ROOT_FEATURES"
+		ROOT_FEATURES="^$FEATURE,$ROOT_FEATURES"
 	fi
-done
-mkdosfs -n boot -F 32 -v "$BOOT_DEV" > /dev/null
-mkfs.ext4 -L rootfs -O "$ROOT_FEATURES" "$ROOT_DEV" > /dev/null
+	done
+	mkdosfs -n boot -F 32 -v "$BOOT_DEV" > /dev/null
+	mkfs.ext4 -L rootfs -O "$ROOT_FEATURES" "$ROOT_DEV" > /dev/null
 
-mount -v "$ROOT_DEV" "${ROOTFS_DIR}" -t ext4
-mkdir -p "${ROOTFS_DIR}/boot"
-mount -v "$BOOT_DEV" "${ROOTFS_DIR}/boot" -t vfat
+	mount -v "$ROOT_DEV" "${ROOTFS_DIR}" -t ext4
+	mkdir -p "${ROOTFS_DIR}/boot"
+	mount -v "$BOOT_DEV" "${ROOTFS_DIR}/boot" -t vfat
 
-rsync -aHAXx --exclude /var/cache/apt/archives --exclude /boot "${EXPORT_ROOTFS_DIR}/" "${ROOTFS_DIR}/"
-rsync -rtx "${EXPORT_ROOTFS_DIR}/boot/" "${ROOTFS_DIR}/boot/"
+	rsync -aHAXx --exclude /var/cache/apt/archives --exclude /boot "${EXPORT_ROOTFS_DIR}/" "${ROOTFS_DIR}/"
+	rsync -rtx "${EXPORT_ROOTFS_DIR}/boot/" "${ROOTFS_DIR}/boot/"
+fi

scripts/qcow2_handling

@@ -0,0 +1,256 @@
+#!/bin/bash
+
+# QCOW2 Routines
+
+export CURRENT_IMAGE
+export CURRENT_MOUNTPOINT
+
+export NBD_DEV
+export MAP_BOOT_DEV
+export MAP_ROOT_DEV
+
+# set in build.sh
+# should be fairly enough for the beginning
+# overwrite here by uncommenting following lines
+# BASE_QCOW2_SIZE=12G
+
+# find and initialize free block device nodes
+init_nbd() {
+	modprobe nbd max_part=16
+	if [ -z "${NBD_DEV}" ]; then
+	for x in /sys/class/block/nbd* ; do
+		S=`cat $x/size`
+		if [ "$S" == "0" ] ; then
+			NBD_DEV=/dev/$(basename $x)
+			MAP_BOOT_DEV=/dev/mapper/$(basename $x)p1
+			MAP_ROOT_DEV=/dev/mapper/$(basename $x)p2
+			break
+		fi
+	done
+	fi
+}
+export -f init_nbd
+
+# connect image to block device
+connect_blkdev() {
+	init_nbd
+	qemu-nbd --discard=unmap -c $NBD_DEV "$1"
+	sync
+	kpartx -a $NBD_DEV
+	sync
+	CURRENT_IMAGE="$1"
+}
+export -f connect_blkdev
+
+# disconnect image from block device
+disconnect_blkdev() {
+	kpartx -d $NBD_DEV
+	qemu-nbd -d $NBD_DEV
+	NBD_DEV=
+	MAP_BOOT_DEV=
+	MAP_ROOT_DEV=
+	CURRENT_IMAGE=
+}
+export -f disconnect_blkdev
+
+# mount qcow2 image: mount_image <image file> <mountpoint>
+mount_qimage() {
+	connect_blkdev "$1"
+	mount -v -t ext4 $MAP_ROOT_DEV "$2"
+	mkdir -p "${ROOTFS_DIR}/boot"
+	mount -v -t vfat $MAP_BOOT_DEV "$2/boot"
+	CURRENT_MOUNTPOINT="$2"
+}
+export -f mount_qimage
+
+# umount qcow2 image: umount_image <current mountpoint>
+umount_qimage() {
+	sync
+	#umount "$1/boot"
+	while mount | grep -q "$1"; do
+		local LOCS
+		LOCS=$(mount | grep "$1" | cut -f 3 -d ' ' | sort -r)
+		for loc in $LOCS; do
+			echo "$loc"
+			while mountpoint -q "$loc" && ! umount "$loc"; do
+				sleep 0.1
+			done
+		done
+	done
+	CURRENT_MOUNTPOINT=
+	disconnect_blkdev
+}
+export -f umount_qimage
+
+# create base image / backing image / mount image
+load_qimage() {
+	if [ -z "${CURRENT_MOUNTPOINT}" ]; then
+		if [ ! -d "${ROOTFS_DIR}" ]; then
+			mkdir -p "${ROOTFS_DIR}";
+		fi
+
+		if [ "${CLEAN}" = "1" ] && [ -f "${WORK_DIR}/image-${STAGE}.qcow2" ]; then
+			rm -f "${WORK_DIR}/image-${STAGE}.qcow2";
+		fi
+
+		if [ ! -f "${WORK_DIR}/image-${STAGE}.qcow2" ]; then
+			pushd ${WORK_DIR} > /dev/null
+			init_nbd
+			if [ -z "${PREV_STAGE}" ]; then
+				echo "Creating base image: image-${STAGE}.qcow2"
+				#  -o preallocation=falloc
+				qemu-img create -f qcow2 image-${STAGE}.qcow2 $BASE_QCOW2_SIZE
+				sync
+				qemu-nbd --discard=unmap -c $NBD_DEV image-${STAGE}.qcow2
+				sync
+				sfdisk $NBD_DEV << EOF
+4MiB,250MiB,c,*
+254MiB,,83;
+EOF
+				sync
+				kpartx -a $NBD_DEV
+				mkdosfs -n boot -F 32 -v $MAP_BOOT_DEV
+				mkfs.ext4 -L rootfs -O "^huge_file,^metadata_csum,^64bit" $MAP_ROOT_DEV
+				sync
+			else
+				if [ ! -f "${WORK_DIR}/image-${PREV_STAGE}.qcow2" ]; then
+					exit 1;
+				fi
+				echo "Creating backing image: image-${STAGE}.qcow2 <- ${WORK_DIR}/image-${PREV_STAGE}.qcow2"
+				qemu-img create -f qcow2 \
+					-o backing_file=${WORK_DIR}/image-${PREV_STAGE}.qcow2 \
+					${WORK_DIR}/image-${STAGE}.qcow2
+				sync
+				qemu-nbd --discard=unmap -c $NBD_DEV image-${STAGE}.qcow2
+				sync
+				kpartx -a $NBD_DEV
+			fi
+
+			mount -v -t ext4 $MAP_ROOT_DEV "${ROOTFS_DIR}"
+			mkdir -p "${ROOTFS_DIR}/boot"
+			mount -v -t vfat $MAP_BOOT_DEV "${ROOTFS_DIR}/boot"
+			CURRENT_IMAGE=${WORK_DIR}/image-${STAGE}.qcow2
+			CURRENT_MOUNTPOINT=${ROOTFS_DIR}
+			popd > /dev/null
+		else
+			mount_qimage "${WORK_DIR}/image-${STAGE}.qcow2" "${ROOTFS_DIR}"
+		fi
+		echo "Current image in use: ${CURRENT_IMAGE} (MP: ${CURRENT_MOUNTPOINT})"
+	fi
+}
+export -f load_qimage
+
+# umount current image and refresh mount point env var
+unload_qimage() {
+	if [ ! -z "${CURRENT_MOUNTPOINT}" ]; then
+		fstrim -v "${CURRENT_MOUNTPOINT}" || true
+		umount_qimage "${CURRENT_MOUNTPOINT}"
+	fi
+}
+export -f unload_qimage
+
+# based on: https://github.com/SirLagz/RaspberryPi-ImgAutoSizer
+# helper function for make_bootable_image, do not call directly
+function resize_qcow2() {
+	if [ -z "$CALL_FROM_MBI" ]; then
+		echo "resize_qcow2: cannot be called directly, use make_bootable_image instead"
+		return 1
+	fi
+
+	# ROOT_MARGIN=$((800*1024*1024))
+	ROOT_MARGIN=$((1*1024*1024))
+	PARTED_OUT=`parted -s -m "$NBD_DEV" unit B print`
+	PART_NO=`echo "$PARTED_OUT" | grep ext4 | awk -F: ' { print $1 } '`
+	PART_START=`echo "$PARTED_OUT" | grep ext4 | awk -F: ' { print substr($2,1,length($2)-1) } '`
+
+	e2fsck -y -f $MAP_ROOT_DEV || true
+
+	DATA_SIZE=`resize2fs -P $MAP_ROOT_DEV | awk -F': ' ' { print $2 } '`
+	BLOCK_SIZE=$(dumpe2fs -h $MAP_ROOT_DEV | grep 'Block size' | awk -F': ' ' { print $2 }')
+	BLOCK_SIZE=${BLOCK_SIZE// /}
+
+	let DATA_SIZE=$DATA_SIZE+$ROOT_MARGIN/$BLOCK_SIZE
+	resize2fs -p $MAP_ROOT_DEV $DATA_SIZE
+	sleep 1
+
+	let PART_NEW_SIZE=$DATA_SIZE*$BLOCK_SIZE
+	let PART_NEW_END=$PART_START+$PART_NEW_SIZE
+	ACT1=`parted -s "$NBD_DEV" rm 2`
+	ACT2=`parted -s "$NBD_DEV" unit B mkpart primary $PART_START $PART_NEW_END`
+	NEW_IMG_SIZE=`parted -s -m "$NBD_DEV" unit B print free | tail -1 | awk -F: ' { print substr($2,1,length($2)-1) } '`
+}
+export -f resize_qcow2
+
+# create raw img from qcow2: make_bootable_image <in.qcow2> <out.img>
+function make_bootable_image() {
+    
+	EXPORT_QCOW2="$1"
+	EXPORT_IMAGE="$2"
+
+	echo "Connect block device to source qcow2"
+	connect_blkdev "${EXPORT_QCOW2}"
+
+	echo "Resize fs and partition"
+	CALL_FROM_MBI=1
+	resize_qcow2
+	sync
+	CALL_FROM_MBI=
+
+	echo "Disconnect block device"
+	disconnect_blkdev
+
+	if [ -z "$NEW_IMG_SIZE" ]; then
+		echo "NEW_IMG_SIZE could not be calculated, cannot process image. Exit."
+		exit 1
+	fi
+
+	echo "Shrinking qcow2 image"
+	qemu-img resize --shrink "${EXPORT_QCOW2}" $NEW_IMG_SIZE
+	sync
+
+	echo "Convert qcow2 to raw image"
+	qemu-img convert -f qcow2 -O raw "${EXPORT_QCOW2}" "${EXPORT_IMAGE}"
+	sync
+
+	echo "Get PARTUUIDs from image"
+	IMGID="$(blkid -o value -s PTUUID "${EXPORT_IMAGE}")"
+
+	BOOT_PARTUUID="${IMGID}-01"
+	echo "Boot: $BOOT_PARTUUID"
+	ROOT_PARTUUID="${IMGID}-02"
+	echo "Root: $ROOT_PARTUUID"
+
+	echo "Mount image"
+	MOUNTROOT=${WORK_DIR}/tmpimage
+	mkdir -p $MOUNTROOT
+
+	MOUNTPT=$MOUNTROOT
+	PARTITION=2
+	mount "${EXPORT_IMAGE}" "$MOUNTPT" -o loop,offset=$[ `/sbin/sfdisk -d "${EXPORT_IMAGE}" | grep "start=" | head -n $PARTITION | tail -n1 | sed 's/.*start=[ ]*//' | sed 's/,.*//'` * 512 ],sizelimit=$[ `/sbin/sfdisk -d "${EXPORT_IMAGE}" | grep "start=" | head -n $PARTITION | tail -n1 | sed 's/.*size=[ ]*//' | sed 's/,.*//'` * 512 ] || exit 1
+
+	MOUNTPT=$MOUNTROOT/boot
+	PARTITION=1
+	mount "${EXPORT_IMAGE}" "$MOUNTPT" -o loop,offset=$[ `/sbin/sfdisk -d "${EXPORT_IMAGE}" | grep "start=" | head -n $PARTITION | tail -n1 | sed 's/.*start=[ ]*//' | sed 's/,.*//'` * 512 ],sizelimit=$[ `/sbin/sfdisk -d "${EXPORT_IMAGE}" | grep "start=" | head -n $PARTITION | tail -n1 | sed 's/.*size=[ ]*//' | sed 's/,.*//'` * 512 ] || exit 1
+
+	if [ ! -d "${MOUNTROOT}/root" ]; then
+		echo "Image damaged or not mounted. Exit."
+		exit 1
+	fi
+
+	echo "Setup PARTUUIDs"
+	if [ ! -z "$BOOT_PARTUUID" ] && [ ! -z "$ROOT_PARTUUID" ]; then
+		echo "Set UUIDs to make it bootable"
+		sed -i "s/BOOTDEV/PARTUUID=${BOOT_PARTUUID}/" "${MOUNTROOT}/etc/fstab"
+		sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${MOUNTROOT}/etc/fstab"
+		sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${MOUNTROOT}/boot/cmdline.txt"
+	fi
+
+	echo "Umount image"
+	sync
+	umount "${MOUNTROOT}/boot" || exit 1
+	umount "${MOUNTROOT}" || exit 1
+
+	echo "Remove qcow2 export image"
+	rm -f "${EXPORT_QCOW2}"
+}
+export -f make_bootable_image

stage0/prerun.sh

@@ -1,5 +1,5 @@
 #!/bin/bash -e
 
-if [ ! -d "${ROOTFS_DIR}" ]; then
+if [ ! -d "${ROOTFS_DIR}" ] || [ "${USE_QCOW2}" = "1" ]; then
 	bootstrap ${RELEASE} "${ROOTFS_DIR}" http://raspbian.raspberrypi.org/raspbian/
-fi
+fi

stage2/01-sys-tweaks/00-packages

@@ -2,6 +2,8 @@ ssh less fbset sudo psmisc strace ed ncdu crda
 console-setup keyboard-configuration debconf-utils parted unzip
 build-essential manpages-dev python bash-completion gdb pkg-config
 python-rpi.gpio v4l-utils
+python-gpiozero
+python3-gpiozero
 avahi-daemon
 lua5.1
 luajit

stage2/01-sys-tweaks/01-run.sh

@@ -11,6 +11,18 @@ install -m 644 files/console-setup   	"${ROOTFS_DIR}/etc/default/"
 
 install -m 755 files/rc.local		"${ROOTFS_DIR}/etc/"
 
+if [ -n "${PUBKEY_SSH_FIRST_USER}" ]; then
+	install -v -m 0700 -o 1000 -g 1000 -d "${ROOTFS_DIR}"/home/"${FIRST_USER_NAME}"/.ssh
+	echo "${PUBKEY_SSH_FIRST_USER}" >"${ROOTFS_DIR}"/home/"${FIRST_USER_NAME}"/.ssh/authorized_keys
+	chown 1000:1000 "${ROOTFS_DIR}"/home/"${FIRST_USER_NAME}"/.ssh/authorized_keys
+	chmod 0600 "${ROOTFS_DIR}"/home/"${FIRST_USER_NAME}"/.ssh/authorized_keys
+fi
+
+if [ "${PUBKEY_ONLY_SSH}" = "1" ]; then
+	sed -i -Ee 's/^#?[[:blank:]]*PubkeyAuthentication[[:blank:]]*no[[:blank:]]*$/PubkeyAuthentication yes/
+s/^#?[[:blank:]]*PasswordAuthentication[[:blank:]]*yes[[:blank:]]*$/PasswordAuthentication no/' "${ROOTFS_DIR}"/etc/ssh/sshd_config
+fi
+
 on_chroot << EOF
 systemctl disable hwclock.sh
 systemctl disable nfs-common

stage2/02-net-tweaks/01-run.sh

@@ -25,7 +25,12 @@ network={
 EOL
 fi
 
-# Disable wifi on 5GHz models
+# Disable wifi on 5GHz models if WPA_COUNTRY is not set
 mkdir -p "${ROOTFS_DIR}/var/lib/systemd/rfkill/"
-echo 1 > "${ROOTFS_DIR}/var/lib/systemd/rfkill/platform-3f300000.mmcnr:wlan"
-echo 1 > "${ROOTFS_DIR}/var/lib/systemd/rfkill/platform-fe300000.mmcnr:wlan"
+if [ -n "$WPA_COUNTRY" ]; then
+    echo 0 > "${ROOTFS_DIR}/var/lib/systemd/rfkill/platform-3f300000.mmcnr:wlan"
+    echo 0 > "${ROOTFS_DIR}/var/lib/systemd/rfkill/platform-fe300000.mmcnr:wlan"
+else
+    echo 1 > "${ROOTFS_DIR}/var/lib/systemd/rfkill/platform-3f300000.mmcnr:wlan"
+    echo 1 > "${ROOTFS_DIR}/var/lib/systemd/rfkill/platform-fe300000.mmcnr:wlan"
+fi

stage4/00-install-packages/00-packages

@@ -1,4 +1,4 @@
-python python-tk
+python python3-pygame python-pygame python-tk
 python3 python3-tk thonny
 python3-pgzero
 python-serial python3-serial
@@ -13,9 +13,11 @@ tree
 libgl1-mesa-dri libgles1 libgles2-mesa xcompmgr
 geany
 piclone
+wiringpi pigpio python-pigpio python3-pigpio raspi-gpio python3-rpi.gpio
 python-spidev python3-spidev
 python-twython python3-twython
 python-smbus python3-smbus
+python-flask python3-flask
 pprompt
 piwiz
 rp-prefapps

stage4/00-install-packages/01-packages

@@ -1,4 +1,22 @@
-hunspell-en-gb
-hyphen-en-gb
-wamerican
-wbritish
+python-automationhat python3-automationhat
+python-blinkt python3-blinkt
+python-cap1xxx python3-cap1xxx
+python-drumhat python3-drumhat
+python-envirophat python3-envirophat
+python-explorerhat python3-explorerhat
+python-fourletterphat python3-fourletterphat
+python-microdotphat python3-microdotphat
+python-mote python3-mote
+python-motephat python3-motephat
+python-phatbeat python3-phatbeat
+python-pianohat python3-pianohat
+python-piglow python3-piglow
+python-rainbowhat python3-rainbowhat
+python-scrollphat python3-scrollphat
+python-scrollphathd python3-scrollphathd
+python-sn3218 python3-sn3218
+python-skywriter python3-skywriter
+python-touchphat python3-touchphat
+python-buttonshim python3-buttonshim
+python-unicornhathd  python3-unicornhathd
+python-pantilthat python3-pantilthat

stage4/04-enable-xcompmgr/00-run.sh

@@ -0,0 +1,5 @@
+#!/bin/bash -e
+
+on_chroot << EOF
+	raspi-config nonint do_xcompmgr 0
+EOF