From 969e43a41fde21824d1e6362d61690c80e02494e Mon Sep 17 00:00:00 2001 From: Luigi Cruz Date: Sat, 8 May 2021 00:14:00 -0300 Subject: [PATCH] Merge pi-gen changes. --- .gitignore | 1 + Dockerfile | 5 +- build-docker.sh | 20 +- build.sh | 153 ++++++++++++++- export-image/01-set-sources/01-run.sh | 2 + export-image/03-set-partuuid/00-run.sh | 19 +- export-image/04-finalise/01-run.sh | 30 +-- export-image/prerun.sh | 112 ++++++----- scripts/qcow2_handling | 256 +++++++++++++++++++++++++ stage0/prerun.sh | 4 +- stage2/01-sys-tweaks/00-packages | 2 + stage2/01-sys-tweaks/01-run.sh | 12 ++ stage2/02-net-tweaks/01-run.sh | 11 +- stage4/00-install-packages/00-packages | 4 +- stage4/00-install-packages/01-packages | 26 ++- stage4/04-enable-xcompmgr/00-run.sh | 5 + 16 files changed, 578 insertions(+), 84 deletions(-) create mode 100644 scripts/qcow2_handling create mode 100755 stage4/04-enable-xcompmgr/00-run.sh diff --git a/.gitignore b/.gitignore index 476eccf..bbb0a9c 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ node_modules .DS_Store deploy/* work/* +config postrun.sh SKIP SKIP_IMAGES diff --git a/Dockerfile b/Dockerfile index 2e53149..cb2b4ba 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,5 @@ -FROM debian:buster +ARG BASE_IMAGE=debian:buster +FROM ${BASE_IMAGE} ENV DEBIAN_FRONTEND noninteractive @@ -7,7 +8,7 @@ RUN apt-get -y update && \ git vim parted \ quilt coreutils qemu-user-static debootstrap zerofree zip dosfstools \ bsdtar libcap2-bin rsync grep udev xz-utils curl xxd file kmod bc\ - binfmt-support ca-certificates \ + binfmt-support ca-certificates qemu-utils kpartx \ && rm -rf /var/lib/apt/lists/* COPY . /pi-gen/ diff --git a/build-docker.sh b/build-docker.sh index b6a9ea3..5d31f86 100755 --- a/build-docker.sh +++ b/build-docker.sh @@ -1,4 +1,5 @@ #!/bin/bash -eu + DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" BUILD_OPTS="$*" @@ -73,10 +74,23 @@ fi # Modify original build-options to allow config file to be mounted in the docker container BUILD_OPTS="$(echo "${BUILD_OPTS:-}" | sed -E 's@\-c\s?([^ ]+)@-c /config@')" -${DOCKER} build -t pi-gen "${DIR}" +# Check the arch of the machine we're running on. If it's 64-bit, use a 32-bit base image instead +case "$(uname -m)" in + x86_64|aarch64) + BASE_IMAGE=i386/debian:buster + ;; + *) + BASE_IMAGE=debian:buster + ;; +esac +${DOCKER} build --build-arg BASE_IMAGE=${BASE_IMAGE} -t pi-gen "${DIR}" + if [ "${CONTAINER_EXISTS}" != "" ]; then trap 'echo "got CTRL+C... please wait 5s" && ${DOCKER} stop -t 5 ${CONTAINER_NAME}_cont' SIGINT SIGTERM time ${DOCKER} run --rm --privileged \ + --cap-add=ALL \ + -v /dev:/dev \ + -v /lib/modules:/lib/modules \ --volume "${CONFIG_FILE}":/config:ro \ -e "GIT_HASH=${GIT_HASH}" \ --volumes-from="${CONTAINER_NAME}" --name "${CONTAINER_NAME}_cont" \ @@ -88,6 +102,9 @@ if [ "${CONTAINER_EXISTS}" != "" ]; then else trap 'echo "got CTRL+C... please wait 5s" && ${DOCKER} stop -t 5 ${CONTAINER_NAME}' SIGINT SIGTERM time ${DOCKER} run --name "${CONTAINER_NAME}" --privileged \ + --cap-add=ALL \ + -v /dev:/dev \ + -v /lib/modules:/lib/modules \ --volume "${CONFIG_FILE}":/config:ro \ -e "GIT_HASH=${GIT_HASH}" \ pi-gen \ @@ -96,6 +113,7 @@ else rsync -av work/*/build.log deploy/" & wait "$!" fi + echo "copying results from deploy/" ${DOCKER} cp "${CONTAINER_NAME}":/pi-gen/deploy . ls -lah deploy diff --git a/build.sh b/build.sh index 3b4e44d..9ebbdbf 100755 --- a/build.sh +++ b/build.sh @@ -1,4 +1,5 @@ #!/bin/bash -e + # shellcheck disable=SC2119 run_sub_stage() { @@ -28,6 +29,11 @@ do sleep 15 done EOF + if [ "${USE_QCOW2}" = "1" ]; then + on_chroot << EOF +apt-get clean +EOF + fi fi log "End ${SUB_STAGE_DIR}/${i}-packages-nr" fi @@ -44,6 +50,11 @@ do sleep 15 done EOF + if [ "${USE_QCOW2}" = "1" ]; then + on_chroot << EOF +apt-get clean +EOF + fi fi log "End ${SUB_STAGE_DIR}/${i}-packages" fi @@ -94,17 +105,30 @@ EOF run_stage(){ log "Begin ${STAGE_DIR}" STAGE="$(basename "${STAGE_DIR}")" + pushd "${STAGE_DIR}" > /dev/null - unmount "${WORK_DIR}/${STAGE}" + STAGE_WORK_DIR="${WORK_DIR}/${STAGE}" ROOTFS_DIR="${STAGE_WORK_DIR}"/rootfs + + if [ "${USE_QCOW2}" = "1" ]; then + if [ ! -f SKIP ]; then + load_qimage + fi + else + # make sure we are not umounting during export-image stage + if [ "${USE_QCOW2}" = "0" ] && [ "${NO_PRERUN_QCOW2}" = "0" ]; then + unmount "${WORK_DIR}/${STAGE}" + fi + fi + if [ ! -f SKIP_IMAGES ]; then if [ -f "${STAGE_DIR}/EXPORT_IMAGE" ]; then EXPORT_DIRS="${EXPORT_DIRS} ${STAGE_DIR}" fi fi if [ ! -f SKIP ]; then - if [ "${CLEAN}" = "1" ]; then + if [ "${CLEAN}" = "1" ] && [ "${USE_QCOW2}" = "0" ] ; then if [ -d "${ROOTFS_DIR}" ]; then rm -rf "${ROOTFS_DIR}" fi @@ -115,13 +139,21 @@ run_stage(){ log "End ${STAGE_DIR}/prerun.sh" fi for SUB_STAGE_DIR in "${STAGE_DIR}"/*; do - if [ -d "${SUB_STAGE_DIR}" ] && - [ ! -f "${SUB_STAGE_DIR}/SKIP" ]; then + if [ -d "${SUB_STAGE_DIR}" ] && [ ! -f "${SUB_STAGE_DIR}/SKIP" ]; then run_sub_stage fi done fi - unmount "${WORK_DIR}/${STAGE}" + + if [ "${USE_QCOW2}" = "1" ]; then + unload_qimage + else + # make sure we are not umounting during export-image stage + if [ "${USE_QCOW2}" = "0" ] && [ "${NO_PRERUN_QCOW2}" = "0" ]; then + unmount "${WORK_DIR}/${STAGE}" + fi + fi + PREV_STAGE="${STAGE}" PREV_STAGE_DIR="${STAGE_DIR}" PREV_ROOTFS_DIR="${ROOTFS_DIR}" @@ -155,6 +187,15 @@ do esac done +term() { + if [ "${USE_QCOW2}" = "1" ]; then + log "Unloading image" + unload_qimage + fi +} + +trap term EXIT INT TERM + export PI_GEN=${PI_GEN:-pi-gen} export PI_GEN_REPO=${PI_GEN_REPO:-https://github.com/RPi-Distro/pi-gen} @@ -183,6 +224,7 @@ export WPA_ESSID export WPA_PASSWORD export WPA_COUNTRY export ENABLE_SSH="${ENABLE_SSH:-0}" +export PUBKEY_ONLY_SSH="${PUBKEY_ONLY_SSH:-0}" export LOCALE_DEFAULT="${LOCALE_DEFAULT:-en_GB.UTF-8}" @@ -193,6 +235,8 @@ export TIMEZONE_DEFAULT="${TIMEZONE_DEFAULT:-Europe/London}" export GIT_HASH=${GIT_HASH:-"$(git rev-parse HEAD)"} +export PUBKEY_SSH_FIRST_USER + export CLEAN export IMG_NAME export APT_PROXY @@ -220,6 +264,18 @@ source "${SCRIPT_DIR}/common" # shellcheck source=scripts/dependencies_check source "${SCRIPT_DIR}/dependencies_check" +export NO_PRERUN_QCOW2="${NO_PRERUN_QCOW2:-1}" +export USE_QCOW2="${USE_QCOW2:-1}" +export BASE_QCOW2_SIZE=${BASE_QCOW2_SIZE:-12G} +source "${SCRIPT_DIR}/qcow2_handling" +if [ "${USE_QCOW2}" = "1" ]; then + NO_PRERUN_QCOW2=1 +else + NO_PRERUN_QCOW2=0 +fi + +export NO_PRERUN_QCOW2="${NO_PRERUN_QCOW2:-1}" + dependencies_check "${BASE_DIR}/depends" #check username is valid @@ -238,6 +294,11 @@ if [[ -n "${WPA_PASSWORD}" && ${#WPA_PASSWORD} -lt 8 || ${#WPA_PASSWORD} -gt 63 exit 1 fi +if [[ "${PUBKEY_ONLY_SSH}" = "1" && -z "${PUBKEY_SSH_FIRST_USER}" ]]; then + echo "Must set 'PUBKEY_SSH_FIRST_USER' to a valid SSH public key if using PUBKEY_ONLY_SSH" + exit 1 +fi + mkdir -p "${WORK_DIR}" log "Begin ${BASE_DIR}" @@ -254,22 +315,98 @@ for EXPORT_DIR in ${EXPORT_DIRS}; do # shellcheck source=/dev/null source "${EXPORT_DIR}/EXPORT_IMAGE" EXPORT_ROOTFS_DIR=${WORK_DIR}/$(basename "${EXPORT_DIR}")/rootfs - run_stage + if [ "${USE_QCOW2}" = "1" ]; then + USE_QCOW2=0 + EXPORT_NAME="${IMG_FILENAME}${IMG_SUFFIX}" + echo "------------------------------------------------------------------------" + echo "Running export stage for ${EXPORT_NAME}" + rm -f "${WORK_DIR}/export-image/${EXPORT_NAME}.img" || true + rm -f "${WORK_DIR}/export-image/${EXPORT_NAME}.qcow2" || true + rm -f "${WORK_DIR}/${EXPORT_NAME}.img" || true + rm -f "${WORK_DIR}/${EXPORT_NAME}.qcow2" || true + EXPORT_STAGE=$(basename "${EXPORT_DIR}") + for s in $STAGE_LIST; do + TMP_LIST=${TMP_LIST:+$TMP_LIST }$(basename "${s}") + done + FIRST_STAGE=${TMP_LIST%% *} + FIRST_IMAGE="image-${FIRST_STAGE}.qcow2" + + pushd "${WORK_DIR}" > /dev/null + echo "Creating new base "${EXPORT_NAME}.qcow2" from ${FIRST_IMAGE}" + cp "./${FIRST_IMAGE}" "${EXPORT_NAME}.qcow2" + + ARR=($TMP_LIST) + # rebase stage images to new export base + for CURR_STAGE in "${ARR[@]}"; do + if [ "${CURR_STAGE}" = "${FIRST_STAGE}" ]; then + PREV_IMG="${EXPORT_NAME}" + continue + fi + echo "Rebasing image-${CURR_STAGE}.qcow2 onto ${PREV_IMG}.qcow2" + qemu-img rebase -f qcow2 -u -b ${PREV_IMG}.qcow2 image-${CURR_STAGE}.qcow2 + if [ "${CURR_STAGE}" = "${EXPORT_STAGE}" ]; then + break + fi + PREV_IMG="image-${CURR_STAGE}" + done + + # commit current export stage into base export image + echo "Committing image-${EXPORT_STAGE}.qcow2 to ${EXPORT_NAME}.qcow2" + qemu-img commit -f qcow2 -p -b "${EXPORT_NAME}.qcow2" image-${EXPORT_STAGE}.qcow2 + + # rebase stage images back to original first stage for easy re-run + for CURR_STAGE in "${ARR[@]}"; do + if [ "${CURR_STAGE}" = "${FIRST_STAGE}" ]; then + PREV_IMG="image-${CURR_STAGE}" + continue + fi + echo "Rebasing back image-${CURR_STAGE}.qcow2 onto ${PREV_IMG}.qcow2" + qemu-img rebase -f qcow2 -u -b ${PREV_IMG}.qcow2 image-${CURR_STAGE}.qcow2 + if [ "${CURR_STAGE}" = "${EXPORT_STAGE}" ]; then + break + fi + PREV_IMG="image-${CURR_STAGE}" + done + popd > /dev/null + + mkdir -p "${WORK_DIR}/export-image/rootfs" + mv "${WORK_DIR}/${EXPORT_NAME}.qcow2" "${WORK_DIR}/export-image/" + echo "Mounting image ${WORK_DIR}/export-image/${EXPORT_NAME}.qcow2 to rootfs ${WORK_DIR}/export-image/rootfs" + mount_qimage "${WORK_DIR}/export-image/${EXPORT_NAME}.qcow2" "${WORK_DIR}/export-image/rootfs" + + CLEAN=0 + run_stage + CLEAN=1 + USE_QCOW2=1 + + else + run_stage + fi if [ "${USE_QEMU}" != "1" ]; then if [ -e "${EXPORT_DIR}/EXPORT_NOOBS" ]; then # shellcheck source=/dev/null source "${EXPORT_DIR}/EXPORT_NOOBS" STAGE_DIR="${BASE_DIR}/export-noobs" - run_stage + if [ "${USE_QCOW2}" = "1" ]; then + USE_QCOW2=0 + run_stage + USE_QCOW2=1 + else + run_stage + fi fi fi done -if [ -x ${BASE_DIR}/postrun.sh ]; then +if [ -x postrun.sh ]; then log "Begin postrun.sh" cd "${BASE_DIR}" ./postrun.sh log "End postrun.sh" fi +if [ "${USE_QCOW2}" = "1" ]; then + unload_qimage +fi + log "End ${BASE_DIR}" diff --git a/export-image/01-set-sources/01-run.sh b/export-image/01-set-sources/01-run.sh index 86dea2d..5f51209 100755 --- a/export-image/01-set-sources/01-run.sh +++ b/export-image/01-set-sources/01-run.sh @@ -1,5 +1,7 @@ #!/bin/bash -e +rm -f "${ROOTFS_DIR}/etc/apt/apt.conf.d/51cache" +find "${ROOTFS_DIR}/var/lib/apt/lists/" -type f -delete on_chroot << EOF apt-get update apt-get -y dist-upgrade diff --git a/export-image/03-set-partuuid/00-run.sh b/export-image/03-set-partuuid/00-run.sh index 1538c07..16e1b15 100755 --- a/export-image/03-set-partuuid/00-run.sh +++ b/export-image/03-set-partuuid/00-run.sh @@ -1,13 +1,18 @@ #!/bin/bash -e -IMG_FILE="${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img" +if [ "${NO_PRERUN_QCOW2}" = "0" ]; then -IMGID="$(dd if="${IMG_FILE}" skip=440 bs=1 count=4 2>/dev/null | xxd -e | cut -f 2 -d' ')" + IMG_FILE="${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img" -BOOT_PARTUUID="${IMGID}-01" -ROOT_PARTUUID="${IMGID}-02" + IMGID="$(dd if="${IMG_FILE}" skip=440 bs=1 count=4 2>/dev/null | xxd -e | cut -f 2 -d' ')" -sed -i "s/BOOTDEV/PARTUUID=${BOOT_PARTUUID}/" "${ROOTFS_DIR}/etc/fstab" -sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${ROOTFS_DIR}/etc/fstab" + BOOT_PARTUUID="${IMGID}-01" + ROOT_PARTUUID="${IMGID}-02" + + sed -i "s/BOOTDEV/PARTUUID=${BOOT_PARTUUID}/" "${ROOTFS_DIR}/etc/fstab" + sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${ROOTFS_DIR}/etc/fstab" + + sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${ROOTFS_DIR}/boot/cmdline.txt" + +fi -sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${ROOTFS_DIR}/boot/cmdline.txt" diff --git a/export-image/04-finalise/01-run.sh b/export-image/04-finalise/01-run.sh index ca08af0..d7ace58 100755 --- a/export-image/04-finalise/01-run.sh +++ b/export-image/04-finalise/01-run.sh @@ -16,7 +16,6 @@ if [ -d "${ROOTFS_DIR}/home/${FIRST_USER_NAME}/.config" ]; then chmod 700 "${ROOTFS_DIR}/home/${FIRST_USER_NAME}/.config" fi -rm -f "${ROOTFS_DIR}/etc/apt/apt.conf.d/51cache" rm -f "${ROOTFS_DIR}/usr/bin/qemu-arm-static" if [ "${USE_QEMU}" != "1" ]; then @@ -77,25 +76,30 @@ cp "$ROOTFS_DIR/etc/rpi-issue" "$INFO_FILE" dpkg -l --root "$ROOTFS_DIR" } >> "$INFO_FILE" -ROOT_DEV="$(mount | grep "${ROOTFS_DIR} " | cut -f1 -d' ')" +mkdir -p "${DEPLOY_DIR}" -unmount "${ROOTFS_DIR}" -zerofree "${ROOT_DEV}" +rm -f "${DEPLOY_DIR}/${ZIP_FILENAME}${IMG_SUFFIX}.zip" +rm -f "${DEPLOY_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img" -unmount_image "${IMG_FILE}" +mv "$INFO_FILE" "$DEPLOY_DIR/" -mkdir -p "${DEPLOY_DIR}" +if [ "${USE_QCOW2}" = "0" ] && [ "${NO_PRERUN_QCOW2}" = "0" ]; then + ROOT_DEV="$(mount | grep "${ROOTFS_DIR} " | cut -f1 -d' ')" -rm -f "${DEPLOY_DIR}/${ZIP_FILENAME}${IMG_SUFFIX}.img.xz" -rm -f "${DEPLOY_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img" + unmount "${ROOTFS_DIR}" + zerofree "${ROOT_DEV}" + + unmount_image "${IMG_FILE}" +else + unload_qimage + make_bootable_image "${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.qcow2" "$IMG_FILE" +fi if [ "${DEPLOY_ZIP}" == "1" ]; then pushd "${STAGE_WORK_DIR}" > /dev/null - xz -c "$(basename "${IMG_FILE}")" > "${DEPLOY_DIR}/${ZIP_FILENAME}${IMG_SUFFIX}.img.xz" + zip "${DEPLOY_DIR}/${ZIP_FILENAME}${IMG_SUFFIX}.zip" \ + "$(basename "${IMG_FILE}")" popd > /dev/null - rm -f "${DEPLOY_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img" else - cp "$IMG_FILE" "$DEPLOY_DIR" + mv "$IMG_FILE" "$DEPLOY_DIR/" fi - -cp "$INFO_FILE" "$DEPLOY_DIR" diff --git a/export-image/prerun.sh b/export-image/prerun.sh index 8bbc566..fad7f80 100755 --- a/export-image/prerun.sh +++ b/export-image/prerun.sh @@ -1,61 +1,87 @@ #!/bin/bash -e -IMG_FILE="${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img" +if [ "${NO_PRERUN_QCOW2}" = "0" ]; then + IMG_FILE="${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img" -unmount_image "${IMG_FILE}" + unmount_image "${IMG_FILE}" -rm -f "${IMG_FILE}" + rm -f "${IMG_FILE}" -rm -rf "${ROOTFS_DIR}" -mkdir -p "${ROOTFS_DIR}" + rm -rf "${ROOTFS_DIR}" + mkdir -p "${ROOTFS_DIR}" -BOOT_SIZE="$((256 * 1024 * 1024))" -ROOT_SIZE=$(du --apparent-size -s "${EXPORT_ROOTFS_DIR}" --exclude var/cache/apt/archives --exclude boot --block-size=1 | cut -f 1) + BOOT_SIZE="$((256 * 1024 * 1024))" + ROOT_SIZE=$(du --apparent-size -s "${EXPORT_ROOTFS_DIR}" --exclude var/cache/apt/archives --exclude boot --block-size=1 | cut -f 1) -# All partition sizes and starts will be aligned to this size -ALIGN="$((4 * 1024 * 1024))" -# Add this much space to the calculated file size. This allows for -# some overhead (since actual space usage is usually rounded up to the -# filesystem block size) and gives some free space on the resulting -# image. -ROOT_MARGIN="$(echo "($ROOT_SIZE * 0.2 + 200 * 1024 * 1024) / 1" | bc)" + # All partition sizes and starts will be aligned to this size + ALIGN="$((4 * 1024 * 1024))" + # Add this much space to the calculated file size. This allows for + # some overhead (since actual space usage is usually rounded up to the + # filesystem block size) and gives some free space on the resulting + # image. + ROOT_MARGIN="$(echo "($ROOT_SIZE * 0.2 + 200 * 1024 * 1024) / 1" | bc)" -BOOT_PART_START=$((ALIGN)) -BOOT_PART_SIZE=$(((BOOT_SIZE + ALIGN - 1) / ALIGN * ALIGN)) -ROOT_PART_START=$((BOOT_PART_START + BOOT_PART_SIZE)) -ROOT_PART_SIZE=$(((ROOT_SIZE + ROOT_MARGIN + ALIGN - 1) / ALIGN * ALIGN)) -IMG_SIZE=$((BOOT_PART_START + BOOT_PART_SIZE + ROOT_PART_SIZE)) + BOOT_PART_START=$((ALIGN)) + BOOT_PART_SIZE=$(((BOOT_SIZE + ALIGN - 1) / ALIGN * ALIGN)) + ROOT_PART_START=$((BOOT_PART_START + BOOT_PART_SIZE)) + ROOT_PART_SIZE=$(((ROOT_SIZE + ROOT_MARGIN + ALIGN - 1) / ALIGN * ALIGN)) + IMG_SIZE=$((BOOT_PART_START + BOOT_PART_SIZE + ROOT_PART_SIZE)) -truncate -s "${IMG_SIZE}" "${IMG_FILE}" + truncate -s "${IMG_SIZE}" "${IMG_FILE}" -parted --script "${IMG_FILE}" mklabel msdos -parted --script "${IMG_FILE}" unit B mkpart primary fat32 "${BOOT_PART_START}" "$((BOOT_PART_START + BOOT_PART_SIZE - 1))" -parted --script "${IMG_FILE}" unit B mkpart primary ext4 "${ROOT_PART_START}" "$((ROOT_PART_START + ROOT_PART_SIZE - 1))" + parted --script "${IMG_FILE}" mklabel msdos + parted --script "${IMG_FILE}" unit B mkpart primary fat32 "${BOOT_PART_START}" "$((BOOT_PART_START + BOOT_PART_SIZE - 1))" + parted --script "${IMG_FILE}" unit B mkpart primary ext4 "${ROOT_PART_START}" "$((ROOT_PART_START + ROOT_PART_SIZE - 1))" -PARTED_OUT=$(parted -sm "${IMG_FILE}" unit b print) -BOOT_OFFSET=$(echo "$PARTED_OUT" | grep -e '^1:' | cut -d':' -f 2 | tr -d B) -BOOT_LENGTH=$(echo "$PARTED_OUT" | grep -e '^1:' | cut -d':' -f 4 | tr -d B) + PARTED_OUT=$(parted -sm "${IMG_FILE}" unit b print) + BOOT_OFFSET=$(echo "$PARTED_OUT" | grep -e '^1:' | cut -d':' -f 2 | tr -d B) + BOOT_LENGTH=$(echo "$PARTED_OUT" | grep -e '^1:' | cut -d':' -f 4 | tr -d B) -ROOT_OFFSET=$(echo "$PARTED_OUT" | grep -e '^2:' | cut -d':' -f 2 | tr -d B) -ROOT_LENGTH=$(echo "$PARTED_OUT" | grep -e '^2:' | cut -d':' -f 4 | tr -d B) + ROOT_OFFSET=$(echo "$PARTED_OUT" | grep -e '^2:' | cut -d':' -f 2 | tr -d B) + ROOT_LENGTH=$(echo "$PARTED_OUT" | grep -e '^2:' | cut -d':' -f 4 | tr -d B) -BOOT_DEV=$(losetup --show -f -o "${BOOT_OFFSET}" --sizelimit "${BOOT_LENGTH}" "${IMG_FILE}") -ROOT_DEV=$(losetup --show -f -o "${ROOT_OFFSET}" --sizelimit "${ROOT_LENGTH}" "${IMG_FILE}") -echo "/boot: offset $BOOT_OFFSET, length $BOOT_LENGTH" -echo "/: offset $ROOT_OFFSET, length $ROOT_LENGTH" + echo "Mounting BOOT_DEV..." + cnt=0 + until BOOT_DEV=$(losetup --show -f -o "${BOOT_OFFSET}" --sizelimit "${BOOT_LENGTH}" "${IMG_FILE}"); do + if [ $cnt -lt 5 ]; then + cnt=$((cnt + 1)) + echo "Error in losetup for BOOT_DEV. Retrying..." + sleep 5 + else + echo "ERROR: losetup for BOOT_DEV failed; exiting" + exit 1 + fi + done -ROOT_FEATURES="^huge_file" -for FEATURE in metadata_csum 64bit; do + echo "Mounting ROOT_DEV..." + cnt=0 + until ROOT_DEV=$(losetup --show -f -o "${ROOT_OFFSET}" --sizelimit "${ROOT_LENGTH}" "${IMG_FILE}"); do + if [ $cnt -lt 5 ]; then + cnt=$((cnt + 1)) + echo "Error in losetup for ROOT_DEV. Retrying..." + sleep 5 + else + echo "ERROR: losetup for ROOT_DEV failed; exiting" + exit 1 + fi + done + + echo "/boot: offset $BOOT_OFFSET, length $BOOT_LENGTH" + echo "/: offset $ROOT_OFFSET, length $ROOT_LENGTH" + + ROOT_FEATURES="^huge_file" + for FEATURE in metadata_csum 64bit; do if grep -q "$FEATURE" /etc/mke2fs.conf; then - ROOT_FEATURES="^$FEATURE,$ROOT_FEATURES" + ROOT_FEATURES="^$FEATURE,$ROOT_FEATURES" fi -done -mkdosfs -n boot -F 32 -v "$BOOT_DEV" > /dev/null -mkfs.ext4 -L rootfs -O "$ROOT_FEATURES" "$ROOT_DEV" > /dev/null + done + mkdosfs -n boot -F 32 -v "$BOOT_DEV" > /dev/null + mkfs.ext4 -L rootfs -O "$ROOT_FEATURES" "$ROOT_DEV" > /dev/null -mount -v "$ROOT_DEV" "${ROOTFS_DIR}" -t ext4 -mkdir -p "${ROOTFS_DIR}/boot" -mount -v "$BOOT_DEV" "${ROOTFS_DIR}/boot" -t vfat + mount -v "$ROOT_DEV" "${ROOTFS_DIR}" -t ext4 + mkdir -p "${ROOTFS_DIR}/boot" + mount -v "$BOOT_DEV" "${ROOTFS_DIR}/boot" -t vfat -rsync -aHAXx --exclude /var/cache/apt/archives --exclude /boot "${EXPORT_ROOTFS_DIR}/" "${ROOTFS_DIR}/" -rsync -rtx "${EXPORT_ROOTFS_DIR}/boot/" "${ROOTFS_DIR}/boot/" + rsync -aHAXx --exclude /var/cache/apt/archives --exclude /boot "${EXPORT_ROOTFS_DIR}/" "${ROOTFS_DIR}/" + rsync -rtx "${EXPORT_ROOTFS_DIR}/boot/" "${ROOTFS_DIR}/boot/" +fi diff --git a/scripts/qcow2_handling b/scripts/qcow2_handling new file mode 100644 index 0000000..f722937 --- /dev/null +++ b/scripts/qcow2_handling @@ -0,0 +1,256 @@ +#!/bin/bash + +# QCOW2 Routines + +export CURRENT_IMAGE +export CURRENT_MOUNTPOINT + +export NBD_DEV +export MAP_BOOT_DEV +export MAP_ROOT_DEV + +# set in build.sh +# should be fairly enough for the beginning +# overwrite here by uncommenting following lines +# BASE_QCOW2_SIZE=12G + +# find and initialize free block device nodes +init_nbd() { + modprobe nbd max_part=16 + if [ -z "${NBD_DEV}" ]; then + for x in /sys/class/block/nbd* ; do + S=`cat $x/size` + if [ "$S" == "0" ] ; then + NBD_DEV=/dev/$(basename $x) + MAP_BOOT_DEV=/dev/mapper/$(basename $x)p1 + MAP_ROOT_DEV=/dev/mapper/$(basename $x)p2 + break + fi + done + fi +} +export -f init_nbd + +# connect image to block device +connect_blkdev() { + init_nbd + qemu-nbd --discard=unmap -c $NBD_DEV "$1" + sync + kpartx -a $NBD_DEV + sync + CURRENT_IMAGE="$1" +} +export -f connect_blkdev + +# disconnect image from block device +disconnect_blkdev() { + kpartx -d $NBD_DEV + qemu-nbd -d $NBD_DEV + NBD_DEV= + MAP_BOOT_DEV= + MAP_ROOT_DEV= + CURRENT_IMAGE= +} +export -f disconnect_blkdev + +# mount qcow2 image: mount_image +mount_qimage() { + connect_blkdev "$1" + mount -v -t ext4 $MAP_ROOT_DEV "$2" + mkdir -p "${ROOTFS_DIR}/boot" + mount -v -t vfat $MAP_BOOT_DEV "$2/boot" + CURRENT_MOUNTPOINT="$2" +} +export -f mount_qimage + +# umount qcow2 image: umount_image +umount_qimage() { + sync + #umount "$1/boot" + while mount | grep -q "$1"; do + local LOCS + LOCS=$(mount | grep "$1" | cut -f 3 -d ' ' | sort -r) + for loc in $LOCS; do + echo "$loc" + while mountpoint -q "$loc" && ! umount "$loc"; do + sleep 0.1 + done + done + done + CURRENT_MOUNTPOINT= + disconnect_blkdev +} +export -f umount_qimage + +# create base image / backing image / mount image +load_qimage() { + if [ -z "${CURRENT_MOUNTPOINT}" ]; then + if [ ! -d "${ROOTFS_DIR}" ]; then + mkdir -p "${ROOTFS_DIR}"; + fi + + if [ "${CLEAN}" = "1" ] && [ -f "${WORK_DIR}/image-${STAGE}.qcow2" ]; then + rm -f "${WORK_DIR}/image-${STAGE}.qcow2"; + fi + + if [ ! -f "${WORK_DIR}/image-${STAGE}.qcow2" ]; then + pushd ${WORK_DIR} > /dev/null + init_nbd + if [ -z "${PREV_STAGE}" ]; then + echo "Creating base image: image-${STAGE}.qcow2" + # -o preallocation=falloc + qemu-img create -f qcow2 image-${STAGE}.qcow2 $BASE_QCOW2_SIZE + sync + qemu-nbd --discard=unmap -c $NBD_DEV image-${STAGE}.qcow2 + sync + sfdisk $NBD_DEV << EOF +4MiB,250MiB,c,* +254MiB,,83; +EOF + sync + kpartx -a $NBD_DEV + mkdosfs -n boot -F 32 -v $MAP_BOOT_DEV + mkfs.ext4 -L rootfs -O "^huge_file,^metadata_csum,^64bit" $MAP_ROOT_DEV + sync + else + if [ ! -f "${WORK_DIR}/image-${PREV_STAGE}.qcow2" ]; then + exit 1; + fi + echo "Creating backing image: image-${STAGE}.qcow2 <- ${WORK_DIR}/image-${PREV_STAGE}.qcow2" + qemu-img create -f qcow2 \ + -o backing_file=${WORK_DIR}/image-${PREV_STAGE}.qcow2 \ + ${WORK_DIR}/image-${STAGE}.qcow2 + sync + qemu-nbd --discard=unmap -c $NBD_DEV image-${STAGE}.qcow2 + sync + kpartx -a $NBD_DEV + fi + + mount -v -t ext4 $MAP_ROOT_DEV "${ROOTFS_DIR}" + mkdir -p "${ROOTFS_DIR}/boot" + mount -v -t vfat $MAP_BOOT_DEV "${ROOTFS_DIR}/boot" + CURRENT_IMAGE=${WORK_DIR}/image-${STAGE}.qcow2 + CURRENT_MOUNTPOINT=${ROOTFS_DIR} + popd > /dev/null + else + mount_qimage "${WORK_DIR}/image-${STAGE}.qcow2" "${ROOTFS_DIR}" + fi + echo "Current image in use: ${CURRENT_IMAGE} (MP: ${CURRENT_MOUNTPOINT})" + fi +} +export -f load_qimage + +# umount current image and refresh mount point env var +unload_qimage() { + if [ ! -z "${CURRENT_MOUNTPOINT}" ]; then + fstrim -v "${CURRENT_MOUNTPOINT}" || true + umount_qimage "${CURRENT_MOUNTPOINT}" + fi +} +export -f unload_qimage + +# based on: https://github.com/SirLagz/RaspberryPi-ImgAutoSizer +# helper function for make_bootable_image, do not call directly +function resize_qcow2() { + if [ -z "$CALL_FROM_MBI" ]; then + echo "resize_qcow2: cannot be called directly, use make_bootable_image instead" + return 1 + fi + + # ROOT_MARGIN=$((800*1024*1024)) + ROOT_MARGIN=$((1*1024*1024)) + PARTED_OUT=`parted -s -m "$NBD_DEV" unit B print` + PART_NO=`echo "$PARTED_OUT" | grep ext4 | awk -F: ' { print $1 } '` + PART_START=`echo "$PARTED_OUT" | grep ext4 | awk -F: ' { print substr($2,1,length($2)-1) } '` + + e2fsck -y -f $MAP_ROOT_DEV || true + + DATA_SIZE=`resize2fs -P $MAP_ROOT_DEV | awk -F': ' ' { print $2 } '` + BLOCK_SIZE=$(dumpe2fs -h $MAP_ROOT_DEV | grep 'Block size' | awk -F': ' ' { print $2 }') + BLOCK_SIZE=${BLOCK_SIZE// /} + + let DATA_SIZE=$DATA_SIZE+$ROOT_MARGIN/$BLOCK_SIZE + resize2fs -p $MAP_ROOT_DEV $DATA_SIZE + sleep 1 + + let PART_NEW_SIZE=$DATA_SIZE*$BLOCK_SIZE + let PART_NEW_END=$PART_START+$PART_NEW_SIZE + ACT1=`parted -s "$NBD_DEV" rm 2` + ACT2=`parted -s "$NBD_DEV" unit B mkpart primary $PART_START $PART_NEW_END` + NEW_IMG_SIZE=`parted -s -m "$NBD_DEV" unit B print free | tail -1 | awk -F: ' { print substr($2,1,length($2)-1) } '` +} +export -f resize_qcow2 + +# create raw img from qcow2: make_bootable_image +function make_bootable_image() { + + EXPORT_QCOW2="$1" + EXPORT_IMAGE="$2" + + echo "Connect block device to source qcow2" + connect_blkdev "${EXPORT_QCOW2}" + + echo "Resize fs and partition" + CALL_FROM_MBI=1 + resize_qcow2 + sync + CALL_FROM_MBI= + + echo "Disconnect block device" + disconnect_blkdev + + if [ -z "$NEW_IMG_SIZE" ]; then + echo "NEW_IMG_SIZE could not be calculated, cannot process image. Exit." + exit 1 + fi + + echo "Shrinking qcow2 image" + qemu-img resize --shrink "${EXPORT_QCOW2}" $NEW_IMG_SIZE + sync + + echo "Convert qcow2 to raw image" + qemu-img convert -f qcow2 -O raw "${EXPORT_QCOW2}" "${EXPORT_IMAGE}" + sync + + echo "Get PARTUUIDs from image" + IMGID="$(blkid -o value -s PTUUID "${EXPORT_IMAGE}")" + + BOOT_PARTUUID="${IMGID}-01" + echo "Boot: $BOOT_PARTUUID" + ROOT_PARTUUID="${IMGID}-02" + echo "Root: $ROOT_PARTUUID" + + echo "Mount image" + MOUNTROOT=${WORK_DIR}/tmpimage + mkdir -p $MOUNTROOT + + MOUNTPT=$MOUNTROOT + PARTITION=2 + mount "${EXPORT_IMAGE}" "$MOUNTPT" -o loop,offset=$[ `/sbin/sfdisk -d "${EXPORT_IMAGE}" | grep "start=" | head -n $PARTITION | tail -n1 | sed 's/.*start=[ ]*//' | sed 's/,.*//'` * 512 ],sizelimit=$[ `/sbin/sfdisk -d "${EXPORT_IMAGE}" | grep "start=" | head -n $PARTITION | tail -n1 | sed 's/.*size=[ ]*//' | sed 's/,.*//'` * 512 ] || exit 1 + + MOUNTPT=$MOUNTROOT/boot + PARTITION=1 + mount "${EXPORT_IMAGE}" "$MOUNTPT" -o loop,offset=$[ `/sbin/sfdisk -d "${EXPORT_IMAGE}" | grep "start=" | head -n $PARTITION | tail -n1 | sed 's/.*start=[ ]*//' | sed 's/,.*//'` * 512 ],sizelimit=$[ `/sbin/sfdisk -d "${EXPORT_IMAGE}" | grep "start=" | head -n $PARTITION | tail -n1 | sed 's/.*size=[ ]*//' | sed 's/,.*//'` * 512 ] || exit 1 + + if [ ! -d "${MOUNTROOT}/root" ]; then + echo "Image damaged or not mounted. Exit." + exit 1 + fi + + echo "Setup PARTUUIDs" + if [ ! -z "$BOOT_PARTUUID" ] && [ ! -z "$ROOT_PARTUUID" ]; then + echo "Set UUIDs to make it bootable" + sed -i "s/BOOTDEV/PARTUUID=${BOOT_PARTUUID}/" "${MOUNTROOT}/etc/fstab" + sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${MOUNTROOT}/etc/fstab" + sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${MOUNTROOT}/boot/cmdline.txt" + fi + + echo "Umount image" + sync + umount "${MOUNTROOT}/boot" || exit 1 + umount "${MOUNTROOT}" || exit 1 + + echo "Remove qcow2 export image" + rm -f "${EXPORT_QCOW2}" +} +export -f make_bootable_image diff --git a/stage0/prerun.sh b/stage0/prerun.sh index cc29e5c..d4dd0a1 100755 --- a/stage0/prerun.sh +++ b/stage0/prerun.sh @@ -1,5 +1,5 @@ #!/bin/bash -e -if [ ! -d "${ROOTFS_DIR}" ]; then +if [ ! -d "${ROOTFS_DIR}" ] || [ "${USE_QCOW2}" = "1" ]; then bootstrap ${RELEASE} "${ROOTFS_DIR}" http://raspbian.raspberrypi.org/raspbian/ -fi \ No newline at end of file +fi diff --git a/stage2/01-sys-tweaks/00-packages b/stage2/01-sys-tweaks/00-packages index 83ec74c..a1f4b59 100644 --- a/stage2/01-sys-tweaks/00-packages +++ b/stage2/01-sys-tweaks/00-packages @@ -2,6 +2,8 @@ ssh less fbset sudo psmisc strace ed ncdu crda console-setup keyboard-configuration debconf-utils parted unzip build-essential manpages-dev python bash-completion gdb pkg-config python-rpi.gpio v4l-utils +python-gpiozero +python3-gpiozero avahi-daemon lua5.1 luajit diff --git a/stage2/01-sys-tweaks/01-run.sh b/stage2/01-sys-tweaks/01-run.sh index 8d28adc..c1836f9 100755 --- a/stage2/01-sys-tweaks/01-run.sh +++ b/stage2/01-sys-tweaks/01-run.sh @@ -11,6 +11,18 @@ install -m 644 files/console-setup "${ROOTFS_DIR}/etc/default/" install -m 755 files/rc.local "${ROOTFS_DIR}/etc/" +if [ -n "${PUBKEY_SSH_FIRST_USER}" ]; then + install -v -m 0700 -o 1000 -g 1000 -d "${ROOTFS_DIR}"/home/"${FIRST_USER_NAME}"/.ssh + echo "${PUBKEY_SSH_FIRST_USER}" >"${ROOTFS_DIR}"/home/"${FIRST_USER_NAME}"/.ssh/authorized_keys + chown 1000:1000 "${ROOTFS_DIR}"/home/"${FIRST_USER_NAME}"/.ssh/authorized_keys + chmod 0600 "${ROOTFS_DIR}"/home/"${FIRST_USER_NAME}"/.ssh/authorized_keys +fi + +if [ "${PUBKEY_ONLY_SSH}" = "1" ]; then + sed -i -Ee 's/^#?[[:blank:]]*PubkeyAuthentication[[:blank:]]*no[[:blank:]]*$/PubkeyAuthentication yes/ +s/^#?[[:blank:]]*PasswordAuthentication[[:blank:]]*yes[[:blank:]]*$/PasswordAuthentication no/' "${ROOTFS_DIR}"/etc/ssh/sshd_config +fi + on_chroot << EOF systemctl disable hwclock.sh systemctl disable nfs-common diff --git a/stage2/02-net-tweaks/01-run.sh b/stage2/02-net-tweaks/01-run.sh index 0929755..d82381b 100755 --- a/stage2/02-net-tweaks/01-run.sh +++ b/stage2/02-net-tweaks/01-run.sh @@ -25,7 +25,12 @@ network={ EOL fi -# Disable wifi on 5GHz models +# Disable wifi on 5GHz models if WPA_COUNTRY is not set mkdir -p "${ROOTFS_DIR}/var/lib/systemd/rfkill/" -echo 1 > "${ROOTFS_DIR}/var/lib/systemd/rfkill/platform-3f300000.mmcnr:wlan" -echo 1 > "${ROOTFS_DIR}/var/lib/systemd/rfkill/platform-fe300000.mmcnr:wlan" +if [ -n "$WPA_COUNTRY" ]; then + echo 0 > "${ROOTFS_DIR}/var/lib/systemd/rfkill/platform-3f300000.mmcnr:wlan" + echo 0 > "${ROOTFS_DIR}/var/lib/systemd/rfkill/platform-fe300000.mmcnr:wlan" +else + echo 1 > "${ROOTFS_DIR}/var/lib/systemd/rfkill/platform-3f300000.mmcnr:wlan" + echo 1 > "${ROOTFS_DIR}/var/lib/systemd/rfkill/platform-fe300000.mmcnr:wlan" +fi diff --git a/stage4/00-install-packages/00-packages b/stage4/00-install-packages/00-packages index 80f1a91..85a9f80 100644 --- a/stage4/00-install-packages/00-packages +++ b/stage4/00-install-packages/00-packages @@ -1,4 +1,4 @@ -python python-tk +python python3-pygame python-pygame python-tk python3 python3-tk thonny python3-pgzero python-serial python3-serial @@ -13,9 +13,11 @@ tree libgl1-mesa-dri libgles1 libgles2-mesa xcompmgr geany piclone +wiringpi pigpio python-pigpio python3-pigpio raspi-gpio python3-rpi.gpio python-spidev python3-spidev python-twython python3-twython python-smbus python3-smbus +python-flask python3-flask pprompt piwiz rp-prefapps diff --git a/stage4/00-install-packages/01-packages b/stage4/00-install-packages/01-packages index fc43d17..fdf5e73 100644 --- a/stage4/00-install-packages/01-packages +++ b/stage4/00-install-packages/01-packages @@ -1,4 +1,22 @@ -hunspell-en-gb -hyphen-en-gb -wamerican -wbritish +python-automationhat python3-automationhat +python-blinkt python3-blinkt +python-cap1xxx python3-cap1xxx +python-drumhat python3-drumhat +python-envirophat python3-envirophat +python-explorerhat python3-explorerhat +python-fourletterphat python3-fourletterphat +python-microdotphat python3-microdotphat +python-mote python3-mote +python-motephat python3-motephat +python-phatbeat python3-phatbeat +python-pianohat python3-pianohat +python-piglow python3-piglow +python-rainbowhat python3-rainbowhat +python-scrollphat python3-scrollphat +python-scrollphathd python3-scrollphathd +python-sn3218 python3-sn3218 +python-skywriter python3-skywriter +python-touchphat python3-touchphat +python-buttonshim python3-buttonshim +python-unicornhathd python3-unicornhathd +python-pantilthat python3-pantilthat diff --git a/stage4/04-enable-xcompmgr/00-run.sh b/stage4/04-enable-xcompmgr/00-run.sh new file mode 100755 index 0000000..ad6298d --- /dev/null +++ b/stage4/04-enable-xcompmgr/00-run.sh @@ -0,0 +1,5 @@ +#!/bin/bash -e + +on_chroot << EOF + raspi-config nonint do_xcompmgr 0 +EOF