Browse Source

Merge pi-gen changes.

tags/5.1.0
Luigi Cruz 3 years ago
parent
commit
969e43a41f
16 changed files with 578 additions and 84 deletions
  1. +1
    -0
      .gitignore
  2. +3
    -2
      Dockerfile
  3. +19
    -1
      build-docker.sh
  4. +145
    -8
      build.sh
  5. +2
    -0
      export-image/01-set-sources/01-run.sh
  6. +12
    -7
      export-image/03-set-partuuid/00-run.sh
  7. +17
    -13
      export-image/04-finalise/01-run.sh
  8. +69
    -43
      export-image/prerun.sh
  9. +256
    -0
      scripts/qcow2_handling
  10. +2
    -2
      stage0/prerun.sh
  11. +2
    -0
      stage2/01-sys-tweaks/00-packages
  12. +12
    -0
      stage2/01-sys-tweaks/01-run.sh
  13. +8
    -3
      stage2/02-net-tweaks/01-run.sh
  14. +3
    -1
      stage4/00-install-packages/00-packages
  15. +22
    -4
      stage4/00-install-packages/01-packages
  16. +5
    -0
      stage4/04-enable-xcompmgr/00-run.sh

+ 1
- 0
.gitignore View File

@@ -7,6 +7,7 @@ node_modules
.DS_Store
deploy/*
work/*
config
postrun.sh
SKIP
SKIP_IMAGES


+ 3
- 2
Dockerfile View File

@@ -1,4 +1,5 @@
FROM debian:buster
ARG BASE_IMAGE=debian:buster
FROM ${BASE_IMAGE}

ENV DEBIAN_FRONTEND noninteractive

@@ -7,7 +8,7 @@ RUN apt-get -y update && \
git vim parted \
quilt coreutils qemu-user-static debootstrap zerofree zip dosfstools \
bsdtar libcap2-bin rsync grep udev xz-utils curl xxd file kmod bc\
binfmt-support ca-certificates \
binfmt-support ca-certificates qemu-utils kpartx \
&& rm -rf /var/lib/apt/lists/*

COPY . /pi-gen/


+ 19
- 1
build-docker.sh View File

@@ -1,4 +1,5 @@
#!/bin/bash -eu

DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"

BUILD_OPTS="$*"
@@ -73,10 +74,23 @@ fi
# Modify original build-options to allow config file to be mounted in the docker container
BUILD_OPTS="$(echo "${BUILD_OPTS:-}" | sed -E 's@\-c\s?([^ ]+)@-c /config@')"

${DOCKER} build -t pi-gen "${DIR}"
# Check the arch of the machine we're running on. If it's 64-bit, use a 32-bit base image instead
case "$(uname -m)" in
x86_64|aarch64)
BASE_IMAGE=i386/debian:buster
;;
*)
BASE_IMAGE=debian:buster
;;
esac
${DOCKER} build --build-arg BASE_IMAGE=${BASE_IMAGE} -t pi-gen "${DIR}"

if [ "${CONTAINER_EXISTS}" != "" ]; then
trap 'echo "got CTRL+C... please wait 5s" && ${DOCKER} stop -t 5 ${CONTAINER_NAME}_cont' SIGINT SIGTERM
time ${DOCKER} run --rm --privileged \
--cap-add=ALL \
-v /dev:/dev \
-v /lib/modules:/lib/modules \
--volume "${CONFIG_FILE}":/config:ro \
-e "GIT_HASH=${GIT_HASH}" \
--volumes-from="${CONTAINER_NAME}" --name "${CONTAINER_NAME}_cont" \
@@ -88,6 +102,9 @@ if [ "${CONTAINER_EXISTS}" != "" ]; then
else
trap 'echo "got CTRL+C... please wait 5s" && ${DOCKER} stop -t 5 ${CONTAINER_NAME}' SIGINT SIGTERM
time ${DOCKER} run --name "${CONTAINER_NAME}" --privileged \
--cap-add=ALL \
-v /dev:/dev \
-v /lib/modules:/lib/modules \
--volume "${CONFIG_FILE}":/config:ro \
-e "GIT_HASH=${GIT_HASH}" \
pi-gen \
@@ -96,6 +113,7 @@ else
rsync -av work/*/build.log deploy/" &
wait "$!"
fi

echo "copying results from deploy/"
${DOCKER} cp "${CONTAINER_NAME}":/pi-gen/deploy .
ls -lah deploy


+ 145
- 8
build.sh View File

@@ -1,4 +1,5 @@
#!/bin/bash -e

# shellcheck disable=SC2119
run_sub_stage()
{
@@ -28,6 +29,11 @@ do
sleep 15
done
EOF
if [ "${USE_QCOW2}" = "1" ]; then
on_chroot << EOF
apt-get clean
EOF
fi
fi
log "End ${SUB_STAGE_DIR}/${i}-packages-nr"
fi
@@ -44,6 +50,11 @@ do
sleep 15
done
EOF
if [ "${USE_QCOW2}" = "1" ]; then
on_chroot << EOF
apt-get clean
EOF
fi
fi
log "End ${SUB_STAGE_DIR}/${i}-packages"
fi
@@ -94,17 +105,30 @@ EOF
run_stage(){
log "Begin ${STAGE_DIR}"
STAGE="$(basename "${STAGE_DIR}")"

pushd "${STAGE_DIR}" > /dev/null
unmount "${WORK_DIR}/${STAGE}"
STAGE_WORK_DIR="${WORK_DIR}/${STAGE}"
ROOTFS_DIR="${STAGE_WORK_DIR}"/rootfs

if [ "${USE_QCOW2}" = "1" ]; then
if [ ! -f SKIP ]; then
load_qimage
fi
else
# make sure we are not umounting during export-image stage
if [ "${USE_QCOW2}" = "0" ] && [ "${NO_PRERUN_QCOW2}" = "0" ]; then
unmount "${WORK_DIR}/${STAGE}"
fi
fi
if [ ! -f SKIP_IMAGES ]; then
if [ -f "${STAGE_DIR}/EXPORT_IMAGE" ]; then
EXPORT_DIRS="${EXPORT_DIRS} ${STAGE_DIR}"
fi
fi
if [ ! -f SKIP ]; then
if [ "${CLEAN}" = "1" ]; then
if [ "${CLEAN}" = "1" ] && [ "${USE_QCOW2}" = "0" ] ; then
if [ -d "${ROOTFS_DIR}" ]; then
rm -rf "${ROOTFS_DIR}"
fi
@@ -115,13 +139,21 @@ run_stage(){
log "End ${STAGE_DIR}/prerun.sh"
fi
for SUB_STAGE_DIR in "${STAGE_DIR}"/*; do
if [ -d "${SUB_STAGE_DIR}" ] &&
[ ! -f "${SUB_STAGE_DIR}/SKIP" ]; then
if [ -d "${SUB_STAGE_DIR}" ] && [ ! -f "${SUB_STAGE_DIR}/SKIP" ]; then
run_sub_stage
fi
done
fi
unmount "${WORK_DIR}/${STAGE}"

if [ "${USE_QCOW2}" = "1" ]; then
unload_qimage
else
# make sure we are not umounting during export-image stage
if [ "${USE_QCOW2}" = "0" ] && [ "${NO_PRERUN_QCOW2}" = "0" ]; then
unmount "${WORK_DIR}/${STAGE}"
fi
fi

PREV_STAGE="${STAGE}"
PREV_STAGE_DIR="${STAGE_DIR}"
PREV_ROOTFS_DIR="${ROOTFS_DIR}"
@@ -155,6 +187,15 @@ do
esac
done

term() {
if [ "${USE_QCOW2}" = "1" ]; then
log "Unloading image"
unload_qimage
fi
}

trap term EXIT INT TERM

export PI_GEN=${PI_GEN:-pi-gen}
export PI_GEN_REPO=${PI_GEN_REPO:-https://github.com/RPi-Distro/pi-gen}

@@ -183,6 +224,7 @@ export WPA_ESSID
export WPA_PASSWORD
export WPA_COUNTRY
export ENABLE_SSH="${ENABLE_SSH:-0}"
export PUBKEY_ONLY_SSH="${PUBKEY_ONLY_SSH:-0}"

export LOCALE_DEFAULT="${LOCALE_DEFAULT:-en_GB.UTF-8}"

@@ -193,6 +235,8 @@ export TIMEZONE_DEFAULT="${TIMEZONE_DEFAULT:-Europe/London}"

export GIT_HASH=${GIT_HASH:-"$(git rev-parse HEAD)"}

export PUBKEY_SSH_FIRST_USER

export CLEAN
export IMG_NAME
export APT_PROXY
@@ -220,6 +264,18 @@ source "${SCRIPT_DIR}/common"
# shellcheck source=scripts/dependencies_check
source "${SCRIPT_DIR}/dependencies_check"

export NO_PRERUN_QCOW2="${NO_PRERUN_QCOW2:-1}"
export USE_QCOW2="${USE_QCOW2:-1}"
export BASE_QCOW2_SIZE=${BASE_QCOW2_SIZE:-12G}
source "${SCRIPT_DIR}/qcow2_handling"
if [ "${USE_QCOW2}" = "1" ]; then
NO_PRERUN_QCOW2=1
else
NO_PRERUN_QCOW2=0
fi

export NO_PRERUN_QCOW2="${NO_PRERUN_QCOW2:-1}"

dependencies_check "${BASE_DIR}/depends"

#check username is valid
@@ -238,6 +294,11 @@ if [[ -n "${WPA_PASSWORD}" && ${#WPA_PASSWORD} -lt 8 || ${#WPA_PASSWORD} -gt 63
exit 1
fi

if [[ "${PUBKEY_ONLY_SSH}" = "1" && -z "${PUBKEY_SSH_FIRST_USER}" ]]; then
echo "Must set 'PUBKEY_SSH_FIRST_USER' to a valid SSH public key if using PUBKEY_ONLY_SSH"
exit 1
fi

mkdir -p "${WORK_DIR}"
log "Begin ${BASE_DIR}"

@@ -254,22 +315,98 @@ for EXPORT_DIR in ${EXPORT_DIRS}; do
# shellcheck source=/dev/null
source "${EXPORT_DIR}/EXPORT_IMAGE"
EXPORT_ROOTFS_DIR=${WORK_DIR}/$(basename "${EXPORT_DIR}")/rootfs
run_stage
if [ "${USE_QCOW2}" = "1" ]; then
USE_QCOW2=0
EXPORT_NAME="${IMG_FILENAME}${IMG_SUFFIX}"
echo "------------------------------------------------------------------------"
echo "Running export stage for ${EXPORT_NAME}"
rm -f "${WORK_DIR}/export-image/${EXPORT_NAME}.img" || true
rm -f "${WORK_DIR}/export-image/${EXPORT_NAME}.qcow2" || true
rm -f "${WORK_DIR}/${EXPORT_NAME}.img" || true
rm -f "${WORK_DIR}/${EXPORT_NAME}.qcow2" || true
EXPORT_STAGE=$(basename "${EXPORT_DIR}")
for s in $STAGE_LIST; do
TMP_LIST=${TMP_LIST:+$TMP_LIST }$(basename "${s}")
done
FIRST_STAGE=${TMP_LIST%% *}
FIRST_IMAGE="image-${FIRST_STAGE}.qcow2"

pushd "${WORK_DIR}" > /dev/null
echo "Creating new base "${EXPORT_NAME}.qcow2" from ${FIRST_IMAGE}"
cp "./${FIRST_IMAGE}" "${EXPORT_NAME}.qcow2"

ARR=($TMP_LIST)
# rebase stage images to new export base
for CURR_STAGE in "${ARR[@]}"; do
if [ "${CURR_STAGE}" = "${FIRST_STAGE}" ]; then
PREV_IMG="${EXPORT_NAME}"
continue
fi
echo "Rebasing image-${CURR_STAGE}.qcow2 onto ${PREV_IMG}.qcow2"
qemu-img rebase -f qcow2 -u -b ${PREV_IMG}.qcow2 image-${CURR_STAGE}.qcow2
if [ "${CURR_STAGE}" = "${EXPORT_STAGE}" ]; then
break
fi
PREV_IMG="image-${CURR_STAGE}"
done

# commit current export stage into base export image
echo "Committing image-${EXPORT_STAGE}.qcow2 to ${EXPORT_NAME}.qcow2"
qemu-img commit -f qcow2 -p -b "${EXPORT_NAME}.qcow2" image-${EXPORT_STAGE}.qcow2

# rebase stage images back to original first stage for easy re-run
for CURR_STAGE in "${ARR[@]}"; do
if [ "${CURR_STAGE}" = "${FIRST_STAGE}" ]; then
PREV_IMG="image-${CURR_STAGE}"
continue
fi
echo "Rebasing back image-${CURR_STAGE}.qcow2 onto ${PREV_IMG}.qcow2"
qemu-img rebase -f qcow2 -u -b ${PREV_IMG}.qcow2 image-${CURR_STAGE}.qcow2
if [ "${CURR_STAGE}" = "${EXPORT_STAGE}" ]; then
break
fi
PREV_IMG="image-${CURR_STAGE}"
done
popd > /dev/null

mkdir -p "${WORK_DIR}/export-image/rootfs"
mv "${WORK_DIR}/${EXPORT_NAME}.qcow2" "${WORK_DIR}/export-image/"
echo "Mounting image ${WORK_DIR}/export-image/${EXPORT_NAME}.qcow2 to rootfs ${WORK_DIR}/export-image/rootfs"
mount_qimage "${WORK_DIR}/export-image/${EXPORT_NAME}.qcow2" "${WORK_DIR}/export-image/rootfs"

CLEAN=0
run_stage
CLEAN=1
USE_QCOW2=1

else
run_stage
fi
if [ "${USE_QEMU}" != "1" ]; then
if [ -e "${EXPORT_DIR}/EXPORT_NOOBS" ]; then
# shellcheck source=/dev/null
source "${EXPORT_DIR}/EXPORT_NOOBS"
STAGE_DIR="${BASE_DIR}/export-noobs"
run_stage
if [ "${USE_QCOW2}" = "1" ]; then
USE_QCOW2=0
run_stage
USE_QCOW2=1
else
run_stage
fi
fi
fi
done

if [ -x ${BASE_DIR}/postrun.sh ]; then
if [ -x postrun.sh ]; then
log "Begin postrun.sh"
cd "${BASE_DIR}"
./postrun.sh
log "End postrun.sh"
fi

if [ "${USE_QCOW2}" = "1" ]; then
unload_qimage
fi

log "End ${BASE_DIR}"

+ 2
- 0
export-image/01-set-sources/01-run.sh View File

@@ -1,5 +1,7 @@
#!/bin/bash -e

rm -f "${ROOTFS_DIR}/etc/apt/apt.conf.d/51cache"
find "${ROOTFS_DIR}/var/lib/apt/lists/" -type f -delete
on_chroot << EOF
apt-get update
apt-get -y dist-upgrade


+ 12
- 7
export-image/03-set-partuuid/00-run.sh View File

@@ -1,13 +1,18 @@
#!/bin/bash -e

IMG_FILE="${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img"
if [ "${NO_PRERUN_QCOW2}" = "0" ]; then

IMGID="$(dd if="${IMG_FILE}" skip=440 bs=1 count=4 2>/dev/null | xxd -e | cut -f 2 -d' ')"
IMG_FILE="${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img"

BOOT_PARTUUID="${IMGID}-01"
ROOT_PARTUUID="${IMGID}-02"
IMGID="$(dd if="${IMG_FILE}" skip=440 bs=1 count=4 2>/dev/null | xxd -e | cut -f 2 -d' ')"

sed -i "s/BOOTDEV/PARTUUID=${BOOT_PARTUUID}/" "${ROOTFS_DIR}/etc/fstab"
sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${ROOTFS_DIR}/etc/fstab"
BOOT_PARTUUID="${IMGID}-01"
ROOT_PARTUUID="${IMGID}-02"

sed -i "s/BOOTDEV/PARTUUID=${BOOT_PARTUUID}/" "${ROOTFS_DIR}/etc/fstab"
sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${ROOTFS_DIR}/etc/fstab"

sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${ROOTFS_DIR}/boot/cmdline.txt"

fi

sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${ROOTFS_DIR}/boot/cmdline.txt"

+ 17
- 13
export-image/04-finalise/01-run.sh View File

@@ -16,7 +16,6 @@ if [ -d "${ROOTFS_DIR}/home/${FIRST_USER_NAME}/.config" ]; then
chmod 700 "${ROOTFS_DIR}/home/${FIRST_USER_NAME}/.config"
fi

rm -f "${ROOTFS_DIR}/etc/apt/apt.conf.d/51cache"
rm -f "${ROOTFS_DIR}/usr/bin/qemu-arm-static"

if [ "${USE_QEMU}" != "1" ]; then
@@ -77,25 +76,30 @@ cp "$ROOTFS_DIR/etc/rpi-issue" "$INFO_FILE"
dpkg -l --root "$ROOTFS_DIR"
} >> "$INFO_FILE"

ROOT_DEV="$(mount | grep "${ROOTFS_DIR} " | cut -f1 -d' ')"
mkdir -p "${DEPLOY_DIR}"

unmount "${ROOTFS_DIR}"
zerofree "${ROOT_DEV}"
rm -f "${DEPLOY_DIR}/${ZIP_FILENAME}${IMG_SUFFIX}.zip"
rm -f "${DEPLOY_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img"

unmount_image "${IMG_FILE}"
mv "$INFO_FILE" "$DEPLOY_DIR/"

mkdir -p "${DEPLOY_DIR}"
if [ "${USE_QCOW2}" = "0" ] && [ "${NO_PRERUN_QCOW2}" = "0" ]; then
ROOT_DEV="$(mount | grep "${ROOTFS_DIR} " | cut -f1 -d' ')"

rm -f "${DEPLOY_DIR}/${ZIP_FILENAME}${IMG_SUFFIX}.img.xz"
rm -f "${DEPLOY_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img"
unmount "${ROOTFS_DIR}"
zerofree "${ROOT_DEV}"

unmount_image "${IMG_FILE}"
else
unload_qimage
make_bootable_image "${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.qcow2" "$IMG_FILE"
fi

if [ "${DEPLOY_ZIP}" == "1" ]; then
pushd "${STAGE_WORK_DIR}" > /dev/null
xz -c "$(basename "${IMG_FILE}")" > "${DEPLOY_DIR}/${ZIP_FILENAME}${IMG_SUFFIX}.img.xz"
zip "${DEPLOY_DIR}/${ZIP_FILENAME}${IMG_SUFFIX}.zip" \
"$(basename "${IMG_FILE}")"
popd > /dev/null
rm -f "${DEPLOY_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img"
else
cp "$IMG_FILE" "$DEPLOY_DIR"
mv "$IMG_FILE" "$DEPLOY_DIR/"
fi

cp "$INFO_FILE" "$DEPLOY_DIR"

+ 69
- 43
export-image/prerun.sh View File

@@ -1,61 +1,87 @@
#!/bin/bash -e

IMG_FILE="${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img"
if [ "${NO_PRERUN_QCOW2}" = "0" ]; then
IMG_FILE="${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img"

unmount_image "${IMG_FILE}"
unmount_image "${IMG_FILE}"

rm -f "${IMG_FILE}"
rm -f "${IMG_FILE}"

rm -rf "${ROOTFS_DIR}"
mkdir -p "${ROOTFS_DIR}"
rm -rf "${ROOTFS_DIR}"
mkdir -p "${ROOTFS_DIR}"

BOOT_SIZE="$((256 * 1024 * 1024))"
ROOT_SIZE=$(du --apparent-size -s "${EXPORT_ROOTFS_DIR}" --exclude var/cache/apt/archives --exclude boot --block-size=1 | cut -f 1)
BOOT_SIZE="$((256 * 1024 * 1024))"
ROOT_SIZE=$(du --apparent-size -s "${EXPORT_ROOTFS_DIR}" --exclude var/cache/apt/archives --exclude boot --block-size=1 | cut -f 1)

# All partition sizes and starts will be aligned to this size
ALIGN="$((4 * 1024 * 1024))"
# Add this much space to the calculated file size. This allows for
# some overhead (since actual space usage is usually rounded up to the
# filesystem block size) and gives some free space on the resulting
# image.
ROOT_MARGIN="$(echo "($ROOT_SIZE * 0.2 + 200 * 1024 * 1024) / 1" | bc)"
# All partition sizes and starts will be aligned to this size
ALIGN="$((4 * 1024 * 1024))"
# Add this much space to the calculated file size. This allows for
# some overhead (since actual space usage is usually rounded up to the
# filesystem block size) and gives some free space on the resulting
# image.
ROOT_MARGIN="$(echo "($ROOT_SIZE * 0.2 + 200 * 1024 * 1024) / 1" | bc)"

BOOT_PART_START=$((ALIGN))
BOOT_PART_SIZE=$(((BOOT_SIZE + ALIGN - 1) / ALIGN * ALIGN))
ROOT_PART_START=$((BOOT_PART_START + BOOT_PART_SIZE))
ROOT_PART_SIZE=$(((ROOT_SIZE + ROOT_MARGIN + ALIGN - 1) / ALIGN * ALIGN))
IMG_SIZE=$((BOOT_PART_START + BOOT_PART_SIZE + ROOT_PART_SIZE))
BOOT_PART_START=$((ALIGN))
BOOT_PART_SIZE=$(((BOOT_SIZE + ALIGN - 1) / ALIGN * ALIGN))
ROOT_PART_START=$((BOOT_PART_START + BOOT_PART_SIZE))
ROOT_PART_SIZE=$(((ROOT_SIZE + ROOT_MARGIN + ALIGN - 1) / ALIGN * ALIGN))
IMG_SIZE=$((BOOT_PART_START + BOOT_PART_SIZE + ROOT_PART_SIZE))

truncate -s "${IMG_SIZE}" "${IMG_FILE}"
truncate -s "${IMG_SIZE}" "${IMG_FILE}"

parted --script "${IMG_FILE}" mklabel msdos
parted --script "${IMG_FILE}" unit B mkpart primary fat32 "${BOOT_PART_START}" "$((BOOT_PART_START + BOOT_PART_SIZE - 1))"
parted --script "${IMG_FILE}" unit B mkpart primary ext4 "${ROOT_PART_START}" "$((ROOT_PART_START + ROOT_PART_SIZE - 1))"
parted --script "${IMG_FILE}" mklabel msdos
parted --script "${IMG_FILE}" unit B mkpart primary fat32 "${BOOT_PART_START}" "$((BOOT_PART_START + BOOT_PART_SIZE - 1))"
parted --script "${IMG_FILE}" unit B mkpart primary ext4 "${ROOT_PART_START}" "$((ROOT_PART_START + ROOT_PART_SIZE - 1))"

PARTED_OUT=$(parted -sm "${IMG_FILE}" unit b print)
BOOT_OFFSET=$(echo "$PARTED_OUT" | grep -e '^1:' | cut -d':' -f 2 | tr -d B)
BOOT_LENGTH=$(echo "$PARTED_OUT" | grep -e '^1:' | cut -d':' -f 4 | tr -d B)
PARTED_OUT=$(parted -sm "${IMG_FILE}" unit b print)
BOOT_OFFSET=$(echo "$PARTED_OUT" | grep -e '^1:' | cut -d':' -f 2 | tr -d B)
BOOT_LENGTH=$(echo "$PARTED_OUT" | grep -e '^1:' | cut -d':' -f 4 | tr -d B)

ROOT_OFFSET=$(echo "$PARTED_OUT" | grep -e '^2:' | cut -d':' -f 2 | tr -d B)
ROOT_LENGTH=$(echo "$PARTED_OUT" | grep -e '^2:' | cut -d':' -f 4 | tr -d B)
ROOT_OFFSET=$(echo "$PARTED_OUT" | grep -e '^2:' | cut -d':' -f 2 | tr -d B)
ROOT_LENGTH=$(echo "$PARTED_OUT" | grep -e '^2:' | cut -d':' -f 4 | tr -d B)

BOOT_DEV=$(losetup --show -f -o "${BOOT_OFFSET}" --sizelimit "${BOOT_LENGTH}" "${IMG_FILE}")
ROOT_DEV=$(losetup --show -f -o "${ROOT_OFFSET}" --sizelimit "${ROOT_LENGTH}" "${IMG_FILE}")
echo "/boot: offset $BOOT_OFFSET, length $BOOT_LENGTH"
echo "/: offset $ROOT_OFFSET, length $ROOT_LENGTH"
echo "Mounting BOOT_DEV..."
cnt=0
until BOOT_DEV=$(losetup --show -f -o "${BOOT_OFFSET}" --sizelimit "${BOOT_LENGTH}" "${IMG_FILE}"); do
if [ $cnt -lt 5 ]; then
cnt=$((cnt + 1))
echo "Error in losetup for BOOT_DEV. Retrying..."
sleep 5
else
echo "ERROR: losetup for BOOT_DEV failed; exiting"
exit 1
fi
done

ROOT_FEATURES="^huge_file"
for FEATURE in metadata_csum 64bit; do
echo "Mounting ROOT_DEV..."
cnt=0
until ROOT_DEV=$(losetup --show -f -o "${ROOT_OFFSET}" --sizelimit "${ROOT_LENGTH}" "${IMG_FILE}"); do
if [ $cnt -lt 5 ]; then
cnt=$((cnt + 1))
echo "Error in losetup for ROOT_DEV. Retrying..."
sleep 5
else
echo "ERROR: losetup for ROOT_DEV failed; exiting"
exit 1
fi
done

echo "/boot: offset $BOOT_OFFSET, length $BOOT_LENGTH"
echo "/: offset $ROOT_OFFSET, length $ROOT_LENGTH"

ROOT_FEATURES="^huge_file"
for FEATURE in metadata_csum 64bit; do
if grep -q "$FEATURE" /etc/mke2fs.conf; then
ROOT_FEATURES="^$FEATURE,$ROOT_FEATURES"
ROOT_FEATURES="^$FEATURE,$ROOT_FEATURES"
fi
done
mkdosfs -n boot -F 32 -v "$BOOT_DEV" > /dev/null
mkfs.ext4 -L rootfs -O "$ROOT_FEATURES" "$ROOT_DEV" > /dev/null
done
mkdosfs -n boot -F 32 -v "$BOOT_DEV" > /dev/null
mkfs.ext4 -L rootfs -O "$ROOT_FEATURES" "$ROOT_DEV" > /dev/null

mount -v "$ROOT_DEV" "${ROOTFS_DIR}" -t ext4
mkdir -p "${ROOTFS_DIR}/boot"
mount -v "$BOOT_DEV" "${ROOTFS_DIR}/boot" -t vfat
mount -v "$ROOT_DEV" "${ROOTFS_DIR}" -t ext4
mkdir -p "${ROOTFS_DIR}/boot"
mount -v "$BOOT_DEV" "${ROOTFS_DIR}/boot" -t vfat

rsync -aHAXx --exclude /var/cache/apt/archives --exclude /boot "${EXPORT_ROOTFS_DIR}/" "${ROOTFS_DIR}/"
rsync -rtx "${EXPORT_ROOTFS_DIR}/boot/" "${ROOTFS_DIR}/boot/"
rsync -aHAXx --exclude /var/cache/apt/archives --exclude /boot "${EXPORT_ROOTFS_DIR}/" "${ROOTFS_DIR}/"
rsync -rtx "${EXPORT_ROOTFS_DIR}/boot/" "${ROOTFS_DIR}/boot/"
fi

+ 256
- 0
scripts/qcow2_handling View File

@@ -0,0 +1,256 @@
#!/bin/bash

# QCOW2 Routines

export CURRENT_IMAGE
export CURRENT_MOUNTPOINT

export NBD_DEV
export MAP_BOOT_DEV
export MAP_ROOT_DEV

# set in build.sh
# should be fairly enough for the beginning
# overwrite here by uncommenting following lines
# BASE_QCOW2_SIZE=12G

# find and initialize free block device nodes
init_nbd() {
modprobe nbd max_part=16
if [ -z "${NBD_DEV}" ]; then
for x in /sys/class/block/nbd* ; do
S=`cat $x/size`
if [ "$S" == "0" ] ; then
NBD_DEV=/dev/$(basename $x)
MAP_BOOT_DEV=/dev/mapper/$(basename $x)p1
MAP_ROOT_DEV=/dev/mapper/$(basename $x)p2
break
fi
done
fi
}
export -f init_nbd

# connect image to block device
connect_blkdev() {
init_nbd
qemu-nbd --discard=unmap -c $NBD_DEV "$1"
sync
kpartx -a $NBD_DEV
sync
CURRENT_IMAGE="$1"
}
export -f connect_blkdev

# disconnect image from block device
disconnect_blkdev() {
kpartx -d $NBD_DEV
qemu-nbd -d $NBD_DEV
NBD_DEV=
MAP_BOOT_DEV=
MAP_ROOT_DEV=
CURRENT_IMAGE=
}
export -f disconnect_blkdev

# mount qcow2 image: mount_image <image file> <mountpoint>
mount_qimage() {
connect_blkdev "$1"
mount -v -t ext4 $MAP_ROOT_DEV "$2"
mkdir -p "${ROOTFS_DIR}/boot"
mount -v -t vfat $MAP_BOOT_DEV "$2/boot"
CURRENT_MOUNTPOINT="$2"
}
export -f mount_qimage

# umount qcow2 image: umount_image <current mountpoint>
umount_qimage() {
sync
#umount "$1/boot"
while mount | grep -q "$1"; do
local LOCS
LOCS=$(mount | grep "$1" | cut -f 3 -d ' ' | sort -r)
for loc in $LOCS; do
echo "$loc"
while mountpoint -q "$loc" && ! umount "$loc"; do
sleep 0.1
done
done
done
CURRENT_MOUNTPOINT=
disconnect_blkdev
}
export -f umount_qimage

# create base image / backing image / mount image
load_qimage() {
if [ -z "${CURRENT_MOUNTPOINT}" ]; then
if [ ! -d "${ROOTFS_DIR}" ]; then
mkdir -p "${ROOTFS_DIR}";
fi

if [ "${CLEAN}" = "1" ] && [ -f "${WORK_DIR}/image-${STAGE}.qcow2" ]; then
rm -f "${WORK_DIR}/image-${STAGE}.qcow2";
fi

if [ ! -f "${WORK_DIR}/image-${STAGE}.qcow2" ]; then
pushd ${WORK_DIR} > /dev/null
init_nbd
if [ -z "${PREV_STAGE}" ]; then
echo "Creating base image: image-${STAGE}.qcow2"
# -o preallocation=falloc
qemu-img create -f qcow2 image-${STAGE}.qcow2 $BASE_QCOW2_SIZE
sync
qemu-nbd --discard=unmap -c $NBD_DEV image-${STAGE}.qcow2
sync
sfdisk $NBD_DEV << EOF
4MiB,250MiB,c,*
254MiB,,83;
EOF
sync
kpartx -a $NBD_DEV
mkdosfs -n boot -F 32 -v $MAP_BOOT_DEV
mkfs.ext4 -L rootfs -O "^huge_file,^metadata_csum,^64bit" $MAP_ROOT_DEV
sync
else
if [ ! -f "${WORK_DIR}/image-${PREV_STAGE}.qcow2" ]; then
exit 1;
fi
echo "Creating backing image: image-${STAGE}.qcow2 <- ${WORK_DIR}/image-${PREV_STAGE}.qcow2"
qemu-img create -f qcow2 \
-o backing_file=${WORK_DIR}/image-${PREV_STAGE}.qcow2 \
${WORK_DIR}/image-${STAGE}.qcow2
sync
qemu-nbd --discard=unmap -c $NBD_DEV image-${STAGE}.qcow2
sync
kpartx -a $NBD_DEV
fi

mount -v -t ext4 $MAP_ROOT_DEV "${ROOTFS_DIR}"
mkdir -p "${ROOTFS_DIR}/boot"
mount -v -t vfat $MAP_BOOT_DEV "${ROOTFS_DIR}/boot"
CURRENT_IMAGE=${WORK_DIR}/image-${STAGE}.qcow2
CURRENT_MOUNTPOINT=${ROOTFS_DIR}
popd > /dev/null
else
mount_qimage "${WORK_DIR}/image-${STAGE}.qcow2" "${ROOTFS_DIR}"
fi
echo "Current image in use: ${CURRENT_IMAGE} (MP: ${CURRENT_MOUNTPOINT})"
fi
}
export -f load_qimage

# umount current image and refresh mount point env var
unload_qimage() {
if [ ! -z "${CURRENT_MOUNTPOINT}" ]; then
fstrim -v "${CURRENT_MOUNTPOINT}" || true
umount_qimage "${CURRENT_MOUNTPOINT}"
fi
}
export -f unload_qimage

# based on: https://github.com/SirLagz/RaspberryPi-ImgAutoSizer
# helper function for make_bootable_image, do not call directly
function resize_qcow2() {
if [ -z "$CALL_FROM_MBI" ]; then
echo "resize_qcow2: cannot be called directly, use make_bootable_image instead"
return 1
fi

# ROOT_MARGIN=$((800*1024*1024))
ROOT_MARGIN=$((1*1024*1024))
PARTED_OUT=`parted -s -m "$NBD_DEV" unit B print`
PART_NO=`echo "$PARTED_OUT" | grep ext4 | awk -F: ' { print $1 } '`
PART_START=`echo "$PARTED_OUT" | grep ext4 | awk -F: ' { print substr($2,1,length($2)-1) } '`

e2fsck -y -f $MAP_ROOT_DEV || true

DATA_SIZE=`resize2fs -P $MAP_ROOT_DEV | awk -F': ' ' { print $2 } '`
BLOCK_SIZE=$(dumpe2fs -h $MAP_ROOT_DEV | grep 'Block size' | awk -F': ' ' { print $2 }')
BLOCK_SIZE=${BLOCK_SIZE// /}

let DATA_SIZE=$DATA_SIZE+$ROOT_MARGIN/$BLOCK_SIZE
resize2fs -p $MAP_ROOT_DEV $DATA_SIZE
sleep 1

let PART_NEW_SIZE=$DATA_SIZE*$BLOCK_SIZE
let PART_NEW_END=$PART_START+$PART_NEW_SIZE
ACT1=`parted -s "$NBD_DEV" rm 2`
ACT2=`parted -s "$NBD_DEV" unit B mkpart primary $PART_START $PART_NEW_END`
NEW_IMG_SIZE=`parted -s -m "$NBD_DEV" unit B print free | tail -1 | awk -F: ' { print substr($2,1,length($2)-1) } '`
}
export -f resize_qcow2

# create raw img from qcow2: make_bootable_image <in.qcow2> <out.img>
function make_bootable_image() {
EXPORT_QCOW2="$1"
EXPORT_IMAGE="$2"

echo "Connect block device to source qcow2"
connect_blkdev "${EXPORT_QCOW2}"

echo "Resize fs and partition"
CALL_FROM_MBI=1
resize_qcow2
sync
CALL_FROM_MBI=

echo "Disconnect block device"
disconnect_blkdev

if [ -z "$NEW_IMG_SIZE" ]; then
echo "NEW_IMG_SIZE could not be calculated, cannot process image. Exit."
exit 1
fi

echo "Shrinking qcow2 image"
qemu-img resize --shrink "${EXPORT_QCOW2}" $NEW_IMG_SIZE
sync

echo "Convert qcow2 to raw image"
qemu-img convert -f qcow2 -O raw "${EXPORT_QCOW2}" "${EXPORT_IMAGE}"
sync

echo "Get PARTUUIDs from image"
IMGID="$(blkid -o value -s PTUUID "${EXPORT_IMAGE}")"

BOOT_PARTUUID="${IMGID}-01"
echo "Boot: $BOOT_PARTUUID"
ROOT_PARTUUID="${IMGID}-02"
echo "Root: $ROOT_PARTUUID"

echo "Mount image"
MOUNTROOT=${WORK_DIR}/tmpimage
mkdir -p $MOUNTROOT

MOUNTPT=$MOUNTROOT
PARTITION=2
mount "${EXPORT_IMAGE}" "$MOUNTPT" -o loop,offset=$[ `/sbin/sfdisk -d "${EXPORT_IMAGE}" | grep "start=" | head -n $PARTITION | tail -n1 | sed 's/.*start=[ ]*//' | sed 's/,.*//'` * 512 ],sizelimit=$[ `/sbin/sfdisk -d "${EXPORT_IMAGE}" | grep "start=" | head -n $PARTITION | tail -n1 | sed 's/.*size=[ ]*//' | sed 's/,.*//'` * 512 ] || exit 1

MOUNTPT=$MOUNTROOT/boot
PARTITION=1
mount "${EXPORT_IMAGE}" "$MOUNTPT" -o loop,offset=$[ `/sbin/sfdisk -d "${EXPORT_IMAGE}" | grep "start=" | head -n $PARTITION | tail -n1 | sed 's/.*start=[ ]*//' | sed 's/,.*//'` * 512 ],sizelimit=$[ `/sbin/sfdisk -d "${EXPORT_IMAGE}" | grep "start=" | head -n $PARTITION | tail -n1 | sed 's/.*size=[ ]*//' | sed 's/,.*//'` * 512 ] || exit 1

if [ ! -d "${MOUNTROOT}/root" ]; then
echo "Image damaged or not mounted. Exit."
exit 1
fi

echo "Setup PARTUUIDs"
if [ ! -z "$BOOT_PARTUUID" ] && [ ! -z "$ROOT_PARTUUID" ]; then
echo "Set UUIDs to make it bootable"
sed -i "s/BOOTDEV/PARTUUID=${BOOT_PARTUUID}/" "${MOUNTROOT}/etc/fstab"
sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${MOUNTROOT}/etc/fstab"
sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${MOUNTROOT}/boot/cmdline.txt"
fi

echo "Umount image"
sync
umount "${MOUNTROOT}/boot" || exit 1
umount "${MOUNTROOT}" || exit 1

echo "Remove qcow2 export image"
rm -f "${EXPORT_QCOW2}"
}
export -f make_bootable_image

+ 2
- 2
stage0/prerun.sh View File

@@ -1,5 +1,5 @@
#!/bin/bash -e

if [ ! -d "${ROOTFS_DIR}" ]; then
if [ ! -d "${ROOTFS_DIR}" ] || [ "${USE_QCOW2}" = "1" ]; then
bootstrap ${RELEASE} "${ROOTFS_DIR}" http://raspbian.raspberrypi.org/raspbian/
fi
fi

+ 2
- 0
stage2/01-sys-tweaks/00-packages View File

@@ -2,6 +2,8 @@ ssh less fbset sudo psmisc strace ed ncdu crda
console-setup keyboard-configuration debconf-utils parted unzip
build-essential manpages-dev python bash-completion gdb pkg-config
python-rpi.gpio v4l-utils
python-gpiozero
python3-gpiozero
avahi-daemon
lua5.1
luajit


+ 12
- 0
stage2/01-sys-tweaks/01-run.sh View File

@@ -11,6 +11,18 @@ install -m 644 files/console-setup "${ROOTFS_DIR}/etc/default/"

install -m 755 files/rc.local "${ROOTFS_DIR}/etc/"

if [ -n "${PUBKEY_SSH_FIRST_USER}" ]; then
install -v -m 0700 -o 1000 -g 1000 -d "${ROOTFS_DIR}"/home/"${FIRST_USER_NAME}"/.ssh
echo "${PUBKEY_SSH_FIRST_USER}" >"${ROOTFS_DIR}"/home/"${FIRST_USER_NAME}"/.ssh/authorized_keys
chown 1000:1000 "${ROOTFS_DIR}"/home/"${FIRST_USER_NAME}"/.ssh/authorized_keys
chmod 0600 "${ROOTFS_DIR}"/home/"${FIRST_USER_NAME}"/.ssh/authorized_keys
fi

if [ "${PUBKEY_ONLY_SSH}" = "1" ]; then
sed -i -Ee 's/^#?[[:blank:]]*PubkeyAuthentication[[:blank:]]*no[[:blank:]]*$/PubkeyAuthentication yes/
s/^#?[[:blank:]]*PasswordAuthentication[[:blank:]]*yes[[:blank:]]*$/PasswordAuthentication no/' "${ROOTFS_DIR}"/etc/ssh/sshd_config
fi

on_chroot << EOF
systemctl disable hwclock.sh
systemctl disable nfs-common


+ 8
- 3
stage2/02-net-tweaks/01-run.sh View File

@@ -25,7 +25,12 @@ network={
EOL
fi

# Disable wifi on 5GHz models
# Disable wifi on 5GHz models if WPA_COUNTRY is not set
mkdir -p "${ROOTFS_DIR}/var/lib/systemd/rfkill/"
echo 1 > "${ROOTFS_DIR}/var/lib/systemd/rfkill/platform-3f300000.mmcnr:wlan"
echo 1 > "${ROOTFS_DIR}/var/lib/systemd/rfkill/platform-fe300000.mmcnr:wlan"
if [ -n "$WPA_COUNTRY" ]; then
echo 0 > "${ROOTFS_DIR}/var/lib/systemd/rfkill/platform-3f300000.mmcnr:wlan"
echo 0 > "${ROOTFS_DIR}/var/lib/systemd/rfkill/platform-fe300000.mmcnr:wlan"
else
echo 1 > "${ROOTFS_DIR}/var/lib/systemd/rfkill/platform-3f300000.mmcnr:wlan"
echo 1 > "${ROOTFS_DIR}/var/lib/systemd/rfkill/platform-fe300000.mmcnr:wlan"
fi

+ 3
- 1
stage4/00-install-packages/00-packages View File

@@ -1,4 +1,4 @@
python python-tk
python python3-pygame python-pygame python-tk
python3 python3-tk thonny
python3-pgzero
python-serial python3-serial
@@ -13,9 +13,11 @@ tree
libgl1-mesa-dri libgles1 libgles2-mesa xcompmgr
geany
piclone
wiringpi pigpio python-pigpio python3-pigpio raspi-gpio python3-rpi.gpio
python-spidev python3-spidev
python-twython python3-twython
python-smbus python3-smbus
python-flask python3-flask
pprompt
piwiz
rp-prefapps


+ 22
- 4
stage4/00-install-packages/01-packages View File

@@ -1,4 +1,22 @@
hunspell-en-gb
hyphen-en-gb
wamerican
wbritish
python-automationhat python3-automationhat
python-blinkt python3-blinkt
python-cap1xxx python3-cap1xxx
python-drumhat python3-drumhat
python-envirophat python3-envirophat
python-explorerhat python3-explorerhat
python-fourletterphat python3-fourletterphat
python-microdotphat python3-microdotphat
python-mote python3-mote
python-motephat python3-motephat
python-phatbeat python3-phatbeat
python-pianohat python3-pianohat
python-piglow python3-piglow
python-rainbowhat python3-rainbowhat
python-scrollphat python3-scrollphat
python-scrollphathd python3-scrollphathd
python-sn3218 python3-sn3218
python-skywriter python3-skywriter
python-touchphat python3-touchphat
python-buttonshim python3-buttonshim
python-unicornhathd python3-unicornhathd
python-pantilthat python3-pantilthat

+ 5
- 0
stage4/04-enable-xcompmgr/00-run.sh View File

@@ -0,0 +1,5 @@
#!/bin/bash -e

on_chroot << EOF
raspi-config nonint do_xcompmgr 0
EOF

Loading…
Cancel
Save