reterminal-dm4/emmc-provisioning/host/build-cloudinit-image.sh

#!/usr/bin/env bash
# Run on the Proxmox host when build_cloudinit_request.json appears in the provisioning dir.
# Downloads Raspberry Pi OS image from URL, injects cloud-init NoCloud files, saves to BACKUPS_DIR.
# Uses loop devices and mount (available on host, not in unprivileged LXC).
# Triggered by systemd path unit cm4-build-cloudinit.path.

set -e
PROV_DIR="${CM4_PROVISIONING_DIR:-/var/lib/cm4-provisioning}"
REQUEST_FILE="$PROV_DIR/build_cloudinit_request.json"
STATUS_FILE="$PROV_DIR/build_cloudinit_status.json"
[[ -f /opt/cm4-provisioning/env ]] && source /opt/cm4-provisioning/env
BACKUPS_DIR="${BACKUPS_DIR:-$PROV_DIR/backups}"
CLOUDINIT_IMAGES_DIR="${CLOUDINIT_IMAGES_DIR:-$PROV_DIR/cloudinit-images}"
DOWNLOAD_CACHE_DIR="${CM4_DOWNLOAD_CACHE_DIR:-$PROV_DIR/download-cache}"
# Write built cloud-init images to CLOUDINIT_IMAGES_DIR (separate from backups)
OUTPUT_DIR="${CLOUDINIT_IMAGES_DIR}"
GOLDEN_IMAGE="${GOLDEN_IMAGE:-$PROV_DIR/golden.img}"

write_status() {
    local phase="$1" message="$2" output_name="$3" error="$4"
    printf '{"phase":"%s","message":"%s","output_name":%s,"error":%s,"updated":%s}\n' \
        "$phase" "$message" \
        "$([ -n "$output_name" ] && echo "\"$output_name\"" || echo "null")" \
        "$([ -n "$error" ] && echo "\"${error//\"/\\\"}\"" || echo "null")" \
        "$(date +%s)" > "$STATUS_FILE"
}

[[ -f "$REQUEST_FILE" ]] || { echo "No request file"; exit 0; }

# Use temp dir on provisioning dir (not /tmp) so we have enough space for decompress (~3GB+)
mkdir -p "$PROV_DIR"
TEMP_DIR=$(mktemp -d -p "$PROV_DIR" build.XXXXXX 2>/dev/null) || TEMP_DIR="$PROV_DIR/build.$$" && mkdir -p "$TEMP_DIR"
trap 'rm -rf "$TEMP_DIR"; rm -f "$REQUEST_FILE"' EXIT

# Extract fields from JSON into temp files (handles multi-line content)
python3 - "$REQUEST_FILE" "$TEMP_DIR" << 'PY'
import json, sys
with open(sys.argv[1]) as f:
    d = json.load(f)
out = sys.argv[2]
for name, key in [("user-data", "user_data"), ("meta-data", "meta_data"), ("network-config", "network_config")]:
    with open(f"{out}/{name}", "w") as f:
        f.write(d.get(key, ""))
with open(f"{out}/variant", "w") as f:
    f.write(d.get("variant", "lite"))
with open(f"{out}/url", "w") as f:
    f.write(d.get("url", ""))
with open(f"{out}/set_golden", "w") as f:
    f.write("1" if d.get("set_as_golden_after") else "0")
PY

URL=$(cat "$TEMP_DIR/url")
VARIANT=$(cat "$TEMP_DIR/variant")
SET_AS_GOLDEN=$(cat "$TEMP_DIR/set_golden")
[[ -n "$URL" ]] || { write_status "error" "" "" "Missing url in request"; exit 1; }

OUT_NAME="raspios-${VARIANT}-cloudinit-$(date +%Y%m%d-%H%M%S).img"
OUT_PATH="$OUTPUT_DIR/$OUT_NAME"

BASENAME=$(basename "$URL")
SHA256_URL="${URL}.sha256"
CACHED="$DOWNLOAD_CACHE_DIR/$BASENAME"
XZ_FILE="$TEMP_DIR/image.img.xz"
CURL_ERR="$TEMP_DIR/curl_err.txt"
SHA256_FILE="$TEMP_DIR/expected.sha256"

# Fetch official SHA256 checksum if available (Raspberry Pi OS publishes .img.xz.sha256 next to each image)
write_status "downloading" "Checking for existing image and checksum…" "" ""
EXPECTED_HASH=""
if curl -sfL -o "$SHA256_FILE" "$SHA256_URL" 2>/dev/null && [[ -s "$SHA256_FILE" ]]; then
    # Format is typically "hash  filename" or just "hash"; take first 64-char hex token
    EXPECTED_HASH=$(head -1 "$SHA256_FILE" | awk '{print $1}')
    [[ ${#EXPECTED_HASH} -eq 64 && "$EXPECTED_HASH" =~ ^[0-9a-fA-F]+$ ]] || EXPECTED_HASH=""
fi

USE_CACHED=0
mkdir -p "$DOWNLOAD_CACHE_DIR"
if [[ -f "$CACHED" && -s "$CACHED" ]]; then
    if [[ -n "$EXPECTED_HASH" ]]; then
        ACTUAL_HASH=$(sha256sum -b "$CACHED" 2>/dev/null | awk '{print $1}')
        if [[ "$ACTUAL_HASH" == "$EXPECTED_HASH" ]]; then
            USE_CACHED=1
        else
            rm -f "$CACHED"
        fi
    else
        USE_CACHED=1
    fi
fi

if [[ $USE_CACHED -eq 1 ]]; then
    write_status "downloading" "Using cached image (checksum OK): $BASENAME" "" ""
    cp "$CACHED" "$XZ_FILE"
else
    write_status "downloading" "Downloading $(basename "$URL")…" "" ""
    CURL_OPTS=(-f -L -o "$XZ_FILE" --connect-timeout 30 --max-time 7200)
    [[ "${CURL_INSECURE:-}" == "1" ]] && CURL_OPTS+=(-k)
    if ! curl "${CURL_OPTS[@]}" "$URL" 2>"$CURL_ERR"; then
        err_detail=$(head -c 200 "$CURL_ERR" | tr '\n' ' ' | sed 's/"/\\"/g')
        write_status "error" "" "" "Download failed: ${err_detail:-curl exited with error}"
        exit 1
    fi
    # Verify with official checksum if we have it
    if [[ -n "$EXPECTED_HASH" ]]; then
        ACTUAL_HASH=$(sha256sum -b "$XZ_FILE" 2>/dev/null | awk '{print $1}')
        if [[ "$ACTUAL_HASH" != "$EXPECTED_HASH" ]]; then
            write_status "error" "" "" "Checksum mismatch: download may be corrupted (expected ${EXPECTED_HASH:0:16}..., got ${ACTUAL_HASH:0:16}...). Try again."
            exit 1
        fi
        write_status "downloading" "Checksum OK, caching image…" "" ""
    fi
    cp "$XZ_FILE" "$CACHED"
fi

write_status "decompressing" "Decompressing image…" "" ""
# Check we have a real xz file (not HTML error page)
if ! command -v xz >/dev/null 2>&1; then
    write_status "error" "" "" "Decompress failed: xz not installed. Install xz-utils (apt install xz-utils)"
    exit 1
fi
if [[ ! -s "$XZ_FILE" ]]; then
    write_status "error" "" "" "Decompress failed: downloaded file is empty"
    exit 1
fi
FILE_TYPE=$(file -b "$XZ_FILE" 2>/dev/null || true)
if [[ "$FILE_TYPE" == *"HTML"* ]] || [[ "$FILE_TYPE" == *"text"* ]] && [[ "$FILE_TYPE" != *"XZ"* ]]; then
    write_status "error" "" "" "Decompress failed: download is not an image (got: ${FILE_TYPE:0:80})"
    exit 1
fi
# Need ~3GB+ free for decompressing typical Raspios .img.xz; use same filesystem as TEMP_DIR
AVAIL_KB=$(df -P "$TEMP_DIR" 2>/dev/null | awk 'NR==2 {print $4}')
AVAIL_GB=0
[[ -n "$AVAIL_KB" ]] && AVAIL_GB=$((AVAIL_KB / 1024 / 1024))
if [[ -n "$AVAIL_KB" && "$AVAIL_KB" -lt 3145728 ]]; then
    write_status "error" "" "" "Decompress failed: not enough disk space (${AVAIL_GB}GB free on $(df -P "$TEMP_DIR" 2>/dev/null | awk 'NR==2 {print $6}'); need ~3GB). Free space or use a larger volume."
    exit 1
fi
IMG_FILE="$TEMP_DIR/image.img"
XZ_ERR="$TEMP_DIR/xz_err.txt"
# -T 1: single-threaded to reduce memory use; -d -k -f: decompress, keep source, force
if ! xz -T 1 -d -k -f "$XZ_FILE" 2>"$XZ_ERR"; then
    err_detail=$(cat "$XZ_ERR" 2>/dev/null | tr '\n' ' ' | sed 's/"/\\"/g' | head -c 400)
    [[ -z "$err_detail" ]] && err_detail="out of disk space or memory?"
    write_status "error" "" "" "Decompress failed: ${err_detail} (free space: ${AVAIL_GB}GB)"
    exit 1
fi
[[ -f "$IMG_FILE" ]] || { write_status "error" "" "" "image.img not found after decompress"; exit 1; }

write_status "injecting" "Mounting boot partition and injecting cloud-init…" "" ""
LOOP=$(losetup -f --show -P "$IMG_FILE")
boot_part="${LOOP}p1"
[[ -b "$boot_part" ]] || boot_part="${LOOP}p2"
[[ -b "$boot_part" ]] || { write_status "error" "" "" "Boot partition not found"; losetup -d "$LOOP"; exit 1; }

MNT="$TEMP_DIR/mnt"
mkdir -p "$MNT"
mount "$boot_part" "$MNT"
cp "$TEMP_DIR/user-data" "$MNT/user-data"
cp "$TEMP_DIR/meta-data" "$MNT/meta-data"
cp "$TEMP_DIR/network-config" "$MNT/network-config"
umount "$MNT"
losetup -d "$LOOP"

write_status "finalizing" "Copying image to cloud-init images…" "" ""
mkdir -p "$OUTPUT_DIR"
cp "$IMG_FILE" "$OUT_PATH"

# Compress to .img.xz to reduce size (deploy supports decompress on the fly)
write_status "finalizing" "Compressing image (xz)…" "" ""
OUT_XZ="${OUT_PATH}.xz"
# -T 0 = use all cores; fallback to -T 1 if old xz
if ! xz -T 0 -z -k -f "$OUT_PATH" 2>/dev/null; then
    xz -T 1 -z -k -f "$OUT_PATH" 2>/dev/null || true
fi
if [[ -f "$OUT_XZ" ]]; then
    rm -f "$OUT_PATH"
    OUT_NAME="${OUT_NAME}.xz"
    OUT_PATH="$OUT_XZ"
else
    write_status "finalizing" "Compression skipped (xz failed), keeping raw image." "" ""
fi

if [[ "$SET_AS_GOLDEN" == "1" ]]; then
    rm -f "$GOLDEN_IMAGE"
    ln -sf "$OUT_PATH" "$GOLDEN_IMAGE"
    write_status "done" "Built $OUT_NAME and set as golden image." "$OUT_NAME" ""
else
    write_status "done" "Built $OUT_NAME" "$OUT_NAME" ""
fi