Enhance build-cloudinit-image.sh and deploy-to-proxmox.sh: add download cache directory support, implement checksum verification for downloaded images, and improve error handling for decompression failures. Update deploy script to ensure xz-utils installation and create necessary directories for cloud-init image management.

chromium-setup/emmc-provisioning/host/build-cloudinit-image.sh

@@ -11,6 +11,7 @@ STATUS_FILE="$PROV_DIR/build_cloudinit_status.json"
 [[ -f /opt/cm4-provisioning/env ]] && source /opt/cm4-provisioning/env
 BACKUPS_DIR="${BACKUPS_DIR:-$PROV_DIR/backups}"
 CLOUDINIT_IMAGES_DIR="${CLOUDINIT_IMAGES_DIR:-$PROV_DIR/cloudinit-images}"
+DOWNLOAD_CACHE_DIR="${CM4_DOWNLOAD_CACHE_DIR:-$PROV_DIR/download-cache}"
 # Write built cloud-init images to CLOUDINIT_IMAGES_DIR (separate from backups)
 OUTPUT_DIR="${CLOUDINIT_IMAGES_DIR}"
 GOLDEN_IMAGE="${GOLDEN_IMAGE:-$PROV_DIR/golden.img}"
@@ -26,7 +27,9 @@ write_status() {
 
 [[ -f "$REQUEST_FILE" ]] || { echo "No request file"; exit 0; }
 
-TEMP_DIR=$(mktemp -d)
+# Use temp dir on provisioning dir (not /tmp) so we have enough space for decompress (~3GB+)
+mkdir -p "$PROV_DIR"
+TEMP_DIR=$(mktemp -d -p "$PROV_DIR" build.XXXXXX 2>/dev/null) || TEMP_DIR="$PROV_DIR/build.$$" && mkdir -p "$TEMP_DIR"
 trap 'rm -rf "$TEMP_DIR"; rm -f "$REQUEST_FILE"' EXIT
 
 # Extract fields from JSON into temp files (handles multi-line content)
@@ -54,15 +57,59 @@ SET_AS_GOLDEN=$(cat "$TEMP_DIR/set_golden")
 OUT_NAME="raspios-${VARIANT}-cloudinit-$(date +%Y%m%d-%H%M%S).img"
 OUT_PATH="$OUTPUT_DIR/$OUT_NAME"
 
-write_status "downloading" "Downloading $(basename "$URL")…" "" ""
+BASENAME=$(basename "$URL")
+SHA256_URL="${URL}.sha256"
+CACHED="$DOWNLOAD_CACHE_DIR/$BASENAME"
 XZ_FILE="$TEMP_DIR/image.img.xz"
 CURL_ERR="$TEMP_DIR/curl_err.txt"
-CURL_OPTS=(-f -L -o "$XZ_FILE" --connect-timeout 30 --max-time 7200)
+SHA256_FILE="$TEMP_DIR/expected.sha256"
-[[ "${CURL_INSECURE:-}" == "1" ]] && CURL_OPTS+=(-k)
+
-if ! curl "${CURL_OPTS[@]}" "$URL" 2>"$CURL_ERR"; then
+# Fetch official SHA256 checksum if available (Raspberry Pi OS publishes .img.xz.sha256 next to each image)
+write_status "downloading" "Checking for existing image and checksum…" "" ""
+EXPECTED_HASH=""
+if curl -sfL -o "$SHA256_FILE" "$SHA256_URL" 2>/dev/null && [[ -s "$SHA256_FILE" ]]; then
+    # Format is typically "hash  filename" or just "hash"; take first 64-char hex token
+    EXPECTED_HASH=$(head -1 "$SHA256_FILE" | awk '{print $1}')
+    [[ ${#EXPECTED_HASH} -eq 64 && "$EXPECTED_HASH" =~ ^[0-9a-fA-F]+$ ]] || EXPECTED_HASH=""
+fi
+
+USE_CACHED=0
+mkdir -p "$DOWNLOAD_CACHE_DIR"
+if [[ -f "$CACHED" && -s "$CACHED" ]]; then
+    if [[ -n "$EXPECTED_HASH" ]]; then
+        ACTUAL_HASH=$(sha256sum -b "$CACHED" 2>/dev/null | awk '{print $1}')
+        if [[ "$ACTUAL_HASH" == "$EXPECTED_HASH" ]]; then
+            USE_CACHED=1
+        else
+            rm -f "$CACHED"
+        fi
+    else
+        USE_CACHED=1
+    fi
+fi
+
+if [[ $USE_CACHED -eq 1 ]]; then
+    write_status "downloading" "Using cached image (checksum OK): $BASENAME" "" ""
+    cp "$CACHED" "$XZ_FILE"
+else
+    write_status "downloading" "Downloading $(basename "$URL")…" "" ""
+    CURL_OPTS=(-f -L -o "$XZ_FILE" --connect-timeout 30 --max-time 7200)
+    [[ "${CURL_INSECURE:-}" == "1" ]] && CURL_OPTS+=(-k)
+    if ! curl "${CURL_OPTS[@]}" "$URL" 2>"$CURL_ERR"; then
         err_detail=$(head -c 200 "$CURL_ERR" | tr '\n' ' ' | sed 's/"/\\"/g')
         write_status "error" "" "" "Download failed: ${err_detail:-curl exited with error}"
         exit 1
+    fi
+    # Verify with official checksum if we have it
+    if [[ -n "$EXPECTED_HASH" ]]; then
+        ACTUAL_HASH=$(sha256sum -b "$XZ_FILE" 2>/dev/null | awk '{print $1}')
+        if [[ "$ACTUAL_HASH" != "$EXPECTED_HASH" ]]; then
+            write_status "error" "" "" "Checksum mismatch: download may be corrupted (expected ${EXPECTED_HASH:0:16}..., got ${ACTUAL_HASH:0:16}...). Try again."
+            exit 1
+        fi
+        write_status "downloading" "Checksum OK, caching image…" "" ""
+    fi
+    cp "$XZ_FILE" "$CACHED"
 fi
 
 write_status "decompressing" "Decompressing image…" "" ""
@@ -80,11 +127,21 @@ if [[ "$FILE_TYPE" == *"HTML"* ]] || [[ "$FILE_TYPE" == *"text"* ]] && [[ "$FILE
     write_status "error" "" "" "Decompress failed: download is not an image (got: ${FILE_TYPE:0:80})"
     exit 1
 fi
+# Need ~3GB+ free for decompressing typical Raspios .img.xz; use same filesystem as TEMP_DIR
+AVAIL_KB=$(df -P "$TEMP_DIR" 2>/dev/null | awk 'NR==2 {print $4}')
+AVAIL_GB=0
+[[ -n "$AVAIL_KB" ]] && AVAIL_GB=$((AVAIL_KB / 1024 / 1024))
+if [[ -n "$AVAIL_KB" && "$AVAIL_KB" -lt 3145728 ]]; then
+    write_status "error" "" "" "Decompress failed: not enough disk space (${AVAIL_GB}GB free on $(df -P "$TEMP_DIR" 2>/dev/null | awk 'NR==2 {print $6}'); need ~3GB). Free space or use a larger volume."
+    exit 1
+fi
 IMG_FILE="$TEMP_DIR/image.img"
 XZ_ERR="$TEMP_DIR/xz_err.txt"
-if ! xz -d -k -f "$XZ_FILE" 2>"$XZ_ERR"; then
+# -T 1: single-threaded to reduce memory use; -d -k -f: decompress, keep source, force
-    err_detail=$(head -c 300 "$XZ_ERR" 2>/dev/null | tr '\n' ' ' | sed 's/"/\\"/g')
+if ! xz -T 1 -d -k -f "$XZ_FILE" 2>"$XZ_ERR"; then
-    write_status "error" "" "" "Decompress failed: ${err_detail:-xz exited with error}"
+    err_detail=$(cat "$XZ_ERR" 2>/dev/null | tr '\n' ' ' | sed 's/"/\\"/g' | head -c 400)
+    [[ -z "$err_detail" ]] && err_detail="out of disk space or memory?"
+    write_status "error" "" "" "Decompress failed: ${err_detail} (free space: ${AVAIL_GB}GB)"
     exit 1
 fi
 [[ -f "$IMG_FILE" ]] || { write_status "error" "" "" "image.img not found after decompress"; exit 1; }

chromium-setup/emmc-provisioning/scripts/deploy-to-proxmox.sh

@@ -251,9 +251,18 @@ EMMC_SIZE_BYTES=8589934592
 ENV
 [[ -n "$BACKUPS_HOST_PATH" ]] && echo "BACKUPS_DIR=$BACKUPS_HOST_PATH" >> /opt/cm4-provisioning/env
 touch /etc/cm4-provisioning/enabled
-mkdir -p /var/lib/cm4-provisioning/backups /var/lib/cm4-provisioning/cloudinit-images /var/lib/cm4-provisioning/portal-files
+mkdir -p /var/lib/cm4-provisioning/backups /var/lib/cm4-provisioning/cloudinit-images /var/lib/cm4-provisioning/portal-files /var/lib/cm4-provisioning/download-cache
 [[ -n "$BACKUPS_HOST_PATH" ]] && mkdir -p "$BACKUPS_HOST_PATH"
 grep -q "CLOUDINIT_IMAGES_DIR" /opt/cm4-provisioning/env || echo "CLOUDINIT_IMAGES_DIR=/var/lib/cm4-provisioning/cloudinit-images" >> /opt/cm4-provisioning/env
+grep -q "CM4_DOWNLOAD_CACHE_DIR" /opt/cm4-provisioning/env || echo "CM4_DOWNLOAD_CACHE_DIR=/var/lib/cm4-provisioning/download-cache" >> /opt/cm4-provisioning/env
+
+# --- Host: ensure xz-utils (and file) for cloud-init image build (decompress .img.xz) ---
+if command -v xz >/dev/null 2>&1; then
+  log "Host: xz-utils already present, skipping."
+else
+  log "Host: installing xz-utils and file (required for cloud-init image decompress)..."
+  apt-get update -qq && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq xz-utils file 2>/dev/null || log "Warning: could not install xz-utils (no internet?). Cloud-init build will fail at decompress until you run: apt install -y xz-utils file"
+fi
 
 # --- Host: install usbboot (rpiboot) only if not already present ---
 if [[ -x /opt/usbboot/rpiboot ]] || [[ -f /opt/usbboot/rpiboot ]]; then