Enhance build-cloudinit-image.sh and deploy-to-proxmox.sh: add download cache directory support, implement checksum verification for downloaded images, and improve error handling for decompression failures. Update deploy script to ensure xz-utils installation and create necessary directories for cloud-init image management.
This commit is contained in:
nearxos
@@ -11,6 +11,7 @@ STATUS_FILE="$PROV_DIR/build_cloudinit_status.json"
|
||||
[[ -f /opt/cm4-provisioning/env ]] && source /opt/cm4-provisioning/env
|
||||
BACKUPS_DIR="${BACKUPS_DIR:-$PROV_DIR/backups}"
|
||||
CLOUDINIT_IMAGES_DIR="${CLOUDINIT_IMAGES_DIR:-$PROV_DIR/cloudinit-images}"
|
||||
DOWNLOAD_CACHE_DIR="${CM4_DOWNLOAD_CACHE_DIR:-$PROV_DIR/download-cache}"
|
||||
# Write built cloud-init images to CLOUDINIT_IMAGES_DIR (separate from backups)
|
||||
OUTPUT_DIR="${CLOUDINIT_IMAGES_DIR}"
|
||||
GOLDEN_IMAGE="${GOLDEN_IMAGE:-$PROV_DIR/golden.img}"
|
||||
@@ -26,7 +27,9 @@ write_status() {
|
||||
|
||||
[[ -f "$REQUEST_FILE" ]] || { echo "No request file"; exit 0; }
|
||||
|
||||
TEMP_DIR=$(mktemp -d)
|
||||
# Use temp dir on provisioning dir (not /tmp) so we have enough space for decompress (~3GB+)
|
||||
mkdir -p "$PROV_DIR"
|
||||
TEMP_DIR=$(mktemp -d -p "$PROV_DIR" build.XXXXXX 2>/dev/null) || TEMP_DIR="$PROV_DIR/build.$$" && mkdir -p "$TEMP_DIR"
|
||||
trap 'rm -rf "$TEMP_DIR"; rm -f "$REQUEST_FILE"' EXIT
|
||||
|
||||
# Extract fields from JSON into temp files (handles multi-line content)
|
||||
@@ -54,15 +57,59 @@ SET_AS_GOLDEN=$(cat "$TEMP_DIR/set_golden")
|
||||
OUT_NAME="raspios-${VARIANT}-cloudinit-$(date +%Y%m%d-%H%M%S).img"
|
||||
OUT_PATH="$OUTPUT_DIR/$OUT_NAME"
|
||||
|
||||
write_status "downloading" "Downloading $(basename "$URL")…" "" ""
|
||||
BASENAME=$(basename "$URL")
|
||||
SHA256_URL="${URL}.sha256"
|
||||
CACHED="$DOWNLOAD_CACHE_DIR/$BASENAME"
|
||||
XZ_FILE="$TEMP_DIR/image.img.xz"
|
||||
CURL_ERR="$TEMP_DIR/curl_err.txt"
|
||||
CURL_OPTS=(-f -L -o "$XZ_FILE" --connect-timeout 30 --max-time 7200)
|
||||
[[ "${CURL_INSECURE:-}" == "1" ]] && CURL_OPTS+=(-k)
|
||||
if ! curl "${CURL_OPTS[@]}" "$URL" 2>"$CURL_ERR"; then
|
||||
SHA256_FILE="$TEMP_DIR/expected.sha256"
|
||||
|
||||
# Fetch official SHA256 checksum if available (Raspberry Pi OS publishes .img.xz.sha256 next to each image)
|
||||
write_status "downloading" "Checking for existing image and checksum…" "" ""
|
||||
EXPECTED_HASH=""
|
||||
if curl -sfL -o "$SHA256_FILE" "$SHA256_URL" 2>/dev/null && [[ -s "$SHA256_FILE" ]]; then
|
||||
# Format is typically "hash filename" or just "hash"; take first 64-char hex token
|
||||
EXPECTED_HASH=$(head -1 "$SHA256_FILE" | awk '{print $1}')
|
||||
[[ ${#EXPECTED_HASH} -eq 64 && "$EXPECTED_HASH" =~ ^[0-9a-fA-F]+$ ]] || EXPECTED_HASH=""
|
||||
fi
|
||||
|
||||
USE_CACHED=0
|
||||
mkdir -p "$DOWNLOAD_CACHE_DIR"
|
||||
if [[ -f "$CACHED" && -s "$CACHED" ]]; then
|
||||
if [[ -n "$EXPECTED_HASH" ]]; then
|
||||
ACTUAL_HASH=$(sha256sum -b "$CACHED" 2>/dev/null | awk '{print $1}')
|
||||
if [[ "$ACTUAL_HASH" == "$EXPECTED_HASH" ]]; then
|
||||
USE_CACHED=1
|
||||
else
|
||||
rm -f "$CACHED"
|
||||
fi
|
||||
else
|
||||
USE_CACHED=1
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ $USE_CACHED -eq 1 ]]; then
|
||||
write_status "downloading" "Using cached image (checksum OK): $BASENAME" "" ""
|
||||
cp "$CACHED" "$XZ_FILE"
|
||||
else
|
||||
write_status "downloading" "Downloading $(basename "$URL")…" "" ""
|
||||
CURL_OPTS=(-f -L -o "$XZ_FILE" --connect-timeout 30 --max-time 7200)
|
||||
[[ "${CURL_INSECURE:-}" == "1" ]] && CURL_OPTS+=(-k)
|
||||
if ! curl "${CURL_OPTS[@]}" "$URL" 2>"$CURL_ERR"; then
|
||||
err_detail=$(head -c 200 "$CURL_ERR" | tr '\n' ' ' | sed 's/"/\\"/g')
|
||||
write_status "error" "" "" "Download failed: ${err_detail:-curl exited with error}"
|
||||
exit 1
|
||||
fi
|
||||
# Verify with official checksum if we have it
|
||||
if [[ -n "$EXPECTED_HASH" ]]; then
|
||||
ACTUAL_HASH=$(sha256sum -b "$XZ_FILE" 2>/dev/null | awk '{print $1}')
|
||||
if [[ "$ACTUAL_HASH" != "$EXPECTED_HASH" ]]; then
|
||||
write_status "error" "" "" "Checksum mismatch: download may be corrupted (expected ${EXPECTED_HASH:0:16}..., got ${ACTUAL_HASH:0:16}...). Try again."
|
||||
exit 1
|
||||
fi
|
||||
write_status "downloading" "Checksum OK, caching image…" "" ""
|
||||
fi
|
||||
cp "$XZ_FILE" "$CACHED"
|
||||
fi
|
||||
|
||||
write_status "decompressing" "Decompressing image…" "" ""
|
||||
@@ -80,11 +127,21 @@ if [[ "$FILE_TYPE" == *"HTML"* ]] || [[ "$FILE_TYPE" == *"text"* ]] && [[ "$FILE
|
||||
write_status "error" "" "" "Decompress failed: download is not an image (got: ${FILE_TYPE:0:80})"
|
||||
exit 1
|
||||
fi
|
||||
# Need ~3GB+ free for decompressing typical Raspios .img.xz; use same filesystem as TEMP_DIR
|
||||
AVAIL_KB=$(df -P "$TEMP_DIR" 2>/dev/null | awk 'NR==2 {print $4}')
|
||||
AVAIL_GB=0
|
||||
[[ -n "$AVAIL_KB" ]] && AVAIL_GB=$((AVAIL_KB / 1024 / 1024))
|
||||
if [[ -n "$AVAIL_KB" && "$AVAIL_KB" -lt 3145728 ]]; then
|
||||
write_status "error" "" "" "Decompress failed: not enough disk space (${AVAIL_GB}GB free on $(df -P "$TEMP_DIR" 2>/dev/null | awk 'NR==2 {print $6}'); need ~3GB). Free space or use a larger volume."
|
||||
exit 1
|
||||
fi
|
||||
IMG_FILE="$TEMP_DIR/image.img"
|
||||
XZ_ERR="$TEMP_DIR/xz_err.txt"
|
||||
if ! xz -d -k -f "$XZ_FILE" 2>"$XZ_ERR"; then
|
||||
err_detail=$(head -c 300 "$XZ_ERR" 2>/dev/null | tr '\n' ' ' | sed 's/"/\\"/g')
|
||||
write_status "error" "" "" "Decompress failed: ${err_detail:-xz exited with error}"
|
||||
# -T 1: single-threaded to reduce memory use; -d -k -f: decompress, keep source, force
|
||||
if ! xz -T 1 -d -k -f "$XZ_FILE" 2>"$XZ_ERR"; then
|
||||
err_detail=$(cat "$XZ_ERR" 2>/dev/null | tr '\n' ' ' | sed 's/"/\\"/g' | head -c 400)
|
||||
[[ -z "$err_detail" ]] && err_detail="out of disk space or memory?"
|
||||
write_status "error" "" "" "Decompress failed: ${err_detail} (free space: ${AVAIL_GB}GB)"
|
||||
exit 1
|
||||
fi
|
||||
[[ -f "$IMG_FILE" ]] || { write_status "error" "" "" "image.img not found after decompress"; exit 1; }
|
||||
|
||||
@@ -251,9 +251,18 @@ EMMC_SIZE_BYTES=8589934592
|
||||
ENV
|
||||
[[ -n "$BACKUPS_HOST_PATH" ]] && echo "BACKUPS_DIR=$BACKUPS_HOST_PATH" >> /opt/cm4-provisioning/env
|
||||
touch /etc/cm4-provisioning/enabled
|
||||
mkdir -p /var/lib/cm4-provisioning/backups /var/lib/cm4-provisioning/cloudinit-images /var/lib/cm4-provisioning/portal-files
|
||||
mkdir -p /var/lib/cm4-provisioning/backups /var/lib/cm4-provisioning/cloudinit-images /var/lib/cm4-provisioning/portal-files /var/lib/cm4-provisioning/download-cache
|
||||
[[ -n "$BACKUPS_HOST_PATH" ]] && mkdir -p "$BACKUPS_HOST_PATH"
|
||||
grep -q "CLOUDINIT_IMAGES_DIR" /opt/cm4-provisioning/env || echo "CLOUDINIT_IMAGES_DIR=/var/lib/cm4-provisioning/cloudinit-images" >> /opt/cm4-provisioning/env
|
||||
grep -q "CM4_DOWNLOAD_CACHE_DIR" /opt/cm4-provisioning/env || echo "CM4_DOWNLOAD_CACHE_DIR=/var/lib/cm4-provisioning/download-cache" >> /opt/cm4-provisioning/env
|
||||
|
||||
# --- Host: ensure xz-utils (and file) for cloud-init image build (decompress .img.xz) ---
|
||||
if command -v xz >/dev/null 2>&1; then
|
||||
log "Host: xz-utils already present, skipping."
|
||||
else
|
||||
log "Host: installing xz-utils and file (required for cloud-init image decompress)..."
|
||||
apt-get update -qq && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq xz-utils file 2>/dev/null || log "Warning: could not install xz-utils (no internet?). Cloud-init build will fail at decompress until you run: apt install -y xz-utils file"
|
||||
fi
|
||||
|
||||
# --- Host: install usbboot (rpiboot) only if not already present ---
|
||||
if [[ -x /opt/usbboot/rpiboot ]] || [[ -f /opt/usbboot/rpiboot ]]; then
|
||||
|
||||
Reference in New Issue
Block a user