5f05663706
<message>Add a cleanup function to handle cancellation of the xz compression process in the build-cloudinit-image.sh script. This enhancement allows for a more robust response to cancellation requests, ensuring that resources are properly released and status messages are updated accordingly. The script now traps termination signals and cleans up temporary files, improving the overall reliability of the cloud-init image building workflow.
237 lines
9.0 KiB
Bash
237 lines
9.0 KiB
Bash
#!/usr/bin/env bash
|
|
# Run on the Proxmox host when build_cloudinit_request.json appears in the provisioning dir.
|
|
# Downloads Raspberry Pi OS image from URL, injects cloud-init NoCloud files, saves to BACKUPS_DIR.
|
|
# Uses loop devices and mount (available on host, not in unprivileged LXC).
|
|
# Triggered by systemd path unit cm4-build-cloudinit.path.
|
|
|
|
set -e
|
|
PROV_DIR="${CM4_PROVISIONING_DIR:-/var/lib/cm4-provisioning}"
|
|
REQUEST_FILE="$PROV_DIR/build_cloudinit_request.json"
|
|
STATUS_FILE="$PROV_DIR/build_cloudinit_status.json"
|
|
CANCEL_FILE="$PROV_DIR/build_cloudinit_cancel"
|
|
[[ -f /opt/cm4-provisioning/env ]] && source /opt/cm4-provisioning/env
|
|
BACKUPS_DIR="${BACKUPS_DIR:-$PROV_DIR/backups}"
|
|
CLOUDINIT_IMAGES_DIR="${CLOUDINIT_IMAGES_DIR:-$PROV_DIR/cloudinit-images}"
|
|
DOWNLOAD_CACHE_DIR="${CM4_DOWNLOAD_CACHE_DIR:-$PROV_DIR/download-cache}"
|
|
# Write built cloud-init images to CLOUDINIT_IMAGES_DIR (separate from backups)
|
|
OUTPUT_DIR="${CLOUDINIT_IMAGES_DIR}"
|
|
GOLDEN_IMAGE="${GOLDEN_IMAGE:-$PROV_DIR/golden.img}"
|
|
|
|
write_status() {
|
|
local phase="$1" message="$2" output_name="$3" error="$4"
|
|
printf '{"phase":"%s","message":"%s","output_name":%s,"error":%s,"updated":%s}\n' \
|
|
"$phase" "$message" \
|
|
"$([ -n "$output_name" ] && echo "\"$output_name\"" || echo "null")" \
|
|
"$([ -n "$error" ] && echo "\"${error//\"/\\\"}\"" || echo "null")" \
|
|
"$(date +%s)" > "$STATUS_FILE"
|
|
}
|
|
|
|
check_cancel() {
|
|
if [[ -f "$CANCEL_FILE" ]]; then
|
|
write_status "cancelled" "Build cancelled." "" ""
|
|
rm -f "$REQUEST_FILE" "$CANCEL_FILE"
|
|
exit 0
|
|
fi
|
|
}
|
|
|
|
[[ -f "$REQUEST_FILE" ]] || { echo "No request file"; exit 0; }
|
|
check_cancel
|
|
|
|
# Use temp dir on provisioning dir (not /tmp) so we have enough space for decompress (~3GB+)
|
|
mkdir -p "$PROV_DIR"
|
|
TEMP_DIR=$(mktemp -d -p "$PROV_DIR" build.XXXXXX 2>/dev/null) || TEMP_DIR="$PROV_DIR/build.$$" && mkdir -p "$TEMP_DIR"
|
|
trap 'rm -rf "$TEMP_DIR"; rm -f "$REQUEST_FILE"' EXIT
|
|
|
|
# Extract fields from JSON into temp files (handles multi-line content)
|
|
python3 - "$REQUEST_FILE" "$TEMP_DIR" << 'PY'
|
|
import json, sys
|
|
with open(sys.argv[1]) as f:
|
|
d = json.load(f)
|
|
out = sys.argv[2]
|
|
for name, key in [("user-data", "user_data"), ("meta-data", "meta_data"), ("network-config", "network_config")]:
|
|
with open(f"{out}/{name}", "w") as f:
|
|
f.write(d.get(key, ""))
|
|
with open(f"{out}/variant", "w") as f:
|
|
f.write(d.get("variant", "lite"))
|
|
with open(f"{out}/url", "w") as f:
|
|
f.write(d.get("url", ""))
|
|
with open(f"{out}/set_golden", "w") as f:
|
|
f.write("1" if d.get("set_as_golden_after") else "0")
|
|
name = (d.get("image_name") or "").strip()[:64]
|
|
name = "".join(c for c in name if c.isalnum() or c in "_-")
|
|
with open(f"{out}/image_name", "w") as f:
|
|
f.write(name)
|
|
PY
|
|
|
|
URL=$(cat "$TEMP_DIR/url")
|
|
VARIANT=$(cat "$TEMP_DIR/variant")
|
|
SET_AS_GOLDEN=$(cat "$TEMP_DIR/set_golden")
|
|
IMAGE_NAME="$(cat "$TEMP_DIR/image_name" 2>/dev/null || true)"
|
|
[[ -n "$URL" ]] || { write_status "error" "" "" "Missing url in request"; exit 1; }
|
|
|
|
DATE_SUFFIX="$(date +%Y%m%d-%H%M%S)"
|
|
if [[ -n "$IMAGE_NAME" ]]; then
|
|
OUT_NAME="${IMAGE_NAME}-${DATE_SUFFIX}.img"
|
|
else
|
|
OUT_NAME="raspios-${VARIANT}-cloudinit-${DATE_SUFFIX}.img"
|
|
fi
|
|
OUT_PATH="$OUTPUT_DIR/$OUT_NAME"
|
|
|
|
BASENAME=$(basename "$URL")
|
|
SHA256_URL="${URL}.sha256"
|
|
CACHED="$DOWNLOAD_CACHE_DIR/$BASENAME"
|
|
XZ_FILE="$TEMP_DIR/image.img.xz"
|
|
CURL_ERR="$TEMP_DIR/curl_err.txt"
|
|
SHA256_FILE="$TEMP_DIR/expected.sha256"
|
|
|
|
# Fetch official SHA256 checksum if available (Raspberry Pi OS publishes .img.xz.sha256 next to each image)
|
|
check_cancel
|
|
write_status "downloading" "Checking for existing image and checksum…" "" ""
|
|
EXPECTED_HASH=""
|
|
if curl -sfL -o "$SHA256_FILE" "$SHA256_URL" 2>/dev/null && [[ -s "$SHA256_FILE" ]]; then
|
|
# Format is typically "hash filename" or just "hash"; take first 64-char hex token
|
|
EXPECTED_HASH=$(head -1 "$SHA256_FILE" | awk '{print $1}')
|
|
[[ ${#EXPECTED_HASH} -eq 64 && "$EXPECTED_HASH" =~ ^[0-9a-fA-F]+$ ]] || EXPECTED_HASH=""
|
|
fi
|
|
|
|
USE_CACHED=0
|
|
mkdir -p "$DOWNLOAD_CACHE_DIR"
|
|
if [[ -f "$CACHED" && -s "$CACHED" ]]; then
|
|
if [[ -n "$EXPECTED_HASH" ]]; then
|
|
ACTUAL_HASH=$(sha256sum -b "$CACHED" 2>/dev/null | awk '{print $1}')
|
|
if [[ "$ACTUAL_HASH" == "$EXPECTED_HASH" ]]; then
|
|
USE_CACHED=1
|
|
else
|
|
rm -f "$CACHED"
|
|
fi
|
|
else
|
|
USE_CACHED=1
|
|
fi
|
|
fi
|
|
|
|
if [[ $USE_CACHED -eq 1 ]]; then
|
|
write_status "downloading" "Using cached image (checksum OK): $BASENAME" "" ""
|
|
cp "$CACHED" "$XZ_FILE"
|
|
else
|
|
write_status "downloading" "Downloading $(basename "$URL")…" "" ""
|
|
CURL_OPTS=(-f -L -o "$XZ_FILE" --connect-timeout 30 --max-time 7200)
|
|
[[ "${CURL_INSECURE:-}" == "1" ]] && CURL_OPTS+=(-k)
|
|
if ! curl "${CURL_OPTS[@]}" "$URL" 2>"$CURL_ERR"; then
|
|
err_detail=$(head -c 200 "$CURL_ERR" | tr '\n' ' ' | sed 's/"/\\"/g')
|
|
write_status "error" "" "" "Download failed: ${err_detail:-curl exited with error}"
|
|
exit 1
|
|
fi
|
|
# Verify with official checksum if we have it
|
|
if [[ -n "$EXPECTED_HASH" ]]; then
|
|
ACTUAL_HASH=$(sha256sum -b "$XZ_FILE" 2>/dev/null | awk '{print $1}')
|
|
if [[ "$ACTUAL_HASH" != "$EXPECTED_HASH" ]]; then
|
|
write_status "error" "" "" "Checksum mismatch: download may be corrupted (expected ${EXPECTED_HASH:0:16}..., got ${ACTUAL_HASH:0:16}...). Try again."
|
|
exit 1
|
|
fi
|
|
write_status "downloading" "Checksum OK, caching image…" "" ""
|
|
fi
|
|
cp "$XZ_FILE" "$CACHED"
|
|
fi
|
|
|
|
check_cancel
|
|
write_status "decompressing" "Decompressing image…" "" ""
|
|
# Check we have a real xz file (not HTML error page)
|
|
if ! command -v xz >/dev/null 2>&1; then
|
|
write_status "error" "" "" "Decompress failed: xz not installed. Install xz-utils (apt install xz-utils)"
|
|
exit 1
|
|
fi
|
|
if [[ ! -s "$XZ_FILE" ]]; then
|
|
write_status "error" "" "" "Decompress failed: downloaded file is empty"
|
|
exit 1
|
|
fi
|
|
FILE_TYPE=$(file -b "$XZ_FILE" 2>/dev/null || true)
|
|
if [[ "$FILE_TYPE" == *"HTML"* ]] || [[ "$FILE_TYPE" == *"text"* ]] && [[ "$FILE_TYPE" != *"XZ"* ]]; then
|
|
write_status "error" "" "" "Decompress failed: download is not an image (got: ${FILE_TYPE:0:80})"
|
|
exit 1
|
|
fi
|
|
# Need ~3GB+ free for decompressing typical Raspios .img.xz; use same filesystem as TEMP_DIR
|
|
AVAIL_KB=$(df -P "$TEMP_DIR" 2>/dev/null | awk 'NR==2 {print $4}')
|
|
AVAIL_GB=0
|
|
[[ -n "$AVAIL_KB" ]] && AVAIL_GB=$((AVAIL_KB / 1024 / 1024))
|
|
if [[ -n "$AVAIL_KB" && "$AVAIL_KB" -lt 3145728 ]]; then
|
|
write_status "error" "" "" "Decompress failed: not enough disk space (${AVAIL_GB}GB free on $(df -P "$TEMP_DIR" 2>/dev/null | awk 'NR==2 {print $6}'); need ~3GB). Free space or use a larger volume."
|
|
exit 1
|
|
fi
|
|
IMG_FILE="$TEMP_DIR/image.img"
|
|
XZ_ERR="$TEMP_DIR/xz_err.txt"
|
|
# -T 1: single-threaded to reduce memory use; -d -k -f: decompress, keep source, force
|
|
if ! xz -T 1 -d -k -f "$XZ_FILE" 2>"$XZ_ERR"; then
|
|
err_detail=$(cat "$XZ_ERR" 2>/dev/null | tr '\n' ' ' | sed 's/"/\\"/g' | head -c 400)
|
|
[[ -z "$err_detail" ]] && err_detail="out of disk space or memory?"
|
|
write_status "error" "" "" "Decompress failed: ${err_detail} (free space: ${AVAIL_GB}GB)"
|
|
exit 1
|
|
fi
|
|
[[ -f "$IMG_FILE" ]] || { write_status "error" "" "" "image.img not found after decompress"; exit 1; }
|
|
|
|
check_cancel
|
|
write_status "injecting" "Mounting boot partition and injecting cloud-init…" "" ""
|
|
LOOP=$(losetup -f --show -P "$IMG_FILE")
|
|
boot_part="${LOOP}p1"
|
|
[[ -b "$boot_part" ]] || boot_part="${LOOP}p2"
|
|
[[ -b "$boot_part" ]] || { write_status "error" "" "" "Boot partition not found"; losetup -d "$LOOP"; exit 1; }
|
|
|
|
MNT="$TEMP_DIR/mnt"
|
|
mkdir -p "$MNT"
|
|
mount "$boot_part" "$MNT"
|
|
cp "$TEMP_DIR/user-data" "$MNT/user-data"
|
|
cp "$TEMP_DIR/meta-data" "$MNT/meta-data"
|
|
cp "$TEMP_DIR/network-config" "$MNT/network-config"
|
|
umount "$MNT"
|
|
losetup -d "$LOOP"
|
|
|
|
check_cancel
|
|
write_status "finalizing" "Copying image to cloud-init images…" "" ""
|
|
mkdir -p "$OUTPUT_DIR"
|
|
cp "$IMG_FILE" "$OUT_PATH"
|
|
|
|
# Compress to .img.xz to reduce size (deploy supports decompress on the fly)
|
|
check_cancel
|
|
write_status "finalizing" "Compressing image (xz)…" "" ""
|
|
OUT_XZ="${OUT_PATH}.xz"
|
|
XZ_PID=""
|
|
cleanup_xz_and_exit() {
|
|
local phase="${1:-cancelled}"
|
|
if [[ -n "$XZ_PID" ]] && kill -0 "$XZ_PID" 2>/dev/null; then
|
|
kill "$XZ_PID" 2>/dev/null
|
|
wait "$XZ_PID" 2>/dev/null || true
|
|
fi
|
|
write_status "$phase" "Build cancelled." "" ""
|
|
rm -f "$REQUEST_FILE" "$CANCEL_FILE"
|
|
exit 0
|
|
}
|
|
trap 'cleanup_xz_and_exit "cancelled"' TERM INT
|
|
xz -T 0 -z -k -f "$OUT_PATH" 2>/dev/null &
|
|
XZ_PID=$!
|
|
while kill -0 "$XZ_PID" 2>/dev/null; do
|
|
if [[ -f "$CANCEL_FILE" ]]; then
|
|
cleanup_xz_and_exit "cancelled"
|
|
fi
|
|
sleep 2
|
|
done
|
|
wait "$XZ_PID" 2>/dev/null || true
|
|
trap - TERM INT
|
|
if [[ ! -f "$OUT_XZ" ]]; then
|
|
xz -T 1 -z -k -f "$OUT_PATH" 2>/dev/null || true
|
|
|
|
fi
|
|
if [[ -f "$OUT_XZ" ]]; then
|
|
rm -f "$OUT_PATH"
|
|
OUT_NAME="${OUT_NAME}.xz"
|
|
OUT_PATH="$OUT_XZ"
|
|
else
|
|
write_status "finalizing" "Compression skipped (xz failed), keeping raw image." "" ""
|
|
fi
|
|
|
|
if [[ "$SET_AS_GOLDEN" == "1" ]]; then
|
|
rm -f "$GOLDEN_IMAGE"
|
|
ln -sf "$OUT_PATH" "$GOLDEN_IMAGE"
|
|
write_status "done" "Built $OUT_NAME and set as golden image." "$OUT_NAME" ""
|
|
else
|
|
write_status "done" "Built $OUT_NAME" "$OUT_NAME" ""
|
|
fi
|