From 0bbd62213cb7f8c043ae8e2e41218d3853d29642 Mon Sep 17 00:00:00 2001 From: nearxos Date: Mon, 23 Feb 2026 10:11:15 +0200 Subject: [PATCH] Enhance Raspberry Pi OS image handling and dashboard UI Update the API to support streaming and decompressing of golden images in .img.xz and .img.gz formats on the fly. Modify the cloud-init build process to compress images to .img.xz for size reduction. Revise the dashboard templates to set 'desktop' as the default variant for Raspberry Pi OS builds, improving user experience and clarity in options. Update related scripts to ensure compatibility with the new image handling features. --- emmc-provisioning/dashboard/app.py | 70 +++++++++++++++---- .../dashboard/templates/cloudinit_build.html | 2 +- .../dashboard/templates/index.html | 5 +- .../host/build-cloudinit-image.sh | 15 ++++ .../host/flash-emmc-on-connect.sh | 10 ++- 5 files changed, 86 insertions(+), 16 deletions(-) diff --git a/emmc-provisioning/dashboard/app.py b/emmc-provisioning/dashboard/app.py index 4110a05..21ef57e 100644 --- a/emmc-provisioning/dashboard/app.py +++ b/emmc-provisioning/dashboard/app.py @@ -18,7 +18,7 @@ import urllib.request from functools import wraps from pathlib import Path -from flask import Flask, render_template, jsonify, request, send_file, redirect, url_for, session +from flask import Flask, render_template, jsonify, request, send_file, redirect, url_for, session, Response, stream_with_context from werkzeug.security import generate_password_hash, check_password_hash @@ -871,9 +871,50 @@ def api_device_action_poll(): @app.route("/api/golden-image") def api_golden_image(): - """Stream the golden image for network deploy (device pulls and writes to eMMC).""" + """Stream the golden image for network deploy (device pulls and writes to eMMC). Decompresses .img.xz/.img.gz on the fly.""" if not GOLDEN_IMAGE.is_file(): return jsonify({"error": "Golden image not found"}), 404 + resolved = GOLDEN_IMAGE.resolve() + if str(resolved).endswith(".img.xz"): + def stream(): + proc = subprocess.Popen( + ["xz", "-d", "-c", str(resolved)], + stdout=subprocess.PIPE, + bufsize=65536, + ) + try: + while True: + chunk = proc.stdout.read(65536) + if not chunk: + break + yield chunk + finally: + proc.wait() + return Response( + stream_with_context(stream()), + mimetype="application/octet-stream", + headers={"Content-Disposition": "attachment; filename=golden.img"}, + ) + if str(resolved).endswith(".img.gz"): + def stream(): + proc = subprocess.Popen( + ["gzip", "-c", "-d", str(resolved)], + stdout=subprocess.PIPE, + bufsize=65536, + ) + try: + while True: + chunk = proc.stdout.read(65536) + if not chunk: + break + yield chunk + finally: + proc.wait() + return Response( + stream_with_context(stream()), + mimetype="application/octet-stream", + headers={"Content-Disposition": "attachment; filename=golden.img"}, + ) return send_file( GOLDEN_IMAGE, mimetype="application/octet-stream", @@ -1515,9 +1556,14 @@ def _build_status_write(phase, message, output_name=None, error=None): pass -def _raspios_latest_url(variant="lite"): - """Resolve latest Raspberry Pi OS (arm64) .img.xz URL. variant=lite|full.""" - slug = "raspios_lite_arm64" if variant == "lite" else "raspios_full_arm64" +def _raspios_latest_url(variant="desktop"): + """Resolve latest Raspberry Pi OS (arm64) .img.xz URL. variant=lite|desktop|full.""" + if variant == "lite": + slug = "raspios_lite_arm64" + elif variant == "full": + slug = "raspios_full_arm64" + else: + slug = "raspios_arm64" # desktop (with desktop, not full) base = f"https://downloads.raspberrypi.com/{slug}/images" headers = {"User-Agent": "Mozilla/5.0 (compatible; CM4-Provisioning/1.0)"} try: @@ -1578,9 +1624,9 @@ def api_build_cloudinit(): if st.get("phase") in ("downloading", "decompressing", "injecting", "finalizing", "resolving"): return jsonify({"ok": False, "error": "A build is already in progress"}), 409 body = request.get_json(silent=True) or {} - variant = (body.get("variant") or "lite").strip().lower() - if variant not in ("lite", "full"): - variant = "lite" + variant = (body.get("variant") or "desktop").strip().lower() + if variant not in ("lite", "desktop", "full"): + variant = "desktop" _build_status_write("resolving", "Resolving latest Raspberry Pi OS image URL…") url = _raspios_latest_url(variant) if not url: @@ -1610,10 +1656,10 @@ def api_build_cloudinit(): @app.route("/api/raspios-latest-url") @require_admin def api_raspios_latest_url(): - """Return the URL of the latest Raspberry Pi OS (arm64) image. Query: variant=lite|full.""" - variant = (request.args.get("variant") or "lite").strip().lower() - if variant not in ("lite", "full"): - variant = "lite" + """Return the URL of the latest Raspberry Pi OS (arm64) image. Query: variant=lite|desktop|full.""" + variant = (request.args.get("variant") or "desktop").strip().lower() + if variant not in ("lite", "desktop", "full"): + variant = "desktop" url = _raspios_latest_url(variant) if not url: return jsonify({"ok": False, "url": None, "error": "Could not resolve latest image URL"}), 503 diff --git a/emmc-provisioning/dashboard/templates/cloudinit_build.html b/emmc-provisioning/dashboard/templates/cloudinit_build.html index 767e48f..1e8b8db 100644 --- a/emmc-provisioning/dashboard/templates/cloudinit_build.html +++ b/emmc-provisioning/dashboard/templates/cloudinit_build.html @@ -67,7 +67,7 @@

Download latest Raspberry Pi OS (arm64), inject cloud-init. Output goes to Cloud-init images on the Admin page.

- +
diff --git a/emmc-provisioning/dashboard/templates/index.html b/emmc-provisioning/dashboard/templates/index.html index 569fd6c..045a331 100644 --- a/emmc-provisioning/dashboard/templates/index.html +++ b/emmc-provisioning/dashboard/templates/index.html @@ -431,8 +431,9 @@
@@ -800,7 +801,7 @@ function getBuildVariant() { var sel = document.getElementById('buildVariant'); - return (sel && sel.value) ? sel.value : 'lite'; + return (sel && sel.value) ? sel.value : 'desktop'; } function fetchRaspiosUrl() { var variant = getBuildVariant(); diff --git a/emmc-provisioning/host/build-cloudinit-image.sh b/emmc-provisioning/host/build-cloudinit-image.sh index 8f6eedf..c709a59 100644 --- a/emmc-provisioning/host/build-cloudinit-image.sh +++ b/emmc-provisioning/host/build-cloudinit-image.sh @@ -165,6 +165,21 @@ write_status "finalizing" "Copying image to cloud-init images…" "" "" mkdir -p "$OUTPUT_DIR" cp "$IMG_FILE" "$OUT_PATH" +# Compress to .img.xz to reduce size (deploy supports decompress on the fly) +write_status "finalizing" "Compressing image (xz)…" "" "" +OUT_XZ="${OUT_PATH}.xz" +# -T 0 = use all cores; fallback to -T 1 if old xz +if ! xz -T 0 -z -k -f "$OUT_PATH" 2>/dev/null; then + xz -T 1 -z -k -f "$OUT_PATH" 2>/dev/null || true +fi +if [[ -f "$OUT_XZ" ]]; then + rm -f "$OUT_PATH" + OUT_NAME="${OUT_NAME}.xz" + OUT_PATH="$OUT_XZ" +else + write_status "finalizing" "Compression skipped (xz failed), keeping raw image." "" "" +fi + if [[ "$SET_AS_GOLDEN" == "1" ]]; then rm -f "$GOLDEN_IMAGE" ln -sf "$OUT_PATH" "$GOLDEN_IMAGE" diff --git a/emmc-provisioning/host/flash-emmc-on-connect.sh b/emmc-provisioning/host/flash-emmc-on-connect.sh index e608b3b..d25206e 100644 --- a/emmc-provisioning/host/flash-emmc-on-connect.sh +++ b/emmc-provisioning/host/flash-emmc-on-connect.sh @@ -217,7 +217,15 @@ for (( i = 0; i < WAIT_TIMEOUT; i += 2 )); do fi write_status "flashing" "Writing golden image…" "null" log "Flashing $GOLDEN_IMAGE to $target_dev..." - if dd if="$GOLDEN_IMAGE" of="$target_dev" bs=4M status=progress conv=fsync; then + GOLDEN_RESOLVED="$(readlink -f "$GOLDEN_IMAGE" 2>/dev/null || echo "$GOLDEN_IMAGE")" + if [[ "$GOLDEN_RESOLVED" == *.img.xz ]]; then + decompress_cmd="xz -d -c" + elif [[ "$GOLDEN_RESOLVED" == *.img.gz ]]; then + decompress_cmd="gzip -c -d" + else + decompress_cmd="cat" + fi + if $decompress_cmd "$GOLDEN_IMAGE" 2>/dev/null | dd of="$target_dev" bs=4M status=progress conv=fsync; then log "Flash complete. Remove eMMC disable jumper and power cycle the reTerminal." write_status "done" "Flash complete. Remove eMMC disable jumper and power cycle the reTerminal." "100" # Auto-reset status to idle after 90s so dashboard does not stay on this message (dashboard also auto-clears after 60s when open)