Enhance Raspberry Pi OS image handling and dashboard UI

Update the API to support streaming and decompressing of golden images in .img.xz and .img.gz formats on the fly. Modify the cloud-init build process to compress images to .img.xz for size reduction. Revise the dashboard templates to set 'desktop' as the default variant for Raspberry Pi OS builds, improving user experience and clarity in options. Update related scripts to ensure compatibility with the new image handling features.

emmc-provisioning/dashboard/app.py

@@ -18,7 +18,7 @@ import urllib.request
 from functools import wraps
 from pathlib import Path
 
-from flask import Flask, render_template, jsonify, request, send_file, redirect, url_for, session
+from flask import Flask, render_template, jsonify, request, send_file, redirect, url_for, session, Response, stream_with_context
 
 from werkzeug.security import generate_password_hash, check_password_hash
 
@@ -871,9 +871,50 @@ def api_device_action_poll():
 
 @app.route("/api/golden-image")
 def api_golden_image():
-    """Stream the golden image for network deploy (device pulls and writes to eMMC)."""
+    """Stream the golden image for network deploy (device pulls and writes to eMMC). Decompresses .img.xz/.img.gz on the fly."""
     if not GOLDEN_IMAGE.is_file():
         return jsonify({"error": "Golden image not found"}), 404
+    resolved = GOLDEN_IMAGE.resolve()
+    if str(resolved).endswith(".img.xz"):
+        def stream():
+            proc = subprocess.Popen(
+                ["xz", "-d", "-c", str(resolved)],
+                stdout=subprocess.PIPE,
+                bufsize=65536,
+            )
+            try:
+                while True:
+                    chunk = proc.stdout.read(65536)
+                    if not chunk:
+                        break
+                    yield chunk
+            finally:
+                proc.wait()
+        return Response(
+            stream_with_context(stream()),
+            mimetype="application/octet-stream",
+            headers={"Content-Disposition": "attachment; filename=golden.img"},
+        )
+    if str(resolved).endswith(".img.gz"):
+        def stream():
+            proc = subprocess.Popen(
+                ["gzip", "-c", "-d", str(resolved)],
+                stdout=subprocess.PIPE,
+                bufsize=65536,
+            )
+            try:
+                while True:
+                    chunk = proc.stdout.read(65536)
+                    if not chunk:
+                        break
+                    yield chunk
+            finally:
+                proc.wait()
+        return Response(
+            stream_with_context(stream()),
+            mimetype="application/octet-stream",
+            headers={"Content-Disposition": "attachment; filename=golden.img"},
+        )
     return send_file(
         GOLDEN_IMAGE,
         mimetype="application/octet-stream",
@@ -1515,9 +1556,14 @@ def _build_status_write(phase, message, output_name=None, error=None):
         pass
 
 
-def _raspios_latest_url(variant="lite"):
-    """Resolve latest Raspberry Pi OS (arm64) .img.xz URL. variant=lite|full."""
-    slug = "raspios_lite_arm64" if variant == "lite" else "raspios_full_arm64"
+def _raspios_latest_url(variant="desktop"):
+    """Resolve latest Raspberry Pi OS (arm64) .img.xz URL. variant=lite|desktop|full."""
+    if variant == "lite":
+        slug = "raspios_lite_arm64"
+    elif variant == "full":
+        slug = "raspios_full_arm64"
+    else:
+        slug = "raspios_arm64"  # desktop (with desktop, not full)
     base = f"https://downloads.raspberrypi.com/{slug}/images"
     headers = {"User-Agent": "Mozilla/5.0 (compatible; CM4-Provisioning/1.0)"}
     try:
@@ -1578,9 +1624,9 @@ def api_build_cloudinit():
     if st.get("phase") in ("downloading", "decompressing", "injecting", "finalizing", "resolving"):
         return jsonify({"ok": False, "error": "A build is already in progress"}), 409
     body = request.get_json(silent=True) or {}
-    variant = (body.get("variant") or "lite").strip().lower()
-    if variant not in ("lite", "full"):
-        variant = "lite"
+    variant = (body.get("variant") or "desktop").strip().lower()
+    if variant not in ("lite", "desktop", "full"):
+        variant = "desktop"
     _build_status_write("resolving", "Resolving latest Raspberry Pi OS image URL…")
     url = _raspios_latest_url(variant)
     if not url:
@@ -1610,10 +1656,10 @@ def api_build_cloudinit():
 @app.route("/api/raspios-latest-url")
 @require_admin
 def api_raspios_latest_url():
-    """Return the URL of the latest Raspberry Pi OS (arm64) image. Query: variant=lite|full."""
-    variant = (request.args.get("variant") or "lite").strip().lower()
-    if variant not in ("lite", "full"):
-        variant = "lite"
+    """Return the URL of the latest Raspberry Pi OS (arm64) image. Query: variant=lite|desktop|full."""
+    variant = (request.args.get("variant") or "desktop").strip().lower()
+    if variant not in ("lite", "desktop", "full"):
+        variant = "desktop"
     url = _raspios_latest_url(variant)
     if not url:
         return jsonify({"ok": False, "url": None, "error": "Could not resolve latest image URL"}), 503

emmc-provisioning/dashboard/templates/cloudinit_build.html

@@ -67,7 +67,7 @@
       <p style="font-size:0.9rem; color:var(--text-dim); margin-bottom:0.75rem;">Download latest Raspberry Pi OS (arm64), inject cloud-init. Output goes to <strong>Cloud-init images</strong> on the Admin page.</p>
       <div style="margin-bottom:0.5rem;">
         <label>Variant:</label>
-        <select id="buildVariant"><option value="lite">Lite</option><option value="full">Full</option></select>
+        <select id="buildVariant"><option value="desktop" selected>Desktop (recommended)</option><option value="lite">Lite</option><option value="full">Full</option></select>
         <span id="buildRaspiosUrl" class="mono" style="margin-left:0.5rem;"></span>
       </div>
       <div style="margin-bottom:0.5rem;"><label><input type="checkbox" id="buildSetGolden" /> Set as golden after build</label></div>

emmc-provisioning/dashboard/templates/index.html

@@ -431,8 +431,9 @@
       <div style="margin-bottom:0.5rem;">
         <label>Variant: </label>
         <select id="buildVariant">
+          <option value="desktop" selected>Desktop (recommended)</option>
           <option value="lite">Lite (no desktop)</option>
-          <option value="full">Full (desktop)</option>
+          <option value="full">Full (desktop + software)</option>
         </select>
         <span id="buildRaspiosUrl" class="backups-mono" style="font-size:0.8rem; margin-left:0.5rem;"></span>
       </div>
@@ -800,7 +801,7 @@
 
     function getBuildVariant() {
       var sel = document.getElementById('buildVariant');
-      return (sel && sel.value) ? sel.value : 'lite';
+      return (sel && sel.value) ? sel.value : 'desktop';
     }
     function fetchRaspiosUrl() {
       var variant = getBuildVariant();

emmc-provisioning/host/build-cloudinit-image.sh

@@ -165,6 +165,21 @@ write_status "finalizing" "Copying image to cloud-init images…" "" ""
 mkdir -p "$OUTPUT_DIR"
 cp "$IMG_FILE" "$OUT_PATH"
 
+# Compress to .img.xz to reduce size (deploy supports decompress on the fly)
+write_status "finalizing" "Compressing image (xz)…" "" ""
+OUT_XZ="${OUT_PATH}.xz"
+# -T 0 = use all cores; fallback to -T 1 if old xz
+if ! xz -T 0 -z -k -f "$OUT_PATH" 2>/dev/null; then
+    xz -T 1 -z -k -f "$OUT_PATH" 2>/dev/null || true
+fi
+if [[ -f "$OUT_XZ" ]]; then
+    rm -f "$OUT_PATH"
+    OUT_NAME="${OUT_NAME}.xz"
+    OUT_PATH="$OUT_XZ"
+else
+    write_status "finalizing" "Compression skipped (xz failed), keeping raw image." "" ""
+fi
+
 if [[ "$SET_AS_GOLDEN" == "1" ]]; then
     rm -f "$GOLDEN_IMAGE"
     ln -sf "$OUT_PATH" "$GOLDEN_IMAGE"

emmc-provisioning/host/flash-emmc-on-connect.sh

@@ -217,7 +217,15 @@ for (( i = 0; i < WAIT_TIMEOUT; i += 2 )); do
       fi
       write_status "flashing" "Writing golden image…" "null"
       log "Flashing $GOLDEN_IMAGE to $target_dev..."
-      if dd if="$GOLDEN_IMAGE" of="$target_dev" bs=4M status=progress conv=fsync; then
+      GOLDEN_RESOLVED="$(readlink -f "$GOLDEN_IMAGE" 2>/dev/null || echo "$GOLDEN_IMAGE")"
+      if [[ "$GOLDEN_RESOLVED" == *.img.xz ]]; then
+        decompress_cmd="xz -d -c"
+      elif [[ "$GOLDEN_RESOLVED" == *.img.gz ]]; then
+        decompress_cmd="gzip -c -d"
+      else
+        decompress_cmd="cat"
+      fi
+      if $decompress_cmd "$GOLDEN_IMAGE" 2>/dev/null | dd of="$target_dev" bs=4M status=progress conv=fsync; then
         log "Flash complete. Remove eMMC disable jumper and power cycle the reTerminal."
         write_status "done" "Flash complete. Remove eMMC disable jumper and power cycle the reTerminal." "100"
         # Auto-reset status to idle after 90s so dashboard does not stay on this message (dashboard also auto-clears after 60s when open)