Add cloud-init image building functionality to eMMC provisioning dashboard: implement API for downloading the latest Raspberry Pi OS Lite, injecting cloud-init files, and updating UI for cloud-init image creation. Enhance backup options with compression support for raw .img files using PiShrink, and update documentation to reflect new features and usage instructions.

chromium-setup/emmc-provisioning/dashboard/README.md

@@ -4,7 +4,8 @@ Flask web UI to monitor the eMMC deployment process and show device connection s
 
 - **Connection steps**: Numbered instructions for putting the reTerminal in boot mode and connecting it.
 - **Live status**: Idle / Connecting (rpiboot) / Flashing / Backup / Done / Error, with optional progress.
-- **Backup / Restore**: Toggle between **Flash** (deploy golden image) and **Backup** (save eMMC to a timestamped file when device is connected in boot mode). List and download saved backups.
+- **Backup / Restore**: Toggle between **Flash** (deploy golden image) and **Backup** (save eMMC to a timestamped file when device is connected in boot mode). List and download saved backups. For each raw `.img` backup you can click **Shrink** (PiShrink) or **Compress** (shrink + xz) to reduce size.
+- **Build cloud-init image**: Download the latest Raspberry Pi OS Lite (arm64) from the official repository and inject cloud-init NoCloud files so the image is ready for first-boot configuration. The result appears in the backups list; set it as golden to deploy. *Requires* loop devices and mount (if the dashboard runs in an LXC, the container may need privileged mode or loop support).
 - **Recent log**: Tail of the flash log (from the host, via the shared bind mount).
 
 The dashboard reads `/var/lib/cm4-provisioning/status.json` and `flash.log`, which the flash script (running on the Proxmox host) updates. When the dashboard runs inside the LXC, that directory is bind-mounted from the host, so it sees the same files.
@@ -36,3 +37,5 @@ systemctl enable --now cm4-dashboard
 
 - `CM4_STATUS_FILE` – path to status JSON (default: `/var/lib/cm4-provisioning/status.json`).
 - `CM4_LOG_FILE` – path to flash log (default: `/var/lib/cm4-provisioning/flash.log`).
+- `CM4_BACKUPS_DIR` – path to backups directory (default: `…/backups`).
+- `CM4_BUILD_STATUS_FILE` – path to build-cloudinit status JSON (default: `…/build_cloudinit_status.json`).

chromium-setup/emmc-provisioning/dashboard/app.py

@@ -7,9 +7,13 @@ Supports USB boot mode (ask Backup/Deploy) and network-booted devices (register,
 
 import json
 import os
+import re
 import shutil
 import subprocess
+import tempfile
+import threading
 import time
+import urllib.request
 from pathlib import Path
 
 from flask import Flask, render_template, jsonify, request, send_file, Response
@@ -34,6 +38,30 @@ DEVICE_SOURCE_FILE = os.environ.get("CM4_DEVICE_SOURCE_FILE", str(BASE_DIR / "de
 BACKUPS_DIR = Path(os.environ.get("CM4_BACKUPS_DIR", str(BASE_DIR / "backups")))
 GOLDEN_IMAGE = Path(os.environ.get("CM4_GOLDEN_IMAGE", str(BASE_DIR / "golden.img")))
 NETWORK_DEVICES_FILE = Path(os.environ.get("CM4_NETWORK_DEVICES_FILE", str(BASE_DIR / "network_devices.json")))
+BUILD_STATUS_FILE = Path(os.environ.get("CM4_BUILD_STATUS_FILE", str(BASE_DIR / "build_cloudinit_status.json")))
+
+# Default cloud-init user-data for Raspberry Pi OS (NoCloud on boot partition)
+DEFAULT_USER_DATA = """#cloud-config
+package_update: true
+package_upgrade: false
+packages:
+  - curl
+
+runcmd:
+  - curl -fsSL "http://YOUR_FILE_SERVER/provisioning/bootstrap.sh" -o /tmp/bootstrap.sh
+  - chmod +x /tmp/bootstrap.sh
+  - /tmp/bootstrap.sh
+"""
+
+DEFAULT_META_DATA = """instance-id: raspios-cloudinit-001
+local-hostname: reterminal
+"""
+
+DEFAULT_NETWORK_CONFIG = """version: 2
+ethernets:
+  eth0:
+    dhcp4: true
+"""
 
 DEFAULT_STATUS = {
     "phase": "idle",
@@ -355,6 +383,48 @@ def api_backup_shrink(name):
         return jsonify({"ok": False, "error": str(e)}), 500
 
 
+@app.route("/api/backups/<path:name>/compress", methods=["POST"])
+def api_backup_compress(name):
+    """Run PiShrink with compression (shrink + gz or xz) on a raw .img backup. Produces .img.gz or .img.xz."""
+    if not _safe_backup_name(name):
+        return jsonify({"ok": False, "error": "invalid backup name"}), 400
+    if not name.endswith(".img") or name.endswith(".img.gz") or name.endswith(".img.xz"):
+        return jsonify({"ok": False, "error": "only raw .img files can be compressed (not .img.gz / .img.xz)"}), 400
+    path = BACKUPS_DIR / name
+    if not path.is_file():
+        return jsonify({"ok": False, "error": "backup not found"}), 404
+    body = request.get_json(force=True, silent=True) or {}
+    fmt = (body.get("format") or request.args.get("format") or "xz").strip().lower()
+    if fmt not in ("gz", "gzip", "xz"):
+        fmt = "xz"
+    if fmt == "gzip":
+        fmt = "gz"
+    pishrink = shutil.which("pishrink.sh") or "/usr/local/bin/pishrink.sh"
+    if not pishrink or not os.path.isfile(pishrink):
+        return jsonify({"ok": False, "error": "PiShrink not installed"}), 503
+    opts = ["-n", "-Z", "-a"] if fmt == "xz" else ["-n", "-z", "-a"]
+    try:
+        proc = subprocess.run(
+            [pishrink] + opts + [name],
+            cwd=str(BACKUPS_DIR),
+            capture_output=True,
+            text=True,
+            timeout=3600,
+        )
+        if proc.returncode != 0:
+            return jsonify({
+                "ok": False,
+                "error": "PiShrink failed",
+                "detail": (proc.stderr or proc.stdout or "").strip() or f"exit code {proc.returncode}",
+            }), 500
+        ext = ".xz" if fmt == "xz" else ".gz"
+        return jsonify({"ok": True, "message": f"Compressed to {name}{ext}"})
+    except subprocess.TimeoutExpired:
+        return jsonify({"ok": False, "error": "PiShrink timed out"}), 504
+    except OSError as e:
+        return jsonify({"ok": False, "error": str(e)}), 500
+
+
 @app.route("/api/backups/<path:name>", methods=["PATCH"])
 def api_backup_update(name):
     """Update backup metadata (display name, description) or rename the file."""
@@ -405,6 +475,184 @@ def api_backup_download(name):
     return send_file(path, as_attachment=True, download_name=name)
 
 
+def _build_status_read():
+    try:
+        if BUILD_STATUS_FILE.is_file():
+            with open(BUILD_STATUS_FILE, "r") as f:
+                return json.load(f)
+    except (json.JSONDecodeError, OSError):
+        pass
+    return {"phase": "idle", "message": "", "output_name": None, "error": None}
+
+
+def _build_status_write(phase, message, output_name=None, error=None):
+    try:
+        BUILD_STATUS_FILE.parent.mkdir(parents=True, exist_ok=True)
+        with open(BUILD_STATUS_FILE, "w") as f:
+            json.dump({
+                "phase": phase,
+                "message": message,
+                "output_name": output_name,
+                "error": error,
+                "updated": time.time(),
+            }, f, indent=2)
+    except OSError:
+        pass
+
+
+def _raspios_latest_lite_url():
+    """Resolve latest Raspberry Pi OS Lite (arm64) .img.xz URL from official index."""
+    base = "https://downloads.raspberrypi.com/raspios_lite_arm64/images"
+    try:
+        with urllib.request.urlopen(base + "/", timeout=15) as r:
+            html = r.read().decode("utf-8", errors="ignore")
+        # Match raspios_lite_arm64-YYYY-MM-DD/
+        folders = re.findall(r"raspios_lite_arm64-(\d{4}-\d{2}-\d{2})/", html)
+        if not folders:
+            return None
+        latest = sorted(folders)[-1]
+        folder_url = f"{base}/raspios_lite_arm64-{latest}/"
+        with urllib.request.urlopen(folder_url, timeout=15) as r:
+            folder_html = r.read().decode("utf-8", errors="ignore")
+        # Match .img.xz link (skip .sha256, .torrent, etc.)
+        m = re.search(r'href="([^"]+\.img\.xz)"', folder_html)
+        if not m:
+            return None
+        return folder_url + m.group(1)
+    except Exception:
+        return None
+
+
+def _build_cloudinit_worker(variant, user_data, meta_data, network_config):
+    """Background: download Raspios, inject cloud-init, save to BACKUPS_DIR."""
+    out_name = f"raspios-{variant}-cloudinit-{time.strftime('%Y%m%d-%H%M%S')}.img"
+    out_path = BACKUPS_DIR / out_name
+    mount_point = None
+    loop_dev = None
+    temp_dir = None
+    img_path = None
+    xz_path = None
+    try:
+        _build_status_write("resolving", "Resolving latest Raspberry Pi OS image URL…")
+        url = _raspios_latest_lite_url()
+        if not url:
+            _build_status_write("error", "", output_name=None, error="Could not resolve latest Raspios Lite URL")
+            return
+        _build_status_write("downloading", f"Downloading {url.split('/')[-1]}… (this may take several minutes)")
+        temp_dir = Path(tempfile.mkdtemp(prefix="cloudinit-build-", dir=str(BACKUPS_DIR)))
+        xz_path = temp_dir / "image.img.xz"
+        with urllib.request.urlopen(url, timeout=3600) as resp:
+            total = int(resp.headers.get("Content-Length", 0)) or 0
+            chunk_size = 1024 * 1024
+            with open(xz_path, "wb") as f:
+                done = 0
+                while True:
+                    chunk = resp.read(chunk_size)
+                    if not chunk:
+                        break
+                    f.write(chunk)
+                    done += len(chunk)
+                    if total and done % (50 * 1024 * 1024) < chunk_size:
+                        _build_status_write("downloading", f"Downloaded {done // (1024*1024)} MB…")
+        _build_status_write("decompressing", "Decompressing image…")
+        img_path = temp_dir / "image.img"
+        subprocess.run(["xz", "-d", "-k", "-f", str(xz_path)], check=True, capture_output=True, timeout=1800, cwd=str(temp_dir))
+        if not img_path.exists():
+            _build_status_write("error", "", output_name=None, error="Decompress failed: image.img not found")
+            return
+        _build_status_write("injecting", "Mounting boot partition and injecting cloud-init…")
+        loop_out = subprocess.run(["losetup", "-f", "--show", "-P", str(img_path)], capture_output=True, text=True, timeout=10)
+        if loop_out.returncode != 0:
+            _build_status_write("error", "", output_name=None, error="losetup failed (loop device may not be available in this container)")
+            return
+        loop_dev = loop_out.stdout.strip()
+        # First partition is usually boot (FAT32)
+        boot_part = f"{loop_dev}p1"
+        if not Path(boot_part).exists():
+            boot_part = f"{loop_dev}p2"
+        if not Path(boot_part).exists():
+            _build_status_write("error", "", output_name=None, error="Boot partition not found")
+            return
+        mount_point = temp_dir / "mnt"
+        mount_point.mkdir(exist_ok=True)
+        subprocess.run(["mount", boot_part, str(mount_point)], check=True, capture_output=True, timeout=10)
+        try:
+            (mount_point / "user-data").write_text(user_data or DEFAULT_USER_DATA)
+            (mount_point / "meta-data").write_text(meta_data or DEFAULT_META_DATA)
+            (mount_point / "network-config").write_text(network_config or DEFAULT_NETWORK_CONFIG)
+        finally:
+            subprocess.run(["umount", str(mount_point)], capture_output=True, timeout=10)
+        subprocess.run(["losetup", "-d", loop_dev], capture_output=True, timeout=10)
+        loop_dev = None
+        _build_status_write("finalizing", "Copying image to backups…")
+        shutil.copy2(str(img_path), str(out_path))
+        _build_status_write("done", f"Built {out_name}", output_name=out_name)
+    except subprocess.TimeoutExpired as e:
+        _build_status_write("error", "", output_name=None, error=f"Timeout: {e}")
+    except subprocess.CalledProcessError as e:
+        _build_status_write("error", "", output_name=None, error=f"Command failed: {e.stderr or e}")
+    except Exception as e:
+        _build_status_write("error", "", output_name=None, error=str(e))
+    finally:
+        if loop_dev:
+            try:
+                subprocess.run(["losetup", "-d", loop_dev], capture_output=True, timeout=5)
+            except Exception:
+                pass
+        if mount_point and mount_point.exists():
+            try:
+                subprocess.run(["umount", str(mount_point)], capture_output=True, timeout=5)
+            except Exception:
+                pass
+        if temp_dir and temp_dir.exists():
+            try:
+                shutil.rmtree(temp_dir, ignore_errors=True)
+            except Exception:
+                pass
+
+
+_build_lock = threading.Lock()
+_build_thread = None
+
+
+@app.route("/api/build-cloudinit-status")
+def api_build_cloudinit_status():
+    """Return current build status (phase, message, output_name, error)."""
+    return jsonify(_build_status_read())
+
+
+@app.route("/api/build-cloudinit", methods=["POST"])
+def api_build_cloudinit():
+    """Start building a cloud-init ready Raspberry Pi OS image (download latest Lite, inject NoCloud). Runs in background."""
+    global _build_thread
+    with _build_lock:
+        st = _build_status_read()
+        if st.get("phase") in ("downloading", "decompressing", "injecting", "finalizing", "resolving"):
+            return jsonify({"ok": False, "error": "A build is already in progress"}), 409
+        _build_thread = threading.Thread(
+            target=_build_cloudinit_worker,
+            kwargs={
+                "variant": "lite",
+                "user_data": (request.get_json(silent=True) or {}).get("user_data") or DEFAULT_USER_DATA,
+                "meta_data": (request.get_json(silent=True) or {}).get("meta_data") or DEFAULT_META_DATA,
+                "network_config": (request.get_json(silent=True) or {}).get("network_config") or DEFAULT_NETWORK_CONFIG,
+            },
+            daemon=True,
+        )
+        _build_thread.start()
+    _build_status_write("resolving", "Starting…")
+    return jsonify({"ok": True, "message": "Build started. Download and inject may take 15–30 min. Poll /api/build-cloudinit-status or refresh the page."}), 202
+
+
+@app.route("/api/raspios-latest-url")
+def api_raspios_latest_url():
+    """Return the URL of the latest Raspberry Pi OS Lite arm64 image (for display only)."""
+    url = _raspios_latest_lite_url()
+    if not url:
+        return jsonify({"ok": False, "url": None, "error": "Could not resolve latest image URL"}), 503
+    return jsonify({"ok": True, "url": url, "filename": url.split("/")[-1]})
+
+
 @app.route("/api/golden-info")
 def api_golden_info():
     """Return whether golden image exists and its size/mtime for UI."""

chromium-setup/emmc-provisioning/dashboard/templates/index.html

@@ -416,6 +416,28 @@
       <p id="backupsEmpty" class="empty-msg" style="display:none;">No backups yet. Capture one from a device (Backup above), then set it as golden for future deploys.</p>
     </section>
 
+    <!-- 3b. Build cloud-init image from official Raspberry Pi OS -->
+    <section class="section">
+      <h2 class="section-title">Build cloud-init image</h2>
+      <p class="backup-deploy-hint">Download the latest <strong>Raspberry Pi OS Lite (arm64)</strong> from the official repository and inject cloud-init NoCloud files so the image is ready for first-boot configuration.</p>
+      <p id="buildRaspiosUrl" class="backups-mono" style="font-size:0.8rem; margin-bottom:0.5rem;"></p>
+      <div style="margin-bottom:0.75rem;">
+        <button type="button" id="buildCloudInitBtn" class="btn btn-primary">Download latest Raspberry Pi OS Lite &amp; build cloud-init image</button>
+      </div>
+      <div id="buildCloudInitStatus" class="backups-mono" style="font-size:0.85rem; min-height:1.5em; margin-bottom:0.5rem;"></div>
+      <details style="margin-top:0.5rem;">
+        <summary>Customize cloud-init (optional)</summary>
+        <div class="inner" style="margin-top:0.5rem;">
+          <label>user-data (YAML)</label>
+          <textarea id="buildUserData" rows="8" style="width:100%; font-family:monospace; font-size:0.85rem; margin-bottom:0.5rem;" placeholder="Leave empty to use default (remote bootstrap example)"></textarea>
+          <label>meta-data (optional)</label>
+          <textarea id="buildMetaData" rows="3" style="width:100%; font-family:monospace; font-size:0.85rem; margin-bottom:0.5rem;"></textarea>
+          <label>network-config (optional)</label>
+          <textarea id="buildNetworkConfig" rows="5" style="width:100%; font-family:monospace; font-size:0.85rem;"></textarea>
+        </div>
+      </details>
+    </section>
+
     <!-- 4. How to connect (collapsible) -->
     <details class="section" style="padding:0;">
       <summary>How to connect</summary>
@@ -568,6 +590,7 @@
         const desc = (b.description || '');
         const isRawImg = b.name.endsWith('.img') && !b.name.endsWith('.img.gz') && !b.name.endsWith('.img.xz');
         const shrinkBtn = isRawImg ? '<button type="button" class="btn btn-outline btn-sm shrink-btn" data-name="' + escapeHtml(b.name) + '" title="Shrink image (PiShrink)">Shrink</button> ' : '';
+        const compressBtn = isRawImg ? '<button type="button" class="btn btn-outline btn-sm compress-btn" data-name="' + escapeHtml(b.name) + '" data-format="xz" title="Shrink and compress to .img.xz (minimum size)">Compress</button> ' : '';
         tr.innerHTML =
           '<td class="backup-name-edit" data-field="name" title="Click to rename">' + escapeHtml(displayName) + '</td>' +
           '<td class="backup-desc-edit" data-field="description" title="Click to add description">' + escapeHtml(desc) + '</td>' +
@@ -575,6 +598,7 @@
           '<td class="backups-mono">' + fmtDate(b.mtime) + '</td>' +
           '<td class="actions-cell">' +
             shrinkBtn +
+            compressBtn +
             '<button type="button" class="btn btn-primary btn-sm set-golden-btn" data-name="' + escapeHtml(b.name) + '">Set as golden</button> ' +
             '<button type="button" class="btn btn-outline btn-sm rename-file-btn" data-name="' + escapeHtml(b.name) + '" title="Rename file">Rename file</button> ' +
             '<a href="/api/backups/' + encodeURIComponent(b.name) + '" download class="btn btn-outline btn-sm download-link">Download</a>' +
@@ -585,6 +609,7 @@
       bindSetGolden();
       bindRenameFile();
       bindShrink();
+      bindCompress();
     }
 
     function bindRenameFile() {
@@ -594,7 +619,7 @@
           const newName = prompt('New filename (e.g. production-v1.img)', name);
           if (newName == null || newName.trim() === '') return;
           const n = newName.trim();
-          if (!/\.(img|img\.gz)$/i.test(n)) { alert('Filename must end with .img or .img.gz'); return; }
+          if (!/\.(img|img\.gz|img\.xz)$/i.test(n)) { alert('Filename must end with .img, .img.gz or .img.xz'); return; }
           if (n === name) return;
           fetch('/api/backups/' + encodeURIComponent(name), { method: 'PATCH', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ filename: n }) })
             .then(function(r) { return r.json(); })
@@ -680,6 +705,26 @@
       });
     }
 
+    function bindCompress() {
+      document.querySelectorAll('.compress-btn').forEach(function(btn) {
+        btn.onclick = function() {
+          const name = btn.getAttribute('data-name');
+          const format = btn.getAttribute('data-format') || 'xz';
+          if (!confirm('Shrink and compress this image to .img.' + format + '? This minimizes size and may take several minutes.\n\n' + name)) return;
+          btn.disabled = true;
+          btn.textContent = 'Compressing…';
+          fetch('/api/backups/' + encodeURIComponent(name) + '/compress', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ format: format }) })
+            .then(function(r) { return r.json(); })
+            .then(function(data) {
+              if (data.ok) { fetchBackups(); }
+              else alert(data.error || data.detail || 'Failed');
+            })
+            .catch(function() { alert('Request failed'); })
+            .finally(function() { btn.disabled = false; btn.textContent = 'Compress'; });
+        };
+      });
+    }
+
     function fetchGoldenInfo() {
       fetch('/api/golden-info').then(function(r) { return r.json(); }).then(function(d) {
         const el = document.getElementById('goldenHint');
@@ -700,6 +745,58 @@
       fetch('/api/backups').then(function(r) { return r.json(); }).then(function(d) { renderBackups(d.backups || []); }).catch(function() {});
     }
 
+    function fetchRaspiosUrl() {
+      fetch('/api/raspios-latest-url').then(function(r) { return r.json(); }).then(function(d) {
+        var el = document.getElementById('buildRaspiosUrl');
+        if (el) el.textContent = d.ok && d.filename ? ('Latest: ' + d.filename) : (d.error || 'Could not resolve latest image URL');
+      }).catch(function() {});
+    }
+
+    function fetchBuildStatus() {
+      fetch('/api/build-cloudinit-status').then(function(r) { return r.json(); }).then(function(d) {
+        var el = document.getElementById('buildCloudInitStatus');
+        var btn = document.getElementById('buildCloudInitBtn');
+        if (!el) return;
+        var busy = ['resolving','downloading','decompressing','injecting','finalizing'].indexOf(d.phase) >= 0;
+        if (btn) btn.disabled = busy;
+        if (d.phase === 'idle' && !d.message) {
+          el.textContent = '';
+        } else if (d.phase === 'done') {
+          el.textContent = 'Done: ' + (d.output_name || '') + ' — refresh the backups list below.';
+          fetchBackups();
+        } else if (d.phase === 'error') {
+          el.textContent = 'Error: ' + (d.error || d.message || 'Unknown');
+        } else {
+          el.textContent = (d.phase || '') + ': ' + (d.message || '');
+        }
+        if (busy) setTimeout(fetchBuildStatus, 5000);
+      }).catch(function() {});
+    }
+
+    function startBuildCloudInit() {
+      var btn = document.getElementById('buildCloudInitBtn');
+      if (btn) btn.disabled = true;
+      var body = {};
+      var ud = document.getElementById('buildUserData');
+      var md = document.getElementById('buildMetaData');
+      var nc = document.getElementById('buildNetworkConfig');
+      if (ud && ud.value.trim()) body.user_data = ud.value.trim();
+      if (md && md.value.trim()) body.meta_data = md.value.trim();
+      if (nc && nc.value.trim()) body.network_config = nc.value.trim();
+      fetch('/api/build-cloudinit', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body) })
+        .then(function(r) { return r.json(); })
+        .then(function(data) {
+          if (data.ok) {
+            document.getElementById('buildCloudInitStatus').textContent = 'Build started. Waiting…';
+            setTimeout(fetchBuildStatus, 2000);
+          } else {
+            alert(data.error || 'Failed to start build');
+            if (btn) btn.disabled = false;
+          }
+        })
+        .catch(function() { alert('Request failed'); if (btn) btn.disabled = false; });
+    }
+
     function fmtSize(n) {
       if (n >= 1e9) return (n / 1e9).toFixed(1) + ' GB';
       if (n >= 1e6) return (n / 1e6).toFixed(1) + ' MB';
@@ -721,10 +818,15 @@
     fetchPending();
     fetchBackups();
     fetchGoldenInfo();
+    fetchRaspiosUrl();
+    fetchBuildStatus();
+    var buildBtn = document.getElementById('buildCloudInitBtn');
+    if (buildBtn) buildBtn.onclick = startBuildCloudInit;
     setInterval(fetchStatus, 2000);
     setInterval(fetchLog, 4000);
     setInterval(fetchPending, 2000);
     setInterval(fetchBackups, 5000);
+    setInterval(fetchBuildStatus, 15000);
     setInterval(fetchGoldenInfo, 10000);
   </script>
 </body>