Enhance eMMC provisioning dashboard: implement cloud-init template management with options to load, save, and delete templates. Update UI to support selecting Raspberry Pi OS variant for image building and improve user instructions for cloud-init image creation. Add new API endpoints for managing cloud-init templates and fetching the latest Raspberry Pi OS URLs.

chromium-setup/emmc-provisioning/dashboard/app.py

@@ -10,8 +10,6 @@ import os
 import re
 import shutil
 import subprocess
-import tempfile
-import threading
 import time
 import urllib.request
 from pathlib import Path
@@ -39,6 +37,8 @@ BACKUPS_DIR = Path(os.environ.get("CM4_BACKUPS_DIR", str(BASE_DIR / "backups")))
 GOLDEN_IMAGE = Path(os.environ.get("CM4_GOLDEN_IMAGE", str(BASE_DIR / "golden.img")))
 NETWORK_DEVICES_FILE = Path(os.environ.get("CM4_NETWORK_DEVICES_FILE", str(BASE_DIR / "network_devices.json")))
 BUILD_STATUS_FILE = Path(os.environ.get("CM4_BUILD_STATUS_FILE", str(BASE_DIR / "build_cloudinit_status.json")))
+BUILD_REQUEST_FILE = Path(os.environ.get("CM4_BUILD_REQUEST_FILE", str(BASE_DIR / "build_cloudinit_request.json")))
+CLOUDINIT_TEMPLATES_FILE = Path(os.environ.get("CM4_CLOUDINIT_TEMPLATES_FILE", str(BASE_DIR / "cloudinit_templates.json")))
 
 # Default cloud-init user-data for Raspberry Pi OS (NoCloud on boot partition)
 DEFAULT_USER_DATA = """#cloud-config
@@ -500,21 +500,20 @@ def _build_status_write(phase, message, output_name=None, error=None):
         pass
 
 
-def _raspios_latest_lite_url():
-    """Resolve latest Raspberry Pi OS Lite (arm64) .img.xz URL from official index."""
-    base = "https://downloads.raspberrypi.com/raspios_lite_arm64/images"
+def _raspios_latest_url(variant="lite"):
+    """Resolve latest Raspberry Pi OS (arm64) .img.xz URL. variant=lite|full."""
+    slug = "raspios_lite_arm64" if variant == "lite" else "raspios_full_arm64"
+    base = f"https://downloads.raspberrypi.com/{slug}/images"
     try:
         with urllib.request.urlopen(base + "/", timeout=15) as r:
             html = r.read().decode("utf-8", errors="ignore")
-        # Match raspios_lite_arm64-YYYY-MM-DD/
-        folders = re.findall(r"raspios_lite_arm64-(\d{4}-\d{2}-\d{2})/", html)
+        folders = re.findall(rf"{re.escape(slug)}-(\d{{4}}-\d{{2}}-\d{{2}})/", html)
         if not folders:
             return None
         latest = sorted(folders)[-1]
-        folder_url = f"{base}/raspios_lite_arm64-{latest}/"
+        folder_url = f"{base}/{slug}-{latest}/"
         with urllib.request.urlopen(folder_url, timeout=15) as r:
             folder_html = r.read().decode("utf-8", errors="ignore")
-        # Match .img.xz link (skip .sha256, .torrent, etc.)
         m = re.search(r'href="([^"]+\.img\.xz)"', folder_html)
         if not m:
             return None
@@ -523,96 +522,24 @@ def _raspios_latest_lite_url():
         return None
 
 
-def _build_cloudinit_worker(variant, user_data, meta_data, network_config):
-    """Background: download Raspios, inject cloud-init, save to BACKUPS_DIR."""
-    out_name = f"raspios-{variant}-cloudinit-{time.strftime('%Y%m%d-%H%M%S')}.img"
-    out_path = BACKUPS_DIR / out_name
-    mount_point = None
-    loop_dev = None
-    temp_dir = None
-    img_path = None
-    xz_path = None
+def _load_cloudinit_templates():
     try:
-        _build_status_write("resolving", "Resolving latest Raspberry Pi OS image URL…")
-        url = _raspios_latest_lite_url()
-        if not url:
-            _build_status_write("error", "", output_name=None, error="Could not resolve latest Raspios Lite URL")
-            return
-        _build_status_write("downloading", f"Downloading {url.split('/')[-1]}… (this may take several minutes)")
-        temp_dir = Path(tempfile.mkdtemp(prefix="cloudinit-build-", dir=str(BACKUPS_DIR)))
-        xz_path = temp_dir / "image.img.xz"
-        with urllib.request.urlopen(url, timeout=3600) as resp:
-            total = int(resp.headers.get("Content-Length", 0)) or 0
-            chunk_size = 1024 * 1024
-            with open(xz_path, "wb") as f:
-                done = 0
-                while True:
-                    chunk = resp.read(chunk_size)
-                    if not chunk:
-                        break
-                    f.write(chunk)
-                    done += len(chunk)
-                    if total and done % (50 * 1024 * 1024) < chunk_size:
-                        _build_status_write("downloading", f"Downloaded {done // (1024*1024)} MB…")
-        _build_status_write("decompressing", "Decompressing image…")
-        img_path = temp_dir / "image.img"
-        subprocess.run(["xz", "-d", "-k", "-f", str(xz_path)], check=True, capture_output=True, timeout=1800, cwd=str(temp_dir))
-        if not img_path.exists():
-            _build_status_write("error", "", output_name=None, error="Decompress failed: image.img not found")
-            return
-        _build_status_write("injecting", "Mounting boot partition and injecting cloud-init…")
-        loop_out = subprocess.run(["losetup", "-f", "--show", "-P", str(img_path)], capture_output=True, text=True, timeout=10)
-        if loop_out.returncode != 0:
-            _build_status_write("error", "", output_name=None, error="losetup failed (loop device may not be available in this container)")
-            return
-        loop_dev = loop_out.stdout.strip()
-        # First partition is usually boot (FAT32)
-        boot_part = f"{loop_dev}p1"
-        if not Path(boot_part).exists():
-            boot_part = f"{loop_dev}p2"
-        if not Path(boot_part).exists():
-            _build_status_write("error", "", output_name=None, error="Boot partition not found")
-            return
-        mount_point = temp_dir / "mnt"
-        mount_point.mkdir(exist_ok=True)
-        subprocess.run(["mount", boot_part, str(mount_point)], check=True, capture_output=True, timeout=10)
-        try:
-            (mount_point / "user-data").write_text(user_data or DEFAULT_USER_DATA)
-            (mount_point / "meta-data").write_text(meta_data or DEFAULT_META_DATA)
-            (mount_point / "network-config").write_text(network_config or DEFAULT_NETWORK_CONFIG)
-        finally:
-            subprocess.run(["umount", str(mount_point)], capture_output=True, timeout=10)
-        subprocess.run(["losetup", "-d", loop_dev], capture_output=True, timeout=10)
-        loop_dev = None
-        _build_status_write("finalizing", "Copying image to backups…")
-        shutil.copy2(str(img_path), str(out_path))
-        _build_status_write("done", f"Built {out_name}", output_name=out_name)
-    except subprocess.TimeoutExpired as e:
-        _build_status_write("error", "", output_name=None, error=f"Timeout: {e}")
-    except subprocess.CalledProcessError as e:
-        _build_status_write("error", "", output_name=None, error=f"Command failed: {e.stderr or e}")
-    except Exception as e:
-        _build_status_write("error", "", output_name=None, error=str(e))
-    finally:
-        if loop_dev:
-            try:
-                subprocess.run(["losetup", "-d", loop_dev], capture_output=True, timeout=5)
-            except Exception:
-                pass
-        if mount_point and mount_point.exists():
-            try:
-                subprocess.run(["umount", str(mount_point)], capture_output=True, timeout=5)
-            except Exception:
-                pass
-        if temp_dir and temp_dir.exists():
-            try:
-                shutil.rmtree(temp_dir, ignore_errors=True)
-            except Exception:
-                pass
+        if CLOUDINIT_TEMPLATES_FILE.is_file():
+            with open(CLOUDINIT_TEMPLATES_FILE, "r") as f:
+                return json.load(f)
+    except (json.JSONDecodeError, OSError):
+        pass
+    return {"templates": []}
 
 
-_build_lock = threading.Lock()
-_build_thread = None
+def _save_cloudinit_templates(data):
+    try:
+        CLOUDINIT_TEMPLATES_FILE.parent.mkdir(parents=True, exist_ok=True)
+        with open(CLOUDINIT_TEMPLATES_FILE, "w") as f:
+            json.dump(data, f, indent=2)
+        return True
+    except OSError:
+        return False
 
 
 @app.route("/api/build-cloudinit-status")
@@ -623,34 +550,103 @@ def api_build_cloudinit_status():
 
 @app.route("/api/build-cloudinit", methods=["POST"])
 def api_build_cloudinit():
-    """Start building a cloud-init ready Raspberry Pi OS image (download latest Lite, inject NoCloud). Runs in background."""
-    global _build_thread
-    with _build_lock:
-        st = _build_status_read()
-        if st.get("phase") in ("downloading", "decompressing", "injecting", "finalizing", "resolving"):
-            return jsonify({"ok": False, "error": "A build is already in progress"}), 409
-        _build_thread = threading.Thread(
-            target=_build_cloudinit_worker,
-            kwargs={
-                "variant": "lite",
-                "user_data": (request.get_json(silent=True) or {}).get("user_data") or DEFAULT_USER_DATA,
-                "meta_data": (request.get_json(silent=True) or {}).get("meta_data") or DEFAULT_META_DATA,
-                "network_config": (request.get_json(silent=True) or {}).get("network_config") or DEFAULT_NETWORK_CONFIG,
-            },
-            daemon=True,
-        )
-        _build_thread.start()
-    _build_status_write("resolving", "Starting…")
-    return jsonify({"ok": True, "message": "Build started. Download and inject may take 15–30 min. Poll /api/build-cloudinit-status or refresh the page."}), 202
+    """Start building a cloud-init image: write request file; host runs build (has loop devices)."""
+    st = _build_status_read()
+    if st.get("phase") in ("downloading", "decompressing", "injecting", "finalizing", "resolving"):
+        return jsonify({"ok": False, "error": "A build is already in progress"}), 409
+    body = request.get_json(silent=True) or {}
+    variant = (body.get("variant") or "lite").strip().lower()
+    if variant not in ("lite", "full"):
+        variant = "lite"
+    _build_status_write("resolving", "Resolving latest Raspberry Pi OS image URL…")
+    url = _raspios_latest_url(variant)
+    if not url:
+        _build_status_write("idle", "", error="Could not resolve latest Raspios URL")
+        return jsonify({"ok": False, "error": "Could not resolve latest image URL"}), 503
+    user_data = body.get("user_data") or DEFAULT_USER_DATA
+    meta_data = body.get("meta_data") or DEFAULT_META_DATA
+    network_config = body.get("network_config") or DEFAULT_NETWORK_CONFIG
+    set_as_golden_after = bool(body.get("set_as_golden_after"))
+    try:
+        BUILD_REQUEST_FILE.parent.mkdir(parents=True, exist_ok=True)
+        with open(BUILD_REQUEST_FILE, "w") as f:
+            json.dump({
+                "url": url,
+                "variant": variant,
+                "user_data": user_data,
+                "meta_data": meta_data,
+                "network_config": network_config,
+                "set_as_golden_after": set_as_golden_after,
+            }, f, indent=2)
+    except OSError as e:
+        return jsonify({"ok": False, "error": str(e)}), 500
+    _build_status_write("resolving", "Build requested; host will run download and inject (check status).")
+    return jsonify({"ok": True, "message": "Build started on host. Download and inject may take 15–45 min. Poll status or refresh the page."}), 202
 
 
 @app.route("/api/raspios-latest-url")
 def api_raspios_latest_url():
-    """Return the URL of the latest Raspberry Pi OS Lite arm64 image (for display only)."""
-    url = _raspios_latest_lite_url()
+    """Return the URL of the latest Raspberry Pi OS (arm64) image. Query: variant=lite|full."""
+    variant = (request.args.get("variant") or "lite").strip().lower()
+    if variant not in ("lite", "full"):
+        variant = "lite"
+    url = _raspios_latest_url(variant)
     if not url:
         return jsonify({"ok": False, "url": None, "error": "Could not resolve latest image URL"}), 503
-    return jsonify({"ok": True, "url": url, "filename": url.split("/")[-1]})
+    return jsonify({"ok": True, "url": url, "filename": url.split("/")[-1], "variant": variant})
+
+
+@app.route("/api/cloudinit-templates", methods=["GET"])
+def api_cloudinit_templates_list():
+    """List saved cloud-init templates."""
+    data = _load_cloudinit_templates()
+    return jsonify({"templates": data.get("templates", [])})
+
+
+@app.route("/api/cloudinit-templates", methods=["POST"])
+def api_cloudinit_templates_create():
+    """Save a new cloud-init template."""
+    body = request.get_json(force=True, silent=True) or {}
+    name = (body.get("name") or "").strip()
+    if not name:
+        return jsonify({"ok": False, "error": "name required"}), 400
+    data = _load_cloudinit_templates()
+    templates = data.get("templates", [])
+    tid = str(int(time.time() * 1000))
+    templates.append({
+        "id": tid,
+        "name": name,
+        "user_data": body.get("user_data", ""),
+        "meta_data": body.get("meta_data", ""),
+        "network_config": body.get("network_config", ""),
+    })
+    data["templates"] = templates
+    if not _save_cloudinit_templates(data):
+        return jsonify({"ok": False, "error": "Failed to save"}), 500
+    return jsonify({"ok": True, "id": tid, "name": name})
+
+
+@app.route("/api/cloudinit-templates/<tid>")
+def api_cloudinit_templates_get(tid):
+    """Get one template by id."""
+    data = _load_cloudinit_templates()
+    for t in data.get("templates", []):
+        if t.get("id") == tid:
+            return jsonify(t)
+    return jsonify({"error": "not found"}), 404
+
+
+@app.route("/api/cloudinit-templates/<tid>", methods=["DELETE"])
+def api_cloudinit_templates_delete(tid):
+    """Delete a template."""
+    data = _load_cloudinit_templates()
+    templates = [t for t in data.get("templates", []) if t.get("id") != tid]
+    if len(templates) == len(data.get("templates", [])):
+        return jsonify({"ok": False, "error": "not found"}), 404
+    data["templates"] = templates
+    if not _save_cloudinit_templates(data):
+        return jsonify({"ok": False, "error": "Failed to save"}), 500
+    return jsonify({"ok": True})
 
 
 @app.route("/api/golden-info")

chromium-setup/emmc-provisioning/dashboard/templates/index.html

@@ -419,15 +419,30 @@
     <!-- 3b. Build cloud-init image from official Raspberry Pi OS -->
     <section class="section">
       <h2 class="section-title">Build cloud-init image</h2>
-      <p class="backup-deploy-hint">Download the latest <strong>Raspberry Pi OS Lite (arm64)</strong> from the official repository and inject cloud-init NoCloud files so the image is ready for first-boot configuration.</p>
-      <p id="buildRaspiosUrl" class="backups-mono" style="font-size:0.8rem; margin-bottom:0.5rem;"></p>
+      <p class="backup-deploy-hint">Download the latest <strong>Raspberry Pi OS (arm64)</strong> from the official repository and inject cloud-init NoCloud files. The built image appears in Saved backups; you can then choose it and click <strong>Set as golden</strong> for deployment.</p>
+      <div style="margin-bottom:0.5rem;">
+        <label>Variant: </label>
+        <select id="buildVariant">
+          <option value="lite">Lite (no desktop)</option>
+          <option value="full">Full (desktop)</option>
+        </select>
+        <span id="buildRaspiosUrl" class="backups-mono" style="font-size:0.8rem; margin-left:0.5rem;"></span>
+      </div>
+      <div style="margin-bottom:0.5rem;">
+        <label><input type="checkbox" id="buildSetGolden" /> Set as golden image after build</label>
+        <span class="backups-mono" style="font-size:0.8rem;"> (use for Deploy without clicking manually)</span>
+      </div>
       <div style="margin-bottom:0.75rem;">
-        <button type="button" id="buildCloudInitBtn" class="btn btn-primary">Download latest Raspberry Pi OS Lite &amp; build cloud-init image</button>
+        <button type="button" id="buildCloudInitBtn" class="btn btn-primary">Download &amp; build cloud-init image</button>
       </div>
       <div id="buildCloudInitStatus" class="backups-mono" style="font-size:0.85rem; min-height:1.5em; margin-bottom:0.5rem;"></div>
       <details style="margin-top:0.5rem;">
-        <summary>Customize cloud-init (optional)</summary>
+        <summary>Cloud-init templates &amp; customize</summary>
         <div class="inner" style="margin-top:0.5rem;">
+          <p><strong>Templates:</strong> <select id="buildTemplateSelect"><option value="">— Load a template —</option></select>
+          <button type="button" id="buildTemplateLoad" class="btn btn-outline btn-sm">Load</button>
+          <button type="button" id="buildTemplateSave" class="btn btn-outline btn-sm">Save current as template…</button></p>
+          <ul id="buildTemplateList" class="backups-mono" style="font-size:0.85rem; list-style:none; padding:0;"></ul>
           <label>user-data (YAML)</label>
           <textarea id="buildUserData" rows="8" style="width:100%; font-family:monospace; font-size:0.85rem; margin-bottom:0.5rem;" placeholder="Leave empty to use default (remote bootstrap example)"></textarea>
           <label>meta-data (optional)</label>
@@ -745,12 +760,74 @@
       fetch('/api/backups').then(function(r) { return r.json(); }).then(function(d) { renderBackups(d.backups || []); }).catch(function() {});
     }
 
+    function getBuildVariant() {
+      var sel = document.getElementById('buildVariant');
+      return (sel && sel.value) ? sel.value : 'lite';
+    }
     function fetchRaspiosUrl() {
-      fetch('/api/raspios-latest-url').then(function(r) { return r.json(); }).then(function(d) {
+      var variant = getBuildVariant();
+      fetch('/api/raspios-latest-url?variant=' + encodeURIComponent(variant)).then(function(r) { return r.json(); }).then(function(d) {
         var el = document.getElementById('buildRaspiosUrl');
         if (el) el.textContent = d.ok && d.filename ? ('Latest: ' + d.filename) : (d.error || 'Could not resolve latest image URL');
       }).catch(function() {});
     }
+    function fetchCloudInitTemplates() {
+      fetch('/api/cloudinit-templates').then(function(r) { return r.json(); }).then(function(d) {
+        var list = d.templates || [];
+        var sel = document.getElementById('buildTemplateSelect');
+        var listEl = document.getElementById('buildTemplateList');
+        if (sel) {
+          sel.innerHTML = '<option value="">— Load a template —</option>';
+          list.forEach(function(t) {
+            var opt = document.createElement('option');
+            opt.value = t.id;
+            opt.textContent = t.name;
+            sel.appendChild(opt);
+          });
+        }
+        if (listEl) {
+          listEl.innerHTML = list.map(function(t) {
+            return '<li><span>' + escapeHtml(t.name) + '</span> <button type="button" class="btn btn-outline btn-sm template-load-btn" data-id="' + escapeHtml(t.id) + '">Load</button> <button type="button" class="btn btn-outline btn-sm template-del-btn" data-id="' + escapeHtml(t.id) + '">Delete</button></li>';
+          }).join('') || '<li>No templates saved.</li>';
+          listEl.querySelectorAll('.template-load-btn').forEach(function(btn) {
+            btn.onclick = function() { loadTemplate(btn.getAttribute('data-id')); };
+          });
+          listEl.querySelectorAll('.template-del-btn').forEach(function(btn) {
+            btn.onclick = function() {
+              if (!confirm('Delete this template?')) return;
+              fetch('/api/cloudinit-templates/' + encodeURIComponent(btn.getAttribute('data-id')), { method: 'DELETE' })
+                .then(function(r) { return r.json(); }).then(function(data) { if (data.ok) fetchCloudInitTemplates(); });
+            };
+          });
+        }
+      }).catch(function() {});
+    }
+    function loadTemplate(id) {
+      fetch('/api/cloudinit-templates/' + encodeURIComponent(id)).then(function(r) { return r.json(); }).then(function(t) {
+        var ud = document.getElementById('buildUserData');
+        var md = document.getElementById('buildMetaData');
+        var nc = document.getElementById('buildNetworkConfig');
+        if (ud) ud.value = t.user_data || '';
+        if (md) md.value = t.meta_data || '';
+        if (nc) nc.value = t.network_config || '';
+      }).catch(function() {});
+    }
+    function saveTemplate() {
+      var name = prompt('Template name');
+      if (!name || !name.trim()) return;
+      var ud = document.getElementById('buildUserData');
+      var md = document.getElementById('buildMetaData');
+      var nc = document.getElementById('buildNetworkConfig');
+      fetch('/api/cloudinit-templates', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({
+        name: name.trim(),
+        user_data: ud ? ud.value : '',
+        meta_data: md ? md.value : '',
+        network_config: nc ? nc.value : ''
+      })}).then(function(r) { return r.json(); }).then(function(d) {
+        if (d.ok) { fetchCloudInitTemplates(); alert('Saved as "' + name + '"'); }
+        else alert(d.error || 'Failed');
+      }).catch(function() { alert('Failed'); });
+    }
 
     function fetchBuildStatus() {
       fetch('/api/build-cloudinit-status').then(function(r) { return r.json(); }).then(function(d) {
@@ -762,8 +839,9 @@
         if (d.phase === 'idle' && !d.message) {
           el.textContent = '';
         } else if (d.phase === 'done') {
-          el.textContent = 'Done: ' + (d.output_name || '') + ' — refresh the backups list below.';
+          el.textContent = 'Done: ' + (d.output_name || '') + ' — see Saved backups below. Set as golden to use for Deploy.';
           fetchBackups();
+          fetchGoldenInfo();
         } else if (d.phase === 'error') {
           el.textContent = 'Error: ' + (d.error || d.message || 'Unknown');
         } else {
@@ -776,18 +854,22 @@
     function startBuildCloudInit() {
       var btn = document.getElementById('buildCloudInitBtn');
       if (btn) btn.disabled = true;
-      var body = {};
       var ud = document.getElementById('buildUserData');
       var md = document.getElementById('buildMetaData');
       var nc = document.getElementById('buildNetworkConfig');
-      if (ud && ud.value.trim()) body.user_data = ud.value.trim();
-      if (md && md.value.trim()) body.meta_data = md.value.trim();
-      if (nc && nc.value.trim()) body.network_config = nc.value.trim();
+      var setGolden = document.getElementById('buildSetGolden');
+      var body = {
+        variant: getBuildVariant(),
+        set_as_golden_after: setGolden && setGolden.checked,
+        user_data: (ud && ud.value.trim()) ? ud.value.trim() : undefined,
+        meta_data: (md && md.value.trim()) ? md.value.trim() : undefined,
+        network_config: (nc && nc.value.trim()) ? nc.value.trim() : undefined
+      };
       fetch('/api/build-cloudinit', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body) })
         .then(function(r) { return r.json(); })
         .then(function(data) {
           if (data.ok) {
-            document.getElementById('buildCloudInitStatus').textContent = 'Build started. Waiting…';
+            document.getElementById('buildCloudInitStatus').textContent = 'Build started on host. Waiting…';
             setTimeout(fetchBuildStatus, 2000);
           } else {
             alert(data.error || 'Failed to start build');
@@ -820,8 +902,15 @@
     fetchGoldenInfo();
     fetchRaspiosUrl();
     fetchBuildStatus();
+    fetchCloudInitTemplates();
     var buildBtn = document.getElementById('buildCloudInitBtn');
     if (buildBtn) buildBtn.onclick = startBuildCloudInit;
+    var variantSel = document.getElementById('buildVariant');
+    if (variantSel) variantSel.onchange = fetchRaspiosUrl;
+    var templateLoadBtn = document.getElementById('buildTemplateLoad');
+    if (templateLoadBtn) templateLoadBtn.onclick = function() { var s = document.getElementById('buildTemplateSelect'); if (s && s.value) loadTemplate(s.value); };
+    var templateSaveBtn = document.getElementById('buildTemplateSave');
+    if (templateSaveBtn) templateSaveBtn.onclick = saveTemplate;
     setInterval(fetchStatus, 2000);
     setInterval(fetchLog, 4000);
     setInterval(fetchPending, 2000);