Enhance eMMC provisioning dashboard: implement cloud-init template management with options to load, save, and delete templates. Update UI to support selecting Raspberry Pi OS variant for image building and improve user instructions for cloud-init image creation. Add new API endpoints for managing cloud-init templates and fetching the latest Raspberry Pi OS URLs.

2026-02-18 23:06:36 +02:00
parent 40b8e15e75
commit 262bc3e515
6 changed files with 332 additions and 129 deletions
--- a/chromium-setup/emmc-provisioning/dashboard/app.py
+++ b/chromium-setup/emmc-provisioning/dashboard/app.py
@@ -10,8 +10,6 @@ import os
 import re
 import shutil
 import subprocess
-import tempfile
-import threading
 import time
 import urllib.request
 from pathlib import Path
@@ -39,6 +37,8 @@ BACKUPS_DIR = Path(os.environ.get("CM4_BACKUPS_DIR", str(BASE_DIR / "backups")))
 GOLDEN_IMAGE = Path(os.environ.get("CM4_GOLDEN_IMAGE", str(BASE_DIR / "golden.img")))
 NETWORK_DEVICES_FILE = Path(os.environ.get("CM4_NETWORK_DEVICES_FILE", str(BASE_DIR / "network_devices.json")))
 BUILD_STATUS_FILE = Path(os.environ.get("CM4_BUILD_STATUS_FILE", str(BASE_DIR / "build_cloudinit_status.json")))
+BUILD_REQUEST_FILE = Path(os.environ.get("CM4_BUILD_REQUEST_FILE", str(BASE_DIR / "build_cloudinit_request.json")))
+CLOUDINIT_TEMPLATES_FILE = Path(os.environ.get("CM4_CLOUDINIT_TEMPLATES_FILE", str(BASE_DIR / "cloudinit_templates.json")))

 # Default cloud-init user-data for Raspberry Pi OS (NoCloud on boot partition)
 DEFAULT_USER_DATA = """#cloud-config
@@ -500,21 +500,20 @@ def _build_status_write(phase, message, output_name=None, error=None):
        pass


-def _raspios_latest_lite_url():
-    """Resolve latest Raspberry Pi OS Lite (arm64) .img.xz URL from official index."""
-    base = "https://downloads.raspberrypi.com/raspios_lite_arm64/images"
+def _raspios_latest_url(variant="lite"):
+    """Resolve latest Raspberry Pi OS (arm64) .img.xz URL. variant=lite|full."""
+    slug = "raspios_lite_arm64" if variant == "lite" else "raspios_full_arm64"
+    base = f"https://downloads.raspberrypi.com/{slug}/images"
    try:
        with urllib.request.urlopen(base + "/", timeout=15) as r:
            html = r.read().decode("utf-8", errors="ignore")
-        # Match raspios_lite_arm64-YYYY-MM-DD/
-        folders = re.findall(r"raspios_lite_arm64-(\d{4}-\d{2}-\d{2})/", html)
+        folders = re.findall(rf"{re.escape(slug)}-(\d{{4}}-\d{{2}}-\d{{2}})/", html)
        if not folders:
            return None
        latest = sorted(folders)[-1]
-        folder_url = f"{base}/raspios_lite_arm64-{latest}/"
+        folder_url = f"{base}/{slug}-{latest}/"
        with urllib.request.urlopen(folder_url, timeout=15) as r:
            folder_html = r.read().decode("utf-8", errors="ignore")
-        # Match .img.xz link (skip .sha256, .torrent, etc.)
        m = re.search(r'href="([^"]+\.img\.xz)"', folder_html)
        if not m:
            return None
@@ -523,96 +522,24 @@ def _raspios_latest_lite_url():
        return None


-def _build_cloudinit_worker(variant, user_data, meta_data, network_config):
-    """Background: download Raspios, inject cloud-init, save to BACKUPS_DIR."""
-    out_name = f"raspios-{variant}-cloudinit-{time.strftime('%Y%m%d-%H%M%S')}.img"
-    out_path = BACKUPS_DIR / out_name
-    mount_point = None
-    loop_dev = None
-    temp_dir = None
-    img_path = None
-    xz_path = None
+def _load_cloudinit_templates():
    try:
-        _build_status_write("resolving", "Resolving latest Raspberry Pi OS image URL…")
-        url = _raspios_latest_lite_url()
-        if not url:
-            _build_status_write("error", "", output_name=None, error="Could not resolve latest Raspios Lite URL")
-            return
-        _build_status_write("downloading", f"Downloading {url.split('/')[-1]}… (this may take several minutes)")
-        temp_dir = Path(tempfile.mkdtemp(prefix="cloudinit-build-", dir=str(BACKUPS_DIR)))
-        xz_path = temp_dir / "image.img.xz"
-        with urllib.request.urlopen(url, timeout=3600) as resp:
-            total = int(resp.headers.get("Content-Length", 0)) or 0
-            chunk_size = 1024 * 1024
-            with open(xz_path, "wb") as f:
-                done = 0
-                while True:
-                    chunk = resp.read(chunk_size)
-                    if not chunk:
-                        break
-                    f.write(chunk)
-                    done += len(chunk)
-                    if total and done % (50 * 1024 * 1024) < chunk_size:
-                        _build_status_write("downloading", f"Downloaded {done // (1024*1024)} MB…")
-        _build_status_write("decompressing", "Decompressing image…")
-        img_path = temp_dir / "image.img"
-        subprocess.run(["xz", "-d", "-k", "-f", str(xz_path)], check=True, capture_output=True, timeout=1800, cwd=str(temp_dir))
-        if not img_path.exists():
-            _build_status_write("error", "", output_name=None, error="Decompress failed: image.img not found")
-            return
-        _build_status_write("injecting", "Mounting boot partition and injecting cloud-init…")
-        loop_out = subprocess.run(["losetup", "-f", "--show", "-P", str(img_path)], capture_output=True, text=True, timeout=10)
-        if loop_out.returncode != 0:
-            _build_status_write("error", "", output_name=None, error="losetup failed (loop device may not be available in this container)")
-            return
-        loop_dev = loop_out.stdout.strip()
-        # First partition is usually boot (FAT32)
-        boot_part = f"{loop_dev}p1"
-        if not Path(boot_part).exists():
-            boot_part = f"{loop_dev}p2"
-        if not Path(boot_part).exists():
-            _build_status_write("error", "", output_name=None, error="Boot partition not found")
-            return
-        mount_point = temp_dir / "mnt"
-        mount_point.mkdir(exist_ok=True)
-        subprocess.run(["mount", boot_part, str(mount_point)], check=True, capture_output=True, timeout=10)
-        try:
-            (mount_point / "user-data").write_text(user_data or DEFAULT_USER_DATA)
-            (mount_point / "meta-data").write_text(meta_data or DEFAULT_META_DATA)
-            (mount_point / "network-config").write_text(network_config or DEFAULT_NETWORK_CONFIG)
-        finally:
-            subprocess.run(["umount", str(mount_point)], capture_output=True, timeout=10)
-        subprocess.run(["losetup", "-d", loop_dev], capture_output=True, timeout=10)
-        loop_dev = None
-        _build_status_write("finalizing", "Copying image to backups…")
-        shutil.copy2(str(img_path), str(out_path))
-        _build_status_write("done", f"Built {out_name}", output_name=out_name)
-    except subprocess.TimeoutExpired as e:
-        _build_status_write("error", "", output_name=None, error=f"Timeout: {e}")
-    except subprocess.CalledProcessError as e:
-        _build_status_write("error", "", output_name=None, error=f"Command failed: {e.stderr or e}")
-    except Exception as e:
-        _build_status_write("error", "", output_name=None, error=str(e))
-    finally:
-        if loop_dev:
-            try:
-                subprocess.run(["losetup", "-d", loop_dev], capture_output=True, timeout=5)
-            except Exception:
-                pass
-        if mount_point and mount_point.exists():
-            try:
-                subprocess.run(["umount", str(mount_point)], capture_output=True, timeout=5)
-            except Exception:
-                pass
-        if temp_dir and temp_dir.exists():
-            try:
-                shutil.rmtree(temp_dir, ignore_errors=True)
-            except Exception:
-                pass
+        if CLOUDINIT_TEMPLATES_FILE.is_file():
+            with open(CLOUDINIT_TEMPLATES_FILE, "r") as f:
+                return json.load(f)
+    except (json.JSONDecodeError, OSError):
+        pass
+    return {"templates": []}


-_build_lock = threading.Lock()
-_build_thread = None
+def _save_cloudinit_templates(data):
+    try:
+        CLOUDINIT_TEMPLATES_FILE.parent.mkdir(parents=True, exist_ok=True)
+        with open(CLOUDINIT_TEMPLATES_FILE, "w") as f:
+            json.dump(data, f, indent=2)
+        return True
+    except OSError:
+        return False


@app.route("/api/build-cloudinit-status")
@@ -623,34 +550,103 @@ def api_build_cloudinit_status():

@app.route("/api/build-cloudinit", methods=["POST"])
 def api_build_cloudinit():
-    """Start building a cloud-init ready Raspberry Pi OS image (download latest Lite, inject NoCloud). Runs in background."""
-    global _build_thread
-    with _build_lock:
-        st = _build_status_read()
-        if st.get("phase") in ("downloading", "decompressing", "injecting", "finalizing", "resolving"):
-            return jsonify({"ok": False, "error": "A build is already in progress"}), 409
-        _build_thread = threading.Thread(
-            target=_build_cloudinit_worker,
-            kwargs={
-                "variant": "lite",
-                "user_data": (request.get_json(silent=True) or {}).get("user_data") or DEFAULT_USER_DATA,
-                "meta_data": (request.get_json(silent=True) or {}).get("meta_data") or DEFAULT_META_DATA,
-                "network_config": (request.get_json(silent=True) or {}).get("network_config") or DEFAULT_NETWORK_CONFIG,
-            },
-            daemon=True,
-        )
-        _build_thread.start()
-    _build_status_write("resolving", "Starting…")
-    return jsonify({"ok": True, "message": "Build started. Download and inject may take 15–30 min. Poll /api/build-cloudinit-status or refresh the page."}), 202
+    """Start building a cloud-init image: write request file; host runs build (has loop devices)."""
+    st = _build_status_read()
+    if st.get("phase") in ("downloading", "decompressing", "injecting", "finalizing", "resolving"):
+        return jsonify({"ok": False, "error": "A build is already in progress"}), 409
+    body = request.get_json(silent=True) or {}
+    variant = (body.get("variant") or "lite").strip().lower()
+    if variant not in ("lite", "full"):
+        variant = "lite"
+    _build_status_write("resolving", "Resolving latest Raspberry Pi OS image URL…")
+    url = _raspios_latest_url(variant)
+    if not url:
+        _build_status_write("idle", "", error="Could not resolve latest Raspios URL")
+        return jsonify({"ok": False, "error": "Could not resolve latest image URL"}), 503
+    user_data = body.get("user_data") or DEFAULT_USER_DATA
+    meta_data = body.get("meta_data") or DEFAULT_META_DATA
+    network_config = body.get("network_config") or DEFAULT_NETWORK_CONFIG
+    set_as_golden_after = bool(body.get("set_as_golden_after"))
+    try:
+        BUILD_REQUEST_FILE.parent.mkdir(parents=True, exist_ok=True)
+        with open(BUILD_REQUEST_FILE, "w") as f:
+            json.dump({
+                "url": url,
+                "variant": variant,
+                "user_data": user_data,
+                "meta_data": meta_data,
+                "network_config": network_config,
+                "set_as_golden_after": set_as_golden_after,
+            }, f, indent=2)
+    except OSError as e:
+        return jsonify({"ok": False, "error": str(e)}), 500
+    _build_status_write("resolving", "Build requested; host will run download and inject (check status).")
+    return jsonify({"ok": True, "message": "Build started on host. Download and inject may take 15–45 min. Poll status or refresh the page."}), 202


@app.route("/api/raspios-latest-url")
 def api_raspios_latest_url():
-    """Return the URL of the latest Raspberry Pi OS Lite arm64 image (for display only)."""
-    url = _raspios_latest_lite_url()
+    """Return the URL of the latest Raspberry Pi OS (arm64) image. Query: variant=lite|full."""
+    variant = (request.args.get("variant") or "lite").strip().lower()
+    if variant not in ("lite", "full"):
+        variant = "lite"
+    url = _raspios_latest_url(variant)
    if not url:
        return jsonify({"ok": False, "url": None, "error": "Could not resolve latest image URL"}), 503
-    return jsonify({"ok": True, "url": url, "filename": url.split("/")[-1]})
+    return jsonify({"ok": True, "url": url, "filename": url.split("/")[-1], "variant": variant})
+
+
+@app.route("/api/cloudinit-templates", methods=["GET"])
+def api_cloudinit_templates_list():
+    """List saved cloud-init templates."""
+    data = _load_cloudinit_templates()
+    return jsonify({"templates": data.get("templates", [])})
+
+
+@app.route("/api/cloudinit-templates", methods=["POST"])
+def api_cloudinit_templates_create():
+    """Save a new cloud-init template."""
+    body = request.get_json(force=True, silent=True) or {}
+    name = (body.get("name") or "").strip()
+    if not name:
+        return jsonify({"ok": False, "error": "name required"}), 400
+    data = _load_cloudinit_templates()
+    templates = data.get("templates", [])
+    tid = str(int(time.time() * 1000))
+    templates.append({
+        "id": tid,
+        "name": name,
+        "user_data": body.get("user_data", ""),
+        "meta_data": body.get("meta_data", ""),
+        "network_config": body.get("network_config", ""),
+    })
+    data["templates"] = templates
+    if not _save_cloudinit_templates(data):
+        return jsonify({"ok": False, "error": "Failed to save"}), 500
+    return jsonify({"ok": True, "id": tid, "name": name})
+
+
+@app.route("/api/cloudinit-templates/<tid>")
+def api_cloudinit_templates_get(tid):
+    """Get one template by id."""
+    data = _load_cloudinit_templates()
+    for t in data.get("templates", []):
+        if t.get("id") == tid:
+            return jsonify(t)
+    return jsonify({"error": "not found"}), 404
+
+
+@app.route("/api/cloudinit-templates/<tid>", methods=["DELETE"])
+def api_cloudinit_templates_delete(tid):
+    """Delete a template."""
+    data = _load_cloudinit_templates()
+    templates = [t for t in data.get("templates", []) if t.get("id") != tid]
+    if len(templates) == len(data.get("templates", [])):
+        return jsonify({"ok": False, "error": "not found"}), 404
+    data["templates"] = templates
+    if not _save_cloudinit_templates(data):
+        return jsonify({"ok": False, "error": "Failed to save"}), 500
+    return jsonify({"ok": True})


@app.route("/api/golden-info")