diff --git a/chromium-setup/emmc-provisioning/dashboard/README.md b/chromium-setup/emmc-provisioning/dashboard/README.md index ff37f93..c62a11a 100644 --- a/chromium-setup/emmc-provisioning/dashboard/README.md +++ b/chromium-setup/emmc-provisioning/dashboard/README.md @@ -4,7 +4,8 @@ Flask web UI to monitor the eMMC deployment process and show device connection s - **Connection steps**: Numbered instructions for putting the reTerminal in boot mode and connecting it. - **Live status**: Idle / Connecting (rpiboot) / Flashing / Backup / Done / Error, with optional progress. -- **Backup / Restore**: Toggle between **Flash** (deploy golden image) and **Backup** (save eMMC to a timestamped file when device is connected in boot mode). List and download saved backups. +- **Backup / Restore**: Toggle between **Flash** (deploy golden image) and **Backup** (save eMMC to a timestamped file when device is connected in boot mode). List and download saved backups. For each raw `.img` backup you can click **Shrink** (PiShrink) or **Compress** (shrink + xz) to reduce size. +- **Build cloud-init image**: Download the latest Raspberry Pi OS Lite (arm64) from the official repository and inject cloud-init NoCloud files so the image is ready for first-boot configuration. The result appears in the backups list; set it as golden to deploy. *Requires* loop devices and mount (if the dashboard runs in an LXC, the container may need privileged mode or loop support). - **Recent log**: Tail of the flash log (from the host, via the shared bind mount). The dashboard reads `/var/lib/cm4-provisioning/status.json` and `flash.log`, which the flash script (running on the Proxmox host) updates. When the dashboard runs inside the LXC, that directory is bind-mounted from the host, so it sees the same files. @@ -36,3 +37,5 @@ systemctl enable --now cm4-dashboard - `CM4_STATUS_FILE` – path to status JSON (default: `/var/lib/cm4-provisioning/status.json`). - `CM4_LOG_FILE` – path to flash log (default: `/var/lib/cm4-provisioning/flash.log`). +- `CM4_BACKUPS_DIR` – path to backups directory (default: `…/backups`). +- `CM4_BUILD_STATUS_FILE` – path to build-cloudinit status JSON (default: `…/build_cloudinit_status.json`). diff --git a/chromium-setup/emmc-provisioning/dashboard/app.py b/chromium-setup/emmc-provisioning/dashboard/app.py index d1e221f..e5de91f 100644 --- a/chromium-setup/emmc-provisioning/dashboard/app.py +++ b/chromium-setup/emmc-provisioning/dashboard/app.py @@ -7,9 +7,13 @@ Supports USB boot mode (ask Backup/Deploy) and network-booted devices (register, import json import os +import re import shutil import subprocess +import tempfile +import threading import time +import urllib.request from pathlib import Path from flask import Flask, render_template, jsonify, request, send_file, Response @@ -34,6 +38,30 @@ DEVICE_SOURCE_FILE = os.environ.get("CM4_DEVICE_SOURCE_FILE", str(BASE_DIR / "de BACKUPS_DIR = Path(os.environ.get("CM4_BACKUPS_DIR", str(BASE_DIR / "backups"))) GOLDEN_IMAGE = Path(os.environ.get("CM4_GOLDEN_IMAGE", str(BASE_DIR / "golden.img"))) NETWORK_DEVICES_FILE = Path(os.environ.get("CM4_NETWORK_DEVICES_FILE", str(BASE_DIR / "network_devices.json"))) +BUILD_STATUS_FILE = Path(os.environ.get("CM4_BUILD_STATUS_FILE", str(BASE_DIR / "build_cloudinit_status.json"))) + +# Default cloud-init user-data for Raspberry Pi OS (NoCloud on boot partition) +DEFAULT_USER_DATA = """#cloud-config +package_update: true +package_upgrade: false +packages: + - curl + +runcmd: + - curl -fsSL "http://YOUR_FILE_SERVER/provisioning/bootstrap.sh" -o /tmp/bootstrap.sh + - chmod +x /tmp/bootstrap.sh + - /tmp/bootstrap.sh +""" + +DEFAULT_META_DATA = """instance-id: raspios-cloudinit-001 +local-hostname: reterminal +""" + +DEFAULT_NETWORK_CONFIG = """version: 2 +ethernets: + eth0: + dhcp4: true +""" DEFAULT_STATUS = { "phase": "idle", @@ -355,6 +383,48 @@ def api_backup_shrink(name): return jsonify({"ok": False, "error": str(e)}), 500 +@app.route("/api/backups//compress", methods=["POST"]) +def api_backup_compress(name): + """Run PiShrink with compression (shrink + gz or xz) on a raw .img backup. Produces .img.gz or .img.xz.""" + if not _safe_backup_name(name): + return jsonify({"ok": False, "error": "invalid backup name"}), 400 + if not name.endswith(".img") or name.endswith(".img.gz") or name.endswith(".img.xz"): + return jsonify({"ok": False, "error": "only raw .img files can be compressed (not .img.gz / .img.xz)"}), 400 + path = BACKUPS_DIR / name + if not path.is_file(): + return jsonify({"ok": False, "error": "backup not found"}), 404 + body = request.get_json(force=True, silent=True) or {} + fmt = (body.get("format") or request.args.get("format") or "xz").strip().lower() + if fmt not in ("gz", "gzip", "xz"): + fmt = "xz" + if fmt == "gzip": + fmt = "gz" + pishrink = shutil.which("pishrink.sh") or "/usr/local/bin/pishrink.sh" + if not pishrink or not os.path.isfile(pishrink): + return jsonify({"ok": False, "error": "PiShrink not installed"}), 503 + opts = ["-n", "-Z", "-a"] if fmt == "xz" else ["-n", "-z", "-a"] + try: + proc = subprocess.run( + [pishrink] + opts + [name], + cwd=str(BACKUPS_DIR), + capture_output=True, + text=True, + timeout=3600, + ) + if proc.returncode != 0: + return jsonify({ + "ok": False, + "error": "PiShrink failed", + "detail": (proc.stderr or proc.stdout or "").strip() or f"exit code {proc.returncode}", + }), 500 + ext = ".xz" if fmt == "xz" else ".gz" + return jsonify({"ok": True, "message": f"Compressed to {name}{ext}"}) + except subprocess.TimeoutExpired: + return jsonify({"ok": False, "error": "PiShrink timed out"}), 504 + except OSError as e: + return jsonify({"ok": False, "error": str(e)}), 500 + + @app.route("/api/backups/", methods=["PATCH"]) def api_backup_update(name): """Update backup metadata (display name, description) or rename the file.""" @@ -405,6 +475,184 @@ def api_backup_download(name): return send_file(path, as_attachment=True, download_name=name) +def _build_status_read(): + try: + if BUILD_STATUS_FILE.is_file(): + with open(BUILD_STATUS_FILE, "r") as f: + return json.load(f) + except (json.JSONDecodeError, OSError): + pass + return {"phase": "idle", "message": "", "output_name": None, "error": None} + + +def _build_status_write(phase, message, output_name=None, error=None): + try: + BUILD_STATUS_FILE.parent.mkdir(parents=True, exist_ok=True) + with open(BUILD_STATUS_FILE, "w") as f: + json.dump({ + "phase": phase, + "message": message, + "output_name": output_name, + "error": error, + "updated": time.time(), + }, f, indent=2) + except OSError: + pass + + +def _raspios_latest_lite_url(): + """Resolve latest Raspberry Pi OS Lite (arm64) .img.xz URL from official index.""" + base = "https://downloads.raspberrypi.com/raspios_lite_arm64/images" + try: + with urllib.request.urlopen(base + "/", timeout=15) as r: + html = r.read().decode("utf-8", errors="ignore") + # Match raspios_lite_arm64-YYYY-MM-DD/ + folders = re.findall(r"raspios_lite_arm64-(\d{4}-\d{2}-\d{2})/", html) + if not folders: + return None + latest = sorted(folders)[-1] + folder_url = f"{base}/raspios_lite_arm64-{latest}/" + with urllib.request.urlopen(folder_url, timeout=15) as r: + folder_html = r.read().decode("utf-8", errors="ignore") + # Match .img.xz link (skip .sha256, .torrent, etc.) + m = re.search(r'href="([^"]+\.img\.xz)"', folder_html) + if not m: + return None + return folder_url + m.group(1) + except Exception: + return None + + +def _build_cloudinit_worker(variant, user_data, meta_data, network_config): + """Background: download Raspios, inject cloud-init, save to BACKUPS_DIR.""" + out_name = f"raspios-{variant}-cloudinit-{time.strftime('%Y%m%d-%H%M%S')}.img" + out_path = BACKUPS_DIR / out_name + mount_point = None + loop_dev = None + temp_dir = None + img_path = None + xz_path = None + try: + _build_status_write("resolving", "Resolving latest Raspberry Pi OS image URL…") + url = _raspios_latest_lite_url() + if not url: + _build_status_write("error", "", output_name=None, error="Could not resolve latest Raspios Lite URL") + return + _build_status_write("downloading", f"Downloading {url.split('/')[-1]}… (this may take several minutes)") + temp_dir = Path(tempfile.mkdtemp(prefix="cloudinit-build-", dir=str(BACKUPS_DIR))) + xz_path = temp_dir / "image.img.xz" + with urllib.request.urlopen(url, timeout=3600) as resp: + total = int(resp.headers.get("Content-Length", 0)) or 0 + chunk_size = 1024 * 1024 + with open(xz_path, "wb") as f: + done = 0 + while True: + chunk = resp.read(chunk_size) + if not chunk: + break + f.write(chunk) + done += len(chunk) + if total and done % (50 * 1024 * 1024) < chunk_size: + _build_status_write("downloading", f"Downloaded {done // (1024*1024)} MB…") + _build_status_write("decompressing", "Decompressing image…") + img_path = temp_dir / "image.img" + subprocess.run(["xz", "-d", "-k", "-f", str(xz_path)], check=True, capture_output=True, timeout=1800, cwd=str(temp_dir)) + if not img_path.exists(): + _build_status_write("error", "", output_name=None, error="Decompress failed: image.img not found") + return + _build_status_write("injecting", "Mounting boot partition and injecting cloud-init…") + loop_out = subprocess.run(["losetup", "-f", "--show", "-P", str(img_path)], capture_output=True, text=True, timeout=10) + if loop_out.returncode != 0: + _build_status_write("error", "", output_name=None, error="losetup failed (loop device may not be available in this container)") + return + loop_dev = loop_out.stdout.strip() + # First partition is usually boot (FAT32) + boot_part = f"{loop_dev}p1" + if not Path(boot_part).exists(): + boot_part = f"{loop_dev}p2" + if not Path(boot_part).exists(): + _build_status_write("error", "", output_name=None, error="Boot partition not found") + return + mount_point = temp_dir / "mnt" + mount_point.mkdir(exist_ok=True) + subprocess.run(["mount", boot_part, str(mount_point)], check=True, capture_output=True, timeout=10) + try: + (mount_point / "user-data").write_text(user_data or DEFAULT_USER_DATA) + (mount_point / "meta-data").write_text(meta_data or DEFAULT_META_DATA) + (mount_point / "network-config").write_text(network_config or DEFAULT_NETWORK_CONFIG) + finally: + subprocess.run(["umount", str(mount_point)], capture_output=True, timeout=10) + subprocess.run(["losetup", "-d", loop_dev], capture_output=True, timeout=10) + loop_dev = None + _build_status_write("finalizing", "Copying image to backups…") + shutil.copy2(str(img_path), str(out_path)) + _build_status_write("done", f"Built {out_name}", output_name=out_name) + except subprocess.TimeoutExpired as e: + _build_status_write("error", "", output_name=None, error=f"Timeout: {e}") + except subprocess.CalledProcessError as e: + _build_status_write("error", "", output_name=None, error=f"Command failed: {e.stderr or e}") + except Exception as e: + _build_status_write("error", "", output_name=None, error=str(e)) + finally: + if loop_dev: + try: + subprocess.run(["losetup", "-d", loop_dev], capture_output=True, timeout=5) + except Exception: + pass + if mount_point and mount_point.exists(): + try: + subprocess.run(["umount", str(mount_point)], capture_output=True, timeout=5) + except Exception: + pass + if temp_dir and temp_dir.exists(): + try: + shutil.rmtree(temp_dir, ignore_errors=True) + except Exception: + pass + + +_build_lock = threading.Lock() +_build_thread = None + + +@app.route("/api/build-cloudinit-status") +def api_build_cloudinit_status(): + """Return current build status (phase, message, output_name, error).""" + return jsonify(_build_status_read()) + + +@app.route("/api/build-cloudinit", methods=["POST"]) +def api_build_cloudinit(): + """Start building a cloud-init ready Raspberry Pi OS image (download latest Lite, inject NoCloud). Runs in background.""" + global _build_thread + with _build_lock: + st = _build_status_read() + if st.get("phase") in ("downloading", "decompressing", "injecting", "finalizing", "resolving"): + return jsonify({"ok": False, "error": "A build is already in progress"}), 409 + _build_thread = threading.Thread( + target=_build_cloudinit_worker, + kwargs={ + "variant": "lite", + "user_data": (request.get_json(silent=True) or {}).get("user_data") or DEFAULT_USER_DATA, + "meta_data": (request.get_json(silent=True) or {}).get("meta_data") or DEFAULT_META_DATA, + "network_config": (request.get_json(silent=True) or {}).get("network_config") or DEFAULT_NETWORK_CONFIG, + }, + daemon=True, + ) + _build_thread.start() + _build_status_write("resolving", "Starting…") + return jsonify({"ok": True, "message": "Build started. Download and inject may take 15–30 min. Poll /api/build-cloudinit-status or refresh the page."}), 202 + + +@app.route("/api/raspios-latest-url") +def api_raspios_latest_url(): + """Return the URL of the latest Raspberry Pi OS Lite arm64 image (for display only).""" + url = _raspios_latest_lite_url() + if not url: + return jsonify({"ok": False, "url": None, "error": "Could not resolve latest image URL"}), 503 + return jsonify({"ok": True, "url": url, "filename": url.split("/")[-1]}) + + @app.route("/api/golden-info") def api_golden_info(): """Return whether golden image exists and its size/mtime for UI.""" diff --git a/chromium-setup/emmc-provisioning/dashboard/templates/index.html b/chromium-setup/emmc-provisioning/dashboard/templates/index.html index 141f371..2aada0d 100644 --- a/chromium-setup/emmc-provisioning/dashboard/templates/index.html +++ b/chromium-setup/emmc-provisioning/dashboard/templates/index.html @@ -416,6 +416,28 @@ + +
+

Build cloud-init image

+

Download the latest Raspberry Pi OS Lite (arm64) from the official repository and inject cloud-init NoCloud files so the image is ready for first-boot configuration.

+

+
+ +
+
+
+ Customize cloud-init (optional) +
+ + + + + + +
+
+
+
How to connect @@ -568,6 +590,7 @@ const desc = (b.description || ''); const isRawImg = b.name.endsWith('.img') && !b.name.endsWith('.img.gz') && !b.name.endsWith('.img.xz'); const shrinkBtn = isRawImg ? ' ' : ''; + const compressBtn = isRawImg ? ' ' : ''; tr.innerHTML = '' + escapeHtml(displayName) + '' + '' + escapeHtml(desc) + '' + @@ -575,6 +598,7 @@ '' + fmtDate(b.mtime) + '' + '' + shrinkBtn + + compressBtn + ' ' + ' ' + 'Download' + @@ -585,6 +609,7 @@ bindSetGolden(); bindRenameFile(); bindShrink(); + bindCompress(); } function bindRenameFile() { @@ -594,7 +619,7 @@ const newName = prompt('New filename (e.g. production-v1.img)', name); if (newName == null || newName.trim() === '') return; const n = newName.trim(); - if (!/\.(img|img\.gz)$/i.test(n)) { alert('Filename must end with .img or .img.gz'); return; } + if (!/\.(img|img\.gz|img\.xz)$/i.test(n)) { alert('Filename must end with .img, .img.gz or .img.xz'); return; } if (n === name) return; fetch('/api/backups/' + encodeURIComponent(name), { method: 'PATCH', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ filename: n }) }) .then(function(r) { return r.json(); }) @@ -680,6 +705,26 @@ }); } + function bindCompress() { + document.querySelectorAll('.compress-btn').forEach(function(btn) { + btn.onclick = function() { + const name = btn.getAttribute('data-name'); + const format = btn.getAttribute('data-format') || 'xz'; + if (!confirm('Shrink and compress this image to .img.' + format + '? This minimizes size and may take several minutes.\n\n' + name)) return; + btn.disabled = true; + btn.textContent = 'Compressing…'; + fetch('/api/backups/' + encodeURIComponent(name) + '/compress', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ format: format }) }) + .then(function(r) { return r.json(); }) + .then(function(data) { + if (data.ok) { fetchBackups(); } + else alert(data.error || data.detail || 'Failed'); + }) + .catch(function() { alert('Request failed'); }) + .finally(function() { btn.disabled = false; btn.textContent = 'Compress'; }); + }; + }); + } + function fetchGoldenInfo() { fetch('/api/golden-info').then(function(r) { return r.json(); }).then(function(d) { const el = document.getElementById('goldenHint'); @@ -700,6 +745,58 @@ fetch('/api/backups').then(function(r) { return r.json(); }).then(function(d) { renderBackups(d.backups || []); }).catch(function() {}); } + function fetchRaspiosUrl() { + fetch('/api/raspios-latest-url').then(function(r) { return r.json(); }).then(function(d) { + var el = document.getElementById('buildRaspiosUrl'); + if (el) el.textContent = d.ok && d.filename ? ('Latest: ' + d.filename) : (d.error || 'Could not resolve latest image URL'); + }).catch(function() {}); + } + + function fetchBuildStatus() { + fetch('/api/build-cloudinit-status').then(function(r) { return r.json(); }).then(function(d) { + var el = document.getElementById('buildCloudInitStatus'); + var btn = document.getElementById('buildCloudInitBtn'); + if (!el) return; + var busy = ['resolving','downloading','decompressing','injecting','finalizing'].indexOf(d.phase) >= 0; + if (btn) btn.disabled = busy; + if (d.phase === 'idle' && !d.message) { + el.textContent = ''; + } else if (d.phase === 'done') { + el.textContent = 'Done: ' + (d.output_name || '') + ' — refresh the backups list below.'; + fetchBackups(); + } else if (d.phase === 'error') { + el.textContent = 'Error: ' + (d.error || d.message || 'Unknown'); + } else { + el.textContent = (d.phase || '') + ': ' + (d.message || ''); + } + if (busy) setTimeout(fetchBuildStatus, 5000); + }).catch(function() {}); + } + + function startBuildCloudInit() { + var btn = document.getElementById('buildCloudInitBtn'); + if (btn) btn.disabled = true; + var body = {}; + var ud = document.getElementById('buildUserData'); + var md = document.getElementById('buildMetaData'); + var nc = document.getElementById('buildNetworkConfig'); + if (ud && ud.value.trim()) body.user_data = ud.value.trim(); + if (md && md.value.trim()) body.meta_data = md.value.trim(); + if (nc && nc.value.trim()) body.network_config = nc.value.trim(); + fetch('/api/build-cloudinit', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body) }) + .then(function(r) { return r.json(); }) + .then(function(data) { + if (data.ok) { + document.getElementById('buildCloudInitStatus').textContent = 'Build started. Waiting…'; + setTimeout(fetchBuildStatus, 2000); + } else { + alert(data.error || 'Failed to start build'); + if (btn) btn.disabled = false; + } + }) + .catch(function() { alert('Request failed'); if (btn) btn.disabled = false; }); + } + function fmtSize(n) { if (n >= 1e9) return (n / 1e9).toFixed(1) + ' GB'; if (n >= 1e6) return (n / 1e6).toFixed(1) + ' MB'; @@ -721,10 +818,15 @@ fetchPending(); fetchBackups(); fetchGoldenInfo(); + fetchRaspiosUrl(); + fetchBuildStatus(); + var buildBtn = document.getElementById('buildCloudInitBtn'); + if (buildBtn) buildBtn.onclick = startBuildCloudInit; setInterval(fetchStatus, 2000); setInterval(fetchLog, 4000); setInterval(fetchPending, 2000); setInterval(fetchBackups, 5000); + setInterval(fetchBuildStatus, 15000); setInterval(fetchGoldenInfo, 10000); diff --git a/chromium-setup/emmc-provisioning/docs/EMMC-PROVISIONING-GUIDE.md b/chromium-setup/emmc-provisioning/docs/EMMC-PROVISIONING-GUIDE.md index 847d7c3..65f7cfc 100644 --- a/chromium-setup/emmc-provisioning/docs/EMMC-PROVISIONING-GUIDE.md +++ b/chromium-setup/emmc-provisioning/docs/EMMC-PROVISIONING-GUIDE.md @@ -208,11 +208,12 @@ Raw full-disk backups and golden images are the full size of the eMMC (e.g. 32 ```bash echo 'SHRINK_BACKUP=1' | sudo tee -a /opt/cm4-provisioning/env - # Optional: compress after shrinking (smaller file, but must decompress before using as golden image) - # echo 'PISHRINK_COMPRESS=gz' | sudo tee -a /opt/cm4-provisioning/env # or xz + # Optional: compress after shrinking (smaller file; must decompress before using as golden image) + # echo 'PISHRINK_COMPRESS=gz' | sudo tee -a /opt/cm4-provisioning/env # good balance + echo 'PISHRINK_COMPRESS=xz' | sudo tee -a /opt/cm4-provisioning/env # minimum size (slower) ``` - With `SHRINK_BACKUP=1`, once a backup finishes, the script runs PiShrink on the `.img` file. Use `PISHRINK_COMPRESS=gz` or `xz` for maximum size reduction; the file becomes `.img.gz` or `.img.xz` and must be decompressed before deploy (e.g. `gunzip -c backup.img.gz > golden.img`). + With `SHRINK_BACKUP=1`, once a backup finishes, the script runs PiShrink on the `.img` file. Use **`PISHRINK_COMPRESS=xz`** for **minimum size** (smallest file, slower); or **`gz`** for a good balance. The file becomes `.img.xz` or `.img.gz` and must be decompressed before deploy (e.g. `xz -dk backup.img.xz` then copy to `golden.img`). 3. **Shrink a golden image manually** (e.g. after building from Raspberry Pi OS):