Enhance eMMC provisioning dashboard: implement cloud-init template management with options to load, save, and delete templates. Update UI to support selecting Raspberry Pi OS variant for image building and improve user instructions for cloud-init image creation. Add new API endpoints for managing cloud-init templates and fetching the latest Raspberry Pi OS URLs.

This commit is contained in:
nearxos
2026-02-18 23:06:36 +02:00
parent 40b8e15e75
commit 262bc3e515
6 changed files with 332 additions and 129 deletions

View File

@@ -10,8 +10,6 @@ import os
import re import re
import shutil import shutil
import subprocess import subprocess
import tempfile
import threading
import time import time
import urllib.request import urllib.request
from pathlib import Path from pathlib import Path
@@ -39,6 +37,8 @@ BACKUPS_DIR = Path(os.environ.get("CM4_BACKUPS_DIR", str(BASE_DIR / "backups")))
GOLDEN_IMAGE = Path(os.environ.get("CM4_GOLDEN_IMAGE", str(BASE_DIR / "golden.img"))) GOLDEN_IMAGE = Path(os.environ.get("CM4_GOLDEN_IMAGE", str(BASE_DIR / "golden.img")))
NETWORK_DEVICES_FILE = Path(os.environ.get("CM4_NETWORK_DEVICES_FILE", str(BASE_DIR / "network_devices.json"))) NETWORK_DEVICES_FILE = Path(os.environ.get("CM4_NETWORK_DEVICES_FILE", str(BASE_DIR / "network_devices.json")))
BUILD_STATUS_FILE = Path(os.environ.get("CM4_BUILD_STATUS_FILE", str(BASE_DIR / "build_cloudinit_status.json"))) BUILD_STATUS_FILE = Path(os.environ.get("CM4_BUILD_STATUS_FILE", str(BASE_DIR / "build_cloudinit_status.json")))
BUILD_REQUEST_FILE = Path(os.environ.get("CM4_BUILD_REQUEST_FILE", str(BASE_DIR / "build_cloudinit_request.json")))
CLOUDINIT_TEMPLATES_FILE = Path(os.environ.get("CM4_CLOUDINIT_TEMPLATES_FILE", str(BASE_DIR / "cloudinit_templates.json")))
# Default cloud-init user-data for Raspberry Pi OS (NoCloud on boot partition) # Default cloud-init user-data for Raspberry Pi OS (NoCloud on boot partition)
DEFAULT_USER_DATA = """#cloud-config DEFAULT_USER_DATA = """#cloud-config
@@ -500,21 +500,20 @@ def _build_status_write(phase, message, output_name=None, error=None):
pass pass
def _raspios_latest_lite_url(): def _raspios_latest_url(variant="lite"):
"""Resolve latest Raspberry Pi OS Lite (arm64) .img.xz URL from official index.""" """Resolve latest Raspberry Pi OS (arm64) .img.xz URL. variant=lite|full."""
base = "https://downloads.raspberrypi.com/raspios_lite_arm64/images" slug = "raspios_lite_arm64" if variant == "lite" else "raspios_full_arm64"
base = f"https://downloads.raspberrypi.com/{slug}/images"
try: try:
with urllib.request.urlopen(base + "/", timeout=15) as r: with urllib.request.urlopen(base + "/", timeout=15) as r:
html = r.read().decode("utf-8", errors="ignore") html = r.read().decode("utf-8", errors="ignore")
# Match raspios_lite_arm64-YYYY-MM-DD/ folders = re.findall(rf"{re.escape(slug)}-(\d{{4}}-\d{{2}}-\d{{2}})/", html)
folders = re.findall(r"raspios_lite_arm64-(\d{4}-\d{2}-\d{2})/", html)
if not folders: if not folders:
return None return None
latest = sorted(folders)[-1] latest = sorted(folders)[-1]
folder_url = f"{base}/raspios_lite_arm64-{latest}/" folder_url = f"{base}/{slug}-{latest}/"
with urllib.request.urlopen(folder_url, timeout=15) as r: with urllib.request.urlopen(folder_url, timeout=15) as r:
folder_html = r.read().decode("utf-8", errors="ignore") folder_html = r.read().decode("utf-8", errors="ignore")
# Match .img.xz link (skip .sha256, .torrent, etc.)
m = re.search(r'href="([^"]+\.img\.xz)"', folder_html) m = re.search(r'href="([^"]+\.img\.xz)"', folder_html)
if not m: if not m:
return None return None
@@ -523,96 +522,24 @@ def _raspios_latest_lite_url():
return None return None
def _build_cloudinit_worker(variant, user_data, meta_data, network_config): def _load_cloudinit_templates():
"""Background: download Raspios, inject cloud-init, save to BACKUPS_DIR."""
out_name = f"raspios-{variant}-cloudinit-{time.strftime('%Y%m%d-%H%M%S')}.img"
out_path = BACKUPS_DIR / out_name
mount_point = None
loop_dev = None
temp_dir = None
img_path = None
xz_path = None
try: try:
_build_status_write("resolving", "Resolving latest Raspberry Pi OS image URL…") if CLOUDINIT_TEMPLATES_FILE.is_file():
url = _raspios_latest_lite_url() with open(CLOUDINIT_TEMPLATES_FILE, "r") as f:
if not url: return json.load(f)
_build_status_write("error", "", output_name=None, error="Could not resolve latest Raspios Lite URL") except (json.JSONDecodeError, OSError):
return pass
_build_status_write("downloading", f"Downloading {url.split('/')[-1]}… (this may take several minutes)") return {"templates": []}
temp_dir = Path(tempfile.mkdtemp(prefix="cloudinit-build-", dir=str(BACKUPS_DIR)))
xz_path = temp_dir / "image.img.xz"
with urllib.request.urlopen(url, timeout=3600) as resp:
total = int(resp.headers.get("Content-Length", 0)) or 0
chunk_size = 1024 * 1024
with open(xz_path, "wb") as f:
done = 0
while True:
chunk = resp.read(chunk_size)
if not chunk:
break
f.write(chunk)
done += len(chunk)
if total and done % (50 * 1024 * 1024) < chunk_size:
_build_status_write("downloading", f"Downloaded {done // (1024*1024)} MB…")
_build_status_write("decompressing", "Decompressing image…")
img_path = temp_dir / "image.img"
subprocess.run(["xz", "-d", "-k", "-f", str(xz_path)], check=True, capture_output=True, timeout=1800, cwd=str(temp_dir))
if not img_path.exists():
_build_status_write("error", "", output_name=None, error="Decompress failed: image.img not found")
return
_build_status_write("injecting", "Mounting boot partition and injecting cloud-init…")
loop_out = subprocess.run(["losetup", "-f", "--show", "-P", str(img_path)], capture_output=True, text=True, timeout=10)
if loop_out.returncode != 0:
_build_status_write("error", "", output_name=None, error="losetup failed (loop device may not be available in this container)")
return
loop_dev = loop_out.stdout.strip()
# First partition is usually boot (FAT32)
boot_part = f"{loop_dev}p1"
if not Path(boot_part).exists():
boot_part = f"{loop_dev}p2"
if not Path(boot_part).exists():
_build_status_write("error", "", output_name=None, error="Boot partition not found")
return
mount_point = temp_dir / "mnt"
mount_point.mkdir(exist_ok=True)
subprocess.run(["mount", boot_part, str(mount_point)], check=True, capture_output=True, timeout=10)
try:
(mount_point / "user-data").write_text(user_data or DEFAULT_USER_DATA)
(mount_point / "meta-data").write_text(meta_data or DEFAULT_META_DATA)
(mount_point / "network-config").write_text(network_config or DEFAULT_NETWORK_CONFIG)
finally:
subprocess.run(["umount", str(mount_point)], capture_output=True, timeout=10)
subprocess.run(["losetup", "-d", loop_dev], capture_output=True, timeout=10)
loop_dev = None
_build_status_write("finalizing", "Copying image to backups…")
shutil.copy2(str(img_path), str(out_path))
_build_status_write("done", f"Built {out_name}", output_name=out_name)
except subprocess.TimeoutExpired as e:
_build_status_write("error", "", output_name=None, error=f"Timeout: {e}")
except subprocess.CalledProcessError as e:
_build_status_write("error", "", output_name=None, error=f"Command failed: {e.stderr or e}")
except Exception as e:
_build_status_write("error", "", output_name=None, error=str(e))
finally:
if loop_dev:
try:
subprocess.run(["losetup", "-d", loop_dev], capture_output=True, timeout=5)
except Exception:
pass
if mount_point and mount_point.exists():
try:
subprocess.run(["umount", str(mount_point)], capture_output=True, timeout=5)
except Exception:
pass
if temp_dir and temp_dir.exists():
try:
shutil.rmtree(temp_dir, ignore_errors=True)
except Exception:
pass
_build_lock = threading.Lock() def _save_cloudinit_templates(data):
_build_thread = None try:
CLOUDINIT_TEMPLATES_FILE.parent.mkdir(parents=True, exist_ok=True)
with open(CLOUDINIT_TEMPLATES_FILE, "w") as f:
json.dump(data, f, indent=2)
return True
except OSError:
return False
@app.route("/api/build-cloudinit-status") @app.route("/api/build-cloudinit-status")
@@ -623,34 +550,103 @@ def api_build_cloudinit_status():
@app.route("/api/build-cloudinit", methods=["POST"]) @app.route("/api/build-cloudinit", methods=["POST"])
def api_build_cloudinit(): def api_build_cloudinit():
"""Start building a cloud-init ready Raspberry Pi OS image (download latest Lite, inject NoCloud). Runs in background.""" """Start building a cloud-init image: write request file; host runs build (has loop devices)."""
global _build_thread st = _build_status_read()
with _build_lock: if st.get("phase") in ("downloading", "decompressing", "injecting", "finalizing", "resolving"):
st = _build_status_read() return jsonify({"ok": False, "error": "A build is already in progress"}), 409
if st.get("phase") in ("downloading", "decompressing", "injecting", "finalizing", "resolving"): body = request.get_json(silent=True) or {}
return jsonify({"ok": False, "error": "A build is already in progress"}), 409 variant = (body.get("variant") or "lite").strip().lower()
_build_thread = threading.Thread( if variant not in ("lite", "full"):
target=_build_cloudinit_worker, variant = "lite"
kwargs={ _build_status_write("resolving", "Resolving latest Raspberry Pi OS image URL…")
"variant": "lite", url = _raspios_latest_url(variant)
"user_data": (request.get_json(silent=True) or {}).get("user_data") or DEFAULT_USER_DATA, if not url:
"meta_data": (request.get_json(silent=True) or {}).get("meta_data") or DEFAULT_META_DATA, _build_status_write("idle", "", error="Could not resolve latest Raspios URL")
"network_config": (request.get_json(silent=True) or {}).get("network_config") or DEFAULT_NETWORK_CONFIG, return jsonify({"ok": False, "error": "Could not resolve latest image URL"}), 503
}, user_data = body.get("user_data") or DEFAULT_USER_DATA
daemon=True, meta_data = body.get("meta_data") or DEFAULT_META_DATA
) network_config = body.get("network_config") or DEFAULT_NETWORK_CONFIG
_build_thread.start() set_as_golden_after = bool(body.get("set_as_golden_after"))
_build_status_write("resolving", "Starting…") try:
return jsonify({"ok": True, "message": "Build started. Download and inject may take 1530 min. Poll /api/build-cloudinit-status or refresh the page."}), 202 BUILD_REQUEST_FILE.parent.mkdir(parents=True, exist_ok=True)
with open(BUILD_REQUEST_FILE, "w") as f:
json.dump({
"url": url,
"variant": variant,
"user_data": user_data,
"meta_data": meta_data,
"network_config": network_config,
"set_as_golden_after": set_as_golden_after,
}, f, indent=2)
except OSError as e:
return jsonify({"ok": False, "error": str(e)}), 500
_build_status_write("resolving", "Build requested; host will run download and inject (check status).")
return jsonify({"ok": True, "message": "Build started on host. Download and inject may take 1545 min. Poll status or refresh the page."}), 202
@app.route("/api/raspios-latest-url") @app.route("/api/raspios-latest-url")
def api_raspios_latest_url(): def api_raspios_latest_url():
"""Return the URL of the latest Raspberry Pi OS Lite arm64 image (for display only).""" """Return the URL of the latest Raspberry Pi OS (arm64) image. Query: variant=lite|full."""
url = _raspios_latest_lite_url() variant = (request.args.get("variant") or "lite").strip().lower()
if variant not in ("lite", "full"):
variant = "lite"
url = _raspios_latest_url(variant)
if not url: if not url:
return jsonify({"ok": False, "url": None, "error": "Could not resolve latest image URL"}), 503 return jsonify({"ok": False, "url": None, "error": "Could not resolve latest image URL"}), 503
return jsonify({"ok": True, "url": url, "filename": url.split("/")[-1]}) return jsonify({"ok": True, "url": url, "filename": url.split("/")[-1], "variant": variant})
@app.route("/api/cloudinit-templates", methods=["GET"])
def api_cloudinit_templates_list():
"""List saved cloud-init templates."""
data = _load_cloudinit_templates()
return jsonify({"templates": data.get("templates", [])})
@app.route("/api/cloudinit-templates", methods=["POST"])
def api_cloudinit_templates_create():
"""Save a new cloud-init template."""
body = request.get_json(force=True, silent=True) or {}
name = (body.get("name") or "").strip()
if not name:
return jsonify({"ok": False, "error": "name required"}), 400
data = _load_cloudinit_templates()
templates = data.get("templates", [])
tid = str(int(time.time() * 1000))
templates.append({
"id": tid,
"name": name,
"user_data": body.get("user_data", ""),
"meta_data": body.get("meta_data", ""),
"network_config": body.get("network_config", ""),
})
data["templates"] = templates
if not _save_cloudinit_templates(data):
return jsonify({"ok": False, "error": "Failed to save"}), 500
return jsonify({"ok": True, "id": tid, "name": name})
@app.route("/api/cloudinit-templates/<tid>")
def api_cloudinit_templates_get(tid):
"""Get one template by id."""
data = _load_cloudinit_templates()
for t in data.get("templates", []):
if t.get("id") == tid:
return jsonify(t)
return jsonify({"error": "not found"}), 404
@app.route("/api/cloudinit-templates/<tid>", methods=["DELETE"])
def api_cloudinit_templates_delete(tid):
"""Delete a template."""
data = _load_cloudinit_templates()
templates = [t for t in data.get("templates", []) if t.get("id") != tid]
if len(templates) == len(data.get("templates", [])):
return jsonify({"ok": False, "error": "not found"}), 404
data["templates"] = templates
if not _save_cloudinit_templates(data):
return jsonify({"ok": False, "error": "Failed to save"}), 500
return jsonify({"ok": True})
@app.route("/api/golden-info") @app.route("/api/golden-info")

View File

@@ -419,15 +419,30 @@
<!-- 3b. Build cloud-init image from official Raspberry Pi OS --> <!-- 3b. Build cloud-init image from official Raspberry Pi OS -->
<section class="section"> <section class="section">
<h2 class="section-title">Build cloud-init image</h2> <h2 class="section-title">Build cloud-init image</h2>
<p class="backup-deploy-hint">Download the latest <strong>Raspberry Pi OS Lite (arm64)</strong> from the official repository and inject cloud-init NoCloud files so the image is ready for first-boot configuration.</p> <p class="backup-deploy-hint">Download the latest <strong>Raspberry Pi OS (arm64)</strong> from the official repository and inject cloud-init NoCloud files. The built image appears in Saved backups; you can then choose it and click <strong>Set as golden</strong> for deployment.</p>
<p id="buildRaspiosUrl" class="backups-mono" style="font-size:0.8rem; margin-bottom:0.5rem;"></p> <div style="margin-bottom:0.5rem;">
<label>Variant: </label>
<select id="buildVariant">
<option value="lite">Lite (no desktop)</option>
<option value="full">Full (desktop)</option>
</select>
<span id="buildRaspiosUrl" class="backups-mono" style="font-size:0.8rem; margin-left:0.5rem;"></span>
</div>
<div style="margin-bottom:0.5rem;">
<label><input type="checkbox" id="buildSetGolden" /> Set as golden image after build</label>
<span class="backups-mono" style="font-size:0.8rem;"> (use for Deploy without clicking manually)</span>
</div>
<div style="margin-bottom:0.75rem;"> <div style="margin-bottom:0.75rem;">
<button type="button" id="buildCloudInitBtn" class="btn btn-primary">Download latest Raspberry Pi OS Lite &amp; build cloud-init image</button> <button type="button" id="buildCloudInitBtn" class="btn btn-primary">Download &amp; build cloud-init image</button>
</div> </div>
<div id="buildCloudInitStatus" class="backups-mono" style="font-size:0.85rem; min-height:1.5em; margin-bottom:0.5rem;"></div> <div id="buildCloudInitStatus" class="backups-mono" style="font-size:0.85rem; min-height:1.5em; margin-bottom:0.5rem;"></div>
<details style="margin-top:0.5rem;"> <details style="margin-top:0.5rem;">
<summary>Customize cloud-init (optional)</summary> <summary>Cloud-init templates &amp; customize</summary>
<div class="inner" style="margin-top:0.5rem;"> <div class="inner" style="margin-top:0.5rem;">
<p><strong>Templates:</strong> <select id="buildTemplateSelect"><option value="">— Load a template —</option></select>
<button type="button" id="buildTemplateLoad" class="btn btn-outline btn-sm">Load</button>
<button type="button" id="buildTemplateSave" class="btn btn-outline btn-sm">Save current as template…</button></p>
<ul id="buildTemplateList" class="backups-mono" style="font-size:0.85rem; list-style:none; padding:0;"></ul>
<label>user-data (YAML)</label> <label>user-data (YAML)</label>
<textarea id="buildUserData" rows="8" style="width:100%; font-family:monospace; font-size:0.85rem; margin-bottom:0.5rem;" placeholder="Leave empty to use default (remote bootstrap example)"></textarea> <textarea id="buildUserData" rows="8" style="width:100%; font-family:monospace; font-size:0.85rem; margin-bottom:0.5rem;" placeholder="Leave empty to use default (remote bootstrap example)"></textarea>
<label>meta-data (optional)</label> <label>meta-data (optional)</label>
@@ -745,12 +760,74 @@
fetch('/api/backups').then(function(r) { return r.json(); }).then(function(d) { renderBackups(d.backups || []); }).catch(function() {}); fetch('/api/backups').then(function(r) { return r.json(); }).then(function(d) { renderBackups(d.backups || []); }).catch(function() {});
} }
function getBuildVariant() {
var sel = document.getElementById('buildVariant');
return (sel && sel.value) ? sel.value : 'lite';
}
function fetchRaspiosUrl() { function fetchRaspiosUrl() {
fetch('/api/raspios-latest-url').then(function(r) { return r.json(); }).then(function(d) { var variant = getBuildVariant();
fetch('/api/raspios-latest-url?variant=' + encodeURIComponent(variant)).then(function(r) { return r.json(); }).then(function(d) {
var el = document.getElementById('buildRaspiosUrl'); var el = document.getElementById('buildRaspiosUrl');
if (el) el.textContent = d.ok && d.filename ? ('Latest: ' + d.filename) : (d.error || 'Could not resolve latest image URL'); if (el) el.textContent = d.ok && d.filename ? ('Latest: ' + d.filename) : (d.error || 'Could not resolve latest image URL');
}).catch(function() {}); }).catch(function() {});
} }
function fetchCloudInitTemplates() {
fetch('/api/cloudinit-templates').then(function(r) { return r.json(); }).then(function(d) {
var list = d.templates || [];
var sel = document.getElementById('buildTemplateSelect');
var listEl = document.getElementById('buildTemplateList');
if (sel) {
sel.innerHTML = '<option value="">— Load a template —</option>';
list.forEach(function(t) {
var opt = document.createElement('option');
opt.value = t.id;
opt.textContent = t.name;
sel.appendChild(opt);
});
}
if (listEl) {
listEl.innerHTML = list.map(function(t) {
return '<li><span>' + escapeHtml(t.name) + '</span> <button type="button" class="btn btn-outline btn-sm template-load-btn" data-id="' + escapeHtml(t.id) + '">Load</button> <button type="button" class="btn btn-outline btn-sm template-del-btn" data-id="' + escapeHtml(t.id) + '">Delete</button></li>';
}).join('') || '<li>No templates saved.</li>';
listEl.querySelectorAll('.template-load-btn').forEach(function(btn) {
btn.onclick = function() { loadTemplate(btn.getAttribute('data-id')); };
});
listEl.querySelectorAll('.template-del-btn').forEach(function(btn) {
btn.onclick = function() {
if (!confirm('Delete this template?')) return;
fetch('/api/cloudinit-templates/' + encodeURIComponent(btn.getAttribute('data-id')), { method: 'DELETE' })
.then(function(r) { return r.json(); }).then(function(data) { if (data.ok) fetchCloudInitTemplates(); });
};
});
}
}).catch(function() {});
}
function loadTemplate(id) {
fetch('/api/cloudinit-templates/' + encodeURIComponent(id)).then(function(r) { return r.json(); }).then(function(t) {
var ud = document.getElementById('buildUserData');
var md = document.getElementById('buildMetaData');
var nc = document.getElementById('buildNetworkConfig');
if (ud) ud.value = t.user_data || '';
if (md) md.value = t.meta_data || '';
if (nc) nc.value = t.network_config || '';
}).catch(function() {});
}
function saveTemplate() {
var name = prompt('Template name');
if (!name || !name.trim()) return;
var ud = document.getElementById('buildUserData');
var md = document.getElementById('buildMetaData');
var nc = document.getElementById('buildNetworkConfig');
fetch('/api/cloudinit-templates', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({
name: name.trim(),
user_data: ud ? ud.value : '',
meta_data: md ? md.value : '',
network_config: nc ? nc.value : ''
})}).then(function(r) { return r.json(); }).then(function(d) {
if (d.ok) { fetchCloudInitTemplates(); alert('Saved as "' + name + '"'); }
else alert(d.error || 'Failed');
}).catch(function() { alert('Failed'); });
}
function fetchBuildStatus() { function fetchBuildStatus() {
fetch('/api/build-cloudinit-status').then(function(r) { return r.json(); }).then(function(d) { fetch('/api/build-cloudinit-status').then(function(r) { return r.json(); }).then(function(d) {
@@ -762,8 +839,9 @@
if (d.phase === 'idle' && !d.message) { if (d.phase === 'idle' && !d.message) {
el.textContent = ''; el.textContent = '';
} else if (d.phase === 'done') { } else if (d.phase === 'done') {
el.textContent = 'Done: ' + (d.output_name || '') + ' — refresh the backups list below.'; el.textContent = 'Done: ' + (d.output_name || '') + ' — see Saved backups below. Set as golden to use for Deploy.';
fetchBackups(); fetchBackups();
fetchGoldenInfo();
} else if (d.phase === 'error') { } else if (d.phase === 'error') {
el.textContent = 'Error: ' + (d.error || d.message || 'Unknown'); el.textContent = 'Error: ' + (d.error || d.message || 'Unknown');
} else { } else {
@@ -776,18 +854,22 @@
function startBuildCloudInit() { function startBuildCloudInit() {
var btn = document.getElementById('buildCloudInitBtn'); var btn = document.getElementById('buildCloudInitBtn');
if (btn) btn.disabled = true; if (btn) btn.disabled = true;
var body = {};
var ud = document.getElementById('buildUserData'); var ud = document.getElementById('buildUserData');
var md = document.getElementById('buildMetaData'); var md = document.getElementById('buildMetaData');
var nc = document.getElementById('buildNetworkConfig'); var nc = document.getElementById('buildNetworkConfig');
if (ud && ud.value.trim()) body.user_data = ud.value.trim(); var setGolden = document.getElementById('buildSetGolden');
if (md && md.value.trim()) body.meta_data = md.value.trim(); var body = {
if (nc && nc.value.trim()) body.network_config = nc.value.trim(); variant: getBuildVariant(),
set_as_golden_after: setGolden && setGolden.checked,
user_data: (ud && ud.value.trim()) ? ud.value.trim() : undefined,
meta_data: (md && md.value.trim()) ? md.value.trim() : undefined,
network_config: (nc && nc.value.trim()) ? nc.value.trim() : undefined
};
fetch('/api/build-cloudinit', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body) }) fetch('/api/build-cloudinit', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body) })
.then(function(r) { return r.json(); }) .then(function(r) { return r.json(); })
.then(function(data) { .then(function(data) {
if (data.ok) { if (data.ok) {
document.getElementById('buildCloudInitStatus').textContent = 'Build started. Waiting…'; document.getElementById('buildCloudInitStatus').textContent = 'Build started on host. Waiting…';
setTimeout(fetchBuildStatus, 2000); setTimeout(fetchBuildStatus, 2000);
} else { } else {
alert(data.error || 'Failed to start build'); alert(data.error || 'Failed to start build');
@@ -820,8 +902,15 @@
fetchGoldenInfo(); fetchGoldenInfo();
fetchRaspiosUrl(); fetchRaspiosUrl();
fetchBuildStatus(); fetchBuildStatus();
fetchCloudInitTemplates();
var buildBtn = document.getElementById('buildCloudInitBtn'); var buildBtn = document.getElementById('buildCloudInitBtn');
if (buildBtn) buildBtn.onclick = startBuildCloudInit; if (buildBtn) buildBtn.onclick = startBuildCloudInit;
var variantSel = document.getElementById('buildVariant');
if (variantSel) variantSel.onchange = fetchRaspiosUrl;
var templateLoadBtn = document.getElementById('buildTemplateLoad');
if (templateLoadBtn) templateLoadBtn.onclick = function() { var s = document.getElementById('buildTemplateSelect'); if (s && s.value) loadTemplate(s.value); };
var templateSaveBtn = document.getElementById('buildTemplateSave');
if (templateSaveBtn) templateSaveBtn.onclick = saveTemplate;
setInterval(fetchStatus, 2000); setInterval(fetchStatus, 2000);
setInterval(fetchLog, 4000); setInterval(fetchLog, 4000);
setInterval(fetchPending, 2000); setInterval(fetchPending, 2000);

View File

@@ -0,0 +1,90 @@
#!/usr/bin/env bash
# Run on the Proxmox host when build_cloudinit_request.json appears in the provisioning dir.
# Downloads Raspberry Pi OS image from URL, injects cloud-init NoCloud files, saves to BACKUPS_DIR.
# Uses loop devices and mount (available on host, not in unprivileged LXC).
# Triggered by systemd path unit cm4-build-cloudinit.path.
set -e
PROV_DIR="${CM4_PROVISIONING_DIR:-/var/lib/cm4-provisioning}"
REQUEST_FILE="$PROV_DIR/build_cloudinit_request.json"
STATUS_FILE="$PROV_DIR/build_cloudinit_status.json"
[[ -f /opt/cm4-provisioning/env ]] && source /opt/cm4-provisioning/env
BACKUPS_DIR="${BACKUPS_DIR:-$PROV_DIR/backups}"
GOLDEN_IMAGE="${GOLDEN_IMAGE:-$PROV_DIR/golden.img}"
write_status() {
local phase="$1" message="$2" output_name="$3" error="$4"
printf '{"phase":"%s","message":"%s","output_name":%s,"error":%s,"updated":%s}\n' \
"$phase" "$message" \
"$([ -n "$output_name" ] && echo "\"$output_name\"" || echo "null")" \
"$([ -n "$error" ] && echo "\"${error//\"/\\\"}\"" || echo "null")" \
"$(date +%s)" > "$STATUS_FILE"
}
[[ -f "$REQUEST_FILE" ]] || { echo "No request file"; exit 0; }
TEMP_DIR=$(mktemp -d -p "$BACKUPS_DIR" cloudinit-build.XXXXXX)
trap 'rm -rf "$TEMP_DIR"; rm -f "$REQUEST_FILE"' EXIT
# Extract fields from JSON into temp files (handles multi-line content)
python3 - "$REQUEST_FILE" "$TEMP_DIR" << 'PY'
import json, sys
with open(sys.argv[1]) as f:
d = json.load(f)
out = sys.argv[2]
for name, key in [("user-data", "user_data"), ("meta-data", "meta_data"), ("network-config", "network_config")]:
with open(f"{out}/{name}", "w") as f:
f.write(d.get(key, ""))
with open(f"{out}/variant", "w") as f:
f.write(d.get("variant", "lite"))
with open(f"{out}/url", "w") as f:
f.write(d.get("url", ""))
with open(f"{out}/set_golden", "w") as f:
f.write("1" if d.get("set_as_golden_after") else "0")
PY
URL=$(cat "$TEMP_DIR/url")
VARIANT=$(cat "$TEMP_DIR/variant")
SET_AS_GOLDEN=$(cat "$TEMP_DIR/set_golden")
[[ -n "$URL" ]] || { write_status "error" "" "" "Missing url in request"; exit 1; }
OUT_NAME="raspios-${VARIANT}-cloudinit-$(date +%Y%m%d-%H%M%S).img"
OUT_PATH="$BACKUPS_DIR/$OUT_NAME"
write_status "downloading" "Downloading $(basename "$URL")" "" ""
XZ_FILE="$TEMP_DIR/image.img.xz"
if ! curl -f -sS -L -o "$XZ_FILE" "$URL"; then
write_status "error" "" "" "Download failed"
exit 1
fi
write_status "decompressing" "Decompressing image…" "" ""
IMG_FILE="$TEMP_DIR/image.img"
xz -d -k -f "$XZ_FILE" || { write_status "error" "" "" "Decompress failed"; exit 1; }
[[ -f "$IMG_FILE" ]] || { write_status "error" "" "" "image.img not found after decompress"; exit 1; }
write_status "injecting" "Mounting boot partition and injecting cloud-init…" "" ""
LOOP=$(losetup -f --show -P "$IMG_FILE")
boot_part="${LOOP}p1"
[[ -b "$boot_part" ]] || boot_part="${LOOP}p2"
[[ -b "$boot_part" ]] || { write_status "error" "" "" "Boot partition not found"; losetup -d "$LOOP"; exit 1; }
MNT="$TEMP_DIR/mnt"
mkdir -p "$MNT"
mount "$boot_part" "$MNT"
cp "$TEMP_DIR/user-data" "$MNT/user-data"
cp "$TEMP_DIR/meta-data" "$MNT/meta-data"
cp "$TEMP_DIR/network-config" "$MNT/network-config"
umount "$MNT"
losetup -d "$LOOP"
write_status "finalizing" "Copying image to backups…" "" ""
mkdir -p "$BACKUPS_DIR"
cp "$IMG_FILE" "$OUT_PATH"
if [[ "$SET_AS_GOLDEN" == "1" ]]; then
cp "$OUT_PATH" "$GOLDEN_IMAGE"
write_status "done" "Built $OUT_NAME and set as golden image." "$OUT_NAME" ""
else
write_status "done" "Built $OUT_NAME" "$OUT_NAME" ""
fi

View File

@@ -0,0 +1,9 @@
[Unit]
Description=CM4 build cloud-init image (when request file appears)
After=local-fs.target
[Path]
PathExists=/var/lib/cm4-provisioning/build_cloudinit_request.json
[Install]
WantedBy=multi-user.target

View File

@@ -0,0 +1,14 @@
[Unit]
Description=CM4 build cloud-init image (download Raspios, inject NoCloud)
After=network-online.target
Wants=network-online.target
[Service]
Type=oneshot
Environment=CM4_PROVISIONING_DIR=/var/lib/cm4-provisioning
ExecStart=/opt/cm4-provisioning/build-cloudinit-image.sh
# Run as root; script uses losetup, mount
User=root
StandardOutput=journal
StandardError=journal
TimeoutStartSec=7200

View File

@@ -73,10 +73,15 @@ log "Host: installing scripts and udev ..."
mkdir -p /opt/cm4-provisioning /etc/cm4-provisioning mkdir -p /opt/cm4-provisioning /etc/cm4-provisioning
cp "$DEPLOY/host/flash-emmc-on-connect.sh" /opt/cm4-provisioning/ cp "$DEPLOY/host/flash-emmc-on-connect.sh" /opt/cm4-provisioning/
chmod +x /opt/cm4-provisioning/flash-emmc-on-connect.sh chmod +x /opt/cm4-provisioning/flash-emmc-on-connect.sh
cp "$DEPLOY/host/build-cloudinit-image.sh" /opt/cm4-provisioning/
chmod +x /opt/cm4-provisioning/build-cloudinit-image.sh
cp "$DEPLOY/host/cm4-flash-trigger.sh" /usr/local/bin/ cp "$DEPLOY/host/cm4-flash-trigger.sh" /usr/local/bin/
chmod +x /usr/local/bin/cm4-flash-trigger.sh chmod +x /usr/local/bin/cm4-flash-trigger.sh
cp "$DEPLOY/host/cm4-flash.service" /etc/systemd/system/ cp "$DEPLOY/host/cm4-flash.service" /etc/systemd/system/
cp "$DEPLOY/host/cm4-build-cloudinit.path" /etc/systemd/system/
cp "$DEPLOY/host/cm4-build-cloudinit.service" /etc/systemd/system/
systemctl daemon-reload systemctl daemon-reload
systemctl enable --now cm4-build-cloudinit.path 2>/dev/null || true
cp "$DEPLOY/host/89-cm4-boot-mode-permissions.rules" /etc/udev/rules.d/ 2>/dev/null || true cp "$DEPLOY/host/89-cm4-boot-mode-permissions.rules" /etc/udev/rules.d/ 2>/dev/null || true
cp "$DEPLOY/host/90-cm4-boot-mode.rules" /etc/udev/rules.d/ cp "$DEPLOY/host/90-cm4-boot-mode.rules" /etc/udev/rules.d/
udevadm control --reload-rules udevadm control --reload-rules