Add cloud-init image building functionality to eMMC provisioning dashboard: implement API for downloading the latest Raspberry Pi OS Lite, injecting cloud-init files, and updating UI for cloud-init image creation. Enhance backup options with compression support for raw .img files using PiShrink, and update documentation to reflect new features and usage instructions.
This commit is contained in:
nearxos
@@ -7,9 +7,13 @@ Supports USB boot mode (ask Backup/Deploy) and network-booted devices (register,
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
from flask import Flask, render_template, jsonify, request, send_file, Response
|
||||
@@ -34,6 +38,30 @@ DEVICE_SOURCE_FILE = os.environ.get("CM4_DEVICE_SOURCE_FILE", str(BASE_DIR / "de
|
||||
BACKUPS_DIR = Path(os.environ.get("CM4_BACKUPS_DIR", str(BASE_DIR / "backups")))
|
||||
GOLDEN_IMAGE = Path(os.environ.get("CM4_GOLDEN_IMAGE", str(BASE_DIR / "golden.img")))
|
||||
NETWORK_DEVICES_FILE = Path(os.environ.get("CM4_NETWORK_DEVICES_FILE", str(BASE_DIR / "network_devices.json")))
|
||||
BUILD_STATUS_FILE = Path(os.environ.get("CM4_BUILD_STATUS_FILE", str(BASE_DIR / "build_cloudinit_status.json")))
|
||||
|
||||
# Default cloud-init user-data for Raspberry Pi OS (NoCloud on boot partition)
|
||||
DEFAULT_USER_DATA = """#cloud-config
|
||||
package_update: true
|
||||
package_upgrade: false
|
||||
packages:
|
||||
- curl
|
||||
|
||||
runcmd:
|
||||
- curl -fsSL "http://YOUR_FILE_SERVER/provisioning/bootstrap.sh" -o /tmp/bootstrap.sh
|
||||
- chmod +x /tmp/bootstrap.sh
|
||||
- /tmp/bootstrap.sh
|
||||
"""
|
||||
|
||||
DEFAULT_META_DATA = """instance-id: raspios-cloudinit-001
|
||||
local-hostname: reterminal
|
||||
"""
|
||||
|
||||
DEFAULT_NETWORK_CONFIG = """version: 2
|
||||
ethernets:
|
||||
eth0:
|
||||
dhcp4: true
|
||||
"""
|
||||
|
||||
DEFAULT_STATUS = {
|
||||
"phase": "idle",
|
||||
@@ -355,6 +383,48 @@ def api_backup_shrink(name):
|
||||
return jsonify({"ok": False, "error": str(e)}), 500
|
||||
|
||||
|
||||
@app.route("/api/backups/<path:name>/compress", methods=["POST"])
|
||||
def api_backup_compress(name):
|
||||
"""Run PiShrink with compression (shrink + gz or xz) on a raw .img backup. Produces .img.gz or .img.xz."""
|
||||
if not _safe_backup_name(name):
|
||||
return jsonify({"ok": False, "error": "invalid backup name"}), 400
|
||||
if not name.endswith(".img") or name.endswith(".img.gz") or name.endswith(".img.xz"):
|
||||
return jsonify({"ok": False, "error": "only raw .img files can be compressed (not .img.gz / .img.xz)"}), 400
|
||||
path = BACKUPS_DIR / name
|
||||
if not path.is_file():
|
||||
return jsonify({"ok": False, "error": "backup not found"}), 404
|
||||
body = request.get_json(force=True, silent=True) or {}
|
||||
fmt = (body.get("format") or request.args.get("format") or "xz").strip().lower()
|
||||
if fmt not in ("gz", "gzip", "xz"):
|
||||
fmt = "xz"
|
||||
if fmt == "gzip":
|
||||
fmt = "gz"
|
||||
pishrink = shutil.which("pishrink.sh") or "/usr/local/bin/pishrink.sh"
|
||||
if not pishrink or not os.path.isfile(pishrink):
|
||||
return jsonify({"ok": False, "error": "PiShrink not installed"}), 503
|
||||
opts = ["-n", "-Z", "-a"] if fmt == "xz" else ["-n", "-z", "-a"]
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
[pishrink] + opts + [name],
|
||||
cwd=str(BACKUPS_DIR),
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=3600,
|
||||
)
|
||||
if proc.returncode != 0:
|
||||
return jsonify({
|
||||
"ok": False,
|
||||
"error": "PiShrink failed",
|
||||
"detail": (proc.stderr or proc.stdout or "").strip() or f"exit code {proc.returncode}",
|
||||
}), 500
|
||||
ext = ".xz" if fmt == "xz" else ".gz"
|
||||
return jsonify({"ok": True, "message": f"Compressed to {name}{ext}"})
|
||||
except subprocess.TimeoutExpired:
|
||||
return jsonify({"ok": False, "error": "PiShrink timed out"}), 504
|
||||
except OSError as e:
|
||||
return jsonify({"ok": False, "error": str(e)}), 500
|
||||
|
||||
|
||||
@app.route("/api/backups/<path:name>", methods=["PATCH"])
|
||||
def api_backup_update(name):
|
||||
"""Update backup metadata (display name, description) or rename the file."""
|
||||
@@ -405,6 +475,184 @@ def api_backup_download(name):
|
||||
return send_file(path, as_attachment=True, download_name=name)
|
||||
|
||||
|
||||
def _build_status_read():
|
||||
try:
|
||||
if BUILD_STATUS_FILE.is_file():
|
||||
with open(BUILD_STATUS_FILE, "r") as f:
|
||||
return json.load(f)
|
||||
except (json.JSONDecodeError, OSError):
|
||||
pass
|
||||
return {"phase": "idle", "message": "", "output_name": None, "error": None}
|
||||
|
||||
|
||||
def _build_status_write(phase, message, output_name=None, error=None):
|
||||
try:
|
||||
BUILD_STATUS_FILE.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(BUILD_STATUS_FILE, "w") as f:
|
||||
json.dump({
|
||||
"phase": phase,
|
||||
"message": message,
|
||||
"output_name": output_name,
|
||||
"error": error,
|
||||
"updated": time.time(),
|
||||
}, f, indent=2)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def _raspios_latest_lite_url():
|
||||
"""Resolve latest Raspberry Pi OS Lite (arm64) .img.xz URL from official index."""
|
||||
base = "https://downloads.raspberrypi.com/raspios_lite_arm64/images"
|
||||
try:
|
||||
with urllib.request.urlopen(base + "/", timeout=15) as r:
|
||||
html = r.read().decode("utf-8", errors="ignore")
|
||||
# Match raspios_lite_arm64-YYYY-MM-DD/
|
||||
folders = re.findall(r"raspios_lite_arm64-(\d{4}-\d{2}-\d{2})/", html)
|
||||
if not folders:
|
||||
return None
|
||||
latest = sorted(folders)[-1]
|
||||
folder_url = f"{base}/raspios_lite_arm64-{latest}/"
|
||||
with urllib.request.urlopen(folder_url, timeout=15) as r:
|
||||
folder_html = r.read().decode("utf-8", errors="ignore")
|
||||
# Match .img.xz link (skip .sha256, .torrent, etc.)
|
||||
m = re.search(r'href="([^"]+\.img\.xz)"', folder_html)
|
||||
if not m:
|
||||
return None
|
||||
return folder_url + m.group(1)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _build_cloudinit_worker(variant, user_data, meta_data, network_config):
|
||||
"""Background: download Raspios, inject cloud-init, save to BACKUPS_DIR."""
|
||||
out_name = f"raspios-{variant}-cloudinit-{time.strftime('%Y%m%d-%H%M%S')}.img"
|
||||
out_path = BACKUPS_DIR / out_name
|
||||
mount_point = None
|
||||
loop_dev = None
|
||||
temp_dir = None
|
||||
img_path = None
|
||||
xz_path = None
|
||||
try:
|
||||
_build_status_write("resolving", "Resolving latest Raspberry Pi OS image URL…")
|
||||
url = _raspios_latest_lite_url()
|
||||
if not url:
|
||||
_build_status_write("error", "", output_name=None, error="Could not resolve latest Raspios Lite URL")
|
||||
return
|
||||
_build_status_write("downloading", f"Downloading {url.split('/')[-1]}… (this may take several minutes)")
|
||||
temp_dir = Path(tempfile.mkdtemp(prefix="cloudinit-build-", dir=str(BACKUPS_DIR)))
|
||||
xz_path = temp_dir / "image.img.xz"
|
||||
with urllib.request.urlopen(url, timeout=3600) as resp:
|
||||
total = int(resp.headers.get("Content-Length", 0)) or 0
|
||||
chunk_size = 1024 * 1024
|
||||
with open(xz_path, "wb") as f:
|
||||
done = 0
|
||||
while True:
|
||||
chunk = resp.read(chunk_size)
|
||||
if not chunk:
|
||||
break
|
||||
f.write(chunk)
|
||||
done += len(chunk)
|
||||
if total and done % (50 * 1024 * 1024) < chunk_size:
|
||||
_build_status_write("downloading", f"Downloaded {done // (1024*1024)} MB…")
|
||||
_build_status_write("decompressing", "Decompressing image…")
|
||||
img_path = temp_dir / "image.img"
|
||||
subprocess.run(["xz", "-d", "-k", "-f", str(xz_path)], check=True, capture_output=True, timeout=1800, cwd=str(temp_dir))
|
||||
if not img_path.exists():
|
||||
_build_status_write("error", "", output_name=None, error="Decompress failed: image.img not found")
|
||||
return
|
||||
_build_status_write("injecting", "Mounting boot partition and injecting cloud-init…")
|
||||
loop_out = subprocess.run(["losetup", "-f", "--show", "-P", str(img_path)], capture_output=True, text=True, timeout=10)
|
||||
if loop_out.returncode != 0:
|
||||
_build_status_write("error", "", output_name=None, error="losetup failed (loop device may not be available in this container)")
|
||||
return
|
||||
loop_dev = loop_out.stdout.strip()
|
||||
# First partition is usually boot (FAT32)
|
||||
boot_part = f"{loop_dev}p1"
|
||||
if not Path(boot_part).exists():
|
||||
boot_part = f"{loop_dev}p2"
|
||||
if not Path(boot_part).exists():
|
||||
_build_status_write("error", "", output_name=None, error="Boot partition not found")
|
||||
return
|
||||
mount_point = temp_dir / "mnt"
|
||||
mount_point.mkdir(exist_ok=True)
|
||||
subprocess.run(["mount", boot_part, str(mount_point)], check=True, capture_output=True, timeout=10)
|
||||
try:
|
||||
(mount_point / "user-data").write_text(user_data or DEFAULT_USER_DATA)
|
||||
(mount_point / "meta-data").write_text(meta_data or DEFAULT_META_DATA)
|
||||
(mount_point / "network-config").write_text(network_config or DEFAULT_NETWORK_CONFIG)
|
||||
finally:
|
||||
subprocess.run(["umount", str(mount_point)], capture_output=True, timeout=10)
|
||||
subprocess.run(["losetup", "-d", loop_dev], capture_output=True, timeout=10)
|
||||
loop_dev = None
|
||||
_build_status_write("finalizing", "Copying image to backups…")
|
||||
shutil.copy2(str(img_path), str(out_path))
|
||||
_build_status_write("done", f"Built {out_name}", output_name=out_name)
|
||||
except subprocess.TimeoutExpired as e:
|
||||
_build_status_write("error", "", output_name=None, error=f"Timeout: {e}")
|
||||
except subprocess.CalledProcessError as e:
|
||||
_build_status_write("error", "", output_name=None, error=f"Command failed: {e.stderr or e}")
|
||||
except Exception as e:
|
||||
_build_status_write("error", "", output_name=None, error=str(e))
|
||||
finally:
|
||||
if loop_dev:
|
||||
try:
|
||||
subprocess.run(["losetup", "-d", loop_dev], capture_output=True, timeout=5)
|
||||
except Exception:
|
||||
pass
|
||||
if mount_point and mount_point.exists():
|
||||
try:
|
||||
subprocess.run(["umount", str(mount_point)], capture_output=True, timeout=5)
|
||||
except Exception:
|
||||
pass
|
||||
if temp_dir and temp_dir.exists():
|
||||
try:
|
||||
shutil.rmtree(temp_dir, ignore_errors=True)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
_build_lock = threading.Lock()
|
||||
_build_thread = None
|
||||
|
||||
|
||||
@app.route("/api/build-cloudinit-status")
|
||||
def api_build_cloudinit_status():
|
||||
"""Return current build status (phase, message, output_name, error)."""
|
||||
return jsonify(_build_status_read())
|
||||
|
||||
|
||||
@app.route("/api/build-cloudinit", methods=["POST"])
|
||||
def api_build_cloudinit():
|
||||
"""Start building a cloud-init ready Raspberry Pi OS image (download latest Lite, inject NoCloud). Runs in background."""
|
||||
global _build_thread
|
||||
with _build_lock:
|
||||
st = _build_status_read()
|
||||
if st.get("phase") in ("downloading", "decompressing", "injecting", "finalizing", "resolving"):
|
||||
return jsonify({"ok": False, "error": "A build is already in progress"}), 409
|
||||
_build_thread = threading.Thread(
|
||||
target=_build_cloudinit_worker,
|
||||
kwargs={
|
||||
"variant": "lite",
|
||||
"user_data": (request.get_json(silent=True) or {}).get("user_data") or DEFAULT_USER_DATA,
|
||||
"meta_data": (request.get_json(silent=True) or {}).get("meta_data") or DEFAULT_META_DATA,
|
||||
"network_config": (request.get_json(silent=True) or {}).get("network_config") or DEFAULT_NETWORK_CONFIG,
|
||||
},
|
||||
daemon=True,
|
||||
)
|
||||
_build_thread.start()
|
||||
_build_status_write("resolving", "Starting…")
|
||||
return jsonify({"ok": True, "message": "Build started. Download and inject may take 15–30 min. Poll /api/build-cloudinit-status or refresh the page."}), 202
|
||||
|
||||
|
||||
@app.route("/api/raspios-latest-url")
|
||||
def api_raspios_latest_url():
|
||||
"""Return the URL of the latest Raspberry Pi OS Lite arm64 image (for display only)."""
|
||||
url = _raspios_latest_lite_url()
|
||||
if not url:
|
||||
return jsonify({"ok": False, "url": None, "error": "Could not resolve latest image URL"}), 503
|
||||
return jsonify({"ok": True, "url": url, "filename": url.split("/")[-1]})
|
||||
|
||||
|
||||
@app.route("/api/golden-info")
|
||||
def api_golden_info():
|
||||
"""Return whether golden image exists and its size/mtime for UI."""
|
||||
|
||||
Reference in New Issue
Block a user