reterminal-dm4/emmc-provisioning/scripts/deploy-to-proxmox.sh

#!/usr/bin/env bash
# Revision: 2
# Deploy CM4 eMMC provisioning to a Proxmox host: host scripts, udev, systemd units,
# LXC container (dashboard), usbboot (rpiboot), and PiShrink. Uses hostname "cm4-provisioning"
# to find the container on redeploy; creates with next available ID if not found.
#
# Redeploy (re-run) behaviour: checks first if the cm4-provisioning container exists;
# if so, skips storage selection. Skips other steps that are already configured so you
# can update only what changed. Always updates: host scripts, dashboard files, env,
# systemd/udev. Skips when present: LXC creation, backups bind-mount (if same path),
# usbboot, PiShrink, LXC python3-flask and openssh-server apt installs. Set
# DEPLOY_ROOTFS_STORAGE to avoid storage prompt on first deploy (when container is new).
#
# With host internet: installs usbboot and PiShrink so USB flash/backup and dashboard
# Shrink/Compress work. The only manual step left is to add a golden image for Deploy.
#
# Usage: ./deploy-to-proxmox.sh [proxmox_host]
# Example: ./deploy-to-proxmox.sh root@10.20.30.152
#
# Optional env:
#   DEPLOY_ROOTFS_STORAGE=name        — LXC rootfs storage (if set and valid, skips interactive choice; otherwise script lists storages and asks for number)
#   CM4_BACKUPS_HOST_PATH=/path       — host dir for backups; bind-mounted into LXC
#   DEPLOY_LXC_ROOT_PASSWORD=secret   — set root password in LXC and enable SSH
#   DEPLOY_LXC_SSH_KEY=/path/to/pub   — copy this key to LXC root (default: ~/.ssh/id_ed25519.pub or id_rsa.pub)
#   DEPLOY_LOG=1                      — also log to deploy-YYYYMMDD-HHMMSS.log
#   DEPLOY_LXC_WAN_BRIDGE=vmbr0       — Proxmox bridge for WAN (eth0); default vmbr0
#   DEPLOY_LXC_WAN_IP=dhcp            — WAN address: dhcp (default) or static e.g. 192.168.1.10/24
#   DEPLOY_LXC_LAN_BRIDGE=vmbr1       — If set, add eth1 as LAN on this bridge (e.g. provisioning / network-boot)
#   DEPLOY_LXC_LAN_SUBNET=10.20.50.1/24 — LXC IP on LAN (gateway); used only if DEPLOY_LXC_LAN_BRIDGE is set; default 10.20.50.1/24
#   DEPLOY_EMMC_SIZE_GB=32              — optional: eMMC size in GB (used only when multiple new devices appear; default 32). Detection is dynamic — single new device is used regardless of size.
#
# Legacy: DEPLOY_LXC_NET1="name=eth1,bridge=vmbr1,ip=10.20.50.1/24" still works; overridden by DEPLOY_LXC_LAN_BRIDGE + DEPLOY_LXC_LAN_SUBNET if both are set.
#
# Requires: ssh key access to root@<host>. For full install (usbboot, PiShrink), host needs internet.

set -e
PROXMOX="${1:-root@10.130.60.224}"
# ROOTFS_STORAGE set later: from DEPLOY_ROOTFS_STORAGE (if in host list) or from interactive choice
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_DIR="$(dirname "$SCRIPT_DIR")"
LOG_FILE=""
if [[ -n "${DEPLOY_LOG:-}" ]]; then
  LOG_FILE="$SCRIPT_DIR/deploy-$(date +%Y%m%d-%H%M%S).log"
  exec > >(tee -a "$LOG_FILE") 2>&1
  echo "[$(date -Iseconds)] Logging to $LOG_FILE"
fi

log() { echo "[$(date -Iseconds)] $*"; }

# Optional: gather SSH key and LXC root password for setup inside deploy
# Default LXC root password (base64); override with DEPLOY_LXC_ROOT_PASSWORD. Not secure if repo is shared.
DEFAULT_LXC_PWD_B64="c2gxcGIweDE="
DEPLOY_SSH_KEY_B64=""
DEPLOY_LXC_PWD_B64=""
if [[ -n "${DEPLOY_LXC_ROOT_PASSWORD:-}" ]]; then
  DEPLOY_LXC_PWD_B64=$(echo -n "$DEPLOY_LXC_ROOT_PASSWORD" | base64 -w 0 2>/dev/null || base64 2>/dev/null | tr -d '\n')
  log "Will set LXC root password (from DEPLOY_LXC_ROOT_PASSWORD)."
else
  DEPLOY_LXC_PWD_B64="$DEFAULT_LXC_PWD_B64"
  log "Will set LXC root password (default)."
fi
KEY_FILE="${DEPLOY_LXC_SSH_KEY:-}"
if [[ -z "$KEY_FILE" ]]; then
  for f in ~/.ssh/id_ed25519.pub ~/.ssh/id_rsa.pub; do
    [[ -f "$f" ]] && { KEY_FILE="$f"; break; }
  done
fi
if [[ -n "$KEY_FILE" && -f "$KEY_FILE" ]]; then
  DEPLOY_SSH_KEY_B64=$(base64 -w 0 < "$KEY_FILE" 2>/dev/null || base64 < "$KEY_FILE" 2>/dev/null | tr -d '\n')
  log "Will copy SSH key to LXC: $KEY_FILE"
fi

log "Deploying to $PROXMOX ..."
log "[1/5] Checking if cm4-provisioning container exists and listing storage ..."
REMOTE_CHECK=$(ssh "$PROXMOX" "bash -s" << 'REMOTECHECK'
LXC_HOSTNAME="cm4-provisioning"
LXC_EXISTS=0
for id in $(pct list 2>/dev/null | awk 'NR>1 {print $1}'); do
  h=$(pct config "$id" 2>/dev/null | sed -n 's/^hostname: *//p')
  if [[ "$h" == "$LXC_HOSTNAME" ]]; then LXC_EXISTS=1; break; fi
done
echo "LXC_EXISTS=$LXC_EXISTS"
pvesm status 2>/dev/null | awk 'NR>1 { sub(/^[ \t]+/, ""); if ($2!="pbs" && $3~/active/) print $1 }'
REMOTECHECK
)

first_line=
storages=()
while IFS= read -r line; do
  if [[ -z "$first_line" ]]; then
    first_line="$line"
  else
    [[ -n "$line" ]] && storages+=("$line")
  fi
done <<< "$REMOTE_CHECK"

if [[ "$first_line" == "LXC_EXISTS=1" ]]; then
  LXC_ALREADY_EXISTS=1
  log "Container cm4-provisioning already exists; no storage selection needed."
  if [[ ${#storages[@]} -eq 0 ]]; then
    log "Error: no Proxmox storage found on host (needed for validation). Run on host: pvesm status"
    exit 1
  fi
  ROOTFS_STORAGE="${storages[0]}"
  log "Using storage $ROOTFS_STORAGE for validation only."
else
  LXC_ALREADY_EXISTS=0
  if [[ ${#storages[@]} -eq 0 ]]; then
    log "Error: no Proxmox storage found on host (PBS excluded). Run on host: pvesm status"
    exit 1
  fi
  # --- Ask user to select LXC rootfs storage only when creating new container ---
  if [[ -n "${DEPLOY_ROOTFS_STORAGE:-}" ]]; then
    for s in "${storages[@]}"; do
      if [[ "$s" == "$DEPLOY_ROOTFS_STORAGE" ]]; then
        ROOTFS_STORAGE="$DEPLOY_ROOTFS_STORAGE"
        log "Using storage: $ROOTFS_STORAGE (from DEPLOY_ROOTFS_STORAGE)"
        break
      fi
    done
  fi
  if [[ -z "${ROOTFS_STORAGE:-}" ]]; then
    echo ""
    echo "Available storage on $PROXMOX (PBS excluded):"
    for i in "${!storages[@]}"; do echo "  $((i+1))) ${storages[i]}"; done
    echo ""
    if [[ ${#storages[@]} -eq 1 ]]; then
      ROOTFS_STORAGE="${storages[0]}"
      log "Using only available storage: $ROOTFS_STORAGE"
    elif [[ ! -t 0 ]]; then
      ROOTFS_STORAGE="${storages[0]}"
      log "No TTY: using first storage $ROOTFS_STORAGE"
    else
      while true; do
        read -r -p "Select storage (1-${#storages[@]}): " num
        if [[ "$num" =~ ^[0-9]+$ ]] && [[ "$num" -ge 1 ]] && [[ "$num" -le ${#storages[@]} ]]; then
          ROOTFS_STORAGE="${storages[num-1]}"
          log "Using storage: $ROOTFS_STORAGE"
          break
        fi
        echo "Invalid choice. Enter a number from 1 to ${#storages[@]}."
      done
    fi
  fi
fi

log "[2/5] Cleaning remote staging dir ..."
ssh "$PROXMOX" "rm -rf /tmp/emmc-provisioning-deploy"
log "[3/5] Rsync repo to $PROXMOX ..."
rsync -a "$REPO_DIR/" "$PROXMOX:/tmp/emmc-provisioning-deploy/" --exclude='.git' --exclude='scripts/deploy-to-proxmox.sh' --exclude='scripts/deploy-*.log'

log "[4/5] Running remote install (host + LXC) ..."

# Pass optional LXC SSH vars (base64), selected storage, network (WAN/LAN), and eMMC size
EMMC_GB="${DEPLOY_EMMC_SIZE_GB:-32}"
EMMC_SIZE_BYTES=$(( EMMC_GB * 1024 * 1024 * 1024 ))
ssh "$PROXMOX" "ROOTFS_STORAGE='$ROOTFS_STORAGE' CM4_BACKUPS_HOST_PATH='${CM4_BACKUPS_HOST_PATH:-}' DEPLOY_SSH_KEY_B64='${DEPLOY_SSH_KEY_B64:-}' DEPLOY_LXC_PWD_B64='${DEPLOY_LXC_PWD_B64:-}' DEPLOY_LXC_WAN_BRIDGE='${DEPLOY_LXC_WAN_BRIDGE:-}' DEPLOY_LXC_WAN_IP='${DEPLOY_LXC_WAN_IP:-}' DEPLOY_LXC_LAN_BRIDGE='${DEPLOY_LXC_LAN_BRIDGE:-}' DEPLOY_LXC_LAN_SUBNET='${DEPLOY_LXC_LAN_SUBNET:-}' DEPLOY_LXC_NET1='${DEPLOY_LXC_NET1:-}' EMMC_SIZE_BYTES='$EMMC_SIZE_BYTES' EMMC_GB='$EMMC_GB'" bash -s << 'REMOTE'
set -e
DEPLOY=/tmp/emmc-provisioning-deploy
ROOTFS_STORAGE="${ROOTFS_STORAGE:?ROOTFS_STORAGE not set}"
BACKUPS_HOST_PATH="${CM4_BACKUPS_HOST_PATH:-}"
LXC_HOSTNAME="cm4-provisioning"
log() { echo "[$(date -Iseconds)] $*"; }

# Storage already chosen interactively; validate it exists on host (match by name + active, robust to column alignment)
if ! pvesm status 2>/dev/null | grep -w "$ROOTFS_STORAGE" | grep -q active; then
  log "Error: storage $ROOTFS_STORAGE not found or not valid on host"
  exit 1
fi
log "Using storage: $ROOTFS_STORAGE"

# --- Find existing LXC by hostname or use next available ID ---
CTID=""
for id in $(pct list 2>/dev/null | awk 'NR>1 {print $1}'); do
  h=$(pct config "$id" 2>/dev/null | sed -n 's/^hostname: *//p')
  if [[ "$h" == "$LXC_HOSTNAME" ]]; then
    CTID="$id"
    break
  fi
done
if [[ -n "$CTID" ]]; then
  log "Found existing LXC $CTID (hostname: $LXC_HOSTNAME)."
  pct set "$CTID" -nameserver 8.8.8.8
else
  MAX_ID=$(pct list 2>/dev/null | awk 'NR>1 {print $1}' | sort -n | tail -1)
  [[ -z "$MAX_ID" ]] && MAX_ID=0
  CTID=$((MAX_ID + 1))
  log "Creating LXC $CTID ($LXC_HOSTNAME) (rootfs on ${ROOTFS_STORAGE})..."
  VZTMPL_DIR=/var/lib/vz/template/cache
  DEBIAN12_TMPL=$(ls "$VZTMPL_DIR"/debian-12-standard_*.tar.zst 2>/dev/null | head -1)
  if [[ -z "$DEBIAN12_TMPL" ]]; then
    log "Downloading Debian 12 LXC template..."
    pveam download local debian-12-standard_12.12-1_amd64.tar.zst || pveam download local debian-12-standard_12.7-1_amd64.tar.zst
    DEBIAN12_TMPL=$(ls "$VZTMPL_DIR"/debian-12-standard_*.tar.zst 2>/dev/null | head -1)
  fi
  [[ -z "$DEBIAN12_TMPL" ]] && { log "Error: no Debian 12 template found"; exit 1; }
  TMPL_NAME=$(basename "$DEBIAN12_TMPL")
  # WAN (eth0): bridge and IP from env; default vmbr0 + dhcp
  WAN_BRIDGE="${DEPLOY_LXC_WAN_BRIDGE:-vmbr0}"
  WAN_IP="${DEPLOY_LXC_WAN_IP:-dhcp}"
  # LAN (eth1): optional; use DEPLOY_LXC_LAN_BRIDGE + DEPLOY_LXC_LAN_SUBNET, or legacy DEPLOY_LXC_NET1
  NET1_OPT=""
  if [[ -n "${DEPLOY_LXC_LAN_BRIDGE:-}" ]]; then
    LAN_SUBNET="${DEPLOY_LXC_LAN_SUBNET:-10.20.50.1/24}"
    NET1_OPT="--net1 name=eth1,bridge=${DEPLOY_LXC_LAN_BRIDGE},ip=${LAN_SUBNET}"
    log "LXC network: eth0 WAN bridge=$WAN_BRIDGE ip=$WAN_IP; eth1 LAN bridge=$DEPLOY_LXC_LAN_BRIDGE ip=$LAN_SUBNET"
  elif [[ -n "${DEPLOY_LXC_NET1:-}" ]]; then
    NET1_OPT="--net1 $DEPLOY_LXC_NET1"
    log "LXC network: eth0 WAN bridge=$WAN_BRIDGE ip=$WAN_IP; eth1 from DEPLOY_LXC_NET1"
  else
    log "LXC network: eth0 WAN bridge=$WAN_BRIDGE ip=$WAN_IP (no LAN interface)"
  fi
  pct create "$CTID" "local:vztmpl/${TMPL_NAME}" \
    --hostname "$LXC_HOSTNAME" --memory 1024 --swap 0 --cores 1 \
    --rootfs "${ROOTFS_STORAGE}:8" --net0 name=eth0,bridge="$WAN_BRIDGE",ip="$WAN_IP" $NET1_OPT \
    --unprivileged 0 --features nesting=1 -tag cm4-provisioning
  pct set "$CTID" -nameserver 8.8.8.8
  mkdir -p /var/lib/cm4-provisioning
  pct set "$CTID" -mp0 /var/lib/cm4-provisioning,mp=/var/lib/cm4-provisioning
  log "LXC $CTID created and mount configured (DNS 8.8.8.8)."
fi

# Optional: bind-mount host directory for backup images (skip if already mounted with same path)
if [[ -n "$BACKUPS_HOST_PATH" ]]; then
  BACKUPS_PATH_NORM="${BACKUPS_HOST_PATH%/}"
  mkdir -p "$BACKUPS_HOST_PATH"
  CURRENT_MP1=$(pct config "$CTID" 2>/dev/null | sed -n 's/^mp1: *//p')
  NEED_MOUNT=1
  if [[ -n "$CURRENT_MP1" ]]; then
    if [[ "$CURRENT_MP1" == *"mp=/var/lib/cm4-provisioning/backups"* ]] && { [[ "$CURRENT_MP1" == *"$BACKUPS_PATH_NORM"* ]] || [[ "$CURRENT_MP1" == *"$BACKUPS_HOST_PATH"* ]]; }; then
      NEED_MOUNT=0
      log "Backups mount already configured (host $BACKUPS_PATH_NORM), skipping."
    fi
  fi
  if [[ "$NEED_MOUNT" -eq 1 ]]; then
    pct stop "$CTID" 2>/dev/null || true
    pct set "$CTID" -mp1 "$BACKUPS_HOST_PATH",mp=/var/lib/cm4-provisioning/backups
    pct start "$CTID" 2>/dev/null || true
    log "Backups mount: host $BACKUPS_HOST_PATH -> LXC $CTID /var/lib/cm4-provisioning/backups"
  fi
fi

# --- Host: scripts, systemd, udev (always update so changes are applied) ---
HOST_PROV_EXISTS=0
[[ -f /opt/cm4-provisioning/flash-emmc-on-connect.sh ]] && HOST_PROV_EXISTS=1
if [[ "$HOST_PROV_EXISTS" -eq 1 ]]; then
  log "Host: updating scripts and systemd units (already configured) ..."
else
  log "Host: installing scripts and systemd units ..."
fi
mkdir -p /opt/cm4-provisioning /etc/cm4-provisioning
cp "$DEPLOY/host/flash-emmc-on-connect.sh" /opt/cm4-provisioning/
chmod +x /opt/cm4-provisioning/flash-emmc-on-connect.sh
cp "$DEPLOY/host/build-cloudinit-image.sh" /opt/cm4-provisioning/
chmod +x /opt/cm4-provisioning/build-cloudinit-image.sh
cp "$DEPLOY/host/run-shrink-on-host.sh" /opt/cm4-provisioning/
chmod +x /opt/cm4-provisioning/run-shrink-on-host.sh
cp "$DEPLOY/scripts/fix-gadget-bootcode-on-host.sh" /opt/cm4-provisioning/ 2>/dev/null && chmod +x /opt/cm4-provisioning/fix-gadget-bootcode-on-host.sh || true
cp "$DEPLOY/host/cm4-flash-trigger.sh" /usr/local/bin/
chmod +x /usr/local/bin/cm4-flash-trigger.sh
cp "$DEPLOY/host/cm4-flash.service" /etc/systemd/system/
cp "$DEPLOY/host/cm4-build-cloudinit.path" /etc/systemd/system/
cp "$DEPLOY/host/cm4-build-cloudinit.service" /etc/systemd/system/
cp "$DEPLOY/host/cm4-shrink.path" /etc/systemd/system/
cp "$DEPLOY/host/cm4-shrink.service" /etc/systemd/system/
systemctl daemon-reload
systemctl enable --now cm4-build-cloudinit.path 2>/dev/null || true
systemctl enable --now cm4-shrink.path 2>/dev/null || true
cp "$DEPLOY/host/89-cm4-boot-mode-permissions.rules" /etc/udev/rules.d/ 2>/dev/null || true
cp "$DEPLOY/host/90-cm4-boot-mode.rules" /etc/udev/rules.d/
udevadm control --reload-rules 2>/dev/null || true

log "Host: env and dirs (EMMC ${EMMC_GB:-32}GB = $EMMC_SIZE_BYTES bytes) ..."
cat > /opt/cm4-provisioning/env << ENV
GOLDEN_IMAGE=/var/lib/cm4-provisioning/golden.img
RPIBOOT_DIR=/opt/usbboot
EMMC_SIZE_BYTES=${EMMC_SIZE_BYTES:-34359738368}
ENV
[[ -n "$BACKUPS_HOST_PATH" ]] && echo "BACKUPS_DIR=$BACKUPS_HOST_PATH" >> /opt/cm4-provisioning/env
touch /etc/cm4-provisioning/enabled
mkdir -p /var/lib/cm4-provisioning/backups /var/lib/cm4-provisioning/cloudinit-images /var/lib/cm4-provisioning/portal-files /var/lib/cm4-provisioning/download-cache
[[ -n "$BACKUPS_HOST_PATH" ]] && mkdir -p "$BACKUPS_HOST_PATH"
grep -q "CLOUDINIT_IMAGES_DIR" /opt/cm4-provisioning/env || echo "CLOUDINIT_IMAGES_DIR=/var/lib/cm4-provisioning/cloudinit-images" >> /opt/cm4-provisioning/env
grep -q "CM4_DOWNLOAD_CACHE_DIR" /opt/cm4-provisioning/env || echo "CM4_DOWNLOAD_CACHE_DIR=/var/lib/cm4-provisioning/download-cache" >> /opt/cm4-provisioning/env

# --- Host: ensure xz-utils (and file) for cloud-init image build (decompress .img.xz) ---
if command -v xz >/dev/null 2>&1; then
  log "Host: xz-utils already present, skipping."
else
  log "Host: installing xz-utils and file (required for cloud-init image decompress)..."
  apt-get update -qq && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq xz-utils file 2>/dev/null || log "Warning: could not install xz-utils (no internet?). Cloud-init build will fail at decompress until you run: apt install -y xz-utils file"
fi

# --- Host: install usbboot (rpiboot) only if not already present ---
if [[ -x /opt/usbboot/rpiboot ]] || [[ -f /opt/usbboot/rpiboot ]]; then
  log "Host: usbboot already installed at /opt/usbboot/rpiboot, skipping."
else
  log "Host: installing usbboot (rpiboot)..."
  if bash "$DEPLOY/scripts/install-usbboot-on-host.sh" 2>&1; then
    log "Host: usbboot installed at /opt/usbboot/rpiboot"
  else
    log "Warning: usbboot install failed (e.g. no internet). USB flash/backup will not work until you run: bash /tmp/emmc-provisioning-deploy/scripts/install-usbboot-on-host.sh"
  fi
fi

# --- Host: install PiShrink only if not already present ---
if [[ -x /usr/local/bin/pishrink.sh ]] || [[ -f /usr/local/bin/pishrink.sh ]]; then
  log "Host: PiShrink already installed, skipping."
  grep -q "SHRINK_BACKUP" /opt/cm4-provisioning/env || echo "SHRINK_BACKUP=1" >> /opt/cm4-provisioning/env
  grep -q "PISHRINK_COMPRESS" /opt/cm4-provisioning/env || echo "PISHRINK_COMPRESS=xz" >> /opt/cm4-provisioning/env
else
  log "Host: installing PiShrink..."
  if bash "$DEPLOY/scripts/install-pishrink-on-host.sh" 2>&1; then
    log "Host: PiShrink installed"
    grep -q "SHRINK_BACKUP" /opt/cm4-provisioning/env || echo "SHRINK_BACKUP=1" >> /opt/cm4-provisioning/env
    grep -q "PISHRINK_COMPRESS" /opt/cm4-provisioning/env || echo "PISHRINK_COMPRESS=xz" >> /opt/cm4-provisioning/env
  else
    log "Warning: PiShrink install failed (e.g. no internet). Dashboard Shrink/Compress will report 'PiShrink not installed' until you run: bash /tmp/emmc-provisioning-deploy/scripts/install-pishrink-on-host.sh"
  fi
fi

# --- Start LXC if stopped ---
log "Starting LXC $CTID if stopped ..."
pct start "$CTID" 2>/dev/null || true

# --- LXC: write lan-subnet.conf when LAN bridge/subnet is set (so dnsmasq/NAT/toggle use same subnet) ---
LAN_SUBNET_FOR_CONF="${DEPLOY_LXC_LAN_SUBNET:-}"
[[ -z "$LAN_SUBNET_FOR_CONF" && -n "${DEPLOY_LXC_LAN_BRIDGE:-}" ]] && LAN_SUBNET_FOR_CONF="10.20.50.1/24"
if [[ -n "$LAN_SUBNET_FOR_CONF" ]]; then
  if [[ "$LAN_SUBNET_FOR_CONF" =~ ^([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)/([0-9]+)$ ]]; then
    LAN_GW="${BASH_REMATCH[1]}"
    PREFIX="${BASH_REMATCH[2]}"
    BASE_3="${LAN_GW%.*}"
    LAN_CIDR="${BASE_3}.0/${PREFIX}"
    DHCP_RANGE_START="${BASE_3}.100"
    DHCP_RANGE_END="${BASE_3}.200"
    pct exec "$CTID" -- bash -c "mkdir -p /opt/cm4-provisioning && echo 'LAN_GW=$LAN_GW' > /opt/cm4-provisioning/lan-subnet.conf && echo 'LAN_CIDR=$LAN_CIDR' >> /opt/cm4-provisioning/lan-subnet.conf && echo 'DHCP_RANGE_START=$DHCP_RANGE_START' >> /opt/cm4-provisioning/lan-subnet.conf && echo 'DHCP_RANGE_END=$DHCP_RANGE_END' >> /opt/cm4-provisioning/lan-subnet.conf"
    echo "$LAN_GW" > "$DEPLOY/lxc_lan_ip.txt"
    log "LXC: wrote /opt/cm4-provisioning/lan-subnet.conf (LAN_GW=$LAN_GW); dashboard will be reachable on LAN at http://${LAN_GW}:5000"
  else
    log "Warning: DEPLOY_LXC_LAN_SUBNET=$LAN_SUBNET_FOR_CONF not in form A.B.C.D/PREFIX; skipping lan-subnet.conf"
  fi
fi

# --- LXC: flash scripts (for reference; actual flash runs on host) ---
log "LXC: installing flash scripts ..."
pct exec "$CTID" -- mkdir -p /opt/cm4-provisioning /etc/cm4-provisioning
pct push "$CTID" "$DEPLOY/host/flash-emmc-on-connect.sh" /opt/cm4-provisioning/flash-emmc-on-connect.sh
pct exec "$CTID" -- chmod +x /opt/cm4-provisioning/flash-emmc-on-connect.sh
pct push "$CTID" "$DEPLOY/host/cm4-flash-trigger.sh" /usr/local/bin/cm4-flash-trigger.sh
pct exec "$CTID" -- chmod +x /usr/local/bin/cm4-flash-trigger.sh
pct exec "$CTID" -- bash -c "echo -e 'GOLDEN_IMAGE=/var/lib/cm4-provisioning/golden.img\nRPIBOOT_DIR=/opt/usbboot\nEMMC_SIZE_BYTES=$EMMC_SIZE_BYTES' > /opt/cm4-provisioning/env"

# --- LXC: dashboard (all files) ---
log "LXC: installing dashboard ..."
pct exec "$CTID" -- mkdir -p /opt/cm4-provisioning/dashboard/templates
pct push "$CTID" "$DEPLOY/dashboard/app.py" /opt/cm4-provisioning/dashboard/app.py
pct push "$CTID" "$DEPLOY/dashboard/templates/home.html" /opt/cm4-provisioning/dashboard/templates/home.html
pct push "$CTID" "$DEPLOY/dashboard/templates/login.html" /opt/cm4-provisioning/dashboard/templates/login.html
pct push "$CTID" "$DEPLOY/dashboard/templates/admin.html" /opt/cm4-provisioning/dashboard/templates/admin.html
pct push "$CTID" "$DEPLOY/dashboard/templates/portal_files.html" /opt/cm4-provisioning/dashboard/templates/portal_files.html
pct push "$CTID" "$DEPLOY/dashboard/templates/cloudinit_build.html" /opt/cm4-provisioning/dashboard/templates/cloudinit_build.html
pct push "$CTID" "$DEPLOY/dashboard/cm4-dashboard.service" /opt/cm4-provisioning/dashboard/cm4-dashboard.service
# Dashboard secret for sessions (create once so logins persist across restarts)
pct exec "$CTID" -- bash -c '[[ -f /opt/cm4-provisioning/dashboard.env ]] || echo "CM4_DASHBOARD_SECRET_KEY=$(openssl rand -hex 24 2>/dev/null || head -c 24 /dev/urandom | xxd -p)" > /opt/cm4-provisioning/dashboard.env'

# --- LXC: Flask and systemd (skip apt install if flask already present) ---
if pct exec "$CTID" -- dpkg -l python3-flask 2>/dev/null | grep -q '^ii'; then
  log "LXC: python3-flask already installed, skipping apt install."
else
  log "LXC: installing python3-flask ..."
  pct exec "$CTID" -- bash -c 'apt-get update -qq && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq python3-flask python3-werkzeug'
fi
pct exec "$CTID" -- cp /opt/cm4-provisioning/dashboard/cm4-dashboard.service /etc/systemd/system/
pct exec "$CTID" -- systemctl daemon-reload
pct exec "$CTID" -- systemctl enable --now cm4-dashboard
pct exec "$CTID" -- systemctl restart cm4-dashboard
log "LXC: cm4-dashboard enabled and restarted (new code loaded)."

# --- LXC: optional SSH (root password + SSH key from deploy env) ---
if [[ -n "${DEPLOY_SSH_KEY_B64:-}" ]] || [[ -n "${DEPLOY_LXC_PWD_B64:-}" ]]; then
  log "LXC: configuring SSH (root login, password, authorized_keys)..."
  if pct exec "$CTID" -- dpkg -l openssh-server 2>/dev/null | grep -q '^ii'; then
    log "LXC: openssh-server already installed, skipping apt install."
  else
    pct exec "$CTID" -- bash -c 'apt-get update -qq && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq openssh-server 2>/dev/null' || true
  fi
  pct exec "$CTID" -- bash -c 'systemctl enable ssh 2>/dev/null; systemctl start ssh 2>/dev/null' || true
  pct exec "$CTID" -- bash -c 'sed -i "s/^#*PermitRootLogin.*/PermitRootLogin yes/" /etc/ssh/sshd_config 2>/dev/null; grep -q "^PermitRootLogin" /etc/ssh/sshd_config || echo "PermitRootLogin yes" >> /etc/ssh/sshd_config; systemctl restart ssh 2>/dev/null || systemctl restart sshd 2>/dev/null' || true
  if [[ -n "${DEPLOY_LXC_PWD_B64:-}" ]]; then
    PWD_RAW=$(echo "$DEPLOY_LXC_PWD_B64" | base64 -d 2>/dev/null)
    echo "root:$PWD_RAW" | pct exec "$CTID" -- chpasswd 2>/dev/null && log "LXC: root password set." || true
  fi
  if [[ -n "${DEPLOY_SSH_KEY_B64:-}" ]]; then
    echo "$DEPLOY_SSH_KEY_B64" | base64 -d 2>/dev/null | pct exec "$CTID" -- bash -c "mkdir -p /root/.ssh; chmod 700 /root/.ssh; cat >> /root/.ssh/authorized_keys; chmod 600 /root/.ssh/authorized_keys" 2>/dev/null && log "LXC: SSH key added to /root/.ssh/authorized_keys." || true
  fi
  LXC_IP=$(pct exec "$CTID" -- hostname -I 2>/dev/null | awk '{print $1}')
  [[ -n "$LXC_IP" ]] && log "LXC SSH: ssh root@$LXC_IP"
fi
# Always capture LXC IP for final summary (write so local script can read it)
LXC_IP=$(pct exec "$CTID" -- hostname -I 2>/dev/null | awk '{print $1}')
echo "${LXC_IP:-}" > "$DEPLOY/lxc_ip.txt"

log "Deploy done on remote. LXC ID: $CTID"
# Heredoc terminator (must be at column 1, no leading space/tab)
REMOTE

# Read LXC IP and optional LAN IP written by remote
LXC_IP=$(ssh "$PROXMOX" "cat /tmp/emmc-provisioning-deploy/lxc_ip.txt 2>/dev/null" | tr -d '\n\r')
LXC_LAN_IP=$(ssh "$PROXMOX" "cat /tmp/emmc-provisioning-deploy/lxc_lan_ip.txt 2>/dev/null" | tr -d '\n\r')

log "[5/5] Deploy finished."
echo ""
echo "=== Deploy complete ==="
echo "Host and LXC are fully set up: usbboot (rpiboot), PiShrink, dashboard, systemd, udev."
[[ -n "$LXC_IP" ]] && echo "  LXC IP (WAN): $LXC_IP"
[[ -n "$LXC_LAN_IP" ]] && echo "  LXC IP (LAN): $LXC_LAN_IP"
echo ""
echo "--- Only remaining step (manual) ---"
echo "  Add a golden image for Deploy (writing image to device):"
echo "    • Dashboard: open http://${LXC_IP:-<LXC-IP>}:5000 → Build cloud-init image → then Set as golden"
echo "    • Or copy your image: scp your-image.img $PROXMOX:/var/lib/cm4-provisioning/golden.img"
echo "  Backup (read from device) works without golden.img."
echo ""
echo "--- You have ---"
echo "  - Dashboard (WAN): http://${LXC_IP:-<LXC-IP>}:5000"
[[ -n "$LXC_LAN_IP" ]] && echo "  - Dashboard (LAN): http://${LXC_LAN_IP}:5000  (use from devices on provisioning LAN)"
[[ -n "${DEPLOY_LXC_ROOT_PASSWORD:-}" || -n "${DEPLOY_SSH_KEY_B64:-}" ]] && [[ -n "$LXC_IP" ]] && echo "  - LXC SSH: ssh root@$LXC_IP (password and/or key were set)"
[[ -n "${DEPLOY_LXC_ROOT_PASSWORD:-}" || -n "${DEPLOY_SSH_KEY_B64:-}" ]] && [[ -z "$LXC_IP" ]] && echo "  - LXC SSH: ssh root@<LXC-IP> (password and/or key were set)"
[[ -n "${CM4_BACKUPS_HOST_PATH:-}" ]] && echo "  - Backups on host: $CM4_BACKUPS_HOST_PATH"
if [[ -n "$LOG_FILE" ]]; then
  echo "  - Log: $LOG_FILE"
fi