Initial commit: Agentic OS troubleshooting platform

Self-hosted, Docker-based agentic troubleshooting platform: FastAPI backend +
LangGraph agent, Next.js UI, tiered LLM routing (local Ollama -> Gemini ->
DeepSeek -> OpenRouter), MCP server manager, encrypted device credentials,
RBAC, audit log, project-memory + Obsidian integrations, and editable
troubleshooting decision rules tuned for the GeneseasX vessel stack.

Co-authored-by: Cursor <cursoragent@cursor.com>

backend/.dockerignore

@@ -0,0 +1,9 @@
+__pycache__/
+*.pyc
+.venv/
+venv/
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/
+tests/
+.env

backend/Dockerfile

@@ -0,0 +1,23 @@
+FROM python:3.12-slim
+
+ENV PYTHONUNBUFFERED=1 \
+    PYTHONDONTWRITEBYTECODE=1 \
+    PIP_NO_CACHE_DIR=1
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends git openssh-client curl ca-certificates \
+    && rm -rf /var/lib/apt/lists/* \
+    && curl -LsSf https://astral.sh/uv/install.sh | sh \
+    && ln -s /root/.local/bin/uv /usr/local/bin/uv
+
+WORKDIR /app
+
+COPY backend/requirements.txt .
+RUN pip install -r requirements.txt
+
+COPY backend/ .
+COPY rules/ /app/rules/
+
+EXPOSE 8000
+
+CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"]

backend/app/__init__.py

@@ -0,0 +1,3 @@
+"""Agentic OS backend application package."""
+
+__version__ = "0.1.0"

backend/app/agent/__init__.py

@@ -0,0 +1,3 @@
+from app.agent.graph import run_task_agent
+
+__all__ = ["run_task_agent"]

backend/app/agent/asterisk_profiles.py

@@ -0,0 +1,84 @@
+"""Asterisk deployment profiles and connect playbooks."""
+from __future__ import annotations
+
+import os
+import shlex
+
+from app.agent.device_connect import ssh_connect_args
+
+# Only one Asterisk slot allowed per vessel.
+ASTERISK_CATALOG_KEYS = frozenset({"asterisk", "asterisk_geneseasx", "asterisk_satbox"})
+
+GENESEASX_KEYS = frozenset({"asterisk", "asterisk_geneseasx"})
+SATBOX_KEYS = frozenset({"asterisk_satbox"})
+
+
+def asterisk_container_name(catalog_key: str | None) -> str:
+    """Docker container name for GeneseasX Asterisk (default: voip)."""
+    if catalog_key not in GENESEASX_KEYS:
+        return ""
+    return (
+        os.environ.get("DEVICE_ASTERISK_GENESEASX_CONTAINER", "").strip()
+        or os.environ.get("DEVICE_ASTERISK_CONTAINER", "").strip()
+        or "voip"
+    )
+
+
+def _ssh_connect(ctx: dict) -> dict:
+    port = ctx.get("port")
+    if port is None:
+        port_raw = os.environ.get("DEVICE_ASTERISK_GENESEASX_PORT", "").strip()
+        port = int(port_raw) if port_raw.isdigit() else 6022
+    return ssh_connect_args({**ctx, "port": port})
+
+
+def _docker_asterisk(command: str):
+    def builder(c: dict) -> dict:
+        container = c.get("asterisk_container") or asterisk_container_name(c.get("catalog_key")) or "voip"
+        return {"command": f"docker exec {container} asterisk -rx {shlex.quote(command)}"}
+
+    return builder
+
+
+def _asterisk_cli(command: str):
+    return lambda c: {"command": f'asterisk -rx "{command}"'}
+
+
+# Native Debian host diagnostics (TMGeneseas / satbox) via SSH.
+SATBOX_ASTERISK_DIAGNOSTICS = [
+    ("ssh_run", _asterisk_cli("pjsip show registrations")),
+    ("ssh_run", _asterisk_cli("pjsip show endpoints")),
+    ("ssh_run", _asterisk_cli("core show channels")),
+    ("ssh_run", lambda c: {"command": "tail -n 100 /var/log/asterisk/full 2>/dev/null || journalctl -u asterisk -n 100 --no-pager 2>/dev/null || true"}),
+]
+
+# GeneseasX: SSH to Docker VM, docker exec into voip (password auth via ssh-generic-mcp).
+GENESEASX_ASTERISK_DIAGNOSTICS = [
+    ("ssh_run", _docker_asterisk("pjsip show registrations")),
+    ("ssh_run", _docker_asterisk("pjsip show endpoints")),
+    ("ssh_run", _docker_asterisk("core show channels")),
+    ("ssh_run", lambda c: {
+        "command": (
+            f"docker logs --tail 100 {shlex.quote(c.get('asterisk_container') or 'voip')} 2>&1 | tail -100"
+        )
+    }),
+]
+
+ASTERISK_PLAYBOOKS: dict[str, dict] = {
+    "asterisk_geneseasx": {
+        "mcp_server": "ssh-generic-mcp",
+        "connect": ("ssh_connect", _ssh_connect),
+        "diagnostics": GENESEASX_ASTERISK_DIAGNOSTICS,
+    },
+    # Legacy catalog key — same as GeneseasX
+    "asterisk": {
+        "mcp_server": "ssh-generic-mcp",
+        "connect": ("ssh_connect", _ssh_connect),
+        "diagnostics": GENESEASX_ASTERISK_DIAGNOSTICS,
+    },
+    "asterisk_satbox": {
+        "mcp_server": "ssh-generic-mcp",
+        "connect": ("ssh_connect", _ssh_connect),
+        "diagnostics": SATBOX_ASTERISK_DIAGNOSTICS,
+    },
+}

backend/app/agent/device_connect.py

@@ -0,0 +1,29 @@
+"""Connect argument builders for layer MCP tools (device ctx + .env fallbacks)."""
+from __future__ import annotations
+
+import os
+
+
+def pfsense_connect_args(ctx: dict) -> dict:
+    port = ctx.get("port") or int(os.environ.get("DEVICE_PFSENSE_PORT", "40443") or 40443)
+    api_key = (
+        ctx.get("secret")
+        or os.environ.get("DEVICE_PFSENSE_API_KEY", "").strip()
+        or os.environ.get("DEVICE_API_KEY", "").strip()
+        or None
+    )
+    return {
+        "url": f"https://{ctx['address']}:{port}",
+        "api_key": api_key,
+        "name": ctx.get("catalog_key") or "pfsense",
+        "verify_ssl": False,
+    }
+
+
+def ssh_connect_args(ctx: dict, *, default_port: int = 22) -> dict:
+    return {
+        "host": ctx["address"],
+        "port": ctx.get("port") or default_port,
+        "username": ctx.get("username") or "root",
+        "password": ctx.get("secret"),
+    }

backend/app/agent/device_routing.py

@@ -0,0 +1,10 @@
+"""Pick which onboard devices to diagnose — delegates to editable troubleshooting rules."""
+from __future__ import annotations
+
+from app.services.troubleshooting_rules import (
+    MatchedRule,
+    match_rule,
+    select_devices_for_issue,
+)
+
+__all__ = ["MatchedRule", "match_rule", "select_devices_for_issue"]

backend/app/agent/graph.py

@@ -0,0 +1,940 @@
+"""LangGraph troubleshooting agent — vessel-aware, multi-device diagnostics."""
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+from datetime import datetime, timezone
+from typing import Any, TypedDict
+
+from langchain_core.messages import HumanMessage, SystemMessage
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy.orm import selectinload
+
+from app.agent.device_routing import select_devices_for_issue
+from app.agent.json_utils import parse_json as _parse_json
+from app.agent.playbooks import playbook_for
+from app.agent.pfsense_profiles import run_pfsense_diagnostics
+from app.agent.proxmox_profiles import enrich_proxmox_device, run_proxmox_diagnostics
+from app.agent.asterisk_profiles import asterisk_container_name
+from app.agent.tool_runner import invoke_connect, invoke_tool
+from app.core.crypto import decrypt_secret
+from app.config import settings
+from app.database import SessionLocal
+from app.inventory_catalog import catalog_by_key
+from app.mcp_manager import get_manager
+from app.models.enums import ApprovalStatus, DeviceType, TaskStatus
+from app.models.inventory import Device, Vessel
+from app.models.task import ApprovalRequest, Task
+from app.services import balance, integrations
+from app.services.diagnostic_format import (
+    diagnostics_to_markdown,
+    prepare_report_diagnostics,
+)
+from app.services.events import publish_event
+from app.services.device_defaults import resolve_slot_defaults
+from app.services.investigation_context import gather_investigation_context
+from app.services.approval_filter import split_proposed_fixes
+from app.services.troubleshooting_rules import rule_guidance_block
+from app.services.llm import triage_model, triage_model_info
+from app.services.llm_usage import LlmUsageTracker, merge_run_history
+from app.services.model_config import get_routing_config
+from app.services.model_router import plan_reasoning_tiers, reason_with_cascade
+from app.services.task_runtime import (
+    clear_task_cancel,
+    clear_task_min_tier,
+    ensure_not_cancelled,
+    get_task_min_tier,
+    is_task_cancel_requested,
+    TaskCancelledError,
+)
+
+logger = logging.getLogger(__name__)
+
+MAX_DIAGNOSTICS_PER_DEVICE = 6
+
+# In-memory LLM usage collectors keyed by task_id (one agent run at a time per task)
+_run_trackers: dict[int, LlmUsageTracker] = {}
+
+
+class AgentState(TypedDict, total=False):
+    task_id: int
+    title: str
+    issue: str
+    details: dict | None
+    vessel_id: int | None
+    vessel_name: str | None
+    vessel_site: str | None
+    vessel_public_ip: str | None
+    devices: list[dict]
+    triage_result: dict
+    diagnostics: list[dict]
+    analysis: dict
+    report: dict
+    prior_context: dict | None
+    investigation_context: dict | None
+    matched_rule: dict | None
+    follow_up: str | None
+    run_number: int
+    prior_runs: list[dict]
+
+
+def _tracker(task_id: int) -> LlmUsageTracker:
+    if task_id not in _run_trackers:
+        _run_trackers[task_id] = LlmUsageTracker()
+    return _run_trackers[task_id]
+
+
+async def _emit(task_id: int, kind: str, message: str, payload: dict | None = None) -> None:
+    await publish_event(task_id, kind, message, payload)
+
+
+async def _progress(task_id: int, phase: str, message: str, **extra) -> None:
+    await _emit(task_id, "progress", message, {"phase": phase, **extra})
+
+
+def _device_dict(d: Device) -> dict:
+    key = d.catalog_key
+    entry = catalog_by_key().get(key) if key else None
+    secret = decrypt_secret(d.secret_enc)
+    username = d.username
+    port = d.port
+
+    if entry:
+        defaults = resolve_slot_defaults(key, entry)
+        if not secret:
+            secret = defaults.secret
+        if not username:
+            username = defaults.username or entry.default_username
+        if port is None and defaults.port is not None:
+            port = defaults.port
+
+    base = {
+        "id": d.id,
+        "name": d.name,
+        "catalog_key": key,
+        "device_type": d.device_type.value,
+        "address": d.address,
+        "port": port,
+        "mcp_server": d.mcp_server,
+        "username": username,
+        "secret": secret,
+        "asterisk_container": asterisk_container_name(key) if key else None,
+    }
+    if key == "proxmox":
+        return enrich_proxmox_device(base)
+    return base
+
+
+async def _load_vessel_devices(db: AsyncSession, vessel_id: int) -> list[dict]:
+    res = await db.execute(
+        select(Vessel).options(selectinload(Vessel.devices)).where(Vessel.id == vessel_id)
+    )
+    vessel = res.scalar_one_or_none()
+    if not vessel:
+        return []
+    devices = [_device_dict(d) for d in vessel.devices]
+    docker = next((d for d in devices if d.get("catalog_key") == "docker_vm"), None)
+    if docker and docker.get("secret"):
+        for d in devices:
+            if d.get("catalog_key") in ("asterisk_geneseasx", "asterisk") and not d.get("secret"):
+                d["secret"] = docker["secret"]
+            if d.get("catalog_key") in ("asterisk_geneseasx", "asterisk") and not d.get("port"):
+                d["port"] = docker.get("port")
+    return devices
+
+
+def _prior_context_block(prior: dict | None, follow_up: str | None) -> str:
+    if not prior and not follow_up:
+        return ""
+    lines = ["\n\n--- Previous investigation context ---"]
+    if prior:
+        if prior.get("summary"):
+            lines.append(f"Prior summary: {prior['summary']}")
+        if prior.get("root_cause"):
+            lines.append(f"Prior root cause: {prior['root_cause']}")
+        if prior.get("resolution"):
+            lines.append(f"Prior resolution attempt: {prior['resolution']}")
+        if prior.get("devices_checked"):
+            lines.append(f"Already checked: {', '.join(prior['devices_checked'])}")
+        if prior.get("resolved") is False:
+            lines.append("Status: issue NOT resolved — continue investigating.")
+    if follow_up:
+        lines.append(f"Follow-up request: {follow_up}")
+    return "\n".join(lines)
+
+
+def _investigation_context_block(ctx: dict | None) -> str:
+    """Format memory + Obsidian hits for LLM prompts."""
+    if not ctx:
+        return ""
+    memory = ctx.get("memory_hits") or []
+    obsidian = ctx.get("obsidian_hits") or []
+    if not memory and not obsidian:
+        return ""
+
+    lines = ["\n\n--- Related past investigations (memory + Obsidian) ---"]
+    lines.append(f"Search query: {ctx.get('query', '')}")
+
+    if memory:
+        lines.append("\nProject memory:")
+        for hit in memory[:6]:
+            title = hit.get("title") or "untitled"
+            mtype = hit.get("memory_type") or "note"
+            content = (hit.get("content") or "").replace("\n", " ")[:350]
+            proj = hit.get("project_id") or "?"
+            vessel = hit.get("vessel")
+            vessel_tag = f" vessel={vessel}" if vessel else ""
+            lines.append(f"- [{proj}/{mtype}{vessel_tag}] {title}: {content}")
+
+    if obsidian:
+        lines.append("\nObsidian notes:")
+        for hit in obsidian[:5]:
+            path = hit.get("path") or "?"
+            title = hit.get("title") or path
+            excerpt = (hit.get("excerpt") or "").replace("\n", " ")[:300]
+            vessel = hit.get("vessel")
+            vessel_tag = f" (vessel: {vessel})" if vessel else ""
+            lines.append(f"- {path}{vessel_tag} — {title}: {excerpt}")
+
+    lines.append(
+        "Use these prior findings to avoid repeating ruled-out causes and to reuse known fixes."
+    )
+    return "\n".join(lines)
+
+
+async def context_node(state: AgentState) -> AgentState:
+    """Load related memory and Obsidian notes before triage."""
+    tid = state["task_id"]
+    await ensure_not_cancelled(tid)
+    await _progress(tid, "context", "Searching project memory and Obsidian…")
+    await _emit(tid, "context", "Searching project memory and Obsidian for related past issues...")
+    try:
+        ctx = await gather_investigation_context(
+            state["title"],
+            state["issue"],
+            vessel_name=state.get("vessel_name"),
+            vessel_id=state.get("vessel_id"),
+        )
+    except Exception as exc:  # noqa: BLE001
+        logger.warning("investigation context failed: %s", exc)
+        ctx = {"memory_hits": [], "obsidian_hits": [], "error": str(exc)}
+
+    mem_n = ctx.get("memory_count", len(ctx.get("memory_hits") or []))
+    obs_n = ctx.get("obsidian_count", len(ctx.get("obsidian_hits") or []))
+    msg = f"Found {mem_n} memory record(s) and {obs_n} Obsidian note(s) related to this issue."
+    await _emit(
+        tid,
+        "context",
+        msg,
+        {
+            "query": ctx.get("query"),
+            "keywords": ctx.get("keywords"),
+            "memory_count": mem_n,
+            "obsidian_count": obs_n,
+            "memory_titles": [h.get("title") for h in (ctx.get("memory_hits") or [])[:5]],
+            "obsidian_paths": [h.get("path") for h in (ctx.get("obsidian_hits") or [])[:5]],
+        },
+    )
+    return {"investigation_context": ctx}
+
+
+async def _run_device_diagnostics(
+    task_id: int,
+    device: dict,
+    manager,
+    *,
+    title: str = "",
+    issue: str = "",
+) -> list[dict]:
+    """Connect to one onboard device and run read-only playbook diagnostics."""
+    from app.models.enums import DeviceType as DT
+
+    results: list[dict] = []
+    dtype = DT(device["device_type"])
+    label = device.get("catalog_key") or device.get("name") or dtype.value
+
+    if device.get("catalog_key") == "proxmox" or dtype == DT.proxmox:
+        return await run_proxmox_diagnostics(
+            task_id,
+            device,
+            manager,
+            _emit,
+            max_diagnostics=MAX_DIAGNOSTICS_PER_DEVICE,
+            task_title=title,
+            task_issue=issue,
+        )
+
+    if device.get("catalog_key") == "pfsense" or dtype == DT.pfsense:
+        return await run_pfsense_diagnostics(
+            task_id,
+            device,
+            manager,
+            _emit,
+            title=title,
+            issue=issue,
+            max_diagnostics=8,
+        )
+
+    pb = playbook_for(dtype, device.get("mcp_server"), device.get("catalog_key"))
+
+    if not pb:
+        await _emit(task_id, "diagnose", f"No playbook for {label}.")
+        return results
+
+    server = pb["mcp_server"]
+    server_state = manager.servers.get(server or "")
+    if not server or not server_state or server_state.status != "loaded":
+        await _emit(
+            task_id,
+            "diagnose",
+            f"MCP '{server}' not available for {label}.",
+            {"device": label, "server": server},
+        )
+        return results
+
+    await _emit(task_id, "connect", f"Connecting to {label} via {server}...", {"device": label})
+    connect = pb.get("connect")
+    if connect:
+        tool, builder = connect
+        await invoke_connect(task_id, label, server, tool, builder(device), manager, _emit)
+
+    for tool, builder in pb["diagnostics"][:MAX_DIAGNOSTICS_PER_DEVICE]:
+        args = builder(device)
+        res, ok = await invoke_tool(
+            task_id,
+            label,
+            server,
+            tool,
+            args,
+            manager,
+            _emit,
+            task_title=title,
+            task_issue=issue,
+        )
+        results.append(
+            {
+                "device": label,
+                "tool": tool,
+                "ok": ok,
+                "output": res.get("text", ""),
+            }
+        )
+    return results
+
+
+async def triage_node(state: AgentState) -> AgentState:
+    tid = state["task_id"]
+    await ensure_not_cancelled(tid)
+    devices = state.get("devices") or []
+    run_number = state.get("run_number", 1)
+    await _progress(tid, "triage", f"Local model triaging (run {run_number})…")
+    await _emit(tid, "triage", f"Local model triaging (run {run_number})...")
+    device_summary = [
+        {"label": d.get("catalog_key") or d.get("device_type"), "name": d.get("name")} for d in devices
+    ]
+    prompt = (
+        "You are the triage brain of a vessel troubleshooting OS. Classify the issue. "
+        "Prefer local handling when possible; only recommend cloud tiers for complex cases. "
+        'Respond ONLY as JSON: {"severity":"low|medium|high|critical",'
+        '"category":"short label","summary":"1-2 sentences",'
+        '"needs_cloud_reasoning":true|false,'
+        '"recommended_tier":"local|economy|premium",'
+        '"plan":["step1","step2"]}'
+    )
+    ctx = (
+        f"Title: {state['title']}\nIssue: {state['issue']}\n"
+        f"Onboard devices to check: {json.dumps(device_summary)}"
+        f"{_investigation_context_block(state.get('investigation_context'))}"
+        f"{rule_guidance_block(state.get('matched_rule'))}"
+        f"{_prior_context_block(state.get('prior_context'), state.get('follow_up'))}"
+    )
+    triage_info = await triage_model_info()
+    try:
+        resp = await _tracker(tid).invoke(
+            "triage",
+            await triage_model(),
+            triage_info,
+            [SystemMessage(content=prompt), HumanMessage(content=ctx)],
+        )
+        triage = _parse_json(resp.content) or {"summary": str(resp.content)[:500]}
+    except Exception as exc:  # noqa: BLE001
+        logger.warning("triage failed: %s", exc)
+        triage = {"severity": "medium", "needs_cloud_reasoning": True, "error": str(exc)}
+    await _emit(tid, "triage", triage.get("summary", "triage complete"), triage)
+    return {"triage_result": triage}
+
+
+async def diagnose_node(state: AgentState) -> AgentState:
+    tid = state["task_id"]
+    await ensure_not_cancelled(tid)
+    devices = state.get("devices") or []
+    diagnostics: list[dict] = []
+
+    if not devices:
+        await _emit(tid, "diagnose", "No vessel devices matched this issue — nothing to diagnose.")
+        return {"diagnostics": diagnostics}
+
+    await _progress(
+        tid,
+        "diagnose",
+        f"Running diagnostics on {len(devices)} device(s)…",
+        device_count=len(devices),
+    )
+    manager = get_manager()
+    for device in devices:
+        await ensure_not_cancelled(tid)
+        diagnostics.extend(
+            await _run_device_diagnostics(
+                tid,
+                device,
+                manager,
+                title=state.get("title") or "",
+                issue=state.get("issue") or "",
+            )
+        )
+    return {"diagnostics": diagnostics}
+
+
+async def reason_node(state: AgentState) -> AgentState:
+    tid = state["task_id"]
+    await ensure_not_cancelled(tid)
+    triage = state.get("triage_result", {})
+    diagnostics = state.get("diagnostics", [])
+    cfg = await get_routing_config()
+    min_tier = await get_task_min_tier(tid)
+
+    await _progress(tid, "reason", "Analyzing diagnostics with LLM…")
+    tiers, routing_reason = plan_reasoning_tiers(
+        triage,
+        diagnostics,
+        prior_context=state.get("prior_context"),
+        follow_up=state.get("follow_up"),
+        run_number=state.get("run_number", 1),
+        cfg=cfg,
+        min_tier=min_tier,
+    )
+    await _emit(
+        tid,
+        "reason",
+        f"Reasoning cascade: {' → '.join(tiers)} ({routing_reason})",
+        {"tiers": tiers, "routing_reason": routing_reason},
+    )
+
+    diag_text = "\n\n".join(
+        f"### [{d.get('device', '?')}] {d['tool']} ({'ok' if d['ok'] else 'fail'})\n{d['output'][:1500]}"
+        for d in diagnostics
+    ) or "No live diagnostics were collected."
+
+    prompt = (
+        "You are an expert vessel infrastructure analyst. Given the issue and diagnostic output "
+        "from onboard devices, determine root cause and resolution. "
+        "Only analyze devices that were actually checked — do not speculate about unrelated layers. "
+        "If prior investigation context or related memory/Obsidian notes are provided, build on them — "
+        "do not repeat already-ruled-out causes. Prefer fixes that worked before when still applicable. "
+        "If the user asked for a list, inventory, status dump, or config details, include the actual data "
+        "from diagnostics in summary and resolution — do not only describe how to find it in the UI. "
+        "If you are uncertain, set needs_escalation true and confidence below 0.65 so a stronger model can retry. "
+        "IMPORTANT for proposed_fixes: list ONLY optional future write actions that would modify production "
+        "config (firewall rules, NAT, Asterisk dialplan, etc.). Set is_config_change true ONLY for those. "
+        "Do NOT list connect steps, diagnostics already run, or manual advice — use steps_taken for those. "
+        "For read-only health/status checks with no change requested, use an empty proposed_fixes array. "
+        'Respond ONLY as JSON: {"summary":"...","root_cause":"... or null",'
+        '"steps_taken":["..."],"proposed_fixes":[{"description":"...","is_config_change":true|false,'
+        '"tool":"optional-mcp-write-tool","args":{}}],"resolved":true|false,"resolution":"...",'
+        '"confidence":0.0-1.0,"needs_escalation":true|false}'
+    )
+    ctx = (
+        f"Title: {state['title']}\nIssue: {state['issue']}\nTriage: {json.dumps(triage)}\n\n{diag_text}"
+        f"{_investigation_context_block(state.get('investigation_context'))}"
+        f"{rule_guidance_block(state.get('matched_rule'))}"
+        f"{_prior_context_block(state.get('prior_context'), state.get('follow_up'))}"
+    )
+    messages = [SystemMessage(content=prompt), HumanMessage(content=ctx)]
+    try:
+        analysis, routing_meta = await reason_with_cascade(
+            _tracker(tid), tiers, messages, _parse_json, cfg=cfg, diagnostics=diagnostics, task_id=tid
+        )
+        analysis["model_routing"] = {**routing_meta, "routing_reason": routing_reason}
+    except Exception as exc:  # noqa: BLE001
+        logger.warning("reason failed: %s", exc)
+        analysis = {"summary": f"reasoning error: {exc}", "resolved": False, "proposed_fixes": []}
+    await _emit(
+        tid,
+        "reason",
+        analysis.get("summary", "analysis complete"),
+        {
+            "model": analysis.get("_model"),
+            "tier": analysis.get("_tier_used"),
+            "routing": analysis.get("model_routing"),
+        },
+    )
+    return {"analysis": analysis}
+
+
+def _unique_strings(values: list[Any]) -> list[str]:
+    out: list[str] = []
+    seen: set[str] = set()
+    for value in values:
+        text = str(value).strip()
+        if not text or text in seen:
+            continue
+        seen.add(text)
+        out.append(text)
+    return out
+
+
+def _scope_checked(devices: list[dict], diagnostics: list[dict]) -> list[dict]:
+    out: list[dict] = []
+    for device in devices:
+        key = device.get("catalog_key") or device.get("name") or "device"
+        tools = [
+            d.get("tool")
+            for d in diagnostics
+            if (d.get("device") == key or d.get("device") == device.get("name")) and d.get("tool")
+        ]
+        out.append(
+            {
+                "device": key,
+                "name": device.get("name") or key,
+                "checks": _unique_strings(tools),
+                "ok": all(bool(d.get("ok")) for d in diagnostics if d.get("device") == key),
+            }
+        )
+    return out
+
+
+def _report_findings(analysis: dict, diagnostics: list[dict], devices_checked: list[str]) -> list[dict]:
+    findings: list[dict] = []
+    if analysis.get("root_cause"):
+        findings.append(
+            {
+                "title": "Root cause",
+                "summary": analysis.get("root_cause"),
+                "description": analysis.get("resolution"),
+            }
+        )
+    failed = [d for d in diagnostics if d.get("ok") is False]
+    for d in failed[:5]:
+        findings.append(
+            {
+                "title": f"{d.get('device', 'device')} check failed",
+                "summary": d.get("tool"),
+                "description": (d.get("output") or "")[:300],
+            }
+        )
+    if not findings and devices_checked:
+        findings.append(
+            {
+                "title": "No critical issue found",
+                "summary": f"Completed checks for {', '.join(devices_checked)}.",
+                "description": analysis.get("summary") or analysis.get("resolution"),
+            }
+        )
+    return findings
+
+
+async def report_node(state: AgentState) -> AgentState:
+    tid = state["task_id"]
+    await ensure_not_cancelled(tid)
+    analysis = state.get("analysis", {})
+    triage = state.get("triage_result", {})
+    diagnostics = state.get("diagnostics", [])
+    run_number = state.get("run_number", 1)
+    follow_up = state.get("follow_up")
+
+    await _progress(tid, "report", "Writing report and saving notes…")
+    planned_steps = _unique_strings(list(triage.get("plan") or []))
+    actions_taken = _unique_strings(list(analysis.get("steps_taken") or []))
+    resolved = bool(analysis.get("resolved"))
+    proposed_all = analysis.get("proposed_fixes") or []
+    proposed, informational = split_proposed_fixes(proposed_all)
+
+    llm_summary = _tracker(tid).summary()
+
+    report_diagnostics = prepare_report_diagnostics(diagnostics)
+    devices_checked = [d.get("catalog_key") or d.get("name") for d in state.get("devices") or []]
+    devices_checked = _unique_strings(devices_checked)
+    scope_checked = _scope_checked(state.get("devices") or [], diagnostics)
+    tools_run = [
+        {"device": d.get("device"), "tool": d.get("tool"), "ok": d.get("ok")}
+        for d in diagnostics
+    ]
+    findings = _report_findings(analysis, diagnostics, devices_checked)
+    executive_summary = analysis.get("summary") or (
+        f"Checked {', '.join(devices_checked)}." if devices_checked else "No device checks were run."
+    )
+
+    report: dict[str, Any] = {
+        "summary": analysis.get("summary"),
+        "executive_summary": executive_summary,
+        "severity": triage.get("severity"),
+        "root_cause": analysis.get("root_cause"),
+        "planned_steps": planned_steps,
+        "actions_taken": actions_taken,
+        "tools_run": tools_run,
+        # Backward compatible, no tool-name spam.
+        "steps": actions_taken or planned_steps,
+        "devices_checked": devices_checked,
+        "scope_checked": scope_checked,
+        "findings": findings,
+        "diagnostics": report_diagnostics,
+        "resolved": resolved,
+        "resolution": analysis.get("resolution"),
+        "proposed_fixes": proposed_all,
+        "proposed_fixes_actionable": proposed,
+        "recommendations": informational,
+        "follow_up": follow_up,
+        "model_routing": analysis.get("model_routing"),
+        "generated_at": datetime.now(timezone.utc).isoformat(),
+        "vessel_id": state.get("vessel_id"),
+        "vessel_name": state.get("vessel_name"),
+        "vessel_site": state.get("vessel_site"),
+        "vessel_public_ip": state.get("vessel_public_ip"),
+    }
+    inv_ctx = state.get("investigation_context")
+    if inv_ctx:
+        report["prior_knowledge"] = {
+            "query": inv_ctx.get("query"),
+            "memory_count": inv_ctx.get("memory_count"),
+            "obsidian_count": inv_ctx.get("obsidian_count"),
+            "memory_titles": [h.get("title") for h in (inv_ctx.get("memory_hits") or [])[:8]],
+            "obsidian_paths": [h.get("path") for h in (inv_ctx.get("obsidian_hits") or [])[:8]],
+        }
+    if state.get("matched_rule"):
+        report["matched_rule"] = state["matched_rule"]
+
+    report = merge_run_history(state.get("prior_runs") or [], run_number, report, llm_summary)
+
+    async with SessionLocal() as db:
+        created: list[ApprovalRequest] = []
+        for fix in proposed:
+            row = ApprovalRequest(
+                task_id=tid,
+                tool_name=fix.get("tool") or "manual-config-change",
+                tool_args=fix.get("args") or {},
+                risk=fix.get("description"),
+                status=ApprovalStatus.pending,
+            )
+            db.add(row)
+            created.append(row)
+        await db.flush()
+        for row in created:
+            await _emit(
+                tid,
+                "approval",
+                f"Approval required: {row.tool_name} — {(row.risk or '')[:120]}",
+                {
+                    "approval_id": row.id,
+                    "tool_name": row.tool_name,
+                    "risk": row.risk,
+                    "status": "pending",
+                },
+            )
+        if created:
+            res = await db.execute(select(Task).where(Task.id == tid))
+            task_row = res.scalar_one_or_none()
+            if task_row:
+                task_row.status = TaskStatus.waiting_approval
+        await db.commit()
+
+    cost_msg = (
+        f"Run cost ${report['run_cost_usd']:.4f} · Total ${report['total_cost_usd']:.4f}"
+        if report["total_cost_usd"] > 0
+        else f"Run cost $0.00 (local) · Total ${report['total_cost_usd']:.4f}"
+    )
+    await _emit(tid, "report", f"Writing memory and Obsidian note… {cost_msg}", {"llm_usage": report["llm_usage"]})
+
+    title = state["title"]
+    run_label = f" (run {run_number})" if run_number > 1 else ""
+    vessel_name = state.get("vessel_name")
+    vessel_label = f"[{vessel_name}] " if vessel_name else ""
+    content = (
+        f"Vessel: {vessel_name or 'unknown'}\n"
+        f"Issue: {state['issue']}\n\n"
+        f"Run: {run_number}\n"
+        f"Devices checked: {', '.join(report['devices_checked']) or 'none'}\n\n"
+        f"Root cause: {report['root_cause']}\n\nResolution: {report['resolution']}\n\n"
+        f"LLM cost this run: ${report['run_cost_usd']:.4f} (total ${report['total_cost_usd']:.4f})\n\n"
+        f"Findings:\n" + "\n".join(f"- {f.get('title')}: {f.get('summary')}" for f in findings)
+    )
+    if report_diagnostics:
+        content += "\n\nDiagnostic tools run:\n" + "\n".join(
+            f"- [{d.get('device')}] {d.get('tool')}" for d in report_diagnostics
+        )
+    memory_id = await integrations.create_task_memory(
+        title=f"{vessel_label}[Task] {title}{run_label}",
+        content=content,
+        memory_type="fix" if resolved else "note",
+        tags=["agentic-os", triage.get("category", "troubleshooting")],
+        importance=4 if triage.get("severity") in ("high", "critical") else 3,
+        vessel_name=vessel_name,
+        task_id=tid,
+    )
+    obsidian_path, obsidian_push_error = await integrations.write_obsidian_note(
+        title=f"{vessel_label}{title}{run_label}".strip(),
+        issue=state["issue"] if not follow_up else f"{state['issue']}\n\nFollow-up: {follow_up}",
+        steps=actions_taken or planned_steps,
+        cause=report["root_cause"],
+        resolution=report["resolution"],
+        resolved=resolved,
+        vessel_name=vessel_name,
+        task_id=tid,
+        diagnostics_md=diagnostics_to_markdown(report_diagnostics),
+    )
+
+    report["memory_id"] = memory_id
+    report["obsidian_path"] = obsidian_path
+    report["obsidian_push_error"] = obsidian_push_error
+    report["memory_project_id"] = settings.memory_task_project_id
+    complete_msg = "Report complete."
+    if memory_id:
+        complete_msg += f" Memory saved ({settings.memory_task_project_id})."
+    if obsidian_path and not obsidian_push_error:
+        complete_msg += f" Obsidian note pushed ({obsidian_path})."
+    elif obsidian_path:
+        complete_msg += f" Obsidian note saved locally ({obsidian_path}); git push failed."
+    await _emit(
+        tid,
+        "report",
+        complete_msg,
+        {
+            "memory_id": memory_id,
+            "memory_project_id": settings.memory_task_project_id,
+            "obsidian_path": obsidian_path,
+            "obsidian_push_error": obsidian_push_error,
+            "llm_usage": report["llm_usage"],
+            "run_cost_usd": report["run_cost_usd"],
+            "total_cost_usd": report["total_cost_usd"],
+        },
+    )
+    return {"report": report}
+
+
+_compiled_graph = None
+
+
+def get_graph():
+    """Return the process-wide compiled LangGraph, building it once."""
+    global _compiled_graph
+    if _compiled_graph is None:
+        _compiled_graph = build_graph()
+    return _compiled_graph
+
+
+def build_graph():
+    from langgraph.graph import END, START, StateGraph
+
+    g = StateGraph(AgentState)
+    g.add_node("context_step", context_node)
+    g.add_node("triage_step", triage_node)
+    g.add_node("diagnose_step", diagnose_node)
+    g.add_node("reason_step", reason_node)
+    g.add_node("report_step", report_node)
+    g.add_edge(START, "context_step")
+    g.add_edge("context_step", "triage_step")
+    g.add_edge("triage_step", "diagnose_step")
+    g.add_edge("diagnose_step", "reason_step")
+    g.add_edge("reason_step", "report_step")
+    g.add_edge("report_step", END)
+    return g.compile()
+
+
+async def _finalize_cancelled(task_id: int) -> None:
+    already_cancelled = False
+    async with SessionLocal() as db:
+        res = await db.execute(select(Task).where(Task.id == task_id))
+        task = res.scalar_one_or_none()
+        if task:
+            already_cancelled = task.status == TaskStatus.cancelled
+            if not already_cancelled:
+                task.status = TaskStatus.cancelled
+                await db.commit()
+    if not already_cancelled:
+        await _emit(task_id, "cancelled", "Task stopped.", persist=True)
+    await clear_task_cancel(task_id)
+
+
+async def run_task_agent(task_id: int) -> None:
+    async with SessionLocal() as db:
+        res = await db.execute(select(Task).where(Task.id == task_id))
+        task = res.scalar_one_or_none()
+        if not task:
+            return
+
+        details = dict(task.details or {})
+        continue_ctx = details.pop("_continue", None)
+        continue_mode = continue_ctx is not None
+
+        prior_runs: list[dict] = details.get("runs") or []
+        if not prior_runs and task.report:
+            prior_runs = task.report.get("runs") or []
+
+        if continue_mode:
+            run_number = len(prior_runs) + 1
+            follow_up = continue_ctx.get("follow_up", "")
+            prior_context = continue_ctx.get("prior_report") or task.report
+        else:
+            run_number = 1
+            follow_up = None
+            prior_context = None
+            prior_runs = []
+
+        issue_text = task.issue
+        routing_text = f"{issue_text}\n{follow_up}" if follow_up else issue_text
+
+        devices: list[dict] = []
+        matched_rule: dict | None = None
+        vessel_name: str | None = None
+        vessel_site: str | None = None
+        vessel_public_ip: str | None = None
+        if task.vessel_id:
+            all_devices = await _load_vessel_devices(db, task.vessel_id)
+            devices, rule = select_devices_for_issue(
+                all_devices, routing_text, title=task.title
+            )
+            matched_rule = rule.as_dict() if rule else None
+            vres = await db.execute(select(Vessel).where(Vessel.id == task.vessel_id))
+            vessel = vres.scalar_one_or_none()
+            if vessel:
+                vessel_name = vessel.name
+                vessel_site = vessel.site
+                vessel_public_ip = vessel.public_ip
+        elif task.device_id:
+            dres = await db.execute(select(Device).where(Device.id == task.device_id))
+            d = dres.scalar_one_or_none()
+            if d:
+                devices = [_device_dict(d)]
+
+        task.status = TaskStatus.running
+        task.details = details
+        await db.commit()
+
+    if await is_task_cancel_requested(task_id):
+        await _finalize_cancelled(task_id)
+        _run_trackers.pop(task_id, None)
+        return
+
+    labels = [d.get("catalog_key") or d.get("name") for d in devices]
+    run_label = f" (run {run_number})" if run_number > 1 else ""
+    await _emit(task_id, "log", f"Agent run started{run_label}.")
+    if follow_up:
+        await _emit(task_id, "log", f"Follow-up: {follow_up}")
+    if vessel_name:
+        await _emit(
+            task_id,
+            "log",
+            f"Vessel: {vessel_name}"
+            + (f" ({vessel_public_ip})" if vessel_public_ip else ""),
+            {"vessel_name": vessel_name, "vessel_public_ip": vessel_public_ip},
+        )
+    await _emit(
+        task_id,
+        "log",
+        f"Will diagnose {len(devices)} device(s): {', '.join(labels) or 'none'}",
+        {"devices": labels, "run_number": run_number},
+    )
+    if matched_rule:
+        await _emit(
+            task_id,
+            "rule",
+            f"Matched rule: {matched_rule.get('name')} → {', '.join(matched_rule.get('devices') or [])}",
+            matched_rule,
+        )
+
+    remaining = await balance.latest_remaining_usd()
+    from app.config import settings
+
+    if remaining is not None and remaining < settings.min_balance_usd:
+        msg = f"Aborting: remaining balance ${remaining:.2f} below minimum ${settings.min_balance_usd:.2f}."
+        await _emit(task_id, "error", msg)
+        async with SessionLocal() as db:
+            res = await db.execute(select(Task).where(Task.id == task_id))
+            task = res.scalar_one()
+            task.status = TaskStatus.failed
+            task.error = msg
+            await db.commit()
+        _run_trackers.pop(task_id, None)
+        return
+
+    initial: AgentState = {
+        "task_id": task_id,
+        "title": task.title,
+        "issue": issue_text,
+        "details": task.details,
+        "vessel_id": task.vessel_id,
+        "vessel_name": vessel_name,
+        "vessel_site": vessel_site,
+        "vessel_public_ip": vessel_public_ip,
+        "devices": devices,
+        "prior_context": prior_context,
+        "follow_up": follow_up,
+        "run_number": run_number,
+        "prior_runs": prior_runs if continue_mode else [],
+        "matched_rule": matched_rule,
+    }
+
+    try:
+        # Create the per-task usage tracker inside the try so the finally below
+        # always cleans it up, even if setup above this point raised.
+        _run_trackers[task_id] = LlmUsageTracker()
+        graph = get_graph()
+        final_state = await graph.ainvoke(initial)
+        report = final_state.get("report", {})
+        async with SessionLocal() as db:
+            res = await db.execute(select(Task).where(Task.id == task_id))
+            task = res.scalar_one()
+            task.report = report
+            task.memory_id = report.get("memory_id")
+            task.obsidian_path = report.get("obsidian_path")
+            task.details = {
+                **(task.details or {}),
+                "runs": report.get("runs", []),
+                "total_cost_usd": report.get("total_cost_usd", 0.0),
+            }
+            pend = await db.execute(
+                select(ApprovalRequest).where(
+                    ApprovalRequest.task_id == task_id,
+                    ApprovalRequest.status == ApprovalStatus.pending,
+                )
+            )
+            has_pending = pend.scalars().first() is not None
+            if report.get("resolved") and not has_pending:
+                task.status = TaskStatus.succeeded
+            elif has_pending:
+                task.status = TaskStatus.waiting_approval
+            elif continue_mode or not report.get("resolved"):
+                task.status = TaskStatus.succeeded  # completed run but unresolved — user can continue
+            else:
+                task.status = TaskStatus.succeeded
+            await db.commit()
+            final_status = task.status.value
+        await _emit(
+            task_id,
+            "log",
+            f"Agent run finished ({final_status}). Cost: ${report.get('run_cost_usd', 0):.4f} this run, "
+            f"${report.get('total_cost_usd', 0):.4f} total.",
+            {"llm_usage": report.get("llm_usage"), "run_number": run_number},
+        )
+    except TaskCancelledError:
+        logger.info("Task %s cancelled by user", task_id)
+        await _finalize_cancelled(task_id)
+    except asyncio.CancelledError:
+        logger.info("Task %s asyncio cancelled", task_id)
+        await _finalize_cancelled(task_id)
+    except Exception as exc:  # noqa: BLE001
+        logger.exception("Agent run failed for task %s", task_id)
+        await _emit(task_id, "error", f"Agent run failed: {exc}")
+        async with SessionLocal() as db:
+            res = await db.execute(select(Task).where(Task.id == task_id))
+            task = res.scalar_one()
+            task.status = TaskStatus.failed
+            task.error = str(exc)
+            await db.commit()
+    finally:
+        _run_trackers.pop(task_id, None)
+        try:
+            await clear_task_min_tier(task_id)
+        except Exception:  # noqa: BLE001
+            logger.debug("clear_task_min_tier failed for task %s", task_id)

backend/app/agent/json_utils.py

@@ -0,0 +1,24 @@
+"""Shared helpers for extracting JSON objects from LLM responses."""
+from __future__ import annotations
+
+import json
+
+
+def parse_json(text: str) -> dict:
+    """Best-effort extraction of a single JSON object from LLM output.
+
+    Handles fenced ```json blocks and leading/trailing prose by slicing
+    between the first '{' and last '}'. Returns {} when nothing parses.
+    """
+    text = text.strip()
+    if text.startswith("```"):
+        text = text.split("```", 2)[1]
+        if text.startswith("json"):
+            text = text[4:]
+    start, end = text.find("{"), text.rfind("}")
+    if start != -1 and end != -1:
+        try:
+            return json.loads(text[start : end + 1])
+        except json.JSONDecodeError:
+            pass
+    return {}

backend/app/agent/mcp_development.py

@@ -0,0 +1,294 @@
+"""Agent-driven MCP extension and bug-fix workflow."""
+from __future__ import annotations
+
+import logging
+from collections.abc import Awaitable, Callable
+from typing import Any
+
+from langchain_core.messages import HumanMessage, SystemMessage
+
+from app.agent.json_utils import parse_json as _parse_json
+from app.config import settings
+from app.services import mcp_workspace
+from app.services.model_config import get_routing_config, resolve_tier_model
+from app.services.model_router import build_chat_model
+
+logger = logging.getLogger(__name__)
+
+EmitFn = Callable[[int, str, str, dict | None], Awaitable[None]]
+
+_REPAIRABLE = (
+    "unknown tool",
+    "traceback",
+    "nameerror",
+    "attributeerror",
+    "typeerror",
+    "not defined",
+    "target' is not defined",
+    "missing required",
+    "unexpected keyword",
+    "has no attribute",
+)
+
+
+def is_mcp_bug_or_gap(tool: str, error: str | None, output: str | None) -> bool:
+    text = f"{error or ''} {output or ''}".lower()
+    if "unknown tool" in text and tool.lower() in text:
+        return True
+    return any(sig in text for sig in _REPAIRABLE)
+
+
+async def _load_context_files(server: str) -> dict[str, str]:
+    files: dict[str, str] = {}
+    for rel in await mcp_workspace.list_source_files(server):
+        if rel == "server.py" or rel.endswith("server.py") or rel == "pyproject.toml":
+            try:
+                files[rel] = await mcp_workspace.read_source_file(server, rel)
+            except OSError:
+                pass
+    if "server.py" not in files:
+        listed = await mcp_workspace.list_source_files(server, max_files=5)
+        for rel in listed[:3]:
+            try:
+                files[rel] = await mcp_workspace.read_source_file(server, rel, max_chars=40_000)
+            except OSError:
+                pass
+    return files
+
+
+async def propose_mcp_patch(
+    *,
+    server: str,
+    tool: str,
+    issue: str,
+    error: str | None,
+    output: str | None,
+    task_title: str = "",
+) -> dict[str, Any]:
+    """Use LLM to propose MCP source file edits."""
+    cfg = await get_routing_config()
+    tier = settings.mcp_dev_model_tier if hasattr(settings, "mcp_dev_model_tier") else "premium"
+    entry = resolve_tier_model(cfg, tier) or resolve_tier_model(cfg, "economy")
+    if not entry:
+        return {"ok": False, "error": "No LLM available for MCP development"}
+
+    files = await _load_context_files(server)
+    tools = mcp_workspace.list_registered_tools(server)
+
+    prompt = (
+        "You are an expert Python MCP (Model Context Protocol) developer using FastMCP patterns. "
+        "Given a failing tool call or missing capability, patch the MCP server source. "
+        "Add new @mcp.tool() functions when a capability is missing; fix bugs when tools crash. "
+        "Match existing code style. Keep changes minimal and safe (read-only tools unless clearly needed). "
+        'Respond ONLY as JSON: {"summary":"...", "commit_message":"fix: ...", '
+        '"files":[{"path":"server.py","content":"FULL file content after edit"}]}'
+    )
+    ctx = (
+        f"MCP server: {server}\n"
+        f"Registered tools: {', '.join(tools) or 'unknown'}\n"
+        f"Task: {task_title}\n"
+        f"Issue: {issue}\n"
+        f"Tool involved: {tool}\n"
+        f"Error: {error or 'none'}\n"
+        f"Output: {(output or '')[:2000]}\n\n"
+        f"Source files:\n"
+        + "\n\n".join(f"--- {p} ---\n{c[:25000]}" for p, c in files.items())
+    )
+
+    model = build_chat_model(entry, temperature=0.1)
+    resp = await model.ainvoke([SystemMessage(content=prompt), HumanMessage(content=ctx)])
+    data = _parse_json(str(resp.content))
+    if not data.get("files"):
+        return {"ok": False, "error": "LLM did not return file edits", "raw": str(resp.content)[:500]}
+    return {"ok": True, **data}
+
+
+async def apply_and_reload_mcp(
+    server: str,
+    edits: list[dict],
+    manager,
+) -> dict[str, Any]:
+    written = await mcp_workspace.apply_file_edits(server, edits)
+    if not written:
+        return {"ok": False, "error": "No files written"}
+    await manager.reload_server(server)
+    return {"ok": True, "files": written, "tools": mcp_workspace.list_registered_tools(server)}
+
+
+async def attempt_mcp_repair(
+    *,
+    task_id: int,
+    server: str,
+    tool: str,
+    args: dict,
+    issue: str,
+    error: str | None,
+    output: str | None,
+    manager,
+    emit: EmitFn,
+    task_title: str = "",
+    push: bool | None = None,
+) -> dict[str, Any]:
+    """Full repair loop: analyze → patch → reload → optional git push."""
+    if not settings.mcp_development_enabled:
+        return {"ok": False, "skipped": True, "reason": "MCP development disabled"}
+
+    await emit(
+        task_id,
+        "mcp_dev",
+        f"Analyzing MCP '{server}' to fix or add capability for {tool}…",
+        {"server": server, "tool": tool, "phase": "analyze"},
+    )
+
+    proposal = await propose_mcp_patch(
+        server=server,
+        tool=tool,
+        issue=issue,
+        error=error,
+        output=output,
+        task_title=task_title,
+    )
+    if not proposal.get("ok"):
+        await emit(task_id, "mcp_dev", f"MCP patch proposal failed: {proposal.get('error')}", proposal)
+        return proposal
+
+    await emit(
+        task_id,
+        "mcp_dev",
+        proposal.get("summary", "Applying MCP patch…"),
+        {"server": server, "phase": "patch", "files": [f.get("path") for f in proposal.get("files", [])]},
+    )
+
+    await emit(
+        task_id,
+        "mcp_dev",
+        f"Reloading MCP '{server}' with patched source…",
+        {"server": server, "phase": "reload"},
+    )
+
+    reload_result = await apply_and_reload_mcp(server, proposal.get("files") or [], manager)
+    if not reload_result.get("ok"):
+        await emit(task_id, "error", reload_result.get("error", "MCP reload failed"), reload_result)
+        return reload_result
+
+    await emit(
+        task_id,
+        "mcp_dev",
+        f"MCP '{server}' reloaded — {len(reload_result.get('tools') or [])} tools registered",
+        {"server": server, "phase": "reload_done", "tools": reload_result.get("tools")},
+    )
+
+    should_push = push if push is not None else settings.mcp_dev_auto_push
+    push_result: dict[str, Any] | None = None
+    commit_msg = proposal.get("commit_message") or f"agentic-os: fix {tool} on {server}"
+    written = reload_result.get("files") or []
+
+    if should_push and settings.gitea_token:
+        push_result = await mcp_workspace.commit_and_push(server, commit_msg, written)
+        await emit(
+            task_id,
+            "mcp_dev",
+            f"Git push {'ok' if push_result.get('ok') else 'failed'}: {push_result.get('commit') or push_result.get('error')}",
+            {"server": server, "phase": "push", **push_result},
+        )
+    elif settings.gitea_token and written and task_id > 0:
+        from app.database import SessionLocal
+        from app.models.enums import ApprovalStatus
+        from app.models.task import ApprovalRequest
+
+        async with SessionLocal() as db:
+            db.add(
+                ApprovalRequest(
+                    task_id=task_id,
+                    tool_name="mcp_git_push",
+                    tool_args={
+                        "server": server,
+                        "files": written,
+                        "commit_message": commit_msg,
+                    },
+                    risk=proposal.get("summary"),
+                    status=ApprovalStatus.pending,
+                )
+            )
+            await db.commit()
+        await emit(
+            task_id,
+            "approval",
+            f"MCP patch ready — approve to push {server} to Gitea",
+            {"server": server, "tool_name": "mcp_git_push", "files": written},
+        )
+
+    return {
+        "ok": True,
+        "summary": proposal.get("summary"),
+        "reload": reload_result,
+        "push": push_result,
+        "tools": reload_result.get("tools"),
+    }
+
+
+async def maybe_repair_after_tool_failure(
+    *,
+    task_id: int,
+    device: str,
+    server: str,
+    tool: str,
+    args: dict,
+    issue: str,
+    error: str | None,
+    output: str | None,
+    manager,
+    emit: EmitFn,
+    task_title: str = "",
+) -> tuple[dict, bool] | None:
+    """If failure looks MCP-related, try repair once and return retry (res, ok) or None."""
+    if not settings.mcp_development_enabled:
+        return None
+    if not is_mcp_bug_or_gap(tool, error, output):
+        return None
+
+    result = await attempt_mcp_repair(
+        task_id=task_id,
+        server=server,
+        tool=tool,
+        args=args,
+        issue=issue,
+        error=error,
+        output=output,
+        manager=manager,
+        emit=emit,
+        task_title=task_title,
+        push=settings.mcp_dev_auto_push,
+    )
+    if not result.get("ok"):
+        return None
+
+    if result.get("push") and not result["push"].get("ok") and result["push"].get("runtime_only"):
+        await emit(
+            task_id,
+            "log",
+            "MCP patched in runtime — no git remote; copy changes to mcp-servers or Gitea manually.",
+            result["push"],
+        )
+
+    from app.agent.mcp_helpers import tool_result_ok
+    from app.agent.tool_events import tool_event_payload
+    from app.mcp_manager.recovery import call_tool_resilient
+
+    await emit(task_id, "log", f"Retrying {tool} after MCP patch…", {"server": server, "tool": tool})
+    res = await call_tool_resilient(manager, server, tool, args, task_id=task_id, emit=emit)
+    ok = tool_result_ok(res)
+    await emit(
+        task_id,
+        "tool_call",
+        f"{device} › {tool} (after MCP patch)",
+        tool_event_payload(
+            device=device,
+            tool=tool,
+            args=args,
+            status="done",
+            ok=ok,
+            output=res.get("text", ""),
+        ),
+    )
+    return res, ok

backend/app/agent/mcp_helpers.py

@@ -0,0 +1,25 @@
+"""Shared helpers for interpreting MCP tool call results."""
+from __future__ import annotations
+
+
+def tool_result_ok(res: dict) -> bool:
+    """Return True when an MCP tool response indicates success."""
+    if res.get("isError"):
+        return False
+    text = (res.get("text") or "").strip()
+    if not text:
+        return True
+    lower = text.lower()
+    if "unknown tool:" in lower:
+        return False
+    if text.startswith("Error executing tool"):
+        return False
+    if '"status": "error"' in text or '"status":"error"' in text:
+        return False
+    if " api error " in lower and any(code in text for code in ("401", "403", "500")):
+        return False
+    if "authentication failed" in lower:
+        return False
+    if "permission denied" in lower:
+        return False
+    return True

backend/app/agent/pfsense_profiles.py

@@ -0,0 +1,132 @@
+"""pfSense diagnostics — issue-aware playbook selection."""
+from __future__ import annotations
+
+import re
+from typing import Awaitable, Callable
+
+from app.agent.tool_runner import invoke_connect, invoke_tool
+
+EmitFn = Callable[[int, str, str, dict | None], Awaitable[None]]
+
+PFSENSE_MCP = "pfsense-mcp"
+
+ArgBuilder = Callable[[dict], dict]
+
+PFSENSE_DEFAULT_DIAGNOSTICS: list[tuple[str, ArgBuilder]] = [
+    ("pfsense_get_system_status", lambda _c: {}),
+    ("pfsense_list_interfaces", lambda _c: {}),
+    ("pfsense_list_gateways", lambda _c: {}),
+    ("pfsense_get_gateway_status", lambda _c: {}),
+]
+
+PFSENSE_FIREWALL_DIAGNOSTICS: list[tuple[str, ArgBuilder]] = [
+    ("pfsense_list_interfaces", lambda _c: {}),
+    ("pfsense_list_firewall_rules", lambda _c: {"limit": 250}),
+    ("pfsense_list_port_forwards", lambda _c: {"limit": 100}),
+    ("pfsense_list_outbound_nat_mappings", lambda _c: {"limit": 100}),
+    ("pfsense_list_aliases", lambda _c: {"limit": 100}),
+]
+
+_FIREWALL_TERMS = (
+    "firewall",
+    "rule",
+    "rules",
+    "policy",
+    "policies",
+    "filter rule",
+    "nat",
+    "port forward",
+    "port-forward",
+    "outbound nat",
+    "inbound",
+    "acl",
+)
+
+
+def _text_has_term(text: str, term: str) -> bool:
+    term = term.lower().strip()
+    if not term:
+        return False
+    if " " in term:
+        return term in text
+    return bool(re.search(rf"\b{re.escape(term)}\b", text))
+
+
+def wants_firewall_inventory(title: str, issue: str) -> bool:
+    text = f"{title} {issue}".lower()
+    return any(_text_has_term(text, term) for term in _FIREWALL_TERMS)
+
+
+def diagnostics_for_issue(title: str, issue: str) -> list[tuple[str, ArgBuilder]]:
+    if wants_firewall_inventory(title, issue):
+        return list(PFSENSE_FIREWALL_DIAGNOSTICS)
+    return list(PFSENSE_DEFAULT_DIAGNOSTICS)
+
+
+async def run_pfsense_diagnostics(
+    task_id: int,
+    device: dict,
+    manager,
+    emit: EmitFn,
+    *,
+    title: str = "",
+    issue: str = "",
+    max_diagnostics: int = 8,
+) -> list[dict]:
+    """Connect to pfSense and run read-only diagnostics matched to the task."""
+    from app.agent.device_connect import pfsense_connect_args
+
+    results: list[dict] = []
+    label = device.get("catalog_key") or device.get("name") or "pfsense"
+    server = PFSENSE_MCP
+
+    server_state = manager.servers.get(server or "")
+    if not server_state or server_state.status != "loaded":
+        await emit(
+            task_id,
+            "diagnose",
+            f"MCP '{server}' not available for {label}.",
+            {"device": label, "server": server},
+        )
+        return results
+
+    diagnostics = diagnostics_for_issue(title, issue)
+    intent = "firewall inventory" if wants_firewall_inventory(title, issue) else "health"
+
+    await emit(
+        task_id,
+        "connect",
+        f"Connecting to {label} via {server} ({intent})...",
+        {"device": label, "intent": intent},
+    )
+
+    ok = await invoke_connect(
+        task_id, label, server, "pfsense_connect", pfsense_connect_args(device), manager, emit
+    )
+    if not ok:
+        return results
+
+    for tool, builder in diagnostics[:max_diagnostics]:
+        args = builder(device)
+        res, tool_ok = await invoke_tool(
+            task_id,
+            label,
+            server,
+            tool,
+            args,
+            manager,
+            emit,
+            task_title=title,
+            task_issue=issue,
+        )
+        results.append(
+            {
+                "device": label,
+                "tool": tool,
+                "ok": tool_ok,
+                "output": res.get("text", ""),
+                "intent": intent,
+            }
+        )
+
+    return results

backend/app/agent/playbooks.py

@@ -0,0 +1,134 @@
+"""Per-device-type connection and read-only diagnostic playbooks.
+
+Each entry maps a DeviceType to:
+  - mcp_server: default MCP server that drives the device
+  - connect: (tool_name, arg_builder) to establish a session (optional)
+  - diagnostics: list of (tool_name, arg_builder) read-only calls to gather state
+
+arg_builder is a callable(device_ctx) -> dict, where device_ctx contains
+address, port, username, secret.
+"""
+from __future__ import annotations
+
+from typing import Callable
+
+from app.agent.asterisk_profiles import ASTERISK_PLAYBOOKS
+from app.agent.device_connect import pfsense_connect_args, ssh_connect_args
+from app.models.enums import DeviceType
+
+ArgBuilder = Callable[[dict], dict]
+
+
+def _ip(ctx: dict) -> dict:
+    return {"ip": ctx["address"]}
+
+
+def _pfsense_connect(ctx: dict) -> dict:
+    return pfsense_connect_args(ctx)
+
+
+def _ssh(ctx: dict) -> dict:
+    return ssh_connect_args(ctx)
+
+
+PLAYBOOKS: dict[DeviceType, dict] = {
+    DeviceType.pfsense: {
+        "mcp_server": "pfsense-mcp",
+        "connect": ("pfsense_connect", _pfsense_connect),
+        "diagnostics": [
+            ("pfsense_get_system_status", lambda c: {}),
+            ("pfsense_list_gateways", lambda c: {}),
+            ("pfsense_get_gateway_status", lambda c: {}),
+            ("pfsense_list_interfaces", lambda c: {}),
+        ],
+    },
+    DeviceType.proxmox: {
+        # Connect + diagnostics handled by proxmox_profiles.run_proxmox_diagnostics (API → SSH).
+        "mcp_server": "proxmox-mcp",
+        "connect": None,
+        "diagnostics": [],
+    },
+    DeviceType.asterisk: {
+        # Legacy key — prefer asterisk_geneseasx catalog slot (see ASTERISK_PLAYBOOKS).
+        "mcp_server": "ssh-generic-mcp",
+        "connect": ("ssh_connect", _ssh),
+        "diagnostics": [],
+    },
+    # GeneseasX / satbox variants are resolved by catalog_key in playbook_for().
+    DeviceType.debian: {
+        "mcp_server": "ssh-generic-mcp",
+        "connect": (
+            "ssh_connect",
+            lambda c: ssh_connect_args(c),
+        ),
+        "diagnostics": [
+            ("ssh_run", lambda c: {"command": "uptime && uname -a"}),
+            ("ssh_run", lambda c: {"command": "df -h"}),
+            ("ssh_run", lambda c: {"command": "free -m"}),
+            ("ssh_run", lambda c: {"command": "systemctl --failed --no-pager"}),
+            ("ssh_run", lambda c: {"command": "journalctl -p err -n 100 --no-pager"}),
+        ],
+    },
+    DeviceType.geneseasx: {
+        "mcp_server": "ssh-generic-mcp",
+        "connect": (
+            "ssh_connect",
+            lambda c: ssh_connect_args(c),
+        ),
+        "diagnostics": [
+            ("ssh_run", lambda c: {"command": "uptime && uname -a"}),
+            ("ssh_run", lambda c: {"command": "docker ps --format '{{.Names}}: {{.Status}}'"}),
+            ("ssh_run", lambda c: {"command": "systemctl --failed --no-pager"}),
+        ],
+    },
+    DeviceType.fortigate: {
+        "mcp_server": "fortigate-mcp",
+        "connect": (
+            "fortigate_connect",
+            lambda c: {
+                "host": c["address"],
+                "api_key": c.get("secret"),
+                "username": c.get("username"),
+            },
+        ),
+        "diagnostics": [
+            ("fortigate_get_system_status", lambda c: {}),
+            ("fortigate_list_interfaces", lambda c: {}),
+            ("fortigate_list_policies", lambda c: {}),
+            ("fortigate_get_logs", lambda c: {"lines": 100}),
+        ],
+    },
+    DeviceType.fortiswitch: {
+        "mcp_server": "fortiswitch-mcp",
+        "connect": (
+            "fortiswitch_connect",
+            lambda c: {
+                "host": c["address"],
+                "username": c.get("username"),
+                "password": c.get("secret"),
+            },
+        ),
+        "diagnostics": [
+            ("fortiswitch_get_system_status", lambda c: {}),
+            ("fortiswitch_list_ports", lambda c: {}),
+            ("fortiswitch_get_port_stats", lambda c: {}),
+        ],
+    },
+}
+
+
+def playbook_for(
+    device_type: DeviceType,
+    mcp_override: str | None,
+    catalog_key: str | None = None,
+) -> dict | None:
+    if catalog_key and catalog_key in ASTERISK_PLAYBOOKS:
+        pb = ASTERISK_PLAYBOOKS[catalog_key]
+    else:
+        pb = PLAYBOOKS.get(device_type)
+    if pb is None:
+        return None
+    # Catalog playbooks define the MCP server; ignore stale device-row overrides.
+    if mcp_override and catalog_key not in ASTERISK_PLAYBOOKS and device_type != DeviceType.proxmox:
+        pb = {**pb, "mcp_server": mcp_override}
+    return pb

backend/app/agent/proxmox_profiles.py

@@ -0,0 +1,596 @@
+"""Proxmox connect strategy: REST API (proxmox-mcp) first, SSH fallback (ssh-generic-mcp)."""
+from __future__ import annotations
+
+import json
+import re
+from typing import Awaitable, Callable
+
+from app.agent.mcp_helpers import tool_result_ok
+from app.agent.tool_runner import invoke_tool
+from app.services.device_defaults import proxmox_api_configured, resolve_proxmox_api_defaults
+
+EmitFn = Callable[[int, str, str, dict | None], Awaitable[None]]
+
+PROXMOX_MCP = "proxmox-mcp"
+SSH_MCP = "ssh-generic-mcp"
+
+MAX_VM_INVENTORY_CALLS = 40
+
+# Node-less probe — VM/LXC calls need a resolved node name.
+PROXMOX_API_DIAGNOSTICS: list[tuple[str, Callable[[dict], dict]]] = [
+    ("proxmox_list_nodes", lambda _c: {}),
+]
+
+# Minimal per-node summary (health-style tasks).
+PROXMOX_API_NODE_SUMMARY: list[str] = [
+    "proxmox_list_qemu",
+    "proxmox_list_lxc",
+    "proxmox_get_node_status",
+]
+
+PROXMOX_SSH_DIAGNOSTICS: list[tuple[str, Callable[[dict], dict]]] = [
+    ("ssh_run", lambda _c: {"command": "pvesh get /nodes --output-format json 2>/dev/null | head -c 4000 || true"}),
+    ("ssh_run", lambda _c: {"command": "qm list 2>/dev/null || true"}),
+    ("ssh_run", lambda _c: {"command": "pct list 2>/dev/null || true"}),
+    ("ssh_run", lambda _c: {"command": "uptime && free -m && df -h /"}),
+]
+
+_VM_INVENTORY_TERMS = (
+    "config",
+    "configuration",
+    "each vm",
+    "every vm",
+    "all vm",
+    "all vms",
+    "running vm",
+    "running vms",
+    "guest",
+    "inventory",
+    "vm status",
+    "vm list",
+    "list vm",
+    "list vms",
+    "qemu",
+    "lxc",
+)
+
+
+def enrich_proxmox_device(device: dict) -> dict:
+    """Attach Proxmox API env defaults; URL uses vessel public IP (:8006) unless overridden in .env."""
+    public_ip = device.get("address") or ""
+    api = resolve_proxmox_api_defaults(public_ip)
+    return {**device, **api}
+
+
+def _text_has_term(text: str, term: str) -> bool:
+    term = term.lower().strip()
+    if not term:
+        return False
+    if " " in term:
+        return term in text
+    if len(term) <= 2:
+        return term in text.split()
+    return bool(re.search(rf"\b{re.escape(term)}\b", text))
+
+
+def wants_vm_inventory(title: str, issue: str) -> bool:
+    """True when the user asked for per-VM status and/or config details."""
+    text = f"{title} {issue}".lower()
+    if any(_text_has_term(text, term) for term in _VM_INVENTORY_TERMS):
+        return True
+    # "VM" / "VMs" in title or issue (e.g. "Proxmox VM status").
+    return bool(re.search(r"\bvms?\b", text))
+
+
+def _ssh_connect_args(ctx: dict) -> dict:
+    return {
+        "host": ctx["address"],
+        "port": ctx.get("port") or 22,
+        "username": ctx.get("username") or "root",
+        "password": ctx.get("secret"),
+        "name": ctx.get("catalog_key") or "proxmox",
+    }
+
+
+def _api_connect_args(ctx: dict) -> dict:
+    return {
+        "url": ctx["proxmox_api_url"],
+        "token_id": ctx["proxmox_token_id"],
+        "token_secret": ctx["proxmox_token_secret"],
+        "verify_ssl": bool(ctx.get("proxmox_verify_ssl")),
+        "name": ctx.get("catalog_key") or "proxmox",
+        "set_active": True,
+    }
+
+
+async def _server_ready(manager, server: str) -> bool:
+    state = manager.servers.get(server or "")
+    return bool(server and state and state.status == "loaded")
+
+
+async def _try_proxmox_api(manager, device: dict) -> bool:
+    if not proxmox_api_configured(device):
+        return False
+    if not await _server_ready(manager, PROXMOX_MCP):
+        return False
+
+    connect_res = await manager.call_tool(PROXMOX_MCP, "proxmox_connect", _api_connect_args(device))
+    if not tool_result_ok(connect_res):
+        return False
+
+    probe = await manager.call_tool(PROXMOX_MCP, "proxmox_list_nodes", {})
+    if tool_result_ok(probe):
+        return True
+
+    name = device.get("catalog_key") or "proxmox"
+    try:
+        await manager.call_tool(PROXMOX_MCP, "proxmox_disconnect_target", {"name": name})
+    except Exception:  # noqa: BLE001
+        pass
+    return False
+
+
+async def _try_proxmox_ssh(manager, device: dict) -> bool:
+    if not await _server_ready(manager, SSH_MCP):
+        return False
+    if not device.get("secret"):
+        return False
+
+    connect_res = await manager.call_tool(SSH_MCP, "ssh_connect", _ssh_connect_args(device))
+    return tool_result_ok(connect_res)
+
+
+def _first_proxmox_node(output: str) -> str | None:
+    text = (output or "").strip()
+    if not text:
+        return None
+    try:
+        data = json.loads(text)
+        if isinstance(data, list) and data:
+            item = data[0]
+            if isinstance(item, dict):
+                return item.get("node") or item.get("name")
+        if isinstance(data, dict):
+            nodes = data.get("data") or data.get("nodes")
+            if isinstance(nodes, list) and nodes:
+                item = nodes[0]
+                if isinstance(item, dict):
+                    return item.get("node") or item.get("name")
+    except json.JSONDecodeError:
+        pass
+    match = re.search(r'"node"\s*:\s*"([^"]+)"', text)
+    return match.group(1) if match else None
+
+
+def _parse_qemu_vms(output: str) -> list[dict]:
+    """Extract VM entries from proxmox_list_qemu JSON or `qm list` text."""
+    text = (output or "").strip()
+    if not text:
+        return []
+
+    parsed = None
+    try:
+        parsed = json.loads(text)
+    except json.JSONDecodeError:
+        start, end = text.find("["), text.rfind("]")
+        if start != -1 and end > start:
+            try:
+                parsed = json.loads(text[start : end + 1])
+            except json.JSONDecodeError:
+                parsed = None
+
+    rows: list[dict] = []
+    if isinstance(parsed, list):
+        for item in parsed:
+            if isinstance(item, dict) and item.get("vmid") is not None:
+                rows.append(item)
+        if rows:
+            return rows
+    if isinstance(parsed, dict):
+        data = parsed.get("data")
+        if isinstance(data, list):
+            for item in data:
+                if isinstance(item, dict) and item.get("vmid") is not None:
+                    rows.append(item)
+            if rows:
+                return rows
+
+    # `qm list` tabular output
+    for line in text.splitlines():
+        line = line.strip()
+        if not line or line.startswith("VMID") or line.startswith("-"):
+            continue
+        parts = line.split()
+        if not parts or not parts[0].isdigit():
+            continue
+        vmid = int(parts[0])
+        status = parts[2] if len(parts) > 2 else ""
+        name = parts[1] if len(parts) > 1 else ""
+        rows.append({"vmid": vmid, "name": name, "status": status})
+
+    return rows
+
+
+def _api_guest_inventory_empty(results: list[dict]) -> bool:
+    """True when API list_qemu/list_lxc ran but returned no guests (common RBAC gap on /vms)."""
+    qemu_out = next((r.get("output") for r in results if r.get("tool") == "proxmox_list_qemu"), None)
+    lxc_out = next((r.get("output") for r in results if r.get("tool") == "proxmox_list_lxc"), None)
+    if qemu_out is None and lxc_out is None:
+        return False
+    qemu_empty = not _parse_qemu_vms(qemu_out or "")
+    lxc_empty = not _parse_qemu_vms(lxc_out or "")
+    return qemu_empty and lxc_empty
+
+
+def _vms_for_detail(vms: list[dict], title: str, issue: str) -> list[dict]:
+    """Pick which guests need per-VM status/config calls."""
+    text = f"{title} {issue}".lower()
+    running_only = "running" in text and "all vm" not in text and "every vm" not in text
+    if running_only:
+        picked = [v for v in vms if str(v.get("status", "")).lower() == "running"]
+        return picked or vms
+    return vms
+
+
+async def _append_tool_result(
+    results: list[dict],
+    *,
+    task_id: int,
+    label: str,
+    server: str,
+    tool: str,
+    args: dict,
+    manager,
+    emit: EmitFn,
+    mode: str,
+    intent: str,
+    task_title: str,
+    task_issue: str,
+) -> bool:
+    """Run one tool and append to results. Returns False if max budget exhausted."""
+    res, tool_ok = await invoke_tool(
+        task_id,
+        label,
+        server,
+        tool,
+        args,
+        manager,
+        emit,
+        task_title=task_title,
+        task_issue=task_issue,
+    )
+    results.append(
+        {
+            "device": label,
+            "tool": tool,
+            "ok": tool_ok,
+            "output": res.get("text", ""),
+            "connect_mode": mode,
+            "intent": intent,
+            "vmid": args.get("vmid"),
+        }
+    )
+    return True
+
+
+async def _run_api_node_tools(
+    results: list[dict],
+    *,
+    task_id: int,
+    label: str,
+    server: str,
+    node: str,
+    manager,
+    emit: EmitFn,
+    mode: str,
+    intent: str,
+    task_title: str,
+    task_issue: str,
+    max_diagnostics: int,
+    vm_inventory: bool,
+) -> None:
+    node_tools = ["proxmox_list_qemu", "proxmox_list_lxc"]
+    if not vm_inventory:
+        node_tools.append("proxmox_get_node_status")
+
+    qemu_output = ""
+    for tool in node_tools:
+        if len(results) >= max_diagnostics:
+            break
+        await _append_tool_result(
+            results,
+            task_id=task_id,
+            label=label,
+            server=server,
+            tool=tool,
+            args={"node": node},
+            manager=manager,
+            emit=emit,
+            mode=mode,
+            intent=intent,
+            task_title=task_title,
+            task_issue=task_issue,
+        )
+        if tool == "proxmox_list_qemu":
+            qemu_output = results[-1].get("output") or ""
+
+    if not vm_inventory or len(results) >= max_diagnostics:
+        return
+
+    vms = _vms_for_detail(_parse_qemu_vms(qemu_output), task_title, task_issue)
+    for vm in vms[:15]:
+        vmid = vm.get("vmid")
+        if vmid is None or len(results) >= max_diagnostics:
+            break
+        for tool in ("proxmox_get_qemu_status", "proxmox_get_qemu_config"):
+            if len(results) >= max_diagnostics:
+                break
+            await _append_tool_result(
+                results,
+                task_id=task_id,
+                label=label,
+                server=server,
+                tool=tool,
+                args={"node": node, "vmid": int(vmid)},
+                manager=manager,
+                emit=emit,
+                mode=mode,
+                intent=intent,
+                task_title=task_title,
+                task_issue=task_issue,
+            )
+
+
+async def _run_ssh_inventory_fallback(
+    results: list[dict],
+    *,
+    task_id: int,
+    label: str,
+    device: dict,
+    manager,
+    emit: EmitFn,
+    intent: str,
+    task_title: str,
+    task_issue: str,
+    max_diagnostics: int,
+    vm_inventory: bool,
+) -> bool:
+    """Use SSH qm/pct when API connected but guest lists are empty (token lacks /vms VM.Audit)."""
+    ctx = enrich_proxmox_device(device)
+    if not ctx.get("secret"):
+        return False
+    if not await _server_ready(manager, SSH_MCP):
+        return False
+    if not await _try_proxmox_ssh(manager, ctx):
+        return False
+
+    await emit(
+        task_id,
+        "connect",
+        f"{label}: API guest list empty (token likely lacks /vms VM.Audit) — using SSH",
+        {"device": label, "mode": "ssh_fallback"},
+    )
+
+    for command in ("qm list 2>/dev/null || true", "pct list 2>/dev/null || true"):
+        if len(results) >= max_diagnostics:
+            break
+        await _append_tool_result(
+            results,
+            task_id=task_id,
+            label=label,
+            server=SSH_MCP,
+            tool="ssh_run",
+            args={"command": command},
+            manager=manager,
+            emit=emit,
+            mode="ssh",
+            intent=intent,
+            task_title=task_title,
+            task_issue=task_issue,
+        )
+
+    if vm_inventory:
+        await _run_ssh_vm_inventory(
+            results,
+            task_id=task_id,
+            label=label,
+            server=SSH_MCP,
+            manager=manager,
+            emit=emit,
+            mode="ssh",
+            intent=intent,
+            task_title=task_title,
+            task_issue=task_issue,
+            max_diagnostics=max_diagnostics,
+        )
+    return True
+
+
+async def _run_ssh_vm_inventory(
+    results: list[dict],
+    *,
+    task_id: int,
+    label: str,
+    server: str,
+    manager,
+    emit: EmitFn,
+    mode: str,
+    intent: str,
+    task_title: str,
+    task_issue: str,
+    max_diagnostics: int,
+) -> None:
+    """After `qm list`, fetch `qm config` for each selected guest."""
+    qemu_output = ""
+    for r in results:
+        if r.get("tool") != "ssh_run":
+            continue
+        out = r.get("output") or ""
+        if re.search(r"^\s*\d+\s+\S+\s+(running|stopped)", out, re.MULTILINE):
+            qemu_output = out
+            break
+
+    vms = _vms_for_detail(_parse_qemu_vms(qemu_output), task_title, task_issue)
+    for vm in vms[:15]:
+        vmid = vm.get("vmid")
+        if vmid is None or len(results) >= max_diagnostics:
+            break
+        await _append_tool_result(
+            results,
+            task_id=task_id,
+            label=label,
+            server=server,
+            tool="ssh_run",
+            args={"command": f"qm config {int(vmid)} 2>/dev/null || true"},
+            manager=manager,
+            emit=emit,
+            mode=mode,
+            intent=intent,
+            task_title=task_title,
+            task_issue=task_issue,
+        )
+
+
+async def run_proxmox_diagnostics(
+    task_id: int,
+    device: dict,
+    manager,
+    emit: EmitFn,
+    *,
+    max_diagnostics: int = 6,
+    task_title: str = "",
+    task_issue: str = "",
+) -> list[dict]:
+    """Connect via Proxmox API when configured; otherwise or on failure use SSH."""
+    results: list[dict] = []
+    label = device.get("catalog_key") or device.get("name") or "proxmox"
+    ctx = enrich_proxmox_device(device)
+    vm_inventory = wants_vm_inventory(task_title, task_issue)
+    intent = "vm inventory" if vm_inventory else "health"
+    budget = MAX_VM_INVENTORY_CALLS if vm_inventory else max_diagnostics
+
+    mode: str | None = None
+    server: str | None = None
+    diagnostics: list[tuple[str, Callable[[dict], dict]]] = []
+
+    await emit(task_id, "connect", f"Connecting to {label}...", {"device": label})
+
+    if proxmox_api_configured(ctx):
+        await emit(
+            task_id,
+            "connect",
+            f"{label}: trying Proxmox API ({ctx['proxmox_api_url']})...",
+            {"device": label, "mode": "api"},
+        )
+        if await _try_proxmox_api(manager, ctx):
+            mode, server, diagnostics = "api", PROXMOX_MCP, PROXMOX_API_DIAGNOSTICS
+            await emit(
+                task_id,
+                "connect",
+                f"{label}: Proxmox API connected ({intent})",
+                {"device": label, "mode": "api", "intent": intent},
+            )
+        else:
+            await emit(
+                task_id,
+                "connect",
+                f"{label}: Proxmox API unavailable — falling back to SSH",
+                {"device": label, "mode": "api_failed"},
+            )
+
+    if mode is None:
+        await emit(
+            task_id,
+            "connect",
+            f"{label}: trying SSH on port {ctx.get('port') or 22}...",
+            {"device": label, "mode": "ssh"},
+        )
+        if await _try_proxmox_ssh(manager, ctx):
+            mode, server, diagnostics = "ssh", SSH_MCP, PROXMOX_SSH_DIAGNOSTICS
+            await emit(
+                task_id,
+                "connect",
+                f"{label}: SSH connected ({intent})",
+                {"device": label, "mode": "ssh", "intent": intent},
+            )
+        else:
+            await emit(
+                task_id,
+                "connect",
+                f"{label}: connect failed (no working API token or SSH credentials)",
+                {"device": label},
+            )
+            return results
+
+    for tool, builder in diagnostics[:budget]:
+        args = builder(ctx)
+        res, tool_ok = await invoke_tool(
+            task_id,
+            label,
+            server,
+            tool,
+            args,
+            manager,
+            emit,
+            task_title=task_title,
+            task_issue=task_issue,
+        )
+        results.append(
+            {
+                "device": label,
+                "tool": tool,
+                "ok": tool_ok,
+                "output": res.get("text", ""),
+                "connect_mode": mode,
+                "intent": intent,
+            }
+        )
+
+    if mode == "api":
+        node = _first_proxmox_node(next((r["output"] for r in results if r["tool"] == "proxmox_list_nodes"), ""))
+        if node:
+            await _run_api_node_tools(
+                results,
+                task_id=task_id,
+                label=label,
+                server=server,
+                node=node,
+                manager=manager,
+                emit=emit,
+                mode=mode,
+                intent=intent,
+                task_title=task_title,
+                task_issue=task_issue,
+                max_diagnostics=budget,
+                vm_inventory=vm_inventory,
+            )
+            if _api_guest_inventory_empty(results):
+                await _run_ssh_inventory_fallback(
+                    results,
+                    task_id=task_id,
+                    label=label,
+                    device=device,
+                    manager=manager,
+                    emit=emit,
+                    intent=intent,
+                    task_title=task_title,
+                    task_issue=task_issue,
+                    max_diagnostics=budget,
+                    vm_inventory=vm_inventory,
+                )
+    elif mode == "ssh" and vm_inventory:
+        await _run_ssh_vm_inventory(
+            results,
+            task_id=task_id,
+            label=label,
+            server=server,
+            manager=manager,
+            emit=emit,
+            mode=mode,
+            intent=intent,
+            task_title=task_title,
+            task_issue=task_issue,
+            max_diagnostics=budget,
+        )
+
+    return results

backend/app/agent/tool_events.py

@@ -0,0 +1,60 @@
+"""Build safe payloads for tool_call / tool_start events."""
+from __future__ import annotations
+
+from app.services.diagnostic_format import MAX_EVENT_OUTPUT
+
+_SECRET_KEYS = frozenset(
+    {
+        "password",
+        "secret",
+        "api_key",
+        "token_secret",
+        "token",
+        "private_key",
+    }
+)
+
+
+def sanitize_tool_args(args: dict | None) -> dict:
+    if not args:
+        return {}
+    out: dict = {}
+    for key, val in args.items():
+        if key.lower() in _SECRET_KEYS:
+            out[key] = "••••"
+        elif isinstance(val, str) and len(val) > 500:
+            out[key] = val[:500] + "…"
+        else:
+            out[key] = val
+    return out
+
+
+def tool_event_payload(
+    *,
+    device: str,
+    tool: str,
+    args: dict | None = None,
+    status: str = "done",
+    ok: bool | None = None,
+    output: str | None = None,
+    error: str | None = None,
+    **extra,
+) -> dict:
+    safe = sanitize_tool_args(args)
+    payload: dict = {
+        "device": device,
+        "tool": tool,
+        "status": status,
+        **extra,
+    }
+    if safe:
+        payload["args"] = safe
+    if safe.get("command"):
+        payload["command"] = safe["command"]
+    if ok is not None:
+        payload["ok"] = ok
+    if output is not None:
+        payload["output"] = output[:MAX_EVENT_OUTPUT]
+    if error is not None:
+        payload["error"] = error
+    return payload

backend/app/agent/tool_runner.py

@@ -0,0 +1,165 @@
+"""Run MCP tools with live events (start → done) and auto-reload on failure."""
+from __future__ import annotations
+
+from collections.abc import Awaitable, Callable
+
+from app.agent.mcp_helpers import tool_result_ok
+from app.agent.tool_events import tool_event_payload
+from app.mcp_manager.recovery import call_tool_resilient
+from app.services.events import get_redis
+from app.services.task_runtime import ensure_not_cancelled
+
+EmitFn = Callable[[int, str, str, dict | None], Awaitable[None]]
+
+
+def _repair_key(task_id: int, server: str, tool: str) -> str:
+    return f"agentic:task:{task_id}:mcp_repair:{server}:{tool}"
+
+
+async def _repair_attempted(task_id: int, server: str, tool: str) -> bool:
+    return bool(await get_redis().get(_repair_key(task_id, server, tool)))
+
+
+async def _mark_repair_attempted(task_id: int, server: str, tool: str) -> None:
+    await get_redis().set(_repair_key(task_id, server, tool), "1", ex=86400)
+
+
+async def invoke_tool(
+    task_id: int,
+    device: str,
+    server: str,
+    tool: str,
+    args: dict,
+    manager,
+    emit: EmitFn,
+    *,
+    task_title: str = "",
+    task_issue: str = "",
+) -> tuple[dict, bool]:
+    await ensure_not_cancelled(task_id)
+    await emit(
+        task_id,
+        "tool_start",
+        f"{device} › {tool}",
+        tool_event_payload(device=device, tool=tool, args=args, status="running"),
+    )
+    try:
+        res = await call_tool_resilient(
+            manager, server, tool, args, task_id=task_id, emit=emit
+        )
+        ok = tool_result_ok(res)
+        if not ok:
+            retry = await _maybe_mcp_repair(
+                task_id,
+                device,
+                server,
+                tool,
+                args,
+                res.get("text"),
+                manager,
+                emit,
+                task_title=task_title,
+                task_issue=task_issue,
+            )
+            if retry is not None:
+                return retry
+        await emit(
+            task_id,
+            "tool_call",
+            f"{device} › {tool}",
+            tool_event_payload(
+                device=device,
+                tool=tool,
+                args=args,
+                status="done",
+                ok=ok,
+                output=res.get("text", ""),
+            ),
+        )
+        return res, ok
+    except Exception as exc:
+        retry = await _maybe_mcp_repair(
+            task_id,
+            device,
+            server,
+            tool,
+            args,
+            str(exc),
+            manager,
+            emit,
+            task_title=task_title,
+            task_issue=task_issue,
+            error=str(exc),
+        )
+        if retry is not None:
+            return retry
+        await emit(
+            task_id,
+            "tool_call",
+            f"{device} › {tool} failed",
+            tool_event_payload(
+                device=device,
+                tool=tool,
+                args=args,
+                status="error",
+                ok=False,
+                error=str(exc),
+            ),
+        )
+        return {"text": str(exc), "isError": True}, False
+
+
+async def _maybe_mcp_repair(
+    task_id: int,
+    device: str,
+    server: str,
+    tool: str,
+    args: dict,
+    output: str | None,
+    manager,
+    emit: EmitFn,
+    *,
+    task_title: str,
+    task_issue: str,
+    error: str | None = None,
+) -> tuple[dict, bool] | None:
+    if await _repair_attempted(task_id, server, tool):
+        return None
+    from app.agent.mcp_development import maybe_repair_after_tool_failure
+
+    await _mark_repair_attempted(task_id, server, tool)
+    return await maybe_repair_after_tool_failure(
+        task_id=task_id,
+        device=device,
+        server=server,
+        tool=tool,
+        args=args,
+        issue=task_issue or task_title,
+        error=error,
+        output=output,
+        manager=manager,
+        emit=emit,
+        task_title=task_title,
+    )
+
+
+async def invoke_connect(
+    task_id: int,
+    device: str,
+    server: str,
+    tool: str,
+    args: dict,
+    manager,
+    emit: EmitFn,
+) -> bool:
+    try:
+        res = await call_tool_resilient(
+            manager, server, tool, args, task_id=task_id, emit=emit
+        )
+        ok = tool_result_ok(res)
+        msg = f"{device}: {tool} ok" if ok else f"{device}: {tool} failed — {res.get('text', '')[:200]}"
+        await emit(task_id, "connect", msg, {"output": res.get("text", "")[:400], "ok": ok, "tool": tool})
+        return ok
+    except Exception as exc:
+        await emit(task_id, "connect", f"{device}: {tool} failed — {exc}", {"ok": False, "tool": tool})
+        return False

backend/app/api/__init__.py

@@ -0,0 +1 @@
+

backend/app/api/auth.py

@@ -0,0 +1,37 @@
+"""Authentication endpoints."""
+from __future__ import annotations
+
+from fastapi import APIRouter, Depends, HTTPException, status
+from fastapi.security import OAuth2PasswordRequestForm
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.core.deps import get_current_user
+from app.core.security import create_access_token, verify_password
+from app.database import get_db
+from app.models.user import User
+from app.schemas import Token, UserOut
+
+router = APIRouter(prefix="/api/auth", tags=["auth"])
+
+
+@router.post("/login", response_model=Token)
+async def login(
+    form: OAuth2PasswordRequestForm = Depends(),
+    db: AsyncSession = Depends(get_db),
+):
+    result = await db.execute(select(User).where(User.email == form.username))
+    user = result.scalar_one_or_none()
+    if not user or not verify_password(form.password, user.hashed_password):
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED, detail="Incorrect email or password"
+        )
+    if not user.is_active:
+        raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="User is disabled")
+    token = create_access_token(subject=user.email, role=user.role.value)
+    return Token(access_token=token, role=user.role, email=user.email)
+
+
+@router.get("/me", response_model=UserOut)
+async def me(user: User = Depends(get_current_user)):
+    return user

backend/app/api/balance.py

@@ -0,0 +1,54 @@
+"""API balance / consumption endpoints."""
+from __future__ import annotations
+
+from fastapi import APIRouter, Depends
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.core.deps import require_readonly, require_support
+from app.database import get_db
+from app.models.balance import BalanceSnapshot
+from app.models.user import User
+from app.schemas import BalanceOut
+from app.services import balance as balance_service
+
+router = APIRouter(prefix="/api/balance", tags=["balance"])
+
+
+async def _latest_snapshot(db: AsyncSession, provider: str) -> BalanceSnapshot | None:
+    """Prefer the newest successful snapshot; fall back to latest error row."""
+    res = await db.execute(
+        select(BalanceSnapshot)
+        .where(BalanceSnapshot.provider == provider)
+        .where(BalanceSnapshot.error.is_(None))
+        .where(BalanceSnapshot.remaining.is_not(None))
+        .order_by(BalanceSnapshot.created_at.desc())
+        .limit(1)
+    )
+    snap = res.scalar_one_or_none()
+    if snap:
+        return snap
+    res = await db.execute(
+        select(BalanceSnapshot)
+        .where(BalanceSnapshot.provider == provider)
+        .order_by(BalanceSnapshot.created_at.desc())
+        .limit(1)
+    )
+    return res.scalar_one_or_none()
+
+
+@router.get("", response_model=list[BalanceOut])
+async def latest_balances(db: AsyncSession = Depends(get_db), _: User = Depends(require_readonly)):
+    """Latest snapshot per provider."""
+    out: list[BalanceSnapshot] = []
+    for provider in ("openrouter", "deepseek"):
+        snap = await _latest_snapshot(db, provider)
+        if snap:
+            out.append(snap)
+    return out
+
+
+@router.post("/refresh", response_model=list[BalanceOut])
+async def refresh_balances(db: AsyncSession = Depends(get_db), _: User = Depends(require_support)):
+    await balance_service.poll_once()
+    return await latest_balances(db)  # type: ignore[arg-type]

backend/app/api/devices.py

@@ -0,0 +1,43 @@
+"""Onboard device updates (credentials / port tweaks)."""
+from __future__ import annotations
+
+from fastapi import APIRouter, Depends, HTTPException, status
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.core.crypto import encrypt_secret
+from app.core.deps import require_readonly, require_support
+from app.database import get_db
+from app.models.inventory import Device
+from app.models.user import User
+from app.schemas import DeviceOut, DeviceUpdate
+from app.services.vessel_devices import device_to_out
+
+router = APIRouter(prefix="/api/devices", tags=["devices"])
+
+
+@router.get("", response_model=list[DeviceOut])
+async def list_devices(db: AsyncSession = Depends(get_db), _: User = Depends(require_readonly)):
+    res = await db.execute(select(Device).order_by(Device.vessel_id, Device.name))
+    return [device_to_out(d) for d in res.scalars().all()]
+
+
+@router.patch("/{device_id}", response_model=DeviceOut)
+async def update_device(
+    device_id: int,
+    body: DeviceUpdate,
+    db: AsyncSession = Depends(get_db),
+    _: User = Depends(require_support),
+):
+    res = await db.execute(select(Device).where(Device.id == device_id))
+    device = res.scalar_one_or_none()
+    if not device:
+        raise HTTPException(status_code=404, detail="Device not found")
+    data = body.model_dump(exclude_unset=True)
+    if "secret" in data:
+        device.secret_enc = encrypt_secret(data.pop("secret"))
+    for k, v in data.items():
+        setattr(device, k, v)
+    await db.commit()
+    await db.refresh(device)
+    return device_to_out(device)

backend/app/api/llm.py

@@ -0,0 +1,85 @@
+"""LLM routing configuration API."""
+from __future__ import annotations
+
+from fastapi import APIRouter, Depends, HTTPException
+
+from app.core.deps import require_admin, require_readonly
+from app.models.user import User
+from app.schemas import LlmRoutingConfigIn, LlmRoutingConfigOut, ProviderModelsOut
+from app.services.model_catalog import catalog_by_tier, get_entry
+from app.services.model_config import (
+    LlmRoutingConfig,
+    available_backends,
+    get_routing_config,
+    save_routing_config,
+)
+from app.services.model_discovery import list_all_provider_models
+
+router = APIRouter(prefix="/api/llm", tags=["llm"])
+
+ROUTING_STRATEGY = (
+    "Local Ollama first for triage and reasoning. On escalation, cloud providers are tried "
+    "in order: Gemini API → DeepSeek → OpenRouter. Within each cost tier (economy, premium), "
+    "each provider is attempted before moving to the next tier."
+)
+
+_MODEL_ID_FIELDS = (
+    "local_model_id",
+    "economy_gemini_id",
+    "economy_deepseek_id",
+    "economy_openrouter_id",
+    "premium_gemini_id",
+    "premium_deepseek_id",
+    "premium_openrouter_id",
+)
+
+
+@router.get("/routing", response_model=LlmRoutingConfigOut)
+async def get_llm_routing(_: User = Depends(require_readonly)):
+    cfg = await get_routing_config()
+    return LlmRoutingConfigOut(
+        config=cfg.to_dict(),
+        catalog=catalog_by_tier(),
+        available_backends=available_backends(),
+        strategy=ROUTING_STRATEGY,
+    )
+
+
+@router.get("/models/available", response_model=ProviderModelsOut)
+async def get_available_models(_: User = Depends(require_readonly)):
+    """Live model lists from Ollama, Gemini, DeepSeek, and OpenRouter APIs."""
+    data = await list_all_provider_models()
+    return ProviderModelsOut(providers=data["providers"], routing_order=data["routing_order"])
+
+
+@router.put("/routing", response_model=LlmRoutingConfigOut)
+async def update_llm_routing(body: LlmRoutingConfigIn, _: User = Depends(require_admin)):
+    current = await get_routing_config()
+    data = body.model_dump(exclude_unset=True)
+
+    for field in (
+        *_MODEL_ID_FIELDS,
+        "auto_escalate",
+        "max_tier",
+    ):
+        if field in data and data[field] is not None:
+            setattr(current, field, data[field])
+
+    if body.cloud_provider_order is not None:
+        current.cloud_provider_order = body.cloud_provider_order
+
+    for fid in _MODEL_ID_FIELDS:
+        val = getattr(current, fid)
+        if val and not get_entry(val):
+            raise HTTPException(status_code=400, detail=f"Unknown model id: {val}")
+
+    if current.max_tier not in ("local", "economy", "premium"):
+        raise HTTPException(status_code=400, detail="max_tier must be local, economy, or premium")
+
+    await save_routing_config(current)
+    return LlmRoutingConfigOut(
+        config=current.to_dict(),
+        catalog=catalog_by_tier(),
+        available_backends=available_backends(),
+        strategy=ROUTING_STRATEGY,
+    )

backend/app/api/mcp.py

@@ -0,0 +1,117 @@
+"""MCP server status panel and maintenance."""
+from __future__ import annotations
+
+import json
+
+from fastapi import APIRouter, Depends, HTTPException
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.core.deps import require_admin, require_readonly
+from app.database import get_db
+from app.mcp_manager.commands import enqueue_mcp_command, enqueue_mcp_command_async
+from app.mcp_manager.manager import MCP_STATUS_KEY
+from app.models.enums import TaskStatus
+from app.models.task import Task
+from app.models.user import User
+from app.schemas import McpCommandResult, McpDevelopRequest, McpDevelopResult, McpServerOut
+from app.services.audit import record_audit
+from app.services.events import get_redis, publish_event
+
+router = APIRouter(prefix="/api/mcp", tags=["mcp"])
+
+
+@router.get("/servers", response_model=list[McpServerOut])
+async def list_mcp_servers(_: User = Depends(require_readonly)):
+    raw = await get_redis().get(MCP_STATUS_KEY)
+    if not raw:
+        return []
+    data = json.loads(raw)
+    return [McpServerOut(**s) for s in data]
+
+
+@router.post("/reload-all", response_model=McpCommandResult)
+async def reload_all_mcps(_: User = Depends(require_admin)):
+    result = await enqueue_mcp_command("reload_all")
+    if not result.get("ok"):
+        raise HTTPException(status_code=502, detail=result.get("error", "Reload failed"))
+    return McpCommandResult(**result)
+
+
+@router.post("/servers/{name}/reload", response_model=McpCommandResult)
+async def reload_mcp_server(name: str, _: User = Depends(require_admin)):
+    result = await enqueue_mcp_command("reload", server=name)
+    if not result.get("ok"):
+        raise HTTPException(status_code=502, detail=result.get("error", "Reload failed"))
+    return McpCommandResult(**result)
+
+
+@router.post("/servers/{name}/upgrade", response_model=McpCommandResult)
+async def upgrade_mcp_server(name: str, _: User = Depends(require_admin)):
+    """Git pull (or re-sync local copy) and restart the MCP server."""
+    result = await enqueue_mcp_command("upgrade", server=name)
+    if not result.get("ok"):
+        raise HTTPException(status_code=502, detail=result.get("error", "Upgrade failed"))
+    return McpCommandResult(**result)
+
+
+@router.post("/servers/{name}/develop", response_model=McpDevelopResult)
+async def develop_mcp_server(
+    name: str,
+    body: McpDevelopRequest,
+    db: AsyncSession = Depends(get_db),
+    user: User = Depends(require_admin),
+):
+    """Research and patch an MCP server; progress is streamed on a monitor task."""
+    task_id = body.task_id
+    if task_id:
+        res = await db.execute(select(Task).where(Task.id == task_id))
+        task = res.scalar_one_or_none()
+        if not task:
+            raise HTTPException(status_code=404, detail="Task not found")
+        task.status = TaskStatus.running
+        task.error = None
+        await db.commit()
+    else:
+        task = Task(
+            title=f"MCP develop: {name}",
+            issue=body.issue.strip(),
+            details={"kind": "mcp_develop", "mcp_server": name, "tool": body.tool},
+            status=TaskStatus.running,
+            created_by_id=user.id,
+        )
+        db.add(task)
+        await db.commit()
+        await db.refresh(task)
+        task_id = task.id
+
+    await publish_event(
+        task_id,
+        "log",
+        f"MCP develop queued for {name}",
+        {"server": name, "kind": "mcp_develop"},
+        persist=True,
+    )
+    await enqueue_mcp_command_async(
+        "develop",
+        server=name,
+        issue=body.issue.strip(),
+        tool=body.tool,
+        task_id=task_id,
+        push=body.push,
+        title=f"MCP develop: {name}",
+    )
+    await record_audit(
+        db,
+        user,
+        "mcp.develop",
+        "mcp",
+        task_id,
+        {"server": name, "issue": body.issue[:500]},
+    )
+
+    return McpDevelopResult(
+        ok=True,
+        task_id=task_id,
+        summary=f"Development started — open task #{task_id} to watch live progress.",
+    )

backend/app/api/rules.py

@@ -0,0 +1,69 @@
+"""Editable troubleshooting rules API."""
+from __future__ import annotations
+
+from fastapi import APIRouter, Depends, HTTPException
+
+from app.core.deps import require_admin, require_readonly
+from app.models.user import User
+from app.schemas import TroubleshootingRulesOut, TroubleshootingRulesUpdate
+from app.services.audit import record_audit
+from app.services.troubleshooting_rules import (
+    load_rules,
+    match_rule,
+    read_rules_raw,
+    save_rules_raw,
+)
+from app.database import get_db
+from sqlalchemy.ext.asyncio import AsyncSession
+
+router = APIRouter(prefix="/api/rules", tags=["rules"])
+
+
+@router.get("/troubleshooting", response_model=TroubleshootingRulesOut)
+async def get_troubleshooting_rules(_: User = Depends(require_readonly)):
+    data = load_rules()  # mtime-cached; reloads only when the file changes
+    return TroubleshootingRulesOut(
+        yaml=read_rules_raw(),
+        parsed=data,
+        path_hint="rules/troubleshooting.yaml",
+    )
+
+
+@router.put("/troubleshooting", response_model=TroubleshootingRulesOut)
+async def update_troubleshooting_rules(
+    body: TroubleshootingRulesUpdate,
+    db: AsyncSession = Depends(get_db),
+    user: User = Depends(require_admin),
+):
+    try:
+        parsed = save_rules_raw(body.yaml)
+    except ValueError as exc:
+        raise HTTPException(status_code=400, detail=str(exc)) from exc
+    except Exception as exc:  # noqa: BLE001
+        raise HTTPException(status_code=400, detail=f"Invalid rules: {exc}") from exc
+
+    await record_audit(
+        db,
+        user,
+        "rules.troubleshooting.update",
+        "rules",
+        0,
+        {"bytes": len(body.yaml)},
+    )
+    return TroubleshootingRulesOut(
+        yaml=read_rules_raw(),
+        parsed=parsed,
+        path_hint="rules/troubleshooting.yaml",
+    )
+
+
+@router.post("/troubleshooting/preview")
+async def preview_rule_match(
+    body: dict,
+    _: User = Depends(require_readonly),
+):
+    """Preview which rule and devices match a title + issue (for the Rules UI)."""
+    title = str(body.get("title") or "")
+    issue = str(body.get("issue") or "")
+    matched = match_rule(title, issue)
+    return {"matched": matched.as_dict() if matched else None}

backend/app/api/tasks.py

@@ -0,0 +1,457 @@
+"""Task creation, listing, approvals, and live event streaming."""
+from __future__ import annotations
+
+import asyncio
+import json
+
+from fastapi import APIRouter, Depends, HTTPException, WebSocket, WebSocketDisconnect, status
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy.orm import selectinload
+
+from app.core.deps import require_readonly, require_support
+from app.core.security import decode_access_token
+from app.database import SessionLocal, get_db
+from app.models.enums import ApprovalStatus, TaskStatus
+from app.models.task import ApprovalRequest, Task
+from app.models.user import User
+from app.schemas import (
+    ApprovalDecision,
+    ApprovalOut,
+    TaskCreate,
+    TaskContinue,
+    TaskDetail,
+    TaskEscalateLlm,
+    TaskEventOut,
+    TaskOverviewOut,
+    TaskOut,
+    TaskPreviewIn,
+    TaskPreviewOut,
+)
+from app.services.audit import record_audit
+from app.services.events import enqueue_task, publish_event, subscribe_events
+from app.inventory_catalog import catalog_by_key
+from app.models.inventory import Vessel
+from app.services.task_present import (
+    task_to_detail,
+    task_to_out,
+    tasks_to_out_list,
+    tasks_to_overview_list,
+    vessel_name_map,
+)
+from app.services.task_runtime import CANCELLABLE_STATUSES, clear_task_cancel, request_task_cancel
+from app.services.troubleshooting_rules import select_devices_for_issue
+
+router = APIRouter(prefix="/api/tasks", tags=["tasks"])
+
+EVENT_REPLAY_LIMIT = 400
+
+CHECK_LABELS = {
+    "proxmox": ["VM and LXC status", "Guest configuration", "Host health"],
+    "pfsense": ["System status", "Interfaces", "Gateways", "Firewall/NAT when requested"],
+    "docker_vm": ["Container status", "Host resources"],
+    "asterisk_geneseasx": ["Asterisk health", "SIP registrations", "Channels"],
+    "asterisk_satbox": ["Asterisk health", "SIP registrations"],
+    "fortigate": ["Firewall status"],
+    "fortiswitch": ["Switch status"],
+}
+
+
+def _preview_device_dict(device) -> dict:
+    key = device.catalog_key or device.device_type.value
+    entry = catalog_by_key().get(device.catalog_key or "")
+    return {
+        "id": device.id,
+        "name": device.name,
+        "catalog_key": key,
+        "device_type": device.device_type.value,
+        "label": entry.label if entry else key.replace("_", " ").title(),
+    }
+
+
+def _checks_for_device(key: str | None) -> list[str]:
+    return CHECK_LABELS.get(key or "", ["Health and status checks"])
+
+
+@router.get("", response_model=list[TaskOut])
+async def list_tasks(
+    limit: int = 100,
+    offset: int = 0,
+    db: AsyncSession = Depends(get_db),
+    _: User = Depends(require_readonly),
+):
+    limit = max(1, min(limit, 500))
+    offset = max(0, offset)
+    res = await db.execute(
+        select(Task).order_by(Task.id.desc()).limit(limit).offset(offset)
+    )
+    return await tasks_to_out_list(db, list(res.scalars().all()))
+
+
+@router.get("/overview", response_model=list[TaskOverviewOut])
+async def task_overview(
+    limit: int = 50,
+    db: AsyncSession = Depends(get_db),
+    _: User = Depends(require_readonly),
+):
+    """Compact task data for dashboard cards. No raw diagnostic payloads."""
+    limit = max(1, min(limit, 200))
+    res = await db.execute(select(Task).order_by(Task.id.desc()).limit(limit))
+    return await tasks_to_overview_list(db, list(res.scalars().all()))
+
+
+@router.post("/preview", response_model=TaskPreviewOut)
+async def preview_task_scope(
+    body: TaskPreviewIn,
+    db: AsyncSession = Depends(get_db),
+    _: User = Depends(require_readonly),
+):
+    """Preview the devices/check groups that task creation will select."""
+    res = await db.execute(
+        select(Vessel).options(selectinload(Vessel.devices)).where(Vessel.id == body.vessel_id)
+    )
+    vessel = res.scalar_one_or_none()
+    if not vessel:
+        raise HTTPException(status_code=404, detail="Vessel not found")
+    device_dicts = [_preview_device_dict(d) for d in vessel.devices]
+    selected, rule = select_devices_for_issue(device_dicts, body.issue, title=body.title)
+    warning = None
+    if not selected:
+        warning = "No onboard device matched this issue. Mention a device type or ask for a full health check."
+    return TaskPreviewOut(
+        vessel_id=vessel.id,
+        vessel_name=vessel.name,
+        devices=[
+            {
+                "catalog_key": d.get("catalog_key") or d.get("device_type") or "device",
+                "name": d.get("name") or d.get("catalog_key") or "device",
+                "label": d.get("label") or (d.get("catalog_key") or "device").replace("_", " ").title(),
+                "checks": _checks_for_device(d.get("catalog_key")),
+            }
+            for d in selected
+        ],
+        rule_id=rule.id if rule else None,
+        rule_name=rule.name if rule else None,
+        severity=rule.severity if rule else None,
+        hints=rule.hints if rule else [],
+        warning=warning,
+    )
+
+
+@router.post("", response_model=TaskOut, status_code=status.HTTP_201_CREATED)
+async def create_task(
+    body: TaskCreate,
+    db: AsyncSession = Depends(get_db),
+    user: User = Depends(require_support),
+):
+    if not body.vessel_id:
+        raise HTTPException(status_code=400, detail="vessel_id is required — pick a vessel; the agent selects devices from the issue")
+    task = Task(
+        title=body.title,
+        issue=body.issue,
+        details=body.details,
+        device_id=None,
+        vessel_id=body.vessel_id,
+        created_by_id=user.id,
+        status=TaskStatus.queued,
+    )
+    db.add(task)
+    await db.commit()
+    await db.refresh(task)
+    await enqueue_task(task.id)
+    await publish_event(task.id, "log", "Task queued.", persist=True)
+    await record_audit(db, user, "task.create", "task", task.id, {"title": task.title})
+    vmap = await vessel_name_map(db, {task.vessel_id} if task.vessel_id else set())
+    return task_to_out(task, vessel_name=vmap.get(task.vessel_id) if task.vessel_id else None)
+
+
+@router.post("/{task_id}/cancel", response_model=TaskOut)
+async def cancel_task(
+    task_id: int,
+    db: AsyncSession = Depends(get_db),
+    user: User = Depends(require_support),
+):
+    """Stop a queued or running task. In-flight work is interrupted at the next checkpoint."""
+    res = await db.execute(select(Task).where(Task.id == task_id))
+    task = res.scalar_one_or_none()
+    if not task:
+        raise HTTPException(status_code=404, detail="Task not found")
+    if task.status.value not in CANCELLABLE_STATUSES:
+        raise HTTPException(
+            status_code=400,
+            detail=f"Cannot stop a task with status '{task.status.value}'",
+        )
+
+    await request_task_cancel(task_id)
+    task.status = TaskStatus.cancelled
+    task.error = None
+    await db.commit()
+    await db.refresh(task)
+    await publish_event(
+        task_id,
+        "cancelled",
+        f"Task stopped by {user.email}.",
+        {"requested_by": user.email},
+        persist=True,
+    )
+    await record_audit(db, user, "task.cancel", "task", task.id)
+    vmap = await vessel_name_map(db, {task.vessel_id} if task.vessel_id else set())
+    return task_to_out(task, vessel_name=vmap.get(task.vessel_id) if task.vessel_id else None)
+
+
+@router.post("/{task_id}/retry", response_model=TaskOut)
+async def retry_task(
+    task_id: int,
+    db: AsyncSession = Depends(get_db),
+    user: User = Depends(require_support),
+):
+    res = await db.execute(select(Task).where(Task.id == task_id))
+    task = res.scalar_one_or_none()
+    if not task:
+        raise HTTPException(status_code=404, detail="Task not found")
+    if task.status in (TaskStatus.running, TaskStatus.queued):
+        raise HTTPException(status_code=400, detail="Task is already running")
+    await clear_task_cancel(task_id)
+    task.status = TaskStatus.queued
+    task.error = None
+    task.report = None
+    task.details = {}
+    await db.commit()
+    await db.refresh(task)
+    await enqueue_task(task.id)
+    await publish_event(task_id, "log", "Task re-queued from scratch (run history cleared).", persist=True)
+    await record_audit(db, user, "task.retry", "task", task.id)
+    return task
+
+
+@router.post("/{task_id}/continue", response_model=TaskOut)
+async def continue_task(
+    task_id: int,
+    body: TaskContinue,
+    db: AsyncSession = Depends(get_db),
+    user: User = Depends(require_support),
+):
+    """Run another investigation pass on the same task with optional follow-up instructions."""
+    res = await db.execute(select(Task).where(Task.id == task_id))
+    task = res.scalar_one_or_none()
+    if not task:
+        raise HTTPException(status_code=404, detail="Task not found")
+    if task.status in (TaskStatus.running, TaskStatus.queued):
+        raise HTTPException(status_code=400, detail="Task is already running")
+    if not body.follow_up.strip():
+        raise HTTPException(status_code=400, detail="follow_up is required — describe what to check next")
+
+    await clear_task_cancel(task_id)
+    details = dict(task.details or {})
+    details["_continue"] = {
+        "follow_up": body.follow_up.strip(),
+        "prior_report": task.report,
+    }
+    task.details = details
+    task.status = TaskStatus.queued
+    task.error = None
+    await db.commit()
+    await db.refresh(task)
+    await enqueue_task(task.id)
+    await publish_event(
+        task_id,
+        "log",
+        f"Task queued for follow-up run: {body.follow_up.strip()[:200]}",
+        persist=True,
+    )
+    await record_audit(db, user, "task.continue", "task", task.id, {"follow_up": body.follow_up[:500]})
+    return task
+
+
+@router.post("/{task_id}/escalate-llm", response_model=TaskOut)
+async def escalate_task_llm(
+    task_id: int,
+    body: TaskEscalateLlm,
+    db: AsyncSession = Depends(get_db),
+    user: User = Depends(require_support),
+):
+    """Request cloud API for reasoning — skips remaining local LLM tiers when possible."""
+    from app.services.task_runtime import set_task_min_tier
+
+    res = await db.execute(select(Task).where(Task.id == task_id))
+    task = res.scalar_one_or_none()
+    if not task:
+        raise HTTPException(status_code=404, detail="Task not found")
+    if task.status != TaskStatus.running:
+        raise HTTPException(status_code=400, detail="Task is not running")
+
+    await set_task_min_tier(task_id, body.tier)
+    label = "Economy" if body.tier == "economy" else "Premium"
+    await publish_event(
+        task_id,
+        "log",
+        f"{label} cloud API requested by {user.email} — will skip local LLM for remaining reasoning steps.",
+        {"min_tier": body.tier, "requested_by": user.email},
+        persist=True,
+    )
+    await record_audit(db, user, "task.escalate_llm", "task", task.id, {"tier": body.tier})
+    await db.refresh(task)
+    return task
+
+
+@router.get("/{task_id}", response_model=TaskDetail)
+async def get_task(
+    task_id: int, db: AsyncSession = Depends(get_db), _: User = Depends(require_readonly)
+):
+    res = await db.execute(
+        select(Task).options(selectinload(Task.events)).where(Task.id == task_id)
+    )
+    task = res.scalar_one_or_none()
+    if not task:
+        raise HTTPException(status_code=404, detail="Task not found")
+    vmap = await vessel_name_map(db, {task.vessel_id} if task.vessel_id else set())
+    return task_to_detail(task, vessel_name=vmap.get(task.vessel_id) if task.vessel_id else None)
+
+
+@router.get("/{task_id}/events", response_model=list[TaskEventOut])
+async def get_task_events(
+    task_id: int, db: AsyncSession = Depends(get_db), _: User = Depends(require_readonly)
+):
+    res = await db.execute(
+        select(Task).options(selectinload(Task.events)).where(Task.id == task_id)
+    )
+    task = res.scalar_one_or_none()
+    if not task:
+        raise HTTPException(status_code=404, detail="Task not found")
+    return task.events
+
+
+@router.get("/{task_id}/approvals", response_model=list[ApprovalOut])
+async def get_approvals(
+    task_id: int, db: AsyncSession = Depends(get_db), _: User = Depends(require_readonly)
+):
+    res = await db.execute(select(ApprovalRequest).where(ApprovalRequest.task_id == task_id))
+    return res.scalars().all()
+
+
+@router.post("/{task_id}/approvals/{approval_id}", response_model=ApprovalOut)
+async def decide_approval(
+    task_id: int,
+    approval_id: int,
+    body: ApprovalDecision,
+    db: AsyncSession = Depends(get_db),
+    user: User = Depends(require_support),
+):
+    res = await db.execute(
+        select(ApprovalRequest).where(
+            ApprovalRequest.id == approval_id, ApprovalRequest.task_id == task_id
+        )
+    )
+    approval = res.scalar_one_or_none()
+    if not approval:
+        raise HTTPException(status_code=404, detail="Approval not found")
+    from datetime import datetime, timezone
+
+    approval.status = ApprovalStatus.approved if body.approve else ApprovalStatus.rejected
+    approval.decided_by_id = user.id
+    approval.decided_at = datetime.now(timezone.utc)
+
+    if body.approve and approval.tool_name == "mcp_git_push" and approval.tool_args:
+        from app.services import mcp_workspace
+
+        args = approval.tool_args
+        push = await mcp_workspace.commit_and_push(
+            args.get("server", ""),
+            args.get("commit_message", "agentic-os: MCP update"),
+            args.get("files") or [],
+        )
+        if not push.get("ok"):
+            if push.get("runtime_only"):
+                raise HTTPException(
+                    status_code=400,
+                    detail=(
+                        push.get("error")
+                        or "Changes exist only in the running MCP — no git repo to push."
+                    ),
+                )
+            raise HTTPException(
+                status_code=502,
+                detail=push.get("error", "Git push failed"),
+            )
+        note = push.get("note") or ""
+        msg = f"MCP pushed to Gitea: {args.get('server')}"
+        if push.get("commit"):
+            msg += f" ({push.get('commit')})"
+        elif note:
+            msg += f" — {note}"
+        await publish_event(
+            task_id,
+            "mcp_dev",
+            msg,
+            push,
+        )
+
+    await db.commit()
+    await db.refresh(approval)
+    await record_audit(
+        db, user, "approval.decide", "approval", approval.id, {"approve": body.approve}
+    )
+    await publish_event(
+        task_id,
+        "approval",
+        f"Approval {approval.id} {'approved' if body.approve else 'rejected'} by {user.email}",
+        {"approval_id": approval.id, "approve": body.approve},
+    )
+
+    # If no more pending approvals, settle task status
+    pend = await db.execute(
+        select(ApprovalRequest).where(
+            ApprovalRequest.task_id == task_id, ApprovalRequest.status == ApprovalStatus.pending
+        )
+    )
+    if pend.scalars().first() is None:
+        tres = await db.execute(select(Task).where(Task.id == task_id))
+        task = tres.scalar_one_or_none()
+        if task and task.status == TaskStatus.waiting_approval:
+            task.status = TaskStatus.succeeded
+            await db.commit()
+    return approval
+
+
+@router.websocket("/{task_id}/stream")
+async def stream_task(websocket: WebSocket, task_id: int):
+    """Live event stream. Authenticate via ?token=<jwt> query param."""
+    token = websocket.query_params.get("token")
+    payload = decode_access_token(token) if token else None
+    if not payload:
+        await websocket.close(code=4401)
+        return
+    await websocket.accept()
+
+    # Replay the most recent persisted events first (bounded), oldest-first.
+    async with SessionLocal() as db:
+        from app.models.task import TaskEvent
+
+        res = await db.execute(
+            select(TaskEvent)
+            .where(TaskEvent.task_id == task_id)
+            .order_by(TaskEvent.id.desc())
+            .limit(EVENT_REPLAY_LIMIT)
+        )
+        recent = list(reversed(res.scalars().all()))
+        for ev in recent:
+            await websocket.send_text(
+                json.dumps(
+                    {
+                        "task_id": task_id,
+                        "kind": ev.kind,
+                        "message": ev.message,
+                        "payload": ev.payload,
+                        "ts": ev.created_at.isoformat(),
+                        "replay": True,
+                    }
+                )
+            )
+
+    try:
+        async for event in subscribe_events(task_id):
+            await websocket.send_text(json.dumps(event))
+    except WebSocketDisconnect:
+        pass
+    except asyncio.CancelledError:
+        pass

backend/app/api/users.py

@@ -0,0 +1,85 @@
+"""User management (admin only)."""
+from __future__ import annotations
+
+from fastapi import APIRouter, Depends, HTTPException, status
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.core.deps import require_admin
+from app.core.security import hash_password
+from app.database import get_db
+from app.models.user import User
+from app.schemas import UserCreate, UserOut, UserUpdate
+from app.services.audit import record_audit
+
+router = APIRouter(prefix="/api/users", tags=["users"])
+
+
+@router.get("", response_model=list[UserOut])
+async def list_users(db: AsyncSession = Depends(get_db), _: User = Depends(require_admin)):
+    res = await db.execute(select(User).order_by(User.id))
+    return res.scalars().all()
+
+
+@router.post("", response_model=UserOut, status_code=status.HTTP_201_CREATED)
+async def create_user(
+    body: UserCreate,
+    db: AsyncSession = Depends(get_db),
+    admin: User = Depends(require_admin),
+):
+    exists = await db.execute(select(User).where(User.email == body.email))
+    if exists.scalar_one_or_none():
+        raise HTTPException(status_code=409, detail="Email already registered")
+    user = User(
+        email=body.email,
+        full_name=body.full_name,
+        hashed_password=hash_password(body.password),
+        role=body.role,
+    )
+    db.add(user)
+    await db.commit()
+    await db.refresh(user)
+    await record_audit(db, admin, "user.create", "user", user.id, {"email": user.email, "role": user.role.value})
+    return user
+
+
+@router.patch("/{user_id}", response_model=UserOut)
+async def update_user(
+    user_id: int,
+    body: UserUpdate,
+    db: AsyncSession = Depends(get_db),
+    admin: User = Depends(require_admin),
+):
+    res = await db.execute(select(User).where(User.id == user_id))
+    user = res.scalar_one_or_none()
+    if not user:
+        raise HTTPException(status_code=404, detail="User not found")
+    if body.full_name is not None:
+        user.full_name = body.full_name
+    if body.role is not None:
+        user.role = body.role
+    if body.is_active is not None:
+        user.is_active = body.is_active
+    if body.password:
+        user.hashed_password = hash_password(body.password)
+    await db.commit()
+    await db.refresh(user)
+    await record_audit(db, admin, "user.update", "user", user.id)
+    return user
+
+
+@router.delete("/{user_id}", status_code=status.HTTP_204_NO_CONTENT)
+async def delete_user(
+    user_id: int,
+    db: AsyncSession = Depends(get_db),
+    admin: User = Depends(require_admin),
+):
+    if user_id == admin.id:
+        raise HTTPException(status_code=400, detail="Cannot delete yourself")
+    res = await db.execute(select(User).where(User.id == user_id))
+    user = res.scalar_one_or_none()
+    if not user:
+        raise HTTPException(status_code=404, detail="User not found")
+    await db.delete(user)
+    await db.commit()
+    await record_audit(db, admin, "user.delete", "user", user_id)

backend/app/api/vessels.py

@@ -0,0 +1,160 @@
+"""Vessel inventory CRUD with onboard device checklist."""
+from __future__ import annotations
+
+from fastapi import APIRouter, Depends, HTTPException, status
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy.orm import selectinload
+
+from app.core.deps import require_readonly, require_support
+from app.core.crypto import decrypt_secret
+from app.database import get_db
+from app.inventory_catalog import catalog_as_dicts
+from app.models.inventory import Device, Vessel
+from app.models.user import User
+from app.schemas import CatalogEntryOut, DeviceOut, VesselCreate, VesselDetail, VesselOut, VesselUpdate
+from app.services.vessel_devices import build_device, device_to_out, validate_selections
+
+router = APIRouter(prefix="/api/vessels", tags=["vessels"])
+
+
+@router.get("/catalog", response_model=list[CatalogEntryOut])
+async def list_device_catalog(_: User = Depends(require_readonly)):
+    """Predefined device/VM slots with default ports for vessel setup."""
+    return catalog_as_dicts()
+
+
+@router.get("", response_model=list[VesselDetail])
+async def list_vessels(db: AsyncSession = Depends(get_db), _: User = Depends(require_readonly)):
+    res = await db.execute(
+        select(Vessel).options(selectinload(Vessel.devices)).order_by(Vessel.name)
+    )
+    vessels = res.scalars().all()
+    out: list[VesselDetail] = []
+    for vessel in vessels:
+        detail = VesselDetail.model_validate(vessel)
+        detail.devices = [device_to_out(d) for d in vessel.devices]
+        out.append(detail)
+    return out
+
+
+@router.get("/{vessel_id}", response_model=VesselDetail)
+async def get_vessel(
+    vessel_id: int, db: AsyncSession = Depends(get_db), _: User = Depends(require_readonly)
+):
+    res = await db.execute(
+        select(Vessel).options(selectinload(Vessel.devices)).where(Vessel.id == vessel_id)
+    )
+    vessel = res.scalar_one_or_none()
+    if not vessel:
+        raise HTTPException(status_code=404, detail="Vessel not found")
+    detail = VesselDetail.model_validate(vessel)
+    detail.devices = [device_to_out(d) for d in vessel.devices]
+    return detail
+
+
+@router.post("", response_model=VesselDetail, status_code=status.HTTP_201_CREATED)
+async def create_vessel(
+    body: VesselCreate,
+    db: AsyncSession = Depends(get_db),
+    _: User = Depends(require_support),
+):
+    try:
+        pairs = validate_selections(body.devices)
+    except ValueError as exc:
+        raise HTTPException(status_code=400, detail=str(exc)) from exc
+
+    if not body.public_ip:
+        needs_ip = any(not s.address for s, _ in pairs)
+        if needs_ip:
+            raise HTTPException(
+                status_code=400,
+                detail="Vessel public IP is required when devices use the shared endpoint",
+            )
+
+    vessel = Vessel(
+        name=body.name,
+        site=body.site,
+        public_ip=body.public_ip,
+        notes=body.notes,
+    )
+    db.add(vessel)
+    await db.flush()
+
+    for sel, entry in pairs:
+        db.add(build_device(vessel, sel, entry))
+
+    await db.commit()
+    await db.refresh(vessel)
+    res = await db.execute(
+        select(Vessel).options(selectinload(Vessel.devices)).where(Vessel.id == vessel.id)
+    )
+    saved = res.scalar_one()
+    detail = VesselDetail.model_validate(saved)
+    detail.devices = [device_to_out(d) for d in saved.devices]
+    return detail
+
+
+@router.patch("/{vessel_id}", response_model=VesselDetail)
+async def update_vessel(
+    vessel_id: int,
+    body: VesselUpdate,
+    db: AsyncSession = Depends(get_db),
+    _: User = Depends(require_support),
+):
+    res = await db.execute(
+        select(Vessel).options(selectinload(Vessel.devices)).where(Vessel.id == vessel_id)
+    )
+    vessel = res.scalar_one_or_none()
+    if not vessel:
+        raise HTTPException(status_code=404, detail="Vessel not found")
+
+    for field in ("name", "site", "public_ip", "notes"):
+        val = getattr(body, field)
+        if val is not None:
+            setattr(vessel, field, val)
+
+    if body.devices is not None:
+        try:
+            pairs = validate_selections(body.devices)
+        except ValueError as exc:
+            raise HTTPException(status_code=400, detail=str(exc)) from exc
+
+        if not vessel.public_ip and any(not s.address for s, _ in pairs):
+            raise HTTPException(status_code=400, detail="Set vessel public IP or per-device addresses")
+
+        # Replace onboard devices from the new checklist
+        old_by_key = {d.catalog_key: d for d in vessel.devices if d.catalog_key}
+        for d in list(vessel.devices):
+            await db.delete(d)
+        await db.flush()
+        for sel, entry in pairs:
+            existing_plain = None
+            if not sel.secret and sel.catalog_key in old_by_key:
+                enc = old_by_key[sel.catalog_key].secret_enc
+                if enc:
+                    existing_plain = decrypt_secret(enc)
+            db.add(build_device(vessel, sel, entry, existing_secret_plain=existing_plain))
+
+    await db.commit()
+    res = await db.execute(
+        select(Vessel).options(selectinload(Vessel.devices)).where(Vessel.id == vessel.id)
+    )
+    saved = res.scalar_one()
+    detail = VesselDetail.model_validate(saved)
+    detail.devices = [device_to_out(d) for d in saved.devices]
+    return detail
+
+
+@router.delete("/{vessel_id}", status_code=status.HTTP_204_NO_CONTENT)
+async def delete_vessel(
+    vessel_id: int,
+    db: AsyncSession = Depends(get_db),
+    _: User = Depends(require_support),
+):
+    res = await db.execute(select(Vessel).where(Vessel.id == vessel_id))
+    vessel = res.scalar_one_or_none()
+    if not vessel:
+        raise HTTPException(status_code=404, detail="Vessel not found")
+    await db.delete(vessel)
+    await db.commit()

backend/app/config.py

@@ -0,0 +1,115 @@
+"""Application configuration, loaded from environment variables."""
+from __future__ import annotations
+
+from functools import lru_cache
+
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class Settings(BaseSettings):
+    model_config = SettingsConfigDict(env_file=".env", extra="ignore", case_sensitive=False)
+
+    # Core security
+    secret_key: str = "dev-insecure-secret-change-me"
+    credential_encryption_key: str = ""
+    access_token_expire_minutes: int = 60 * 12
+    algorithm: str = "HS256"
+
+    first_admin_email: str = "admin@agentic.local"
+    first_admin_password: str = "changeme"
+
+    # Database / cache
+    database_url: str = "postgresql+asyncpg://agentic:agentic@postgres:5432/agentic_os"
+    redis_url: str = "redis://redis:6379/0"
+
+    # CORS: comma-separated allowed browser origins (no wildcard with credentials).
+    cors_allow_origins: str = "http://localhost:3000"
+
+    # LLM routing
+    litellm_base_url: str = "http://litellm:4000"
+    litellm_master_key: str = "sk-litellm-master-change-me"
+    ollama_base_url: str = "http://host.docker.internal:11434"
+    local_triage_model: str = "qwen2.5:7b-instruct"
+    cloud_reasoning_model: str = "openrouter/deepseek/deepseek-chat"
+
+    # Tiered LLM routing (local → gemini → deepseek → openrouter)
+    llm_local_model: str = "qwen2.5:7b-instruct"
+    llm_economy_gemini: str = "gemini-2.5-flash"
+    llm_economy_deepseek: str = "deepseek-chat"
+    llm_economy_openrouter: str = "deepseek/deepseek-chat"
+    llm_premium_gemini: str = "gemini-2.5-pro"
+    llm_premium_deepseek: str = "deepseek-reasoner"
+    llm_premium_openrouter: str = "anthropic/claude-3.5-sonnet"
+    llm_auto_escalate: bool = True
+    llm_max_tier: str = "premium"
+    llm_cloud_provider_order: str = "gemini,deepseek,openrouter"
+
+    # Cloud API keys
+    gemini_api_key: str = ""
+    openrouter_api_key: str = ""
+    openrouter_provisioning_key: str = ""
+    deepseek_api_key: str = ""
+
+    # Balance polling
+    balance_poll_interval_seconds: int = 900
+    min_balance_usd: float = 1.0
+
+    # Gitea / MCP repos
+    gitea_base_url: str = "http://10.77.30.250:3000"
+    gitea_username: str = "nearxos"
+    gitea_token: str = ""
+    mcp_repos: str = "ssh-generic-mcp,fortigate-mcp,fortiswitch-mcp,pfsense-mcp,proxmox-mcp,asterisk-mcp"
+
+    # Obsidian
+    obsidian_vault_repo: str = "homelab-vault"
+    obsidian_tasks_folder: str = "agentic-os-tasks"
+    obsidian_auto_push: bool = True
+
+    # Project memory MCP
+    memory_mcp_url: str = "http://10.77.30.184:8787/mcp"
+    memory_mcp_token: str = ""
+    memory_task_project_id: str = "agentic-os-task"
+    memory_search_projects: str = "agentic-os-task,agentic-os,global"
+    obsidian_search_paths: str = "agentic-os-tasks,Meta/Agent-Memory"
+
+    # Troubleshooting decision rules (YAML, editable via Admin → Rules)
+    troubleshooting_rules_path: str = "/app/rules/troubleshooting.yaml"
+
+    # Runtime paths (inside container)
+    runtime_dir: str = "/runtime"
+
+    # MCP self-development: agent patches MCP source on tool failure / missing capability
+    mcp_development_enabled: bool = True
+    mcp_dev_auto_push: bool = False  # when false, patch applies + reload; git push needs approval
+    mcp_dev_model_tier: str = "premium"
+
+    # Worker: max tasks executed in parallel (MCP sessions are shared — use 1 for same-vessel safety)
+    worker_max_concurrent_tasks: int = 1
+
+    @property
+    def cors_origin_list(self) -> list[str]:
+        return [o.strip() for o in self.cors_allow_origins.split(",") if o.strip()]
+
+    @property
+    def mcp_repo_list(self) -> list[str]:
+        return [r.strip() for r in self.mcp_repos.split(",") if r.strip()]
+
+    def gitea_clone_url(self, repo: str) -> str:
+        """Authenticated clone URL for a Gitea repo (preserves http/https from base URL)."""
+        base = self.gitea_base_url.rstrip("/")
+        if self.gitea_token:
+            from urllib.parse import urlparse
+
+            parsed = urlparse(base if "://" in base else f"http://{base}")
+            scheme = parsed.scheme or "http"
+            host = parsed.netloc or parsed.path
+            return f"{scheme}://{self.gitea_username}:{self.gitea_token}@{host}/{self.gitea_username}/{repo}.git"
+        return f"{base}/{self.gitea_username}/{repo}.git"
+
+
+@lru_cache
+def get_settings() -> Settings:
+    return Settings()
+
+
+settings = get_settings()

backend/app/core/__init__.py

@@ -0,0 +1 @@
+

backend/app/core/crypto.py

@@ -0,0 +1,38 @@
+"""Symmetric encryption of device credentials at rest (Fernet)."""
+from __future__ import annotations
+
+import base64
+import hashlib
+
+from cryptography.fernet import Fernet, InvalidToken
+
+from app.config import settings
+
+
+def _fernet() -> Fernet:
+    key = settings.credential_encryption_key
+    if not key:
+        # Derive a stable (but weak) key from SECRET_KEY so dev works without extra config.
+        digest = hashlib.sha256(settings.secret_key.encode()).digest()
+        key = base64.urlsafe_b64encode(digest).decode()
+    try:
+        return Fernet(key)
+    except (ValueError, TypeError):
+        # Treat provided value as raw material and derive a valid Fernet key
+        digest = hashlib.sha256(key.encode()).digest()
+        return Fernet(base64.urlsafe_b64encode(digest))
+
+
+def encrypt_secret(plaintext: str | None) -> str | None:
+    if not plaintext:
+        return None
+    return _fernet().encrypt(plaintext.encode()).decode()
+
+
+def decrypt_secret(token: str | None) -> str | None:
+    if not token:
+        return None
+    try:
+        return _fernet().decrypt(token.encode()).decode()
+    except InvalidToken:
+        return None

backend/app/core/deps.py

@@ -0,0 +1,57 @@
+"""Auth + RBAC dependencies for FastAPI routes."""
+from __future__ import annotations
+
+from collections.abc import Callable
+
+from fastapi import Depends, HTTPException, status
+from fastapi.security import OAuth2PasswordBearer
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.core.security import decode_access_token
+from app.database import get_db
+from app.models.enums import Role
+from app.models.user import User
+
+oauth2_scheme = OAuth2PasswordBearer(tokenUrl="/api/auth/login")
+
+# Privilege ordering: higher number = more privilege
+_ROLE_RANK = {Role.readonly: 0, Role.support: 1, Role.admin: 2}
+
+
+async def get_current_user(
+    token: str = Depends(oauth2_scheme),
+    db: AsyncSession = Depends(get_db),
+) -> User:
+    creds_error = HTTPException(
+        status_code=status.HTTP_401_UNAUTHORIZED,
+        detail="Could not validate credentials",
+        headers={"WWW-Authenticate": "Bearer"},
+    )
+    payload = decode_access_token(token)
+    if not payload or "sub" not in payload:
+        raise creds_error
+    result = await db.execute(select(User).where(User.email == payload["sub"]))
+    user = result.scalar_one_or_none()
+    if user is None or not user.is_active:
+        raise creds_error
+    return user
+
+
+def require_role(minimum: Role) -> Callable:
+    """Return a dependency that enforces a minimum role."""
+
+    async def _guard(user: User = Depends(get_current_user)) -> User:
+        if _ROLE_RANK[user.role] < _ROLE_RANK[minimum]:
+            raise HTTPException(
+                status_code=status.HTTP_403_FORBIDDEN,
+                detail=f"Requires at least '{minimum.value}' role",
+            )
+        return user
+
+    return _guard
+
+
+require_admin = require_role(Role.admin)
+require_support = require_role(Role.support)
+require_readonly = require_role(Role.readonly)

backend/app/core/security.py

@@ -0,0 +1,32 @@
+"""Password hashing and JWT helpers."""
+from __future__ import annotations
+
+from datetime import datetime, timedelta, timezone
+
+from jose import JWTError, jwt
+from passlib.context import CryptContext
+
+from app.config import settings
+
+pwd_context = CryptContext(schemes=["bcrypt"], deprecated="auto")
+
+
+def hash_password(password: str) -> str:
+    return pwd_context.hash(password)
+
+
+def verify_password(plain: str, hashed: str) -> bool:
+    return pwd_context.verify(plain, hashed)
+
+
+def create_access_token(subject: str, role: str) -> str:
+    expire = datetime.now(timezone.utc) + timedelta(minutes=settings.access_token_expire_minutes)
+    payload = {"sub": subject, "role": role, "exp": expire}
+    return jwt.encode(payload, settings.secret_key, algorithm=settings.algorithm)
+
+
+def decode_access_token(token: str) -> dict | None:
+    try:
+        return jwt.decode(token, settings.secret_key, algorithms=[settings.algorithm])
+    except JWTError:
+        return None

backend/app/database.py

@@ -0,0 +1,65 @@
+"""Async SQLAlchemy engine, session factory, and Base model."""
+from __future__ import annotations
+
+from collections.abc import AsyncGenerator
+
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
+from sqlalchemy.orm import DeclarativeBase
+
+from app.config import settings
+
+engine = create_async_engine(settings.database_url, echo=False, pool_pre_ping=True)
+SessionLocal = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
+
+
+class Base(DeclarativeBase):
+    pass
+
+
+async def get_db() -> AsyncGenerator[AsyncSession, None]:
+    async with SessionLocal() as session:
+        yield session
+
+
+async def init_db() -> None:
+    """Create tables and apply lightweight dev migrations."""
+    from app import models  # noqa: F401
+    from sqlalchemy import text
+
+    async with engine.begin() as conn:
+        await conn.run_sync(Base.metadata.create_all)
+        # Dev migration: locations → vessels (rename legacy table/columns)
+        await conn.execute(
+            text(
+                """
+                DO $$
+                BEGIN
+                  IF EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = 'locations')
+                     AND NOT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = 'vessels')
+                  THEN
+                    ALTER TABLE locations RENAME TO vessels;
+                  END IF;
+                  IF EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'devices' AND column_name = 'location_id')
+                  THEN
+                    ALTER TABLE devices RENAME COLUMN location_id TO vessel_id;
+                  END IF;
+                  IF EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'tasks' AND column_name = 'location_id')
+                  THEN
+                    ALTER TABLE tasks RENAME COLUMN location_id TO vessel_id;
+                  END IF;
+                  IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'devices' AND column_name = 'catalog_key')
+                  THEN
+                    ALTER TABLE devices ADD COLUMN catalog_key VARCHAR(64);
+                  END IF;
+                END $$;
+                """
+            )
+        )
+        # Indexes for hot query paths (idempotent; covers pre-existing DBs).
+        for stmt in (
+            "CREATE INDEX IF NOT EXISTS ix_tasks_vessel_id ON tasks (vessel_id)",
+            "CREATE INDEX IF NOT EXISTS ix_task_events_task_id_id ON task_events (task_id, id)",
+            "CREATE INDEX IF NOT EXISTS ix_approval_requests_task_status "
+            "ON approval_requests (task_id, status)",
+        ):
+            await conn.execute(text(stmt))

backend/app/inventory_catalog.py

@@ -0,0 +1,145 @@
+"""Predefined device/VM slots available when configuring a vessel.
+
+Each catalog entry defines defaults (port, MCP, type). Per-vessel overrides
+are stored on the Device row when the operator enables that slot.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from app.models.enums import DeviceType
+from app.agent.asterisk_profiles import asterisk_container_name
+
+
+@dataclass(frozen=True)
+class CatalogEntry:
+    key: str
+    label: str
+    description: str
+    device_type: DeviceType
+    mcp_server: str | None
+    default_port: int
+    default_username: str | None = None
+    category: str = "core"  # core | network | optional
+
+
+# GeneseasX + common homelab/vessel gear
+VESSEL_DEVICE_CATALOG: tuple[CatalogEntry, ...] = (
+    CatalogEntry(
+        key="proxmox",
+        label="Proxmox host",
+        description="Proxmox VE — API token (proxmox-mcp) with SSH fallback",
+        device_type=DeviceType.proxmox,
+        mcp_server="proxmox-mcp",
+        default_port=22,
+        default_username="root",
+        category="core",
+    ),
+    CatalogEntry(
+        key="pfsense",
+        label="pfSense firewall",
+        description="pfSense REST API (pfSense package)",
+        device_type=DeviceType.pfsense,
+        mcp_server="pfsense-mcp",
+        default_port=40443,
+        category="core",
+    ),
+    CatalogEntry(
+        key="docker_vm",
+        label="Docker VM (GeneseasX)",
+        description="Debian Docker VM hosting GeneseasX services",
+        device_type=DeviceType.geneseasx,
+        mcp_server="ssh-generic-mcp",
+        default_port=22,
+        default_username="root",
+        category="core",
+    ),
+    CatalogEntry(
+        key="asterisk_geneseasx",
+        label="Asterisk VoIP (GeneseasX)",
+        description="Asterisk in Docker container `voip` on the GeneseasX Docker VM (SSH + docker exec)",
+        device_type=DeviceType.asterisk,
+        mcp_server="ssh-generic-mcp",
+        default_port=5022,
+        default_username="root",
+        category="core",
+    ),
+    CatalogEntry(
+        key="asterisk_satbox",
+        label="Asterisk VoIP (TMGeneseas / satbox)",
+        description="Asterisk installed natively on Debian 9/10 host (SSH + asterisk CLI)",
+        device_type=DeviceType.asterisk,
+        mcp_server="ssh-generic-mcp",
+        default_port=22,
+        default_username="root",
+        category="core",
+    ),
+    CatalogEntry(
+        key="debian_host",
+        label="Plain Debian server",
+        description="Standalone Debian 9/10 host (SSH)",
+        device_type=DeviceType.debian,
+        mcp_server="ssh-generic-mcp",
+        default_port=22,
+        default_username="root",
+        category="optional",
+    ),
+    CatalogEntry(
+        key="fortigate",
+        label="FortiGate",
+        description="Standalone FortiGate (FortiOS REST API)",
+        device_type=DeviceType.fortigate,
+        mcp_server="fortigate-mcp",
+        default_port=443,
+        category="network",
+    ),
+    CatalogEntry(
+        key="fortiswitch",
+        label="FortiSwitch",
+        description="FortiSwitch management API",
+        device_type=DeviceType.fortiswitch,
+        mcp_server="fortiswitch-mcp",
+        default_port=443,
+        category="network",
+    ),
+    CatalogEntry(
+        key="tplink",
+        label="TP-Link device",
+        description="TP-Link switch/AP (SSH; MCP pending)",
+        device_type=DeviceType.tplink,
+        mcp_server=None,
+        default_port=22,
+        category="optional",
+    ),
+)
+
+
+def catalog_by_key() -> dict[str, CatalogEntry]:
+    return {e.key: e for e in VESSEL_DEVICE_CATALOG}
+
+
+def catalog_as_dicts() -> list[dict]:
+    from app.services.device_defaults import proxmox_api_configured, resolve_proxmox_api_defaults, resolve_slot_defaults
+
+    rows: list[dict] = []
+    for e in VESSEL_DEVICE_CATALOG:
+        env = resolve_slot_defaults(e.key, e)
+        row = {
+            "key": e.key,
+            "label": e.label,
+            "description": e.description,
+            "device_type": e.device_type.value,
+            "mcp_server": e.mcp_server,
+            "default_port": env.port if env.port is not None else e.default_port,
+            "default_username": env.username or e.default_username,
+            "has_default_secret": bool(env.secret),
+            "secret_kind": env.secret_kind,
+            "env_defaults_configured": env.from_env,
+            "asterisk_container": asterisk_container_name(e.key) or None,
+            "category": e.category,
+        }
+        if e.key == "proxmox":
+            api = resolve_proxmox_api_defaults("")
+            row["has_default_api"] = proxmox_api_configured(api)
+        rows.append(row)
+    return rows

backend/app/main.py

@@ -0,0 +1,65 @@
+"""FastAPI application entrypoint."""
+from __future__ import annotations
+
+import asyncio
+import logging
+from contextlib import asynccontextmanager
+
+from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+from sqlalchemy import select
+
+from app.api import auth, balance, devices, llm, mcp, rules, tasks, users, vessels
+from app.config import settings
+from app.core.security import hash_password
+from app.database import SessionLocal, init_db
+from app.models.enums import Role
+from app.models.user import User
+from app.services.balance import balance_poller_loop
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+async def _seed_admin() -> None:
+    async with SessionLocal() as db:
+        res = await db.execute(select(User).where(User.email == settings.first_admin_email))
+        if res.scalar_one_or_none():
+            return
+        admin = User(
+            email=settings.first_admin_email,
+            full_name="Administrator",
+            hashed_password=hash_password(settings.first_admin_password),
+            role=Role.admin,
+        )
+        db.add(admin)
+        await db.commit()
+        logger.info("Seeded first admin user: %s", settings.first_admin_email)
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    await init_db()
+    await _seed_admin()
+    poller = asyncio.create_task(balance_poller_loop())
+    yield
+    poller.cancel()
+
+
+app = FastAPI(title="Agentic OS", version="0.1.0", lifespan=lifespan)
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=settings.cors_origin_list,
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+for r in (auth.router, users.router, vessels.router, devices.router, tasks.router, mcp.router, balance.router, llm.router, rules.router):
+    app.include_router(r)
+
+
+@app.get("/api/health")
+async def health():
+    return {"status": "ok", "version": app.version}

backend/app/mcp_manager/__init__.py

@@ -0,0 +1,3 @@
+from app.mcp_manager.manager import McpManager, get_manager
+
+__all__ = ["McpManager", "get_manager"]

backend/app/mcp_manager/commands.py

@@ -0,0 +1,191 @@
+"""Redis queue for MCP maintenance commands (worker executes reload/upgrade/develop)."""
+from __future__ import annotations
+
+import asyncio
+import json
+import uuid
+from collections.abc import Awaitable, Callable
+from typing import Any
+
+from app.services.events import get_redis
+
+MCP_COMMAND_QUEUE = "agentic:mcp:commands"
+
+
+def _result_key(request_id: str) -> str:
+    return f"agentic:mcp:cmd_result:{request_id}"
+
+
+async def _wait_result(request_id: str, timeout_seconds: float) -> dict:
+    redis = get_redis()
+    deadline = asyncio.get_event_loop().time() + timeout_seconds
+    while asyncio.get_event_loop().time() < deadline:
+        raw = await redis.get(_result_key(request_id))
+        if raw:
+            await redis.delete(_result_key(request_id))
+            return json.loads(raw)
+        await asyncio.sleep(0.5)
+    return {"ok": False, "error": "Worker did not respond in time — is the worker running?"}
+
+
+async def enqueue_mcp_command(
+    action: str,
+    *,
+    server: str | None = None,
+    timeout_seconds: float = 90,
+    **extra: Any,
+) -> dict:
+    request_id = str(uuid.uuid4())
+    cmd = {"request_id": request_id, "action": action, "server": server, **extra}
+    await get_redis().rpush(MCP_COMMAND_QUEUE, json.dumps(cmd))
+    return await _wait_result(request_id, timeout_seconds)
+
+
+async def enqueue_mcp_command_async(
+    action: str,
+    *,
+    server: str | None = None,
+    **extra: Any,
+) -> str:
+    """Queue a command for the worker without waiting for the result."""
+    request_id = str(uuid.uuid4())
+    cmd = {"request_id": request_id, "action": action, "server": server, **extra}
+    await get_redis().rpush(MCP_COMMAND_QUEUE, json.dumps(cmd))
+    return request_id
+
+
+async def store_mcp_command_result(request_id: str, result: dict) -> None:
+    redis = get_redis()
+    await redis.set(_result_key(request_id), json.dumps(result), ex=300)
+
+
+async def process_mcp_command(cmd: dict) -> dict:
+    from app.mcp_manager.manager import get_manager
+
+    action = cmd.get("action")
+    server = cmd.get("server")
+    manager = get_manager()
+    try:
+        if action == "reload":
+            if not server:
+                raise ValueError("server name required")
+            await manager.reload_server(server)
+        elif action == "upgrade":
+            if not server:
+                raise ValueError("server name required")
+            await manager.upgrade_server(server)
+        elif action == "reload_all":
+            await manager.reload_all()
+        elif action == "develop":
+            if not server:
+                raise ValueError("server name required")
+            return await _execute_develop_command(cmd, manager)
+        else:
+            raise ValueError(f"Unknown action: {action}")
+        return {"ok": True, "action": action, "server": server}
+    except Exception as exc:  # noqa: BLE001
+        return {"ok": False, "action": action, "server": server, "error": str(exc)}
+
+
+async def _execute_develop_command(cmd: dict, manager) -> dict:
+    server = cmd["server"]
+    result = await _develop_on_worker(cmd, manager)
+    task_id = int(cmd.get("task_id") or 0)
+    if task_id:
+        await _finalize_develop_task(task_id, result, server)
+    return result
+
+
+async def run_develop_command_background(cmd: dict, manager) -> None:
+    """Run MCP develop without blocking the worker task queue loop."""
+    request_id = cmd.get("request_id", "")
+    try:
+        result = await _execute_develop_command(cmd, manager)
+        await store_mcp_command_result(request_id, result)
+    except Exception as exc:  # noqa: BLE001
+        await store_mcp_command_result(
+            request_id,
+            {"ok": False, "action": "develop", "server": cmd.get("server"), "error": str(exc)},
+        )
+
+
+async def _finalize_develop_task(task_id: int, result: dict, server: str) -> None:
+    from sqlalchemy import select
+
+    from app.database import SessionLocal
+    from app.models.enums import ApprovalStatus, TaskStatus
+    from app.models.task import ApprovalRequest, Task
+    from app.services.events import publish_event
+
+    async with SessionLocal() as db:
+        res = await db.execute(select(Task).where(Task.id == task_id))
+        task = res.scalar_one_or_none()
+        if not task:
+            return
+
+        pend = await db.execute(
+            select(ApprovalRequest).where(
+                ApprovalRequest.task_id == task_id,
+                ApprovalRequest.status == ApprovalStatus.pending,
+            )
+        )
+        has_pending = pend.scalars().first() is not None
+
+        if result.get("ok"):
+            task.report = {
+                "kind": "mcp_develop",
+                "mcp_server": server,
+                "summary": result.get("summary"),
+                "tools": result.get("tools"),
+                "push": result.get("push"),
+                "resolved": not has_pending,
+            }
+            task.error = None
+            task.status = TaskStatus.waiting_approval if has_pending else TaskStatus.succeeded
+        else:
+            task.status = TaskStatus.failed
+            task.error = result.get("error") or result.get("reason") or "MCP development failed"
+        await db.commit()
+
+    if result.get("ok"):
+        msg = result.get("summary") or f"MCP develop finished for {server}"
+        if has_pending:
+            msg += " — approve git push on this task."
+        await publish_event(task_id, "report", msg, {"kind": "mcp_develop", **result}, persist=True)
+    else:
+        err = result.get("error") or result.get("reason") or "MCP development failed"
+        await publish_event(task_id, "error", err, result, persist=True)
+
+
+async def _develop_on_worker(cmd: dict, manager) -> dict:
+    from app.agent.mcp_development import attempt_mcp_repair
+    from app.services.events import publish_event
+
+    server = cmd["server"]
+    task_id = int(cmd.get("task_id") or 0)
+
+    async def emit(tid: int, kind: str, message: str, payload: dict | None = None):
+        if tid:
+            await publish_event(tid, kind, message, payload)
+
+    if task_id:
+        await emit(
+            task_id,
+            "log",
+            f"MCP develop started for {server}",
+            {"server": server, "kind": "mcp_develop"},
+        )
+
+    return await attempt_mcp_repair(
+        task_id=task_id,
+        server=server,
+        tool=cmd.get("tool") or "unknown",
+        args={},
+        issue=cmd.get("issue") or "",
+        error=cmd.get("error"),
+        output=cmd.get("output"),
+        manager=manager,
+        emit=emit,
+        task_title=cmd.get("title") or f"MCP develop: {server}",
+        push=cmd.get("push"),
+    )

backend/app/mcp_manager/manager.py

@@ -0,0 +1,294 @@
+"""Clone MCP repos from Gitea and run them as stdio MCP subprocess sidecars."""
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+import os
+import shutil
+from contextlib import AsyncExitStack
+from dataclasses import dataclass, field
+
+from app.config import settings
+
+logger = logging.getLogger(__name__)
+
+MCP_STATUS_KEY = "agentic:mcp:status"
+
+
+@dataclass
+class ServerState:
+    name: str
+    status: str = "stopped"
+    tools: list[dict] = field(default_factory=list)
+    error: str | None = None
+    session: object | None = None
+    _stack: AsyncExitStack | None = field(default=None, repr=False)
+
+
+async def ensure_git_remote(repo: str, target: str) -> None:
+    """Point origin at GITEA_BASE_URL — fixes repos cloned when remote was public HTTPS."""
+    if not os.path.isdir(os.path.join(target, ".git")):
+        return
+    url = settings.gitea_clone_url(repo)
+    code, out = await McpManager._run(["git", "remote", "set-url", "origin", url], cwd=target)
+    if code != 0:
+        raise RuntimeError(f"git remote set-url failed: {out[:500]}")
+
+
+class McpManager:
+    def __init__(self) -> None:
+        self.servers: dict[str, ServerState] = {}
+        self._lock = asyncio.Lock()
+        self._started = False
+
+    async def startup(self) -> None:
+        async with self._lock:
+            if self._started:
+                return
+            self._started = True
+            os.makedirs(self.repos_dir, exist_ok=True)
+            for repo in settings.mcp_repo_list:
+                state = ServerState(name=repo)
+                self.servers[repo] = state
+                try:
+                    await self._prepare_repo(repo, state)
+                    await self._start_server(repo, state)
+                except Exception as exc:  # noqa: BLE001
+                    state.status = "error"
+                    state.error = str(exc)
+                    logger.warning("MCP %s failed to start: %s", repo, exc)
+            await self.publish_status()
+
+    async def shutdown(self) -> None:
+        async with self._lock:
+            for state in self.servers.values():
+                await self._stop_server(state)
+
+    @property
+    def repos_dir(self) -> str:
+        return os.path.join(settings.runtime_dir, "mcp-repos")
+
+    @property
+    def local_dir(self) -> str:
+        return os.environ.get("MCP_LOCAL_DIR", "/app/mcp-servers")
+
+    def _is_local_repo(self, repo: str) -> bool:
+        return os.path.isdir(os.path.join(self.local_dir, repo))
+
+    def _repo_dir(self, repo: str) -> str:
+        local_src = os.path.join(self.local_dir, repo)
+        if os.path.isdir(local_src):
+            dst = os.path.join(settings.runtime_dir, "mcp-local", repo)
+            if not os.path.isdir(dst):
+                os.makedirs(os.path.dirname(dst), exist_ok=True)
+                shutil.copytree(local_src, dst)
+            return dst
+        return os.path.join(self.repos_dir, repo)
+
+    async def _sync_local_repo(self, repo: str) -> None:
+        local_src = os.path.join(self.local_dir, repo)
+        if not os.path.isdir(local_src):
+            return
+        dst = os.path.join(settings.runtime_dir, "mcp-local", repo)
+        if os.path.isdir(dst):
+            shutil.rmtree(dst)
+        shutil.copytree(local_src, dst)
+
+    async def _ensure_git_remote(self, repo: str, target: str) -> None:
+        await ensure_git_remote(repo, target)
+
+    async def _prepare_repo(self, repo: str, state: ServerState) -> None:
+        if self._is_local_repo(repo):
+            await self._sync_local_repo(repo)
+            return
+        state.status = "cloning"
+        target = os.path.join(self.repos_dir, repo)
+        url = settings.gitea_clone_url(repo)
+        if os.path.isdir(os.path.join(target, ".git")):
+            await self._ensure_git_remote(repo, target)
+            code, out = await self._run(["git", "pull", "--rebase", "--autostash"], cwd=target)
+            if code != 0:
+                raise RuntimeError(f"git pull failed: {out[:500]}")
+        else:
+            code, out = await self._run(["git", "clone", "--depth", "1", url, target])
+            if code != 0:
+                raise RuntimeError(f"clone failed: {out[:500]}")
+
+    def _detect_start_command(self, repo: str) -> list[str]:
+        override = os.environ.get(f"MCP_START_{repo.upper().replace('-', '_')}")
+        if override:
+            return override.split()
+        target = self._repo_dir(repo)
+        has_pyproject = os.path.isfile(os.path.join(target, "pyproject.toml"))
+        for candidate in ("server.py", "main.py", "app.py", "mcp_server.py", "__main__.py"):
+            if os.path.isfile(os.path.join(target, candidate)):
+                if has_pyproject:
+                    return ["uv", "run", "--directory", target, "python", candidate]
+                return ["uv", "run", "--no-project", "--directory", target, "python", candidate]
+        module = repo.replace("-", "_")
+        if os.path.isdir(os.path.join(target, "src", module)) or os.path.isdir(
+            os.path.join(target, module)
+        ):
+            return ["uv", "run", "--directory", target, "python", "-m", module]
+        return ["uv", "run", "--directory", target, repo]
+
+    async def _stop_server(self, state: ServerState) -> None:
+        if state._stack is not None:
+            try:
+                await state._stack.aclose()
+            except Exception as exc:  # noqa: BLE001
+                logger.debug("stop %s: %s", state.name, exc)
+            state._stack = None
+        state.session = None
+        state.tools = []
+        if state.status not in ("error", "cloning"):
+            state.status = "stopped"
+
+    def _mcp_subprocess_env(self, repo: str) -> dict[str, str]:
+        """Env passed to MCP stdio subprocesses (repo-specific overrides)."""
+        env = {**os.environ}
+        if repo == "proxmox-mcp":
+            verify = os.environ.get("PROXMOX_VERIFY_SSL")
+            if verify is None:
+                verify = os.environ.get("DEVICE_PROXMOX_VERIFY_SSL", "false")
+            env["PROXMOX_VERIFY_SSL"] = verify
+        return env
+
+    async def _start_server(self, repo: str, state: ServerState) -> None:
+        from mcp import ClientSession, StdioServerParameters
+        from mcp.client.stdio import stdio_client
+
+        state.status = "starting"
+        state.error = None
+        target = self._repo_dir(repo)
+        cmd = self._detect_start_command(repo)
+        params = StdioServerParameters(
+            command=cmd[0],
+            args=cmd[1:],
+            cwd=target,
+            env=self._mcp_subprocess_env(repo),
+        )
+        stack = AsyncExitStack()
+        read, write = await stack.enter_async_context(stdio_client(params))
+        session = await stack.enter_async_context(ClientSession(read, write))
+        await asyncio.wait_for(session.initialize(), timeout=90)
+        tools_resp = await session.list_tools()
+        state._stack = stack
+        state.session = session
+        state.tools = [{"name": t.name, "description": t.description} for t in tools_resp.tools]
+        state.status = "loaded"
+        logger.info("MCP %s loaded with %d tools", repo, len(state.tools))
+
+    async def reload_server(self, repo: str) -> None:
+        async with self._lock:
+            state = self.servers.get(repo)
+            if not state:
+                raise ValueError(f"Unknown MCP server: {repo}")
+            await self._stop_server(state)
+            await self._start_server(repo, state)
+            await self.publish_status()
+
+    async def upgrade_server(self, repo: str) -> None:
+        async with self._lock:
+            state = self.servers.get(repo)
+            if not state:
+                raise ValueError(f"Unknown MCP server: {repo}")
+            await self._stop_server(state)
+            await self._prepare_repo(repo, state)
+            await self._start_server(repo, state)
+            await self.publish_status()
+
+    async def reload_all(self) -> None:
+        async with self._lock:
+            for repo in settings.mcp_repo_list:
+                state = self.servers.get(repo)
+                if not state:
+                    continue
+                try:
+                    await self._stop_server(state)
+                    await self._start_server(repo, state)
+                except Exception as exc:  # noqa: BLE001
+                    state.status = "error"
+                    state.error = str(exc)
+            await self.publish_status()
+
+    async def process_command_queue_once(self) -> bool:
+        from app.mcp_manager.commands import (
+            MCP_COMMAND_QUEUE,
+            process_mcp_command,
+            run_develop_command_background,
+            store_mcp_command_result,
+        )
+        from app.services.events import get_redis
+
+        item = await get_redis().lpop(MCP_COMMAND_QUEUE)
+        if not item:
+            return False
+        cmd = json.loads(item)
+        if cmd.get("action") == "develop":
+            asyncio.create_task(run_develop_command_background(cmd, self))
+            return True
+        result = await process_mcp_command(cmd)
+        await store_mcp_command_result(cmd["request_id"], result)
+        return True
+
+    def find_server_for_tool(self, tool_name: str) -> str | None:
+        for name, state in self.servers.items():
+            if any(t["name"] == tool_name for t in state.tools):
+                return name
+        return None
+
+    async def call_tool(self, server: str, tool: str, args: dict) -> dict:
+        state = self.servers.get(server)
+        if not state or state.status != "loaded" or state.session is None:
+            raise RuntimeError(f"MCP server '{server}' is not available")
+        result = await state.session.call_tool(tool, args)  # type: ignore[attr-defined]
+        content = []
+        for block in getattr(result, "content", []) or []:
+            text = getattr(block, "text", None)
+            if text is not None:
+                content.append(text)
+        return {
+            "isError": getattr(result, "isError", False),
+            "text": "\n".join(content),
+            "structured": getattr(result, "structuredContent", None),
+        }
+
+    def status_snapshot(self) -> list[dict]:
+        return [
+            {
+                "name": s.name,
+                "status": s.status,
+                "tools": s.tools,
+                "error": s.error,
+                "source": "local" if self._is_local_repo(s.name) else "git",
+            }
+            for s in self.servers.values()
+        ]
+
+    async def publish_status(self) -> None:
+        from app.services.events import get_redis
+
+        await get_redis().set(MCP_STATUS_KEY, json.dumps(self.status_snapshot()))
+
+    @staticmethod
+    async def _run(cmd: list[str], cwd: str | None = None) -> tuple[int, str]:
+        proc = await asyncio.create_subprocess_exec(
+            *cmd,
+            cwd=cwd,
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.STDOUT,
+        )
+        out, _ = await proc.communicate()
+        return proc.returncode or 0, out.decode(errors="replace")
+
+
+_manager: McpManager | None = None
+
+
+def get_manager() -> McpManager:
+    global _manager
+    if _manager is None:
+        _manager = McpManager()
+    return _manager

backend/app/mcp_manager/recovery.py

@@ -0,0 +1,46 @@
+"""Auto-reload MCP servers when a tool call fails due to server unavailability."""
+from __future__ import annotations
+
+import logging
+from collections.abc import Awaitable, Callable
+
+logger = logging.getLogger(__name__)
+
+EmitFn = Callable[[int, str, str, dict | None], Awaitable[None]]
+
+
+async def call_tool_resilient(
+    manager,
+    server: str,
+    tool: str,
+    args: dict,
+    *,
+    task_id: int | None = None,
+    emit: EmitFn | None = None,
+    max_retries: int = 1,
+) -> dict:
+    """Call MCP tool; on server-unavailable errors, reload that server and retry once."""
+    last_exc: Exception | None = None
+    for attempt in range(max_retries + 1):
+        try:
+            return await manager.call_tool(server, tool, args)
+        except RuntimeError as exc:
+            last_exc = exc
+            msg = str(exc).lower()
+            recoverable = "not available" in msg or "not loaded" in msg
+            if not recoverable or attempt >= max_retries:
+                raise
+            logger.warning("MCP %s unavailable for %s — reloading (attempt %s)", server, tool, attempt + 1)
+            if task_id and emit:
+                await emit(
+                    task_id,
+                    "log",
+                    f"MCP '{server}' unavailable — reloading server and retrying {tool}…",
+                    {"server": server, "tool": tool},
+                )
+            try:
+                await manager.reload_server(server)
+            except Exception as reload_exc:  # noqa: BLE001
+                logger.warning("MCP reload %s failed: %s", server, reload_exc)
+                raise exc from reload_exc
+    raise last_exc or RuntimeError(f"MCP call failed: {server}/{tool}")

backend/app/models/__init__.py

@@ -0,0 +1,17 @@
+"""SQLAlchemy ORM models for Agentic OS."""
+from app.models.audit import AuditLog
+from app.models.balance import BalanceSnapshot
+from app.models.inventory import Device, Vessel
+from app.models.task import ApprovalRequest, Task, TaskEvent
+from app.models.user import User
+
+__all__ = [
+    "User",
+    "Vessel",
+    "Device",
+    "Task",
+    "TaskEvent",
+    "ApprovalRequest",
+    "BalanceSnapshot",
+    "AuditLog",
+]

backend/app/models/audit.py

@@ -0,0 +1,27 @@
+"""Audit log of security-relevant actions."""
+from __future__ import annotations
+
+from datetime import datetime
+
+from sqlalchemy import DateTime, ForeignKey, Integer, String, func
+from sqlalchemy.dialects.postgresql import JSONB
+from sqlalchemy.orm import Mapped, mapped_column
+
+from app.database import Base
+
+
+class AuditLog(Base):
+    __tablename__ = "audit_logs"
+
+    id: Mapped[int] = mapped_column(primary_key=True)
+    actor_id: Mapped[int | None] = mapped_column(
+        ForeignKey("users.id", ondelete="SET NULL"), nullable=True
+    )
+    actor_email: Mapped[str | None] = mapped_column(String(255), nullable=True)
+    action: Mapped[str] = mapped_column(String(128), index=True)
+    target_type: Mapped[str | None] = mapped_column(String(64), nullable=True)
+    target_id: Mapped[int | None] = mapped_column(Integer, nullable=True)
+    detail: Mapped[dict | None] = mapped_column(JSONB, nullable=True)
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), server_default=func.now(), index=True
+    )

backend/app/models/balance.py

@@ -0,0 +1,26 @@
+"""Snapshots of API provider balances/usage."""
+from __future__ import annotations
+
+from datetime import datetime
+
+from sqlalchemy import DateTime, Float, String, func
+from sqlalchemy.dialects.postgresql import JSONB
+from sqlalchemy.orm import Mapped, mapped_column
+
+from app.database import Base
+
+
+class BalanceSnapshot(Base):
+    __tablename__ = "balance_snapshots"
+
+    id: Mapped[int] = mapped_column(primary_key=True)
+    provider: Mapped[str] = mapped_column(String(64), index=True)  # openrouter | deepseek
+    total_credits: Mapped[float | None] = mapped_column(Float, nullable=True)
+    total_usage: Mapped[float | None] = mapped_column(Float, nullable=True)
+    remaining: Mapped[float | None] = mapped_column(Float, nullable=True)
+    currency: Mapped[str] = mapped_column(String(8), default="USD")
+    raw: Mapped[dict | None] = mapped_column(JSONB, nullable=True)
+    error: Mapped[str | None] = mapped_column(String(512), nullable=True)
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), server_default=func.now(), index=True
+    )

backend/app/models/enums.py

@@ -0,0 +1,37 @@
+"""Shared enums."""
+from __future__ import annotations
+
+import enum
+
+
+class Role(str, enum.Enum):
+    admin = "admin"
+    support = "support"
+    readonly = "readonly"
+
+
+class DeviceType(str, enum.Enum):
+    debian = "debian"          # plain Debian/Linux server (ssh-generic-mcp)
+    geneseasx = "geneseasx"    # GeneseasX host
+    proxmox = "proxmox"
+    pfsense = "pfsense"
+    asterisk = "asterisk"
+    fortigate = "fortigate"
+    fortiswitch = "fortiswitch"
+    tplink = "tplink"
+    other = "other"
+
+
+class TaskStatus(str, enum.Enum):
+    queued = "queued"
+    running = "running"
+    waiting_approval = "waiting_approval"
+    succeeded = "succeeded"
+    failed = "failed"
+    cancelled = "cancelled"
+
+
+class ApprovalStatus(str, enum.Enum):
+    pending = "pending"
+    approved = "approved"
+    rejected = "rejected"

backend/app/models/inventory.py

@@ -0,0 +1,48 @@
+"""Vessels and their onboard devices (inventory with encrypted credentials)."""
+from __future__ import annotations
+
+from datetime import datetime
+
+from sqlalchemy import DateTime, Enum, ForeignKey, Integer, String, Text, func
+from sqlalchemy.orm import Mapped, mapped_column, relationship
+
+from app.database import Base
+from app.models.enums import DeviceType
+
+
+class Vessel(Base):
+    __tablename__ = "vessels"
+
+    id: Mapped[int] = mapped_column(primary_key=True)
+    name: Mapped[str] = mapped_column(String(255), index=True)
+    site: Mapped[str | None] = mapped_column(String(255), nullable=True)
+    # Public IP / endpoint used to reach devices on this vessel (port forwards)
+    public_ip: Mapped[str | None] = mapped_column(String(255), nullable=True)
+    notes: Mapped[str | None] = mapped_column(Text, nullable=True)
+    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
+
+    devices: Mapped[list["Device"]] = relationship(
+        back_populates="vessel", cascade="all, delete-orphan"
+    )
+
+
+class Device(Base):
+    __tablename__ = "devices"
+
+    id: Mapped[int] = mapped_column(primary_key=True)
+    name: Mapped[str] = mapped_column(String(255), index=True)
+    # Catalog slot key (e.g. proxmox, pfsense, docker_vm)
+    catalog_key: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
+    device_type: Mapped[DeviceType] = mapped_column(Enum(DeviceType), default=DeviceType.other)
+    address: Mapped[str] = mapped_column(String(255))
+    port: Mapped[int | None] = mapped_column(Integer, nullable=True)
+    mcp_server: Mapped[str | None] = mapped_column(String(128), nullable=True)
+    username: Mapped[str | None] = mapped_column(String(255), nullable=True)
+    secret_enc: Mapped[str | None] = mapped_column(Text, nullable=True)
+    notes: Mapped[str | None] = mapped_column(Text, nullable=True)
+
+    vessel_id: Mapped[int | None] = mapped_column(
+        ForeignKey("vessels.id", ondelete="CASCADE"), nullable=True, index=True
+    )
+    vessel: Mapped[Vessel | None] = relationship(back_populates="devices")
+    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())

backend/app/models/task.py

@@ -0,0 +1,85 @@
+"""Troubleshooting tasks, their event stream, and approval requests."""
+from __future__ import annotations
+
+from datetime import datetime
+
+from sqlalchemy import DateTime, Enum, ForeignKey, Index, Integer, String, Text, func
+from sqlalchemy.dialects.postgresql import JSONB
+from sqlalchemy.orm import Mapped, mapped_column, relationship
+
+from app.database import Base
+from app.models.enums import ApprovalStatus, TaskStatus
+
+
+class Task(Base):
+    __tablename__ = "tasks"
+
+    id: Mapped[int] = mapped_column(primary_key=True)
+    title: Mapped[str] = mapped_column(String(255))
+    issue: Mapped[str] = mapped_column(Text)  # description of the problem
+    details: Mapped[dict | None] = mapped_column(JSONB, nullable=True)  # extra structured context
+    status: Mapped[TaskStatus] = mapped_column(Enum(TaskStatus), default=TaskStatus.queued, index=True)
+
+    device_id: Mapped[int | None] = mapped_column(
+        ForeignKey("devices.id", ondelete="SET NULL"), nullable=True
+    )
+    vessel_id: Mapped[int | None] = mapped_column(
+        ForeignKey("vessels.id", ondelete="SET NULL"), nullable=True, index=True
+    )
+    created_by_id: Mapped[int | None] = mapped_column(
+        ForeignKey("users.id", ondelete="SET NULL"), nullable=True
+    )
+
+    # Final structured report (issue, steps, root cause, resolution)
+    report: Mapped[dict | None] = mapped_column(JSONB, nullable=True)
+    error: Mapped[str | None] = mapped_column(Text, nullable=True)
+    # Links to persisted artifacts
+    memory_id: Mapped[str | None] = mapped_column(String(128), nullable=True)
+    obsidian_path: Mapped[str | None] = mapped_column(String(512), nullable=True)
+
+    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
+    updated_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), server_default=func.now(), onupdate=func.now()
+    )
+
+    events: Mapped[list["TaskEvent"]] = relationship(
+        back_populates="task", cascade="all, delete-orphan", order_by="TaskEvent.id"
+    )
+    approvals: Mapped[list["ApprovalRequest"]] = relationship(
+        back_populates="task", cascade="all, delete-orphan"
+    )
+
+
+class TaskEvent(Base):
+    __tablename__ = "task_events"
+
+    id: Mapped[int] = mapped_column(primary_key=True)
+    task_id: Mapped[int] = mapped_column(ForeignKey("tasks.id", ondelete="CASCADE"), index=True)
+    # node / kind: triage, connect, diagnose, reason, propose_fix, report, tool_call, log, error
+    kind: Mapped[str] = mapped_column(String(64))
+    message: Mapped[str | None] = mapped_column(Text, nullable=True)
+    payload: Mapped[dict | None] = mapped_column(JSONB, nullable=True)
+    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
+
+    task: Mapped[Task] = relationship(back_populates="events")
+
+
+class ApprovalRequest(Base):
+    __tablename__ = "approval_requests"
+    __table_args__ = (
+        Index("ix_approval_requests_task_status", "task_id", "status"),
+    )
+
+    id: Mapped[int] = mapped_column(primary_key=True)
+    task_id: Mapped[int] = mapped_column(ForeignKey("tasks.id", ondelete="CASCADE"), index=True)
+    tool_name: Mapped[str] = mapped_column(String(255))
+    tool_args: Mapped[dict | None] = mapped_column(JSONB, nullable=True)
+    risk: Mapped[str | None] = mapped_column(Text, nullable=True)
+    status: Mapped[ApprovalStatus] = mapped_column(Enum(ApprovalStatus), default=ApprovalStatus.pending)
+    decided_by_id: Mapped[int | None] = mapped_column(
+        ForeignKey("users.id", ondelete="SET NULL"), nullable=True
+    )
+    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
+    decided_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
+
+    task: Mapped[Task] = relationship(back_populates="approvals")

backend/app/models/user.py

@@ -0,0 +1,22 @@
+"""User accounts and roles."""
+from __future__ import annotations
+
+from datetime import datetime
+
+from sqlalchemy import Boolean, DateTime, Enum, String, func
+from sqlalchemy.orm import Mapped, mapped_column
+
+from app.database import Base
+from app.models.enums import Role
+
+
+class User(Base):
+    __tablename__ = "users"
+
+    id: Mapped[int] = mapped_column(primary_key=True)
+    email: Mapped[str] = mapped_column(String(255), unique=True, index=True)
+    full_name: Mapped[str | None] = mapped_column(String(255), nullable=True)
+    hashed_password: Mapped[str] = mapped_column(String(255))
+    role: Mapped[Role] = mapped_column(Enum(Role), default=Role.readonly)
+    is_active: Mapped[bool] = mapped_column(Boolean, default=True)
+    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())

backend/app/schemas.py

@@ -0,0 +1,315 @@
+"""Pydantic request/response schemas."""
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Any, Literal
+
+from pydantic import BaseModel, ConfigDict, EmailStr
+
+from app.models.enums import ApprovalStatus, DeviceType, Role, TaskStatus
+
+
+# ---- Auth / users -----------------------------------------------------
+class Token(BaseModel):
+    access_token: str
+    token_type: str = "bearer"
+    role: Role
+    email: str
+
+
+class UserCreate(BaseModel):
+    email: EmailStr
+    password: str
+    full_name: str | None = None
+    role: Role = Role.readonly
+
+
+class UserUpdate(BaseModel):
+    full_name: str | None = None
+    role: Role | None = None
+    is_active: bool | None = None
+    password: str | None = None
+
+
+class UserOut(BaseModel):
+    model_config = ConfigDict(from_attributes=True)
+    id: int
+    email: str
+    full_name: str | None
+    role: Role
+    is_active: bool
+    created_at: datetime
+
+
+# ---- Inventory (vessels + onboard devices) ------------------------------
+class CatalogEntryOut(BaseModel):
+    key: str
+    label: str
+    description: str
+    device_type: str
+    mcp_server: str | None
+    default_port: int
+    default_username: str | None
+    has_default_secret: bool = False
+    secret_kind: str = "password"  # password | api_key
+    env_defaults_configured: bool = False
+    asterisk_container: str | None = None  # GeneseasX docker container name (e.g. voip)
+    category: str
+
+
+class VesselDeviceSelection(BaseModel):
+    """One catalog slot when creating/updating a vessel."""
+    catalog_key: str
+    enabled: bool = True
+    port: int | None = None  # falls back to catalog default_port
+    address: str | None = None  # falls back to vessel public_ip
+    username: str | None = None
+    secret: str | None = None
+    notes: str | None = None
+
+
+class VesselCreate(BaseModel):
+    name: str
+    site: str | None = None
+    public_ip: str | None = None
+    notes: str | None = None
+    devices: list[VesselDeviceSelection] = []
+
+
+class VesselUpdate(BaseModel):
+    name: str | None = None
+    site: str | None = None
+    public_ip: str | None = None
+    notes: str | None = None
+    devices: list[VesselDeviceSelection] | None = None
+
+
+class DeviceOut(BaseModel):
+    model_config = ConfigDict(from_attributes=True)
+    id: int
+    name: str
+    catalog_key: str | None
+    device_type: DeviceType
+    address: str
+    port: int | None
+    mcp_server: str | None
+    username: str | None
+    has_secret: bool = False
+    notes: str | None
+    vessel_id: int | None
+    created_at: datetime
+
+
+class VesselOut(BaseModel):
+    model_config = ConfigDict(from_attributes=True)
+    id: int
+    name: str
+    site: str | None
+    public_ip: str | None
+    notes: str | None
+    created_at: datetime
+
+
+class VesselDetail(VesselOut):
+    devices: list[DeviceOut] = []
+
+
+class DeviceUpdate(BaseModel):
+    port: int | None = None
+    address: str | None = None
+    username: str | None = None
+    secret: str | None = None
+    notes: str | None = None
+
+
+# ---- Tasks ------------------------------------------------------------
+class TaskCreate(BaseModel):
+    title: str
+    issue: str
+    device_id: int | None = None
+    vessel_id: int | None = None
+    details: dict[str, Any] | None = None
+
+
+class TaskPreviewIn(BaseModel):
+    title: str = ""
+    issue: str
+    vessel_id: int
+
+
+class TaskPreviewDevice(BaseModel):
+    catalog_key: str
+    name: str
+    label: str
+    checks: list[str] = []
+
+
+class TaskPreviewOut(BaseModel):
+    vessel_id: int
+    vessel_name: str | None = None
+    devices: list[TaskPreviewDevice]
+    rule_id: str | None = None
+    rule_name: str | None = None
+    severity: str | None = None
+    hints: list[str] = []
+    warning: str | None = None
+
+
+class TaskContinue(BaseModel):
+    follow_up: str
+
+
+class TaskEscalateLlm(BaseModel):
+    tier: Literal["economy", "premium"]
+
+
+class TaskEventOut(BaseModel):
+    model_config = ConfigDict(from_attributes=True)
+    id: int
+    kind: str
+    message: str | None
+    payload: dict[str, Any] | None
+    created_at: datetime
+
+
+class TaskOut(BaseModel):
+    model_config = ConfigDict(from_attributes=True)
+    id: int
+    title: str
+    issue: str
+    details: dict[str, Any] | None
+    status: TaskStatus
+    device_id: int | None
+    vessel_id: int | None
+    created_by_id: int | None
+    report: dict[str, Any] | None
+    error: str | None
+    memory_id: str | None
+    obsidian_path: str | None
+    vessel_name: str | None = None
+    created_at: datetime
+    updated_at: datetime
+
+
+class TaskOverviewOut(BaseModel):
+    id: int
+    title: str
+    issue: str
+    status: TaskStatus
+    vessel_id: int | None = None
+    vessel_name: str | None = None
+    summary: str | None = None
+    key_finding: str | None = None
+    devices_checked: list[str] = []
+    resolved: bool | None = None
+    memory_id: str | None = None
+    obsidian_path: str | None = None
+    obsidian_push_error: str | None = None
+    run_cost_usd: float | None = None
+    created_at: datetime
+    updated_at: datetime
+
+
+class TaskDetail(TaskOut):
+    events: list[TaskEventOut] = []
+
+
+class ApprovalOut(BaseModel):
+    model_config = ConfigDict(from_attributes=True)
+    id: int
+    task_id: int
+    tool_name: str
+    tool_args: dict[str, Any] | None
+    risk: str | None
+    status: ApprovalStatus
+    created_at: datetime
+
+
+class ApprovalDecision(BaseModel):
+    approve: bool
+
+
+# ---- MCP --------------------------------------------------------------
+class McpToolOut(BaseModel):
+    name: str
+    description: str | None = None
+
+
+class McpServerOut(BaseModel):
+    name: str
+    status: str  # loaded | error | cloning | stopped
+    tools: list[McpToolOut] = []
+    error: str | None = None
+    source: str | None = None  # local | git
+
+
+class McpCommandResult(BaseModel):
+    ok: bool
+    action: str | None = None
+    server: str | None = None
+    error: str | None = None
+
+
+class McpDevelopRequest(BaseModel):
+    issue: str
+    tool: str | None = None
+    task_id: int | None = None
+    push: bool | None = None
+
+
+class McpDevelopResult(BaseModel):
+    ok: bool
+    task_id: int | None = None
+    summary: str | None = None
+    error: str | None = None
+    tools: list[str] | None = None
+    push: dict | None = None
+
+
+# ---- Balance ----------------------------------------------------------
+class BalanceOut(BaseModel):
+    model_config = ConfigDict(from_attributes=True)
+    provider: str
+    total_credits: float | None
+    total_usage: float | None
+    remaining: float | None
+    currency: str
+    error: str | None
+    created_at: datetime
+
+
+# ---- LLM routing ------------------------------------------------------
+class LlmRoutingConfigIn(BaseModel):
+    local_model_id: str | None = None
+    economy_gemini_id: str | None = None
+    economy_deepseek_id: str | None = None
+    economy_openrouter_id: str | None = None
+    premium_gemini_id: str | None = None
+    premium_deepseek_id: str | None = None
+    premium_openrouter_id: str | None = None
+    cloud_provider_order: list[str] | None = None
+    auto_escalate: bool | None = None
+    max_tier: str | None = None
+
+
+class ProviderModelsOut(BaseModel):
+    providers: dict[str, Any]
+    routing_order: str
+
+
+class LlmRoutingConfigOut(BaseModel):
+    config: dict[str, Any]
+    catalog: dict[str, Any]
+    available_backends: dict[str, bool]
+    strategy: str
+
+
+# ---- Troubleshooting rules --------------------------------------------
+class TroubleshootingRulesUpdate(BaseModel):
+    yaml: str
+
+
+class TroubleshootingRulesOut(BaseModel):
+    yaml: str
+    parsed: dict[str, Any]
+    path_hint: str

backend/app/services/__init__.py

@@ -0,0 +1 @@
+

backend/app/services/approval_filter.py

@@ -0,0 +1,57 @@
+"""Filter LLM proposed_fixes — only real write actions need human approval."""
+from __future__ import annotations
+
+# Connect / session tools already run during diagnostics — never gate on these.
+_CONNECT_TOOLS = frozenset(
+    {
+        "proxmox_connect",
+        "pfsense_connect",
+        "asterisk_connect",
+        "ssh_connect",
+        "fortigate_connect",
+        "fortiswitch_connect",
+    }
+)
+
+# Read-only diagnostic tools the agent may mention but must not require approval for.
+_READ_TOOLS = frozenset(
+    {
+        "proxmox_list_nodes",
+        "proxmox_list_qemu",
+        "proxmox_list_lxc",
+        "proxmox_get_node_status",
+        "pfsense_get_system_status",
+        "pfsense_list_gateways",
+        "ssh_run",
+    }
+)
+
+
+def fix_requires_approval(fix: dict) -> bool:
+    """True only for actionable production config changes with a concrete write tool."""
+    if not fix.get("is_config_change"):
+        return False
+
+    tool = (fix.get("tool") or "").strip()
+    if not tool or tool == "manual-config-change":
+        return False
+    if tool in _CONNECT_TOOLS or tool in _READ_TOOLS:
+        return False
+    if tool.endswith("_connect"):
+        return False
+
+    return True
+
+
+def split_proposed_fixes(fixes: list | None) -> tuple[list[dict], list[dict]]:
+    """Return (approval_candidates, informational_only)."""
+    approval: list[dict] = []
+    info: list[dict] = []
+    for fix in fixes or []:
+        if not isinstance(fix, dict):
+            continue
+        if fix_requires_approval(fix):
+            approval.append(fix)
+        else:
+            info.append(fix)
+    return approval, info

backend/app/services/audit.py

@@ -0,0 +1,30 @@
+"""Audit logging helper."""
+from __future__ import annotations
+
+from typing import Any
+
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.models.audit import AuditLog
+from app.models.user import User
+
+
+async def record_audit(
+    db: AsyncSession,
+    actor: User | None,
+    action: str,
+    target_type: str | None = None,
+    target_id: int | None = None,
+    detail: dict[str, Any] | None = None,
+) -> None:
+    db.add(
+        AuditLog(
+            actor_id=actor.id if actor else None,
+            actor_email=actor.email if actor else None,
+            action=action,
+            target_type=target_type,
+            target_id=target_id,
+            detail=detail,
+        )
+    )
+    await db.commit()

backend/app/services/balance.py

@@ -0,0 +1,148 @@
+"""Poll OpenRouter and DeepSeek for credits/balance and store snapshots."""
+from __future__ import annotations
+
+import asyncio
+import logging
+
+import httpx
+from sqlalchemy import select
+
+from app.config import settings
+from app.database import SessionLocal
+from app.models.balance import BalanceSnapshot
+
+logger = logging.getLogger(__name__)
+
+
+async def fetch_openrouter() -> dict:
+    """Return {total_credits, total_usage, remaining} for OpenRouter.
+
+    Prefers the provisioning key against /credits (account totals); falls back to
+    /key (per-key limit/usage) when only the regular API key is available.
+    """
+    key = settings.openrouter_provisioning_key or settings.openrouter_api_key
+    if not key:
+        return {"error": "no OpenRouter key configured"}
+
+    async with httpx.AsyncClient(timeout=20) as client:
+        # Try account-wide credits first
+        if settings.openrouter_provisioning_key:
+            r = await client.get(
+                "https://openrouter.ai/api/v1/credits",
+                headers={"Authorization": f"Bearer {settings.openrouter_provisioning_key}"},
+            )
+            if r.status_code == 200:
+                data = r.json().get("data", {})
+                total = data.get("total_credits")
+                usage = data.get("total_usage")
+                remaining = (total - usage) if total is not None and usage is not None else None
+                return {
+                    "total_credits": total,
+                    "total_usage": usage,
+                    "remaining": remaining,
+                    "raw": data,
+                }
+
+        # Fall back to per-key info
+        r = await client.get(
+            "https://openrouter.ai/api/v1/key",
+            headers={"Authorization": f"Bearer {settings.openrouter_api_key}"},
+        )
+        r.raise_for_status()
+        data = r.json().get("data", {})
+        limit = data.get("limit")
+        usage = data.get("usage")
+        remaining = data.get("limit_remaining")
+        if remaining is None and limit is not None and usage is not None:
+            remaining = limit - usage
+        return {
+            "total_credits": limit,
+            "total_usage": usage,
+            "remaining": remaining,
+            "raw": data,
+        }
+
+
+async def fetch_deepseek() -> dict:
+    if not settings.deepseek_api_key:
+        return {"error": "no DeepSeek key configured"}
+    async with httpx.AsyncClient(timeout=20) as client:
+        r = await client.get(
+            "https://api.deepseek.com/user/balance",
+            headers={"Authorization": f"Bearer {settings.deepseek_api_key}"},
+        )
+        r.raise_for_status()
+        data = r.json()
+        infos = data.get("balance_infos", [])
+        info = infos[0] if infos else {}
+        total = float(info.get("total_balance")) if info.get("total_balance") else None
+        return {
+            "total_credits": None,
+            "total_usage": None,
+            "remaining": total,
+            "currency": info.get("currency", "USD"),
+            "raw": data,
+        }
+
+
+async def _store(provider: str, data: dict) -> None:
+    async with SessionLocal() as db:
+        db.add(
+            BalanceSnapshot(
+                provider=provider,
+                total_credits=data.get("total_credits"),
+                total_usage=data.get("total_usage"),
+                remaining=data.get("remaining"),
+                currency=data.get("currency", "USD"),
+                raw=data.get("raw"),
+                error=data.get("error"),
+            )
+        )
+        await db.commit()
+
+
+async def poll_once() -> None:
+    for provider, fetcher in (("openrouter", fetch_openrouter), ("deepseek", fetch_deepseek)):
+        try:
+            data = await fetcher()
+        except Exception as exc:  # noqa: BLE001
+            data = {"error": str(exc)}
+            logger.warning("Balance poll failed for %s: %s", provider, exc)
+        if data.get("error"):
+            # Keep last good snapshot visible when DNS/network blips.
+            async with SessionLocal() as db:
+                res = await db.execute(
+                    select(BalanceSnapshot)
+                    .where(BalanceSnapshot.provider == provider)
+                    .where(BalanceSnapshot.error.is_(None))
+                    .where(BalanceSnapshot.remaining.is_not(None))
+                    .order_by(BalanceSnapshot.created_at.desc())
+                    .limit(1)
+                )
+                if res.scalar_one_or_none():
+                    logger.info("Skipping error balance snapshot for %s: %s", provider, data["error"])
+                    continue
+        await _store(provider, data)
+
+
+async def latest_remaining_usd() -> float | None:
+    """Best estimate of remaining USD across configured providers (max)."""
+    async with SessionLocal() as db:
+        remainings: list[float] = []
+        for provider in ("openrouter", "deepseek"):
+            res = await db.execute(
+                select(BalanceSnapshot)
+                .where(BalanceSnapshot.provider == provider)
+                .order_by(BalanceSnapshot.created_at.desc())
+                .limit(1)
+            )
+            snap = res.scalar_one_or_none()
+            if snap and snap.remaining is not None:
+                remainings.append(snap.remaining)
+        return max(remainings) if remainings else None
+
+
+async def balance_poller_loop() -> None:
+    while True:
+        await poll_once()
+        await asyncio.sleep(settings.balance_poll_interval_seconds)

backend/app/services/device_defaults.py

@@ -0,0 +1,173 @@
+"""Default device credentials and ports from environment variables.
+
+Naming convention (catalog_key → env prefix):
+  proxmox      → DEVICE_PROXMOX_*  (API token + SSH fallback)
+  docker_vm    → DEVICE_DOCKER_VM_*
+  pfsense      → DEVICE_PFSENSE_*
+
+Per slot:
+  DEVICE_<PREFIX>_PORT
+  DEVICE_<PREFIX>_USERNAME
+  DEVICE_<PREFIX>_PASSWORD      (SSH / login)
+  DEVICE_<PREFIX>_API_KEY       (REST API devices)
+
+Proxmox API (proxmox-mcp — tried before SSH):
+  DEVICE_PROXMOX_API_URL          (optional override; leave blank to use https://<vessel-public-ip>:8006)
+  DEVICE_PROXMOX_API_PORT         (optional; default 8006 when API URL is auto-built)
+  DEVICE_PROXMOX_TOKEN_ID         (e.g. root@pam!agentic)
+  DEVICE_PROXMOX_TOKEN_SECRET
+  DEVICE_PROXMOX_VERIFY_SSL       (true/false, default false)
+
+Global fallbacks for SSH slots:
+  DEVICE_SSH_USERNAME
+  DEVICE_SSH_PASSWORD
+
+Global fallback for API-key slots:
+  DEVICE_API_KEY
+"""
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+
+from app.inventory_catalog import CatalogEntry
+
+# catalog_key → how secrets are stored / resolved
+SLOT_AUTH: dict[str, dict] = {
+    "proxmox": {"auth": "ssh", "secret_field": "PASSWORD"},
+    "docker_vm": {"auth": "ssh", "secret_field": "PASSWORD"},
+    "asterisk": {"auth": "ssh", "secret_field": "PASSWORD"},
+    "asterisk_geneseasx": {"auth": "ssh", "secret_field": "PASSWORD"},
+    "asterisk_satbox": {"auth": "ssh", "secret_field": "PASSWORD"},
+    "debian_host": {"auth": "ssh", "secret_field": "PASSWORD"},
+    "tplink": {"auth": "ssh", "secret_field": "PASSWORD"},
+    "pfsense": {"auth": "api_key", "secret_field": "API_KEY"},
+    "fortigate": {"auth": "api_key", "secret_field": "API_KEY"},
+    "fortiswitch": {"auth": "password", "secret_field": "PASSWORD"},
+}
+
+
+@dataclass
+class SlotDefaults:
+    port: int | None = None
+    username: str | None = None
+    secret: str | None = None
+    secret_kind: str = "password"  # password | api_key
+    from_env: bool = False
+
+
+def _env(name: str) -> str | None:
+    val = os.environ.get(name, "").strip()
+    return val or None
+
+
+def _prefix(catalog_key: str) -> str:
+    return catalog_key.upper()
+
+
+def resolve_slot_defaults(catalog_key: str, entry: CatalogEntry) -> SlotDefaults:
+    """Load defaults for a catalog slot from .env (never exposed via API as plaintext)."""
+    prefix = _prefix(catalog_key)
+    meta = SLOT_AUTH.get(catalog_key, {"auth": "ssh", "secret_field": "PASSWORD"})
+    secret_kind = "api_key" if meta["auth"] == "api_key" else "password"
+
+    port_raw = _env(f"DEVICE_{prefix}_PORT")
+    port = int(port_raw) if port_raw and port_raw.isdigit() else None
+
+    username = _env(f"DEVICE_{prefix}_USERNAME")
+    secret: str | None = None
+
+    if meta["secret_field"] == "API_KEY":
+        secret = _env(f"DEVICE_{prefix}_API_KEY") or _env("DEVICE_API_KEY")
+    else:
+        secret = _env(f"DEVICE_{prefix}_PASSWORD")
+
+    if meta["auth"] == "ssh":
+        username = username or _env("DEVICE_SSH_USERNAME")
+        secret = secret or _env("DEVICE_SSH_PASSWORD")
+
+    if meta["auth"] == "api_key" and not username:
+        username = _env(f"DEVICE_{prefix}_USERNAME")
+
+    from_env = any(
+        [
+            port is not None,
+            bool(username),
+            bool(secret),
+        ]
+    )
+
+    return SlotDefaults(
+        port=port,
+        username=username,
+        secret=secret,
+        secret_kind=secret_kind,
+        from_env=from_env,
+    )
+
+
+def _truthy_env(name: str) -> bool:
+    val = os.environ.get(name, "").strip().lower()
+    return val in ("1", "true", "yes", "on")
+
+
+PROXMOX_DEFAULT_API_PORT = 8006
+
+
+def build_proxmox_api_url(public_ip: str) -> str | None:
+    """Build Proxmox API URL from vessel/device public IP unless DEVICE_PROXMOX_API_URL is set."""
+    override = _env("DEVICE_PROXMOX_API_URL")
+    if override:
+        return override
+    if not public_ip:
+        return None
+    port_raw = _env("DEVICE_PROXMOX_API_PORT")
+    port = int(port_raw) if port_raw and port_raw.isdigit() else PROXMOX_DEFAULT_API_PORT
+    host = public_ip.strip().removeprefix("https://").removeprefix("http://").split("/")[0]
+    if ":" in host and not host.startswith("["):
+        # Already host:port — use as-is with scheme only.
+        return f"https://{host}"
+    return f"https://{host}:{port}"
+
+
+def resolve_proxmox_api_defaults(public_ip: str) -> dict:
+    """Proxmox VE API token settings from env (used by proxmox-mcp connect).
+
+    ``public_ip`` is the vessel public IP stored on the proxmox device row (``device.address``).
+    When DEVICE_PROXMOX_API_URL is blank, the URL becomes ``https://<public_ip>:8006``.
+    """
+    return {
+        "proxmox_api_url": build_proxmox_api_url(public_ip),
+        "proxmox_token_id": _env("DEVICE_PROXMOX_TOKEN_ID"),
+        "proxmox_token_secret": _env("DEVICE_PROXMOX_TOKEN_SECRET"),
+        "proxmox_verify_ssl": _truthy_env("DEVICE_PROXMOX_VERIFY_SSL"),
+    }
+
+
+def proxmox_api_configured(ctx: dict) -> bool:
+    return bool(ctx.get("proxmox_token_id") and ctx.get("proxmox_token_secret"))
+
+
+def merge_selection_with_defaults(
+    catalog_key: str,
+    entry: CatalogEntry,
+    *,
+    port: int | None,
+    username: str | None,
+    secret: str | None,
+    existing_secret: str | None = None,
+) -> tuple[int | None, str | None, str | None]:
+    """Apply env defaults where the UI/API did not supply a value."""
+    defaults = resolve_slot_defaults(catalog_key, entry)
+
+    resolved_port = port if port is not None else defaults.port if defaults.port is not None else entry.default_port
+    resolved_username = username if username is not None else defaults.username or entry.default_username
+
+    if secret:
+        resolved_secret = secret
+    elif existing_secret:
+        resolved_secret = existing_secret
+    else:
+        resolved_secret = defaults.secret
+
+    return resolved_port, resolved_username, resolved_secret

backend/app/services/diagnostic_format.py

@@ -0,0 +1,374 @@
+"""Format MCP diagnostic output for reports, Obsidian, and UI."""
+from __future__ import annotations
+
+import json
+import re
+from typing import Any
+
+MAX_REPORT_OUTPUT_PER_TOOL = 4_000
+MAX_OBSIDIAN_DIAG_CHARS = 120_000
+MAX_EVENT_OUTPUT = 12_000
+
+_TABLE_TOOLS = frozenset(
+    {
+        "pfsense_list_firewall_rules",
+        "pfsense_list_port_forwards",
+        "pfsense_list_outbound_nat_mappings",
+        "pfsense_list_aliases",
+        "pfsense_list_interfaces",
+        "proxmox_list_qemu",
+        "proxmox_list_lxc",
+        "proxmox_get_qemu_status",
+        "proxmox_get_qemu_config",
+    }
+)
+
+
+def _clip(text: str, limit: int) -> str:
+    if len(text) <= limit:
+        return text
+    return text[: limit - 40] + f"\n\n… ({len(text) - limit + 40} chars truncated)"
+
+
+def _parse_json_payload(text: str) -> Any | None:
+    text = (text or "").strip()
+    if not text:
+        return None
+    try:
+        return json.loads(text)
+    except json.JSONDecodeError:
+        pass
+    start, end = text.find("{"), text.rfind("}")
+    if start != -1 and end > start:
+        try:
+            return json.loads(text[start : end + 1])
+        except json.JSONDecodeError:
+            return None
+    return None
+
+
+def _cell(val: Any) -> str:
+    if val is None:
+        return ""
+    if isinstance(val, list):
+        return ", ".join(str(x) for x in val)
+    s = str(val)
+    s = re.sub(r"&amp;+", "&", s)
+    return s.replace("|", "\\|").replace("\n", " ")
+
+
+def _firewall_rules_table(rows: list[dict]) -> str:
+    header = "| # | Action | Interface | Protocol | Source | Destination | Description |"
+    sep = "|---|---|---|---|---|---|---|"
+    lines = [header, sep]
+    for i, row in enumerate(rows[:300], start=1):
+        lines.append(
+            "| "
+            + " | ".join(
+                [
+                    str(i),
+                    _cell(row.get("type")),
+                    _cell(row.get("interface")),
+                    _cell(row.get("protocol")),
+                    _cell(row.get("source")),
+                    _cell(row.get("destination")),
+                    _cell(row.get("descr")),
+                ]
+            )
+            + " |"
+        )
+    if len(rows) > 300:
+        lines.append(f"\n_Showing 300 of {len(rows)} rules._")
+    return "\n".join(lines)
+
+
+def _port_forwards_table(rows: list[dict]) -> str:
+    header = "| # | Interface | Protocol | Destination | Target | Local port | Description |"
+    sep = "|---|---|---|---|---|---|---|"
+    lines = [header, sep]
+    for i, row in enumerate(rows[:200], start=1):
+        lines.append(
+            "| "
+            + " | ".join(
+                [
+                    str(i),
+                    _cell(row.get("interface")),
+                    _cell(row.get("protocol")),
+                    _cell(row.get("destination")),
+                    _cell(row.get("target")),
+                    _cell(row.get("local_port") or row.get("destination_port")),
+                    _cell(row.get("descr")),
+                ]
+            )
+            + " |"
+        )
+    return "\n".join(lines)
+
+
+def _nat_table(rows: list[dict]) -> str:
+    header = "| # | Interface | Source | Destination | Target | Description |"
+    sep = "|---|---|---|---|---|---|"
+    lines = [header, sep]
+    for i, row in enumerate(rows[:200], start=1):
+        lines.append(
+            "| "
+            + " | ".join(
+                [
+                    str(i),
+                    _cell(row.get("interface")),
+                    _cell(row.get("source")),
+                    _cell(row.get("destination")),
+                    _cell(row.get("target")),
+                    _cell(row.get("descr")),
+                ]
+            )
+            + " |"
+        )
+    return "\n".join(lines)
+
+
+def _aliases_table(rows: list[dict]) -> str:
+    header = "| Name | Type | Addresses | Description |"
+    sep = "|---|---|---|---|"
+    lines = [header, sep]
+    for row in rows[:200]:
+        lines.append(
+            "| "
+            + " | ".join(
+                [
+                    _cell(row.get("name")),
+                    _cell(row.get("type")),
+                    _cell(row.get("address")),
+                    _cell(row.get("descr")),
+                ]
+            )
+            + " |"
+        )
+    return "\n".join(lines)
+
+
+def _interfaces_table(rows: list[dict]) -> str:
+    header = "| ID | Interface | Enabled | Description | IPv4 |"
+    sep = "|---|---|---|---|---|"
+    lines = [header, sep]
+    for row in rows[:50]:
+        lines.append(
+            "| "
+            + " | ".join(
+                [
+                    _cell(row.get("id")),
+                    _cell(row.get("if")),
+                    _cell(row.get("enable")),
+                    _cell(row.get("descr")),
+                    _cell(row.get("ipaddr")),
+                ]
+            )
+            + " |"
+        )
+    return "\n".join(lines)
+
+
+def _qemu_list_table(rows: list[dict]) -> str:
+    header = "| VMID | Name | Status | CPU | Memory | Disk |"
+    sep = "|---|---|---|---|---|---|"
+    lines = [header, sep]
+    for row in rows[:100]:
+        mem = row.get("maxmem") or row.get("mem")
+        if isinstance(mem, (int, float)) and mem > 1024 * 1024:
+            mem = f"{mem / (1024 ** 3):.1f} GB"
+        disk = row.get("maxdisk") or row.get("disk")
+        if isinstance(disk, (int, float)) and disk > 1024 * 1024:
+            disk = f"{disk / (1024 ** 3):.1f} GB"
+        lines.append(
+            "| "
+            + " | ".join(
+                [
+                    _cell(row.get("vmid")),
+                    _cell(row.get("name")),
+                    _cell(row.get("status")),
+                    _cell(row.get("cpus") or row.get("cpu")),
+                    _cell(mem),
+                    _cell(disk),
+                ]
+            )
+            + " |"
+        )
+    return "\n".join(lines)
+
+
+def _lxc_list_table(rows: list[dict]) -> str:
+    header = "| VMID | Name | Status |"
+    sep = "|---|---|---|"
+    lines = [header, sep]
+    for row in rows[:100]:
+        lines.append(
+            "| "
+            + " | ".join([_cell(row.get("vmid")), _cell(row.get("name")), _cell(row.get("status"))])
+            + " |"
+        )
+    return "\n".join(lines)
+
+
+def _qemu_config_table(data: dict) -> str:
+    lines = ["| Key | Value |", "|---|---|"]
+    for key in sorted(data.keys()):
+        if key.startswith("digest"):
+            continue
+        lines.append(f"| {_cell(key)} | {_cell(data[key])} |")
+    return "\n".join(lines)
+
+
+def _qemu_status_table(data: dict) -> str:
+    if not data:
+        return "_No status data._"
+    lines = ["| Field | Value |", "|---|---|"]
+    for key in ("name", "status", "qmpstatus", "vmid", "uptime", "cpu", "mem", "maxmem", "disk", "maxdisk", "pid"):
+        if key in data:
+            val = data[key]
+            if isinstance(val, (int, float)) and key in ("mem", "maxmem", "disk", "maxdisk") and val > 1024 * 1024:
+                val = f"{val / (1024 ** 3):.2f} GB"
+            lines.append(f"| {_cell(key)} | {_cell(val)} |")
+    for key, val in sorted(data.items()):
+        if key in ("name", "status", "qmpstatus", "vmid", "uptime", "cpu", "mem", "maxmem", "disk", "maxdisk", "pid"):
+            continue
+        lines.append(f"| {_cell(key)} | {_cell(val)} |")
+    return "\n".join(lines)
+
+
+def _rows_from_payload(parsed: Any) -> list[dict]:
+    if isinstance(parsed, list):
+        return [r for r in parsed if isinstance(r, dict)]
+    if isinstance(parsed, dict):
+        data = parsed.get("data")
+        if isinstance(data, list):
+            return [r for r in data if isinstance(r, dict)]
+        return [parsed]
+    return []
+
+
+def _qm_list_text_table(text: str) -> str:
+    lines = ["| VMID | Name | Status | Mem (MB) | Boot disk (GB) |", "|---|---|---|---|---|"]
+    for row in text.splitlines():
+        row = row.strip()
+        if not row or row.startswith("VMID") or row.startswith("-"):
+            continue
+        parts = row.split()
+        if not parts or not parts[0].isdigit():
+            continue
+        vmid, name, status = parts[0], parts[1] if len(parts) > 1 else "", parts[2] if len(parts) > 2 else ""
+        mem = parts[3] if len(parts) > 3 else ""
+        disk = parts[4] if len(parts) > 4 else ""
+        lines.append(f"| {vmid} | {name} | {status} | {mem} | {disk} |")
+    return "\n".join(lines) if len(lines) > 2 else f"```\n{_clip(text, 8000)}\n```"
+
+
+def _qm_config_text_block(text: str) -> str:
+    lines = ["| Key | Value |", "|---|---|"]
+    for raw in text.splitlines():
+        raw = raw.strip()
+        if not raw or ":" not in raw:
+            continue
+        key, val = raw.split(":", 1)
+        lines.append(f"| {_cell(key.strip())} | {_cell(val.strip())} |")
+    return "\n".join(lines) if len(lines) > 2 else f"```\n{_clip(text, 8000)}\n```"
+
+
+def format_tool_output_markdown(tool: str, output: str) -> str:
+    """Turn raw MCP JSON/text into readable markdown when possible."""
+    parsed = _parse_json_payload(output)
+    if tool == "proxmox_list_qemu":
+        if isinstance(parsed, (dict, list)):
+            rows = _rows_from_payload(parsed)
+            if rows and rows[0].get("vmid") is not None:
+                return _qemu_list_table(rows)
+        if re.search(r"^\s*\d+\s+\S+\s+(running|stopped)", output, re.MULTILINE):
+            return _qm_list_text_table(output)
+    if tool == "proxmox_list_lxc" and isinstance(parsed, (dict, list)):
+        rows = _rows_from_payload(parsed)
+        if rows:
+            return _lxc_list_table(rows)
+    if tool == "proxmox_get_qemu_config":
+        if isinstance(parsed, dict):
+            inner = parsed.get("data") if isinstance(parsed.get("data"), dict) else parsed
+            if isinstance(inner, dict) and inner:
+                return _qemu_config_table(inner)
+        if ":" in output:
+            return _qm_config_text_block(output)
+    if tool == "proxmox_get_qemu_status" and isinstance(parsed, dict):
+        inner = parsed.get("data") if isinstance(parsed.get("data"), dict) else parsed
+        if isinstance(inner, dict):
+            return _qemu_status_table(inner)
+
+    if isinstance(parsed, dict):
+        data = parsed.get("data")
+        if isinstance(data, list) and data:
+            if tool == "pfsense_list_firewall_rules":
+                return _firewall_rules_table(data)
+            if tool == "pfsense_list_port_forwards":
+                return _port_forwards_table(data)
+            if tool == "pfsense_list_outbound_nat_mappings":
+                return _nat_table(data)
+            if tool == "pfsense_list_aliases":
+                return _aliases_table(data)
+            if tool == "pfsense_list_interfaces":
+                return _interfaces_table(data)
+            if tool in _TABLE_TOOLS:
+                return f"```json\n{json.dumps(data[:50], indent=2)}\n```"
+    return f"```\n{_clip(output, 20_000)}\n```"
+
+
+def prepare_report_diagnostics(diagnostics: list[dict]) -> list[dict]:
+    """Normalize diagnostics for persistence in task.report."""
+    out: list[dict] = []
+    for d in diagnostics:
+        text = d.get("output") or ""
+        formatted = ""
+        tool = d.get("tool")
+        if text:
+            if tool in _TABLE_TOOLS:
+                formatted = format_tool_output_markdown(tool, text)
+            elif tool == "ssh_run":
+                if re.search(r"^\s*\d+\s+\S+\s+(running|stopped)", text, re.MULTILINE):
+                    formatted = format_tool_output_markdown("proxmox_list_qemu", text)
+                elif re.match(r"^[a-z][a-z0-9_-]*:", text.strip(), re.IGNORECASE):
+                    formatted = format_tool_output_markdown("proxmox_get_qemu_config", text)
+        entry = {
+            "device": d.get("device"),
+            "tool": tool,
+            "ok": d.get("ok"),
+            "intent": d.get("intent"),
+            "vmid": d.get("vmid"),
+            "output_chars": len(text),
+            "raw_available": bool(text),
+            "truncated": len(text) > MAX_REPORT_OUTPUT_PER_TOOL,
+        }
+        if formatted:
+            entry["formatted"] = formatted
+        elif text:
+            entry["output"] = _clip(text, MAX_REPORT_OUTPUT_PER_TOOL)
+        out.append(entry)
+    return out
+
+
+def diagnostics_to_markdown(diagnostics: list[dict]) -> str:
+    """Build Obsidian section from report-style diagnostics."""
+    if not diagnostics:
+        return "_No live diagnostic output captured._"
+
+    parts: list[str] = []
+    total = 0
+    for d in diagnostics:
+        tool = d.get("tool") or "unknown"
+        device = d.get("device") or "?"
+        ok = d.get("ok")
+        vmid = d.get("vmid")
+        vm_label = f" VM {vmid}" if vmid is not None else ""
+        header = f"### [{device}] {tool}{vm_label} ({'ok' if ok else 'fail'})"
+        body = d.get("formatted") or format_tool_output_markdown(tool, d.get("output") or "")
+        chunk = f"{header}\n\n{body}\n"
+        if total + len(chunk) > MAX_OBSIDIAN_DIAG_CHARS:
+            parts.append("_Additional diagnostic output omitted (size limit)._")
+            break
+        parts.append(chunk)
+        total += len(chunk)
+    return "\n".join(parts)

backend/app/services/events.py

@@ -0,0 +1,96 @@
+"""Task event bus: persist events to DB and publish to Redis for live streaming."""
+from __future__ import annotations
+
+import json
+from datetime import datetime, timezone
+from typing import Any
+
+import redis.asyncio as aioredis
+
+from app.config import settings
+from app.database import SessionLocal
+from app.models.task import TaskEvent
+
+_redis: aioredis.Redis | None = None
+
+# Redis keys
+TASK_QUEUE = "agentic:task_queue"
+
+# High-frequency live-only event kinds: streamed to the UI but not written to
+# Postgres, to avoid a separate transaction per progress/tool-start tick.
+EPHEMERAL_KINDS = frozenset({"progress", "tool_start"})
+
+
+def get_redis() -> aioredis.Redis:
+    global _redis
+    if _redis is None:
+        _redis = aioredis.from_url(
+            settings.redis_url, decode_responses=True, max_connections=32
+        )
+    return _redis
+
+
+def _channel(task_id: int) -> str:
+    return f"agentic:task:{task_id}:events"
+
+
+async def dedupe_task_queue(task_id: int) -> int:
+    """Remove stale duplicate entries for a task id from the Redis queue."""
+    redis = get_redis()
+    needle = str(task_id)
+    items = await redis.lrange(TASK_QUEUE, 0, -1)
+    removed = sum(1 for x in items if x == needle)
+    if removed:
+        await redis.delete(TASK_QUEUE)
+        for item in items:
+            if item != needle:
+                await redis.rpush(TASK_QUEUE, item)
+    return removed
+
+
+async def enqueue_task(task_id: int) -> None:
+    await dedupe_task_queue(task_id)
+    await get_redis().rpush(TASK_QUEUE, str(task_id))
+
+
+async def publish_event(
+    task_id: int,
+    kind: str,
+    message: str | None = None,
+    payload: dict[str, Any] | None = None,
+    persist: bool | None = None,
+) -> None:
+    """Persist an event and publish it to the task's Redis channel.
+
+    When ``persist`` is None, high-frequency ephemeral kinds (progress,
+    tool_start) are streamed live but not written to the DB, while all other
+    kinds are persisted. Pass an explicit bool to override.
+    """
+    if persist is None:
+        persist = kind not in EPHEMERAL_KINDS
+    if persist:
+        async with SessionLocal() as db:
+            db.add(TaskEvent(task_id=task_id, kind=kind, message=message, payload=payload))
+            await db.commit()
+
+    event = {
+        "task_id": task_id,
+        "kind": kind,
+        "message": message,
+        "payload": payload,
+        "ts": datetime.now(timezone.utc).isoformat(),
+    }
+    await get_redis().publish(_channel(task_id), json.dumps(event))
+
+
+async def subscribe_events(task_id: int):
+    """Async generator yielding decoded events for a task."""
+    pubsub = get_redis().pubsub()
+    await pubsub.subscribe(_channel(task_id))
+    try:
+        async for message in pubsub.listen():
+            if message["type"] == "message":
+                yield json.loads(message["data"])
+    finally:
+        await pubsub.unsubscribe(_channel(task_id))
+        await pubsub.close()

backend/app/services/integrations.py

@@ -0,0 +1,353 @@
+"""Per-task persistence: project-memory (HTTP) and Obsidian vault (git)."""
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+import os
+import time
+from datetime import date
+
+import httpx
+from slugify import slugify
+
+from app.config import settings
+
+logger = logging.getLogger(__name__)
+
+MCP_ACCEPT = "application/json, text/event-stream"
+
+
+def _parse_mcp_response(resp: httpx.Response) -> dict:
+    """Parse JSON or SSE body from streamable HTTP MCP responses."""
+    content_type = resp.headers.get("content-type", "")
+    text = resp.text.strip()
+    if "text/event-stream" in content_type or text.startswith("event:") or text.startswith("data:"):
+        for line in text.splitlines():
+            if line.startswith("data:"):
+                payload = line[5:].strip()
+                if payload and payload != "[DONE]":
+                    return json.loads(payload)
+        return {}
+    if text:
+        return resp.json()
+    return {}
+
+
+class _MemoryMcpSession:
+    """Reusable streamable-HTTP session for the project-memory MCP server.
+
+    Performs the initialize handshake once and reuses the HTTP client +
+    Mcp-Session-Id across calls, re-initializing only when the session is
+    rejected (e.g. server restart / expired session).
+    """
+
+    def __init__(self) -> None:
+        self._client: httpx.AsyncClient | None = None
+        self._session_id: str | None = None
+        self._lock = asyncio.Lock()
+        self._req_id = 0
+
+    def _headers(self) -> dict[str, str]:
+        headers = {"Content-Type": "application/json", "Accept": MCP_ACCEPT}
+        if settings.memory_mcp_token:
+            headers["Authorization"] = f"Bearer {settings.memory_mcp_token}"
+        if self._session_id:
+            headers["Mcp-Session-Id"] = self._session_id
+        return headers
+
+    async def _ensure_client(self) -> httpx.AsyncClient:
+        if self._client is None:
+            self._client = httpx.AsyncClient(timeout=30)
+        return self._client
+
+    async def _handshake(self) -> None:
+        client = await self._ensure_client()
+        self._session_id = None
+        init_body = {
+            "jsonrpc": "2.0",
+            "id": 1,
+            "method": "initialize",
+            "params": {
+                "protocolVersion": "2024-11-05",
+                "capabilities": {},
+                "clientInfo": {"name": "agentic-os", "version": "0.1.0"},
+            },
+        }
+        resp = await client.post(settings.memory_mcp_url, headers=self._headers(), json=init_body)
+        resp.raise_for_status()
+        self._session_id = resp.headers.get("mcp-session-id") or resp.headers.get("Mcp-Session-Id")
+        await client.post(
+            settings.memory_mcp_url,
+            headers=self._headers(),
+            json={"jsonrpc": "2.0", "method": "notifications/initialized"},
+        )
+
+    async def _call_once(self, tool_name: str, arguments: dict) -> dict | None:
+        client = await self._ensure_client()
+        self._req_id += 1
+        body = {
+            "jsonrpc": "2.0",
+            "id": self._req_id,
+            "method": "tools/call",
+            "params": {"name": tool_name, "arguments": arguments},
+        }
+        resp = await client.post(settings.memory_mcp_url, headers=self._headers(), json=body)
+        resp.raise_for_status()
+        data = _parse_mcp_response(resp)
+        if data.get("error"):
+            raise RuntimeError(data["error"].get("message", str(data["error"])))
+        return data.get("result")
+
+    async def call(self, tool_name: str, arguments: dict) -> dict | None:
+        async with self._lock:
+            if self._session_id is None:
+                await self._handshake()
+            try:
+                return await self._call_once(tool_name, arguments)
+            except (httpx.HTTPStatusError, RuntimeError) as exc:
+                # Session may have expired; re-handshake once and retry.
+                logger.info("memory MCP retry after error: %s", exc)
+                await self._handshake()
+                return await self._call_once(tool_name, arguments)
+
+
+_memory_session = _MemoryMcpSession()
+
+
+async def memory_mcp_call(tool_name: str, arguments: dict) -> dict | None:
+    """Run one MCP tool call against project-memory over a reused HTTP session."""
+    if not settings.memory_mcp_url:
+        return None
+    return await _memory_session.call(tool_name, arguments)
+
+
+async def create_task_memory(
+    title: str,
+    content: str,
+    memory_type: str = "fix",
+    tags: list[str] | None = None,
+    importance: int = 3,
+    *,
+    vessel_name: str | None = None,
+    task_id: int | None = None,
+) -> str | None:
+    """Call memory_upsert on the project-memory MCP over HTTP JSON-RPC."""
+    tag_list = list(tags or ["agentic-os"])
+    if vessel_name:
+        from slugify import slugify
+
+        tag_list.append(f"vessel:{slugify(vessel_name, lowercase=True)}")
+    if task_id is not None:
+        tag_list.append(f"task:{task_id}")
+
+    body_parts = []
+    if vessel_name:
+        body_parts.append(f"Vessel: {vessel_name}")
+    if task_id is not None:
+        body_parts.append(f"Task ID: {task_id}")
+    body_parts.append(content)
+    full_content = "\n\n".join(body_parts)
+
+    args = {
+        "project_id": settings.memory_task_project_id,
+        "memory_type": memory_type,
+        "title": title,
+        "content": full_content,
+        "tags": tag_list,
+        "importance": importance,
+    }
+
+    try:
+        result = await memory_mcp_call("memory_upsert", args)
+        if not isinstance(result, dict):
+            return None
+        structured = result.get("structuredContent")
+        if isinstance(structured, dict):
+            mid = structured.get("id") or structured.get("memory_id")
+            if mid:
+                return str(mid)
+        for block in result.get("content", []):
+            text = block.get("text") if isinstance(block, dict) else None
+            if text and "id" in text:
+                return text[:128]
+    except Exception as exc:  # noqa: BLE001
+        logger.warning("create_task_memory failed: %s", exc)
+    return None
+
+
+def _vault_dir() -> str:
+    return os.path.join(settings.runtime_dir, "obsidian-vault")
+
+
+async def _run(cmd: list[str], cwd: str | None = None) -> tuple[int, str]:
+    proc = await asyncio.create_subprocess_exec(
+        *cmd,
+        cwd=cwd,
+        stdout=asyncio.subprocess.PIPE,
+        stderr=asyncio.subprocess.STDOUT,
+    )
+    out, _ = await proc.communicate()
+    return proc.returncode or 0, out.decode(errors="replace")
+
+
+async def _repair_vault_git(vault: str) -> None:
+    """Recover from stuck rebase/merge so commits can be pushed to origin/main."""
+    git_dir = os.path.join(vault, ".git")
+    if not os.path.isdir(git_dir):
+        return
+
+    if os.path.isdir(os.path.join(git_dir, "rebase-merge")) or os.path.isdir(
+        os.path.join(git_dir, "rebase-apply")
+    ):
+        logger.warning("Obsidian vault stuck in git rebase; attempting continue/abort")
+        code, _ = await _run(["git", "rebase", "--continue"], cwd=vault)
+        if code != 0:
+            await _run(["git", "rebase", "--abort"], cwd=vault)
+
+    if os.path.isfile(os.path.join(git_dir, "MERGE_HEAD")):
+        logger.warning("Obsidian vault stuck in git merge; aborting")
+        await _run(["git", "merge", "--abort"], cwd=vault)
+
+    code, _ = await _run(["git", "checkout", "main"], cwd=vault)
+    if code != 0:
+        await _run(["git", "checkout", "-B", "main"], cwd=vault)
+
+
+async def _vault_pull(vault: str) -> None:
+    await _repair_vault_git(vault)
+    code, out = await _run(["git", "pull", "--ff-only", "origin", "main"], cwd=vault)
+    if code != 0:
+        logger.warning("vault ff-only pull failed: %s", out[:300])
+        await _repair_vault_git(vault)
+        await _run(["git", "pull", "--no-rebase", "origin", "main"], cwd=vault)
+
+
+async def _vault_commit_and_push(vault: str, rel_path: str, title: str) -> tuple[bool, str | None]:
+    """Stage, commit, and push one note. Returns (pushed, error_message)."""
+    if not settings.obsidian_auto_push or not settings.gitea_token:
+        return False, None
+
+    await _repair_vault_git(vault)
+    await _run(["git", "add", rel_path], cwd=vault)
+    code, out = await _run(
+        [
+            "git",
+            "-c",
+            "user.email=agentic-os@paraskeva.net",
+            "-c",
+            "user.name=Agentic OS",
+            "commit",
+            "-m",
+            f"agentic-os task: {title}",
+        ],
+        cwd=vault,
+    )
+    if code != 0 and "nothing to commit" not in out.lower():
+        return False, out.strip()[:500]
+
+    await _repair_vault_git(vault)
+    code, out = await _run(["git", "push", "origin", "main"], cwd=vault)
+    if code != 0:
+        logger.warning("vault push failed: %s", out)
+        return False, out.strip()[:500]
+    return True, None
+
+
+_VAULT_PULL_TTL = 300.0  # seconds between background `git pull` of the vault
+_vault_pull_lock = asyncio.Lock()
+_vault_last_pull = 0.0
+
+
+async def ensure_vault(*, force_pull: bool = False) -> str | None:
+    global _vault_last_pull
+    vault = _vault_dir()
+    clone_url = settings.gitea_clone_url(settings.obsidian_vault_repo)
+    if os.path.isdir(os.path.join(vault, ".git")):
+        # Throttle pulls: skip if pulled recently (deduped across concurrent tasks).
+        now = time.monotonic()
+        if force_pull or now - _vault_last_pull >= _VAULT_PULL_TTL:
+            async with _vault_pull_lock:
+                if force_pull or time.monotonic() - _vault_last_pull >= _VAULT_PULL_TTL:
+                    await _vault_pull(vault)
+                    _vault_last_pull = time.monotonic()
+        return vault
+    os.makedirs(settings.runtime_dir, exist_ok=True)
+    code, out = await _run(["git", "clone", clone_url, vault])
+    if code != 0:
+        logger.warning("vault clone failed: %s", out)
+        return None
+    _vault_last_pull = time.monotonic()
+    return vault
+
+
+async def write_obsidian_note(
+    title: str,
+    issue: str,
+    steps: list[str] | str,
+    cause: str | None,
+    resolution: str | None,
+    resolved: bool,
+    *,
+    vessel_name: str | None = None,
+    task_id: int | None = None,
+    diagnostics_md: str | None = None,
+) -> tuple[str | None, str | None]:
+    """Write task note to vault. Returns (relative_path, push_error)."""
+    vault = await ensure_vault()
+    if not vault:
+        return None, "Obsidian vault unavailable (clone failed)"
+
+    folder = settings.obsidian_tasks_folder
+    os.makedirs(os.path.join(vault, folder), exist_ok=True)
+    slug = slugify(title)[:60] or "task"
+    filename = f"{date.today().isoformat()}-{slug}.md"
+    rel_path = f"{folder}/{filename}"
+    abs_path = os.path.join(vault, rel_path)
+
+    steps_md = (
+        "\n".join(f"- {s}" for s in steps) if isinstance(steps, list) else str(steps or "")
+    )
+    status_label = "Resolved" if resolved else "Unresolved"
+    vessel_line = f"vessel: {vessel_name}\n" if vessel_name else ""
+    task_line = f"task_id: {task_id}\n" if task_id is not None else ""
+    diag_section = ""
+    if diagnostics_md:
+        diag_section = f"\n## Diagnostic findings\n{diagnostics_md}\n"
+
+    body = f"""---
+tags: [agentic-os, troubleshooting]
+{vessel_line}{task_line}status: {status_label.lower()}
+created: {date.today().isoformat()}
+---
+
+# {title}
+
+## Vessel
+{vessel_name or "_Not specified._"}
+
+## Issue
+{issue}
+
+## Troubleshooting steps
+{steps_md}
+
+## Root cause
+{cause or "_Not determined._"}
+
+## Resolution ({status_label})
+{resolution or "_No resolution recorded._"}
+{diag_section}"""
+    def _write() -> None:
+        with open(abs_path, "w", encoding="utf-8") as fh:
+            fh.write(body)
+
+    await asyncio.to_thread(_write)
+
+    pushed, push_error = await _vault_commit_and_push(vault, rel_path, title)
+    if push_error:
+        logger.warning("Obsidian note written locally at %s but git push failed: %s", rel_path, push_error)
+    elif not pushed and settings.obsidian_auto_push:
+        push_error = "Note saved in vault clone only (auto-push disabled or no Gitea token)"
+
+    return rel_path, push_error

backend/app/services/investigation_context.py

@@ -0,0 +1,317 @@
+"""Load prior knowledge from project-memory and Obsidian before agent triage."""
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+import os
+import re
+from typing import Any
+
+from app.config import settings
+from app.services import integrations
+
+logger = logging.getLogger(__name__)
+
+_STOPWORDS = frozenset(
+    """
+    a an the and or but in on at to for of is are was were be been being
+    with from by as it this that these those i we you they he she
+    check status health task issue vessel ship
+    """.split()
+)
+
+_MAX_MEMORY_HITS = 8
+_MAX_OBSIDIAN_HITS = 6
+_EXCERPT_CHARS = 400
+
+
+def extract_search_keywords(text: str, *, max_terms: int = 8) -> list[str]:
+    """Pull meaningful terms from title + issue for memory/Obsidian search."""
+    raw = re.findall(r"[a-zA-Z0-9][a-zA-Z0-9._/-]{1,}", text.lower())
+    terms: list[str] = []
+    seen: set[str] = set()
+    for tok in raw:
+        if tok in _STOPWORDS or len(tok) < 3:
+            continue
+        if tok.isdigit():
+            continue
+        if tok not in seen:
+            seen.add(tok)
+            terms.append(tok)
+        if len(terms) >= max_terms:
+            break
+    return terms
+
+
+def _parse_memory_tool_result(result: dict | None) -> list[dict]:
+    """Normalize memory MCP tool results into a list of entry dicts."""
+    if not result:
+        return []
+
+    structured = result.get("structuredContent")
+    if isinstance(structured, dict):
+        for key in ("memories", "results", "entries", "items"):
+            val = structured.get(key)
+            if isinstance(val, list):
+                return [x for x in val if isinstance(x, dict)]
+        if structured.get("title") or structured.get("content"):
+            return [structured]
+
+    for block in result.get("content", []):
+        if not isinstance(block, dict):
+            continue
+        text = block.get("text")
+        if not text:
+            continue
+        text = text.strip()
+        try:
+            parsed = json.loads(text)
+        except json.JSONDecodeError:
+            continue
+        if isinstance(parsed, list):
+            return [x for x in parsed if isinstance(x, dict)]
+        if isinstance(parsed, dict):
+            for key in ("memories", "results", "entries", "items"):
+                val = parsed.get(key)
+                if isinstance(val, list):
+                    return [x for x in val if isinstance(x, dict)]
+            if parsed.get("title") or parsed.get("content"):
+                return [parsed]
+    return []
+
+
+def _memory_vessel_from_hit(hit: dict) -> str | None:
+    for tag in hit.get("tags") or []:
+        if isinstance(tag, str) and tag.startswith("vessel:"):
+            return tag.split(":", 1)[1].replace("-", " ")
+    content = hit.get("content") or ""
+    m = re.search(r"^Vessel:\s*(.+)$", content, re.MULTILINE)
+    if m:
+        return m.group(1).strip()
+    title = hit.get("title") or ""
+    m = re.search(r"^\[([^\]]+)\]", title)
+    if m and m.group(1).lower() not in ("task", "fix"):
+        return m.group(1).strip()
+    return None
+
+
+def _obsidian_vessel_from_text(text: str) -> str | None:
+    if not text.startswith("---"):
+        return None
+    end = text.find("\n---", 3)
+    if end == -1:
+        return None
+    front = text[3:end]
+    for line in front.splitlines():
+        if line.lower().startswith("vessel:"):
+            return line.split(":", 1)[1].strip()
+    return None
+
+
+def _memory_hit_from_entry(entry: dict, *, project_id: str) -> dict:
+    hit = {
+        "project_id": entry.get("project_id") or project_id,
+        "title": (entry.get("title") or "").strip(),
+        "content": (entry.get("content") or "")[:_EXCERPT_CHARS],
+        "memory_type": entry.get("memory_type") or entry.get("type") or "note",
+        "importance": entry.get("importance"),
+        "tags": entry.get("tags") or [],
+        "source": "memory",
+    }
+    return _enrich_hit_vessel(hit)
+
+
+def _enrich_hit_vessel(hit: dict, *, vessel_name: str | None = None) -> dict:
+    vessel = hit.get("vessel")
+    if not vessel and hit.get("source") == "memory":
+        vessel = _memory_vessel_from_hit(hit)
+    if not vessel and hit.get("source") == "obsidian":
+        vessel = _obsidian_vessel_from_text(hit.get("excerpt") or "")
+    if vessel:
+        hit["vessel"] = vessel
+    elif vessel_name:
+        hit["vessel"] = vessel_name
+    return hit
+
+
+async def _search_memory_projects(query: str, project_ids: list[str]) -> list[dict]:
+    hits: list[dict] = []
+    seen_titles: set[str] = set()
+
+    for project_id in project_ids:
+        try:
+            result = await integrations.memory_mcp_call(
+                "memory_search",
+                {"project_id": project_id, "query": query, "limit": 5},
+            )
+            for entry in _parse_memory_tool_result(result):
+                hit = _memory_hit_from_entry(entry, project_id=project_id)
+                key = hit["title"].lower()
+                if not hit["title"] or key in seen_titles:
+                    continue
+                seen_titles.add(key)
+                hits.append(hit)
+        except Exception as exc:  # noqa: BLE001
+            logger.warning("memory_search %s failed: %s", project_id, exc)
+
+    return hits[:_MAX_MEMORY_HITS]
+
+
+async def _recent_memory(project_ids: list[str]) -> list[dict]:
+    hits: list[dict] = []
+    for project_id in project_ids[:2]:
+        try:
+            result = await integrations.memory_mcp_call(
+                "memory_list_recent",
+                {"project_id": project_id, "limit": 10},
+            )
+            for entry in _parse_memory_tool_result(result):
+                hits.append(_memory_hit_from_entry(entry, project_id=project_id))
+        except Exception as exc:  # noqa: BLE001
+            logger.warning("memory_list_recent %s failed: %s", project_id, exc)
+    return hits
+
+
+def _obsidian_search_roots(vault: str) -> list[str]:
+    roots: list[str] = []
+    for rel in settings.obsidian_search_paths.split(","):
+        rel = rel.strip()
+        if not rel:
+            continue
+        path = os.path.join(vault, rel)
+        if os.path.isdir(path):
+            roots.append(path)
+    tasks = os.path.join(vault, settings.obsidian_tasks_folder)
+    if os.path.isdir(tasks) and tasks not in roots:
+        roots.insert(0, tasks)
+    return roots
+
+
+def _score_obsidian_file(
+    path: str, text: str, keywords: list[str], *, vessel_name: str | None = None
+) -> int:
+    name = os.path.basename(path).lower()
+    body = text.lower()
+    score = 0
+    for kw in keywords:
+        if kw in name:
+            score += 3
+        if kw in body:
+            score += 1
+    if vessel_name:
+        vslug = vessel_name.lower().replace(" ", "-")
+        note_vessel = _obsidian_vessel_from_text(text)
+        if note_vessel and note_vessel.lower() == vessel_name.lower():
+            score += 5
+        elif vslug in name or vessel_name.lower() in body:
+            score += 2
+    return score
+
+
+def _obsidian_title(text: str, filename: str) -> str:
+    for line in text.splitlines():
+        if line.startswith("# "):
+            return line[2:].strip()
+    return os.path.splitext(filename)[0].replace("-", " ")
+
+
+async def search_obsidian_notes(
+    title: str, issue: str, *, vessel_name: str | None = None
+) -> list[dict]:
+    """Search cloned Obsidian vault for notes related to this issue."""
+    vault = await integrations.ensure_vault()
+    if not vault:
+        return []
+
+    keywords = extract_search_keywords(f"{title} {issue}")
+    if vessel_name:
+        keywords = keywords + [vessel_name.lower().replace(" ", "-"), vessel_name.lower()]
+    if not keywords:
+        return []
+
+    # The vault scan is synchronous disk I/O — run it off the event loop.
+    return await asyncio.to_thread(_scan_obsidian_notes, vault, keywords, vessel_name)
+
+
+def _scan_obsidian_notes(
+    vault: str, keywords: list[str], vessel_name: str | None
+) -> list[dict]:
+    scored: list[tuple[int, dict]] = []
+    for root in _obsidian_search_roots(vault):
+        for dirpath, _, files in os.walk(root):
+            for filename in files:
+                if not filename.endswith(".md"):
+                    continue
+                abs_path = os.path.join(dirpath, filename)
+                try:
+                    with open(abs_path, encoding="utf-8", errors="replace") as fh:
+                        text = fh.read()
+                except OSError:
+                    continue
+                score = _score_obsidian_file(abs_path, text, keywords, vessel_name=vessel_name)
+                if score <= 0:
+                    continue
+                rel = os.path.relpath(abs_path, vault)
+                excerpt = text[:_EXCERPT_CHARS].strip()
+                hit = {
+                    "path": rel,
+                    "title": _obsidian_title(text, filename),
+                    "excerpt": excerpt,
+                    "score": score,
+                    "source": "obsidian",
+                }
+                scored.append((score, _enrich_hit_vessel(hit, vessel_name=vessel_name)))
+
+    scored.sort(key=lambda x: (-x[0], x[1]["path"]))
+    return [hit for _, hit in scored[:_MAX_OBSIDIAN_HITS]]
+
+
+async def gather_investigation_context(
+    title: str,
+    issue: str,
+    *,
+    vessel_name: str | None = None,
+    vessel_id: int | None = None,
+) -> dict[str, Any]:
+    """Search memory MCP + Obsidian vault for prior related investigations."""
+    keywords = extract_search_keywords(f"{title} {issue}")
+    query_parts = list(keywords)
+    if vessel_name:
+        query_parts.insert(0, vessel_name)
+    query = " ".join(query_parts) if query_parts else f"{title} {issue}"[:200]
+
+    project_ids = [
+        p.strip()
+        for p in settings.memory_search_projects.split(",")
+        if p.strip()
+    ]
+    if settings.memory_task_project_id not in project_ids:
+        project_ids.insert(0, settings.memory_task_project_id)
+
+    memory_hits = await _search_memory_projects(query, project_ids)
+
+    # If keyword search is sparse, add recent task memories for continuity.
+    if len(memory_hits) < 3:
+        recent = await _recent_memory(project_ids)
+        seen = {h["title"].lower() for h in memory_hits if h.get("title")}
+        for hit in recent:
+            t = hit.get("title", "").lower()
+            if t and t not in seen:
+                memory_hits.append(hit)
+                seen.add(t)
+            if len(memory_hits) >= _MAX_MEMORY_HITS:
+                break
+
+    obsidian_hits = await search_obsidian_notes(title, issue, vessel_name=vessel_name)
+
+    return {
+        "query": query,
+        "keywords": keywords,
+        "vessel_name": vessel_name,
+        "vessel_id": vessel_id,
+        "memory_hits": memory_hits[:_MAX_MEMORY_HITS],
+        "obsidian_hits": obsidian_hits,
+        "memory_count": len(memory_hits),
+        "obsidian_count": len(obsidian_hits),
+    }

backend/app/services/llm.py

@@ -0,0 +1,24 @@
+"""LLM client builders — delegates tier selection to model_router."""
+from __future__ import annotations
+
+from app.config import settings
+from app.services.llm_usage import ModelInfo
+from app.services.model_catalog import entry_for_model
+from app.services.model_config import get_routing_config, resolve_tier_model
+from app.services.model_router import build_chat_model, entry_to_info
+
+
+async def triage_model():
+    cfg = await get_routing_config()
+    entry = resolve_tier_model(cfg, "local")
+    if not entry:
+        entry = entry_for_model("ollama", settings.llm_local_model, "local")
+    return build_chat_model(entry, temperature=0.0)
+
+
+async def triage_model_info() -> ModelInfo:
+    cfg = await get_routing_config()
+    entry = resolve_tier_model(cfg, "local") or entry_for_model(
+        "ollama", settings.llm_local_model, "local"
+    )
+    return entry_to_info(entry, "triage")

backend/app/services/llm_usage.py

@@ -0,0 +1,140 @@
+"""Track LLM calls per task: provider, model, tokens, and estimated cost."""
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+from langchain_core.messages import BaseMessage
+from langchain_openai import ChatOpenAI
+
+# USD per 1M tokens (input, output). Local/Ollama is always zero.
+MODEL_PRICING: dict[str, tuple[float, float]] = {
+    "deepseek-chat": (0.14, 0.28),
+    "deepseek/deepseek-chat": (0.14, 0.28),
+    "qwen2.5:7b-instruct": (0.0, 0.0),
+    "qwen2.5-coder:14b": (0.0, 0.0),
+    # OpenRouter common fallbacks (approximate)
+    "anthropic/claude-3.5-sonnet": (3.0, 15.0),
+    "anthropic/claude-3.7-sonnet": (3.0, 15.0),
+    "openai/gpt-4o-mini": (0.15, 0.60),
+    "meta-llama/llama-3.1-70b-instruct": (0.52, 0.75),
+}
+
+
+@dataclass
+class ModelInfo:
+    step: str
+    provider: str  # local | api
+    backend: str  # ollama | gemini | deepseek | openrouter
+    model: str
+    display: str
+
+
+@dataclass
+class LlmUsageTracker:
+    """Collects LLM call records for one agent run."""
+
+    calls: list[dict[str, Any]] = field(default_factory=list)
+
+    async def invoke(
+        self,
+        step: str,
+        model: ChatOpenAI,
+        info: ModelInfo,
+        messages: list[BaseMessage],
+    ) -> Any:
+        response = await model.ainvoke(messages)
+        usage = extract_usage(response, info)
+        usage["step"] = step
+        self.calls.append(usage)
+        return response
+
+    def summary(self) -> dict[str, Any]:
+        run_cost = round(sum(c.get("cost_usd", 0.0) for c in self.calls), 6)
+        return {
+            "llm_usage": self.calls,
+            "run_cost_usd": run_cost,
+            "total_input_tokens": sum(c.get("input_tokens", 0) for c in self.calls),
+            "total_output_tokens": sum(c.get("output_tokens", 0) for c in self.calls),
+        }
+
+
+def extract_usage(response: Any, info: ModelInfo) -> dict[str, Any]:
+    meta = getattr(response, "response_metadata", None) or {}
+    token_usage = meta.get("token_usage") or {}
+    if not token_usage:
+        usage_meta = getattr(response, "usage_metadata", None)
+        if usage_meta:
+            token_usage = {
+                "prompt_tokens": usage_meta.get("input_tokens", 0),
+                "completion_tokens": usage_meta.get("output_tokens", 0),
+                "total_tokens": usage_meta.get("total_tokens", 0),
+            }
+
+    input_tokens = int(token_usage.get("prompt_tokens") or token_usage.get("input_tokens") or 0)
+    output_tokens = int(
+        token_usage.get("completion_tokens") or token_usage.get("output_tokens") or 0
+    )
+    model_name = meta.get("model_name") or info.model
+    cost = estimate_cost(model_name, info.backend, input_tokens, output_tokens)
+
+    return {
+        "provider": info.provider,
+        "backend": info.backend,
+        "model": model_name,
+        "display": info.display,
+        "input_tokens": input_tokens,
+        "output_tokens": output_tokens,
+        "total_tokens": input_tokens + output_tokens,
+        "cost_usd": cost,
+    }
+
+
+def estimate_cost(model: str, backend: str, input_tokens: int, output_tokens: int) -> float:
+    if backend == "ollama":
+        return 0.0
+    from app.services.model_catalog import CATALOG, entry_for_model
+
+    key = model.removeprefix("openrouter/")
+    for e in CATALOG:
+        if e.backend == backend and (e.model == model or e.model == key):
+            in_price, out_price = e.input_per_m, e.output_per_m
+            return round((input_tokens * in_price + output_tokens * out_price) / 1_000_000, 6)
+    entry = entry_for_model(backend, model, "economy")
+    in_price, out_price = entry.input_per_m, entry.output_per_m
+    return round((input_tokens * in_price + output_tokens * out_price) / 1_000_000, 6)
+
+
+def merge_run_history(
+    prior_runs: list[dict],
+    run_number: int,
+    run_report: dict,
+    llm_summary: dict,
+) -> dict:
+    """Build cumulative report with per-run and total cost."""
+    run_entry = {
+        "run_number": run_number,
+        "follow_up": run_report.get("follow_up"),
+        "summary": run_report.get("summary"),
+        "resolved": run_report.get("resolved"),
+        "devices_checked": run_report.get("devices_checked"),
+        "llm_usage": llm_summary.get("llm_usage", []),
+        "run_cost_usd": llm_summary.get("run_cost_usd", 0.0),
+        "generated_at": run_report.get("generated_at"),
+    }
+    all_runs = prior_runs + [run_entry]
+    total_cost = round(sum(r.get("run_cost_usd", 0.0) for r in all_runs), 6)
+
+    merged = {**run_report}
+    merged["run_number"] = run_number
+    merged["runs"] = all_runs
+    merged["llm_usage"] = llm_summary.get("llm_usage", [])
+    merged["run_cost_usd"] = llm_summary.get("run_cost_usd", 0.0)
+    merged["total_cost_usd"] = total_cost
+    merged["total_input_tokens"] = sum(
+        t.get("input_tokens", 0) for r in all_runs for t in r.get("llm_usage", [])
+    )
+    merged["total_output_tokens"] = sum(
+        t.get("output_tokens", 0) for r in all_runs for t in r.get("llm_usage", [])
+    )
+    return merged

backend/app/services/mcp_workspace.py

@@ -0,0 +1,225 @@
+"""Read/write MCP server source trees and push changes to Gitea."""
+from __future__ import annotations
+
+import logging
+import os
+from pathlib import Path
+
+from app.config import settings
+from app.mcp_manager.manager import get_manager
+
+logger = logging.getLogger(__name__)
+
+GIT_USER_NAME = "Agentic OS"
+GIT_USER_EMAIL = "agentic-os@paraskeva.net"
+
+
+def _local_src_dir(server: str) -> str:
+    return os.path.join(os.environ.get("MCP_LOCAL_DIR", "/app/mcp-servers"), server)
+
+
+def is_local_shipped(server: str) -> bool:
+    return os.path.isdir(_local_src_dir(server))
+
+
+def workspace_path(server: str) -> str:
+    """Writable directory where the running MCP code lives."""
+    manager = get_manager()
+    return manager._repo_dir(server)
+
+
+def _is_git_repo(path: str) -> bool:
+    return os.path.isdir(os.path.join(path, ".git"))
+
+
+async def _run(cmd: list[str], cwd: str) -> tuple[int, str]:
+    import asyncio
+
+    proc = await asyncio.create_subprocess_exec(
+        *cmd,
+        cwd=cwd,
+        stdout=asyncio.subprocess.PIPE,
+        stderr=asyncio.subprocess.STDOUT,
+    )
+    out, _ = await proc.communicate()
+    return proc.returncode or 0, out.decode(errors="replace")
+
+
+async def ensure_git_ready(server: str) -> tuple[str, bool]:
+    """Return (repo_path, can_push). Clone from Gitea if local-only without git."""
+    path = workspace_path(server)
+    if _is_git_repo(path):
+        await _ensure_remote(server, path)
+        return path, True
+
+    if not settings.gitea_token:
+        return path, False
+
+    clone_url = settings.gitea_clone_url(server)
+    git_dir = os.path.join(settings.runtime_dir, "mcp-repos", server)
+    if _is_git_repo(git_dir):
+        await _ensure_remote(server, git_dir)
+        return git_dir, True
+
+    os.makedirs(os.path.dirname(git_dir), exist_ok=True)
+    code, out = await _run(["git", "clone", "--depth", "1", clone_url, git_dir])
+    if code != 0:
+        logger.warning("MCP dev clone %s failed: %s", server, out[:300])
+        return path, False
+    return git_dir, True
+
+
+async def _ensure_remote(server: str, path: str) -> None:
+    from app.mcp_manager.manager import ensure_git_remote
+
+    await ensure_git_remote(server, path)
+
+
+async def list_source_files(server: str, *, max_files: int = 40) -> list[str]:
+    root = Path(workspace_path(server))
+    if not root.is_dir():
+        return []
+    out: list[str] = []
+    for p in sorted(root.rglob("*")):
+        if not p.is_file():
+            continue
+        rel = p.relative_to(root).as_posix()
+        if any(part in rel for part in (".venv", "__pycache__", ".git", "node_modules")):
+            continue
+        if p.suffix not in {".py", ".toml", ".md", ".json", ".yaml", ".yml"}:
+            continue
+        out.append(rel)
+        if len(out) >= max_files:
+            break
+    return out
+
+
+async def read_source_file(server: str, rel_path: str, *, max_chars: int = 80_000) -> str:
+    root = Path(workspace_path(server)).resolve()
+    target = (root / rel_path).resolve()
+    if not str(target).startswith(str(root)):
+        raise ValueError("Path escapes MCP workspace")
+    if not target.is_file():
+        raise FileNotFoundError(rel_path)
+    text = target.read_text(encoding="utf-8")
+    return text[:max_chars]
+
+
+async def write_source_file(server: str, rel_path: str, content: str) -> None:
+    root = Path(workspace_path(server)).resolve()
+    target = (root / rel_path).resolve()
+    if not str(target).startswith(str(root)):
+        raise ValueError("Path escapes MCP workspace")
+    target.parent.mkdir(parents=True, exist_ok=True)
+    target.write_text(content, encoding="utf-8")
+
+
+async def apply_file_edits(server: str, edits: list[dict]) -> list[str]:
+    written: list[str] = []
+    for edit in edits:
+        path = edit.get("path") or edit.get("file")
+        content = edit.get("content")
+        if not path or content is None:
+            continue
+        await write_source_file(server, path, content)
+        written.append(path)
+    return written
+
+
+async def commit_and_push(
+    server: str,
+    message: str,
+    files: list[str],
+    *,
+    use_git_clone: bool = True,
+) -> dict:
+    """Commit listed paths and push to origin. Returns {ok, commit, error}."""
+    if not files:
+        return {"ok": False, "error": "No files listed for git push"}
+
+    path, can_push = await ensure_git_ready(server)
+    if not can_push:
+        return {
+            "ok": False,
+            "error": "No Gitea git repo for this MCP — changes applied at runtime only",
+            "runtime_only": True,
+        }
+
+    # Sync runtime workspace files into git checkout when they differ
+    runtime = workspace_path(server)
+    if use_git_clone and os.path.abspath(runtime) != os.path.abspath(path):
+        for rel in files:
+            src = os.path.join(runtime, rel)
+            dst = os.path.join(path, rel)
+            if os.path.isfile(src):
+                os.makedirs(os.path.dirname(dst), exist_ok=True)
+                with open(src, encoding="utf-8") as fh:
+                    data = fh.read()
+                with open(dst, "w", encoding="utf-8") as fh:
+                    fh.write(data)
+
+    missing = [rel for rel in files if not os.path.isfile(os.path.join(path, rel))]
+    if missing:
+        return {
+            "ok": False,
+            "error": f"File(s) not found in MCP workspace: {', '.join(missing)}",
+        }
+
+    _, diff_out = await _run(["git", "diff", "HEAD", "--"] + files, cwd=path)
+    _, cached_out = await _run(["git", "diff", "--cached", "HEAD", "--"] + files, cwd=path)
+    if not diff_out.strip() and not cached_out.strip():
+        _, status_sb = await _run(["git", "status", "-sb"], cwd=path)
+        if "ahead" in status_sb:
+            code, out = await _run(["git", "push"], cwd=path)
+            if code != 0:
+                return {"ok": False, "error": f"push failed: {out[:500]}"}
+            commit_hash = (await _run(["git", "rev-parse", "--short", "HEAD"], cwd=path))[1].strip()
+            return {"ok": True, "commit": commit_hash, "repo": server, "note": "pushed existing commits"}
+        return {
+            "ok": True,
+            "commit": None,
+            "note": "no changes to push — files already match the last commit",
+            "already_up_to_date": True,
+        }
+
+    for rel in files:
+        await _run(["git", "add", rel], cwd=path)
+
+    code, out = await _run(
+        [
+            "git",
+            "-c",
+            f"user.email={GIT_USER_EMAIL}",
+            "-c",
+            f"user.name={GIT_USER_NAME}",
+            "commit",
+            "-m",
+            message,
+        ],
+        cwd=path,
+    )
+    if code != 0:
+        lowered = out.lower()
+        if "nothing to commit" in lowered or "nothing added to commit" in lowered:
+            return {
+                "ok": True,
+                "commit": None,
+                "note": "no changes to commit — already up to date",
+                "already_up_to_date": True,
+            }
+        return {"ok": False, "error": out[:500]}
+
+    code, out = await _run(["git", "push"], cwd=path)
+    if code != 0:
+        return {"ok": False, "error": f"push failed: {out[:500]}"}
+
+    commit_hash = (await _run(["git", "rev-parse", "--short", "HEAD"], cwd=path))[1].strip()
+    return {"ok": True, "commit": commit_hash, "repo": server}
+
+
+def list_registered_tools(server: str) -> list[str]:
+    manager = get_manager()
+    state = manager.servers.get(server)
+    if not state:
+        return []
+    return [t["name"] for t in state.tools]

backend/app/services/model_catalog.py

@@ -0,0 +1,106 @@
+"""Curated LLM catalog: tiers, backends, and pricing hints."""
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class ModelEntry:
+    id: str
+    backend: str  # ollama | gemini | deepseek | openrouter
+    model: str
+    tier: str  # local | economy | premium
+    label: str
+    input_per_m: float  # USD per 1M input tokens
+    output_per_m: float
+
+    @property
+    def provider(self) -> str:
+        return "local" if self.backend == "ollama" else "api"
+
+    @property
+    def display(self) -> str:
+        if self.backend == "ollama":
+            return f"Local · Ollama · {self.model}"
+        if self.backend == "gemini":
+            return f"API · Gemini · {self.model}"
+        if self.backend == "deepseek":
+            return f"API · DeepSeek · {self.model}"
+        return f"API · OpenRouter · {self.model}"
+
+
+# Curated models users can pick from the UI / config.
+CATALOG: list[ModelEntry] = [
+    # Local (Ollama)
+    ModelEntry("local-qwen25-7b", "ollama", "qwen2.5:7b-instruct", "local", "Qwen 2.5 7B Instruct", 0, 0),
+    ModelEntry("local-qwen25-coder-14b", "ollama", "qwen2.5-coder:14b", "local", "Qwen 2.5 Coder 14B", 0, 0),
+    ModelEntry("local-llama32-3b", "ollama", "llama3.2:3b", "local", "Llama 3.2 3B", 0, 0),
+    # Gemini economy (Google AI Studio — free tier eligible)
+    ModelEntry("gemini-flash-25", "gemini", "gemini-2.5-flash", "economy", "Gemini 2.5 Flash", 0.0, 0.0),
+    ModelEntry("gemini-flash-lite-25", "gemini", "gemini-2.5-flash-lite", "economy", "Gemini 2.5 Flash-Lite", 0.0, 0.0),
+    ModelEntry("gemini-flash-20", "gemini", "gemini-2.0-flash", "economy", "Gemini 2.0 Flash", 0.0, 0.0),
+    # Gemini premium
+    ModelEntry("gemini-pro-25", "gemini", "gemini-2.5-pro", "premium", "Gemini 2.5 Pro", 0.0, 0.0),
+    # DeepSeek economy
+    ModelEntry("ds-chat", "deepseek", "deepseek-chat", "economy", "DeepSeek Chat", 0.14, 0.28),
+    # DeepSeek premium
+    ModelEntry("ds-reasoner", "deepseek", "deepseek-reasoner", "premium", "DeepSeek Reasoner", 0.55, 2.19),
+    # OpenRouter economy
+    ModelEntry("or-ds-chat", "openrouter", "deepseek/deepseek-chat", "economy", "DeepSeek Chat (OR)", 0.14, 0.28),
+    ModelEntry("or-gemini-flash", "openrouter", "google/gemini-2.5-flash-preview", "economy", "Gemini 2.5 Flash", 0.15, 0.60),
+    ModelEntry("or-gpt4o-mini", "openrouter", "openai/gpt-4o-mini", "economy", "GPT-4o Mini", 0.15, 0.60),
+    ModelEntry("or-qwen25-72b", "openrouter", "qwen/qwen-2.5-72b-instruct", "economy", "Qwen 2.5 72B", 0.35, 0.40),
+    # OpenRouter premium
+    ModelEntry("or-claude-35-sonnet", "openrouter", "anthropic/claude-3.5-sonnet", "premium", "Claude 3.5 Sonnet", 3.0, 15.0),
+    ModelEntry("or-claude-37-sonnet", "openrouter", "anthropic/claude-3.7-sonnet", "premium", "Claude 3.7 Sonnet", 3.0, 15.0),
+    ModelEntry("or-gpt4o", "openrouter", "openai/gpt-4o", "premium", "GPT-4o", 2.5, 10.0),
+]
+
+_BY_ID: dict[str, ModelEntry] = {m.id: m for m in CATALOG}
+
+
+def get_entry(entry_id: str) -> ModelEntry | None:
+    return _BY_ID.get(entry_id)
+
+
+def entry_for_model(backend: str, model: str, tier: str = "economy") -> ModelEntry:
+    """Find catalog entry or synthesize one for custom model IDs."""
+    for e in CATALOG:
+        if e.backend == backend and e.model == model:
+            return e
+    label = model.split("/")[-1] if "/" in model else model
+    pricing = (0.50, 1.50) if tier != "local" else (0.0, 0.0)
+    return ModelEntry(
+        id=f"custom-{backend}-{model.replace('/', '-')}",
+        backend=backend,
+        model=model,
+        tier=tier,
+        label=label,
+        input_per_m=pricing[0],
+        output_per_m=pricing[1],
+    )
+
+
+def catalog_by_tier() -> dict[str, dict[str, list[dict]]]:
+    """Group catalog for API/UI: tier -> backend -> entries."""
+    out: dict[str, dict[str, list[dict]]] = {
+        "local": {"ollama": []},
+        "economy": {"gemini": [], "deepseek": [], "openrouter": []},
+        "premium": {"gemini": [], "deepseek": [], "openrouter": []},
+    }
+    for e in CATALOG:
+        out[e.tier].setdefault(e.backend, []).append(_entry_dict(e))
+    return out
+
+
+def _entry_dict(e: ModelEntry) -> dict:
+    return {
+        "id": e.id,
+        "backend": e.backend,
+        "model": e.model,
+        "tier": e.tier,
+        "label": e.label,
+        "input_per_m": e.input_per_m,
+        "output_per_m": e.output_per_m,
+        "display": e.display,
+    }

backend/app/services/model_config.py

@@ -0,0 +1,181 @@
+"""Runtime LLM routing configuration (env defaults + Redis overrides)."""
+from __future__ import annotations
+
+import json
+import time
+from dataclasses import asdict, dataclass, field
+
+from app.config import settings
+from app.services.model_catalog import ModelEntry, entry_for_model, get_entry
+
+ROUTING_KEY = "agentic:llm:routing"
+
+# Short in-process cache: routing config is read many times per task but
+# changes only via the admin /api/llm/routing endpoint.
+_CONFIG_CACHE_TTL = 30.0
+_config_cache: tuple[float, "LlmRoutingConfig"] | None = None
+
+DEFAULT_CLOUD_PROVIDER_ORDER = ["gemini", "deepseek", "openrouter"]
+
+
+@dataclass
+class LlmRoutingConfig:
+    """Selected models per tier. Always tries local first, then escalates."""
+
+    local_model_id: str = "local-qwen25-7b"
+    economy_gemini_id: str = "gemini-flash-25"
+    economy_deepseek_id: str = "ds-chat"
+    economy_openrouter_id: str = "or-ds-chat"
+    premium_gemini_id: str = "gemini-pro-25"
+    premium_deepseek_id: str = "ds-reasoner"
+    premium_openrouter_id: str = "or-claude-35-sonnet"
+    # Cloud provider cascade within each tier: gemini → deepseek → openrouter
+    cloud_provider_order: list[str] = field(default_factory=lambda: list(DEFAULT_CLOUD_PROVIDER_ORDER))
+    auto_escalate: bool = True
+    # Cap escalation: local | economy | premium
+    max_tier: str = "premium"
+
+    def to_dict(self) -> dict:
+        return asdict(self)
+
+    def resolve(self, entry_id: str) -> ModelEntry | None:
+        entry = get_entry(entry_id)
+        if entry:
+            return entry
+        return None
+
+
+def _defaults_from_env() -> LlmRoutingConfig:
+    """Build defaults using env vars and catalog IDs where possible."""
+    local = settings.llm_local_model
+    cfg = LlmRoutingConfig(
+        local_model_id=_find_id("ollama", local, "local") or "local-qwen25-7b",
+        economy_gemini_id=_find_id("gemini", settings.llm_economy_gemini, "economy") or "gemini-flash-25",
+        economy_deepseek_id=_find_id("deepseek", settings.llm_economy_deepseek, "economy") or "ds-chat",
+        economy_openrouter_id=_find_id("openrouter", settings.llm_economy_openrouter, "economy") or "or-ds-chat",
+        premium_gemini_id=_find_id("gemini", settings.llm_premium_gemini, "premium") or "gemini-pro-25",
+        premium_deepseek_id=_find_id("deepseek", settings.llm_premium_deepseek, "premium") or "ds-reasoner",
+        premium_openrouter_id=_find_id("openrouter", settings.llm_premium_openrouter, "premium")
+        or "or-claude-35-sonnet",
+        auto_escalate=settings.llm_auto_escalate,
+        max_tier=settings.llm_max_tier,
+    )
+    if settings.llm_cloud_provider_order:
+        cfg.cloud_provider_order = [
+            p.strip() for p in settings.llm_cloud_provider_order.split(",") if p.strip()
+        ]
+    return cfg
+
+
+def _find_id(backend: str, model: str, tier: str) -> str | None:
+    from app.services.model_catalog import CATALOG
+
+    for e in CATALOG:
+        if e.backend == backend and e.model == model and e.tier == tier:
+            return e.id
+    return None
+
+
+async def get_routing_config(*, use_cache: bool = True) -> LlmRoutingConfig:
+    from app.services.events import get_redis
+
+    global _config_cache
+    if use_cache and _config_cache is not None:
+        ts, cached = _config_cache
+        if time.monotonic() - ts < _CONFIG_CACHE_TTL:
+            return cached
+
+    cfg = _defaults_from_env()
+    try:
+        raw = await get_redis().get(ROUTING_KEY)
+        if raw:
+            data = json.loads(raw)
+            for k, v in data.items():
+                if hasattr(cfg, k):
+                    setattr(cfg, k, v)
+    except Exception:  # noqa: BLE001
+        pass
+
+    _config_cache = (time.monotonic(), cfg)
+    return cfg
+
+
+async def save_routing_config(cfg: LlmRoutingConfig) -> None:
+    from app.services.events import get_redis
+
+    global _config_cache
+    await get_redis().set(ROUTING_KEY, json.dumps(cfg.to_dict()))
+    _config_cache = (time.monotonic(), cfg)
+
+
+def available_backends() -> dict[str, bool]:
+    return {
+        "ollama": True,
+        "gemini": bool(settings.gemini_api_key),
+        "deepseek": bool(settings.deepseek_api_key),
+        "openrouter": bool(settings.openrouter_api_key),
+    }
+
+
+def _provider_entry(cfg: LlmRoutingConfig, tier: str, provider: str) -> ModelEntry | None:
+    avail = available_backends()
+    if tier == "economy":
+        if provider == "gemini" and avail["gemini"]:
+            return cfg.resolve(cfg.economy_gemini_id) or entry_for_model(
+                "gemini", settings.llm_economy_gemini, "economy"
+            )
+        if provider == "deepseek" and avail["deepseek"]:
+            return cfg.resolve(cfg.economy_deepseek_id) or entry_for_model(
+                "deepseek", settings.llm_economy_deepseek, "economy"
+            )
+        if provider == "openrouter" and avail["openrouter"]:
+            return cfg.resolve(cfg.economy_openrouter_id) or entry_for_model(
+                "openrouter", settings.llm_economy_openrouter, "economy"
+            )
+    elif tier == "premium":
+        if provider == "gemini" and avail["gemini"]:
+            return cfg.resolve(cfg.premium_gemini_id) or entry_for_model(
+                "gemini", settings.llm_premium_gemini, "premium"
+            )
+        if provider == "deepseek" and avail["deepseek"]:
+            return cfg.resolve(cfg.premium_deepseek_id) or entry_for_model(
+                "deepseek", settings.llm_premium_deepseek, "premium"
+            )
+        if provider == "openrouter" and avail["openrouter"]:
+            return cfg.resolve(cfg.premium_openrouter_id) or entry_for_model(
+                "openrouter", settings.llm_premium_openrouter, "premium"
+            )
+    return None
+
+
+def resolve_tier_models(cfg: LlmRoutingConfig, tier: str) -> list[ModelEntry]:
+    """Return ordered models for a tier (local = one entry; cloud = provider cascade)."""
+    if tier == "local":
+        if not available_backends()["ollama"]:
+            return []
+        e = cfg.resolve(cfg.local_model_id) or entry_for_model("ollama", settings.llm_local_model, "local")
+        return [e]
+
+    models: list[ModelEntry] = []
+    seen: set[str] = set()
+    for provider in cfg.cloud_provider_order:
+        entry = _provider_entry(cfg, tier, provider)
+        if entry and entry.id not in seen:
+            models.append(entry)
+            seen.add(entry.id)
+
+    # Fallback: any configured cloud provider with a key
+    for provider in DEFAULT_CLOUD_PROVIDER_ORDER:
+        if provider in cfg.cloud_provider_order:
+            continue
+        entry = _provider_entry(cfg, tier, provider)
+        if entry and entry.id not in seen:
+            models.append(entry)
+            seen.add(entry.id)
+    return models
+
+
+def resolve_tier_model(cfg: LlmRoutingConfig, tier: str) -> ModelEntry | None:
+    """Pick the first available model for a tier (legacy single-model callers)."""
+    models = resolve_tier_models(cfg, tier)
+    return models[0] if models else None

backend/app/services/model_discovery.py

@@ -0,0 +1,182 @@
+"""Live model discovery from Ollama, Gemini, DeepSeek, and OpenRouter APIs."""
+from __future__ import annotations
+
+import asyncio
+import logging
+from typing import Any
+
+import httpx
+
+from app.config import settings
+
+logger = logging.getLogger(__name__)
+
+# DeepSeek catalog fallbacks when /models is unavailable or incomplete.
+KNOWN_DEEPSEEK_MODELS = [
+    {"id": "deepseek-chat", "label": "DeepSeek Chat", "tier_hint": "economy"},
+    {"id": "deepseek-reasoner", "label": "DeepSeek Reasoner", "tier_hint": "premium"},
+]
+
+
+async def list_ollama_models() -> dict[str, Any]:
+    base = settings.ollama_base_url.rstrip("/")
+    try:
+        async with httpx.AsyncClient(timeout=10.0) as client:
+            resp = await client.get(f"{base}/api/tags")
+            resp.raise_for_status()
+            payload = resp.json()
+    except Exception as exc:  # noqa: BLE001
+        logger.debug("Ollama model list failed: %s", exc)
+        return {"backend": "ollama", "available": False, "error": str(exc), "models": []}
+
+    models = []
+    for item in payload.get("models") or []:
+        name = item.get("name") or item.get("model") or ""
+        details = item.get("details") or {}
+        caps = item.get("capabilities") or []
+        models.append(
+            {
+                "id": name,
+                "label": name,
+                "size_bytes": item.get("size"),
+                "parameter_size": details.get("parameter_size"),
+                "quantization": details.get("quantization_level"),
+                "context_length": details.get("context_length"),
+                "capabilities": caps,
+                "supports_tools": "tools" in caps,
+            }
+        )
+    return {"backend": "ollama", "available": True, "models": models}
+
+
+async def list_gemini_models() -> dict[str, Any]:
+    if not settings.gemini_api_key:
+        return {
+            "backend": "gemini",
+            "available": False,
+            "error": "GEMINI_API_KEY not set",
+            "models": [],
+        }
+    url = "https://generativelanguage.googleapis.com/v1beta/models"
+    try:
+        async with httpx.AsyncClient(timeout=15.0) as client:
+            resp = await client.get(url, params={"key": settings.gemini_api_key})
+            resp.raise_for_status()
+            payload = resp.json()
+    except Exception as exc:  # noqa: BLE001
+        logger.debug("Gemini model list failed: %s", exc)
+        return {"backend": "gemini", "available": False, "error": str(exc), "models": []}
+
+    models = []
+    for item in payload.get("models") or []:
+        methods = item.get("supportedGenerationMethods") or []
+        if "generateContent" not in methods:
+            continue
+        raw_name = item.get("name") or ""
+        model_id = raw_name.removeprefix("models/")
+        display = item.get("displayName") or model_id
+        models.append(
+            {
+                "id": model_id,
+                "label": display,
+                "input_token_limit": item.get("inputTokenLimit"),
+                "output_token_limit": item.get("outputTokenLimit"),
+                "description": item.get("description"),
+            }
+        )
+    models.sort(key=lambda m: m["id"])
+    return {"backend": "gemini", "available": True, "models": models}
+
+
+async def list_deepseek_models() -> dict[str, Any]:
+    if not settings.deepseek_api_key:
+        return {
+            "backend": "deepseek",
+            "available": False,
+            "error": "DEEPSEEK_API_KEY not set",
+            "models": [],
+        }
+    try:
+        async with httpx.AsyncClient(timeout=15.0) as client:
+            resp = await client.get(
+                "https://api.deepseek.com/models",
+                headers={"Authorization": f"Bearer {settings.deepseek_api_key}"},
+            )
+            resp.raise_for_status()
+            payload = resp.json()
+        models = [
+            {
+                "id": item.get("id") or "",
+                "label": item.get("id") or "",
+                "owned_by": item.get("owned_by"),
+            }
+            for item in payload.get("data") or []
+            if item.get("id")
+        ]
+        if not models:
+            models = [dict(m) for m in KNOWN_DEEPSEEK_MODELS]
+        return {"backend": "deepseek", "available": True, "models": models}
+    except Exception as exc:  # noqa: BLE001
+        logger.debug("DeepSeek model list failed: %s", exc)
+        return {
+            "backend": "deepseek",
+            "available": True,
+            "error": str(exc),
+            "models": [dict(m) for m in KNOWN_DEEPSEEK_MODELS],
+            "note": "Using known DeepSeek model IDs (API list unavailable)",
+        }
+
+
+async def list_openrouter_models() -> dict[str, Any]:
+    if not settings.openrouter_api_key:
+        return {
+            "backend": "openrouter",
+            "available": False,
+            "error": "OPENROUTER_API_KEY not set",
+            "models": [],
+        }
+    try:
+        async with httpx.AsyncClient(timeout=20.0) as client:
+            resp = await client.get(
+                "https://openrouter.ai/api/v1/models",
+                headers={"Authorization": f"Bearer {settings.openrouter_api_key}"},
+            )
+            resp.raise_for_status()
+            payload = resp.json()
+    except Exception as exc:  # noqa: BLE001
+        logger.debug("OpenRouter model list failed: %s", exc)
+        return {"backend": "openrouter", "available": False, "error": str(exc), "models": []}
+
+    models = []
+    for item in payload.get("data") or []:
+        pricing = item.get("pricing") or {}
+        models.append(
+            {
+                "id": item.get("id") or "",
+                "label": item.get("name") or item.get("id") or "",
+                "context_length": item.get("context_length"),
+                "input_per_token": pricing.get("prompt"),
+                "output_per_token": pricing.get("completion"),
+                "description": item.get("description"),
+            }
+        )
+    models.sort(key=lambda m: m["id"])
+    return {"backend": "openrouter", "available": True, "models": models}
+
+
+async def list_all_provider_models() -> dict[str, Any]:
+    ollama, gemini, deepseek, openrouter = await asyncio.gather(
+        list_ollama_models(),
+        list_gemini_models(),
+        list_deepseek_models(),
+        list_openrouter_models(),
+    )
+    return {
+        "providers": {
+            "ollama": ollama,
+            "gemini": gemini,
+            "deepseek": deepseek,
+            "openrouter": openrouter,
+        },
+        "routing_order": settings.llm_cloud_provider_order,
+    }

backend/app/services/model_router.py

@@ -0,0 +1,268 @@
+"""Tiered model routing: local first, auto-escalate to economy/premium when needed."""
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+from langchain_core.messages import BaseMessage
+from langchain_openai import ChatOpenAI
+
+from app.config import settings
+from app.services.llm_usage import LlmUsageTracker, ModelInfo
+from app.services.model_catalog import ModelEntry
+from app.services.model_config import LlmRoutingConfig, get_routing_config, resolve_tier_models
+
+logger = logging.getLogger(__name__)
+
+TIER_ORDER = ["local", "economy", "premium"]
+TIER_RANK = {t: i for i, t in enumerate(TIER_ORDER)}
+
+GEMINI_OPENAI_BASE = "https://generativelanguage.googleapis.com/v1beta/openai/"
+
+
+def build_chat_model(entry: ModelEntry, temperature: float = 0.1) -> ChatOpenAI:
+    if entry.backend == "ollama":
+        return ChatOpenAI(
+            model=entry.model,
+            temperature=temperature,
+            base_url=f"{settings.ollama_base_url.rstrip('/')}/v1",
+            api_key="ollama",
+            timeout=120,
+            max_retries=2,
+        )
+    if entry.backend == "gemini":
+        return ChatOpenAI(
+            model=entry.model,
+            temperature=temperature,
+            base_url=GEMINI_OPENAI_BASE,
+            api_key=settings.gemini_api_key,
+            timeout=120,
+            max_retries=2,
+        )
+    if entry.backend == "deepseek":
+        return ChatOpenAI(
+            model=entry.model,
+            temperature=temperature,
+            base_url="https://api.deepseek.com/v1",
+            api_key=settings.deepseek_api_key,
+            timeout=120,
+            max_retries=2,
+        )
+    return ChatOpenAI(
+        model=entry.model,
+        temperature=temperature,
+        base_url="https://openrouter.ai/api/v1",
+        api_key=settings.openrouter_api_key,
+        timeout=120,
+        max_retries=2,
+        default_headers={"HTTP-Referer": "http://localhost:3000", "X-Title": "Agentic OS"},
+    )
+
+
+def entry_to_info(entry: ModelEntry, step: str) -> ModelInfo:
+    return ModelInfo(
+        step=step,
+        provider=entry.provider,
+        backend=entry.backend,
+        model=entry.model,
+        display=entry.display,
+    )
+
+
+def plan_reasoning_tiers(
+    triage: dict,
+    diagnostics: list[dict],
+    *,
+    prior_context: dict | None = None,
+    follow_up: str | None = None,
+    run_number: int = 1,
+    cfg: LlmRoutingConfig | None = None,
+    min_tier: str | None = None,
+) -> tuple[list[str], str]:
+    """Return ordered tiers to try and a human-readable routing rationale."""
+    cfg = cfg or LlmRoutingConfig()
+    severity = (triage.get("severity") or "medium").lower()
+    recommended = (triage.get("recommended_tier") or "local").lower()
+    needs_cloud = bool(triage.get("needs_cloud_reasoning"))
+    fail_count = sum(1 for d in diagnostics if not d.get("ok"))
+    diag_count = len(diagnostics)
+
+    max_tier = "local"
+    reasons: list[str] = ["Always start with local LLM"]
+
+    if needs_cloud:
+        max_tier = "economy"
+        reasons.append("Triage flagged cloud reasoning helpful")
+    if severity in ("high", "critical"):
+        max_tier = "premium" if severity == "critical" else max([max_tier, "economy"], key=lambda t: TIER_RANK[t])
+        reasons.append(f"Severity is {severity}")
+    if recommended == "premium":
+        max_tier = "premium"
+        reasons.append("Triage recommended premium tier")
+    elif recommended == "economy" and TIER_RANK[max_tier] < TIER_RANK["economy"]:
+        max_tier = "economy"
+        reasons.append("Triage recommended economy tier")
+    if fail_count >= 2 or (fail_count >= 1 and diag_count >= 4 and fail_count / max(diag_count, 1) > 0.25):
+        max_tier = max([max_tier, "economy"], key=lambda t: TIER_RANK[t])
+        reasons.append(f"{fail_count} diagnostic failure(s)")
+    if fail_count >= 4 or (severity == "critical" and fail_count >= 1):
+        max_tier = "premium"
+        reasons.append("Complex failure pattern — premium model")
+    if prior_context and not prior_context.get("resolved"):
+        max_tier = max([max_tier, "economy"], key=lambda t: TIER_RANK[t])
+        reasons.append("Prior run unresolved")
+    if follow_up:
+        max_tier = max([max_tier, "economy"], key=lambda t: TIER_RANK[t])
+        reasons.append("Follow-up investigation")
+    if run_number > 1:
+        max_tier = max([max_tier, "economy"], key=lambda t: TIER_RANK[t])
+        reasons.append(f"Run {run_number} continuation")
+
+    # Respect configured cap
+    cap = cfg.max_tier if cfg.max_tier in TIER_RANK else "premium"
+    if TIER_RANK[max_tier] > TIER_RANK[cap]:
+        max_tier = cap
+        reasons.append(f"Capped at {cap} tier")
+
+    max_idx = TIER_RANK[max_tier]
+    tiers = TIER_ORDER[: max_idx + 1]
+
+    if not cfg.auto_escalate and len(tiers) > 1:
+        tiers = ["local"]
+        reasons.append("Auto-escalation disabled — local only")
+
+    if min_tier and min_tier in TIER_RANK:
+        from app.services.task_runtime import apply_min_tier_floor
+
+        tiers = apply_min_tier_floor(tiers, min_tier)
+        reasons.append(f"User requested {min_tier} API (skip local)")
+
+    return tiers, "; ".join(reasons)
+
+
+def _should_escalate(
+    analysis: dict,
+    tier: str,
+    tiers: list[str],
+    *,
+    diag_ok_ratio: float = 0.0,
+    is_last_provider: bool = False,
+) -> bool:
+    if is_last_provider and tier == tiers[-1]:
+        return False
+    if analysis.get("error"):
+        return True
+    confidence = analysis.get("confidence")
+    if analysis.get("needs_escalation"):
+        return True
+    if confidence is not None and float(confidence) < 0.65 and diag_ok_ratio < 0.75:
+        return True
+    if not analysis.get("summary") and not analysis.get("root_cause"):
+        return True
+    return False
+
+
+async def reason_with_cascade(
+    tracker: LlmUsageTracker,
+    tiers: list[str],
+    messages: list[BaseMessage],
+    parse_json,
+    cfg: LlmRoutingConfig | None = None,
+    *,
+    diagnostics: list[dict] | None = None,
+    task_id: int | None = None,
+) -> tuple[dict, dict]:
+    """Try tiers in order; within cloud tiers cascade gemini → deepseek → openrouter."""
+    cfg = cfg or await get_routing_config()
+    analysis: dict[str, Any] = {}
+    used_tiers: list[str] = []
+    used_backends: list[str] = []
+    last_entry: ModelEntry | None = None
+    last_error: str | None = None
+    diag_ok_ratio = 0.0
+    if diagnostics:
+        diag_ok_ratio = sum(1 for d in diagnostics if d.get("ok")) / len(diagnostics)
+
+    from app.services.task_runtime import get_task_min_tier
+
+    finished = False
+    for tier in tiers:
+        min_tier = await get_task_min_tier(task_id) if task_id else None
+        if min_tier and TIER_RANK[tier] < TIER_RANK[min_tier]:
+            continue
+
+        entries = resolve_tier_models(cfg, tier)
+        if not entries:
+            logger.info("No models available for tier %s", tier)
+            continue
+
+        for idx, entry in enumerate(entries):
+            min_tier = await get_task_min_tier(task_id) if task_id else None
+            if min_tier and TIER_RANK[tier] < TIER_RANK[min_tier]:
+                break
+
+            last_entry = entry
+            is_last_provider = idx == len(entries) - 1
+            step_label = f"reason-{tier}-{entry.backend}"
+            try:
+                model = build_chat_model(entry, temperature=0.1)
+                info = entry_to_info(entry, step_label)
+                resp = await tracker.invoke(step_label, model, info, messages)
+                analysis = parse_json(resp.content) or {
+                    "summary": str(resp.content)[:1000],
+                    "resolved": False,
+                }
+                analysis["_tier_used"] = tier
+                analysis["_backend_used"] = entry.backend
+                analysis["_model"] = entry.display
+                used_tiers.append(tier)
+                used_backends.append(entry.backend)
+
+                if not cfg.auto_escalate or not _should_escalate(
+                    analysis,
+                    tier,
+                    tiers,
+                    diag_ok_ratio=diag_ok_ratio,
+                    is_last_provider=is_last_provider,
+                ):
+                    min_tier = await get_task_min_tier(task_id) if task_id else None
+                    if not (
+                        min_tier
+                        and TIER_RANK.get(min_tier, 0) > TIER_RANK.get(tier, 0)
+                    ):
+                        finished = True
+                        break
+                logger.info(
+                    "Escalating from %s/%s: confidence=%s needs_esc=%s",
+                    tier,
+                    entry.backend,
+                    analysis.get("confidence"),
+                    analysis.get("needs_escalation"),
+                )
+            except Exception as exc:  # noqa: BLE001
+                last_error = str(exc)
+                logger.warning("Reasoning %s/%s failed: %s", tier, entry.backend, exc)
+                analysis = {
+                    "summary": f"reasoning error ({tier}/{entry.backend}): {exc}",
+                    "resolved": False,
+                    "error": str(exc),
+                }
+                used_tiers.append(tier)
+                used_backends.append(entry.backend)
+                continue
+
+        if finished:
+            break
+
+    if last_error and not analysis.get("summary"):
+        analysis = {"summary": f"reasoning error: {last_error}", "resolved": False, "error": last_error}
+
+    routing_meta = {
+        "tiers_planned": tiers,
+        "tiers_used": used_tiers,
+        "backends_used": used_backends,
+        "final_tier": used_tiers[-1] if used_tiers else None,
+        "final_backend": last_entry.backend if last_entry else None,
+        "final_model": last_entry.display if last_entry else None,
+    }
+    return analysis, routing_meta

backend/app/services/task_present.py

@@ -0,0 +1,85 @@
+"""Serialize tasks with vessel names for API responses."""
+from __future__ import annotations
+
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.models.inventory import Vessel
+from app.models.task import Task
+from app.schemas import TaskDetail, TaskOut, TaskOverviewOut
+
+
+async def vessel_name_map(db: AsyncSession, vessel_ids: set[int]) -> dict[int, str]:
+    if not vessel_ids:
+        return {}
+    res = await db.execute(select(Vessel).where(Vessel.id.in_(vessel_ids)))
+    return {v.id: v.name for v in res.scalars().all()}
+
+
+def task_to_out(task: Task, *, vessel_name: str | None = None) -> TaskOut:
+    base = TaskOut.model_validate(task)
+    name = vessel_name
+    if name is None and task.report:
+        name = task.report.get("vessel_name")
+    return base.model_copy(update={"vessel_name": name})
+
+
+def task_to_detail(task: Task, *, vessel_name: str | None = None) -> TaskDetail:
+    base = TaskDetail.model_validate(task)
+    name = vessel_name
+    if name is None and task.report:
+        name = task.report.get("vessel_name")
+    return base.model_copy(update={"vessel_name": name})
+
+
+def _report_summary(report: dict | None) -> tuple[str | None, str | None]:
+    if not report:
+        return None, None
+    summary = report.get("executive_summary") or report.get("summary")
+    findings = report.get("findings") or []
+    key_finding = None
+    if isinstance(findings, list) and findings:
+        first = findings[0]
+        if isinstance(first, dict):
+            key_finding = first.get("summary") or first.get("description")
+        else:
+            key_finding = str(first)
+    if not key_finding:
+        key_finding = report.get("root_cause") or report.get("resolution")
+    return summary, key_finding
+
+
+def task_to_overview(task: Task, *, vessel_name: str | None = None) -> TaskOverviewOut:
+    report = task.report or {}
+    name = vessel_name or report.get("vessel_name")
+    summary, key_finding = _report_summary(report)
+    return TaskOverviewOut(
+        id=task.id,
+        title=task.title,
+        issue=task.issue,
+        status=task.status,
+        vessel_id=task.vessel_id,
+        vessel_name=name,
+        summary=summary,
+        key_finding=key_finding,
+        devices_checked=list(report.get("devices_checked") or []),
+        resolved=report.get("resolved"),
+        memory_id=task.memory_id or report.get("memory_id"),
+        obsidian_path=task.obsidian_path or report.get("obsidian_path"),
+        obsidian_push_error=report.get("obsidian_push_error"),
+        run_cost_usd=report.get("run_cost_usd"),
+        created_at=task.created_at,
+        updated_at=task.updated_at,
+    )
+
+
+async def tasks_to_out_list(db: AsyncSession, tasks: list[Task]) -> list[TaskOut]:
+    ids = {t.vessel_id for t in tasks if t.vessel_id}
+    vmap = await vessel_name_map(db, ids)
+    return [task_to_out(t, vessel_name=vmap.get(t.vessel_id) if t.vessel_id else None) for t in tasks]
+
+
+async def tasks_to_overview_list(db: AsyncSession, tasks: list[Task]) -> list[TaskOverviewOut]:
+    ids = {t.vessel_id for t in tasks if t.vessel_id}
+    vmap = await vessel_name_map(db, ids)
+    return [task_to_overview(t, vessel_name=vmap.get(t.vessel_id) if t.vessel_id else None) for t in tasks]

backend/app/services/task_runtime.py

@@ -0,0 +1,64 @@
+"""Per-task runtime flags (Redis) — e.g. user-requested LLM tier escalation and cancellation."""
+from __future__ import annotations
+
+from app.services.events import get_redis
+
+TIER_ORDER = ["local", "economy", "premium"]
+TIER_RANK = {t: i for i, t in enumerate(TIER_ORDER)}
+
+CANCELLABLE_STATUSES = frozenset({"queued", "running", "waiting_approval"})
+
+
+class TaskCancelledError(Exception):
+    """Raised when the user stops a running or queued task."""
+
+
+def _min_tier_key(task_id: int) -> str:
+    return f"agentic:task:{task_id}:min_tier"
+
+
+def _cancel_key(task_id: int) -> str:
+    return f"agentic:task:{task_id}:cancel"
+
+
+async def request_task_cancel(task_id: int) -> None:
+    await get_redis().set(_cancel_key(task_id), "1", ex=86400)
+
+
+async def is_task_cancel_requested(task_id: int) -> bool:
+    return bool(await get_redis().get(_cancel_key(task_id)))
+
+
+async def clear_task_cancel(task_id: int) -> None:
+    await get_redis().delete(_cancel_key(task_id))
+
+
+async def ensure_not_cancelled(task_id: int) -> None:
+    if await is_task_cancel_requested(task_id):
+        raise TaskCancelledError("Task cancelled by user")
+
+
+async def set_task_min_tier(task_id: int, tier: str) -> None:
+    if tier not in TIER_RANK:
+        raise ValueError(f"Invalid tier: {tier}")
+    await get_redis().set(_min_tier_key(task_id), tier, ex=86400)
+
+
+async def get_task_min_tier(task_id: int) -> str | None:
+    tier = await get_redis().get(_min_tier_key(task_id))
+    return tier if tier in TIER_RANK else None
+
+
+async def clear_task_min_tier(task_id: int) -> None:
+    await get_redis().delete(_min_tier_key(task_id))
+
+
+def apply_min_tier_floor(tiers: list[str], min_tier: str) -> list[str]:
+    """Drop tiers below min_tier (e.g. skip local when user requests economy)."""
+    if min_tier not in TIER_RANK:
+        return tiers
+    floor = TIER_RANK[min_tier]
+    filtered = [t for t in tiers if TIER_RANK[t] >= floor]
+    if filtered:
+        return filtered
+    return [min_tier]

backend/app/services/troubleshooting_rules.py

@@ -0,0 +1,297 @@
+"""Editable troubleshooting decision rules (YAML on disk).
+
+Rules standardize which devices to check and in what order, plus hints for the LLM.
+"""
+from __future__ import annotations
+
+import logging
+import os
+import re
+from dataclasses import dataclass
+
+import yaml
+
+from app.config import settings
+
+logger = logging.getLogger(__name__)
+
+_rules_cache: dict | None = None
+_rules_mtime: float = -1.0
+
+
+@dataclass
+class MatchedRule:
+    id: str
+    name: str
+    severity: str
+    devices: list[str]
+    order: list[str]
+    hints: list[str]
+    steps: list[str]
+    broad: bool = False
+
+    def as_dict(self) -> dict:
+        return {
+            "id": self.id,
+            "name": self.name,
+            "severity": self.severity,
+            "devices": self.devices,
+            "order": self.order,
+            "hints": self.hints,
+            "steps": self.steps,
+            "broad": self.broad,
+        }
+
+
+def rules_path() -> str:
+    return os.environ.get("TROUBLESHOOTING_RULES_PATH", settings.troubleshooting_rules_path)
+
+
+def _normalize(data: dict) -> dict:
+    data.setdefault("version", 1)
+    data.setdefault("broad_triggers", {"keywords": []})
+    data.setdefault("device_keywords", {})
+    data.setdefault("rules", [])
+    data.setdefault("default", {"severity": "medium", "devices": ["all"], "hints": []})
+    return data
+
+
+def invalidate_rules_cache() -> None:
+    global _rules_cache, _rules_mtime
+    _rules_cache = None
+    _rules_mtime = -1.0
+
+
+def load_rules(*, force: bool = False) -> dict:
+    global _rules_cache, _rules_mtime
+    path = rules_path()
+    try:
+        mtime = os.path.getmtime(path)
+    except OSError:
+        logger.warning("troubleshooting rules file missing: %s", path)
+        return _normalize({})
+
+    if not force and _rules_cache is not None and mtime == _rules_mtime:
+        return _rules_cache
+
+    try:
+        with open(path, encoding="utf-8") as fh:
+            data = _normalize(yaml.safe_load(fh) or {})
+    except yaml.YAMLError as exc:
+        logger.warning("invalid rules YAML %s: %s", path, exc)
+        data = _normalize({})
+
+    _rules_cache = data
+    _rules_mtime = mtime
+    return data
+
+
+def read_rules_raw() -> str:
+    path = rules_path()
+    try:
+        with open(path, encoding="utf-8") as fh:
+            return fh.read()
+    except OSError:
+        return ""
+
+
+def save_rules_raw(content: str) -> dict:
+    parsed = yaml.safe_load(content)
+    if not isinstance(parsed, dict):
+        raise ValueError("Rules file must be a YAML mapping at the top level")
+    normalized = _normalize(parsed)
+    path = rules_path()
+    parent = os.path.dirname(path)
+    if parent:
+        os.makedirs(parent, exist_ok=True)
+    with open(path, "w", encoding="utf-8") as fh:
+        fh.write(content if content.endswith("\n") else content + "\n")
+    invalidate_rules_cache()
+    return load_rules(force=True)
+
+
+def _text_has_term(text: str, term: str) -> bool:
+    term = term.lower().strip()
+    if not term:
+        return False
+    if " " in term:
+        return term in text
+    if len(term) <= 2:
+        return term in text.split()
+    return bool(re.search(rf"\b{re.escape(term)}\b", text))
+
+
+def is_broad_issue(title: str, issue: str, data: dict | None = None) -> bool:
+    data = data or load_rules()
+    text = f"{title} {issue}".lower()
+    keywords = data.get("broad_triggers", {}).get("keywords") or []
+    return any(kw in text for kw in keywords)
+
+
+def _rule_match_score(text: str, rule: dict) -> int:
+    """Higher score = more specific match. Zero means no match."""
+    match = rule.get("match") or {}
+    keywords = match.get("keywords") or []
+    all_of = match.get("all_of") or []
+
+    if all_of and not all(_text_has_term(text, t) for t in all_of):
+        return 0
+
+    score = 0
+    for kw in keywords:
+        if _text_has_term(text, kw):
+            # Multi-word phrases are more specific than single tokens like "sip".
+            score += 2 if " " in kw.strip() else 1
+
+    if all_of:
+        score += len(all_of) * 2
+
+    if score == 0 and not all_of:
+        return 0
+    return score
+
+
+def match_rule(title: str, issue: str, data: dict | None = None) -> MatchedRule | None:
+    data = data or load_rules()
+    text = f"{title} {issue}".lower()
+
+    if is_broad_issue(title, issue, data):
+        default = data.get("default", {})
+        return MatchedRule(
+            id="broad-health",
+            name="Full stack health check",
+            severity=default.get("severity", "medium"),
+            devices=default.get("devices", ["all"]),
+            order=[],
+            hints=list(default.get("hints") or ["Check all onboard devices layer by layer"]),
+            steps=[],
+            broad=True,
+        )
+
+    rules = data.get("rules") or []
+    best: dict | None = None
+    best_score = 0
+    best_priority = -1
+
+    for rule in rules:
+        score = _rule_match_score(text, rule)
+        if score <= 0:
+            continue
+        priority = rule.get("priority") or 0
+        if score > best_score or (score == best_score and priority > best_priority):
+            best = rule
+            best_score = score
+            best_priority = priority
+
+    if not best:
+        return None
+
+    return MatchedRule(
+        id=str(best.get("id") or "unnamed"),
+        name=str(best.get("name") or best.get("id") or "Rule"),
+        severity=str(best.get("severity") or "medium"),
+        devices=list(best.get("devices") or ["all"]),
+        order=list(best.get("order") or []),
+        hints=list(best.get("hints") or []),
+        steps=list(best.get("steps") or []),
+    )
+
+
+def device_keyword_map(data: dict | None = None) -> dict[str, list[str]]:
+    data = data or load_rules()
+    return {k: list(v) for k, v in (data.get("device_keywords") or {}).items()}
+
+
+def devices_mentioned_in_text(text: str, devices: list[dict], data: dict | None = None) -> list[dict]:
+    """Return onboard devices whose catalog keywords appear in *text*."""
+    data = data or load_rules()
+    kw_map = device_keyword_map(data)
+    matched_devices: list[dict] = []
+    for device in devices:
+        key = (device.get("catalog_key") or device.get("device_type") or "").lower()
+        keywords = kw_map.get(key, [])
+        terms = set(keywords + [key, device.get("device_type", ""), device.get("name", "")])
+        terms = {t.lower() for t in terms if t}
+        if any(_text_has_term(text, term) for term in terms if len(term) > 2):
+            matched_devices.append(device)
+    return matched_devices
+
+
+def select_devices_for_issue(
+    devices: list[dict],
+    issue: str,
+    *,
+    title: str = "",
+) -> tuple[list[dict], MatchedRule | None]:
+    if not devices:
+        return [], None
+
+    data = load_rules()
+    matched = match_rule(title, issue, data)
+
+    text = f"{title} {issue}".lower()
+
+    if matched:
+        if "all" in matched.devices:
+            selected = list(devices)
+        else:
+            want = {d.lower() for d in matched.devices}
+            selected = [d for d in devices if (d.get("catalog_key") or "").lower() in want]
+            # Union devices explicitly named in the issue (e.g. "proxmox and pfsense").
+            if not matched.broad:
+                seen = {(d.get("catalog_key") or "").lower() for d in selected}
+                for device in devices_mentioned_in_text(text, devices, data):
+                    key = (device.get("catalog_key") or "").lower()
+                    if key not in seen:
+                        selected.append(device)
+                        seen.add(key)
+
+        if matched.order:
+            order_map = {k.lower(): i for i, k in enumerate(matched.order)}
+            selected.sort(
+                key=lambda d: order_map.get((d.get("catalog_key") or "").lower(), 999)
+            )
+
+        if selected:
+            return selected, matched
+
+    if is_broad_issue(title, issue, data):
+        return devices, matched
+
+    matched_devices = devices_mentioned_in_text(text, devices, data)
+
+    # Never fall back to all devices — only run what the issue mentions.
+    return matched_devices, matched
+
+
+def rule_guidance_block(matched: MatchedRule | dict | None) -> str:
+    if not matched:
+        return ""
+    if isinstance(matched, dict):
+        name = matched.get("name", "")
+        rid = matched.get("id", "")
+        severity = matched.get("severity", "medium")
+        devices = matched.get("devices") or []
+        order = matched.get("order") or []
+        hints = matched.get("hints") or []
+        steps = matched.get("steps") or []
+    else:
+        name, rid, severity = matched.name, matched.id, matched.severity
+        devices, order, hints, steps = matched.devices, matched.order, matched.hints, matched.steps
+
+    lines = [
+        "\n\n--- Troubleshooting rule (standardized) ---",
+        f"Rule: {name} ({rid})",
+        f"Severity hint: {severity}",
+    ]
+    if devices:
+        lines.append(f"Target devices: {', '.join(devices)}")
+    if order:
+        lines.append(f"Diagnose order: {' → '.join(order)}")
+    if hints:
+        lines.append("Hints:")
+        lines.extend(f"- {h}" for h in hints)
+    if steps:
+        lines.append("Suggested steps:")
+        lines.extend(f"- {s}" for s in steps)
+    return "\n".join(lines)

backend/app/services/vessel_devices.py

@@ -0,0 +1,78 @@
+"""Helpers for building vessel device rows from catalog selections."""
+from __future__ import annotations
+
+from app.core.crypto import decrypt_secret, encrypt_secret
+from app.agent.asterisk_profiles import ASTERISK_CATALOG_KEYS
+from app.inventory_catalog import CatalogEntry, catalog_by_key
+from app.models.inventory import Device, Vessel
+from app.schemas import DeviceOut, VesselDeviceSelection
+from app.services.device_defaults import merge_selection_with_defaults
+
+
+def device_to_out(d: Device) -> DeviceOut:
+    out = DeviceOut.model_validate(d)
+    out.has_secret = bool(d.secret_enc)
+    return out
+
+
+def _resolve_address(vessel: Vessel, sel: VesselDeviceSelection) -> str:
+    if sel.address:
+        return sel.address
+    if vessel.public_ip:
+        return vessel.public_ip
+    raise ValueError(f"Device '{sel.catalog_key}' needs an address or vessel public IP")
+
+
+def build_device(
+    vessel: Vessel,
+    sel: VesselDeviceSelection,
+    entry: CatalogEntry,
+    *,
+    existing_secret_plain: str | None = None,
+) -> Device:
+    port, username, secret = merge_selection_with_defaults(
+        entry.key,
+        entry,
+        port=sel.port,
+        username=sel.username,
+        secret=sel.secret,
+        existing_secret=existing_secret_plain,
+    )
+    secret_enc = encrypt_secret(secret) if secret else None
+    return Device(
+        vessel_id=vessel.id,
+        catalog_key=entry.key,
+        name=f"{vessel.name} — {entry.label}",
+        device_type=entry.device_type,
+        address=_resolve_address(vessel, sel),
+        port=port,
+        mcp_server=entry.mcp_server,
+        username=username,
+        secret_enc=secret_enc,
+        notes=sel.notes,
+    )
+
+
+def validate_selections(selections: list[VesselDeviceSelection]) -> list[tuple[VesselDeviceSelection, CatalogEntry]]:
+    catalog = catalog_by_key()
+    enabled = [s for s in selections if s.enabled]
+    if not enabled:
+        raise ValueError("Select at least one device for this vessel")
+    out: list[tuple[VesselDeviceSelection, CatalogEntry]] = []
+    seen: set[str] = set()
+    asterisk_picked = False
+    for sel in enabled:
+        entry = catalog.get(sel.catalog_key)
+        if not entry:
+            raise ValueError(f"Unknown catalog key: {sel.catalog_key}")
+        if sel.catalog_key in seen:
+            raise ValueError(f"Duplicate device slot: {sel.catalog_key}")
+        if sel.catalog_key in ASTERISK_CATALOG_KEYS:
+            if asterisk_picked:
+                raise ValueError(
+                    "Only one Asterisk slot per vessel — pick GeneseasX (docker voip) OR TMGeneseas/satbox (native)"
+                )
+            asterisk_picked = True
+        seen.add(sel.catalog_key)
+        out.append((sel, entry))
+    return out

backend/app/worker.py

@@ -0,0 +1,137 @@
+"""Background worker: owns MCP sessions and runs the troubleshooting agent per task.
+
+Consumes task ids from the Redis queue, executes the LangGraph agent, and keeps the
+MCP status snapshot fresh for the API/UI.
+"""
+from __future__ import annotations
+
+import asyncio
+import logging
+
+from sqlalchemy import select
+
+from app.agent import run_task_agent
+from app.config import settings
+from app.database import SessionLocal, init_db
+from app.mcp_manager import get_manager
+from app.models.enums import TaskStatus
+from app.models.task import Task
+from app.services.events import TASK_QUEUE, get_redis
+from app.services.task_runtime import is_task_cancel_requested
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger("worker")
+
+
+async def _refresh_status_loop() -> None:
+    manager = get_manager()
+    while True:
+        await asyncio.sleep(5)
+        try:
+            while await manager.process_command_queue_once():
+                await asyncio.sleep(0)  # yield so task queue blpop can run
+        except Exception as exc:  # noqa: BLE001
+            logger.warning("MCP command processing failed: %s", exc)
+        try:
+            await manager.publish_status()
+        except Exception as exc:  # noqa: BLE001
+            logger.warning("status publish failed: %s", exc)
+
+
+async def _task_should_run(task_id: int) -> bool:
+    async with SessionLocal() as db:
+        res = await db.execute(select(Task).where(Task.id == task_id))
+        task = res.scalar_one_or_none()
+        if not task:
+            return False
+        if task.status == TaskStatus.cancelled:
+            return False
+    if await is_task_cancel_requested(task_id):
+        return False
+    return True
+
+
+async def _mark_task_running(task_id: int) -> None:
+    async with SessionLocal() as db:
+        res = await db.execute(select(Task).where(Task.id == task_id))
+        task = res.scalar_one_or_none()
+        if task and task.status == TaskStatus.queued:
+            task.status = TaskStatus.running
+            await db.commit()
+
+
+async def _run_task(task_id: int, sem: asyncio.Semaphore) -> None:
+    async with sem:
+        if not await _task_should_run(task_id):
+            logger.info("Skipping task %s (cancelled or missing)", task_id)
+            return
+        logger.info("Running task %s", task_id)
+        try:
+            await run_task_agent(task_id)
+        except asyncio.CancelledError:
+            logger.info("task %s asyncio task cancelled", task_id)
+        except Exception:  # noqa: BLE001
+            logger.exception("task %s crashed", task_id)
+
+
+async def _cancel_watch_loop(in_flight: dict[int, asyncio.Task]) -> None:
+    """Cancel in-flight asyncio tasks when the user requests stop via the API."""
+    while True:
+        await asyncio.sleep(1)
+        for task_id, handle in list(in_flight.items()):
+            if handle.done():
+                continue
+            try:
+                if await is_task_cancel_requested(task_id):
+                    logger.info("Cancelling in-flight asyncio task %s", task_id)
+                    handle.cancel()
+            except Exception as exc:  # noqa: BLE001
+                logger.debug("cancel watch for task %s: %s", task_id, exc)
+
+
+async def main() -> None:
+    await init_db()
+    manager = get_manager()
+    logger.info("Starting MCP manager (cloning + launching MCP servers)...")
+    await manager.startup()
+    logger.info("MCP manager ready: %s", [s["name"] for s in manager.status_snapshot()])
+
+    asyncio.create_task(_refresh_status_loop())
+
+    max_concurrent = max(1, settings.worker_max_concurrent_tasks)
+    sem = asyncio.Semaphore(max_concurrent)
+    in_flight: dict[int, asyncio.Task] = {}
+    asyncio.create_task(_cancel_watch_loop(in_flight))
+
+    redis = get_redis()
+    logger.info(
+        "Worker waiting for tasks on %s (max concurrent: %s)",
+        TASK_QUEUE,
+        max_concurrent,
+    )
+    while True:
+        item = await redis.blpop(TASK_QUEUE, timeout=5)
+        if not item:
+            in_flight = {tid: t for tid, t in in_flight.items() if not t.done()}
+            continue
+        _, task_id_str = item
+        try:
+            task_id = int(task_id_str)
+        except ValueError:
+            continue
+        if not await _task_should_run(task_id):
+            logger.info("Skipped dequeued task %s (already cancelled)", task_id)
+            continue
+        logger.info("Picked up task %s", task_id)
+        await _mark_task_running(task_id)
+        handle = asyncio.create_task(_run_task(task_id, sem))
+        in_flight[task_id] = handle
+
+        def _done(t: asyncio.Task, tid: int = task_id) -> None:
+            in_flight.pop(tid, None)
+
+        handle.add_done_callback(_done)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

backend/requirements.txt

@@ -0,0 +1,32 @@
+fastapi==0.115.6
+uvicorn[standard]==0.34.0
+pydantic==2.10.4
+pydantic-settings==2.7.1
+email-validator==2.2.0
+sqlalchemy[asyncio]==2.0.36
+asyncpg==0.30.0
+alembic==1.14.0
+python-jose[cryptography]==3.3.0
+passlib[bcrypt]==1.7.4
+bcrypt==4.2.1
+python-multipart==0.0.20
+cryptography==44.0.0
+httpx==0.28.1
+redis==5.2.1
+websockets==14.1
+
+# Agent engine
+langgraph==0.2.60
+langchain-core==0.3.28
+langchain-openai==0.2.14
+
+# MCP client (to drive the spawned MCP servers)
+mcp==1.2.0
+
+# Misc
+python-slugify==8.0.4
+pyyaml==6.0.2
+tenacity==9.0.0
+
+# Dev / tests
+pytest==8.3.4

backend/tests/__init__.py

@@ -0,0 +1 @@
+

backend/tests/test_agent.py

@@ -0,0 +1,96 @@
+"""Tests for agent helpers that need no external services."""
+from __future__ import annotations
+
+import os
+
+os.environ.setdefault("SECRET_KEY", "test-secret-key-for-unit-tests-only")
+
+from app.agent.graph import _parse_json, _report_findings, _scope_checked  # noqa: E402
+from app.agent.playbooks import playbook_for  # noqa: E402
+from app.models.enums import DeviceType  # noqa: E402
+from app.services.llm_usage import estimate_cost, merge_run_history  # noqa: E402
+from app.services.model_router import plan_reasoning_tiers  # noqa: E402
+from app.services.model_config import LlmRoutingConfig  # noqa: E402
+
+
+def test_parse_json_plain():
+    assert _parse_json('{"a": 1}') == {"a": 1}
+
+
+def test_parse_json_fenced():
+    text = """Here is the result:\n```json\n{"severity": "high", "plan": ["a"]}\n```"""
+    out = _parse_json(text)
+    assert out["severity"] == "high"
+    assert out["plan"] == ["a"]
+
+
+def test_parse_json_garbage():
+    assert _parse_json("no json here") == {}
+
+
+def test_playbook_lookup():
+    pb = playbook_for(DeviceType.pfsense, None)
+    assert pb is not None
+    assert pb["mcp_server"] == "pfsense-mcp"
+    assert pb["diagnostics"]
+
+
+def test_playbook_override():
+    pb = playbook_for(DeviceType.debian, "custom-ssh-mcp")
+    assert pb["mcp_server"] == "custom-ssh-mcp"
+
+
+def test_estimate_cost_local():
+    assert estimate_cost("qwen2.5:7b-instruct", "ollama", 1000, 500) == 0.0
+
+
+def test_merge_run_history():
+    prior = [{"run_number": 1, "run_cost_usd": 0.001, "llm_usage": []}]
+    report = {"summary": "still broken", "resolved": False, "devices_checked": ["proxmox"]}
+    llm = {"llm_usage": [{"step": "triage", "cost_usd": 0.0}], "run_cost_usd": 0.002}
+    merged = merge_run_history(prior, 2, report, llm)
+    assert merged["run_number"] == 2
+    assert len(merged["runs"]) == 2
+    assert merged["total_cost_usd"] == 0.003
+
+
+def test_plan_reasoning_local_first():
+    tiers, reason = plan_reasoning_tiers(
+        {"severity": "low", "needs_cloud_reasoning": False, "recommended_tier": "local"},
+        [],
+        cfg=LlmRoutingConfig(),
+    )
+    assert tiers == ["local"]
+    assert "local" in reason.lower()
+
+
+def test_plan_reasoning_escalates_critical():
+    tiers, _ = plan_reasoning_tiers(
+        {"severity": "critical", "needs_cloud_reasoning": True, "recommended_tier": "premium"},
+        [{"ok": False}, {"ok": False}],
+        cfg=LlmRoutingConfig(),
+    )
+    assert tiers == ["local", "economy", "premium"]
+
+
+def test_scope_checked_tracks_tools_per_device():
+    devices = [{"catalog_key": "proxmox", "name": "PVE"}, {"catalog_key": "pfsense", "name": "Firewall"}]
+    diagnostics = [
+        {"device": "proxmox", "tool": "proxmox_list_qemu", "ok": True},
+        {"device": "pfsense", "tool": "pfsense_get_system_status", "ok": True},
+    ]
+    scope = _scope_checked(devices, diagnostics)
+    assert scope[0]["device"] == "proxmox"
+    assert scope[0]["checks"] == ["proxmox_list_qemu"]
+    assert scope[1]["device"] == "pfsense"
+
+
+def test_report_findings_does_not_invent_unchecked_devices():
+    findings = _report_findings(
+        {"summary": "done", "resolution": "No critical issue found."},
+        [{"device": "proxmox", "tool": "proxmox_list_qemu", "ok": True}],
+        ["proxmox"],
+    )
+    assert "proxmox" in findings[0]["summary"]
+    assert "pfsense" not in findings[0]["summary"].lower()
+

backend/tests/test_approval_filter.py

@@ -0,0 +1,43 @@
+"""Tests for approval filtering."""
+from app.services.approval_filter import fix_requires_approval, split_proposed_fixes
+
+
+def test_connect_tools_never_require_approval():
+    fix = {
+        "description": "Connect to Proxmox",
+        "is_config_change": True,
+        "tool": "proxmox_connect",
+        "args": {},
+    }
+    assert fix_requires_approval(fix) is False
+
+
+def test_manual_advice_never_requires_approval():
+    fix = {
+        "description": "Install Docker on the VM",
+        "is_config_change": True,
+        "tool": "manual-config-change",
+    }
+    assert fix_requires_approval(fix) is False
+
+
+def test_write_tool_requires_approval():
+    fix = {
+        "description": "Add NAT rule for SIP",
+        "is_config_change": True,
+        "tool": "pfsense_create_firewall_rule",
+        "args": {"descr": "mcp:sip"},
+    }
+    assert fix_requires_approval(fix) is True
+
+
+def test_split_proposed_fixes():
+    fixes = [
+        {"description": "connect", "is_config_change": True, "tool": "pfsense_connect"},
+        {"description": "add rule", "is_config_change": True, "tool": "pfsense_create_firewall_rule"},
+        {"description": "note", "is_config_change": False},
+    ]
+    approval, info = split_proposed_fixes(fixes)
+    assert len(approval) == 1
+    assert approval[0]["tool"] == "pfsense_create_firewall_rule"
+    assert len(info) == 2

backend/tests/test_asterisk_profiles.py

@@ -0,0 +1,60 @@
+"""Tests for Asterisk deployment profiles."""
+from __future__ import annotations
+
+import os
+
+os.environ.setdefault("SECRET_KEY", "test-secret-key-for-unit-tests-only")
+
+from app.agent.asterisk_profiles import (  # noqa: E402
+    _docker_asterisk,
+    _ssh_connect,
+)
+from app.agent.playbooks import playbook_for  # noqa: E402
+from app.models.enums import DeviceType  # noqa: E402
+from app.services.vessel_devices import validate_selections  # noqa: E402
+from app.schemas import VesselDeviceSelection  # noqa: E402
+
+
+def test_geneseasx_ssh_connect_uses_voip_port_default():
+    os.environ["DEVICE_ASTERISK_GENESEASX_PORT"] = "6022"
+    args = _ssh_connect(
+        {"address": "10.0.0.1", "username": "root", "secret": "pw", "catalog_key": "asterisk_geneseasx"}
+    )
+    assert args["port"] == 6022
+    assert args["password"] == "pw"
+
+
+def test_docker_asterisk_command_uses_voip_container():
+    cmd = _docker_asterisk("pjsip show registrations")(
+        {"asterisk_container": "voip", "catalog_key": "asterisk_geneseasx"}
+    )
+    assert "docker exec voip asterisk -rx" in cmd["command"]
+    assert "pjsip show registrations" in cmd["command"]
+
+
+def test_playbook_geneseasx_uses_ssh_mcp():
+    pb = playbook_for(DeviceType.asterisk, None, "asterisk_geneseasx")
+    assert pb is not None
+    assert pb["mcp_server"] == "ssh-generic-mcp"
+    assert pb["connect"][0] == "ssh_connect"
+
+
+def test_playbook_satbox_uses_ssh_mcp():
+    pb = playbook_for(DeviceType.asterisk, None, "asterisk_satbox")
+    assert pb is not None
+    assert pb["mcp_server"] == "ssh-generic-mcp"
+    assert pb["connect"][0] == "ssh_connect"
+
+
+def test_only_one_asterisk_slot_per_vessel():
+    sels = [
+        VesselDeviceSelection(catalog_key="asterisk_geneseasx", enabled=True),
+        VesselDeviceSelection(catalog_key="asterisk_satbox", enabled=True),
+    ]
+    try:
+        validate_selections(sels)
+        raised = False
+    except ValueError as exc:
+        raised = True
+        assert "one Asterisk" in str(exc)
+    assert raised

backend/tests/test_core.py

@@ -0,0 +1,49 @@
+"""Unit tests for core security primitives (no external services required)."""
+from __future__ import annotations
+
+import os
+
+os.environ.setdefault("SECRET_KEY", "test-secret-key-for-unit-tests-only")
+
+from app.core.crypto import decrypt_secret, encrypt_secret  # noqa: E402
+from app.core.security import (  # noqa: E402
+    create_access_token,
+    decode_access_token,
+    hash_password,
+    verify_password,
+)
+from app.models.enums import Role  # noqa: E402
+
+
+def test_password_hash_roundtrip():
+    h = hash_password("s3cret!")
+    assert h != "s3cret!"
+    assert verify_password("s3cret!", h)
+    assert not verify_password("wrong", h)
+
+
+def test_jwt_roundtrip():
+    token = create_access_token(subject="admin@agentic.local", role="admin")
+    payload = decode_access_token(token)
+    assert payload is not None
+    assert payload["sub"] == "admin@agentic.local"
+    assert payload["role"] == "admin"
+
+
+def test_jwt_invalid():
+    assert decode_access_token("not-a-token") is None
+
+
+def test_credential_encryption_roundtrip():
+    token = encrypt_secret("device-password")
+    assert token is not None
+    assert token != "device-password"
+    assert decrypt_secret(token) == "device-password"
+    assert encrypt_secret(None) is None
+    assert decrypt_secret(None) is None
+
+
+def test_role_ranking():
+    from app.core.deps import _ROLE_RANK
+
+    assert _ROLE_RANK[Role.admin] > _ROLE_RANK[Role.support] > _ROLE_RANK[Role.readonly]

backend/tests/test_device_defaults.py

@@ -0,0 +1,68 @@
+"""Tests for device default credentials from env."""
+from __future__ import annotations
+
+import os
+
+os.environ.setdefault("SECRET_KEY", "test-secret-key-for-unit-tests-only")
+os.environ["DEVICE_SSH_USERNAME"] = "shipadmin"
+os.environ["DEVICE_SSH_PASSWORD"] = "ssh-secret"
+os.environ["DEVICE_PFSENSE_PORT"] = "40443"
+os.environ["DEVICE_PFSENSE_API_KEY"] = "pf-key-123"
+os.environ["DEVICE_PROXMOX_PASSWORD"] = "pve-pass"
+os.environ["DEVICE_PROXMOX_TOKEN_ID"] = "root@pam!agentic"
+os.environ["DEVICE_PROXMOX_TOKEN_SECRET"] = "token-uuid"
+
+from app.inventory_catalog import catalog_by_key  # noqa: E402
+from app.services.device_defaults import (  # noqa: E402
+    merge_selection_with_defaults,
+    proxmox_api_configured,
+    resolve_proxmox_api_defaults,
+    resolve_slot_defaults,
+)
+
+
+def test_ssh_slot_uses_global_fallback():
+    entry = catalog_by_key()["docker_vm"]
+    d = resolve_slot_defaults("docker_vm", entry)
+    assert d.username == "shipadmin"
+    assert d.secret == "ssh-secret"
+    assert d.secret_kind == "password"
+
+
+def test_per_slot_overrides_global():
+    entry = catalog_by_key()["proxmox"]
+    d = resolve_slot_defaults("proxmox", entry)
+    assert d.secret == "pve-pass"
+    assert d.username == "shipadmin"
+
+
+def test_api_key_slot():
+    entry = catalog_by_key()["pfsense"]
+    d = resolve_slot_defaults("pfsense", entry)
+    assert d.port == 40443
+    assert d.secret == "pf-key-123"
+    assert d.secret_kind == "api_key"
+
+
+def test_merge_applies_env_when_blank():
+    entry = catalog_by_key()["pfsense"]
+    port, user, secret = merge_selection_with_defaults(
+        "pfsense", entry, port=None, username=None, secret=None
+    )
+    assert port == 40443
+    assert secret == "pf-key-123"
+
+
+def test_proxmox_api_env():
+    api = resolve_proxmox_api_defaults("10.1.2.3")
+    assert api["proxmox_token_id"] == "root@pam!agentic"
+    assert api["proxmox_token_secret"] == "token-uuid"
+    assert proxmox_api_configured(api)
+
+
+def test_merge_ui_override_wins():
+    entry = catalog_by_key()["pfsense"]
+    _, _, secret = merge_selection_with_defaults(
+        "pfsense", entry, port=8443, username="admin", secret="custom-key"
+    )
+    assert secret == "custom-key"

backend/tests/test_diagnostic_format.py

@@ -0,0 +1,82 @@
+"""Tests for diagnostic output formatting."""
+from __future__ import annotations
+
+import json
+import os
+
+os.environ.setdefault("SECRET_KEY", "test-secret-key-for-unit-tests-only")
+
+from app.services.diagnostic_format import (  # noqa: E402
+    diagnostics_to_markdown,
+    format_tool_output_markdown,
+    prepare_report_diagnostics,
+)
+
+
+def test_firewall_rules_table_from_json():
+    payload = {
+        "code": 200,
+        "data": [
+            {
+                "type": "pass",
+                "interface": ["vtnet2"],
+                "protocol": "tcp",
+                "source": "any",
+                "destination": "GENESEASX_LOCAL_SERVICES",
+                "descr": "crew to local services",
+            }
+        ],
+    }
+    md = format_tool_output_markdown("pfsense_list_firewall_rules", json.dumps(payload))
+    assert "| Action |" in md
+    assert "crew to local services" in md
+    assert "pass" in md
+
+
+def test_prepare_report_diagnostics_includes_formatted():
+    diags = [
+        {
+            "device": "pfsense",
+            "tool": "pfsense_list_firewall_rules",
+            "ok": True,
+            "output": json.dumps({"data": [{"type": "pass", "descr": "test rule"}]}),
+        }
+    ]
+    out = prepare_report_diagnostics(diags)
+    assert len(out) == 1
+    assert out[0]["formatted"]
+    assert "test rule" in out[0]["formatted"]
+    assert out[0]["raw_available"] is True
+    assert "output" not in out[0]
+
+
+def test_diagnostics_to_markdown_section():
+    diags = prepare_report_diagnostics(
+        [
+            {
+                "device": "pfsense",
+                "tool": "pfsense_list_interfaces",
+                "ok": True,
+                "output": json.dumps({"data": [{"id": "wan", "if": "vtnet0", "descr": "WAN"}]}),
+            }
+        ]
+    )
+    md = diagnostics_to_markdown(diags)
+    assert "### [pfsense] pfsense_list_interfaces" in md
+    assert "vtnet0" in md
+
+
+def test_prepare_report_diagnostics_fallback_output_is_clipped():
+    out = prepare_report_diagnostics(
+        [
+            {
+                "device": "docker",
+                "tool": "unknown_tool",
+                "ok": True,
+                "output": "x" * 5000,
+            }
+        ]
+    )
+    assert out[0]["raw_available"] is True
+    assert out[0]["truncated"] is True
+    assert len(out[0]["output"]) < 4100

backend/tests/test_investigation_context.py

@@ -0,0 +1,45 @@
+"""Tests for investigation context gathering."""
+from __future__ import annotations
+
+import os
+import tempfile
+
+os.environ.setdefault("SECRET_KEY", "test-secret-key-for-unit-tests-only")
+
+from app.services.investigation_context import (  # noqa: E402
+    _score_obsidian_file,
+    extract_search_keywords,
+    _obsidian_title,
+)
+
+
+def test_extract_search_keywords_from_issue():
+    terms = extract_search_keywords("Asterisk health status on GeneseasX voip container")
+    assert "asterisk" in terms
+    assert "geneseasx" in terms
+    assert "voip" in terms
+    assert "health" not in terms  # stopword
+
+
+def test_obsidian_title_from_markdown():
+    text = "# Asterisk health status\n\nSome body"
+    assert _obsidian_title(text, "2026-06-13-asterisk.md") == "Asterisk health status"
+
+
+def test_memory_vessel_from_tags():
+    from app.services.investigation_context import _memory_vessel_from_hit, _obsidian_vessel_from_text
+
+    hit = {"tags": ["agentic-os", "vessel:geneseasx-test"], "content": "", "title": "[Task] foo"}
+    assert _memory_vessel_from_hit(hit) == "geneseasx test"
+
+    text = "---\ntags: [agentic-os]\nvessel: test\nstatus: unresolved\n---\n# Title"
+    assert _obsidian_vessel_from_text(text) == "test"
+
+
+def test_score_obsidian_file_matches_keywords():
+    score = _score_obsidian_file(
+        "/vault/agentic-os-tasks/2026-06-13-asterisk-health.md",
+        "Asterisk pjsip registration failed on voip container",
+        ["asterisk", "voip"],
+    )
+    assert score >= 3

backend/tests/test_mcp_development.py

@@ -0,0 +1,13 @@
+from app.agent.mcp_development import is_mcp_bug_or_gap
+
+
+def test_unknown_tool_is_repairable():
+    assert is_mcp_bug_or_gap("pfsense_get_firewall_log", "unknown tool: pfsense_get_firewall_log", None)
+
+
+def test_traceback_is_repairable():
+    assert is_mcp_bug_or_gap("ssh_run", None, "Traceback (most recent call last):")
+
+
+def test_normal_failure_not_repairable():
+    assert not is_mcp_bug_or_gap("ssh_run", None, "connection refused")

backend/tests/test_model_routing.py

@@ -0,0 +1,67 @@
+"""Tests for tiered model routing."""
+from __future__ import annotations
+
+import os
+
+os.environ.setdefault("SECRET_KEY", "test-secret-key-for-unit-tests-only")
+
+from app.services.model_config import (  # noqa: E402
+    LlmRoutingConfig,
+    resolve_tier_model,
+    resolve_tier_models,
+)
+from app.services.model_router import plan_reasoning_tiers  # noqa: E402
+
+
+def test_resolve_tier_models_provider_order(monkeypatch):
+    monkeypatch.setenv("GEMINI_API_KEY", "test-gemini")
+    monkeypatch.setenv("DEEPSEEK_API_KEY", "test-deepseek")
+    monkeypatch.setenv("OPENROUTER_API_KEY", "test-openrouter")
+
+    from app.config import get_settings
+
+    get_settings.cache_clear()
+
+    cfg = LlmRoutingConfig(
+        cloud_provider_order=["gemini", "deepseek", "openrouter"],
+    )
+    models = resolve_tier_models(cfg, "economy")
+    backends = [m.backend for m in models]
+    assert backends == ["gemini", "deepseek", "openrouter"]
+
+    get_settings.cache_clear()
+
+
+def test_resolve_tier_model_returns_first_provider(monkeypatch):
+    monkeypatch.setenv("GEMINI_API_KEY", "test-gemini")
+    monkeypatch.setenv("DEEPSEEK_API_KEY", "test-deepseek")
+
+    from app.config import get_settings
+
+    get_settings.cache_clear()
+
+    cfg = LlmRoutingConfig(cloud_provider_order=["gemini", "deepseek", "openrouter"])
+    entry = resolve_tier_model(cfg, "economy")
+    assert entry is not None
+    assert entry.backend == "gemini"
+
+    get_settings.cache_clear()
+
+
+def test_plan_reasoning_local_first():
+    tiers, reason = plan_reasoning_tiers(
+        {"severity": "low", "needs_cloud_reasoning": False, "recommended_tier": "local"},
+        [],
+        cfg=LlmRoutingConfig(),
+    )
+    assert tiers == ["local"]
+    assert "local" in reason.lower()
+
+
+def test_plan_reasoning_escalates_critical():
+    tiers, _ = plan_reasoning_tiers(
+        {"severity": "critical", "needs_cloud_reasoning": True, "recommended_tier": "premium"},
+        [{"ok": False}, {"ok": False}],
+        cfg=LlmRoutingConfig(),
+    )
+    assert tiers == ["local", "economy", "premium"]

backend/tests/test_pfsense_profiles.py

@@ -0,0 +1,24 @@
+"""Tests for pfSense issue-aware diagnostics."""
+from app.agent.pfsense_profiles import (
+    diagnostics_for_issue,
+    wants_firewall_inventory,
+)
+
+
+def test_wants_firewall_inventory():
+    assert wants_firewall_inventory("list Firewall rules", "policies for all interfaces")
+    assert wants_firewall_inventory("", "show me NAT port forwards")
+
+
+def test_default_health_diagnostics():
+    tools = [t for t, _ in diagnostics_for_issue("Asterisk health", "check voip")]
+    assert "pfsense_list_firewall_rules" not in tools
+    assert "pfsense_get_system_status" in tools
+
+
+def test_firewall_diagnostics():
+    tools = [t for t, _ in diagnostics_for_issue("list rules", "firewall policies")]
+    assert tools[0] == "pfsense_list_interfaces"
+    assert "pfsense_list_firewall_rules" in tools
+    assert "pfsense_list_port_forwards" in tools
+    assert "pfsense_list_outbound_nat_mappings" in tools

backend/tests/test_proxmox_profiles.py

@@ -0,0 +1,57 @@
+"""Tests for Proxmox profile helpers."""
+from app.agent.proxmox_profiles import (
+    _api_guest_inventory_empty,
+    _first_proxmox_node,
+    _parse_qemu_vms,
+    wants_vm_inventory,
+)
+
+
+def test_first_proxmox_node_from_json_list():
+    out = '[{"node":"genx","status":"online"}]'
+    assert _first_proxmox_node(out) == "genx"
+
+
+def test_first_proxmox_node_from_regex():
+    out = 'nodes: {"node":"pve1"}'
+    assert _first_proxmox_node(out) == "pve1"
+
+
+def test_wants_vm_inventory_from_task4_issue():
+    assert wants_vm_inventory(
+        "Proxmox VM status",
+        "Check status of all running VMs and provide config for each VM",
+    )
+
+
+def test_parse_qemu_vms_json():
+    out = '[{"vmid":100,"name":"pfsense","status":"running","cpus":2}]'
+    vms = _parse_qemu_vms(out)
+    assert len(vms) == 1
+    assert vms[0]["vmid"] == 100
+
+
+def test_parse_qemu_vms_qm_list_text():
+    out = """      VMID NAME                 STATUS     MEM(MB)    BOOTDISK(GB) PID
+       100 pfsense              running    4096              32.00 1234
+       101 debian-docker        running    8192              64.00 5678"""
+    vms = _parse_qemu_vms(out)
+    assert len(vms) == 2
+    assert vms[0]["vmid"] == 100
+    assert vms[0]["status"] == "running"
+
+
+def test_api_guest_inventory_empty():
+    results = [
+        {"tool": "proxmox_list_qemu", "output": "[]"},
+        {"tool": "proxmox_list_lxc", "output": "[]"},
+    ]
+    assert _api_guest_inventory_empty(results)
+
+
+def test_api_guest_inventory_not_empty_when_qemu_has_rows():
+    results = [
+        {"tool": "proxmox_list_qemu", "output": '[{"vmid":100,"name":"pfsense","status":"running"}]'},
+        {"tool": "proxmox_list_lxc", "output": "[]"},
+    ]
+    assert not _api_guest_inventory_empty(results)

backend/tests/test_task_present.py

@@ -0,0 +1,40 @@
+"""Tests for compact task presentation helpers."""
+from __future__ import annotations
+
+import os
+from datetime import datetime, timezone
+
+os.environ.setdefault("SECRET_KEY", "test-secret-key-for-unit-tests-only")
+
+from app.models.enums import TaskStatus  # noqa: E402
+from app.models.task import Task  # noqa: E402
+from app.services.task_present import task_to_overview  # noqa: E402
+
+
+def test_task_to_overview_omits_heavy_diagnostics():
+    now = datetime.now(timezone.utc)
+    task = Task(
+        id=7,
+        title="Status",
+        issue="check proxmox and pfsense",
+        status=TaskStatus.succeeded,
+        vessel_id=2,
+        report={
+            "executive_summary": "Checked requested systems.",
+            "findings": [{"title": "No critical issue", "summary": "Both devices answered."}],
+            "devices_checked": ["proxmox", "pfsense"],
+            "diagnostics": [{"output": "large raw data"}],
+            "resolved": True,
+            "run_cost_usd": 0.0,
+        },
+        created_at=now,
+        updated_at=now,
+    )
+    task.memory_id = "mem-1"
+    task.obsidian_path = "agentic-os-tasks/status.md"
+
+    overview = task_to_overview(task, vessel_name="IRINAs")
+    assert overview.summary == "Checked requested systems."
+    assert overview.key_finding == "Both devices answered."
+    assert overview.devices_checked == ["proxmox", "pfsense"]
+    assert not hasattr(overview, "diagnostics")

backend/tests/test_task_runtime.py

@@ -0,0 +1,28 @@
+"""Tests for per-task runtime flags."""
+from app.services.task_runtime import (
+    CANCELLABLE_STATUSES,
+    TaskCancelledError,
+    apply_min_tier_floor,
+)
+
+
+def test_cancellable_statuses():
+    assert "running" in CANCELLABLE_STATUSES
+    assert "queued" in CANCELLABLE_STATUSES
+    assert "succeeded" not in CANCELLABLE_STATUSES
+
+
+def test_task_cancelled_error_message():
+    err = TaskCancelledError("Task cancelled by user")
+    assert "cancelled" in str(err).lower()
+
+
+def test_apply_min_tier_floor_skips_local():
+    assert apply_min_tier_floor(["local", "economy", "premium"], "economy") == [
+        "economy",
+        "premium",
+    ]
+
+
+def test_apply_min_tier_floor_premium():
+    assert apply_min_tier_floor(["local", "economy"], "premium") == ["premium"]

backend/tests/test_tool_events.py

@@ -0,0 +1,18 @@
+from app.agent.tool_events import sanitize_tool_args, tool_event_payload
+
+
+def test_sanitize_secrets():
+    safe = sanitize_tool_args({"command": "uptime", "password": "secret123"})
+    assert safe["command"] == "uptime"
+    assert safe["password"] == "••••"
+
+
+def test_tool_event_includes_command():
+    p = tool_event_payload(
+        device="docker_vm",
+        tool="ssh_run",
+        args={"command": "docker ps"},
+        status="running",
+    )
+    assert p["command"] == "docker ps"
+    assert p["device"] == "docker_vm"

backend/tests/test_troubleshooting_rules.py

@@ -0,0 +1,112 @@
+"""Tests for troubleshooting decision rules."""
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+os.environ.setdefault("SECRET_KEY", "test-secret-key-for-unit-tests-only")
+_test_dir = Path(__file__).resolve().parent
+_backend_root = _test_dir.parent
+_rules_file = _backend_root / "rules" / "troubleshooting.yaml"
+if not _rules_file.is_file():
+    _rules_file = _backend_root.parent / "rules" / "troubleshooting.yaml"
+os.environ["TROUBLESHOOTING_RULES_PATH"] = str(_rules_file)
+
+from app.services.troubleshooting_rules import (  # noqa: E402
+    invalidate_rules_cache,
+    load_rules,
+    match_rule,
+    select_devices_for_issue,
+)
+
+
+def _devices():
+    return [
+        {"catalog_key": "proxmox", "name": "pve"},
+        {"catalog_key": "pfsense", "name": "fw"},
+        {"catalog_key": "docker_vm", "name": "docker"},
+        {"catalog_key": "asterisk_geneseasx", "name": "voip"},
+    ]
+
+
+def test_match_no_registration_rule():
+    invalidate_rules_cache()
+    rule = match_rule("SIP phones down", "Phone won't register on crew VLAN")
+    assert rule is not None
+    assert rule.id == "no-registration"
+    assert "pfsense" in rule.devices
+
+
+def test_match_broad_health():
+    invalidate_rules_cache()
+    rule = match_rule("Full check", "health check of all systems")
+    assert rule is not None
+    assert rule.broad is True
+
+
+def test_task4_issue_selects_proxmox_only():
+    """Scoped VM task must not trigger full-stack sweep."""
+    invalidate_rules_cache()
+    selected, rule = select_devices_for_issue(
+        _devices(),
+        "Check status of all running VMs and provide config for each VM",
+        title="Proxmox VM status",
+    )
+    keys = [d["catalog_key"] for d in selected]
+    assert keys == ["proxmox"]
+    assert rule is None or rule.id == "proxmox-vm-inventory"
+
+
+def test_multi_device_status_includes_all_named():
+    """Issue naming two devices must diagnose both even when a single-device rule matches."""
+    invalidate_rules_cache()
+    selected, rule = select_devices_for_issue(
+        _devices(),
+        "check status of proxmox and pfsense",
+        title="Irinas box",
+    )
+    keys = [d["catalog_key"] for d in selected]
+    assert keys == ["proxmox", "pfsense"]
+    assert rule is not None
+    assert rule.id == "proxmox-vm-inventory"
+
+
+def test_unmatched_issue_selects_nothing():
+    invalidate_rules_cache()
+    selected, _rule = select_devices_for_issue(
+        _devices(),
+        "something completely unrelated with no device keywords",
+        title="Mystery",
+    )
+    assert selected == []
+
+
+def test_broad_issue_selects_all_devices():
+    invalidate_rules_cache()
+    selected, rule = select_devices_for_issue(
+        _devices(),
+        "Run a full stack health check on everything",
+        title="Weekly audit",
+    )
+    assert rule is not None
+    assert rule.broad is True
+    assert len(selected) == 4
+
+
+def test_select_devices_order():
+    invalidate_rules_cache()
+    selected, rule = select_devices_for_issue(
+        _devices(),
+        "one-way audio on SIP calls",
+        title="VoIP issue",
+    )
+    assert rule is not None
+    assert rule.id == "one-way-audio"
+    keys = [d["catalog_key"] for d in selected]
+    assert keys == ["pfsense", "asterisk_geneseasx"]
+
+
+def test_load_rules_has_device_keywords():
+    invalidate_rules_cache()
+    data = load_rules(force=True)
+    assert "asterisk_geneseasx" in data.get("device_keywords", {})