Initial commit: Agentic OS troubleshooting platform

Self-hosted, Docker-based agentic troubleshooting platform: FastAPI backend +
LangGraph agent, Next.js UI, tiered LLM routing (local Ollama -> Gemini ->
DeepSeek -> OpenRouter), MCP server manager, encrypted device credentials,
RBAC, audit log, project-memory + Obsidian integrations, and editable
troubleshooting decision rules tuned for the GeneseasX vessel stack.

Co-authored-by: Cursor <cursoragent@cursor.com>

backend/app/agent/__init__.py

@@ -0,0 +1,3 @@
+from app.agent.graph import run_task_agent
+
+__all__ = ["run_task_agent"]

backend/app/agent/asterisk_profiles.py

@@ -0,0 +1,84 @@
+"""Asterisk deployment profiles and connect playbooks."""
+from __future__ import annotations
+
+import os
+import shlex
+
+from app.agent.device_connect import ssh_connect_args
+
+# Only one Asterisk slot allowed per vessel.
+ASTERISK_CATALOG_KEYS = frozenset({"asterisk", "asterisk_geneseasx", "asterisk_satbox"})
+
+GENESEASX_KEYS = frozenset({"asterisk", "asterisk_geneseasx"})
+SATBOX_KEYS = frozenset({"asterisk_satbox"})
+
+
+def asterisk_container_name(catalog_key: str | None) -> str:
+    """Docker container name for GeneseasX Asterisk (default: voip)."""
+    if catalog_key not in GENESEASX_KEYS:
+        return ""
+    return (
+        os.environ.get("DEVICE_ASTERISK_GENESEASX_CONTAINER", "").strip()
+        or os.environ.get("DEVICE_ASTERISK_CONTAINER", "").strip()
+        or "voip"
+    )
+
+
+def _ssh_connect(ctx: dict) -> dict:
+    port = ctx.get("port")
+    if port is None:
+        port_raw = os.environ.get("DEVICE_ASTERISK_GENESEASX_PORT", "").strip()
+        port = int(port_raw) if port_raw.isdigit() else 6022
+    return ssh_connect_args({**ctx, "port": port})
+
+
+def _docker_asterisk(command: str):
+    def builder(c: dict) -> dict:
+        container = c.get("asterisk_container") or asterisk_container_name(c.get("catalog_key")) or "voip"
+        return {"command": f"docker exec {container} asterisk -rx {shlex.quote(command)}"}
+
+    return builder
+
+
+def _asterisk_cli(command: str):
+    return lambda c: {"command": f'asterisk -rx "{command}"'}
+
+
+# Native Debian host diagnostics (TMGeneseas / satbox) via SSH.
+SATBOX_ASTERISK_DIAGNOSTICS = [
+    ("ssh_run", _asterisk_cli("pjsip show registrations")),
+    ("ssh_run", _asterisk_cli("pjsip show endpoints")),
+    ("ssh_run", _asterisk_cli("core show channels")),
+    ("ssh_run", lambda c: {"command": "tail -n 100 /var/log/asterisk/full 2>/dev/null || journalctl -u asterisk -n 100 --no-pager 2>/dev/null || true"}),
+]
+
+# GeneseasX: SSH to Docker VM, docker exec into voip (password auth via ssh-generic-mcp).
+GENESEASX_ASTERISK_DIAGNOSTICS = [
+    ("ssh_run", _docker_asterisk("pjsip show registrations")),
+    ("ssh_run", _docker_asterisk("pjsip show endpoints")),
+    ("ssh_run", _docker_asterisk("core show channels")),
+    ("ssh_run", lambda c: {
+        "command": (
+            f"docker logs --tail 100 {shlex.quote(c.get('asterisk_container') or 'voip')} 2>&1 | tail -100"
+        )
+    }),
+]
+
+ASTERISK_PLAYBOOKS: dict[str, dict] = {
+    "asterisk_geneseasx": {
+        "mcp_server": "ssh-generic-mcp",
+        "connect": ("ssh_connect", _ssh_connect),
+        "diagnostics": GENESEASX_ASTERISK_DIAGNOSTICS,
+    },
+    # Legacy catalog key — same as GeneseasX
+    "asterisk": {
+        "mcp_server": "ssh-generic-mcp",
+        "connect": ("ssh_connect", _ssh_connect),
+        "diagnostics": GENESEASX_ASTERISK_DIAGNOSTICS,
+    },
+    "asterisk_satbox": {
+        "mcp_server": "ssh-generic-mcp",
+        "connect": ("ssh_connect", _ssh_connect),
+        "diagnostics": SATBOX_ASTERISK_DIAGNOSTICS,
+    },
+}

backend/app/agent/device_connect.py

@@ -0,0 +1,29 @@
+"""Connect argument builders for layer MCP tools (device ctx + .env fallbacks)."""
+from __future__ import annotations
+
+import os
+
+
+def pfsense_connect_args(ctx: dict) -> dict:
+    port = ctx.get("port") or int(os.environ.get("DEVICE_PFSENSE_PORT", "40443") or 40443)
+    api_key = (
+        ctx.get("secret")
+        or os.environ.get("DEVICE_PFSENSE_API_KEY", "").strip()
+        or os.environ.get("DEVICE_API_KEY", "").strip()
+        or None
+    )
+    return {
+        "url": f"https://{ctx['address']}:{port}",
+        "api_key": api_key,
+        "name": ctx.get("catalog_key") or "pfsense",
+        "verify_ssl": False,
+    }
+
+
+def ssh_connect_args(ctx: dict, *, default_port: int = 22) -> dict:
+    return {
+        "host": ctx["address"],
+        "port": ctx.get("port") or default_port,
+        "username": ctx.get("username") or "root",
+        "password": ctx.get("secret"),
+    }

backend/app/agent/device_routing.py

@@ -0,0 +1,10 @@
+"""Pick which onboard devices to diagnose — delegates to editable troubleshooting rules."""
+from __future__ import annotations
+
+from app.services.troubleshooting_rules import (
+    MatchedRule,
+    match_rule,
+    select_devices_for_issue,
+)
+
+__all__ = ["MatchedRule", "match_rule", "select_devices_for_issue"]

backend/app/agent/graph.py

@@ -0,0 +1,940 @@
+"""LangGraph troubleshooting agent — vessel-aware, multi-device diagnostics."""
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+from datetime import datetime, timezone
+from typing import Any, TypedDict
+
+from langchain_core.messages import HumanMessage, SystemMessage
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy.orm import selectinload
+
+from app.agent.device_routing import select_devices_for_issue
+from app.agent.json_utils import parse_json as _parse_json
+from app.agent.playbooks import playbook_for
+from app.agent.pfsense_profiles import run_pfsense_diagnostics
+from app.agent.proxmox_profiles import enrich_proxmox_device, run_proxmox_diagnostics
+from app.agent.asterisk_profiles import asterisk_container_name
+from app.agent.tool_runner import invoke_connect, invoke_tool
+from app.core.crypto import decrypt_secret
+from app.config import settings
+from app.database import SessionLocal
+from app.inventory_catalog import catalog_by_key
+from app.mcp_manager import get_manager
+from app.models.enums import ApprovalStatus, DeviceType, TaskStatus
+from app.models.inventory import Device, Vessel
+from app.models.task import ApprovalRequest, Task
+from app.services import balance, integrations
+from app.services.diagnostic_format import (
+    diagnostics_to_markdown,
+    prepare_report_diagnostics,
+)
+from app.services.events import publish_event
+from app.services.device_defaults import resolve_slot_defaults
+from app.services.investigation_context import gather_investigation_context
+from app.services.approval_filter import split_proposed_fixes
+from app.services.troubleshooting_rules import rule_guidance_block
+from app.services.llm import triage_model, triage_model_info
+from app.services.llm_usage import LlmUsageTracker, merge_run_history
+from app.services.model_config import get_routing_config
+from app.services.model_router import plan_reasoning_tiers, reason_with_cascade
+from app.services.task_runtime import (
+    clear_task_cancel,
+    clear_task_min_tier,
+    ensure_not_cancelled,
+    get_task_min_tier,
+    is_task_cancel_requested,
+    TaskCancelledError,
+)
+
+logger = logging.getLogger(__name__)
+
+MAX_DIAGNOSTICS_PER_DEVICE = 6
+
+# In-memory LLM usage collectors keyed by task_id (one agent run at a time per task)
+_run_trackers: dict[int, LlmUsageTracker] = {}
+
+
+class AgentState(TypedDict, total=False):
+    task_id: int
+    title: str
+    issue: str
+    details: dict | None
+    vessel_id: int | None
+    vessel_name: str | None
+    vessel_site: str | None
+    vessel_public_ip: str | None
+    devices: list[dict]
+    triage_result: dict
+    diagnostics: list[dict]
+    analysis: dict
+    report: dict
+    prior_context: dict | None
+    investigation_context: dict | None
+    matched_rule: dict | None
+    follow_up: str | None
+    run_number: int
+    prior_runs: list[dict]
+
+
+def _tracker(task_id: int) -> LlmUsageTracker:
+    if task_id not in _run_trackers:
+        _run_trackers[task_id] = LlmUsageTracker()
+    return _run_trackers[task_id]
+
+
+async def _emit(task_id: int, kind: str, message: str, payload: dict | None = None) -> None:
+    await publish_event(task_id, kind, message, payload)
+
+
+async def _progress(task_id: int, phase: str, message: str, **extra) -> None:
+    await _emit(task_id, "progress", message, {"phase": phase, **extra})
+
+
+def _device_dict(d: Device) -> dict:
+    key = d.catalog_key
+    entry = catalog_by_key().get(key) if key else None
+    secret = decrypt_secret(d.secret_enc)
+    username = d.username
+    port = d.port
+
+    if entry:
+        defaults = resolve_slot_defaults(key, entry)
+        if not secret:
+            secret = defaults.secret
+        if not username:
+            username = defaults.username or entry.default_username
+        if port is None and defaults.port is not None:
+            port = defaults.port
+
+    base = {
+        "id": d.id,
+        "name": d.name,
+        "catalog_key": key,
+        "device_type": d.device_type.value,
+        "address": d.address,
+        "port": port,
+        "mcp_server": d.mcp_server,
+        "username": username,
+        "secret": secret,
+        "asterisk_container": asterisk_container_name(key) if key else None,
+    }
+    if key == "proxmox":
+        return enrich_proxmox_device(base)
+    return base
+
+
+async def _load_vessel_devices(db: AsyncSession, vessel_id: int) -> list[dict]:
+    res = await db.execute(
+        select(Vessel).options(selectinload(Vessel.devices)).where(Vessel.id == vessel_id)
+    )
+    vessel = res.scalar_one_or_none()
+    if not vessel:
+        return []
+    devices = [_device_dict(d) for d in vessel.devices]
+    docker = next((d for d in devices if d.get("catalog_key") == "docker_vm"), None)
+    if docker and docker.get("secret"):
+        for d in devices:
+            if d.get("catalog_key") in ("asterisk_geneseasx", "asterisk") and not d.get("secret"):
+                d["secret"] = docker["secret"]
+            if d.get("catalog_key") in ("asterisk_geneseasx", "asterisk") and not d.get("port"):
+                d["port"] = docker.get("port")
+    return devices
+
+
+def _prior_context_block(prior: dict | None, follow_up: str | None) -> str:
+    if not prior and not follow_up:
+        return ""
+    lines = ["\n\n--- Previous investigation context ---"]
+    if prior:
+        if prior.get("summary"):
+            lines.append(f"Prior summary: {prior['summary']}")
+        if prior.get("root_cause"):
+            lines.append(f"Prior root cause: {prior['root_cause']}")
+        if prior.get("resolution"):
+            lines.append(f"Prior resolution attempt: {prior['resolution']}")
+        if prior.get("devices_checked"):
+            lines.append(f"Already checked: {', '.join(prior['devices_checked'])}")
+        if prior.get("resolved") is False:
+            lines.append("Status: issue NOT resolved — continue investigating.")
+    if follow_up:
+        lines.append(f"Follow-up request: {follow_up}")
+    return "\n".join(lines)
+
+
+def _investigation_context_block(ctx: dict | None) -> str:
+    """Format memory + Obsidian hits for LLM prompts."""
+    if not ctx:
+        return ""
+    memory = ctx.get("memory_hits") or []
+    obsidian = ctx.get("obsidian_hits") or []
+    if not memory and not obsidian:
+        return ""
+
+    lines = ["\n\n--- Related past investigations (memory + Obsidian) ---"]
+    lines.append(f"Search query: {ctx.get('query', '')}")
+
+    if memory:
+        lines.append("\nProject memory:")
+        for hit in memory[:6]:
+            title = hit.get("title") or "untitled"
+            mtype = hit.get("memory_type") or "note"
+            content = (hit.get("content") or "").replace("\n", " ")[:350]
+            proj = hit.get("project_id") or "?"
+            vessel = hit.get("vessel")
+            vessel_tag = f" vessel={vessel}" if vessel else ""
+            lines.append(f"- [{proj}/{mtype}{vessel_tag}] {title}: {content}")
+
+    if obsidian:
+        lines.append("\nObsidian notes:")
+        for hit in obsidian[:5]:
+            path = hit.get("path") or "?"
+            title = hit.get("title") or path
+            excerpt = (hit.get("excerpt") or "").replace("\n", " ")[:300]
+            vessel = hit.get("vessel")
+            vessel_tag = f" (vessel: {vessel})" if vessel else ""
+            lines.append(f"- {path}{vessel_tag} — {title}: {excerpt}")
+
+    lines.append(
+        "Use these prior findings to avoid repeating ruled-out causes and to reuse known fixes."
+    )
+    return "\n".join(lines)
+
+
+async def context_node(state: AgentState) -> AgentState:
+    """Load related memory and Obsidian notes before triage."""
+    tid = state["task_id"]
+    await ensure_not_cancelled(tid)
+    await _progress(tid, "context", "Searching project memory and Obsidian…")
+    await _emit(tid, "context", "Searching project memory and Obsidian for related past issues...")
+    try:
+        ctx = await gather_investigation_context(
+            state["title"],
+            state["issue"],
+            vessel_name=state.get("vessel_name"),
+            vessel_id=state.get("vessel_id"),
+        )
+    except Exception as exc:  # noqa: BLE001
+        logger.warning("investigation context failed: %s", exc)
+        ctx = {"memory_hits": [], "obsidian_hits": [], "error": str(exc)}
+
+    mem_n = ctx.get("memory_count", len(ctx.get("memory_hits") or []))
+    obs_n = ctx.get("obsidian_count", len(ctx.get("obsidian_hits") or []))
+    msg = f"Found {mem_n} memory record(s) and {obs_n} Obsidian note(s) related to this issue."
+    await _emit(
+        tid,
+        "context",
+        msg,
+        {
+            "query": ctx.get("query"),
+            "keywords": ctx.get("keywords"),
+            "memory_count": mem_n,
+            "obsidian_count": obs_n,
+            "memory_titles": [h.get("title") for h in (ctx.get("memory_hits") or [])[:5]],
+            "obsidian_paths": [h.get("path") for h in (ctx.get("obsidian_hits") or [])[:5]],
+        },
+    )
+    return {"investigation_context": ctx}
+
+
+async def _run_device_diagnostics(
+    task_id: int,
+    device: dict,
+    manager,
+    *,
+    title: str = "",
+    issue: str = "",
+) -> list[dict]:
+    """Connect to one onboard device and run read-only playbook diagnostics."""
+    from app.models.enums import DeviceType as DT
+
+    results: list[dict] = []
+    dtype = DT(device["device_type"])
+    label = device.get("catalog_key") or device.get("name") or dtype.value
+
+    if device.get("catalog_key") == "proxmox" or dtype == DT.proxmox:
+        return await run_proxmox_diagnostics(
+            task_id,
+            device,
+            manager,
+            _emit,
+            max_diagnostics=MAX_DIAGNOSTICS_PER_DEVICE,
+            task_title=title,
+            task_issue=issue,
+        )
+
+    if device.get("catalog_key") == "pfsense" or dtype == DT.pfsense:
+        return await run_pfsense_diagnostics(
+            task_id,
+            device,
+            manager,
+            _emit,
+            title=title,
+            issue=issue,
+            max_diagnostics=8,
+        )
+
+    pb = playbook_for(dtype, device.get("mcp_server"), device.get("catalog_key"))
+
+    if not pb:
+        await _emit(task_id, "diagnose", f"No playbook for {label}.")
+        return results
+
+    server = pb["mcp_server"]
+    server_state = manager.servers.get(server or "")
+    if not server or not server_state or server_state.status != "loaded":
+        await _emit(
+            task_id,
+            "diagnose",
+            f"MCP '{server}' not available for {label}.",
+            {"device": label, "server": server},
+        )
+        return results
+
+    await _emit(task_id, "connect", f"Connecting to {label} via {server}...", {"device": label})
+    connect = pb.get("connect")
+    if connect:
+        tool, builder = connect
+        await invoke_connect(task_id, label, server, tool, builder(device), manager, _emit)
+
+    for tool, builder in pb["diagnostics"][:MAX_DIAGNOSTICS_PER_DEVICE]:
+        args = builder(device)
+        res, ok = await invoke_tool(
+            task_id,
+            label,
+            server,
+            tool,
+            args,
+            manager,
+            _emit,
+            task_title=title,
+            task_issue=issue,
+        )
+        results.append(
+            {
+                "device": label,
+                "tool": tool,
+                "ok": ok,
+                "output": res.get("text", ""),
+            }
+        )
+    return results
+
+
+async def triage_node(state: AgentState) -> AgentState:
+    tid = state["task_id"]
+    await ensure_not_cancelled(tid)
+    devices = state.get("devices") or []
+    run_number = state.get("run_number", 1)
+    await _progress(tid, "triage", f"Local model triaging (run {run_number})…")
+    await _emit(tid, "triage", f"Local model triaging (run {run_number})...")
+    device_summary = [
+        {"label": d.get("catalog_key") or d.get("device_type"), "name": d.get("name")} for d in devices
+    ]
+    prompt = (
+        "You are the triage brain of a vessel troubleshooting OS. Classify the issue. "
+        "Prefer local handling when possible; only recommend cloud tiers for complex cases. "
+        'Respond ONLY as JSON: {"severity":"low|medium|high|critical",'
+        '"category":"short label","summary":"1-2 sentences",'
+        '"needs_cloud_reasoning":true|false,'
+        '"recommended_tier":"local|economy|premium",'
+        '"plan":["step1","step2"]}'
+    )
+    ctx = (
+        f"Title: {state['title']}\nIssue: {state['issue']}\n"
+        f"Onboard devices to check: {json.dumps(device_summary)}"
+        f"{_investigation_context_block(state.get('investigation_context'))}"
+        f"{rule_guidance_block(state.get('matched_rule'))}"
+        f"{_prior_context_block(state.get('prior_context'), state.get('follow_up'))}"
+    )
+    triage_info = await triage_model_info()
+    try:
+        resp = await _tracker(tid).invoke(
+            "triage",
+            await triage_model(),
+            triage_info,
+            [SystemMessage(content=prompt), HumanMessage(content=ctx)],
+        )
+        triage = _parse_json(resp.content) or {"summary": str(resp.content)[:500]}
+    except Exception as exc:  # noqa: BLE001
+        logger.warning("triage failed: %s", exc)
+        triage = {"severity": "medium", "needs_cloud_reasoning": True, "error": str(exc)}
+    await _emit(tid, "triage", triage.get("summary", "triage complete"), triage)
+    return {"triage_result": triage}
+
+
+async def diagnose_node(state: AgentState) -> AgentState:
+    tid = state["task_id"]
+    await ensure_not_cancelled(tid)
+    devices = state.get("devices") or []
+    diagnostics: list[dict] = []
+
+    if not devices:
+        await _emit(tid, "diagnose", "No vessel devices matched this issue — nothing to diagnose.")
+        return {"diagnostics": diagnostics}
+
+    await _progress(
+        tid,
+        "diagnose",
+        f"Running diagnostics on {len(devices)} device(s)…",
+        device_count=len(devices),
+    )
+    manager = get_manager()
+    for device in devices:
+        await ensure_not_cancelled(tid)
+        diagnostics.extend(
+            await _run_device_diagnostics(
+                tid,
+                device,
+                manager,
+                title=state.get("title") or "",
+                issue=state.get("issue") or "",
+            )
+        )
+    return {"diagnostics": diagnostics}
+
+
+async def reason_node(state: AgentState) -> AgentState:
+    tid = state["task_id"]
+    await ensure_not_cancelled(tid)
+    triage = state.get("triage_result", {})
+    diagnostics = state.get("diagnostics", [])
+    cfg = await get_routing_config()
+    min_tier = await get_task_min_tier(tid)
+
+    await _progress(tid, "reason", "Analyzing diagnostics with LLM…")
+    tiers, routing_reason = plan_reasoning_tiers(
+        triage,
+        diagnostics,
+        prior_context=state.get("prior_context"),
+        follow_up=state.get("follow_up"),
+        run_number=state.get("run_number", 1),
+        cfg=cfg,
+        min_tier=min_tier,
+    )
+    await _emit(
+        tid,
+        "reason",
+        f"Reasoning cascade: {' → '.join(tiers)} ({routing_reason})",
+        {"tiers": tiers, "routing_reason": routing_reason},
+    )
+
+    diag_text = "\n\n".join(
+        f"### [{d.get('device', '?')}] {d['tool']} ({'ok' if d['ok'] else 'fail'})\n{d['output'][:1500]}"
+        for d in diagnostics
+    ) or "No live diagnostics were collected."
+
+    prompt = (
+        "You are an expert vessel infrastructure analyst. Given the issue and diagnostic output "
+        "from onboard devices, determine root cause and resolution. "
+        "Only analyze devices that were actually checked — do not speculate about unrelated layers. "
+        "If prior investigation context or related memory/Obsidian notes are provided, build on them — "
+        "do not repeat already-ruled-out causes. Prefer fixes that worked before when still applicable. "
+        "If the user asked for a list, inventory, status dump, or config details, include the actual data "
+        "from diagnostics in summary and resolution — do not only describe how to find it in the UI. "
+        "If you are uncertain, set needs_escalation true and confidence below 0.65 so a stronger model can retry. "
+        "IMPORTANT for proposed_fixes: list ONLY optional future write actions that would modify production "
+        "config (firewall rules, NAT, Asterisk dialplan, etc.). Set is_config_change true ONLY for those. "
+        "Do NOT list connect steps, diagnostics already run, or manual advice — use steps_taken for those. "
+        "For read-only health/status checks with no change requested, use an empty proposed_fixes array. "
+        'Respond ONLY as JSON: {"summary":"...","root_cause":"... or null",'
+        '"steps_taken":["..."],"proposed_fixes":[{"description":"...","is_config_change":true|false,'
+        '"tool":"optional-mcp-write-tool","args":{}}],"resolved":true|false,"resolution":"...",'
+        '"confidence":0.0-1.0,"needs_escalation":true|false}'
+    )
+    ctx = (
+        f"Title: {state['title']}\nIssue: {state['issue']}\nTriage: {json.dumps(triage)}\n\n{diag_text}"
+        f"{_investigation_context_block(state.get('investigation_context'))}"
+        f"{rule_guidance_block(state.get('matched_rule'))}"
+        f"{_prior_context_block(state.get('prior_context'), state.get('follow_up'))}"
+    )
+    messages = [SystemMessage(content=prompt), HumanMessage(content=ctx)]
+    try:
+        analysis, routing_meta = await reason_with_cascade(
+            _tracker(tid), tiers, messages, _parse_json, cfg=cfg, diagnostics=diagnostics, task_id=tid
+        )
+        analysis["model_routing"] = {**routing_meta, "routing_reason": routing_reason}
+    except Exception as exc:  # noqa: BLE001
+        logger.warning("reason failed: %s", exc)
+        analysis = {"summary": f"reasoning error: {exc}", "resolved": False, "proposed_fixes": []}
+    await _emit(
+        tid,
+        "reason",
+        analysis.get("summary", "analysis complete"),
+        {
+            "model": analysis.get("_model"),
+            "tier": analysis.get("_tier_used"),
+            "routing": analysis.get("model_routing"),
+        },
+    )
+    return {"analysis": analysis}
+
+
+def _unique_strings(values: list[Any]) -> list[str]:
+    out: list[str] = []
+    seen: set[str] = set()
+    for value in values:
+        text = str(value).strip()
+        if not text or text in seen:
+            continue
+        seen.add(text)
+        out.append(text)
+    return out
+
+
+def _scope_checked(devices: list[dict], diagnostics: list[dict]) -> list[dict]:
+    out: list[dict] = []
+    for device in devices:
+        key = device.get("catalog_key") or device.get("name") or "device"
+        tools = [
+            d.get("tool")
+            for d in diagnostics
+            if (d.get("device") == key or d.get("device") == device.get("name")) and d.get("tool")
+        ]
+        out.append(
+            {
+                "device": key,
+                "name": device.get("name") or key,
+                "checks": _unique_strings(tools),
+                "ok": all(bool(d.get("ok")) for d in diagnostics if d.get("device") == key),
+            }
+        )
+    return out
+
+
+def _report_findings(analysis: dict, diagnostics: list[dict], devices_checked: list[str]) -> list[dict]:
+    findings: list[dict] = []
+    if analysis.get("root_cause"):
+        findings.append(
+            {
+                "title": "Root cause",
+                "summary": analysis.get("root_cause"),
+                "description": analysis.get("resolution"),
+            }
+        )
+    failed = [d for d in diagnostics if d.get("ok") is False]
+    for d in failed[:5]:
+        findings.append(
+            {
+                "title": f"{d.get('device', 'device')} check failed",
+                "summary": d.get("tool"),
+                "description": (d.get("output") or "")[:300],
+            }
+        )
+    if not findings and devices_checked:
+        findings.append(
+            {
+                "title": "No critical issue found",
+                "summary": f"Completed checks for {', '.join(devices_checked)}.",
+                "description": analysis.get("summary") or analysis.get("resolution"),
+            }
+        )
+    return findings
+
+
+async def report_node(state: AgentState) -> AgentState:
+    tid = state["task_id"]
+    await ensure_not_cancelled(tid)
+    analysis = state.get("analysis", {})
+    triage = state.get("triage_result", {})
+    diagnostics = state.get("diagnostics", [])
+    run_number = state.get("run_number", 1)
+    follow_up = state.get("follow_up")
+
+    await _progress(tid, "report", "Writing report and saving notes…")
+    planned_steps = _unique_strings(list(triage.get("plan") or []))
+    actions_taken = _unique_strings(list(analysis.get("steps_taken") or []))
+    resolved = bool(analysis.get("resolved"))
+    proposed_all = analysis.get("proposed_fixes") or []
+    proposed, informational = split_proposed_fixes(proposed_all)
+
+    llm_summary = _tracker(tid).summary()
+
+    report_diagnostics = prepare_report_diagnostics(diagnostics)
+    devices_checked = [d.get("catalog_key") or d.get("name") for d in state.get("devices") or []]
+    devices_checked = _unique_strings(devices_checked)
+    scope_checked = _scope_checked(state.get("devices") or [], diagnostics)
+    tools_run = [
+        {"device": d.get("device"), "tool": d.get("tool"), "ok": d.get("ok")}
+        for d in diagnostics
+    ]
+    findings = _report_findings(analysis, diagnostics, devices_checked)
+    executive_summary = analysis.get("summary") or (
+        f"Checked {', '.join(devices_checked)}." if devices_checked else "No device checks were run."
+    )
+
+    report: dict[str, Any] = {
+        "summary": analysis.get("summary"),
+        "executive_summary": executive_summary,
+        "severity": triage.get("severity"),
+        "root_cause": analysis.get("root_cause"),
+        "planned_steps": planned_steps,
+        "actions_taken": actions_taken,
+        "tools_run": tools_run,
+        # Backward compatible, no tool-name spam.
+        "steps": actions_taken or planned_steps,
+        "devices_checked": devices_checked,
+        "scope_checked": scope_checked,
+        "findings": findings,
+        "diagnostics": report_diagnostics,
+        "resolved": resolved,
+        "resolution": analysis.get("resolution"),
+        "proposed_fixes": proposed_all,
+        "proposed_fixes_actionable": proposed,
+        "recommendations": informational,
+        "follow_up": follow_up,
+        "model_routing": analysis.get("model_routing"),
+        "generated_at": datetime.now(timezone.utc).isoformat(),
+        "vessel_id": state.get("vessel_id"),
+        "vessel_name": state.get("vessel_name"),
+        "vessel_site": state.get("vessel_site"),
+        "vessel_public_ip": state.get("vessel_public_ip"),
+    }
+    inv_ctx = state.get("investigation_context")
+    if inv_ctx:
+        report["prior_knowledge"] = {
+            "query": inv_ctx.get("query"),
+            "memory_count": inv_ctx.get("memory_count"),
+            "obsidian_count": inv_ctx.get("obsidian_count"),
+            "memory_titles": [h.get("title") for h in (inv_ctx.get("memory_hits") or [])[:8]],
+            "obsidian_paths": [h.get("path") for h in (inv_ctx.get("obsidian_hits") or [])[:8]],
+        }
+    if state.get("matched_rule"):
+        report["matched_rule"] = state["matched_rule"]
+
+    report = merge_run_history(state.get("prior_runs") or [], run_number, report, llm_summary)
+
+    async with SessionLocal() as db:
+        created: list[ApprovalRequest] = []
+        for fix in proposed:
+            row = ApprovalRequest(
+                task_id=tid,
+                tool_name=fix.get("tool") or "manual-config-change",
+                tool_args=fix.get("args") or {},
+                risk=fix.get("description"),
+                status=ApprovalStatus.pending,
+            )
+            db.add(row)
+            created.append(row)
+        await db.flush()
+        for row in created:
+            await _emit(
+                tid,
+                "approval",
+                f"Approval required: {row.tool_name} — {(row.risk or '')[:120]}",
+                {
+                    "approval_id": row.id,
+                    "tool_name": row.tool_name,
+                    "risk": row.risk,
+                    "status": "pending",
+                },
+            )
+        if created:
+            res = await db.execute(select(Task).where(Task.id == tid))
+            task_row = res.scalar_one_or_none()
+            if task_row:
+                task_row.status = TaskStatus.waiting_approval
+        await db.commit()
+
+    cost_msg = (
+        f"Run cost ${report['run_cost_usd']:.4f} · Total ${report['total_cost_usd']:.4f}"
+        if report["total_cost_usd"] > 0
+        else f"Run cost $0.00 (local) · Total ${report['total_cost_usd']:.4f}"
+    )
+    await _emit(tid, "report", f"Writing memory and Obsidian note… {cost_msg}", {"llm_usage": report["llm_usage"]})
+
+    title = state["title"]
+    run_label = f" (run {run_number})" if run_number > 1 else ""
+    vessel_name = state.get("vessel_name")
+    vessel_label = f"[{vessel_name}] " if vessel_name else ""
+    content = (
+        f"Vessel: {vessel_name or 'unknown'}\n"
+        f"Issue: {state['issue']}\n\n"
+        f"Run: {run_number}\n"
+        f"Devices checked: {', '.join(report['devices_checked']) or 'none'}\n\n"
+        f"Root cause: {report['root_cause']}\n\nResolution: {report['resolution']}\n\n"
+        f"LLM cost this run: ${report['run_cost_usd']:.4f} (total ${report['total_cost_usd']:.4f})\n\n"
+        f"Findings:\n" + "\n".join(f"- {f.get('title')}: {f.get('summary')}" for f in findings)
+    )
+    if report_diagnostics:
+        content += "\n\nDiagnostic tools run:\n" + "\n".join(
+            f"- [{d.get('device')}] {d.get('tool')}" for d in report_diagnostics
+        )
+    memory_id = await integrations.create_task_memory(
+        title=f"{vessel_label}[Task] {title}{run_label}",
+        content=content,
+        memory_type="fix" if resolved else "note",
+        tags=["agentic-os", triage.get("category", "troubleshooting")],
+        importance=4 if triage.get("severity") in ("high", "critical") else 3,
+        vessel_name=vessel_name,
+        task_id=tid,
+    )
+    obsidian_path, obsidian_push_error = await integrations.write_obsidian_note(
+        title=f"{vessel_label}{title}{run_label}".strip(),
+        issue=state["issue"] if not follow_up else f"{state['issue']}\n\nFollow-up: {follow_up}",
+        steps=actions_taken or planned_steps,
+        cause=report["root_cause"],
+        resolution=report["resolution"],
+        resolved=resolved,
+        vessel_name=vessel_name,
+        task_id=tid,
+        diagnostics_md=diagnostics_to_markdown(report_diagnostics),
+    )
+
+    report["memory_id"] = memory_id
+    report["obsidian_path"] = obsidian_path
+    report["obsidian_push_error"] = obsidian_push_error
+    report["memory_project_id"] = settings.memory_task_project_id
+    complete_msg = "Report complete."
+    if memory_id:
+        complete_msg += f" Memory saved ({settings.memory_task_project_id})."
+    if obsidian_path and not obsidian_push_error:
+        complete_msg += f" Obsidian note pushed ({obsidian_path})."
+    elif obsidian_path:
+        complete_msg += f" Obsidian note saved locally ({obsidian_path}); git push failed."
+    await _emit(
+        tid,
+        "report",
+        complete_msg,
+        {
+            "memory_id": memory_id,
+            "memory_project_id": settings.memory_task_project_id,
+            "obsidian_path": obsidian_path,
+            "obsidian_push_error": obsidian_push_error,
+            "llm_usage": report["llm_usage"],
+            "run_cost_usd": report["run_cost_usd"],
+            "total_cost_usd": report["total_cost_usd"],
+        },
+    )
+    return {"report": report}
+
+
+_compiled_graph = None
+
+
+def get_graph():
+    """Return the process-wide compiled LangGraph, building it once."""
+    global _compiled_graph
+    if _compiled_graph is None:
+        _compiled_graph = build_graph()
+    return _compiled_graph
+
+
+def build_graph():
+    from langgraph.graph import END, START, StateGraph
+
+    g = StateGraph(AgentState)
+    g.add_node("context_step", context_node)
+    g.add_node("triage_step", triage_node)
+    g.add_node("diagnose_step", diagnose_node)
+    g.add_node("reason_step", reason_node)
+    g.add_node("report_step", report_node)
+    g.add_edge(START, "context_step")
+    g.add_edge("context_step", "triage_step")
+    g.add_edge("triage_step", "diagnose_step")
+    g.add_edge("diagnose_step", "reason_step")
+    g.add_edge("reason_step", "report_step")
+    g.add_edge("report_step", END)
+    return g.compile()
+
+
+async def _finalize_cancelled(task_id: int) -> None:
+    already_cancelled = False
+    async with SessionLocal() as db:
+        res = await db.execute(select(Task).where(Task.id == task_id))
+        task = res.scalar_one_or_none()
+        if task:
+            already_cancelled = task.status == TaskStatus.cancelled
+            if not already_cancelled:
+                task.status = TaskStatus.cancelled
+                await db.commit()
+    if not already_cancelled:
+        await _emit(task_id, "cancelled", "Task stopped.", persist=True)
+    await clear_task_cancel(task_id)
+
+
+async def run_task_agent(task_id: int) -> None:
+    async with SessionLocal() as db:
+        res = await db.execute(select(Task).where(Task.id == task_id))
+        task = res.scalar_one_or_none()
+        if not task:
+            return
+
+        details = dict(task.details or {})
+        continue_ctx = details.pop("_continue", None)
+        continue_mode = continue_ctx is not None
+
+        prior_runs: list[dict] = details.get("runs") or []
+        if not prior_runs and task.report:
+            prior_runs = task.report.get("runs") or []
+
+        if continue_mode:
+            run_number = len(prior_runs) + 1
+            follow_up = continue_ctx.get("follow_up", "")
+            prior_context = continue_ctx.get("prior_report") or task.report
+        else:
+            run_number = 1
+            follow_up = None
+            prior_context = None
+            prior_runs = []
+
+        issue_text = task.issue
+        routing_text = f"{issue_text}\n{follow_up}" if follow_up else issue_text
+
+        devices: list[dict] = []
+        matched_rule: dict | None = None
+        vessel_name: str | None = None
+        vessel_site: str | None = None
+        vessel_public_ip: str | None = None
+        if task.vessel_id:
+            all_devices = await _load_vessel_devices(db, task.vessel_id)
+            devices, rule = select_devices_for_issue(
+                all_devices, routing_text, title=task.title
+            )
+            matched_rule = rule.as_dict() if rule else None
+            vres = await db.execute(select(Vessel).where(Vessel.id == task.vessel_id))
+            vessel = vres.scalar_one_or_none()
+            if vessel:
+                vessel_name = vessel.name
+                vessel_site = vessel.site
+                vessel_public_ip = vessel.public_ip
+        elif task.device_id:
+            dres = await db.execute(select(Device).where(Device.id == task.device_id))
+            d = dres.scalar_one_or_none()
+            if d:
+                devices = [_device_dict(d)]
+
+        task.status = TaskStatus.running
+        task.details = details
+        await db.commit()
+
+    if await is_task_cancel_requested(task_id):
+        await _finalize_cancelled(task_id)
+        _run_trackers.pop(task_id, None)
+        return
+
+    labels = [d.get("catalog_key") or d.get("name") for d in devices]
+    run_label = f" (run {run_number})" if run_number > 1 else ""
+    await _emit(task_id, "log", f"Agent run started{run_label}.")
+    if follow_up:
+        await _emit(task_id, "log", f"Follow-up: {follow_up}")
+    if vessel_name:
+        await _emit(
+            task_id,
+            "log",
+            f"Vessel: {vessel_name}"
+            + (f" ({vessel_public_ip})" if vessel_public_ip else ""),
+            {"vessel_name": vessel_name, "vessel_public_ip": vessel_public_ip},
+        )
+    await _emit(
+        task_id,
+        "log",
+        f"Will diagnose {len(devices)} device(s): {', '.join(labels) or 'none'}",
+        {"devices": labels, "run_number": run_number},
+    )
+    if matched_rule:
+        await _emit(
+            task_id,
+            "rule",
+            f"Matched rule: {matched_rule.get('name')} → {', '.join(matched_rule.get('devices') or [])}",
+            matched_rule,
+        )
+
+    remaining = await balance.latest_remaining_usd()
+    from app.config import settings
+
+    if remaining is not None and remaining < settings.min_balance_usd:
+        msg = f"Aborting: remaining balance ${remaining:.2f} below minimum ${settings.min_balance_usd:.2f}."
+        await _emit(task_id, "error", msg)
+        async with SessionLocal() as db:
+            res = await db.execute(select(Task).where(Task.id == task_id))
+            task = res.scalar_one()
+            task.status = TaskStatus.failed
+            task.error = msg
+            await db.commit()
+        _run_trackers.pop(task_id, None)
+        return
+
+    initial: AgentState = {
+        "task_id": task_id,
+        "title": task.title,
+        "issue": issue_text,
+        "details": task.details,
+        "vessel_id": task.vessel_id,
+        "vessel_name": vessel_name,
+        "vessel_site": vessel_site,
+        "vessel_public_ip": vessel_public_ip,
+        "devices": devices,
+        "prior_context": prior_context,
+        "follow_up": follow_up,
+        "run_number": run_number,
+        "prior_runs": prior_runs if continue_mode else [],
+        "matched_rule": matched_rule,
+    }
+
+    try:
+        # Create the per-task usage tracker inside the try so the finally below
+        # always cleans it up, even if setup above this point raised.
+        _run_trackers[task_id] = LlmUsageTracker()
+        graph = get_graph()
+        final_state = await graph.ainvoke(initial)
+        report = final_state.get("report", {})
+        async with SessionLocal() as db:
+            res = await db.execute(select(Task).where(Task.id == task_id))
+            task = res.scalar_one()
+            task.report = report
+            task.memory_id = report.get("memory_id")
+            task.obsidian_path = report.get("obsidian_path")
+            task.details = {
+                **(task.details or {}),
+                "runs": report.get("runs", []),
+                "total_cost_usd": report.get("total_cost_usd", 0.0),
+            }
+            pend = await db.execute(
+                select(ApprovalRequest).where(
+                    ApprovalRequest.task_id == task_id,
+                    ApprovalRequest.status == ApprovalStatus.pending,
+                )
+            )
+            has_pending = pend.scalars().first() is not None
+            if report.get("resolved") and not has_pending:
+                task.status = TaskStatus.succeeded
+            elif has_pending:
+                task.status = TaskStatus.waiting_approval
+            elif continue_mode or not report.get("resolved"):
+                task.status = TaskStatus.succeeded  # completed run but unresolved — user can continue
+            else:
+                task.status = TaskStatus.succeeded
+            await db.commit()
+            final_status = task.status.value
+        await _emit(
+            task_id,
+            "log",
+            f"Agent run finished ({final_status}). Cost: ${report.get('run_cost_usd', 0):.4f} this run, "
+            f"${report.get('total_cost_usd', 0):.4f} total.",
+            {"llm_usage": report.get("llm_usage"), "run_number": run_number},
+        )
+    except TaskCancelledError:
+        logger.info("Task %s cancelled by user", task_id)
+        await _finalize_cancelled(task_id)
+    except asyncio.CancelledError:
+        logger.info("Task %s asyncio cancelled", task_id)
+        await _finalize_cancelled(task_id)
+    except Exception as exc:  # noqa: BLE001
+        logger.exception("Agent run failed for task %s", task_id)
+        await _emit(task_id, "error", f"Agent run failed: {exc}")
+        async with SessionLocal() as db:
+            res = await db.execute(select(Task).where(Task.id == task_id))
+            task = res.scalar_one()
+            task.status = TaskStatus.failed
+            task.error = str(exc)
+            await db.commit()
+    finally:
+        _run_trackers.pop(task_id, None)
+        try:
+            await clear_task_min_tier(task_id)
+        except Exception:  # noqa: BLE001
+            logger.debug("clear_task_min_tier failed for task %s", task_id)

backend/app/agent/json_utils.py

@@ -0,0 +1,24 @@
+"""Shared helpers for extracting JSON objects from LLM responses."""
+from __future__ import annotations
+
+import json
+
+
+def parse_json(text: str) -> dict:
+    """Best-effort extraction of a single JSON object from LLM output.
+
+    Handles fenced ```json blocks and leading/trailing prose by slicing
+    between the first '{' and last '}'. Returns {} when nothing parses.
+    """
+    text = text.strip()
+    if text.startswith("```"):
+        text = text.split("```", 2)[1]
+        if text.startswith("json"):
+            text = text[4:]
+    start, end = text.find("{"), text.rfind("}")
+    if start != -1 and end != -1:
+        try:
+            return json.loads(text[start : end + 1])
+        except json.JSONDecodeError:
+            pass
+    return {}

backend/app/agent/mcp_development.py

@@ -0,0 +1,294 @@
+"""Agent-driven MCP extension and bug-fix workflow."""
+from __future__ import annotations
+
+import logging
+from collections.abc import Awaitable, Callable
+from typing import Any
+
+from langchain_core.messages import HumanMessage, SystemMessage
+
+from app.agent.json_utils import parse_json as _parse_json
+from app.config import settings
+from app.services import mcp_workspace
+from app.services.model_config import get_routing_config, resolve_tier_model
+from app.services.model_router import build_chat_model
+
+logger = logging.getLogger(__name__)
+
+EmitFn = Callable[[int, str, str, dict | None], Awaitable[None]]
+
+_REPAIRABLE = (
+    "unknown tool",
+    "traceback",
+    "nameerror",
+    "attributeerror",
+    "typeerror",
+    "not defined",
+    "target' is not defined",
+    "missing required",
+    "unexpected keyword",
+    "has no attribute",
+)
+
+
+def is_mcp_bug_or_gap(tool: str, error: str | None, output: str | None) -> bool:
+    text = f"{error or ''} {output or ''}".lower()
+    if "unknown tool" in text and tool.lower() in text:
+        return True
+    return any(sig in text for sig in _REPAIRABLE)
+
+
+async def _load_context_files(server: str) -> dict[str, str]:
+    files: dict[str, str] = {}
+    for rel in await mcp_workspace.list_source_files(server):
+        if rel == "server.py" or rel.endswith("server.py") or rel == "pyproject.toml":
+            try:
+                files[rel] = await mcp_workspace.read_source_file(server, rel)
+            except OSError:
+                pass
+    if "server.py" not in files:
+        listed = await mcp_workspace.list_source_files(server, max_files=5)
+        for rel in listed[:3]:
+            try:
+                files[rel] = await mcp_workspace.read_source_file(server, rel, max_chars=40_000)
+            except OSError:
+                pass
+    return files
+
+
+async def propose_mcp_patch(
+    *,
+    server: str,
+    tool: str,
+    issue: str,
+    error: str | None,
+    output: str | None,
+    task_title: str = "",
+) -> dict[str, Any]:
+    """Use LLM to propose MCP source file edits."""
+    cfg = await get_routing_config()
+    tier = settings.mcp_dev_model_tier if hasattr(settings, "mcp_dev_model_tier") else "premium"
+    entry = resolve_tier_model(cfg, tier) or resolve_tier_model(cfg, "economy")
+    if not entry:
+        return {"ok": False, "error": "No LLM available for MCP development"}
+
+    files = await _load_context_files(server)
+    tools = mcp_workspace.list_registered_tools(server)
+
+    prompt = (
+        "You are an expert Python MCP (Model Context Protocol) developer using FastMCP patterns. "
+        "Given a failing tool call or missing capability, patch the MCP server source. "
+        "Add new @mcp.tool() functions when a capability is missing; fix bugs when tools crash. "
+        "Match existing code style. Keep changes minimal and safe (read-only tools unless clearly needed). "
+        'Respond ONLY as JSON: {"summary":"...", "commit_message":"fix: ...", '
+        '"files":[{"path":"server.py","content":"FULL file content after edit"}]}'
+    )
+    ctx = (
+        f"MCP server: {server}\n"
+        f"Registered tools: {', '.join(tools) or 'unknown'}\n"
+        f"Task: {task_title}\n"
+        f"Issue: {issue}\n"
+        f"Tool involved: {tool}\n"
+        f"Error: {error or 'none'}\n"
+        f"Output: {(output or '')[:2000]}\n\n"
+        f"Source files:\n"
+        + "\n\n".join(f"--- {p} ---\n{c[:25000]}" for p, c in files.items())
+    )
+
+    model = build_chat_model(entry, temperature=0.1)
+    resp = await model.ainvoke([SystemMessage(content=prompt), HumanMessage(content=ctx)])
+    data = _parse_json(str(resp.content))
+    if not data.get("files"):
+        return {"ok": False, "error": "LLM did not return file edits", "raw": str(resp.content)[:500]}
+    return {"ok": True, **data}
+
+
+async def apply_and_reload_mcp(
+    server: str,
+    edits: list[dict],
+    manager,
+) -> dict[str, Any]:
+    written = await mcp_workspace.apply_file_edits(server, edits)
+    if not written:
+        return {"ok": False, "error": "No files written"}
+    await manager.reload_server(server)
+    return {"ok": True, "files": written, "tools": mcp_workspace.list_registered_tools(server)}
+
+
+async def attempt_mcp_repair(
+    *,
+    task_id: int,
+    server: str,
+    tool: str,
+    args: dict,
+    issue: str,
+    error: str | None,
+    output: str | None,
+    manager,
+    emit: EmitFn,
+    task_title: str = "",
+    push: bool | None = None,
+) -> dict[str, Any]:
+    """Full repair loop: analyze → patch → reload → optional git push."""
+    if not settings.mcp_development_enabled:
+        return {"ok": False, "skipped": True, "reason": "MCP development disabled"}
+
+    await emit(
+        task_id,
+        "mcp_dev",
+        f"Analyzing MCP '{server}' to fix or add capability for {tool}…",
+        {"server": server, "tool": tool, "phase": "analyze"},
+    )
+
+    proposal = await propose_mcp_patch(
+        server=server,
+        tool=tool,
+        issue=issue,
+        error=error,
+        output=output,
+        task_title=task_title,
+    )
+    if not proposal.get("ok"):
+        await emit(task_id, "mcp_dev", f"MCP patch proposal failed: {proposal.get('error')}", proposal)
+        return proposal
+
+    await emit(
+        task_id,
+        "mcp_dev",
+        proposal.get("summary", "Applying MCP patch…"),
+        {"server": server, "phase": "patch", "files": [f.get("path") for f in proposal.get("files", [])]},
+    )
+
+    await emit(
+        task_id,
+        "mcp_dev",
+        f"Reloading MCP '{server}' with patched source…",
+        {"server": server, "phase": "reload"},
+    )
+
+    reload_result = await apply_and_reload_mcp(server, proposal.get("files") or [], manager)
+    if not reload_result.get("ok"):
+        await emit(task_id, "error", reload_result.get("error", "MCP reload failed"), reload_result)
+        return reload_result
+
+    await emit(
+        task_id,
+        "mcp_dev",
+        f"MCP '{server}' reloaded — {len(reload_result.get('tools') or [])} tools registered",
+        {"server": server, "phase": "reload_done", "tools": reload_result.get("tools")},
+    )
+
+    should_push = push if push is not None else settings.mcp_dev_auto_push
+    push_result: dict[str, Any] | None = None
+    commit_msg = proposal.get("commit_message") or f"agentic-os: fix {tool} on {server}"
+    written = reload_result.get("files") or []
+
+    if should_push and settings.gitea_token:
+        push_result = await mcp_workspace.commit_and_push(server, commit_msg, written)
+        await emit(
+            task_id,
+            "mcp_dev",
+            f"Git push {'ok' if push_result.get('ok') else 'failed'}: {push_result.get('commit') or push_result.get('error')}",
+            {"server": server, "phase": "push", **push_result},
+        )
+    elif settings.gitea_token and written and task_id > 0:
+        from app.database import SessionLocal
+        from app.models.enums import ApprovalStatus
+        from app.models.task import ApprovalRequest
+
+        async with SessionLocal() as db:
+            db.add(
+                ApprovalRequest(
+                    task_id=task_id,
+                    tool_name="mcp_git_push",
+                    tool_args={
+                        "server": server,
+                        "files": written,
+                        "commit_message": commit_msg,
+                    },
+                    risk=proposal.get("summary"),
+                    status=ApprovalStatus.pending,
+                )
+            )
+            await db.commit()
+        await emit(
+            task_id,
+            "approval",
+            f"MCP patch ready — approve to push {server} to Gitea",
+            {"server": server, "tool_name": "mcp_git_push", "files": written},
+        )
+
+    return {
+        "ok": True,
+        "summary": proposal.get("summary"),
+        "reload": reload_result,
+        "push": push_result,
+        "tools": reload_result.get("tools"),
+    }
+
+
+async def maybe_repair_after_tool_failure(
+    *,
+    task_id: int,
+    device: str,
+    server: str,
+    tool: str,
+    args: dict,
+    issue: str,
+    error: str | None,
+    output: str | None,
+    manager,
+    emit: EmitFn,
+    task_title: str = "",
+) -> tuple[dict, bool] | None:
+    """If failure looks MCP-related, try repair once and return retry (res, ok) or None."""
+    if not settings.mcp_development_enabled:
+        return None
+    if not is_mcp_bug_or_gap(tool, error, output):
+        return None
+
+    result = await attempt_mcp_repair(
+        task_id=task_id,
+        server=server,
+        tool=tool,
+        args=args,
+        issue=issue,
+        error=error,
+        output=output,
+        manager=manager,
+        emit=emit,
+        task_title=task_title,
+        push=settings.mcp_dev_auto_push,
+    )
+    if not result.get("ok"):
+        return None
+
+    if result.get("push") and not result["push"].get("ok") and result["push"].get("runtime_only"):
+        await emit(
+            task_id,
+            "log",
+            "MCP patched in runtime — no git remote; copy changes to mcp-servers or Gitea manually.",
+            result["push"],
+        )
+
+    from app.agent.mcp_helpers import tool_result_ok
+    from app.agent.tool_events import tool_event_payload
+    from app.mcp_manager.recovery import call_tool_resilient
+
+    await emit(task_id, "log", f"Retrying {tool} after MCP patch…", {"server": server, "tool": tool})
+    res = await call_tool_resilient(manager, server, tool, args, task_id=task_id, emit=emit)
+    ok = tool_result_ok(res)
+    await emit(
+        task_id,
+        "tool_call",
+        f"{device} › {tool} (after MCP patch)",
+        tool_event_payload(
+            device=device,
+            tool=tool,
+            args=args,
+            status="done",
+            ok=ok,
+            output=res.get("text", ""),
+        ),
+    )
+    return res, ok

backend/app/agent/mcp_helpers.py

@@ -0,0 +1,25 @@
+"""Shared helpers for interpreting MCP tool call results."""
+from __future__ import annotations
+
+
+def tool_result_ok(res: dict) -> bool:
+    """Return True when an MCP tool response indicates success."""
+    if res.get("isError"):
+        return False
+    text = (res.get("text") or "").strip()
+    if not text:
+        return True
+    lower = text.lower()
+    if "unknown tool:" in lower:
+        return False
+    if text.startswith("Error executing tool"):
+        return False
+    if '"status": "error"' in text or '"status":"error"' in text:
+        return False
+    if " api error " in lower and any(code in text for code in ("401", "403", "500")):
+        return False
+    if "authentication failed" in lower:
+        return False
+    if "permission denied" in lower:
+        return False
+    return True

backend/app/agent/pfsense_profiles.py

@@ -0,0 +1,132 @@
+"""pfSense diagnostics — issue-aware playbook selection."""
+from __future__ import annotations
+
+import re
+from typing import Awaitable, Callable
+
+from app.agent.tool_runner import invoke_connect, invoke_tool
+
+EmitFn = Callable[[int, str, str, dict | None], Awaitable[None]]
+
+PFSENSE_MCP = "pfsense-mcp"
+
+ArgBuilder = Callable[[dict], dict]
+
+PFSENSE_DEFAULT_DIAGNOSTICS: list[tuple[str, ArgBuilder]] = [
+    ("pfsense_get_system_status", lambda _c: {}),
+    ("pfsense_list_interfaces", lambda _c: {}),
+    ("pfsense_list_gateways", lambda _c: {}),
+    ("pfsense_get_gateway_status", lambda _c: {}),
+]
+
+PFSENSE_FIREWALL_DIAGNOSTICS: list[tuple[str, ArgBuilder]] = [
+    ("pfsense_list_interfaces", lambda _c: {}),
+    ("pfsense_list_firewall_rules", lambda _c: {"limit": 250}),
+    ("pfsense_list_port_forwards", lambda _c: {"limit": 100}),
+    ("pfsense_list_outbound_nat_mappings", lambda _c: {"limit": 100}),
+    ("pfsense_list_aliases", lambda _c: {"limit": 100}),
+]
+
+_FIREWALL_TERMS = (
+    "firewall",
+    "rule",
+    "rules",
+    "policy",
+    "policies",
+    "filter rule",
+    "nat",
+    "port forward",
+    "port-forward",
+    "outbound nat",
+    "inbound",
+    "acl",
+)
+
+
+def _text_has_term(text: str, term: str) -> bool:
+    term = term.lower().strip()
+    if not term:
+        return False
+    if " " in term:
+        return term in text
+    return bool(re.search(rf"\b{re.escape(term)}\b", text))
+
+
+def wants_firewall_inventory(title: str, issue: str) -> bool:
+    text = f"{title} {issue}".lower()
+    return any(_text_has_term(text, term) for term in _FIREWALL_TERMS)
+
+
+def diagnostics_for_issue(title: str, issue: str) -> list[tuple[str, ArgBuilder]]:
+    if wants_firewall_inventory(title, issue):
+        return list(PFSENSE_FIREWALL_DIAGNOSTICS)
+    return list(PFSENSE_DEFAULT_DIAGNOSTICS)
+
+
+async def run_pfsense_diagnostics(
+    task_id: int,
+    device: dict,
+    manager,
+    emit: EmitFn,
+    *,
+    title: str = "",
+    issue: str = "",
+    max_diagnostics: int = 8,
+) -> list[dict]:
+    """Connect to pfSense and run read-only diagnostics matched to the task."""
+    from app.agent.device_connect import pfsense_connect_args
+
+    results: list[dict] = []
+    label = device.get("catalog_key") or device.get("name") or "pfsense"
+    server = PFSENSE_MCP
+
+    server_state = manager.servers.get(server or "")
+    if not server_state or server_state.status != "loaded":
+        await emit(
+            task_id,
+            "diagnose",
+            f"MCP '{server}' not available for {label}.",
+            {"device": label, "server": server},
+        )
+        return results
+
+    diagnostics = diagnostics_for_issue(title, issue)
+    intent = "firewall inventory" if wants_firewall_inventory(title, issue) else "health"
+
+    await emit(
+        task_id,
+        "connect",
+        f"Connecting to {label} via {server} ({intent})...",
+        {"device": label, "intent": intent},
+    )
+
+    ok = await invoke_connect(
+        task_id, label, server, "pfsense_connect", pfsense_connect_args(device), manager, emit
+    )
+    if not ok:
+        return results
+
+    for tool, builder in diagnostics[:max_diagnostics]:
+        args = builder(device)
+        res, tool_ok = await invoke_tool(
+            task_id,
+            label,
+            server,
+            tool,
+            args,
+            manager,
+            emit,
+            task_title=title,
+            task_issue=issue,
+        )
+        results.append(
+            {
+                "device": label,
+                "tool": tool,
+                "ok": tool_ok,
+                "output": res.get("text", ""),
+                "intent": intent,
+            }
+        )
+
+    return results

backend/app/agent/playbooks.py

@@ -0,0 +1,134 @@
+"""Per-device-type connection and read-only diagnostic playbooks.
+
+Each entry maps a DeviceType to:
+  - mcp_server: default MCP server that drives the device
+  - connect: (tool_name, arg_builder) to establish a session (optional)
+  - diagnostics: list of (tool_name, arg_builder) read-only calls to gather state
+
+arg_builder is a callable(device_ctx) -> dict, where device_ctx contains
+address, port, username, secret.
+"""
+from __future__ import annotations
+
+from typing import Callable
+
+from app.agent.asterisk_profiles import ASTERISK_PLAYBOOKS
+from app.agent.device_connect import pfsense_connect_args, ssh_connect_args
+from app.models.enums import DeviceType
+
+ArgBuilder = Callable[[dict], dict]
+
+
+def _ip(ctx: dict) -> dict:
+    return {"ip": ctx["address"]}
+
+
+def _pfsense_connect(ctx: dict) -> dict:
+    return pfsense_connect_args(ctx)
+
+
+def _ssh(ctx: dict) -> dict:
+    return ssh_connect_args(ctx)
+
+
+PLAYBOOKS: dict[DeviceType, dict] = {
+    DeviceType.pfsense: {
+        "mcp_server": "pfsense-mcp",
+        "connect": ("pfsense_connect", _pfsense_connect),
+        "diagnostics": [
+            ("pfsense_get_system_status", lambda c: {}),
+            ("pfsense_list_gateways", lambda c: {}),
+            ("pfsense_get_gateway_status", lambda c: {}),
+            ("pfsense_list_interfaces", lambda c: {}),
+        ],
+    },
+    DeviceType.proxmox: {
+        # Connect + diagnostics handled by proxmox_profiles.run_proxmox_diagnostics (API → SSH).
+        "mcp_server": "proxmox-mcp",
+        "connect": None,
+        "diagnostics": [],
+    },
+    DeviceType.asterisk: {
+        # Legacy key — prefer asterisk_geneseasx catalog slot (see ASTERISK_PLAYBOOKS).
+        "mcp_server": "ssh-generic-mcp",
+        "connect": ("ssh_connect", _ssh),
+        "diagnostics": [],
+    },
+    # GeneseasX / satbox variants are resolved by catalog_key in playbook_for().
+    DeviceType.debian: {
+        "mcp_server": "ssh-generic-mcp",
+        "connect": (
+            "ssh_connect",
+            lambda c: ssh_connect_args(c),
+        ),
+        "diagnostics": [
+            ("ssh_run", lambda c: {"command": "uptime && uname -a"}),
+            ("ssh_run", lambda c: {"command": "df -h"}),
+            ("ssh_run", lambda c: {"command": "free -m"}),
+            ("ssh_run", lambda c: {"command": "systemctl --failed --no-pager"}),
+            ("ssh_run", lambda c: {"command": "journalctl -p err -n 100 --no-pager"}),
+        ],
+    },
+    DeviceType.geneseasx: {
+        "mcp_server": "ssh-generic-mcp",
+        "connect": (
+            "ssh_connect",
+            lambda c: ssh_connect_args(c),
+        ),
+        "diagnostics": [
+            ("ssh_run", lambda c: {"command": "uptime && uname -a"}),
+            ("ssh_run", lambda c: {"command": "docker ps --format '{{.Names}}: {{.Status}}'"}),
+            ("ssh_run", lambda c: {"command": "systemctl --failed --no-pager"}),
+        ],
+    },
+    DeviceType.fortigate: {
+        "mcp_server": "fortigate-mcp",
+        "connect": (
+            "fortigate_connect",
+            lambda c: {
+                "host": c["address"],
+                "api_key": c.get("secret"),
+                "username": c.get("username"),
+            },
+        ),
+        "diagnostics": [
+            ("fortigate_get_system_status", lambda c: {}),
+            ("fortigate_list_interfaces", lambda c: {}),
+            ("fortigate_list_policies", lambda c: {}),
+            ("fortigate_get_logs", lambda c: {"lines": 100}),
+        ],
+    },
+    DeviceType.fortiswitch: {
+        "mcp_server": "fortiswitch-mcp",
+        "connect": (
+            "fortiswitch_connect",
+            lambda c: {
+                "host": c["address"],
+                "username": c.get("username"),
+                "password": c.get("secret"),
+            },
+        ),
+        "diagnostics": [
+            ("fortiswitch_get_system_status", lambda c: {}),
+            ("fortiswitch_list_ports", lambda c: {}),
+            ("fortiswitch_get_port_stats", lambda c: {}),
+        ],
+    },
+}
+
+
+def playbook_for(
+    device_type: DeviceType,
+    mcp_override: str | None,
+    catalog_key: str | None = None,
+) -> dict | None:
+    if catalog_key and catalog_key in ASTERISK_PLAYBOOKS:
+        pb = ASTERISK_PLAYBOOKS[catalog_key]
+    else:
+        pb = PLAYBOOKS.get(device_type)
+    if pb is None:
+        return None
+    # Catalog playbooks define the MCP server; ignore stale device-row overrides.
+    if mcp_override and catalog_key not in ASTERISK_PLAYBOOKS and device_type != DeviceType.proxmox:
+        pb = {**pb, "mcp_server": mcp_override}
+    return pb

backend/app/agent/proxmox_profiles.py

@@ -0,0 +1,596 @@
+"""Proxmox connect strategy: REST API (proxmox-mcp) first, SSH fallback (ssh-generic-mcp)."""
+from __future__ import annotations
+
+import json
+import re
+from typing import Awaitable, Callable
+
+from app.agent.mcp_helpers import tool_result_ok
+from app.agent.tool_runner import invoke_tool
+from app.services.device_defaults import proxmox_api_configured, resolve_proxmox_api_defaults
+
+EmitFn = Callable[[int, str, str, dict | None], Awaitable[None]]
+
+PROXMOX_MCP = "proxmox-mcp"
+SSH_MCP = "ssh-generic-mcp"
+
+MAX_VM_INVENTORY_CALLS = 40
+
+# Node-less probe — VM/LXC calls need a resolved node name.
+PROXMOX_API_DIAGNOSTICS: list[tuple[str, Callable[[dict], dict]]] = [
+    ("proxmox_list_nodes", lambda _c: {}),
+]
+
+# Minimal per-node summary (health-style tasks).
+PROXMOX_API_NODE_SUMMARY: list[str] = [
+    "proxmox_list_qemu",
+    "proxmox_list_lxc",
+    "proxmox_get_node_status",
+]
+
+PROXMOX_SSH_DIAGNOSTICS: list[tuple[str, Callable[[dict], dict]]] = [
+    ("ssh_run", lambda _c: {"command": "pvesh get /nodes --output-format json 2>/dev/null | head -c 4000 || true"}),
+    ("ssh_run", lambda _c: {"command": "qm list 2>/dev/null || true"}),
+    ("ssh_run", lambda _c: {"command": "pct list 2>/dev/null || true"}),
+    ("ssh_run", lambda _c: {"command": "uptime && free -m && df -h /"}),
+]
+
+_VM_INVENTORY_TERMS = (
+    "config",
+    "configuration",
+    "each vm",
+    "every vm",
+    "all vm",
+    "all vms",
+    "running vm",
+    "running vms",
+    "guest",
+    "inventory",
+    "vm status",
+    "vm list",
+    "list vm",
+    "list vms",
+    "qemu",
+    "lxc",
+)
+
+
+def enrich_proxmox_device(device: dict) -> dict:
+    """Attach Proxmox API env defaults; URL uses vessel public IP (:8006) unless overridden in .env."""
+    public_ip = device.get("address") or ""
+    api = resolve_proxmox_api_defaults(public_ip)
+    return {**device, **api}
+
+
+def _text_has_term(text: str, term: str) -> bool:
+    term = term.lower().strip()
+    if not term:
+        return False
+    if " " in term:
+        return term in text
+    if len(term) <= 2:
+        return term in text.split()
+    return bool(re.search(rf"\b{re.escape(term)}\b", text))
+
+
+def wants_vm_inventory(title: str, issue: str) -> bool:
+    """True when the user asked for per-VM status and/or config details."""
+    text = f"{title} {issue}".lower()
+    if any(_text_has_term(text, term) for term in _VM_INVENTORY_TERMS):
+        return True
+    # "VM" / "VMs" in title or issue (e.g. "Proxmox VM status").
+    return bool(re.search(r"\bvms?\b", text))
+
+
+def _ssh_connect_args(ctx: dict) -> dict:
+    return {
+        "host": ctx["address"],
+        "port": ctx.get("port") or 22,
+        "username": ctx.get("username") or "root",
+        "password": ctx.get("secret"),
+        "name": ctx.get("catalog_key") or "proxmox",
+    }
+
+
+def _api_connect_args(ctx: dict) -> dict:
+    return {
+        "url": ctx["proxmox_api_url"],
+        "token_id": ctx["proxmox_token_id"],
+        "token_secret": ctx["proxmox_token_secret"],
+        "verify_ssl": bool(ctx.get("proxmox_verify_ssl")),
+        "name": ctx.get("catalog_key") or "proxmox",
+        "set_active": True,
+    }
+
+
+async def _server_ready(manager, server: str) -> bool:
+    state = manager.servers.get(server or "")
+    return bool(server and state and state.status == "loaded")
+
+
+async def _try_proxmox_api(manager, device: dict) -> bool:
+    if not proxmox_api_configured(device):
+        return False
+    if not await _server_ready(manager, PROXMOX_MCP):
+        return False
+
+    connect_res = await manager.call_tool(PROXMOX_MCP, "proxmox_connect", _api_connect_args(device))
+    if not tool_result_ok(connect_res):
+        return False
+
+    probe = await manager.call_tool(PROXMOX_MCP, "proxmox_list_nodes", {})
+    if tool_result_ok(probe):
+        return True
+
+    name = device.get("catalog_key") or "proxmox"
+    try:
+        await manager.call_tool(PROXMOX_MCP, "proxmox_disconnect_target", {"name": name})
+    except Exception:  # noqa: BLE001
+        pass
+    return False
+
+
+async def _try_proxmox_ssh(manager, device: dict) -> bool:
+    if not await _server_ready(manager, SSH_MCP):
+        return False
+    if not device.get("secret"):
+        return False
+
+    connect_res = await manager.call_tool(SSH_MCP, "ssh_connect", _ssh_connect_args(device))
+    return tool_result_ok(connect_res)
+
+
+def _first_proxmox_node(output: str) -> str | None:
+    text = (output or "").strip()
+    if not text:
+        return None
+    try:
+        data = json.loads(text)
+        if isinstance(data, list) and data:
+            item = data[0]
+            if isinstance(item, dict):
+                return item.get("node") or item.get("name")
+        if isinstance(data, dict):
+            nodes = data.get("data") or data.get("nodes")
+            if isinstance(nodes, list) and nodes:
+                item = nodes[0]
+                if isinstance(item, dict):
+                    return item.get("node") or item.get("name")
+    except json.JSONDecodeError:
+        pass
+    match = re.search(r'"node"\s*:\s*"([^"]+)"', text)
+    return match.group(1) if match else None
+
+
+def _parse_qemu_vms(output: str) -> list[dict]:
+    """Extract VM entries from proxmox_list_qemu JSON or `qm list` text."""
+    text = (output or "").strip()
+    if not text:
+        return []
+
+    parsed = None
+    try:
+        parsed = json.loads(text)
+    except json.JSONDecodeError:
+        start, end = text.find("["), text.rfind("]")
+        if start != -1 and end > start:
+            try:
+                parsed = json.loads(text[start : end + 1])
+            except json.JSONDecodeError:
+                parsed = None
+
+    rows: list[dict] = []
+    if isinstance(parsed, list):
+        for item in parsed:
+            if isinstance(item, dict) and item.get("vmid") is not None:
+                rows.append(item)
+        if rows:
+            return rows
+    if isinstance(parsed, dict):
+        data = parsed.get("data")
+        if isinstance(data, list):
+            for item in data:
+                if isinstance(item, dict) and item.get("vmid") is not None:
+                    rows.append(item)
+            if rows:
+                return rows
+
+    # `qm list` tabular output
+    for line in text.splitlines():
+        line = line.strip()
+        if not line or line.startswith("VMID") or line.startswith("-"):
+            continue
+        parts = line.split()
+        if not parts or not parts[0].isdigit():
+            continue
+        vmid = int(parts[0])
+        status = parts[2] if len(parts) > 2 else ""
+        name = parts[1] if len(parts) > 1 else ""
+        rows.append({"vmid": vmid, "name": name, "status": status})
+
+    return rows
+
+
+def _api_guest_inventory_empty(results: list[dict]) -> bool:
+    """True when API list_qemu/list_lxc ran but returned no guests (common RBAC gap on /vms)."""
+    qemu_out = next((r.get("output") for r in results if r.get("tool") == "proxmox_list_qemu"), None)
+    lxc_out = next((r.get("output") for r in results if r.get("tool") == "proxmox_list_lxc"), None)
+    if qemu_out is None and lxc_out is None:
+        return False
+    qemu_empty = not _parse_qemu_vms(qemu_out or "")
+    lxc_empty = not _parse_qemu_vms(lxc_out or "")
+    return qemu_empty and lxc_empty
+
+
+def _vms_for_detail(vms: list[dict], title: str, issue: str) -> list[dict]:
+    """Pick which guests need per-VM status/config calls."""
+    text = f"{title} {issue}".lower()
+    running_only = "running" in text and "all vm" not in text and "every vm" not in text
+    if running_only:
+        picked = [v for v in vms if str(v.get("status", "")).lower() == "running"]
+        return picked or vms
+    return vms
+
+
+async def _append_tool_result(
+    results: list[dict],
+    *,
+    task_id: int,
+    label: str,
+    server: str,
+    tool: str,
+    args: dict,
+    manager,
+    emit: EmitFn,
+    mode: str,
+    intent: str,
+    task_title: str,
+    task_issue: str,
+) -> bool:
+    """Run one tool and append to results. Returns False if max budget exhausted."""
+    res, tool_ok = await invoke_tool(
+        task_id,
+        label,
+        server,
+        tool,
+        args,
+        manager,
+        emit,
+        task_title=task_title,
+        task_issue=task_issue,
+    )
+    results.append(
+        {
+            "device": label,
+            "tool": tool,
+            "ok": tool_ok,
+            "output": res.get("text", ""),
+            "connect_mode": mode,
+            "intent": intent,
+            "vmid": args.get("vmid"),
+        }
+    )
+    return True
+
+
+async def _run_api_node_tools(
+    results: list[dict],
+    *,
+    task_id: int,
+    label: str,
+    server: str,
+    node: str,
+    manager,
+    emit: EmitFn,
+    mode: str,
+    intent: str,
+    task_title: str,
+    task_issue: str,
+    max_diagnostics: int,
+    vm_inventory: bool,
+) -> None:
+    node_tools = ["proxmox_list_qemu", "proxmox_list_lxc"]
+    if not vm_inventory:
+        node_tools.append("proxmox_get_node_status")
+
+    qemu_output = ""
+    for tool in node_tools:
+        if len(results) >= max_diagnostics:
+            break
+        await _append_tool_result(
+            results,
+            task_id=task_id,
+            label=label,
+            server=server,
+            tool=tool,
+            args={"node": node},
+            manager=manager,
+            emit=emit,
+            mode=mode,
+            intent=intent,
+            task_title=task_title,
+            task_issue=task_issue,
+        )
+        if tool == "proxmox_list_qemu":
+            qemu_output = results[-1].get("output") or ""
+
+    if not vm_inventory or len(results) >= max_diagnostics:
+        return
+
+    vms = _vms_for_detail(_parse_qemu_vms(qemu_output), task_title, task_issue)
+    for vm in vms[:15]:
+        vmid = vm.get("vmid")
+        if vmid is None or len(results) >= max_diagnostics:
+            break
+        for tool in ("proxmox_get_qemu_status", "proxmox_get_qemu_config"):
+            if len(results) >= max_diagnostics:
+                break
+            await _append_tool_result(
+                results,
+                task_id=task_id,
+                label=label,
+                server=server,
+                tool=tool,
+                args={"node": node, "vmid": int(vmid)},
+                manager=manager,
+                emit=emit,
+                mode=mode,
+                intent=intent,
+                task_title=task_title,
+                task_issue=task_issue,
+            )
+
+
+async def _run_ssh_inventory_fallback(
+    results: list[dict],
+    *,
+    task_id: int,
+    label: str,
+    device: dict,
+    manager,
+    emit: EmitFn,
+    intent: str,
+    task_title: str,
+    task_issue: str,
+    max_diagnostics: int,
+    vm_inventory: bool,
+) -> bool:
+    """Use SSH qm/pct when API connected but guest lists are empty (token lacks /vms VM.Audit)."""
+    ctx = enrich_proxmox_device(device)
+    if not ctx.get("secret"):
+        return False
+    if not await _server_ready(manager, SSH_MCP):
+        return False
+    if not await _try_proxmox_ssh(manager, ctx):
+        return False
+
+    await emit(
+        task_id,
+        "connect",
+        f"{label}: API guest list empty (token likely lacks /vms VM.Audit) — using SSH",
+        {"device": label, "mode": "ssh_fallback"},
+    )
+
+    for command in ("qm list 2>/dev/null || true", "pct list 2>/dev/null || true"):
+        if len(results) >= max_diagnostics:
+            break
+        await _append_tool_result(
+            results,
+            task_id=task_id,
+            label=label,
+            server=SSH_MCP,
+            tool="ssh_run",
+            args={"command": command},
+            manager=manager,
+            emit=emit,
+            mode="ssh",
+            intent=intent,
+            task_title=task_title,
+            task_issue=task_issue,
+        )
+
+    if vm_inventory:
+        await _run_ssh_vm_inventory(
+            results,
+            task_id=task_id,
+            label=label,
+            server=SSH_MCP,
+            manager=manager,
+            emit=emit,
+            mode="ssh",
+            intent=intent,
+            task_title=task_title,
+            task_issue=task_issue,
+            max_diagnostics=max_diagnostics,
+        )
+    return True
+
+
+async def _run_ssh_vm_inventory(
+    results: list[dict],
+    *,
+    task_id: int,
+    label: str,
+    server: str,
+    manager,
+    emit: EmitFn,
+    mode: str,
+    intent: str,
+    task_title: str,
+    task_issue: str,
+    max_diagnostics: int,
+) -> None:
+    """After `qm list`, fetch `qm config` for each selected guest."""
+    qemu_output = ""
+    for r in results:
+        if r.get("tool") != "ssh_run":
+            continue
+        out = r.get("output") or ""
+        if re.search(r"^\s*\d+\s+\S+\s+(running|stopped)", out, re.MULTILINE):
+            qemu_output = out
+            break
+
+    vms = _vms_for_detail(_parse_qemu_vms(qemu_output), task_title, task_issue)
+    for vm in vms[:15]:
+        vmid = vm.get("vmid")
+        if vmid is None or len(results) >= max_diagnostics:
+            break
+        await _append_tool_result(
+            results,
+            task_id=task_id,
+            label=label,
+            server=server,
+            tool="ssh_run",
+            args={"command": f"qm config {int(vmid)} 2>/dev/null || true"},
+            manager=manager,
+            emit=emit,
+            mode=mode,
+            intent=intent,
+            task_title=task_title,
+            task_issue=task_issue,
+        )
+
+
+async def run_proxmox_diagnostics(
+    task_id: int,
+    device: dict,
+    manager,
+    emit: EmitFn,
+    *,
+    max_diagnostics: int = 6,
+    task_title: str = "",
+    task_issue: str = "",
+) -> list[dict]:
+    """Connect via Proxmox API when configured; otherwise or on failure use SSH."""
+    results: list[dict] = []
+    label = device.get("catalog_key") or device.get("name") or "proxmox"
+    ctx = enrich_proxmox_device(device)
+    vm_inventory = wants_vm_inventory(task_title, task_issue)
+    intent = "vm inventory" if vm_inventory else "health"
+    budget = MAX_VM_INVENTORY_CALLS if vm_inventory else max_diagnostics
+
+    mode: str | None = None
+    server: str | None = None
+    diagnostics: list[tuple[str, Callable[[dict], dict]]] = []
+
+    await emit(task_id, "connect", f"Connecting to {label}...", {"device": label})
+
+    if proxmox_api_configured(ctx):
+        await emit(
+            task_id,
+            "connect",
+            f"{label}: trying Proxmox API ({ctx['proxmox_api_url']})...",
+            {"device": label, "mode": "api"},
+        )
+        if await _try_proxmox_api(manager, ctx):
+            mode, server, diagnostics = "api", PROXMOX_MCP, PROXMOX_API_DIAGNOSTICS
+            await emit(
+                task_id,
+                "connect",
+                f"{label}: Proxmox API connected ({intent})",
+                {"device": label, "mode": "api", "intent": intent},
+            )
+        else:
+            await emit(
+                task_id,
+                "connect",
+                f"{label}: Proxmox API unavailable — falling back to SSH",
+                {"device": label, "mode": "api_failed"},
+            )
+
+    if mode is None:
+        await emit(
+            task_id,
+            "connect",
+            f"{label}: trying SSH on port {ctx.get('port') or 22}...",
+            {"device": label, "mode": "ssh"},
+        )
+        if await _try_proxmox_ssh(manager, ctx):
+            mode, server, diagnostics = "ssh", SSH_MCP, PROXMOX_SSH_DIAGNOSTICS
+            await emit(
+                task_id,
+                "connect",
+                f"{label}: SSH connected ({intent})",
+                {"device": label, "mode": "ssh", "intent": intent},
+            )
+        else:
+            await emit(
+                task_id,
+                "connect",
+                f"{label}: connect failed (no working API token or SSH credentials)",
+                {"device": label},
+            )
+            return results
+
+    for tool, builder in diagnostics[:budget]:
+        args = builder(ctx)
+        res, tool_ok = await invoke_tool(
+            task_id,
+            label,
+            server,
+            tool,
+            args,
+            manager,
+            emit,
+            task_title=task_title,
+            task_issue=task_issue,
+        )
+        results.append(
+            {
+                "device": label,
+                "tool": tool,
+                "ok": tool_ok,
+                "output": res.get("text", ""),
+                "connect_mode": mode,
+                "intent": intent,
+            }
+        )
+
+    if mode == "api":
+        node = _first_proxmox_node(next((r["output"] for r in results if r["tool"] == "proxmox_list_nodes"), ""))
+        if node:
+            await _run_api_node_tools(
+                results,
+                task_id=task_id,
+                label=label,
+                server=server,
+                node=node,
+                manager=manager,
+                emit=emit,
+                mode=mode,
+                intent=intent,
+                task_title=task_title,
+                task_issue=task_issue,
+                max_diagnostics=budget,
+                vm_inventory=vm_inventory,
+            )
+            if _api_guest_inventory_empty(results):
+                await _run_ssh_inventory_fallback(
+                    results,
+                    task_id=task_id,
+                    label=label,
+                    device=device,
+                    manager=manager,
+                    emit=emit,
+                    intent=intent,
+                    task_title=task_title,
+                    task_issue=task_issue,
+                    max_diagnostics=budget,
+                    vm_inventory=vm_inventory,
+                )
+    elif mode == "ssh" and vm_inventory:
+        await _run_ssh_vm_inventory(
+            results,
+            task_id=task_id,
+            label=label,
+            server=server,
+            manager=manager,
+            emit=emit,
+            mode=mode,
+            intent=intent,
+            task_title=task_title,
+            task_issue=task_issue,
+            max_diagnostics=budget,
+        )
+
+    return results

backend/app/agent/tool_events.py

@@ -0,0 +1,60 @@
+"""Build safe payloads for tool_call / tool_start events."""
+from __future__ import annotations
+
+from app.services.diagnostic_format import MAX_EVENT_OUTPUT
+
+_SECRET_KEYS = frozenset(
+    {
+        "password",
+        "secret",
+        "api_key",
+        "token_secret",
+        "token",
+        "private_key",
+    }
+)
+
+
+def sanitize_tool_args(args: dict | None) -> dict:
+    if not args:
+        return {}
+    out: dict = {}
+    for key, val in args.items():
+        if key.lower() in _SECRET_KEYS:
+            out[key] = "••••"
+        elif isinstance(val, str) and len(val) > 500:
+            out[key] = val[:500] + "…"
+        else:
+            out[key] = val
+    return out
+
+
+def tool_event_payload(
+    *,
+    device: str,
+    tool: str,
+    args: dict | None = None,
+    status: str = "done",
+    ok: bool | None = None,
+    output: str | None = None,
+    error: str | None = None,
+    **extra,
+) -> dict:
+    safe = sanitize_tool_args(args)
+    payload: dict = {
+        "device": device,
+        "tool": tool,
+        "status": status,
+        **extra,
+    }
+    if safe:
+        payload["args"] = safe
+    if safe.get("command"):
+        payload["command"] = safe["command"]
+    if ok is not None:
+        payload["ok"] = ok
+    if output is not None:
+        payload["output"] = output[:MAX_EVENT_OUTPUT]
+    if error is not None:
+        payload["error"] = error
+    return payload

backend/app/agent/tool_runner.py

@@ -0,0 +1,165 @@
+"""Run MCP tools with live events (start → done) and auto-reload on failure."""
+from __future__ import annotations
+
+from collections.abc import Awaitable, Callable
+
+from app.agent.mcp_helpers import tool_result_ok
+from app.agent.tool_events import tool_event_payload
+from app.mcp_manager.recovery import call_tool_resilient
+from app.services.events import get_redis
+from app.services.task_runtime import ensure_not_cancelled
+
+EmitFn = Callable[[int, str, str, dict | None], Awaitable[None]]
+
+
+def _repair_key(task_id: int, server: str, tool: str) -> str:
+    return f"agentic:task:{task_id}:mcp_repair:{server}:{tool}"
+
+
+async def _repair_attempted(task_id: int, server: str, tool: str) -> bool:
+    return bool(await get_redis().get(_repair_key(task_id, server, tool)))
+
+
+async def _mark_repair_attempted(task_id: int, server: str, tool: str) -> None:
+    await get_redis().set(_repair_key(task_id, server, tool), "1", ex=86400)
+
+
+async def invoke_tool(
+    task_id: int,
+    device: str,
+    server: str,
+    tool: str,
+    args: dict,
+    manager,
+    emit: EmitFn,
+    *,
+    task_title: str = "",
+    task_issue: str = "",
+) -> tuple[dict, bool]:
+    await ensure_not_cancelled(task_id)
+    await emit(
+        task_id,
+        "tool_start",
+        f"{device} › {tool}",
+        tool_event_payload(device=device, tool=tool, args=args, status="running"),
+    )
+    try:
+        res = await call_tool_resilient(
+            manager, server, tool, args, task_id=task_id, emit=emit
+        )
+        ok = tool_result_ok(res)
+        if not ok:
+            retry = await _maybe_mcp_repair(
+                task_id,
+                device,
+                server,
+                tool,
+                args,
+                res.get("text"),
+                manager,
+                emit,
+                task_title=task_title,
+                task_issue=task_issue,
+            )
+            if retry is not None:
+                return retry
+        await emit(
+            task_id,
+            "tool_call",
+            f"{device} › {tool}",
+            tool_event_payload(
+                device=device,
+                tool=tool,
+                args=args,
+                status="done",
+                ok=ok,
+                output=res.get("text", ""),
+            ),
+        )
+        return res, ok
+    except Exception as exc:
+        retry = await _maybe_mcp_repair(
+            task_id,
+            device,
+            server,
+            tool,
+            args,
+            str(exc),
+            manager,
+            emit,
+            task_title=task_title,
+            task_issue=task_issue,
+            error=str(exc),
+        )
+        if retry is not None:
+            return retry
+        await emit(
+            task_id,
+            "tool_call",
+            f"{device} › {tool} failed",
+            tool_event_payload(
+                device=device,
+                tool=tool,
+                args=args,
+                status="error",
+                ok=False,
+                error=str(exc),
+            ),
+        )
+        return {"text": str(exc), "isError": True}, False
+
+
+async def _maybe_mcp_repair(
+    task_id: int,
+    device: str,
+    server: str,
+    tool: str,
+    args: dict,
+    output: str | None,
+    manager,
+    emit: EmitFn,
+    *,
+    task_title: str,
+    task_issue: str,
+    error: str | None = None,
+) -> tuple[dict, bool] | None:
+    if await _repair_attempted(task_id, server, tool):
+        return None
+    from app.agent.mcp_development import maybe_repair_after_tool_failure
+
+    await _mark_repair_attempted(task_id, server, tool)
+    return await maybe_repair_after_tool_failure(
+        task_id=task_id,
+        device=device,
+        server=server,
+        tool=tool,
+        args=args,
+        issue=task_issue or task_title,
+        error=error,
+        output=output,
+        manager=manager,
+        emit=emit,
+        task_title=task_title,
+    )
+
+
+async def invoke_connect(
+    task_id: int,
+    device: str,
+    server: str,
+    tool: str,
+    args: dict,
+    manager,
+    emit: EmitFn,
+) -> bool:
+    try:
+        res = await call_tool_resilient(
+            manager, server, tool, args, task_id=task_id, emit=emit
+        )
+        ok = tool_result_ok(res)
+        msg = f"{device}: {tool} ok" if ok else f"{device}: {tool} failed — {res.get('text', '')[:200]}"
+        await emit(task_id, "connect", msg, {"output": res.get("text", "")[:400], "ok": ok, "tool": tool})
+        return ok
+    except Exception as exc:
+        await emit(task_id, "connect", f"{device}: {tool} failed — {exc}", {"ok": False, "tool": tool})
+        return False