Initial commit: Agentic OS troubleshooting platform

Self-hosted, Docker-based agentic troubleshooting platform: FastAPI backend +
LangGraph agent, Next.js UI, tiered LLM routing (local Ollama -> Gemini ->
DeepSeek -> OpenRouter), MCP server manager, encrypted device credentials,
RBAC, audit log, project-memory + Obsidian integrations, and editable
troubleshooting decision rules tuned for the GeneseasX vessel stack.

Co-authored-by: Cursor <cursoragent@cursor.com>

backend/app/services/__init__.py

@@ -0,0 +1 @@
+

backend/app/services/approval_filter.py

@@ -0,0 +1,57 @@
+"""Filter LLM proposed_fixes — only real write actions need human approval."""
+from __future__ import annotations
+
+# Connect / session tools already run during diagnostics — never gate on these.
+_CONNECT_TOOLS = frozenset(
+    {
+        "proxmox_connect",
+        "pfsense_connect",
+        "asterisk_connect",
+        "ssh_connect",
+        "fortigate_connect",
+        "fortiswitch_connect",
+    }
+)
+
+# Read-only diagnostic tools the agent may mention but must not require approval for.
+_READ_TOOLS = frozenset(
+    {
+        "proxmox_list_nodes",
+        "proxmox_list_qemu",
+        "proxmox_list_lxc",
+        "proxmox_get_node_status",
+        "pfsense_get_system_status",
+        "pfsense_list_gateways",
+        "ssh_run",
+    }
+)
+
+
+def fix_requires_approval(fix: dict) -> bool:
+    """True only for actionable production config changes with a concrete write tool."""
+    if not fix.get("is_config_change"):
+        return False
+
+    tool = (fix.get("tool") or "").strip()
+    if not tool or tool == "manual-config-change":
+        return False
+    if tool in _CONNECT_TOOLS or tool in _READ_TOOLS:
+        return False
+    if tool.endswith("_connect"):
+        return False
+
+    return True
+
+
+def split_proposed_fixes(fixes: list | None) -> tuple[list[dict], list[dict]]:
+    """Return (approval_candidates, informational_only)."""
+    approval: list[dict] = []
+    info: list[dict] = []
+    for fix in fixes or []:
+        if not isinstance(fix, dict):
+            continue
+        if fix_requires_approval(fix):
+            approval.append(fix)
+        else:
+            info.append(fix)
+    return approval, info

backend/app/services/audit.py

@@ -0,0 +1,30 @@
+"""Audit logging helper."""
+from __future__ import annotations
+
+from typing import Any
+
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.models.audit import AuditLog
+from app.models.user import User
+
+
+async def record_audit(
+    db: AsyncSession,
+    actor: User | None,
+    action: str,
+    target_type: str | None = None,
+    target_id: int | None = None,
+    detail: dict[str, Any] | None = None,
+) -> None:
+    db.add(
+        AuditLog(
+            actor_id=actor.id if actor else None,
+            actor_email=actor.email if actor else None,
+            action=action,
+            target_type=target_type,
+            target_id=target_id,
+            detail=detail,
+        )
+    )
+    await db.commit()

backend/app/services/balance.py

@@ -0,0 +1,148 @@
+"""Poll OpenRouter and DeepSeek for credits/balance and store snapshots."""
+from __future__ import annotations
+
+import asyncio
+import logging
+
+import httpx
+from sqlalchemy import select
+
+from app.config import settings
+from app.database import SessionLocal
+from app.models.balance import BalanceSnapshot
+
+logger = logging.getLogger(__name__)
+
+
+async def fetch_openrouter() -> dict:
+    """Return {total_credits, total_usage, remaining} for OpenRouter.
+
+    Prefers the provisioning key against /credits (account totals); falls back to
+    /key (per-key limit/usage) when only the regular API key is available.
+    """
+    key = settings.openrouter_provisioning_key or settings.openrouter_api_key
+    if not key:
+        return {"error": "no OpenRouter key configured"}
+
+    async with httpx.AsyncClient(timeout=20) as client:
+        # Try account-wide credits first
+        if settings.openrouter_provisioning_key:
+            r = await client.get(
+                "https://openrouter.ai/api/v1/credits",
+                headers={"Authorization": f"Bearer {settings.openrouter_provisioning_key}"},
+            )
+            if r.status_code == 200:
+                data = r.json().get("data", {})
+                total = data.get("total_credits")
+                usage = data.get("total_usage")
+                remaining = (total - usage) if total is not None and usage is not None else None
+                return {
+                    "total_credits": total,
+                    "total_usage": usage,
+                    "remaining": remaining,
+                    "raw": data,
+                }
+
+        # Fall back to per-key info
+        r = await client.get(
+            "https://openrouter.ai/api/v1/key",
+            headers={"Authorization": f"Bearer {settings.openrouter_api_key}"},
+        )
+        r.raise_for_status()
+        data = r.json().get("data", {})
+        limit = data.get("limit")
+        usage = data.get("usage")
+        remaining = data.get("limit_remaining")
+        if remaining is None and limit is not None and usage is not None:
+            remaining = limit - usage
+        return {
+            "total_credits": limit,
+            "total_usage": usage,
+            "remaining": remaining,
+            "raw": data,
+        }
+
+
+async def fetch_deepseek() -> dict:
+    if not settings.deepseek_api_key:
+        return {"error": "no DeepSeek key configured"}
+    async with httpx.AsyncClient(timeout=20) as client:
+        r = await client.get(
+            "https://api.deepseek.com/user/balance",
+            headers={"Authorization": f"Bearer {settings.deepseek_api_key}"},
+        )
+        r.raise_for_status()
+        data = r.json()
+        infos = data.get("balance_infos", [])
+        info = infos[0] if infos else {}
+        total = float(info.get("total_balance")) if info.get("total_balance") else None
+        return {
+            "total_credits": None,
+            "total_usage": None,
+            "remaining": total,
+            "currency": info.get("currency", "USD"),
+            "raw": data,
+        }
+
+
+async def _store(provider: str, data: dict) -> None:
+    async with SessionLocal() as db:
+        db.add(
+            BalanceSnapshot(
+                provider=provider,
+                total_credits=data.get("total_credits"),
+                total_usage=data.get("total_usage"),
+                remaining=data.get("remaining"),
+                currency=data.get("currency", "USD"),
+                raw=data.get("raw"),
+                error=data.get("error"),
+            )
+        )
+        await db.commit()
+
+
+async def poll_once() -> None:
+    for provider, fetcher in (("openrouter", fetch_openrouter), ("deepseek", fetch_deepseek)):
+        try:
+            data = await fetcher()
+        except Exception as exc:  # noqa: BLE001
+            data = {"error": str(exc)}
+            logger.warning("Balance poll failed for %s: %s", provider, exc)
+        if data.get("error"):
+            # Keep last good snapshot visible when DNS/network blips.
+            async with SessionLocal() as db:
+                res = await db.execute(
+                    select(BalanceSnapshot)
+                    .where(BalanceSnapshot.provider == provider)
+                    .where(BalanceSnapshot.error.is_(None))
+                    .where(BalanceSnapshot.remaining.is_not(None))
+                    .order_by(BalanceSnapshot.created_at.desc())
+                    .limit(1)
+                )
+                if res.scalar_one_or_none():
+                    logger.info("Skipping error balance snapshot for %s: %s", provider, data["error"])
+                    continue
+        await _store(provider, data)
+
+
+async def latest_remaining_usd() -> float | None:
+    """Best estimate of remaining USD across configured providers (max)."""
+    async with SessionLocal() as db:
+        remainings: list[float] = []
+        for provider in ("openrouter", "deepseek"):
+            res = await db.execute(
+                select(BalanceSnapshot)
+                .where(BalanceSnapshot.provider == provider)
+                .order_by(BalanceSnapshot.created_at.desc())
+                .limit(1)
+            )
+            snap = res.scalar_one_or_none()
+            if snap and snap.remaining is not None:
+                remainings.append(snap.remaining)
+        return max(remainings) if remainings else None
+
+
+async def balance_poller_loop() -> None:
+    while True:
+        await poll_once()
+        await asyncio.sleep(settings.balance_poll_interval_seconds)

backend/app/services/device_defaults.py

@@ -0,0 +1,173 @@
+"""Default device credentials and ports from environment variables.
+
+Naming convention (catalog_key → env prefix):
+  proxmox      → DEVICE_PROXMOX_*  (API token + SSH fallback)
+  docker_vm    → DEVICE_DOCKER_VM_*
+  pfsense      → DEVICE_PFSENSE_*
+
+Per slot:
+  DEVICE_<PREFIX>_PORT
+  DEVICE_<PREFIX>_USERNAME
+  DEVICE_<PREFIX>_PASSWORD      (SSH / login)
+  DEVICE_<PREFIX>_API_KEY       (REST API devices)
+
+Proxmox API (proxmox-mcp — tried before SSH):
+  DEVICE_PROXMOX_API_URL          (optional override; leave blank to use https://<vessel-public-ip>:8006)
+  DEVICE_PROXMOX_API_PORT         (optional; default 8006 when API URL is auto-built)
+  DEVICE_PROXMOX_TOKEN_ID         (e.g. root@pam!agentic)
+  DEVICE_PROXMOX_TOKEN_SECRET
+  DEVICE_PROXMOX_VERIFY_SSL       (true/false, default false)
+
+Global fallbacks for SSH slots:
+  DEVICE_SSH_USERNAME
+  DEVICE_SSH_PASSWORD
+
+Global fallback for API-key slots:
+  DEVICE_API_KEY
+"""
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+
+from app.inventory_catalog import CatalogEntry
+
+# catalog_key → how secrets are stored / resolved
+SLOT_AUTH: dict[str, dict] = {
+    "proxmox": {"auth": "ssh", "secret_field": "PASSWORD"},
+    "docker_vm": {"auth": "ssh", "secret_field": "PASSWORD"},
+    "asterisk": {"auth": "ssh", "secret_field": "PASSWORD"},
+    "asterisk_geneseasx": {"auth": "ssh", "secret_field": "PASSWORD"},
+    "asterisk_satbox": {"auth": "ssh", "secret_field": "PASSWORD"},
+    "debian_host": {"auth": "ssh", "secret_field": "PASSWORD"},
+    "tplink": {"auth": "ssh", "secret_field": "PASSWORD"},
+    "pfsense": {"auth": "api_key", "secret_field": "API_KEY"},
+    "fortigate": {"auth": "api_key", "secret_field": "API_KEY"},
+    "fortiswitch": {"auth": "password", "secret_field": "PASSWORD"},
+}
+
+
+@dataclass
+class SlotDefaults:
+    port: int | None = None
+    username: str | None = None
+    secret: str | None = None
+    secret_kind: str = "password"  # password | api_key
+    from_env: bool = False
+
+
+def _env(name: str) -> str | None:
+    val = os.environ.get(name, "").strip()
+    return val or None
+
+
+def _prefix(catalog_key: str) -> str:
+    return catalog_key.upper()
+
+
+def resolve_slot_defaults(catalog_key: str, entry: CatalogEntry) -> SlotDefaults:
+    """Load defaults for a catalog slot from .env (never exposed via API as plaintext)."""
+    prefix = _prefix(catalog_key)
+    meta = SLOT_AUTH.get(catalog_key, {"auth": "ssh", "secret_field": "PASSWORD"})
+    secret_kind = "api_key" if meta["auth"] == "api_key" else "password"
+
+    port_raw = _env(f"DEVICE_{prefix}_PORT")
+    port = int(port_raw) if port_raw and port_raw.isdigit() else None
+
+    username = _env(f"DEVICE_{prefix}_USERNAME")
+    secret: str | None = None
+
+    if meta["secret_field"] == "API_KEY":
+        secret = _env(f"DEVICE_{prefix}_API_KEY") or _env("DEVICE_API_KEY")
+    else:
+        secret = _env(f"DEVICE_{prefix}_PASSWORD")
+
+    if meta["auth"] == "ssh":
+        username = username or _env("DEVICE_SSH_USERNAME")
+        secret = secret or _env("DEVICE_SSH_PASSWORD")
+
+    if meta["auth"] == "api_key" and not username:
+        username = _env(f"DEVICE_{prefix}_USERNAME")
+
+    from_env = any(
+        [
+            port is not None,
+            bool(username),
+            bool(secret),
+        ]
+    )
+
+    return SlotDefaults(
+        port=port,
+        username=username,
+        secret=secret,
+        secret_kind=secret_kind,
+        from_env=from_env,
+    )
+
+
+def _truthy_env(name: str) -> bool:
+    val = os.environ.get(name, "").strip().lower()
+    return val in ("1", "true", "yes", "on")
+
+
+PROXMOX_DEFAULT_API_PORT = 8006
+
+
+def build_proxmox_api_url(public_ip: str) -> str | None:
+    """Build Proxmox API URL from vessel/device public IP unless DEVICE_PROXMOX_API_URL is set."""
+    override = _env("DEVICE_PROXMOX_API_URL")
+    if override:
+        return override
+    if not public_ip:
+        return None
+    port_raw = _env("DEVICE_PROXMOX_API_PORT")
+    port = int(port_raw) if port_raw and port_raw.isdigit() else PROXMOX_DEFAULT_API_PORT
+    host = public_ip.strip().removeprefix("https://").removeprefix("http://").split("/")[0]
+    if ":" in host and not host.startswith("["):
+        # Already host:port — use as-is with scheme only.
+        return f"https://{host}"
+    return f"https://{host}:{port}"
+
+
+def resolve_proxmox_api_defaults(public_ip: str) -> dict:
+    """Proxmox VE API token settings from env (used by proxmox-mcp connect).
+
+    ``public_ip`` is the vessel public IP stored on the proxmox device row (``device.address``).
+    When DEVICE_PROXMOX_API_URL is blank, the URL becomes ``https://<public_ip>:8006``.
+    """
+    return {
+        "proxmox_api_url": build_proxmox_api_url(public_ip),
+        "proxmox_token_id": _env("DEVICE_PROXMOX_TOKEN_ID"),
+        "proxmox_token_secret": _env("DEVICE_PROXMOX_TOKEN_SECRET"),
+        "proxmox_verify_ssl": _truthy_env("DEVICE_PROXMOX_VERIFY_SSL"),
+    }
+
+
+def proxmox_api_configured(ctx: dict) -> bool:
+    return bool(ctx.get("proxmox_token_id") and ctx.get("proxmox_token_secret"))
+
+
+def merge_selection_with_defaults(
+    catalog_key: str,
+    entry: CatalogEntry,
+    *,
+    port: int | None,
+    username: str | None,
+    secret: str | None,
+    existing_secret: str | None = None,
+) -> tuple[int | None, str | None, str | None]:
+    """Apply env defaults where the UI/API did not supply a value."""
+    defaults = resolve_slot_defaults(catalog_key, entry)
+
+    resolved_port = port if port is not None else defaults.port if defaults.port is not None else entry.default_port
+    resolved_username = username if username is not None else defaults.username or entry.default_username
+
+    if secret:
+        resolved_secret = secret
+    elif existing_secret:
+        resolved_secret = existing_secret
+    else:
+        resolved_secret = defaults.secret
+
+    return resolved_port, resolved_username, resolved_secret

backend/app/services/diagnostic_format.py

@@ -0,0 +1,374 @@
+"""Format MCP diagnostic output for reports, Obsidian, and UI."""
+from __future__ import annotations
+
+import json
+import re
+from typing import Any
+
+MAX_REPORT_OUTPUT_PER_TOOL = 4_000
+MAX_OBSIDIAN_DIAG_CHARS = 120_000
+MAX_EVENT_OUTPUT = 12_000
+
+_TABLE_TOOLS = frozenset(
+    {
+        "pfsense_list_firewall_rules",
+        "pfsense_list_port_forwards",
+        "pfsense_list_outbound_nat_mappings",
+        "pfsense_list_aliases",
+        "pfsense_list_interfaces",
+        "proxmox_list_qemu",
+        "proxmox_list_lxc",
+        "proxmox_get_qemu_status",
+        "proxmox_get_qemu_config",
+    }
+)
+
+
+def _clip(text: str, limit: int) -> str:
+    if len(text) <= limit:
+        return text
+    return text[: limit - 40] + f"\n\n… ({len(text) - limit + 40} chars truncated)"
+
+
+def _parse_json_payload(text: str) -> Any | None:
+    text = (text or "").strip()
+    if not text:
+        return None
+    try:
+        return json.loads(text)
+    except json.JSONDecodeError:
+        pass
+    start, end = text.find("{"), text.rfind("}")
+    if start != -1 and end > start:
+        try:
+            return json.loads(text[start : end + 1])
+        except json.JSONDecodeError:
+            return None
+    return None
+
+
+def _cell(val: Any) -> str:
+    if val is None:
+        return ""
+    if isinstance(val, list):
+        return ", ".join(str(x) for x in val)
+    s = str(val)
+    s = re.sub(r"&amp;+", "&", s)
+    return s.replace("|", "\\|").replace("\n", " ")
+
+
+def _firewall_rules_table(rows: list[dict]) -> str:
+    header = "| # | Action | Interface | Protocol | Source | Destination | Description |"
+    sep = "|---|---|---|---|---|---|---|"
+    lines = [header, sep]
+    for i, row in enumerate(rows[:300], start=1):
+        lines.append(
+            "| "
+            + " | ".join(
+                [
+                    str(i),
+                    _cell(row.get("type")),
+                    _cell(row.get("interface")),
+                    _cell(row.get("protocol")),
+                    _cell(row.get("source")),
+                    _cell(row.get("destination")),
+                    _cell(row.get("descr")),
+                ]
+            )
+            + " |"
+        )
+    if len(rows) > 300:
+        lines.append(f"\n_Showing 300 of {len(rows)} rules._")
+    return "\n".join(lines)
+
+
+def _port_forwards_table(rows: list[dict]) -> str:
+    header = "| # | Interface | Protocol | Destination | Target | Local port | Description |"
+    sep = "|---|---|---|---|---|---|---|"
+    lines = [header, sep]
+    for i, row in enumerate(rows[:200], start=1):
+        lines.append(
+            "| "
+            + " | ".join(
+                [
+                    str(i),
+                    _cell(row.get("interface")),
+                    _cell(row.get("protocol")),
+                    _cell(row.get("destination")),
+                    _cell(row.get("target")),
+                    _cell(row.get("local_port") or row.get("destination_port")),
+                    _cell(row.get("descr")),
+                ]
+            )
+            + " |"
+        )
+    return "\n".join(lines)
+
+
+def _nat_table(rows: list[dict]) -> str:
+    header = "| # | Interface | Source | Destination | Target | Description |"
+    sep = "|---|---|---|---|---|---|"
+    lines = [header, sep]
+    for i, row in enumerate(rows[:200], start=1):
+        lines.append(
+            "| "
+            + " | ".join(
+                [
+                    str(i),
+                    _cell(row.get("interface")),
+                    _cell(row.get("source")),
+                    _cell(row.get("destination")),
+                    _cell(row.get("target")),
+                    _cell(row.get("descr")),
+                ]
+            )
+            + " |"
+        )
+    return "\n".join(lines)
+
+
+def _aliases_table(rows: list[dict]) -> str:
+    header = "| Name | Type | Addresses | Description |"
+    sep = "|---|---|---|---|"
+    lines = [header, sep]
+    for row in rows[:200]:
+        lines.append(
+            "| "
+            + " | ".join(
+                [
+                    _cell(row.get("name")),
+                    _cell(row.get("type")),
+                    _cell(row.get("address")),
+                    _cell(row.get("descr")),
+                ]
+            )
+            + " |"
+        )
+    return "\n".join(lines)
+
+
+def _interfaces_table(rows: list[dict]) -> str:
+    header = "| ID | Interface | Enabled | Description | IPv4 |"
+    sep = "|---|---|---|---|---|"
+    lines = [header, sep]
+    for row in rows[:50]:
+        lines.append(
+            "| "
+            + " | ".join(
+                [
+                    _cell(row.get("id")),
+                    _cell(row.get("if")),
+                    _cell(row.get("enable")),
+                    _cell(row.get("descr")),
+                    _cell(row.get("ipaddr")),
+                ]
+            )
+            + " |"
+        )
+    return "\n".join(lines)
+
+
+def _qemu_list_table(rows: list[dict]) -> str:
+    header = "| VMID | Name | Status | CPU | Memory | Disk |"
+    sep = "|---|---|---|---|---|---|"
+    lines = [header, sep]
+    for row in rows[:100]:
+        mem = row.get("maxmem") or row.get("mem")
+        if isinstance(mem, (int, float)) and mem > 1024 * 1024:
+            mem = f"{mem / (1024 ** 3):.1f} GB"
+        disk = row.get("maxdisk") or row.get("disk")
+        if isinstance(disk, (int, float)) and disk > 1024 * 1024:
+            disk = f"{disk / (1024 ** 3):.1f} GB"
+        lines.append(
+            "| "
+            + " | ".join(
+                [
+                    _cell(row.get("vmid")),
+                    _cell(row.get("name")),
+                    _cell(row.get("status")),
+                    _cell(row.get("cpus") or row.get("cpu")),
+                    _cell(mem),
+                    _cell(disk),
+                ]
+            )
+            + " |"
+        )
+    return "\n".join(lines)
+
+
+def _lxc_list_table(rows: list[dict]) -> str:
+    header = "| VMID | Name | Status |"
+    sep = "|---|---|---|"
+    lines = [header, sep]
+    for row in rows[:100]:
+        lines.append(
+            "| "
+            + " | ".join([_cell(row.get("vmid")), _cell(row.get("name")), _cell(row.get("status"))])
+            + " |"
+        )
+    return "\n".join(lines)
+
+
+def _qemu_config_table(data: dict) -> str:
+    lines = ["| Key | Value |", "|---|---|"]
+    for key in sorted(data.keys()):
+        if key.startswith("digest"):
+            continue
+        lines.append(f"| {_cell(key)} | {_cell(data[key])} |")
+    return "\n".join(lines)
+
+
+def _qemu_status_table(data: dict) -> str:
+    if not data:
+        return "_No status data._"
+    lines = ["| Field | Value |", "|---|---|"]
+    for key in ("name", "status", "qmpstatus", "vmid", "uptime", "cpu", "mem", "maxmem", "disk", "maxdisk", "pid"):
+        if key in data:
+            val = data[key]
+            if isinstance(val, (int, float)) and key in ("mem", "maxmem", "disk", "maxdisk") and val > 1024 * 1024:
+                val = f"{val / (1024 ** 3):.2f} GB"
+            lines.append(f"| {_cell(key)} | {_cell(val)} |")
+    for key, val in sorted(data.items()):
+        if key in ("name", "status", "qmpstatus", "vmid", "uptime", "cpu", "mem", "maxmem", "disk", "maxdisk", "pid"):
+            continue
+        lines.append(f"| {_cell(key)} | {_cell(val)} |")
+    return "\n".join(lines)
+
+
+def _rows_from_payload(parsed: Any) -> list[dict]:
+    if isinstance(parsed, list):
+        return [r for r in parsed if isinstance(r, dict)]
+    if isinstance(parsed, dict):
+        data = parsed.get("data")
+        if isinstance(data, list):
+            return [r for r in data if isinstance(r, dict)]
+        return [parsed]
+    return []
+
+
+def _qm_list_text_table(text: str) -> str:
+    lines = ["| VMID | Name | Status | Mem (MB) | Boot disk (GB) |", "|---|---|---|---|---|"]
+    for row in text.splitlines():
+        row = row.strip()
+        if not row or row.startswith("VMID") or row.startswith("-"):
+            continue
+        parts = row.split()
+        if not parts or not parts[0].isdigit():
+            continue
+        vmid, name, status = parts[0], parts[1] if len(parts) > 1 else "", parts[2] if len(parts) > 2 else ""
+        mem = parts[3] if len(parts) > 3 else ""
+        disk = parts[4] if len(parts) > 4 else ""
+        lines.append(f"| {vmid} | {name} | {status} | {mem} | {disk} |")
+    return "\n".join(lines) if len(lines) > 2 else f"```\n{_clip(text, 8000)}\n```"
+
+
+def _qm_config_text_block(text: str) -> str:
+    lines = ["| Key | Value |", "|---|---|"]
+    for raw in text.splitlines():
+        raw = raw.strip()
+        if not raw or ":" not in raw:
+            continue
+        key, val = raw.split(":", 1)
+        lines.append(f"| {_cell(key.strip())} | {_cell(val.strip())} |")
+    return "\n".join(lines) if len(lines) > 2 else f"```\n{_clip(text, 8000)}\n```"
+
+
+def format_tool_output_markdown(tool: str, output: str) -> str:
+    """Turn raw MCP JSON/text into readable markdown when possible."""
+    parsed = _parse_json_payload(output)
+    if tool == "proxmox_list_qemu":
+        if isinstance(parsed, (dict, list)):
+            rows = _rows_from_payload(parsed)
+            if rows and rows[0].get("vmid") is not None:
+                return _qemu_list_table(rows)
+        if re.search(r"^\s*\d+\s+\S+\s+(running|stopped)", output, re.MULTILINE):
+            return _qm_list_text_table(output)
+    if tool == "proxmox_list_lxc" and isinstance(parsed, (dict, list)):
+        rows = _rows_from_payload(parsed)
+        if rows:
+            return _lxc_list_table(rows)
+    if tool == "proxmox_get_qemu_config":
+        if isinstance(parsed, dict):
+            inner = parsed.get("data") if isinstance(parsed.get("data"), dict) else parsed
+            if isinstance(inner, dict) and inner:
+                return _qemu_config_table(inner)
+        if ":" in output:
+            return _qm_config_text_block(output)
+    if tool == "proxmox_get_qemu_status" and isinstance(parsed, dict):
+        inner = parsed.get("data") if isinstance(parsed.get("data"), dict) else parsed
+        if isinstance(inner, dict):
+            return _qemu_status_table(inner)
+
+    if isinstance(parsed, dict):
+        data = parsed.get("data")
+        if isinstance(data, list) and data:
+            if tool == "pfsense_list_firewall_rules":
+                return _firewall_rules_table(data)
+            if tool == "pfsense_list_port_forwards":
+                return _port_forwards_table(data)
+            if tool == "pfsense_list_outbound_nat_mappings":
+                return _nat_table(data)
+            if tool == "pfsense_list_aliases":
+                return _aliases_table(data)
+            if tool == "pfsense_list_interfaces":
+                return _interfaces_table(data)
+            if tool in _TABLE_TOOLS:
+                return f"```json\n{json.dumps(data[:50], indent=2)}\n```"
+    return f"```\n{_clip(output, 20_000)}\n```"
+
+
+def prepare_report_diagnostics(diagnostics: list[dict]) -> list[dict]:
+    """Normalize diagnostics for persistence in task.report."""
+    out: list[dict] = []
+    for d in diagnostics:
+        text = d.get("output") or ""
+        formatted = ""
+        tool = d.get("tool")
+        if text:
+            if tool in _TABLE_TOOLS:
+                formatted = format_tool_output_markdown(tool, text)
+            elif tool == "ssh_run":
+                if re.search(r"^\s*\d+\s+\S+\s+(running|stopped)", text, re.MULTILINE):
+                    formatted = format_tool_output_markdown("proxmox_list_qemu", text)
+                elif re.match(r"^[a-z][a-z0-9_-]*:", text.strip(), re.IGNORECASE):
+                    formatted = format_tool_output_markdown("proxmox_get_qemu_config", text)
+        entry = {
+            "device": d.get("device"),
+            "tool": tool,
+            "ok": d.get("ok"),
+            "intent": d.get("intent"),
+            "vmid": d.get("vmid"),
+            "output_chars": len(text),
+            "raw_available": bool(text),
+            "truncated": len(text) > MAX_REPORT_OUTPUT_PER_TOOL,
+        }
+        if formatted:
+            entry["formatted"] = formatted
+        elif text:
+            entry["output"] = _clip(text, MAX_REPORT_OUTPUT_PER_TOOL)
+        out.append(entry)
+    return out
+
+
+def diagnostics_to_markdown(diagnostics: list[dict]) -> str:
+    """Build Obsidian section from report-style diagnostics."""
+    if not diagnostics:
+        return "_No live diagnostic output captured._"
+
+    parts: list[str] = []
+    total = 0
+    for d in diagnostics:
+        tool = d.get("tool") or "unknown"
+        device = d.get("device") or "?"
+        ok = d.get("ok")
+        vmid = d.get("vmid")
+        vm_label = f" VM {vmid}" if vmid is not None else ""
+        header = f"### [{device}] {tool}{vm_label} ({'ok' if ok else 'fail'})"
+        body = d.get("formatted") or format_tool_output_markdown(tool, d.get("output") or "")
+        chunk = f"{header}\n\n{body}\n"
+        if total + len(chunk) > MAX_OBSIDIAN_DIAG_CHARS:
+            parts.append("_Additional diagnostic output omitted (size limit)._")
+            break
+        parts.append(chunk)
+        total += len(chunk)
+    return "\n".join(parts)

backend/app/services/events.py

@@ -0,0 +1,96 @@
+"""Task event bus: persist events to DB and publish to Redis for live streaming."""
+from __future__ import annotations
+
+import json
+from datetime import datetime, timezone
+from typing import Any
+
+import redis.asyncio as aioredis
+
+from app.config import settings
+from app.database import SessionLocal
+from app.models.task import TaskEvent
+
+_redis: aioredis.Redis | None = None
+
+# Redis keys
+TASK_QUEUE = "agentic:task_queue"
+
+# High-frequency live-only event kinds: streamed to the UI but not written to
+# Postgres, to avoid a separate transaction per progress/tool-start tick.
+EPHEMERAL_KINDS = frozenset({"progress", "tool_start"})
+
+
+def get_redis() -> aioredis.Redis:
+    global _redis
+    if _redis is None:
+        _redis = aioredis.from_url(
+            settings.redis_url, decode_responses=True, max_connections=32
+        )
+    return _redis
+
+
+def _channel(task_id: int) -> str:
+    return f"agentic:task:{task_id}:events"
+
+
+async def dedupe_task_queue(task_id: int) -> int:
+    """Remove stale duplicate entries for a task id from the Redis queue."""
+    redis = get_redis()
+    needle = str(task_id)
+    items = await redis.lrange(TASK_QUEUE, 0, -1)
+    removed = sum(1 for x in items if x == needle)
+    if removed:
+        await redis.delete(TASK_QUEUE)
+        for item in items:
+            if item != needle:
+                await redis.rpush(TASK_QUEUE, item)
+    return removed
+
+
+async def enqueue_task(task_id: int) -> None:
+    await dedupe_task_queue(task_id)
+    await get_redis().rpush(TASK_QUEUE, str(task_id))
+
+
+async def publish_event(
+    task_id: int,
+    kind: str,
+    message: str | None = None,
+    payload: dict[str, Any] | None = None,
+    persist: bool | None = None,
+) -> None:
+    """Persist an event and publish it to the task's Redis channel.
+
+    When ``persist`` is None, high-frequency ephemeral kinds (progress,
+    tool_start) are streamed live but not written to the DB, while all other
+    kinds are persisted. Pass an explicit bool to override.
+    """
+    if persist is None:
+        persist = kind not in EPHEMERAL_KINDS
+    if persist:
+        async with SessionLocal() as db:
+            db.add(TaskEvent(task_id=task_id, kind=kind, message=message, payload=payload))
+            await db.commit()
+
+    event = {
+        "task_id": task_id,
+        "kind": kind,
+        "message": message,
+        "payload": payload,
+        "ts": datetime.now(timezone.utc).isoformat(),
+    }
+    await get_redis().publish(_channel(task_id), json.dumps(event))
+
+
+async def subscribe_events(task_id: int):
+    """Async generator yielding decoded events for a task."""
+    pubsub = get_redis().pubsub()
+    await pubsub.subscribe(_channel(task_id))
+    try:
+        async for message in pubsub.listen():
+            if message["type"] == "message":
+                yield json.loads(message["data"])
+    finally:
+        await pubsub.unsubscribe(_channel(task_id))
+        await pubsub.close()

backend/app/services/integrations.py

@@ -0,0 +1,353 @@
+"""Per-task persistence: project-memory (HTTP) and Obsidian vault (git)."""
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+import os
+import time
+from datetime import date
+
+import httpx
+from slugify import slugify
+
+from app.config import settings
+
+logger = logging.getLogger(__name__)
+
+MCP_ACCEPT = "application/json, text/event-stream"
+
+
+def _parse_mcp_response(resp: httpx.Response) -> dict:
+    """Parse JSON or SSE body from streamable HTTP MCP responses."""
+    content_type = resp.headers.get("content-type", "")
+    text = resp.text.strip()
+    if "text/event-stream" in content_type or text.startswith("event:") or text.startswith("data:"):
+        for line in text.splitlines():
+            if line.startswith("data:"):
+                payload = line[5:].strip()
+                if payload and payload != "[DONE]":
+                    return json.loads(payload)
+        return {}
+    if text:
+        return resp.json()
+    return {}
+
+
+class _MemoryMcpSession:
+    """Reusable streamable-HTTP session for the project-memory MCP server.
+
+    Performs the initialize handshake once and reuses the HTTP client +
+    Mcp-Session-Id across calls, re-initializing only when the session is
+    rejected (e.g. server restart / expired session).
+    """
+
+    def __init__(self) -> None:
+        self._client: httpx.AsyncClient | None = None
+        self._session_id: str | None = None
+        self._lock = asyncio.Lock()
+        self._req_id = 0
+
+    def _headers(self) -> dict[str, str]:
+        headers = {"Content-Type": "application/json", "Accept": MCP_ACCEPT}
+        if settings.memory_mcp_token:
+            headers["Authorization"] = f"Bearer {settings.memory_mcp_token}"
+        if self._session_id:
+            headers["Mcp-Session-Id"] = self._session_id
+        return headers
+
+    async def _ensure_client(self) -> httpx.AsyncClient:
+        if self._client is None:
+            self._client = httpx.AsyncClient(timeout=30)
+        return self._client
+
+    async def _handshake(self) -> None:
+        client = await self._ensure_client()
+        self._session_id = None
+        init_body = {
+            "jsonrpc": "2.0",
+            "id": 1,
+            "method": "initialize",
+            "params": {
+                "protocolVersion": "2024-11-05",
+                "capabilities": {},
+                "clientInfo": {"name": "agentic-os", "version": "0.1.0"},
+            },
+        }
+        resp = await client.post(settings.memory_mcp_url, headers=self._headers(), json=init_body)
+        resp.raise_for_status()
+        self._session_id = resp.headers.get("mcp-session-id") or resp.headers.get("Mcp-Session-Id")
+        await client.post(
+            settings.memory_mcp_url,
+            headers=self._headers(),
+            json={"jsonrpc": "2.0", "method": "notifications/initialized"},
+        )
+
+    async def _call_once(self, tool_name: str, arguments: dict) -> dict | None:
+        client = await self._ensure_client()
+        self._req_id += 1
+        body = {
+            "jsonrpc": "2.0",
+            "id": self._req_id,
+            "method": "tools/call",
+            "params": {"name": tool_name, "arguments": arguments},
+        }
+        resp = await client.post(settings.memory_mcp_url, headers=self._headers(), json=body)
+        resp.raise_for_status()
+        data = _parse_mcp_response(resp)
+        if data.get("error"):
+            raise RuntimeError(data["error"].get("message", str(data["error"])))
+        return data.get("result")
+
+    async def call(self, tool_name: str, arguments: dict) -> dict | None:
+        async with self._lock:
+            if self._session_id is None:
+                await self._handshake()
+            try:
+                return await self._call_once(tool_name, arguments)
+            except (httpx.HTTPStatusError, RuntimeError) as exc:
+                # Session may have expired; re-handshake once and retry.
+                logger.info("memory MCP retry after error: %s", exc)
+                await self._handshake()
+                return await self._call_once(tool_name, arguments)
+
+
+_memory_session = _MemoryMcpSession()
+
+
+async def memory_mcp_call(tool_name: str, arguments: dict) -> dict | None:
+    """Run one MCP tool call against project-memory over a reused HTTP session."""
+    if not settings.memory_mcp_url:
+        return None
+    return await _memory_session.call(tool_name, arguments)
+
+
+async def create_task_memory(
+    title: str,
+    content: str,
+    memory_type: str = "fix",
+    tags: list[str] | None = None,
+    importance: int = 3,
+    *,
+    vessel_name: str | None = None,
+    task_id: int | None = None,
+) -> str | None:
+    """Call memory_upsert on the project-memory MCP over HTTP JSON-RPC."""
+    tag_list = list(tags or ["agentic-os"])
+    if vessel_name:
+        from slugify import slugify
+
+        tag_list.append(f"vessel:{slugify(vessel_name, lowercase=True)}")
+    if task_id is not None:
+        tag_list.append(f"task:{task_id}")
+
+    body_parts = []
+    if vessel_name:
+        body_parts.append(f"Vessel: {vessel_name}")
+    if task_id is not None:
+        body_parts.append(f"Task ID: {task_id}")
+    body_parts.append(content)
+    full_content = "\n\n".join(body_parts)
+
+    args = {
+        "project_id": settings.memory_task_project_id,
+        "memory_type": memory_type,
+        "title": title,
+        "content": full_content,
+        "tags": tag_list,
+        "importance": importance,
+    }
+
+    try:
+        result = await memory_mcp_call("memory_upsert", args)
+        if not isinstance(result, dict):
+            return None
+        structured = result.get("structuredContent")
+        if isinstance(structured, dict):
+            mid = structured.get("id") or structured.get("memory_id")
+            if mid:
+                return str(mid)
+        for block in result.get("content", []):
+            text = block.get("text") if isinstance(block, dict) else None
+            if text and "id" in text:
+                return text[:128]
+    except Exception as exc:  # noqa: BLE001
+        logger.warning("create_task_memory failed: %s", exc)
+    return None
+
+
+def _vault_dir() -> str:
+    return os.path.join(settings.runtime_dir, "obsidian-vault")
+
+
+async def _run(cmd: list[str], cwd: str | None = None) -> tuple[int, str]:
+    proc = await asyncio.create_subprocess_exec(
+        *cmd,
+        cwd=cwd,
+        stdout=asyncio.subprocess.PIPE,
+        stderr=asyncio.subprocess.STDOUT,
+    )
+    out, _ = await proc.communicate()
+    return proc.returncode or 0, out.decode(errors="replace")
+
+
+async def _repair_vault_git(vault: str) -> None:
+    """Recover from stuck rebase/merge so commits can be pushed to origin/main."""
+    git_dir = os.path.join(vault, ".git")
+    if not os.path.isdir(git_dir):
+        return
+
+    if os.path.isdir(os.path.join(git_dir, "rebase-merge")) or os.path.isdir(
+        os.path.join(git_dir, "rebase-apply")
+    ):
+        logger.warning("Obsidian vault stuck in git rebase; attempting continue/abort")
+        code, _ = await _run(["git", "rebase", "--continue"], cwd=vault)
+        if code != 0:
+            await _run(["git", "rebase", "--abort"], cwd=vault)
+
+    if os.path.isfile(os.path.join(git_dir, "MERGE_HEAD")):
+        logger.warning("Obsidian vault stuck in git merge; aborting")
+        await _run(["git", "merge", "--abort"], cwd=vault)
+
+    code, _ = await _run(["git", "checkout", "main"], cwd=vault)
+    if code != 0:
+        await _run(["git", "checkout", "-B", "main"], cwd=vault)
+
+
+async def _vault_pull(vault: str) -> None:
+    await _repair_vault_git(vault)
+    code, out = await _run(["git", "pull", "--ff-only", "origin", "main"], cwd=vault)
+    if code != 0:
+        logger.warning("vault ff-only pull failed: %s", out[:300])
+        await _repair_vault_git(vault)
+        await _run(["git", "pull", "--no-rebase", "origin", "main"], cwd=vault)
+
+
+async def _vault_commit_and_push(vault: str, rel_path: str, title: str) -> tuple[bool, str | None]:
+    """Stage, commit, and push one note. Returns (pushed, error_message)."""
+    if not settings.obsidian_auto_push or not settings.gitea_token:
+        return False, None
+
+    await _repair_vault_git(vault)
+    await _run(["git", "add", rel_path], cwd=vault)
+    code, out = await _run(
+        [
+            "git",
+            "-c",
+            "user.email=agentic-os@paraskeva.net",
+            "-c",
+            "user.name=Agentic OS",
+            "commit",
+            "-m",
+            f"agentic-os task: {title}",
+        ],
+        cwd=vault,
+    )
+    if code != 0 and "nothing to commit" not in out.lower():
+        return False, out.strip()[:500]
+
+    await _repair_vault_git(vault)
+    code, out = await _run(["git", "push", "origin", "main"], cwd=vault)
+    if code != 0:
+        logger.warning("vault push failed: %s", out)
+        return False, out.strip()[:500]
+    return True, None
+
+
+_VAULT_PULL_TTL = 300.0  # seconds between background `git pull` of the vault
+_vault_pull_lock = asyncio.Lock()
+_vault_last_pull = 0.0
+
+
+async def ensure_vault(*, force_pull: bool = False) -> str | None:
+    global _vault_last_pull
+    vault = _vault_dir()
+    clone_url = settings.gitea_clone_url(settings.obsidian_vault_repo)
+    if os.path.isdir(os.path.join(vault, ".git")):
+        # Throttle pulls: skip if pulled recently (deduped across concurrent tasks).
+        now = time.monotonic()
+        if force_pull or now - _vault_last_pull >= _VAULT_PULL_TTL:
+            async with _vault_pull_lock:
+                if force_pull or time.monotonic() - _vault_last_pull >= _VAULT_PULL_TTL:
+                    await _vault_pull(vault)
+                    _vault_last_pull = time.monotonic()
+        return vault
+    os.makedirs(settings.runtime_dir, exist_ok=True)
+    code, out = await _run(["git", "clone", clone_url, vault])
+    if code != 0:
+        logger.warning("vault clone failed: %s", out)
+        return None
+    _vault_last_pull = time.monotonic()
+    return vault
+
+
+async def write_obsidian_note(
+    title: str,
+    issue: str,
+    steps: list[str] | str,
+    cause: str | None,
+    resolution: str | None,
+    resolved: bool,
+    *,
+    vessel_name: str | None = None,
+    task_id: int | None = None,
+    diagnostics_md: str | None = None,
+) -> tuple[str | None, str | None]:
+    """Write task note to vault. Returns (relative_path, push_error)."""
+    vault = await ensure_vault()
+    if not vault:
+        return None, "Obsidian vault unavailable (clone failed)"
+
+    folder = settings.obsidian_tasks_folder
+    os.makedirs(os.path.join(vault, folder), exist_ok=True)
+    slug = slugify(title)[:60] or "task"
+    filename = f"{date.today().isoformat()}-{slug}.md"
+    rel_path = f"{folder}/{filename}"
+    abs_path = os.path.join(vault, rel_path)
+
+    steps_md = (
+        "\n".join(f"- {s}" for s in steps) if isinstance(steps, list) else str(steps or "")
+    )
+    status_label = "Resolved" if resolved else "Unresolved"
+    vessel_line = f"vessel: {vessel_name}\n" if vessel_name else ""
+    task_line = f"task_id: {task_id}\n" if task_id is not None else ""
+    diag_section = ""
+    if diagnostics_md:
+        diag_section = f"\n## Diagnostic findings\n{diagnostics_md}\n"
+
+    body = f"""---
+tags: [agentic-os, troubleshooting]
+{vessel_line}{task_line}status: {status_label.lower()}
+created: {date.today().isoformat()}
+---
+
+# {title}
+
+## Vessel
+{vessel_name or "_Not specified._"}
+
+## Issue
+{issue}
+
+## Troubleshooting steps
+{steps_md}
+
+## Root cause
+{cause or "_Not determined._"}
+
+## Resolution ({status_label})
+{resolution or "_No resolution recorded._"}
+{diag_section}"""
+    def _write() -> None:
+        with open(abs_path, "w", encoding="utf-8") as fh:
+            fh.write(body)
+
+    await asyncio.to_thread(_write)
+
+    pushed, push_error = await _vault_commit_and_push(vault, rel_path, title)
+    if push_error:
+        logger.warning("Obsidian note written locally at %s but git push failed: %s", rel_path, push_error)
+    elif not pushed and settings.obsidian_auto_push:
+        push_error = "Note saved in vault clone only (auto-push disabled or no Gitea token)"
+
+    return rel_path, push_error

backend/app/services/investigation_context.py

@@ -0,0 +1,317 @@
+"""Load prior knowledge from project-memory and Obsidian before agent triage."""
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+import os
+import re
+from typing import Any
+
+from app.config import settings
+from app.services import integrations
+
+logger = logging.getLogger(__name__)
+
+_STOPWORDS = frozenset(
+    """
+    a an the and or but in on at to for of is are was were be been being
+    with from by as it this that these those i we you they he she
+    check status health task issue vessel ship
+    """.split()
+)
+
+_MAX_MEMORY_HITS = 8
+_MAX_OBSIDIAN_HITS = 6
+_EXCERPT_CHARS = 400
+
+
+def extract_search_keywords(text: str, *, max_terms: int = 8) -> list[str]:
+    """Pull meaningful terms from title + issue for memory/Obsidian search."""
+    raw = re.findall(r"[a-zA-Z0-9][a-zA-Z0-9._/-]{1,}", text.lower())
+    terms: list[str] = []
+    seen: set[str] = set()
+    for tok in raw:
+        if tok in _STOPWORDS or len(tok) < 3:
+            continue
+        if tok.isdigit():
+            continue
+        if tok not in seen:
+            seen.add(tok)
+            terms.append(tok)
+        if len(terms) >= max_terms:
+            break
+    return terms
+
+
+def _parse_memory_tool_result(result: dict | None) -> list[dict]:
+    """Normalize memory MCP tool results into a list of entry dicts."""
+    if not result:
+        return []
+
+    structured = result.get("structuredContent")
+    if isinstance(structured, dict):
+        for key in ("memories", "results", "entries", "items"):
+            val = structured.get(key)
+            if isinstance(val, list):
+                return [x for x in val if isinstance(x, dict)]
+        if structured.get("title") or structured.get("content"):
+            return [structured]
+
+    for block in result.get("content", []):
+        if not isinstance(block, dict):
+            continue
+        text = block.get("text")
+        if not text:
+            continue
+        text = text.strip()
+        try:
+            parsed = json.loads(text)
+        except json.JSONDecodeError:
+            continue
+        if isinstance(parsed, list):
+            return [x for x in parsed if isinstance(x, dict)]
+        if isinstance(parsed, dict):
+            for key in ("memories", "results", "entries", "items"):
+                val = parsed.get(key)
+                if isinstance(val, list):
+                    return [x for x in val if isinstance(x, dict)]
+            if parsed.get("title") or parsed.get("content"):
+                return [parsed]
+    return []
+
+
+def _memory_vessel_from_hit(hit: dict) -> str | None:
+    for tag in hit.get("tags") or []:
+        if isinstance(tag, str) and tag.startswith("vessel:"):
+            return tag.split(":", 1)[1].replace("-", " ")
+    content = hit.get("content") or ""
+    m = re.search(r"^Vessel:\s*(.+)$", content, re.MULTILINE)
+    if m:
+        return m.group(1).strip()
+    title = hit.get("title") or ""
+    m = re.search(r"^\[([^\]]+)\]", title)
+    if m and m.group(1).lower() not in ("task", "fix"):
+        return m.group(1).strip()
+    return None
+
+
+def _obsidian_vessel_from_text(text: str) -> str | None:
+    if not text.startswith("---"):
+        return None
+    end = text.find("\n---", 3)
+    if end == -1:
+        return None
+    front = text[3:end]
+    for line in front.splitlines():
+        if line.lower().startswith("vessel:"):
+            return line.split(":", 1)[1].strip()
+    return None
+
+
+def _memory_hit_from_entry(entry: dict, *, project_id: str) -> dict:
+    hit = {
+        "project_id": entry.get("project_id") or project_id,
+        "title": (entry.get("title") or "").strip(),
+        "content": (entry.get("content") or "")[:_EXCERPT_CHARS],
+        "memory_type": entry.get("memory_type") or entry.get("type") or "note",
+        "importance": entry.get("importance"),
+        "tags": entry.get("tags") or [],
+        "source": "memory",
+    }
+    return _enrich_hit_vessel(hit)
+
+
+def _enrich_hit_vessel(hit: dict, *, vessel_name: str | None = None) -> dict:
+    vessel = hit.get("vessel")
+    if not vessel and hit.get("source") == "memory":
+        vessel = _memory_vessel_from_hit(hit)
+    if not vessel and hit.get("source") == "obsidian":
+        vessel = _obsidian_vessel_from_text(hit.get("excerpt") or "")
+    if vessel:
+        hit["vessel"] = vessel
+    elif vessel_name:
+        hit["vessel"] = vessel_name
+    return hit
+
+
+async def _search_memory_projects(query: str, project_ids: list[str]) -> list[dict]:
+    hits: list[dict] = []
+    seen_titles: set[str] = set()
+
+    for project_id in project_ids:
+        try:
+            result = await integrations.memory_mcp_call(
+                "memory_search",
+                {"project_id": project_id, "query": query, "limit": 5},
+            )
+            for entry in _parse_memory_tool_result(result):
+                hit = _memory_hit_from_entry(entry, project_id=project_id)
+                key = hit["title"].lower()
+                if not hit["title"] or key in seen_titles:
+                    continue
+                seen_titles.add(key)
+                hits.append(hit)
+        except Exception as exc:  # noqa: BLE001
+            logger.warning("memory_search %s failed: %s", project_id, exc)
+
+    return hits[:_MAX_MEMORY_HITS]
+
+
+async def _recent_memory(project_ids: list[str]) -> list[dict]:
+    hits: list[dict] = []
+    for project_id in project_ids[:2]:
+        try:
+            result = await integrations.memory_mcp_call(
+                "memory_list_recent",
+                {"project_id": project_id, "limit": 10},
+            )
+            for entry in _parse_memory_tool_result(result):
+                hits.append(_memory_hit_from_entry(entry, project_id=project_id))
+        except Exception as exc:  # noqa: BLE001
+            logger.warning("memory_list_recent %s failed: %s", project_id, exc)
+    return hits
+
+
+def _obsidian_search_roots(vault: str) -> list[str]:
+    roots: list[str] = []
+    for rel in settings.obsidian_search_paths.split(","):
+        rel = rel.strip()
+        if not rel:
+            continue
+        path = os.path.join(vault, rel)
+        if os.path.isdir(path):
+            roots.append(path)
+    tasks = os.path.join(vault, settings.obsidian_tasks_folder)
+    if os.path.isdir(tasks) and tasks not in roots:
+        roots.insert(0, tasks)
+    return roots
+
+
+def _score_obsidian_file(
+    path: str, text: str, keywords: list[str], *, vessel_name: str | None = None
+) -> int:
+    name = os.path.basename(path).lower()
+    body = text.lower()
+    score = 0
+    for kw in keywords:
+        if kw in name:
+            score += 3
+        if kw in body:
+            score += 1
+    if vessel_name:
+        vslug = vessel_name.lower().replace(" ", "-")
+        note_vessel = _obsidian_vessel_from_text(text)
+        if note_vessel and note_vessel.lower() == vessel_name.lower():
+            score += 5
+        elif vslug in name or vessel_name.lower() in body:
+            score += 2
+    return score
+
+
+def _obsidian_title(text: str, filename: str) -> str:
+    for line in text.splitlines():
+        if line.startswith("# "):
+            return line[2:].strip()
+    return os.path.splitext(filename)[0].replace("-", " ")
+
+
+async def search_obsidian_notes(
+    title: str, issue: str, *, vessel_name: str | None = None
+) -> list[dict]:
+    """Search cloned Obsidian vault for notes related to this issue."""
+    vault = await integrations.ensure_vault()
+    if not vault:
+        return []
+
+    keywords = extract_search_keywords(f"{title} {issue}")
+    if vessel_name:
+        keywords = keywords + [vessel_name.lower().replace(" ", "-"), vessel_name.lower()]
+    if not keywords:
+        return []
+
+    # The vault scan is synchronous disk I/O — run it off the event loop.
+    return await asyncio.to_thread(_scan_obsidian_notes, vault, keywords, vessel_name)
+
+
+def _scan_obsidian_notes(
+    vault: str, keywords: list[str], vessel_name: str | None
+) -> list[dict]:
+    scored: list[tuple[int, dict]] = []
+    for root in _obsidian_search_roots(vault):
+        for dirpath, _, files in os.walk(root):
+            for filename in files:
+                if not filename.endswith(".md"):
+                    continue
+                abs_path = os.path.join(dirpath, filename)
+                try:
+                    with open(abs_path, encoding="utf-8", errors="replace") as fh:
+                        text = fh.read()
+                except OSError:
+                    continue
+                score = _score_obsidian_file(abs_path, text, keywords, vessel_name=vessel_name)
+                if score <= 0:
+                    continue
+                rel = os.path.relpath(abs_path, vault)
+                excerpt = text[:_EXCERPT_CHARS].strip()
+                hit = {
+                    "path": rel,
+                    "title": _obsidian_title(text, filename),
+                    "excerpt": excerpt,
+                    "score": score,
+                    "source": "obsidian",
+                }
+                scored.append((score, _enrich_hit_vessel(hit, vessel_name=vessel_name)))
+
+    scored.sort(key=lambda x: (-x[0], x[1]["path"]))
+    return [hit for _, hit in scored[:_MAX_OBSIDIAN_HITS]]
+
+
+async def gather_investigation_context(
+    title: str,
+    issue: str,
+    *,
+    vessel_name: str | None = None,
+    vessel_id: int | None = None,
+) -> dict[str, Any]:
+    """Search memory MCP + Obsidian vault for prior related investigations."""
+    keywords = extract_search_keywords(f"{title} {issue}")
+    query_parts = list(keywords)
+    if vessel_name:
+        query_parts.insert(0, vessel_name)
+    query = " ".join(query_parts) if query_parts else f"{title} {issue}"[:200]
+
+    project_ids = [
+        p.strip()
+        for p in settings.memory_search_projects.split(",")
+        if p.strip()
+    ]
+    if settings.memory_task_project_id not in project_ids:
+        project_ids.insert(0, settings.memory_task_project_id)
+
+    memory_hits = await _search_memory_projects(query, project_ids)
+
+    # If keyword search is sparse, add recent task memories for continuity.
+    if len(memory_hits) < 3:
+        recent = await _recent_memory(project_ids)
+        seen = {h["title"].lower() for h in memory_hits if h.get("title")}
+        for hit in recent:
+            t = hit.get("title", "").lower()
+            if t and t not in seen:
+                memory_hits.append(hit)
+                seen.add(t)
+            if len(memory_hits) >= _MAX_MEMORY_HITS:
+                break
+
+    obsidian_hits = await search_obsidian_notes(title, issue, vessel_name=vessel_name)
+
+    return {
+        "query": query,
+        "keywords": keywords,
+        "vessel_name": vessel_name,
+        "vessel_id": vessel_id,
+        "memory_hits": memory_hits[:_MAX_MEMORY_HITS],
+        "obsidian_hits": obsidian_hits,
+        "memory_count": len(memory_hits),
+        "obsidian_count": len(obsidian_hits),
+    }

backend/app/services/llm.py

@@ -0,0 +1,24 @@
+"""LLM client builders — delegates tier selection to model_router."""
+from __future__ import annotations
+
+from app.config import settings
+from app.services.llm_usage import ModelInfo
+from app.services.model_catalog import entry_for_model
+from app.services.model_config import get_routing_config, resolve_tier_model
+from app.services.model_router import build_chat_model, entry_to_info
+
+
+async def triage_model():
+    cfg = await get_routing_config()
+    entry = resolve_tier_model(cfg, "local")
+    if not entry:
+        entry = entry_for_model("ollama", settings.llm_local_model, "local")
+    return build_chat_model(entry, temperature=0.0)
+
+
+async def triage_model_info() -> ModelInfo:
+    cfg = await get_routing_config()
+    entry = resolve_tier_model(cfg, "local") or entry_for_model(
+        "ollama", settings.llm_local_model, "local"
+    )
+    return entry_to_info(entry, "triage")

backend/app/services/llm_usage.py

@@ -0,0 +1,140 @@
+"""Track LLM calls per task: provider, model, tokens, and estimated cost."""
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+from langchain_core.messages import BaseMessage
+from langchain_openai import ChatOpenAI
+
+# USD per 1M tokens (input, output). Local/Ollama is always zero.
+MODEL_PRICING: dict[str, tuple[float, float]] = {
+    "deepseek-chat": (0.14, 0.28),
+    "deepseek/deepseek-chat": (0.14, 0.28),
+    "qwen2.5:7b-instruct": (0.0, 0.0),
+    "qwen2.5-coder:14b": (0.0, 0.0),
+    # OpenRouter common fallbacks (approximate)
+    "anthropic/claude-3.5-sonnet": (3.0, 15.0),
+    "anthropic/claude-3.7-sonnet": (3.0, 15.0),
+    "openai/gpt-4o-mini": (0.15, 0.60),
+    "meta-llama/llama-3.1-70b-instruct": (0.52, 0.75),
+}
+
+
+@dataclass
+class ModelInfo:
+    step: str
+    provider: str  # local | api
+    backend: str  # ollama | gemini | deepseek | openrouter
+    model: str
+    display: str
+
+
+@dataclass
+class LlmUsageTracker:
+    """Collects LLM call records for one agent run."""
+
+    calls: list[dict[str, Any]] = field(default_factory=list)
+
+    async def invoke(
+        self,
+        step: str,
+        model: ChatOpenAI,
+        info: ModelInfo,
+        messages: list[BaseMessage],
+    ) -> Any:
+        response = await model.ainvoke(messages)
+        usage = extract_usage(response, info)
+        usage["step"] = step
+        self.calls.append(usage)
+        return response
+
+    def summary(self) -> dict[str, Any]:
+        run_cost = round(sum(c.get("cost_usd", 0.0) for c in self.calls), 6)
+        return {
+            "llm_usage": self.calls,
+            "run_cost_usd": run_cost,
+            "total_input_tokens": sum(c.get("input_tokens", 0) for c in self.calls),
+            "total_output_tokens": sum(c.get("output_tokens", 0) for c in self.calls),
+        }
+
+
+def extract_usage(response: Any, info: ModelInfo) -> dict[str, Any]:
+    meta = getattr(response, "response_metadata", None) or {}
+    token_usage = meta.get("token_usage") or {}
+    if not token_usage:
+        usage_meta = getattr(response, "usage_metadata", None)
+        if usage_meta:
+            token_usage = {
+                "prompt_tokens": usage_meta.get("input_tokens", 0),
+                "completion_tokens": usage_meta.get("output_tokens", 0),
+                "total_tokens": usage_meta.get("total_tokens", 0),
+            }
+
+    input_tokens = int(token_usage.get("prompt_tokens") or token_usage.get("input_tokens") or 0)
+    output_tokens = int(
+        token_usage.get("completion_tokens") or token_usage.get("output_tokens") or 0
+    )
+    model_name = meta.get("model_name") or info.model
+    cost = estimate_cost(model_name, info.backend, input_tokens, output_tokens)
+
+    return {
+        "provider": info.provider,
+        "backend": info.backend,
+        "model": model_name,
+        "display": info.display,
+        "input_tokens": input_tokens,
+        "output_tokens": output_tokens,
+        "total_tokens": input_tokens + output_tokens,
+        "cost_usd": cost,
+    }
+
+
+def estimate_cost(model: str, backend: str, input_tokens: int, output_tokens: int) -> float:
+    if backend == "ollama":
+        return 0.0
+    from app.services.model_catalog import CATALOG, entry_for_model
+
+    key = model.removeprefix("openrouter/")
+    for e in CATALOG:
+        if e.backend == backend and (e.model == model or e.model == key):
+            in_price, out_price = e.input_per_m, e.output_per_m
+            return round((input_tokens * in_price + output_tokens * out_price) / 1_000_000, 6)
+    entry = entry_for_model(backend, model, "economy")
+    in_price, out_price = entry.input_per_m, entry.output_per_m
+    return round((input_tokens * in_price + output_tokens * out_price) / 1_000_000, 6)
+
+
+def merge_run_history(
+    prior_runs: list[dict],
+    run_number: int,
+    run_report: dict,
+    llm_summary: dict,
+) -> dict:
+    """Build cumulative report with per-run and total cost."""
+    run_entry = {
+        "run_number": run_number,
+        "follow_up": run_report.get("follow_up"),
+        "summary": run_report.get("summary"),
+        "resolved": run_report.get("resolved"),
+        "devices_checked": run_report.get("devices_checked"),
+        "llm_usage": llm_summary.get("llm_usage", []),
+        "run_cost_usd": llm_summary.get("run_cost_usd", 0.0),
+        "generated_at": run_report.get("generated_at"),
+    }
+    all_runs = prior_runs + [run_entry]
+    total_cost = round(sum(r.get("run_cost_usd", 0.0) for r in all_runs), 6)
+
+    merged = {**run_report}
+    merged["run_number"] = run_number
+    merged["runs"] = all_runs
+    merged["llm_usage"] = llm_summary.get("llm_usage", [])
+    merged["run_cost_usd"] = llm_summary.get("run_cost_usd", 0.0)
+    merged["total_cost_usd"] = total_cost
+    merged["total_input_tokens"] = sum(
+        t.get("input_tokens", 0) for r in all_runs for t in r.get("llm_usage", [])
+    )
+    merged["total_output_tokens"] = sum(
+        t.get("output_tokens", 0) for r in all_runs for t in r.get("llm_usage", [])
+    )
+    return merged

backend/app/services/mcp_workspace.py

@@ -0,0 +1,225 @@
+"""Read/write MCP server source trees and push changes to Gitea."""
+from __future__ import annotations
+
+import logging
+import os
+from pathlib import Path
+
+from app.config import settings
+from app.mcp_manager.manager import get_manager
+
+logger = logging.getLogger(__name__)
+
+GIT_USER_NAME = "Agentic OS"
+GIT_USER_EMAIL = "agentic-os@paraskeva.net"
+
+
+def _local_src_dir(server: str) -> str:
+    return os.path.join(os.environ.get("MCP_LOCAL_DIR", "/app/mcp-servers"), server)
+
+
+def is_local_shipped(server: str) -> bool:
+    return os.path.isdir(_local_src_dir(server))
+
+
+def workspace_path(server: str) -> str:
+    """Writable directory where the running MCP code lives."""
+    manager = get_manager()
+    return manager._repo_dir(server)
+
+
+def _is_git_repo(path: str) -> bool:
+    return os.path.isdir(os.path.join(path, ".git"))
+
+
+async def _run(cmd: list[str], cwd: str) -> tuple[int, str]:
+    import asyncio
+
+    proc = await asyncio.create_subprocess_exec(
+        *cmd,
+        cwd=cwd,
+        stdout=asyncio.subprocess.PIPE,
+        stderr=asyncio.subprocess.STDOUT,
+    )
+    out, _ = await proc.communicate()
+    return proc.returncode or 0, out.decode(errors="replace")
+
+
+async def ensure_git_ready(server: str) -> tuple[str, bool]:
+    """Return (repo_path, can_push). Clone from Gitea if local-only without git."""
+    path = workspace_path(server)
+    if _is_git_repo(path):
+        await _ensure_remote(server, path)
+        return path, True
+
+    if not settings.gitea_token:
+        return path, False
+
+    clone_url = settings.gitea_clone_url(server)
+    git_dir = os.path.join(settings.runtime_dir, "mcp-repos", server)
+    if _is_git_repo(git_dir):
+        await _ensure_remote(server, git_dir)
+        return git_dir, True
+
+    os.makedirs(os.path.dirname(git_dir), exist_ok=True)
+    code, out = await _run(["git", "clone", "--depth", "1", clone_url, git_dir])
+    if code != 0:
+        logger.warning("MCP dev clone %s failed: %s", server, out[:300])
+        return path, False
+    return git_dir, True
+
+
+async def _ensure_remote(server: str, path: str) -> None:
+    from app.mcp_manager.manager import ensure_git_remote
+
+    await ensure_git_remote(server, path)
+
+
+async def list_source_files(server: str, *, max_files: int = 40) -> list[str]:
+    root = Path(workspace_path(server))
+    if not root.is_dir():
+        return []
+    out: list[str] = []
+    for p in sorted(root.rglob("*")):
+        if not p.is_file():
+            continue
+        rel = p.relative_to(root).as_posix()
+        if any(part in rel for part in (".venv", "__pycache__", ".git", "node_modules")):
+            continue
+        if p.suffix not in {".py", ".toml", ".md", ".json", ".yaml", ".yml"}:
+            continue
+        out.append(rel)
+        if len(out) >= max_files:
+            break
+    return out
+
+
+async def read_source_file(server: str, rel_path: str, *, max_chars: int = 80_000) -> str:
+    root = Path(workspace_path(server)).resolve()
+    target = (root / rel_path).resolve()
+    if not str(target).startswith(str(root)):
+        raise ValueError("Path escapes MCP workspace")
+    if not target.is_file():
+        raise FileNotFoundError(rel_path)
+    text = target.read_text(encoding="utf-8")
+    return text[:max_chars]
+
+
+async def write_source_file(server: str, rel_path: str, content: str) -> None:
+    root = Path(workspace_path(server)).resolve()
+    target = (root / rel_path).resolve()
+    if not str(target).startswith(str(root)):
+        raise ValueError("Path escapes MCP workspace")
+    target.parent.mkdir(parents=True, exist_ok=True)
+    target.write_text(content, encoding="utf-8")
+
+
+async def apply_file_edits(server: str, edits: list[dict]) -> list[str]:
+    written: list[str] = []
+    for edit in edits:
+        path = edit.get("path") or edit.get("file")
+        content = edit.get("content")
+        if not path or content is None:
+            continue
+        await write_source_file(server, path, content)
+        written.append(path)
+    return written
+
+
+async def commit_and_push(
+    server: str,
+    message: str,
+    files: list[str],
+    *,
+    use_git_clone: bool = True,
+) -> dict:
+    """Commit listed paths and push to origin. Returns {ok, commit, error}."""
+    if not files:
+        return {"ok": False, "error": "No files listed for git push"}
+
+    path, can_push = await ensure_git_ready(server)
+    if not can_push:
+        return {
+            "ok": False,
+            "error": "No Gitea git repo for this MCP — changes applied at runtime only",
+            "runtime_only": True,
+        }
+
+    # Sync runtime workspace files into git checkout when they differ
+    runtime = workspace_path(server)
+    if use_git_clone and os.path.abspath(runtime) != os.path.abspath(path):
+        for rel in files:
+            src = os.path.join(runtime, rel)
+            dst = os.path.join(path, rel)
+            if os.path.isfile(src):
+                os.makedirs(os.path.dirname(dst), exist_ok=True)
+                with open(src, encoding="utf-8") as fh:
+                    data = fh.read()
+                with open(dst, "w", encoding="utf-8") as fh:
+                    fh.write(data)
+
+    missing = [rel for rel in files if not os.path.isfile(os.path.join(path, rel))]
+    if missing:
+        return {
+            "ok": False,
+            "error": f"File(s) not found in MCP workspace: {', '.join(missing)}",
+        }
+
+    _, diff_out = await _run(["git", "diff", "HEAD", "--"] + files, cwd=path)
+    _, cached_out = await _run(["git", "diff", "--cached", "HEAD", "--"] + files, cwd=path)
+    if not diff_out.strip() and not cached_out.strip():
+        _, status_sb = await _run(["git", "status", "-sb"], cwd=path)
+        if "ahead" in status_sb:
+            code, out = await _run(["git", "push"], cwd=path)
+            if code != 0:
+                return {"ok": False, "error": f"push failed: {out[:500]}"}
+            commit_hash = (await _run(["git", "rev-parse", "--short", "HEAD"], cwd=path))[1].strip()
+            return {"ok": True, "commit": commit_hash, "repo": server, "note": "pushed existing commits"}
+        return {
+            "ok": True,
+            "commit": None,
+            "note": "no changes to push — files already match the last commit",
+            "already_up_to_date": True,
+        }
+
+    for rel in files:
+        await _run(["git", "add", rel], cwd=path)
+
+    code, out = await _run(
+        [
+            "git",
+            "-c",
+            f"user.email={GIT_USER_EMAIL}",
+            "-c",
+            f"user.name={GIT_USER_NAME}",
+            "commit",
+            "-m",
+            message,
+        ],
+        cwd=path,
+    )
+    if code != 0:
+        lowered = out.lower()
+        if "nothing to commit" in lowered or "nothing added to commit" in lowered:
+            return {
+                "ok": True,
+                "commit": None,
+                "note": "no changes to commit — already up to date",
+                "already_up_to_date": True,
+            }
+        return {"ok": False, "error": out[:500]}
+
+    code, out = await _run(["git", "push"], cwd=path)
+    if code != 0:
+        return {"ok": False, "error": f"push failed: {out[:500]}"}
+
+    commit_hash = (await _run(["git", "rev-parse", "--short", "HEAD"], cwd=path))[1].strip()
+    return {"ok": True, "commit": commit_hash, "repo": server}
+
+
+def list_registered_tools(server: str) -> list[str]:
+    manager = get_manager()
+    state = manager.servers.get(server)
+    if not state:
+        return []
+    return [t["name"] for t in state.tools]

backend/app/services/model_catalog.py

@@ -0,0 +1,106 @@
+"""Curated LLM catalog: tiers, backends, and pricing hints."""
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class ModelEntry:
+    id: str
+    backend: str  # ollama | gemini | deepseek | openrouter
+    model: str
+    tier: str  # local | economy | premium
+    label: str
+    input_per_m: float  # USD per 1M input tokens
+    output_per_m: float
+
+    @property
+    def provider(self) -> str:
+        return "local" if self.backend == "ollama" else "api"
+
+    @property
+    def display(self) -> str:
+        if self.backend == "ollama":
+            return f"Local · Ollama · {self.model}"
+        if self.backend == "gemini":
+            return f"API · Gemini · {self.model}"
+        if self.backend == "deepseek":
+            return f"API · DeepSeek · {self.model}"
+        return f"API · OpenRouter · {self.model}"
+
+
+# Curated models users can pick from the UI / config.
+CATALOG: list[ModelEntry] = [
+    # Local (Ollama)
+    ModelEntry("local-qwen25-7b", "ollama", "qwen2.5:7b-instruct", "local", "Qwen 2.5 7B Instruct", 0, 0),
+    ModelEntry("local-qwen25-coder-14b", "ollama", "qwen2.5-coder:14b", "local", "Qwen 2.5 Coder 14B", 0, 0),
+    ModelEntry("local-llama32-3b", "ollama", "llama3.2:3b", "local", "Llama 3.2 3B", 0, 0),
+    # Gemini economy (Google AI Studio — free tier eligible)
+    ModelEntry("gemini-flash-25", "gemini", "gemini-2.5-flash", "economy", "Gemini 2.5 Flash", 0.0, 0.0),
+    ModelEntry("gemini-flash-lite-25", "gemini", "gemini-2.5-flash-lite", "economy", "Gemini 2.5 Flash-Lite", 0.0, 0.0),
+    ModelEntry("gemini-flash-20", "gemini", "gemini-2.0-flash", "economy", "Gemini 2.0 Flash", 0.0, 0.0),
+    # Gemini premium
+    ModelEntry("gemini-pro-25", "gemini", "gemini-2.5-pro", "premium", "Gemini 2.5 Pro", 0.0, 0.0),
+    # DeepSeek economy
+    ModelEntry("ds-chat", "deepseek", "deepseek-chat", "economy", "DeepSeek Chat", 0.14, 0.28),
+    # DeepSeek premium
+    ModelEntry("ds-reasoner", "deepseek", "deepseek-reasoner", "premium", "DeepSeek Reasoner", 0.55, 2.19),
+    # OpenRouter economy
+    ModelEntry("or-ds-chat", "openrouter", "deepseek/deepseek-chat", "economy", "DeepSeek Chat (OR)", 0.14, 0.28),
+    ModelEntry("or-gemini-flash", "openrouter", "google/gemini-2.5-flash-preview", "economy", "Gemini 2.5 Flash", 0.15, 0.60),
+    ModelEntry("or-gpt4o-mini", "openrouter", "openai/gpt-4o-mini", "economy", "GPT-4o Mini", 0.15, 0.60),
+    ModelEntry("or-qwen25-72b", "openrouter", "qwen/qwen-2.5-72b-instruct", "economy", "Qwen 2.5 72B", 0.35, 0.40),
+    # OpenRouter premium
+    ModelEntry("or-claude-35-sonnet", "openrouter", "anthropic/claude-3.5-sonnet", "premium", "Claude 3.5 Sonnet", 3.0, 15.0),
+    ModelEntry("or-claude-37-sonnet", "openrouter", "anthropic/claude-3.7-sonnet", "premium", "Claude 3.7 Sonnet", 3.0, 15.0),
+    ModelEntry("or-gpt4o", "openrouter", "openai/gpt-4o", "premium", "GPT-4o", 2.5, 10.0),
+]
+
+_BY_ID: dict[str, ModelEntry] = {m.id: m for m in CATALOG}
+
+
+def get_entry(entry_id: str) -> ModelEntry | None:
+    return _BY_ID.get(entry_id)
+
+
+def entry_for_model(backend: str, model: str, tier: str = "economy") -> ModelEntry:
+    """Find catalog entry or synthesize one for custom model IDs."""
+    for e in CATALOG:
+        if e.backend == backend and e.model == model:
+            return e
+    label = model.split("/")[-1] if "/" in model else model
+    pricing = (0.50, 1.50) if tier != "local" else (0.0, 0.0)
+    return ModelEntry(
+        id=f"custom-{backend}-{model.replace('/', '-')}",
+        backend=backend,
+        model=model,
+        tier=tier,
+        label=label,
+        input_per_m=pricing[0],
+        output_per_m=pricing[1],
+    )
+
+
+def catalog_by_tier() -> dict[str, dict[str, list[dict]]]:
+    """Group catalog for API/UI: tier -> backend -> entries."""
+    out: dict[str, dict[str, list[dict]]] = {
+        "local": {"ollama": []},
+        "economy": {"gemini": [], "deepseek": [], "openrouter": []},
+        "premium": {"gemini": [], "deepseek": [], "openrouter": []},
+    }
+    for e in CATALOG:
+        out[e.tier].setdefault(e.backend, []).append(_entry_dict(e))
+    return out
+
+
+def _entry_dict(e: ModelEntry) -> dict:
+    return {
+        "id": e.id,
+        "backend": e.backend,
+        "model": e.model,
+        "tier": e.tier,
+        "label": e.label,
+        "input_per_m": e.input_per_m,
+        "output_per_m": e.output_per_m,
+        "display": e.display,
+    }

backend/app/services/model_config.py

@@ -0,0 +1,181 @@
+"""Runtime LLM routing configuration (env defaults + Redis overrides)."""
+from __future__ import annotations
+
+import json
+import time
+from dataclasses import asdict, dataclass, field
+
+from app.config import settings
+from app.services.model_catalog import ModelEntry, entry_for_model, get_entry
+
+ROUTING_KEY = "agentic:llm:routing"
+
+# Short in-process cache: routing config is read many times per task but
+# changes only via the admin /api/llm/routing endpoint.
+_CONFIG_CACHE_TTL = 30.0
+_config_cache: tuple[float, "LlmRoutingConfig"] | None = None
+
+DEFAULT_CLOUD_PROVIDER_ORDER = ["gemini", "deepseek", "openrouter"]
+
+
+@dataclass
+class LlmRoutingConfig:
+    """Selected models per tier. Always tries local first, then escalates."""
+
+    local_model_id: str = "local-qwen25-7b"
+    economy_gemini_id: str = "gemini-flash-25"
+    economy_deepseek_id: str = "ds-chat"
+    economy_openrouter_id: str = "or-ds-chat"
+    premium_gemini_id: str = "gemini-pro-25"
+    premium_deepseek_id: str = "ds-reasoner"
+    premium_openrouter_id: str = "or-claude-35-sonnet"
+    # Cloud provider cascade within each tier: gemini → deepseek → openrouter
+    cloud_provider_order: list[str] = field(default_factory=lambda: list(DEFAULT_CLOUD_PROVIDER_ORDER))
+    auto_escalate: bool = True
+    # Cap escalation: local | economy | premium
+    max_tier: str = "premium"
+
+    def to_dict(self) -> dict:
+        return asdict(self)
+
+    def resolve(self, entry_id: str) -> ModelEntry | None:
+        entry = get_entry(entry_id)
+        if entry:
+            return entry
+        return None
+
+
+def _defaults_from_env() -> LlmRoutingConfig:
+    """Build defaults using env vars and catalog IDs where possible."""
+    local = settings.llm_local_model
+    cfg = LlmRoutingConfig(
+        local_model_id=_find_id("ollama", local, "local") or "local-qwen25-7b",
+        economy_gemini_id=_find_id("gemini", settings.llm_economy_gemini, "economy") or "gemini-flash-25",
+        economy_deepseek_id=_find_id("deepseek", settings.llm_economy_deepseek, "economy") or "ds-chat",
+        economy_openrouter_id=_find_id("openrouter", settings.llm_economy_openrouter, "economy") or "or-ds-chat",
+        premium_gemini_id=_find_id("gemini", settings.llm_premium_gemini, "premium") or "gemini-pro-25",
+        premium_deepseek_id=_find_id("deepseek", settings.llm_premium_deepseek, "premium") or "ds-reasoner",
+        premium_openrouter_id=_find_id("openrouter", settings.llm_premium_openrouter, "premium")
+        or "or-claude-35-sonnet",
+        auto_escalate=settings.llm_auto_escalate,
+        max_tier=settings.llm_max_tier,
+    )
+    if settings.llm_cloud_provider_order:
+        cfg.cloud_provider_order = [
+            p.strip() for p in settings.llm_cloud_provider_order.split(",") if p.strip()
+        ]
+    return cfg
+
+
+def _find_id(backend: str, model: str, tier: str) -> str | None:
+    from app.services.model_catalog import CATALOG
+
+    for e in CATALOG:
+        if e.backend == backend and e.model == model and e.tier == tier:
+            return e.id
+    return None
+
+
+async def get_routing_config(*, use_cache: bool = True) -> LlmRoutingConfig:
+    from app.services.events import get_redis
+
+    global _config_cache
+    if use_cache and _config_cache is not None:
+        ts, cached = _config_cache
+        if time.monotonic() - ts < _CONFIG_CACHE_TTL:
+            return cached
+
+    cfg = _defaults_from_env()
+    try:
+        raw = await get_redis().get(ROUTING_KEY)
+        if raw:
+            data = json.loads(raw)
+            for k, v in data.items():
+                if hasattr(cfg, k):
+                    setattr(cfg, k, v)
+    except Exception:  # noqa: BLE001
+        pass
+
+    _config_cache = (time.monotonic(), cfg)
+    return cfg
+
+
+async def save_routing_config(cfg: LlmRoutingConfig) -> None:
+    from app.services.events import get_redis
+
+    global _config_cache
+    await get_redis().set(ROUTING_KEY, json.dumps(cfg.to_dict()))
+    _config_cache = (time.monotonic(), cfg)
+
+
+def available_backends() -> dict[str, bool]:
+    return {
+        "ollama": True,
+        "gemini": bool(settings.gemini_api_key),
+        "deepseek": bool(settings.deepseek_api_key),
+        "openrouter": bool(settings.openrouter_api_key),
+    }
+
+
+def _provider_entry(cfg: LlmRoutingConfig, tier: str, provider: str) -> ModelEntry | None:
+    avail = available_backends()
+    if tier == "economy":
+        if provider == "gemini" and avail["gemini"]:
+            return cfg.resolve(cfg.economy_gemini_id) or entry_for_model(
+                "gemini", settings.llm_economy_gemini, "economy"
+            )
+        if provider == "deepseek" and avail["deepseek"]:
+            return cfg.resolve(cfg.economy_deepseek_id) or entry_for_model(
+                "deepseek", settings.llm_economy_deepseek, "economy"
+            )
+        if provider == "openrouter" and avail["openrouter"]:
+            return cfg.resolve(cfg.economy_openrouter_id) or entry_for_model(
+                "openrouter", settings.llm_economy_openrouter, "economy"
+            )
+    elif tier == "premium":
+        if provider == "gemini" and avail["gemini"]:
+            return cfg.resolve(cfg.premium_gemini_id) or entry_for_model(
+                "gemini", settings.llm_premium_gemini, "premium"
+            )
+        if provider == "deepseek" and avail["deepseek"]:
+            return cfg.resolve(cfg.premium_deepseek_id) or entry_for_model(
+                "deepseek", settings.llm_premium_deepseek, "premium"
+            )
+        if provider == "openrouter" and avail["openrouter"]:
+            return cfg.resolve(cfg.premium_openrouter_id) or entry_for_model(
+                "openrouter", settings.llm_premium_openrouter, "premium"
+            )
+    return None
+
+
+def resolve_tier_models(cfg: LlmRoutingConfig, tier: str) -> list[ModelEntry]:
+    """Return ordered models for a tier (local = one entry; cloud = provider cascade)."""
+    if tier == "local":
+        if not available_backends()["ollama"]:
+            return []
+        e = cfg.resolve(cfg.local_model_id) or entry_for_model("ollama", settings.llm_local_model, "local")
+        return [e]
+
+    models: list[ModelEntry] = []
+    seen: set[str] = set()
+    for provider in cfg.cloud_provider_order:
+        entry = _provider_entry(cfg, tier, provider)
+        if entry and entry.id not in seen:
+            models.append(entry)
+            seen.add(entry.id)
+
+    # Fallback: any configured cloud provider with a key
+    for provider in DEFAULT_CLOUD_PROVIDER_ORDER:
+        if provider in cfg.cloud_provider_order:
+            continue
+        entry = _provider_entry(cfg, tier, provider)
+        if entry and entry.id not in seen:
+            models.append(entry)
+            seen.add(entry.id)
+    return models
+
+
+def resolve_tier_model(cfg: LlmRoutingConfig, tier: str) -> ModelEntry | None:
+    """Pick the first available model for a tier (legacy single-model callers)."""
+    models = resolve_tier_models(cfg, tier)
+    return models[0] if models else None

backend/app/services/model_discovery.py

@@ -0,0 +1,182 @@
+"""Live model discovery from Ollama, Gemini, DeepSeek, and OpenRouter APIs."""
+from __future__ import annotations
+
+import asyncio
+import logging
+from typing import Any
+
+import httpx
+
+from app.config import settings
+
+logger = logging.getLogger(__name__)
+
+# DeepSeek catalog fallbacks when /models is unavailable or incomplete.
+KNOWN_DEEPSEEK_MODELS = [
+    {"id": "deepseek-chat", "label": "DeepSeek Chat", "tier_hint": "economy"},
+    {"id": "deepseek-reasoner", "label": "DeepSeek Reasoner", "tier_hint": "premium"},
+]
+
+
+async def list_ollama_models() -> dict[str, Any]:
+    base = settings.ollama_base_url.rstrip("/")
+    try:
+        async with httpx.AsyncClient(timeout=10.0) as client:
+            resp = await client.get(f"{base}/api/tags")
+            resp.raise_for_status()
+            payload = resp.json()
+    except Exception as exc:  # noqa: BLE001
+        logger.debug("Ollama model list failed: %s", exc)
+        return {"backend": "ollama", "available": False, "error": str(exc), "models": []}
+
+    models = []
+    for item in payload.get("models") or []:
+        name = item.get("name") or item.get("model") or ""
+        details = item.get("details") or {}
+        caps = item.get("capabilities") or []
+        models.append(
+            {
+                "id": name,
+                "label": name,
+                "size_bytes": item.get("size"),
+                "parameter_size": details.get("parameter_size"),
+                "quantization": details.get("quantization_level"),
+                "context_length": details.get("context_length"),
+                "capabilities": caps,
+                "supports_tools": "tools" in caps,
+            }
+        )
+    return {"backend": "ollama", "available": True, "models": models}
+
+
+async def list_gemini_models() -> dict[str, Any]:
+    if not settings.gemini_api_key:
+        return {
+            "backend": "gemini",
+            "available": False,
+            "error": "GEMINI_API_KEY not set",
+            "models": [],
+        }
+    url = "https://generativelanguage.googleapis.com/v1beta/models"
+    try:
+        async with httpx.AsyncClient(timeout=15.0) as client:
+            resp = await client.get(url, params={"key": settings.gemini_api_key})
+            resp.raise_for_status()
+            payload = resp.json()
+    except Exception as exc:  # noqa: BLE001
+        logger.debug("Gemini model list failed: %s", exc)
+        return {"backend": "gemini", "available": False, "error": str(exc), "models": []}
+
+    models = []
+    for item in payload.get("models") or []:
+        methods = item.get("supportedGenerationMethods") or []
+        if "generateContent" not in methods:
+            continue
+        raw_name = item.get("name") or ""
+        model_id = raw_name.removeprefix("models/")
+        display = item.get("displayName") or model_id
+        models.append(
+            {
+                "id": model_id,
+                "label": display,
+                "input_token_limit": item.get("inputTokenLimit"),
+                "output_token_limit": item.get("outputTokenLimit"),
+                "description": item.get("description"),
+            }
+        )
+    models.sort(key=lambda m: m["id"])
+    return {"backend": "gemini", "available": True, "models": models}
+
+
+async def list_deepseek_models() -> dict[str, Any]:
+    if not settings.deepseek_api_key:
+        return {
+            "backend": "deepseek",
+            "available": False,
+            "error": "DEEPSEEK_API_KEY not set",
+            "models": [],
+        }
+    try:
+        async with httpx.AsyncClient(timeout=15.0) as client:
+            resp = await client.get(
+                "https://api.deepseek.com/models",
+                headers={"Authorization": f"Bearer {settings.deepseek_api_key}"},
+            )
+            resp.raise_for_status()
+            payload = resp.json()
+        models = [
+            {
+                "id": item.get("id") or "",
+                "label": item.get("id") or "",
+                "owned_by": item.get("owned_by"),
+            }
+            for item in payload.get("data") or []
+            if item.get("id")
+        ]
+        if not models:
+            models = [dict(m) for m in KNOWN_DEEPSEEK_MODELS]
+        return {"backend": "deepseek", "available": True, "models": models}
+    except Exception as exc:  # noqa: BLE001
+        logger.debug("DeepSeek model list failed: %s", exc)
+        return {
+            "backend": "deepseek",
+            "available": True,
+            "error": str(exc),
+            "models": [dict(m) for m in KNOWN_DEEPSEEK_MODELS],
+            "note": "Using known DeepSeek model IDs (API list unavailable)",
+        }
+
+
+async def list_openrouter_models() -> dict[str, Any]:
+    if not settings.openrouter_api_key:
+        return {
+            "backend": "openrouter",
+            "available": False,
+            "error": "OPENROUTER_API_KEY not set",
+            "models": [],
+        }
+    try:
+        async with httpx.AsyncClient(timeout=20.0) as client:
+            resp = await client.get(
+                "https://openrouter.ai/api/v1/models",
+                headers={"Authorization": f"Bearer {settings.openrouter_api_key}"},
+            )
+            resp.raise_for_status()
+            payload = resp.json()
+    except Exception as exc:  # noqa: BLE001
+        logger.debug("OpenRouter model list failed: %s", exc)
+        return {"backend": "openrouter", "available": False, "error": str(exc), "models": []}
+
+    models = []
+    for item in payload.get("data") or []:
+        pricing = item.get("pricing") or {}
+        models.append(
+            {
+                "id": item.get("id") or "",
+                "label": item.get("name") or item.get("id") or "",
+                "context_length": item.get("context_length"),
+                "input_per_token": pricing.get("prompt"),
+                "output_per_token": pricing.get("completion"),
+                "description": item.get("description"),
+            }
+        )
+    models.sort(key=lambda m: m["id"])
+    return {"backend": "openrouter", "available": True, "models": models}
+
+
+async def list_all_provider_models() -> dict[str, Any]:
+    ollama, gemini, deepseek, openrouter = await asyncio.gather(
+        list_ollama_models(),
+        list_gemini_models(),
+        list_deepseek_models(),
+        list_openrouter_models(),
+    )
+    return {
+        "providers": {
+            "ollama": ollama,
+            "gemini": gemini,
+            "deepseek": deepseek,
+            "openrouter": openrouter,
+        },
+        "routing_order": settings.llm_cloud_provider_order,
+    }

backend/app/services/model_router.py

@@ -0,0 +1,268 @@
+"""Tiered model routing: local first, auto-escalate to economy/premium when needed."""
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+from langchain_core.messages import BaseMessage
+from langchain_openai import ChatOpenAI
+
+from app.config import settings
+from app.services.llm_usage import LlmUsageTracker, ModelInfo
+from app.services.model_catalog import ModelEntry
+from app.services.model_config import LlmRoutingConfig, get_routing_config, resolve_tier_models
+
+logger = logging.getLogger(__name__)
+
+TIER_ORDER = ["local", "economy", "premium"]
+TIER_RANK = {t: i for i, t in enumerate(TIER_ORDER)}
+
+GEMINI_OPENAI_BASE = "https://generativelanguage.googleapis.com/v1beta/openai/"
+
+
+def build_chat_model(entry: ModelEntry, temperature: float = 0.1) -> ChatOpenAI:
+    if entry.backend == "ollama":
+        return ChatOpenAI(
+            model=entry.model,
+            temperature=temperature,
+            base_url=f"{settings.ollama_base_url.rstrip('/')}/v1",
+            api_key="ollama",
+            timeout=120,
+            max_retries=2,
+        )
+    if entry.backend == "gemini":
+        return ChatOpenAI(
+            model=entry.model,
+            temperature=temperature,
+            base_url=GEMINI_OPENAI_BASE,
+            api_key=settings.gemini_api_key,
+            timeout=120,
+            max_retries=2,
+        )
+    if entry.backend == "deepseek":
+        return ChatOpenAI(
+            model=entry.model,
+            temperature=temperature,
+            base_url="https://api.deepseek.com/v1",
+            api_key=settings.deepseek_api_key,
+            timeout=120,
+            max_retries=2,
+        )
+    return ChatOpenAI(
+        model=entry.model,
+        temperature=temperature,
+        base_url="https://openrouter.ai/api/v1",
+        api_key=settings.openrouter_api_key,
+        timeout=120,
+        max_retries=2,
+        default_headers={"HTTP-Referer": "http://localhost:3000", "X-Title": "Agentic OS"},
+    )
+
+
+def entry_to_info(entry: ModelEntry, step: str) -> ModelInfo:
+    return ModelInfo(
+        step=step,
+        provider=entry.provider,
+        backend=entry.backend,
+        model=entry.model,
+        display=entry.display,
+    )
+
+
+def plan_reasoning_tiers(
+    triage: dict,
+    diagnostics: list[dict],
+    *,
+    prior_context: dict | None = None,
+    follow_up: str | None = None,
+    run_number: int = 1,
+    cfg: LlmRoutingConfig | None = None,
+    min_tier: str | None = None,
+) -> tuple[list[str], str]:
+    """Return ordered tiers to try and a human-readable routing rationale."""
+    cfg = cfg or LlmRoutingConfig()
+    severity = (triage.get("severity") or "medium").lower()
+    recommended = (triage.get("recommended_tier") or "local").lower()
+    needs_cloud = bool(triage.get("needs_cloud_reasoning"))
+    fail_count = sum(1 for d in diagnostics if not d.get("ok"))
+    diag_count = len(diagnostics)
+
+    max_tier = "local"
+    reasons: list[str] = ["Always start with local LLM"]
+
+    if needs_cloud:
+        max_tier = "economy"
+        reasons.append("Triage flagged cloud reasoning helpful")
+    if severity in ("high", "critical"):
+        max_tier = "premium" if severity == "critical" else max([max_tier, "economy"], key=lambda t: TIER_RANK[t])
+        reasons.append(f"Severity is {severity}")
+    if recommended == "premium":
+        max_tier = "premium"
+        reasons.append("Triage recommended premium tier")
+    elif recommended == "economy" and TIER_RANK[max_tier] < TIER_RANK["economy"]:
+        max_tier = "economy"
+        reasons.append("Triage recommended economy tier")
+    if fail_count >= 2 or (fail_count >= 1 and diag_count >= 4 and fail_count / max(diag_count, 1) > 0.25):
+        max_tier = max([max_tier, "economy"], key=lambda t: TIER_RANK[t])
+        reasons.append(f"{fail_count} diagnostic failure(s)")
+    if fail_count >= 4 or (severity == "critical" and fail_count >= 1):
+        max_tier = "premium"
+        reasons.append("Complex failure pattern — premium model")
+    if prior_context and not prior_context.get("resolved"):
+        max_tier = max([max_tier, "economy"], key=lambda t: TIER_RANK[t])
+        reasons.append("Prior run unresolved")
+    if follow_up:
+        max_tier = max([max_tier, "economy"], key=lambda t: TIER_RANK[t])
+        reasons.append("Follow-up investigation")
+    if run_number > 1:
+        max_tier = max([max_tier, "economy"], key=lambda t: TIER_RANK[t])
+        reasons.append(f"Run {run_number} continuation")
+
+    # Respect configured cap
+    cap = cfg.max_tier if cfg.max_tier in TIER_RANK else "premium"
+    if TIER_RANK[max_tier] > TIER_RANK[cap]:
+        max_tier = cap
+        reasons.append(f"Capped at {cap} tier")
+
+    max_idx = TIER_RANK[max_tier]
+    tiers = TIER_ORDER[: max_idx + 1]
+
+    if not cfg.auto_escalate and len(tiers) > 1:
+        tiers = ["local"]
+        reasons.append("Auto-escalation disabled — local only")
+
+    if min_tier and min_tier in TIER_RANK:
+        from app.services.task_runtime import apply_min_tier_floor
+
+        tiers = apply_min_tier_floor(tiers, min_tier)
+        reasons.append(f"User requested {min_tier} API (skip local)")
+
+    return tiers, "; ".join(reasons)
+
+
+def _should_escalate(
+    analysis: dict,
+    tier: str,
+    tiers: list[str],
+    *,
+    diag_ok_ratio: float = 0.0,
+    is_last_provider: bool = False,
+) -> bool:
+    if is_last_provider and tier == tiers[-1]:
+        return False
+    if analysis.get("error"):
+        return True
+    confidence = analysis.get("confidence")
+    if analysis.get("needs_escalation"):
+        return True
+    if confidence is not None and float(confidence) < 0.65 and diag_ok_ratio < 0.75:
+        return True
+    if not analysis.get("summary") and not analysis.get("root_cause"):
+        return True
+    return False
+
+
+async def reason_with_cascade(
+    tracker: LlmUsageTracker,
+    tiers: list[str],
+    messages: list[BaseMessage],
+    parse_json,
+    cfg: LlmRoutingConfig | None = None,
+    *,
+    diagnostics: list[dict] | None = None,
+    task_id: int | None = None,
+) -> tuple[dict, dict]:
+    """Try tiers in order; within cloud tiers cascade gemini → deepseek → openrouter."""
+    cfg = cfg or await get_routing_config()
+    analysis: dict[str, Any] = {}
+    used_tiers: list[str] = []
+    used_backends: list[str] = []
+    last_entry: ModelEntry | None = None
+    last_error: str | None = None
+    diag_ok_ratio = 0.0
+    if diagnostics:
+        diag_ok_ratio = sum(1 for d in diagnostics if d.get("ok")) / len(diagnostics)
+
+    from app.services.task_runtime import get_task_min_tier
+
+    finished = False
+    for tier in tiers:
+        min_tier = await get_task_min_tier(task_id) if task_id else None
+        if min_tier and TIER_RANK[tier] < TIER_RANK[min_tier]:
+            continue
+
+        entries = resolve_tier_models(cfg, tier)
+        if not entries:
+            logger.info("No models available for tier %s", tier)
+            continue
+
+        for idx, entry in enumerate(entries):
+            min_tier = await get_task_min_tier(task_id) if task_id else None
+            if min_tier and TIER_RANK[tier] < TIER_RANK[min_tier]:
+                break
+
+            last_entry = entry
+            is_last_provider = idx == len(entries) - 1
+            step_label = f"reason-{tier}-{entry.backend}"
+            try:
+                model = build_chat_model(entry, temperature=0.1)
+                info = entry_to_info(entry, step_label)
+                resp = await tracker.invoke(step_label, model, info, messages)
+                analysis = parse_json(resp.content) or {
+                    "summary": str(resp.content)[:1000],
+                    "resolved": False,
+                }
+                analysis["_tier_used"] = tier
+                analysis["_backend_used"] = entry.backend
+                analysis["_model"] = entry.display
+                used_tiers.append(tier)
+                used_backends.append(entry.backend)
+
+                if not cfg.auto_escalate or not _should_escalate(
+                    analysis,
+                    tier,
+                    tiers,
+                    diag_ok_ratio=diag_ok_ratio,
+                    is_last_provider=is_last_provider,
+                ):
+                    min_tier = await get_task_min_tier(task_id) if task_id else None
+                    if not (
+                        min_tier
+                        and TIER_RANK.get(min_tier, 0) > TIER_RANK.get(tier, 0)
+                    ):
+                        finished = True
+                        break
+                logger.info(
+                    "Escalating from %s/%s: confidence=%s needs_esc=%s",
+                    tier,
+                    entry.backend,
+                    analysis.get("confidence"),
+                    analysis.get("needs_escalation"),
+                )
+            except Exception as exc:  # noqa: BLE001
+                last_error = str(exc)
+                logger.warning("Reasoning %s/%s failed: %s", tier, entry.backend, exc)
+                analysis = {
+                    "summary": f"reasoning error ({tier}/{entry.backend}): {exc}",
+                    "resolved": False,
+                    "error": str(exc),
+                }
+                used_tiers.append(tier)
+                used_backends.append(entry.backend)
+                continue
+
+        if finished:
+            break
+
+    if last_error and not analysis.get("summary"):
+        analysis = {"summary": f"reasoning error: {last_error}", "resolved": False, "error": last_error}
+
+    routing_meta = {
+        "tiers_planned": tiers,
+        "tiers_used": used_tiers,
+        "backends_used": used_backends,
+        "final_tier": used_tiers[-1] if used_tiers else None,
+        "final_backend": last_entry.backend if last_entry else None,
+        "final_model": last_entry.display if last_entry else None,
+    }
+    return analysis, routing_meta

backend/app/services/task_present.py

@@ -0,0 +1,85 @@
+"""Serialize tasks with vessel names for API responses."""
+from __future__ import annotations
+
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.models.inventory import Vessel
+from app.models.task import Task
+from app.schemas import TaskDetail, TaskOut, TaskOverviewOut
+
+
+async def vessel_name_map(db: AsyncSession, vessel_ids: set[int]) -> dict[int, str]:
+    if not vessel_ids:
+        return {}
+    res = await db.execute(select(Vessel).where(Vessel.id.in_(vessel_ids)))
+    return {v.id: v.name for v in res.scalars().all()}
+
+
+def task_to_out(task: Task, *, vessel_name: str | None = None) -> TaskOut:
+    base = TaskOut.model_validate(task)
+    name = vessel_name
+    if name is None and task.report:
+        name = task.report.get("vessel_name")
+    return base.model_copy(update={"vessel_name": name})
+
+
+def task_to_detail(task: Task, *, vessel_name: str | None = None) -> TaskDetail:
+    base = TaskDetail.model_validate(task)
+    name = vessel_name
+    if name is None and task.report:
+        name = task.report.get("vessel_name")
+    return base.model_copy(update={"vessel_name": name})
+
+
+def _report_summary(report: dict | None) -> tuple[str | None, str | None]:
+    if not report:
+        return None, None
+    summary = report.get("executive_summary") or report.get("summary")
+    findings = report.get("findings") or []
+    key_finding = None
+    if isinstance(findings, list) and findings:
+        first = findings[0]
+        if isinstance(first, dict):
+            key_finding = first.get("summary") or first.get("description")
+        else:
+            key_finding = str(first)
+    if not key_finding:
+        key_finding = report.get("root_cause") or report.get("resolution")
+    return summary, key_finding
+
+
+def task_to_overview(task: Task, *, vessel_name: str | None = None) -> TaskOverviewOut:
+    report = task.report or {}
+    name = vessel_name or report.get("vessel_name")
+    summary, key_finding = _report_summary(report)
+    return TaskOverviewOut(
+        id=task.id,
+        title=task.title,
+        issue=task.issue,
+        status=task.status,
+        vessel_id=task.vessel_id,
+        vessel_name=name,
+        summary=summary,
+        key_finding=key_finding,
+        devices_checked=list(report.get("devices_checked") or []),
+        resolved=report.get("resolved"),
+        memory_id=task.memory_id or report.get("memory_id"),
+        obsidian_path=task.obsidian_path or report.get("obsidian_path"),
+        obsidian_push_error=report.get("obsidian_push_error"),
+        run_cost_usd=report.get("run_cost_usd"),
+        created_at=task.created_at,
+        updated_at=task.updated_at,
+    )
+
+
+async def tasks_to_out_list(db: AsyncSession, tasks: list[Task]) -> list[TaskOut]:
+    ids = {t.vessel_id for t in tasks if t.vessel_id}
+    vmap = await vessel_name_map(db, ids)
+    return [task_to_out(t, vessel_name=vmap.get(t.vessel_id) if t.vessel_id else None) for t in tasks]
+
+
+async def tasks_to_overview_list(db: AsyncSession, tasks: list[Task]) -> list[TaskOverviewOut]:
+    ids = {t.vessel_id for t in tasks if t.vessel_id}
+    vmap = await vessel_name_map(db, ids)
+    return [task_to_overview(t, vessel_name=vmap.get(t.vessel_id) if t.vessel_id else None) for t in tasks]

backend/app/services/task_runtime.py

@@ -0,0 +1,64 @@
+"""Per-task runtime flags (Redis) — e.g. user-requested LLM tier escalation and cancellation."""
+from __future__ import annotations
+
+from app.services.events import get_redis
+
+TIER_ORDER = ["local", "economy", "premium"]
+TIER_RANK = {t: i for i, t in enumerate(TIER_ORDER)}
+
+CANCELLABLE_STATUSES = frozenset({"queued", "running", "waiting_approval"})
+
+
+class TaskCancelledError(Exception):
+    """Raised when the user stops a running or queued task."""
+
+
+def _min_tier_key(task_id: int) -> str:
+    return f"agentic:task:{task_id}:min_tier"
+
+
+def _cancel_key(task_id: int) -> str:
+    return f"agentic:task:{task_id}:cancel"
+
+
+async def request_task_cancel(task_id: int) -> None:
+    await get_redis().set(_cancel_key(task_id), "1", ex=86400)
+
+
+async def is_task_cancel_requested(task_id: int) -> bool:
+    return bool(await get_redis().get(_cancel_key(task_id)))
+
+
+async def clear_task_cancel(task_id: int) -> None:
+    await get_redis().delete(_cancel_key(task_id))
+
+
+async def ensure_not_cancelled(task_id: int) -> None:
+    if await is_task_cancel_requested(task_id):
+        raise TaskCancelledError("Task cancelled by user")
+
+
+async def set_task_min_tier(task_id: int, tier: str) -> None:
+    if tier not in TIER_RANK:
+        raise ValueError(f"Invalid tier: {tier}")
+    await get_redis().set(_min_tier_key(task_id), tier, ex=86400)
+
+
+async def get_task_min_tier(task_id: int) -> str | None:
+    tier = await get_redis().get(_min_tier_key(task_id))
+    return tier if tier in TIER_RANK else None
+
+
+async def clear_task_min_tier(task_id: int) -> None:
+    await get_redis().delete(_min_tier_key(task_id))
+
+
+def apply_min_tier_floor(tiers: list[str], min_tier: str) -> list[str]:
+    """Drop tiers below min_tier (e.g. skip local when user requests economy)."""
+    if min_tier not in TIER_RANK:
+        return tiers
+    floor = TIER_RANK[min_tier]
+    filtered = [t for t in tiers if TIER_RANK[t] >= floor]
+    if filtered:
+        return filtered
+    return [min_tier]

backend/app/services/troubleshooting_rules.py

@@ -0,0 +1,297 @@
+"""Editable troubleshooting decision rules (YAML on disk).
+
+Rules standardize which devices to check and in what order, plus hints for the LLM.
+"""
+from __future__ import annotations
+
+import logging
+import os
+import re
+from dataclasses import dataclass
+
+import yaml
+
+from app.config import settings
+
+logger = logging.getLogger(__name__)
+
+_rules_cache: dict | None = None
+_rules_mtime: float = -1.0
+
+
+@dataclass
+class MatchedRule:
+    id: str
+    name: str
+    severity: str
+    devices: list[str]
+    order: list[str]
+    hints: list[str]
+    steps: list[str]
+    broad: bool = False
+
+    def as_dict(self) -> dict:
+        return {
+            "id": self.id,
+            "name": self.name,
+            "severity": self.severity,
+            "devices": self.devices,
+            "order": self.order,
+            "hints": self.hints,
+            "steps": self.steps,
+            "broad": self.broad,
+        }
+
+
+def rules_path() -> str:
+    return os.environ.get("TROUBLESHOOTING_RULES_PATH", settings.troubleshooting_rules_path)
+
+
+def _normalize(data: dict) -> dict:
+    data.setdefault("version", 1)
+    data.setdefault("broad_triggers", {"keywords": []})
+    data.setdefault("device_keywords", {})
+    data.setdefault("rules", [])
+    data.setdefault("default", {"severity": "medium", "devices": ["all"], "hints": []})
+    return data
+
+
+def invalidate_rules_cache() -> None:
+    global _rules_cache, _rules_mtime
+    _rules_cache = None
+    _rules_mtime = -1.0
+
+
+def load_rules(*, force: bool = False) -> dict:
+    global _rules_cache, _rules_mtime
+    path = rules_path()
+    try:
+        mtime = os.path.getmtime(path)
+    except OSError:
+        logger.warning("troubleshooting rules file missing: %s", path)
+        return _normalize({})
+
+    if not force and _rules_cache is not None and mtime == _rules_mtime:
+        return _rules_cache
+
+    try:
+        with open(path, encoding="utf-8") as fh:
+            data = _normalize(yaml.safe_load(fh) or {})
+    except yaml.YAMLError as exc:
+        logger.warning("invalid rules YAML %s: %s", path, exc)
+        data = _normalize({})
+
+    _rules_cache = data
+    _rules_mtime = mtime
+    return data
+
+
+def read_rules_raw() -> str:
+    path = rules_path()
+    try:
+        with open(path, encoding="utf-8") as fh:
+            return fh.read()
+    except OSError:
+        return ""
+
+
+def save_rules_raw(content: str) -> dict:
+    parsed = yaml.safe_load(content)
+    if not isinstance(parsed, dict):
+        raise ValueError("Rules file must be a YAML mapping at the top level")
+    normalized = _normalize(parsed)
+    path = rules_path()
+    parent = os.path.dirname(path)
+    if parent:
+        os.makedirs(parent, exist_ok=True)
+    with open(path, "w", encoding="utf-8") as fh:
+        fh.write(content if content.endswith("\n") else content + "\n")
+    invalidate_rules_cache()
+    return load_rules(force=True)
+
+
+def _text_has_term(text: str, term: str) -> bool:
+    term = term.lower().strip()
+    if not term:
+        return False
+    if " " in term:
+        return term in text
+    if len(term) <= 2:
+        return term in text.split()
+    return bool(re.search(rf"\b{re.escape(term)}\b", text))
+
+
+def is_broad_issue(title: str, issue: str, data: dict | None = None) -> bool:
+    data = data or load_rules()
+    text = f"{title} {issue}".lower()
+    keywords = data.get("broad_triggers", {}).get("keywords") or []
+    return any(kw in text for kw in keywords)
+
+
+def _rule_match_score(text: str, rule: dict) -> int:
+    """Higher score = more specific match. Zero means no match."""
+    match = rule.get("match") or {}
+    keywords = match.get("keywords") or []
+    all_of = match.get("all_of") or []
+
+    if all_of and not all(_text_has_term(text, t) for t in all_of):
+        return 0
+
+    score = 0
+    for kw in keywords:
+        if _text_has_term(text, kw):
+            # Multi-word phrases are more specific than single tokens like "sip".
+            score += 2 if " " in kw.strip() else 1
+
+    if all_of:
+        score += len(all_of) * 2
+
+    if score == 0 and not all_of:
+        return 0
+    return score
+
+
+def match_rule(title: str, issue: str, data: dict | None = None) -> MatchedRule | None:
+    data = data or load_rules()
+    text = f"{title} {issue}".lower()
+
+    if is_broad_issue(title, issue, data):
+        default = data.get("default", {})
+        return MatchedRule(
+            id="broad-health",
+            name="Full stack health check",
+            severity=default.get("severity", "medium"),
+            devices=default.get("devices", ["all"]),
+            order=[],
+            hints=list(default.get("hints") or ["Check all onboard devices layer by layer"]),
+            steps=[],
+            broad=True,
+        )
+
+    rules = data.get("rules") or []
+    best: dict | None = None
+    best_score = 0
+    best_priority = -1
+
+    for rule in rules:
+        score = _rule_match_score(text, rule)
+        if score <= 0:
+            continue
+        priority = rule.get("priority") or 0
+        if score > best_score or (score == best_score and priority > best_priority):
+            best = rule
+            best_score = score
+            best_priority = priority
+
+    if not best:
+        return None
+
+    return MatchedRule(
+        id=str(best.get("id") or "unnamed"),
+        name=str(best.get("name") or best.get("id") or "Rule"),
+        severity=str(best.get("severity") or "medium"),
+        devices=list(best.get("devices") or ["all"]),
+        order=list(best.get("order") or []),
+        hints=list(best.get("hints") or []),
+        steps=list(best.get("steps") or []),
+    )
+
+
+def device_keyword_map(data: dict | None = None) -> dict[str, list[str]]:
+    data = data or load_rules()
+    return {k: list(v) for k, v in (data.get("device_keywords") or {}).items()}
+
+
+def devices_mentioned_in_text(text: str, devices: list[dict], data: dict | None = None) -> list[dict]:
+    """Return onboard devices whose catalog keywords appear in *text*."""
+    data = data or load_rules()
+    kw_map = device_keyword_map(data)
+    matched_devices: list[dict] = []
+    for device in devices:
+        key = (device.get("catalog_key") or device.get("device_type") or "").lower()
+        keywords = kw_map.get(key, [])
+        terms = set(keywords + [key, device.get("device_type", ""), device.get("name", "")])
+        terms = {t.lower() for t in terms if t}
+        if any(_text_has_term(text, term) for term in terms if len(term) > 2):
+            matched_devices.append(device)
+    return matched_devices
+
+
+def select_devices_for_issue(
+    devices: list[dict],
+    issue: str,
+    *,
+    title: str = "",
+) -> tuple[list[dict], MatchedRule | None]:
+    if not devices:
+        return [], None
+
+    data = load_rules()
+    matched = match_rule(title, issue, data)
+
+    text = f"{title} {issue}".lower()
+
+    if matched:
+        if "all" in matched.devices:
+            selected = list(devices)
+        else:
+            want = {d.lower() for d in matched.devices}
+            selected = [d for d in devices if (d.get("catalog_key") or "").lower() in want]
+            # Union devices explicitly named in the issue (e.g. "proxmox and pfsense").
+            if not matched.broad:
+                seen = {(d.get("catalog_key") or "").lower() for d in selected}
+                for device in devices_mentioned_in_text(text, devices, data):
+                    key = (device.get("catalog_key") or "").lower()
+                    if key not in seen:
+                        selected.append(device)
+                        seen.add(key)
+
+        if matched.order:
+            order_map = {k.lower(): i for i, k in enumerate(matched.order)}
+            selected.sort(
+                key=lambda d: order_map.get((d.get("catalog_key") or "").lower(), 999)
+            )
+
+        if selected:
+            return selected, matched
+
+    if is_broad_issue(title, issue, data):
+        return devices, matched
+
+    matched_devices = devices_mentioned_in_text(text, devices, data)
+
+    # Never fall back to all devices — only run what the issue mentions.
+    return matched_devices, matched
+
+
+def rule_guidance_block(matched: MatchedRule | dict | None) -> str:
+    if not matched:
+        return ""
+    if isinstance(matched, dict):
+        name = matched.get("name", "")
+        rid = matched.get("id", "")
+        severity = matched.get("severity", "medium")
+        devices = matched.get("devices") or []
+        order = matched.get("order") or []
+        hints = matched.get("hints") or []
+        steps = matched.get("steps") or []
+    else:
+        name, rid, severity = matched.name, matched.id, matched.severity
+        devices, order, hints, steps = matched.devices, matched.order, matched.hints, matched.steps
+
+    lines = [
+        "\n\n--- Troubleshooting rule (standardized) ---",
+        f"Rule: {name} ({rid})",
+        f"Severity hint: {severity}",
+    ]
+    if devices:
+        lines.append(f"Target devices: {', '.join(devices)}")
+    if order:
+        lines.append(f"Diagnose order: {' → '.join(order)}")
+    if hints:
+        lines.append("Hints:")
+        lines.extend(f"- {h}" for h in hints)
+    if steps:
+        lines.append("Suggested steps:")
+        lines.extend(f"- {s}" for s in steps)
+    return "\n".join(lines)

backend/app/services/vessel_devices.py

@@ -0,0 +1,78 @@
+"""Helpers for building vessel device rows from catalog selections."""
+from __future__ import annotations
+
+from app.core.crypto import decrypt_secret, encrypt_secret
+from app.agent.asterisk_profiles import ASTERISK_CATALOG_KEYS
+from app.inventory_catalog import CatalogEntry, catalog_by_key
+from app.models.inventory import Device, Vessel
+from app.schemas import DeviceOut, VesselDeviceSelection
+from app.services.device_defaults import merge_selection_with_defaults
+
+
+def device_to_out(d: Device) -> DeviceOut:
+    out = DeviceOut.model_validate(d)
+    out.has_secret = bool(d.secret_enc)
+    return out
+
+
+def _resolve_address(vessel: Vessel, sel: VesselDeviceSelection) -> str:
+    if sel.address:
+        return sel.address
+    if vessel.public_ip:
+        return vessel.public_ip
+    raise ValueError(f"Device '{sel.catalog_key}' needs an address or vessel public IP")
+
+
+def build_device(
+    vessel: Vessel,
+    sel: VesselDeviceSelection,
+    entry: CatalogEntry,
+    *,
+    existing_secret_plain: str | None = None,
+) -> Device:
+    port, username, secret = merge_selection_with_defaults(
+        entry.key,
+        entry,
+        port=sel.port,
+        username=sel.username,
+        secret=sel.secret,
+        existing_secret=existing_secret_plain,
+    )
+    secret_enc = encrypt_secret(secret) if secret else None
+    return Device(
+        vessel_id=vessel.id,
+        catalog_key=entry.key,
+        name=f"{vessel.name} — {entry.label}",
+        device_type=entry.device_type,
+        address=_resolve_address(vessel, sel),
+        port=port,
+        mcp_server=entry.mcp_server,
+        username=username,
+        secret_enc=secret_enc,
+        notes=sel.notes,
+    )
+
+
+def validate_selections(selections: list[VesselDeviceSelection]) -> list[tuple[VesselDeviceSelection, CatalogEntry]]:
+    catalog = catalog_by_key()
+    enabled = [s for s in selections if s.enabled]
+    if not enabled:
+        raise ValueError("Select at least one device for this vessel")
+    out: list[tuple[VesselDeviceSelection, CatalogEntry]] = []
+    seen: set[str] = set()
+    asterisk_picked = False
+    for sel in enabled:
+        entry = catalog.get(sel.catalog_key)
+        if not entry:
+            raise ValueError(f"Unknown catalog key: {sel.catalog_key}")
+        if sel.catalog_key in seen:
+            raise ValueError(f"Duplicate device slot: {sel.catalog_key}")
+        if sel.catalog_key in ASTERISK_CATALOG_KEYS:
+            if asterisk_picked:
+                raise ValueError(
+                    "Only one Asterisk slot per vessel — pick GeneseasX (docker voip) OR TMGeneseas/satbox (native)"
+                )
+            asterisk_picked = True
+        seen.add(sel.catalog_key)
+        out.append((sel, entry))
+    return out