Initial commit: Agentic OS troubleshooting platform
Self-hosted, Docker-based agentic troubleshooting platform: FastAPI backend + LangGraph agent, Next.js UI, tiered LLM routing (local Ollama -> Gemini -> DeepSeek -> OpenRouter), MCP server manager, encrypted device credentials, RBAC, audit log, project-memory + Obsidian integrations, and editable troubleshooting decision rules tuned for the GeneseasX vessel stack. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
3
backend/app/mcp_manager/__init__.py
Normal file
3
backend/app/mcp_manager/__init__.py
Normal file
@@ -0,0 +1,3 @@
|
||||
from app.mcp_manager.manager import McpManager, get_manager
|
||||
|
||||
__all__ = ["McpManager", "get_manager"]
|
||||
191
backend/app/mcp_manager/commands.py
Normal file
191
backend/app/mcp_manager/commands.py
Normal file
@@ -0,0 +1,191 @@
|
||||
"""Redis queue for MCP maintenance commands (worker executes reload/upgrade/develop)."""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import uuid
|
||||
from collections.abc import Awaitable, Callable
|
||||
from typing import Any
|
||||
|
||||
from app.services.events import get_redis
|
||||
|
||||
MCP_COMMAND_QUEUE = "agentic:mcp:commands"
|
||||
|
||||
|
||||
def _result_key(request_id: str) -> str:
|
||||
return f"agentic:mcp:cmd_result:{request_id}"
|
||||
|
||||
|
||||
async def _wait_result(request_id: str, timeout_seconds: float) -> dict:
|
||||
redis = get_redis()
|
||||
deadline = asyncio.get_event_loop().time() + timeout_seconds
|
||||
while asyncio.get_event_loop().time() < deadline:
|
||||
raw = await redis.get(_result_key(request_id))
|
||||
if raw:
|
||||
await redis.delete(_result_key(request_id))
|
||||
return json.loads(raw)
|
||||
await asyncio.sleep(0.5)
|
||||
return {"ok": False, "error": "Worker did not respond in time — is the worker running?"}
|
||||
|
||||
|
||||
async def enqueue_mcp_command(
|
||||
action: str,
|
||||
*,
|
||||
server: str | None = None,
|
||||
timeout_seconds: float = 90,
|
||||
**extra: Any,
|
||||
) -> dict:
|
||||
request_id = str(uuid.uuid4())
|
||||
cmd = {"request_id": request_id, "action": action, "server": server, **extra}
|
||||
await get_redis().rpush(MCP_COMMAND_QUEUE, json.dumps(cmd))
|
||||
return await _wait_result(request_id, timeout_seconds)
|
||||
|
||||
|
||||
async def enqueue_mcp_command_async(
|
||||
action: str,
|
||||
*,
|
||||
server: str | None = None,
|
||||
**extra: Any,
|
||||
) -> str:
|
||||
"""Queue a command for the worker without waiting for the result."""
|
||||
request_id = str(uuid.uuid4())
|
||||
cmd = {"request_id": request_id, "action": action, "server": server, **extra}
|
||||
await get_redis().rpush(MCP_COMMAND_QUEUE, json.dumps(cmd))
|
||||
return request_id
|
||||
|
||||
|
||||
async def store_mcp_command_result(request_id: str, result: dict) -> None:
|
||||
redis = get_redis()
|
||||
await redis.set(_result_key(request_id), json.dumps(result), ex=300)
|
||||
|
||||
|
||||
async def process_mcp_command(cmd: dict) -> dict:
|
||||
from app.mcp_manager.manager import get_manager
|
||||
|
||||
action = cmd.get("action")
|
||||
server = cmd.get("server")
|
||||
manager = get_manager()
|
||||
try:
|
||||
if action == "reload":
|
||||
if not server:
|
||||
raise ValueError("server name required")
|
||||
await manager.reload_server(server)
|
||||
elif action == "upgrade":
|
||||
if not server:
|
||||
raise ValueError("server name required")
|
||||
await manager.upgrade_server(server)
|
||||
elif action == "reload_all":
|
||||
await manager.reload_all()
|
||||
elif action == "develop":
|
||||
if not server:
|
||||
raise ValueError("server name required")
|
||||
return await _execute_develop_command(cmd, manager)
|
||||
else:
|
||||
raise ValueError(f"Unknown action: {action}")
|
||||
return {"ok": True, "action": action, "server": server}
|
||||
except Exception as exc: # noqa: BLE001
|
||||
return {"ok": False, "action": action, "server": server, "error": str(exc)}
|
||||
|
||||
|
||||
async def _execute_develop_command(cmd: dict, manager) -> dict:
|
||||
server = cmd["server"]
|
||||
result = await _develop_on_worker(cmd, manager)
|
||||
task_id = int(cmd.get("task_id") or 0)
|
||||
if task_id:
|
||||
await _finalize_develop_task(task_id, result, server)
|
||||
return result
|
||||
|
||||
|
||||
async def run_develop_command_background(cmd: dict, manager) -> None:
|
||||
"""Run MCP develop without blocking the worker task queue loop."""
|
||||
request_id = cmd.get("request_id", "")
|
||||
try:
|
||||
result = await _execute_develop_command(cmd, manager)
|
||||
await store_mcp_command_result(request_id, result)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
await store_mcp_command_result(
|
||||
request_id,
|
||||
{"ok": False, "action": "develop", "server": cmd.get("server"), "error": str(exc)},
|
||||
)
|
||||
|
||||
|
||||
async def _finalize_develop_task(task_id: int, result: dict, server: str) -> None:
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.database import SessionLocal
|
||||
from app.models.enums import ApprovalStatus, TaskStatus
|
||||
from app.models.task import ApprovalRequest, Task
|
||||
from app.services.events import publish_event
|
||||
|
||||
async with SessionLocal() as db:
|
||||
res = await db.execute(select(Task).where(Task.id == task_id))
|
||||
task = res.scalar_one_or_none()
|
||||
if not task:
|
||||
return
|
||||
|
||||
pend = await db.execute(
|
||||
select(ApprovalRequest).where(
|
||||
ApprovalRequest.task_id == task_id,
|
||||
ApprovalRequest.status == ApprovalStatus.pending,
|
||||
)
|
||||
)
|
||||
has_pending = pend.scalars().first() is not None
|
||||
|
||||
if result.get("ok"):
|
||||
task.report = {
|
||||
"kind": "mcp_develop",
|
||||
"mcp_server": server,
|
||||
"summary": result.get("summary"),
|
||||
"tools": result.get("tools"),
|
||||
"push": result.get("push"),
|
||||
"resolved": not has_pending,
|
||||
}
|
||||
task.error = None
|
||||
task.status = TaskStatus.waiting_approval if has_pending else TaskStatus.succeeded
|
||||
else:
|
||||
task.status = TaskStatus.failed
|
||||
task.error = result.get("error") or result.get("reason") or "MCP development failed"
|
||||
await db.commit()
|
||||
|
||||
if result.get("ok"):
|
||||
msg = result.get("summary") or f"MCP develop finished for {server}"
|
||||
if has_pending:
|
||||
msg += " — approve git push on this task."
|
||||
await publish_event(task_id, "report", msg, {"kind": "mcp_develop", **result}, persist=True)
|
||||
else:
|
||||
err = result.get("error") or result.get("reason") or "MCP development failed"
|
||||
await publish_event(task_id, "error", err, result, persist=True)
|
||||
|
||||
|
||||
async def _develop_on_worker(cmd: dict, manager) -> dict:
|
||||
from app.agent.mcp_development import attempt_mcp_repair
|
||||
from app.services.events import publish_event
|
||||
|
||||
server = cmd["server"]
|
||||
task_id = int(cmd.get("task_id") or 0)
|
||||
|
||||
async def emit(tid: int, kind: str, message: str, payload: dict | None = None):
|
||||
if tid:
|
||||
await publish_event(tid, kind, message, payload)
|
||||
|
||||
if task_id:
|
||||
await emit(
|
||||
task_id,
|
||||
"log",
|
||||
f"MCP develop started for {server}",
|
||||
{"server": server, "kind": "mcp_develop"},
|
||||
)
|
||||
|
||||
return await attempt_mcp_repair(
|
||||
task_id=task_id,
|
||||
server=server,
|
||||
tool=cmd.get("tool") or "unknown",
|
||||
args={},
|
||||
issue=cmd.get("issue") or "",
|
||||
error=cmd.get("error"),
|
||||
output=cmd.get("output"),
|
||||
manager=manager,
|
||||
emit=emit,
|
||||
task_title=cmd.get("title") or f"MCP develop: {server}",
|
||||
push=cmd.get("push"),
|
||||
)
|
||||
294
backend/app/mcp_manager/manager.py
Normal file
294
backend/app/mcp_manager/manager.py
Normal file
@@ -0,0 +1,294 @@
|
||||
"""Clone MCP repos from Gitea and run them as stdio MCP subprocess sidecars."""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
from contextlib import AsyncExitStack
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from app.config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
MCP_STATUS_KEY = "agentic:mcp:status"
|
||||
|
||||
|
||||
@dataclass
|
||||
class ServerState:
|
||||
name: str
|
||||
status: str = "stopped"
|
||||
tools: list[dict] = field(default_factory=list)
|
||||
error: str | None = None
|
||||
session: object | None = None
|
||||
_stack: AsyncExitStack | None = field(default=None, repr=False)
|
||||
|
||||
|
||||
async def ensure_git_remote(repo: str, target: str) -> None:
|
||||
"""Point origin at GITEA_BASE_URL — fixes repos cloned when remote was public HTTPS."""
|
||||
if not os.path.isdir(os.path.join(target, ".git")):
|
||||
return
|
||||
url = settings.gitea_clone_url(repo)
|
||||
code, out = await McpManager._run(["git", "remote", "set-url", "origin", url], cwd=target)
|
||||
if code != 0:
|
||||
raise RuntimeError(f"git remote set-url failed: {out[:500]}")
|
||||
|
||||
|
||||
class McpManager:
|
||||
def __init__(self) -> None:
|
||||
self.servers: dict[str, ServerState] = {}
|
||||
self._lock = asyncio.Lock()
|
||||
self._started = False
|
||||
|
||||
async def startup(self) -> None:
|
||||
async with self._lock:
|
||||
if self._started:
|
||||
return
|
||||
self._started = True
|
||||
os.makedirs(self.repos_dir, exist_ok=True)
|
||||
for repo in settings.mcp_repo_list:
|
||||
state = ServerState(name=repo)
|
||||
self.servers[repo] = state
|
||||
try:
|
||||
await self._prepare_repo(repo, state)
|
||||
await self._start_server(repo, state)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
state.status = "error"
|
||||
state.error = str(exc)
|
||||
logger.warning("MCP %s failed to start: %s", repo, exc)
|
||||
await self.publish_status()
|
||||
|
||||
async def shutdown(self) -> None:
|
||||
async with self._lock:
|
||||
for state in self.servers.values():
|
||||
await self._stop_server(state)
|
||||
|
||||
@property
|
||||
def repos_dir(self) -> str:
|
||||
return os.path.join(settings.runtime_dir, "mcp-repos")
|
||||
|
||||
@property
|
||||
def local_dir(self) -> str:
|
||||
return os.environ.get("MCP_LOCAL_DIR", "/app/mcp-servers")
|
||||
|
||||
def _is_local_repo(self, repo: str) -> bool:
|
||||
return os.path.isdir(os.path.join(self.local_dir, repo))
|
||||
|
||||
def _repo_dir(self, repo: str) -> str:
|
||||
local_src = os.path.join(self.local_dir, repo)
|
||||
if os.path.isdir(local_src):
|
||||
dst = os.path.join(settings.runtime_dir, "mcp-local", repo)
|
||||
if not os.path.isdir(dst):
|
||||
os.makedirs(os.path.dirname(dst), exist_ok=True)
|
||||
shutil.copytree(local_src, dst)
|
||||
return dst
|
||||
return os.path.join(self.repos_dir, repo)
|
||||
|
||||
async def _sync_local_repo(self, repo: str) -> None:
|
||||
local_src = os.path.join(self.local_dir, repo)
|
||||
if not os.path.isdir(local_src):
|
||||
return
|
||||
dst = os.path.join(settings.runtime_dir, "mcp-local", repo)
|
||||
if os.path.isdir(dst):
|
||||
shutil.rmtree(dst)
|
||||
shutil.copytree(local_src, dst)
|
||||
|
||||
async def _ensure_git_remote(self, repo: str, target: str) -> None:
|
||||
await ensure_git_remote(repo, target)
|
||||
|
||||
async def _prepare_repo(self, repo: str, state: ServerState) -> None:
|
||||
if self._is_local_repo(repo):
|
||||
await self._sync_local_repo(repo)
|
||||
return
|
||||
state.status = "cloning"
|
||||
target = os.path.join(self.repos_dir, repo)
|
||||
url = settings.gitea_clone_url(repo)
|
||||
if os.path.isdir(os.path.join(target, ".git")):
|
||||
await self._ensure_git_remote(repo, target)
|
||||
code, out = await self._run(["git", "pull", "--rebase", "--autostash"], cwd=target)
|
||||
if code != 0:
|
||||
raise RuntimeError(f"git pull failed: {out[:500]}")
|
||||
else:
|
||||
code, out = await self._run(["git", "clone", "--depth", "1", url, target])
|
||||
if code != 0:
|
||||
raise RuntimeError(f"clone failed: {out[:500]}")
|
||||
|
||||
def _detect_start_command(self, repo: str) -> list[str]:
|
||||
override = os.environ.get(f"MCP_START_{repo.upper().replace('-', '_')}")
|
||||
if override:
|
||||
return override.split()
|
||||
target = self._repo_dir(repo)
|
||||
has_pyproject = os.path.isfile(os.path.join(target, "pyproject.toml"))
|
||||
for candidate in ("server.py", "main.py", "app.py", "mcp_server.py", "__main__.py"):
|
||||
if os.path.isfile(os.path.join(target, candidate)):
|
||||
if has_pyproject:
|
||||
return ["uv", "run", "--directory", target, "python", candidate]
|
||||
return ["uv", "run", "--no-project", "--directory", target, "python", candidate]
|
||||
module = repo.replace("-", "_")
|
||||
if os.path.isdir(os.path.join(target, "src", module)) or os.path.isdir(
|
||||
os.path.join(target, module)
|
||||
):
|
||||
return ["uv", "run", "--directory", target, "python", "-m", module]
|
||||
return ["uv", "run", "--directory", target, repo]
|
||||
|
||||
async def _stop_server(self, state: ServerState) -> None:
|
||||
if state._stack is not None:
|
||||
try:
|
||||
await state._stack.aclose()
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.debug("stop %s: %s", state.name, exc)
|
||||
state._stack = None
|
||||
state.session = None
|
||||
state.tools = []
|
||||
if state.status not in ("error", "cloning"):
|
||||
state.status = "stopped"
|
||||
|
||||
def _mcp_subprocess_env(self, repo: str) -> dict[str, str]:
|
||||
"""Env passed to MCP stdio subprocesses (repo-specific overrides)."""
|
||||
env = {**os.environ}
|
||||
if repo == "proxmox-mcp":
|
||||
verify = os.environ.get("PROXMOX_VERIFY_SSL")
|
||||
if verify is None:
|
||||
verify = os.environ.get("DEVICE_PROXMOX_VERIFY_SSL", "false")
|
||||
env["PROXMOX_VERIFY_SSL"] = verify
|
||||
return env
|
||||
|
||||
async def _start_server(self, repo: str, state: ServerState) -> None:
|
||||
from mcp import ClientSession, StdioServerParameters
|
||||
from mcp.client.stdio import stdio_client
|
||||
|
||||
state.status = "starting"
|
||||
state.error = None
|
||||
target = self._repo_dir(repo)
|
||||
cmd = self._detect_start_command(repo)
|
||||
params = StdioServerParameters(
|
||||
command=cmd[0],
|
||||
args=cmd[1:],
|
||||
cwd=target,
|
||||
env=self._mcp_subprocess_env(repo),
|
||||
)
|
||||
stack = AsyncExitStack()
|
||||
read, write = await stack.enter_async_context(stdio_client(params))
|
||||
session = await stack.enter_async_context(ClientSession(read, write))
|
||||
await asyncio.wait_for(session.initialize(), timeout=90)
|
||||
tools_resp = await session.list_tools()
|
||||
state._stack = stack
|
||||
state.session = session
|
||||
state.tools = [{"name": t.name, "description": t.description} for t in tools_resp.tools]
|
||||
state.status = "loaded"
|
||||
logger.info("MCP %s loaded with %d tools", repo, len(state.tools))
|
||||
|
||||
async def reload_server(self, repo: str) -> None:
|
||||
async with self._lock:
|
||||
state = self.servers.get(repo)
|
||||
if not state:
|
||||
raise ValueError(f"Unknown MCP server: {repo}")
|
||||
await self._stop_server(state)
|
||||
await self._start_server(repo, state)
|
||||
await self.publish_status()
|
||||
|
||||
async def upgrade_server(self, repo: str) -> None:
|
||||
async with self._lock:
|
||||
state = self.servers.get(repo)
|
||||
if not state:
|
||||
raise ValueError(f"Unknown MCP server: {repo}")
|
||||
await self._stop_server(state)
|
||||
await self._prepare_repo(repo, state)
|
||||
await self._start_server(repo, state)
|
||||
await self.publish_status()
|
||||
|
||||
async def reload_all(self) -> None:
|
||||
async with self._lock:
|
||||
for repo in settings.mcp_repo_list:
|
||||
state = self.servers.get(repo)
|
||||
if not state:
|
||||
continue
|
||||
try:
|
||||
await self._stop_server(state)
|
||||
await self._start_server(repo, state)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
state.status = "error"
|
||||
state.error = str(exc)
|
||||
await self.publish_status()
|
||||
|
||||
async def process_command_queue_once(self) -> bool:
|
||||
from app.mcp_manager.commands import (
|
||||
MCP_COMMAND_QUEUE,
|
||||
process_mcp_command,
|
||||
run_develop_command_background,
|
||||
store_mcp_command_result,
|
||||
)
|
||||
from app.services.events import get_redis
|
||||
|
||||
item = await get_redis().lpop(MCP_COMMAND_QUEUE)
|
||||
if not item:
|
||||
return False
|
||||
cmd = json.loads(item)
|
||||
if cmd.get("action") == "develop":
|
||||
asyncio.create_task(run_develop_command_background(cmd, self))
|
||||
return True
|
||||
result = await process_mcp_command(cmd)
|
||||
await store_mcp_command_result(cmd["request_id"], result)
|
||||
return True
|
||||
|
||||
def find_server_for_tool(self, tool_name: str) -> str | None:
|
||||
for name, state in self.servers.items():
|
||||
if any(t["name"] == tool_name for t in state.tools):
|
||||
return name
|
||||
return None
|
||||
|
||||
async def call_tool(self, server: str, tool: str, args: dict) -> dict:
|
||||
state = self.servers.get(server)
|
||||
if not state or state.status != "loaded" or state.session is None:
|
||||
raise RuntimeError(f"MCP server '{server}' is not available")
|
||||
result = await state.session.call_tool(tool, args) # type: ignore[attr-defined]
|
||||
content = []
|
||||
for block in getattr(result, "content", []) or []:
|
||||
text = getattr(block, "text", None)
|
||||
if text is not None:
|
||||
content.append(text)
|
||||
return {
|
||||
"isError": getattr(result, "isError", False),
|
||||
"text": "\n".join(content),
|
||||
"structured": getattr(result, "structuredContent", None),
|
||||
}
|
||||
|
||||
def status_snapshot(self) -> list[dict]:
|
||||
return [
|
||||
{
|
||||
"name": s.name,
|
||||
"status": s.status,
|
||||
"tools": s.tools,
|
||||
"error": s.error,
|
||||
"source": "local" if self._is_local_repo(s.name) else "git",
|
||||
}
|
||||
for s in self.servers.values()
|
||||
]
|
||||
|
||||
async def publish_status(self) -> None:
|
||||
from app.services.events import get_redis
|
||||
|
||||
await get_redis().set(MCP_STATUS_KEY, json.dumps(self.status_snapshot()))
|
||||
|
||||
@staticmethod
|
||||
async def _run(cmd: list[str], cwd: str | None = None) -> tuple[int, str]:
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
*cmd,
|
||||
cwd=cwd,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.STDOUT,
|
||||
)
|
||||
out, _ = await proc.communicate()
|
||||
return proc.returncode or 0, out.decode(errors="replace")
|
||||
|
||||
|
||||
_manager: McpManager | None = None
|
||||
|
||||
|
||||
def get_manager() -> McpManager:
|
||||
global _manager
|
||||
if _manager is None:
|
||||
_manager = McpManager()
|
||||
return _manager
|
||||
46
backend/app/mcp_manager/recovery.py
Normal file
46
backend/app/mcp_manager/recovery.py
Normal file
@@ -0,0 +1,46 @@
|
||||
"""Auto-reload MCP servers when a tool call fails due to server unavailability."""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from collections.abc import Awaitable, Callable
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
EmitFn = Callable[[int, str, str, dict | None], Awaitable[None]]
|
||||
|
||||
|
||||
async def call_tool_resilient(
|
||||
manager,
|
||||
server: str,
|
||||
tool: str,
|
||||
args: dict,
|
||||
*,
|
||||
task_id: int | None = None,
|
||||
emit: EmitFn | None = None,
|
||||
max_retries: int = 1,
|
||||
) -> dict:
|
||||
"""Call MCP tool; on server-unavailable errors, reload that server and retry once."""
|
||||
last_exc: Exception | None = None
|
||||
for attempt in range(max_retries + 1):
|
||||
try:
|
||||
return await manager.call_tool(server, tool, args)
|
||||
except RuntimeError as exc:
|
||||
last_exc = exc
|
||||
msg = str(exc).lower()
|
||||
recoverable = "not available" in msg or "not loaded" in msg
|
||||
if not recoverable or attempt >= max_retries:
|
||||
raise
|
||||
logger.warning("MCP %s unavailable for %s — reloading (attempt %s)", server, tool, attempt + 1)
|
||||
if task_id and emit:
|
||||
await emit(
|
||||
task_id,
|
||||
"log",
|
||||
f"MCP '{server}' unavailable — reloading server and retrying {tool}…",
|
||||
{"server": server, "tool": tool},
|
||||
)
|
||||
try:
|
||||
await manager.reload_server(server)
|
||||
except Exception as reload_exc: # noqa: BLE001
|
||||
logger.warning("MCP reload %s failed: %s", server, reload_exc)
|
||||
raise exc from reload_exc
|
||||
raise last_exc or RuntimeError(f"MCP call failed: {server}/{tool}")
|
||||
Reference in New Issue
Block a user