Initial commit: Agentic OS troubleshooting platform
Self-hosted, Docker-based agentic troubleshooting platform: FastAPI backend + LangGraph agent, Next.js UI, tiered LLM routing (local Ollama -> Gemini -> DeepSeek -> OpenRouter), MCP server manager, encrypted device credentials, RBAC, audit log, project-memory + Obsidian integrations, and editable troubleshooting decision rules tuned for the GeneseasX vessel stack. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
137
backend/app/worker.py
Normal file
137
backend/app/worker.py
Normal file
@@ -0,0 +1,137 @@
|
||||
"""Background worker: owns MCP sessions and runs the troubleshooting agent per task.
|
||||
|
||||
Consumes task ids from the Redis queue, executes the LangGraph agent, and keeps the
|
||||
MCP status snapshot fresh for the API/UI.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.agent import run_task_agent
|
||||
from app.config import settings
|
||||
from app.database import SessionLocal, init_db
|
||||
from app.mcp_manager import get_manager
|
||||
from app.models.enums import TaskStatus
|
||||
from app.models.task import Task
|
||||
from app.services.events import TASK_QUEUE, get_redis
|
||||
from app.services.task_runtime import is_task_cancel_requested
|
||||
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logger = logging.getLogger("worker")
|
||||
|
||||
|
||||
async def _refresh_status_loop() -> None:
|
||||
manager = get_manager()
|
||||
while True:
|
||||
await asyncio.sleep(5)
|
||||
try:
|
||||
while await manager.process_command_queue_once():
|
||||
await asyncio.sleep(0) # yield so task queue blpop can run
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("MCP command processing failed: %s", exc)
|
||||
try:
|
||||
await manager.publish_status()
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("status publish failed: %s", exc)
|
||||
|
||||
|
||||
async def _task_should_run(task_id: int) -> bool:
|
||||
async with SessionLocal() as db:
|
||||
res = await db.execute(select(Task).where(Task.id == task_id))
|
||||
task = res.scalar_one_or_none()
|
||||
if not task:
|
||||
return False
|
||||
if task.status == TaskStatus.cancelled:
|
||||
return False
|
||||
if await is_task_cancel_requested(task_id):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
async def _mark_task_running(task_id: int) -> None:
|
||||
async with SessionLocal() as db:
|
||||
res = await db.execute(select(Task).where(Task.id == task_id))
|
||||
task = res.scalar_one_or_none()
|
||||
if task and task.status == TaskStatus.queued:
|
||||
task.status = TaskStatus.running
|
||||
await db.commit()
|
||||
|
||||
|
||||
async def _run_task(task_id: int, sem: asyncio.Semaphore) -> None:
|
||||
async with sem:
|
||||
if not await _task_should_run(task_id):
|
||||
logger.info("Skipping task %s (cancelled or missing)", task_id)
|
||||
return
|
||||
logger.info("Running task %s", task_id)
|
||||
try:
|
||||
await run_task_agent(task_id)
|
||||
except asyncio.CancelledError:
|
||||
logger.info("task %s asyncio task cancelled", task_id)
|
||||
except Exception: # noqa: BLE001
|
||||
logger.exception("task %s crashed", task_id)
|
||||
|
||||
|
||||
async def _cancel_watch_loop(in_flight: dict[int, asyncio.Task]) -> None:
|
||||
"""Cancel in-flight asyncio tasks when the user requests stop via the API."""
|
||||
while True:
|
||||
await asyncio.sleep(1)
|
||||
for task_id, handle in list(in_flight.items()):
|
||||
if handle.done():
|
||||
continue
|
||||
try:
|
||||
if await is_task_cancel_requested(task_id):
|
||||
logger.info("Cancelling in-flight asyncio task %s", task_id)
|
||||
handle.cancel()
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.debug("cancel watch for task %s: %s", task_id, exc)
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
await init_db()
|
||||
manager = get_manager()
|
||||
logger.info("Starting MCP manager (cloning + launching MCP servers)...")
|
||||
await manager.startup()
|
||||
logger.info("MCP manager ready: %s", [s["name"] for s in manager.status_snapshot()])
|
||||
|
||||
asyncio.create_task(_refresh_status_loop())
|
||||
|
||||
max_concurrent = max(1, settings.worker_max_concurrent_tasks)
|
||||
sem = asyncio.Semaphore(max_concurrent)
|
||||
in_flight: dict[int, asyncio.Task] = {}
|
||||
asyncio.create_task(_cancel_watch_loop(in_flight))
|
||||
|
||||
redis = get_redis()
|
||||
logger.info(
|
||||
"Worker waiting for tasks on %s (max concurrent: %s)",
|
||||
TASK_QUEUE,
|
||||
max_concurrent,
|
||||
)
|
||||
while True:
|
||||
item = await redis.blpop(TASK_QUEUE, timeout=5)
|
||||
if not item:
|
||||
in_flight = {tid: t for tid, t in in_flight.items() if not t.done()}
|
||||
continue
|
||||
_, task_id_str = item
|
||||
try:
|
||||
task_id = int(task_id_str)
|
||||
except ValueError:
|
||||
continue
|
||||
if not await _task_should_run(task_id):
|
||||
logger.info("Skipped dequeued task %s (already cancelled)", task_id)
|
||||
continue
|
||||
logger.info("Picked up task %s", task_id)
|
||||
await _mark_task_running(task_id)
|
||||
handle = asyncio.create_task(_run_task(task_id, sem))
|
||||
in_flight[task_id] = handle
|
||||
|
||||
def _done(t: asyncio.Task, tid: int = task_id) -> None:
|
||||
in_flight.pop(tid, None)
|
||||
|
||||
handle.add_done_callback(_done)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
Reference in New Issue
Block a user