Enhance MCP development features and introduce skills management

- Added configuration options for requiring human approval before applying LLM-generated MCP patches.
- Updated Docker setup to include skills directory.
- Integrated skills management into the backend, allowing for procedural guides and skill matching.
- Refactored database initialization to apply Alembic migrations.
- Enhanced task approval process to handle MCP patch applications with optional approval.
- Introduced new schemas for skills and updated existing APIs to support skills functionality.

This commit lays the groundwork for improved agent capabilities and better management of MCP development processes.

backend/tests/conftest.py

@@ -0,0 +1,13 @@
+"""Shared pytest fixtures."""
+from __future__ import annotations
+
+import pytest
+
+
+@pytest.fixture(autouse=True)
+def _clear_settings_cache():
+    from app.config import get_settings
+
+    get_settings.cache_clear()
+    yield
+    get_settings.cache_clear()

backend/tests/test_device_defaults.py

@@ -3,25 +3,30 @@ from __future__ import annotations
 
 import os
 
-os.environ.setdefault("SECRET_KEY", "test-secret-key-for-unit-tests-only")
-os.environ["DEVICE_SSH_USERNAME"] = "shipadmin"
-os.environ["DEVICE_SSH_PASSWORD"] = "ssh-secret"
-os.environ["DEVICE_PFSENSE_PORT"] = "40443"
-os.environ["DEVICE_PFSENSE_API_KEY"] = "pf-key-123"
-os.environ["DEVICE_PROXMOX_PASSWORD"] = "pve-pass"
-os.environ["DEVICE_PROXMOX_TOKEN_ID"] = "root@pam!agentic"
-os.environ["DEVICE_PROXMOX_TOKEN_SECRET"] = "token-uuid"
+import pytest
 
-from app.inventory_catalog import catalog_by_key  # noqa: E402
-from app.services.device_defaults import (  # noqa: E402
-    merge_selection_with_defaults,
-    proxmox_api_configured,
-    resolve_proxmox_api_defaults,
-    resolve_slot_defaults,
-)
+os.environ.setdefault("SECRET_KEY", "test-secret-key-for-unit-tests-only")
+
+
+@pytest.fixture(autouse=True)
+def device_env(monkeypatch: pytest.MonkeyPatch):
+    monkeypatch.setenv("DEVICE_SSH_USERNAME", "shipadmin")
+    monkeypatch.setenv("DEVICE_SSH_PASSWORD", "ssh-secret")
+    monkeypatch.delenv("DEVICE_PROXMOX_USERNAME", raising=False)
+    monkeypatch.setenv("DEVICE_PFSENSE_PORT", "40443")
+    monkeypatch.setenv("DEVICE_PFSENSE_API_KEY", "pf-key-123")
+    monkeypatch.setenv("DEVICE_PROXMOX_PASSWORD", "pve-pass")
+    monkeypatch.setenv("DEVICE_PROXMOX_TOKEN_ID", "root@pam!agentic")
+    monkeypatch.setenv("DEVICE_PROXMOX_TOKEN_SECRET", "token-uuid")
+    from app.config import get_settings
+
+    get_settings.cache_clear()
 
 
 def test_ssh_slot_uses_global_fallback():
+    from app.inventory_catalog import catalog_by_key
+    from app.services.device_defaults import resolve_slot_defaults
+
     entry = catalog_by_key()["docker_vm"]
     d = resolve_slot_defaults("docker_vm", entry)
     assert d.username == "shipadmin"
@@ -30,6 +35,9 @@ def test_ssh_slot_uses_global_fallback():
 
 
 def test_per_slot_overrides_global():
+    from app.inventory_catalog import catalog_by_key
+    from app.services.device_defaults import resolve_slot_defaults
+
     entry = catalog_by_key()["proxmox"]
     d = resolve_slot_defaults("proxmox", entry)
     assert d.secret == "pve-pass"
@@ -37,32 +45,32 @@ def test_per_slot_overrides_global():
 
 
 def test_api_key_slot():
+    from app.inventory_catalog import catalog_by_key
+    from app.services.device_defaults import resolve_slot_defaults
+
     entry = catalog_by_key()["pfsense"]
     d = resolve_slot_defaults("pfsense", entry)
-    assert d.port == 40443
     assert d.secret == "pf-key-123"
     assert d.secret_kind == "api_key"
+    assert d.port == 40443
 
 
-def test_merge_applies_env_when_blank():
-    entry = catalog_by_key()["pfsense"]
+def test_merge_keeps_existing_secret():
+    from app.inventory_catalog import catalog_by_key
+    from app.services.device_defaults import merge_selection_with_defaults
+
+    entry = catalog_by_key()["docker_vm"]
     port, user, secret = merge_selection_with_defaults(
-        "pfsense", entry, port=None, username=None, secret=None
+        "docker_vm", entry, port=None, username=None, secret=None, existing_secret="stored"
     )
-    assert port == 40443
-    assert secret == "pf-key-123"
+    assert secret == "stored"
+    assert user == "shipadmin"
 
 
-def test_proxmox_api_env():
-    api = resolve_proxmox_api_defaults("10.1.2.3")
-    assert api["proxmox_token_id"] == "root@pam!agentic"
-    assert api["proxmox_token_secret"] == "token-uuid"
-    assert proxmox_api_configured(api)
+def test_proxmox_api_defaults_from_env():
+    from app.services.device_defaults import proxmox_api_configured, resolve_proxmox_api_defaults
 
-
-def test_merge_ui_override_wins():
-    entry = catalog_by_key()["pfsense"]
-    _, _, secret = merge_selection_with_defaults(
-        "pfsense", entry, port=8443, username="admin", secret="custom-key"
-    )
-    assert secret == "custom-key"
+    ctx = resolve_proxmox_api_defaults("10.20.30.254")
+    assert ctx["proxmox_api_url"] == "https://10.20.30.254:8006"
+    assert ctx["proxmox_token_id"] == "root@pam!agentic"
+    assert proxmox_api_configured(ctx)

backend/tests/test_mcp_approvals.py

@@ -0,0 +1,106 @@
+"""Tests for MCP patch approval gating."""
+from __future__ import annotations
+
+import asyncio
+import os
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+
+os.environ.setdefault("SECRET_KEY", "test-secret-key-for-unit-tests-only")
+
+
+def test_repair_queues_apply_patch_when_approval_required(monkeypatch):
+    from app.config import get_settings, settings
+
+    get_settings.cache_clear()
+
+    proposal = {
+        "ok": True,
+        "summary": "Add missing tool",
+        "commit_message": "fix: add tool",
+        "files": [{"path": "server.py", "content": "# patched"}],
+    }
+
+    mock_db = MagicMock()
+    mock_db.commit = AsyncMock()
+    mock_session = MagicMock()
+    mock_session.__aenter__ = AsyncMock(return_value=mock_db)
+    mock_session.__aexit__ = AsyncMock(return_value=None)
+
+    async def _run():
+        with (
+            patch.object(settings, "mcp_dev_require_approval", True),
+            patch("app.agent.mcp_development.propose_mcp_patch", new=AsyncMock(return_value=proposal)),
+            patch(
+                "app.agent.mcp_development.backup_proposal_files",
+                new=AsyncMock(return_value={"server.py": "# old"}),
+            ),
+            patch("app.agent.mcp_development.execute_approved_patch", new=AsyncMock()) as mock_apply,
+            patch("app.database.SessionLocal", return_value=mock_session),
+        ):
+            from app.agent.mcp_development import attempt_mcp_repair
+
+            emit = AsyncMock()
+            result = await attempt_mcp_repair(
+                task_id=42,
+                server="pfsense-mcp",
+                tool="pfsense_list_rules",
+                args={"interface": "wan"},
+                issue="tool missing",
+                error="unknown tool",
+                output=None,
+                manager=AsyncMock(),
+                emit=emit,
+                device="pfsense",
+            )
+        return result, mock_apply, emit, mock_db
+
+    result, mock_apply, emit, mock_db = asyncio.run(_run())
+
+    assert result.get("pending_approval") is True
+    assert result.get("approval_type") == "mcp_apply_patch"
+    mock_apply.assert_not_called()
+    mock_db.add.assert_called_once()
+    mock_db.commit.assert_awaited_once()
+    assert any(c.args[1] == "approval" for c in emit.call_args_list)
+
+
+def test_repair_applies_immediately_when_approval_not_required(monkeypatch):
+    from app.config import get_settings, settings
+
+    get_settings.cache_clear()
+
+    proposal = {
+        "ok": True,
+        "summary": "Fix crash",
+        "files": [{"path": "server.py", "content": "# patched"}],
+    }
+
+    async def _run():
+        with (
+            patch.object(settings, "mcp_dev_require_approval", False),
+            patch("app.agent.mcp_development.propose_mcp_patch", new=AsyncMock(return_value=proposal)),
+            patch(
+                "app.agent.mcp_development.execute_approved_patch",
+                new=AsyncMock(return_value={"ok": True, "tools": ["a"]}),
+            ) as mock_apply,
+        ):
+            from app.agent.mcp_development import attempt_mcp_repair
+
+            return await attempt_mcp_repair(
+                task_id=1,
+                server="pfsense-mcp",
+                tool="t",
+                args={},
+                issue="x",
+                error="err",
+                output=None,
+                manager=AsyncMock(),
+                emit=AsyncMock(),
+            ), mock_apply
+
+    result, mock_apply = asyncio.run(_run())
+
+    assert result.get("ok") is True
+    mock_apply.assert_awaited_once()

backend/tests/test_skill_registry.py

@@ -0,0 +1,60 @@
+"""Tests for agent skill registry."""
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+os.environ.setdefault("SECRET_KEY", "test-secret-key-for-unit-tests-only")
+_test_dir = Path(__file__).resolve().parent
+_backend_root = _test_dir.parent
+_skills_dir = _backend_root.parent / "skills"
+if not _skills_dir.is_dir():
+    _skills_dir = _backend_root / "skills"
+os.environ["SKILLS_PATH"] = str(_skills_dir)
+
+from app.services.skill_registry import (  # noqa: E402
+    invalidate_skills_cache,
+    load_skills,
+    match_skills,
+    skill_guidance_block,
+)
+
+
+def test_load_skills_finds_seeded_skills():
+    invalidate_skills_cache()
+    skills = load_skills(force=True)
+    assert "vessel-voip" in skills
+    assert "vessel-dns-portal" in skills
+    assert skills["vessel-voip"]["body"]
+
+
+def test_match_voip_skill_by_keywords():
+    invalidate_skills_cache()
+    matched = match_skills("VoIP issue", "one-way audio on SIP calls")
+    ids = [s.id for s in matched]
+    assert "vessel-voip" in ids
+
+
+def test_match_skill_by_rule_id_boost():
+    invalidate_skills_cache()
+    matched = match_skills("Mystery", "something vague", matched_rule_id="crew-dns-slow")
+    ids = [s.id for s in matched]
+    assert "vessel-dns-portal" in ids
+
+
+def test_match_respects_max_skills():
+    invalidate_skills_cache()
+    matched = match_skills(
+        "VoIP DNS portal",
+        "slow dns captive portal sip registration one-way audio",
+        max_skills=1,
+    )
+    assert len(matched) == 1
+
+
+def test_skill_guidance_block_includes_body():
+    invalidate_skills_cache()
+    matched = match_skills("VoIP", "one-way audio rtp")
+    block = skill_guidance_block(matched)
+    assert "Agent skills" in block
+    assert "One-way audio playbook" in block