From c96ea82a962164c9ae2346062b0a207d044f1517 Mon Sep 17 00:00:00 2001 From: Emperiusm Date: Wed, 15 Apr 2026 00:51:56 -0400 Subject: [PATCH 01/41] fix: add missing timeline/artifacts stubs, fix recipe routing and error handling Co-Authored-By: Claude Sonnet 4.6 --- .../web/backend/app/routes/engagements.py | 28 +++++++++++++++++ packages/web/backend/app/routes/recipes.py | 30 +++++++++++++++++++ .../backend/app/services/recipe_service.py | 4 ++- 3 files changed, 61 insertions(+), 1 deletion(-) diff --git a/packages/web/backend/app/routes/engagements.py b/packages/web/backend/app/routes/engagements.py index 6610a90..5448eb1 100644 --- a/packages/web/backend/app/routes/engagements.py +++ b/packages/web/backend/app/routes/engagements.py @@ -110,3 +110,31 @@ async def update_engagement_status( if not engagement: raise HTTPException(status_code=404, detail="Engagement not found") return engagement + + +@router.get("/{engagement_id}/timeline") +async def get_engagement_timeline( + engagement_id: str, + db: AsyncSession = Depends(get_db), + user: User = Depends(get_current_user), +): + """Return timeline events for an engagement.""" + service = EngagementService(db, user) + summary = await service.get_summary(engagement_id) + if not summary: + raise HTTPException(status_code=404, detail="Engagement not found") + return {"items": []} + + +@router.get("/{engagement_id}/artifacts") +async def get_engagement_artifacts( + engagement_id: str, + db: AsyncSession = Depends(get_db), + user: User = Depends(get_current_user), +): + """Return artifacts for an engagement.""" + service = EngagementService(db, user) + summary = await service.get_summary(engagement_id) + if not summary: + raise HTTPException(status_code=404, detail="Engagement not found") + return {"items": []} diff --git a/packages/web/backend/app/routes/recipes.py b/packages/web/backend/app/routes/recipes.py index e73acce..398af71 100644 --- a/packages/web/backend/app/routes/recipes.py +++ b/packages/web/backend/app/routes/recipes.py @@ -27,6 +27,36 @@ async def list_recipes( return {"items": recipes} +@router.get("/{recipe_id}") +async def get_recipe( + recipe_id: str, + user: User = Depends(get_current_user), +): + service = RecipeService(user) + recipes = await service.list_recipes() + recipe = next((r for r in recipes if r.get("id") == recipe_id), None) + if not recipe: + raise HTTPException(status_code=404, detail="Recipe not found") + return recipe + + +@router.post("/{recipe_id}/run") +async def run_recipe_by_id( + recipe_id: str, + body: dict | None = None, + user: User = Depends(get_current_user), +): + service = RecipeService(user) + variables = (body or {}).get("variables", {}) + dry_run = (body or {}).get("dry_run", False) + task_id = await service.run( + recipe_id=recipe_id, + variables=variables, + dry_run=dry_run, + ) + return {"task_id": task_id, "status": "submitted"} + + @router.post("/run") async def run_recipe( body: RecipeRunRequest, diff --git a/packages/web/backend/app/services/recipe_service.py b/packages/web/backend/app/services/recipe_service.py index d7706e8..1fbde13 100644 --- a/packages/web/backend/app/services/recipe_service.py +++ b/packages/web/backend/app/services/recipe_service.py @@ -30,7 +30,9 @@ async def list_recipes(self) -> list[dict]: for r in recipes ] except Exception as exc: - return [{"error": str(exc)}] + import logging + logging.getLogger(__name__).warning("Failed to load recipes: %s", exc) + return [] async def run( self, From 2d29942d5443ea4d365a622123995c306f1a3cc3 Mon Sep 17 00:00:00 2001 From: Emperiusm Date: Wed, 15 Apr 2026 01:12:03 -0400 Subject: [PATCH 02/41] fix: bridge API container to security hub tool containers - Mount Docker socket into API container for docker exec access - Install Docker CLI in API image - Join mcp-security-hub_mcp-network as external network - API can now exec into nmap-mcp, nuclei-mcp, ffuf-mcp etc. Co-Authored-By: Claude Opus 4.6 (1M context) --- packages/web/backend/Dockerfile | 14 ++++++++++++++ packages/web/docker-compose.yml | 11 +++++++++++ 2 files changed, 25 insertions(+) diff --git a/packages/web/backend/Dockerfile b/packages/web/backend/Dockerfile index 708a605..b08e8fa 100644 --- a/packages/web/backend/Dockerfile +++ b/packages/web/backend/Dockerfile @@ -2,10 +2,24 @@ FROM python:3.14-slim WORKDIR /app +# Install Docker CLI (for docker exec into tool containers) +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates curl gnupg && \ + install -m 0755 -d /etc/apt/keyrings && \ + curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc && \ + chmod a+r /etc/apt/keyrings/docker.asc && \ + echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian $(. /etc/os-release && echo "$VERSION_CODENAME") stable" > /etc/apt/sources.list.d/docker.list && \ + apt-get update && apt-get install -y --no-install-recommends docker-ce-cli && \ + apt-get clean && rm -rf /var/lib/apt/lists/* + # Install shared CLI library (for models, parsers, etc.) COPY packages/cli/ /app/packages/cli/ RUN pip install --no-cache-dir /app/packages/cli/ +# Copy plugin directory (recipes, config, tools) +COPY packages/plugin/ /app/packages/plugin/ +ENV OPENTOOLS_PLUGIN_DIR=/app/packages/plugin + # Install web backend COPY packages/web/backend/ /app/packages/web/backend/ RUN pip install --no-cache-dir /app/packages/web/backend/ diff --git a/packages/web/docker-compose.yml b/packages/web/docker-compose.yml index 86fbde5..0ff4b51 100644 --- a/packages/web/docker-compose.yml +++ b/packages/web/docker-compose.yml @@ -36,13 +36,18 @@ services: SECRET_KEY: ${SECRET_KEY:?Set SECRET_KEY in .env} ENVIRONMENT: ${ENVIRONMENT:-production} ALLOWED_ORIGINS: ${ALLOWED_ORIGINS:-} + OPENTOOLS_PLUGIN_DIR: /app/packages/plugin depends_on: migrate: condition: service_completed_successfully volumes: - appdata:/app/data + - //var/run/docker.sock:/var/run/docker.sock ports: - "8000:8000" + networks: + - default + - mcp-network nginx: image: nginx:alpine @@ -57,3 +62,9 @@ services: volumes: pgdata: appdata: + +networks: + default: + mcp-network: + external: true + name: mcp-security-hub_mcp-network From 925dc7c741db6755a4f3c351c60733aeb5fe640b Mon Sep 17 00:00:00 2001 From: Emperiusm Date: Wed, 15 Apr 2026 01:23:17 -0400 Subject: [PATCH 03/41] feat: add scan UI page with target type selector and file upload Co-Authored-By: Claude Sonnet 4.6 --- .../web/frontend/src/components/AppLayout.vue | 1 + packages/web/frontend/src/router/index.ts | 1 + packages/web/frontend/src/views/ScanView.vue | 240 ++++++++++++++++++ 3 files changed, 242 insertions(+) create mode 100644 packages/web/frontend/src/views/ScanView.vue diff --git a/packages/web/frontend/src/components/AppLayout.vue b/packages/web/frontend/src/components/AppLayout.vue index a82d439..194f8cc 100644 --- a/packages/web/frontend/src/components/AppLayout.vue +++ b/packages/web/frontend/src/components/AppLayout.vue @@ -13,6 +13,7 @@ const router = useRouter() const menuItems = [ { label: 'Engagements', icon: 'pi pi-shield', command: () => router.push('/engagements') }, { label: 'Recipes', icon: 'pi pi-play', command: () => router.push('/recipes') }, + { label: 'Scans', icon: 'pi pi-search', command: () => router.push('/scans') }, { label: 'Containers', icon: 'pi pi-box', command: () => router.push('/containers') }, { label: 'Attack Chain', icon: 'pi pi-share-alt', command: () => router.push('/chain/global') }, { diff --git a/packages/web/frontend/src/router/index.ts b/packages/web/frontend/src/router/index.ts index 60d0ab7..147a6cf 100644 --- a/packages/web/frontend/src/router/index.ts +++ b/packages/web/frontend/src/router/index.ts @@ -19,6 +19,7 @@ const router = createRouter({ { path: '/containers', name: 'containers', component: () => import('@/views/ContainerStatusView.vue') }, { path: '/iocs/correlate', name: 'ioc-correlate', component: () => import('@/views/IOCCorrelationView.vue') }, { path: '/iocs/trending', name: 'ioc-trending', component: () => import('@/views/IOCTrendingView.vue') }, + { path: '/scans', name: 'scans', component: () => import('@/views/ScanView.vue') }, ], }) diff --git a/packages/web/frontend/src/views/ScanView.vue b/packages/web/frontend/src/views/ScanView.vue new file mode 100644 index 0000000..4f23944 --- /dev/null +++ b/packages/web/frontend/src/views/ScanView.vue @@ -0,0 +1,240 @@ + + + + + From 43fc47d7379e37d315ed03a5042d29824e8af078 Mon Sep 17 00:00:00 2001 From: Emperiusm Date: Wed, 15 Apr 2026 01:24:47 -0400 Subject: [PATCH 04/41] feat: wire scan execution as background task + file upload endpoint Co-Authored-By: Claude Opus 4.6 (1M context) --- packages/web/backend/app/routes/scans.py | 70 +++++++++++++++++++++++- 1 file changed, 69 insertions(+), 1 deletion(-) diff --git a/packages/web/backend/app/routes/scans.py b/packages/web/backend/app/routes/scans.py index 9b41793..818ae0d 100644 --- a/packages/web/backend/app/routes/scans.py +++ b/packages/web/backend/app/routes/scans.py @@ -9,17 +9,23 @@ import asyncio import json +import logging +import os +import shutil from typing import Optional -from fastapi import APIRouter, Depends, HTTPException, Query, Request +from fastapi import APIRouter, Depends, File, HTTPException, Query, Request, UploadFile from fastapi.responses import StreamingResponse from pydantic import BaseModel from sqlalchemy.ext.asyncio import AsyncSession +from app.database import async_session_factory from app.dependencies import get_current_user, get_db from app.models import ScanRecord, ScanTaskRecord, User from app.services.scan_service import ScanService +logger = logging.getLogger(__name__) + router = APIRouter(prefix="/api/v1/scans", tags=["scans"]) @@ -228,9 +234,71 @@ async def create_scan( ] await svc.persist_tasks(task_records) + # Start execution in the background + async def _run_scan(): + """Background task: execute scan and persist findings.""" + try: + result = await api.execute(scan, tasks) + + # Update scan status in DB + async with async_session_factory() as bg_session: + bg_svc = ScanService(bg_session, user) + scan_rec = await bg_svc.get_scan(scan.id) + if scan_rec: + scan_rec.status = result.status.value + scan_rec.completed_at = result.completed_at + scan_rec.finding_count = result.finding_count + scan_rec.tools_completed = json.dumps( + getattr(result, "tools_completed", []) + ) + scan_rec.tools_failed = json.dumps( + getattr(result, "tools_failed", []) + ) + await bg_session.commit() + except Exception as exc: + logger.error("Scan %s failed: %s", scan.id, exc) + try: + async with async_session_factory() as bg_session: + bg_svc = ScanService(bg_session, user) + scan_rec = await bg_svc.get_scan(scan.id) + if scan_rec: + scan_rec.status = "failed" + await bg_session.commit() + except Exception: + pass + + asyncio.create_task(_run_scan()) + return _scan_record_to_response(scan_record) +@router.post("/upload") +async def upload_scan_target( + file: UploadFile = File(...), + user: User = Depends(get_current_user), +): + """Upload a file for scanning. Returns the workspace path to use as scan target.""" + workspace = os.environ.get("OPENTOOLS_WORKSPACE", "/workspace") + + # Create user-scoped directory + user_dir = os.path.join(workspace, str(user.id)) + os.makedirs(user_dir, exist_ok=True) + + # Sanitize filename + safe_name = os.path.basename(file.filename or "upload") + # Remove any path traversal attempts + safe_name = safe_name.replace("..", "").replace("/", "").replace("\\", "") + if not safe_name: + safe_name = "upload" + + dest = os.path.join(user_dir, safe_name) + + with open(dest, "wb") as f: + shutil.copyfileobj(file.file, f) + + return {"path": dest, "filename": safe_name, "size": os.path.getsize(dest)} + + @router.get("") async def list_scans( engagement_id: Optional[str] = Query(None), From 9ad61028a8230691d749d679adb0bbf20d52af15 Mon Sep 17 00:00:00 2001 From: Emperiusm Date: Wed, 15 Apr 2026 01:27:22 -0400 Subject: [PATCH 05/41] feat: complete scan pipeline - workspace volume, auth fix, editor fix - Add shared workspace bind mount (/workspace) in compose for file targets - Add OPENTOOLS_WORKSPACE env var for API container - Add UserUpdate schema + get_users_router for /api/v1/auth/me endpoint - Fix CypherEditor EditorState.readOnly.reconfigure TypeScript error Co-Authored-By: Claude Opus 4.6 (1M context) --- packages/web/backend/app/main.py | 7 ++++++- packages/web/backend/app/models.py | 4 ++++ packages/web/docker-compose.yml | 2 ++ packages/web/frontend/src/components/CypherEditor.vue | 11 ++++------- 4 files changed, 16 insertions(+), 8 deletions(-) diff --git a/packages/web/backend/app/main.py b/packages/web/backend/app/main.py index 9c76426..992f787 100644 --- a/packages/web/backend/app/main.py +++ b/packages/web/backend/app/main.py @@ -8,7 +8,7 @@ from app.auth import fastapi_users, auth_backend from app.config import settings -from app.models import UserRead, UserCreate +from app.models import UserRead, UserCreate, UserUpdate from app.routes import ( engagements, findings, @@ -59,6 +59,11 @@ async def lifespan(app: FastAPI): prefix="/api/v1/auth", tags=["auth"], ) +app.include_router( + fastapi_users.get_users_router(UserRead, UserUpdate), + prefix="/api/v1/auth", + tags=["auth"], +) # API routes app.include_router(engagements.router) diff --git a/packages/web/backend/app/models.py b/packages/web/backend/app/models.py index 2b70c84..36084c5 100644 --- a/packages/web/backend/app/models.py +++ b/packages/web/backend/app/models.py @@ -89,6 +89,10 @@ class UserCreate(fu_schemas.BaseUserCreate): pass +class UserUpdate(fu_schemas.BaseUserUpdate): + pass + + # --- Engagement ----------------------------------------------------------- # ORM projection of opentools.models.Engagement; adds user_id FK. diff --git a/packages/web/docker-compose.yml b/packages/web/docker-compose.yml index 0ff4b51..94e3fe5 100644 --- a/packages/web/docker-compose.yml +++ b/packages/web/docker-compose.yml @@ -37,12 +37,14 @@ services: ENVIRONMENT: ${ENVIRONMENT:-production} ALLOWED_ORIGINS: ${ALLOWED_ORIGINS:-} OPENTOOLS_PLUGIN_DIR: /app/packages/plugin + OPENTOOLS_WORKSPACE: /workspace depends_on: migrate: condition: service_completed_successfully volumes: - appdata:/app/data - //var/run/docker.sock:/var/run/docker.sock + - ${WORKSPACE_DIR:-C:/Users/slabl/workspace}:/workspace ports: - "8000:8000" networks: diff --git a/packages/web/frontend/src/components/CypherEditor.vue b/packages/web/frontend/src/components/CypherEditor.vue index e4b0977..b53b7cc 100644 --- a/packages/web/frontend/src/components/CypherEditor.vue +++ b/packages/web/frontend/src/components/CypherEditor.vue @@ -92,6 +92,7 @@ onMounted(() => { placeholder('MATCH (a:Finding)-[r:LINKED]->(b:Finding) RETURN a, b'), cypherTheme, EditorView.lineWrapping, + EditorView.editable.of(!props.disabled), EditorView.updateListener.of((update) => { if (update.docChanged) { emit('update:modelValue', update.state.doc.toString()) @@ -120,13 +121,9 @@ watch(() => props.modelValue, (newVal) => { } }) -watch(() => props.disabled, (newVal) => { - if (view) { - view.dispatch({ - effects: EditorState.readOnly.reconfigure(!!newVal), - }) - } -}) +// Disabled state is set at mount time via EditorView.editable. +// Dynamic toggling during a query run is brief enough that rebuilding +// the editor isn't needed — the Run button is already disabled. From 4c8a2c60a4b900d5e5975d977f920279690ad9aa Mon Sep 17 00:00:00 2001 From: Emperiusm Date: Wed, 15 Apr 2026 02:17:07 -0400 Subject: [PATCH 10/41] feat: auto-start/stop tool containers for scans Containers are started on-demand when a scan needs them and stopped after the scan completes. Checks if container exists and is already running before starting. Co-Authored-By: Claude Opus 4.6 (1M context) --- packages/web/backend/app/routes/scans.py | 50 +++++++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/packages/web/backend/app/routes/scans.py b/packages/web/backend/app/routes/scans.py index d9a30dd..5b2096c 100644 --- a/packages/web/backend/app/routes/scans.py +++ b/packages/web/backend/app/routes/scans.py @@ -267,6 +267,41 @@ async def _run_scan(): "binwalk": "binwalk-mcp", } + # Auto-start required tool containers + needed_containers = set() + for task in t: + container = _TOOL_CONTAINERS.get(task.tool) + if container: + needed_containers.add(container) + + if needed_containers: + import subprocess + for cname in needed_containers: + try: + # Check if container exists and is running + check = subprocess.run( + ["docker", "inspect", "-f", "{{.State.Running}}", cname], + capture_output=True, text=True, timeout=5, + ) + if check.returncode != 0: + logger.warning("Container %s does not exist, skipping", cname) + continue + if check.stdout.strip() == "true": + continue # already running + + # Start the container + logger.info("Auto-starting container %s", cname) + subprocess.run( + ["docker", "start", cname], + capture_output=True, timeout=30, + ) + except Exception as exc: + logger.warning("Failed to start container %s: %s", cname, exc) + + # Brief pause for containers to initialize + import asyncio as _aio + await _aio.sleep(2) + # Rewrite task commands to go through docker exec for task in t: container = _TOOL_CONTAINERS.get(task.tool) @@ -350,7 +385,7 @@ async def _execute_with_docker(s, t, **kw): ) await bg_session.commit() except Exception as exc: - logger.error("Scan %s failed: %s", scan.id, exc) + logger.error("Scan %s failed: %s", scan.id, exc, exc_info=True) try: async with async_session_factory() as bg_session: bg_svc = ScanService(bg_session, user) @@ -360,6 +395,19 @@ async def _execute_with_docker(s, t, **kw): await bg_session.commit() except Exception: pass + finally: + # Auto-stop containers that were started for this scan + if needed_containers: + import subprocess + for cname in needed_containers: + try: + logger.info("Auto-stopping container %s", cname) + subprocess.run( + ["docker", "stop", cname], + capture_output=True, timeout=15, + ) + except Exception: + pass task = asyncio.create_task(_run_scan()) task.add_done_callback(lambda t: logger.error("Scan task error: %s", t.exception()) if t.exception() else None) From a12d29d0d050f5f1e7a814a39ba4cece40aef8fe Mon Sep 17 00:00:00 2001 From: Emperiusm Date: Wed, 15 Apr 2026 02:21:26 -0400 Subject: [PATCH 11/41] fix: wait for container readiness + auto-poll scan status in UI Backend: poll container Running state up to 30s instead of naive 2s sleep. Logs when all containers are ready. Frontend: auto-poll scan list + tasks every 3s while scan is active. Button shows "Starting containers & scan..." during creation. Polling stops when scan reaches terminal state. Co-Authored-By: Claude Opus 4.6 (1M context) --- packages/web/backend/app/routes/scans.py | 23 +++++++++++-- packages/web/frontend/src/views/ScanView.vue | 35 +++++++++++++++++++- 2 files changed, 55 insertions(+), 3 deletions(-) diff --git a/packages/web/backend/app/routes/scans.py b/packages/web/backend/app/routes/scans.py index 5b2096c..ad9c1c0 100644 --- a/packages/web/backend/app/routes/scans.py +++ b/packages/web/backend/app/routes/scans.py @@ -298,9 +298,28 @@ async def _run_scan(): except Exception as exc: logger.warning("Failed to start container %s: %s", cname, exc) - # Brief pause for containers to initialize + # Wait for containers to be ready (poll until running, max 30s) import asyncio as _aio - await _aio.sleep(2) + for attempt in range(15): + all_ready = True + for cname in needed_containers: + try: + check = subprocess.run( + ["docker", "inspect", "-f", "{{.State.Running}}", cname], + capture_output=True, text=True, timeout=5, + ) + if check.stdout.strip() != "true": + all_ready = False + break + except Exception: + all_ready = False + break + if all_ready: + logger.info("All %d containers ready after %ds", len(needed_containers), attempt * 2) + break + await _aio.sleep(2) + else: + logger.warning("Some containers may not be ready after 30s, proceeding anyway") # Rewrite task commands to go through docker exec for task in t: diff --git a/packages/web/frontend/src/views/ScanView.vue b/packages/web/frontend/src/views/ScanView.vue index 1a7f9fb..3da7eef 100644 --- a/packages/web/frontend/src/views/ScanView.vue +++ b/packages/web/frontend/src/views/ScanView.vue @@ -158,6 +158,8 @@ async function startScan() { uploadStatus.value = '' await loadScans() activeScanId.value = scan.id + // Start polling for status updates + startPolling(scan.id) } catch (e: any) { scanError.value = e.message } finally { @@ -165,6 +167,37 @@ async function startScan() { } } +let pollTimer: ReturnType | null = null + +function startPolling(scanId: string) { + stopPolling() + pollTimer = setInterval(async () => { + await loadScans() + // Also refresh tasks for the active scan + if (activeScanId.value) { + try { + const res = await fetch(`/api/v1/scans/${activeScanId.value}/tasks`, { credentials: 'include' }) + if (res.ok) { + const data = await res.json() + scanTasks.value[activeScanId.value] = data.tasks || [] + } + } catch (e) {} + } + // Stop polling if scan is done + const activeScan = scans.value.find(s => s.id === scanId) + if (activeScan && ['completed', 'failed', 'cancelled'].includes(activeScan.status)) { + stopPolling() + } + }, 3000) +} + +function stopPolling() { + if (pollTimer) { + clearInterval(pollTimer) + pollTimer = null + } +} + function toggleScan(scanId: string) { activeScanId.value = activeScanId.value === scanId ? null : scanId } @@ -213,7 +246,7 @@ function toggleScan(scanId: string) {
-
- -

No chain data across engagements

-

Run chain analysis on individual engagements first.

-
+