Hackspace

.github/workflows/tests.yml

@@ -15,7 +15,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
     services:
       mongo:
         image: mongo

README.md

@@ -1,23 +1,26 @@
-![bbot-server](https://github.com/user-attachments/assets/3041001f-5135-4f69-a585-fea30341d803)
+![Image](https://github.com/user-attachments/assets/c63cc32c-8823-4b60-a990-12b544dd99ba)
 
-# BBOT Server
+# BBOT Server [BETA]
 
-<!-- ![bbot-server](https://github.com/user-attachments/assets/f97648ad-fc72-4fbf-8f85-3896b9f8f02c) -->
+***NOTE**: This is an early-access preview of **BBOT Server**. Basic features are documented below. Expect updates as development progresses, including blog posts and documentation describing the full range of features.*
 
-BBOT Server is a central database and multiplayer hub for all your [BBOT](https://github.com/blacklanternsecurity/bbot) scanning activities!
+---
 
+BBOT Server is a database and multiplayer hub for all your [BBOT](https://github.com/blacklanternsecurity/bbot) activities!
+
+- [x] **Asset Tracking and Alerting**
+    - [x] Get detailed history for each individual asset
+    - [ ] Instantly alert on new vulnerabilities, open ports, etc.
 - [x] **Scan Management**
     - [x] Kick off concurrent scans on remote servers
     - [x] Monitor scan progress, statistics
-- [x] **Asset Tracking and Alerting**
-    - [x] Detailed history for each individual asset
-    - [ ] Instant alerting on new vulnerabilities, open ports, etc.
 - [x] **Collaboration**
     - [x] Multi-user CLI
     - [x] Multiple concurrent scans
 - [x] **Advanced Querying**
     - [x] REST API
     - [x] Python SDK
+    - [x] Export to JSON/CSV
 
 ## Installation
 

bbot_server/agent/agent.py

@@ -8,7 +8,8 @@
 from typing import Callable, Any
 from urllib.parse import urlparse, urlunparse, urljoin
 
-from bbot import Scanner, Preset
+from bbot.scanner import Scanner, Preset
+from bbot.scanner.dispatcher import Dispatcher
 
 from bbot_server.config import BBOT_SERVER_CONFIG
 from bbot_server.errors import BBOTServerValueError
@@ -36,9 +37,6 @@ def command(fn: Callable) -> Callable:
     return fn
 
 
-from bbot.scanner.dispatcher import Dispatcher
-
-
 class AgentDispatcher(Dispatcher):
     def __init__(self, agent, *args, **kwargs):
         super().__init__(*args, **kwargs)

bbot_server/api/__init__.py

@@ -1,86 +0,0 @@
-from fastapi import FastAPI, Request
-from contextlib import asynccontextmanager
-from fastapi.responses import RedirectResponse, ORJSONResponse
-
-from bbot_server.errors import BBOTServerError
-
-# from fastapi.openapi.docs import get_swagger_ui_html, get_redoc_html
-
-app_kwargs = {
-    "title": "BBOT Server",
-    "description": "A central database for all your BBOT activities 🧡",
-    "debug": True,
-}
-
-
-def make_app(config=None):
-    from bbot_server.applets import BBOTServerRootApplet
-
-    app_root = BBOTServerRootApplet(config=config)
-    app_root._is_main_server = True
-
-    @asynccontextmanager
-    async def lifespan(app: FastAPI):
-        await app_root.setup()
-        yield
-        await app_root.cleanup()
-
-    app = FastAPI(
-        lifespan=lifespan,
-        prefix="/v1",
-        openapi_tags=app_root.tags_metadata,
-        default_response_class=ORJSONResponse,
-        **app_kwargs,
-    )
-    app.include_router(app_root.router)
-
-    @app.get("/", include_in_schema=False)
-    async def docs_redirect():
-        return RedirectResponse(url="docs")
-
-    @app.exception_handler(BBOTServerError)
-    async def exception_handler(request: Request, exc: Exception):
-        status_code = exc.http_status_code
-        error_message = str(exc)
-        message = error_message if error_message else exc.default_message
-        return ORJSONResponse(
-            status_code=status_code,
-            content={"error": message, "detail": getattr(exc, "detail", {})},
-        )
-
-    # favicon overrides - not working
-
-    # @app.get("/docs", include_in_schema=False)
-    # async def custom_swagger_ui_html():
-    #     return get_swagger_ui_html(
-    #         openapi_url=app.openapi_url,
-    #         title=app.title + " - Swagger UI",
-    #         swagger_favicon_url="https://www.blacklanternsecurity.com/bbot/Stable/bbot.png"
-    #     )
-
-    # @app.get("/redoc", include_in_schema=False)
-    # async def custom_redoc_html():
-    #     return get_redoc_html(
-    #         openapi_url=app.openapi_url,
-    #         title=app.title + " - ReDoc",
-    #         redoc_favicon_url="https://www.blacklanternsecurity.com/bbot/Stable/bbot.png"
-    #     )
-
-    return app, lifespan
-
-
-def make_server_app(config=None):
-    app, lifespan = make_app(config=config)
-
-    # includes the /v1 prefix
-    server_app = FastAPI(
-        lifespan=lifespan,
-        **app_kwargs,
-    )
-
-    @server_app.get("/", include_in_schema=False)
-    async def docs_redirect():
-        return RedirectResponse(url="/v1/docs")
-
-    server_app.mount("/v1", app)
-    return server_app

bbot_server/api/_fastapi.py

@@ -0,0 +1,85 @@
+from fastapi import FastAPI
+from contextlib import asynccontextmanager
+from fastapi.responses import RedirectResponse, ORJSONResponse
+
+from bbot_server.errors import BBOTServerError, handle_bbot_server_error
+
+# from fastapi.openapi.docs import get_swagger_ui_html, get_redoc_html
+
+app_kwargs = {
+    "title": "BBOT Server",
+    "description": "A central database for all your BBOT activities 🧡",
+    "debug": True,
+}
+
+
+def make_app(config=None):
+    from bbot_server.api.mcp import make_mcp_server
+    from bbot_server.applets import BBOTServerRootApplet
+
+    app_root = BBOTServerRootApplet(config=config)
+    app_root._is_main_server = True
+
+    @asynccontextmanager
+    async def lifespan(app: FastAPI):
+        await app_root.setup()
+        yield
+        await app_root.cleanup()
+
+    app = FastAPI(
+        lifespan=lifespan,
+        root_path="/v1",
+        openapi_tags=app_root.tags_metadata,
+        default_response_class=ORJSONResponse,
+        **app_kwargs,
+    )
+
+    app.include_router(app_root.router)
+
+    # add MCP server to the fastapi app
+    make_mcp_server(app, app_root.router)
+
+    @app.get("/", include_in_schema=False)
+    async def docs_redirect():
+        return RedirectResponse(url="docs")
+
+    # Register the exception handler
+    app.exception_handler(BBOTServerError)(handle_bbot_server_error)
+
+    # favicon overrides - not working
+
+    # @app.get("/docs", include_in_schema=False)
+    # async def custom_swagger_ui_html():
+    #     return get_swagger_ui_html(
+    #         openapi_url=app.openapi_url,
+    #         title=app.title + " - Swagger UI",
+    #         swagger_favicon_url="https://www.blacklanternsecurity.com/bbot/Stable/bbot.png"
+    #     )
+
+    # @app.get("/redoc", include_in_schema=False)
+    # async def custom_redoc_html():
+    #     return get_redoc_html(
+    #         openapi_url=app.openapi_url,
+    #         title=app.title + " - ReDoc",
+    #         redoc_favicon_url="https://www.blacklanternsecurity.com/bbot/Stable/bbot.png"
+    #     )
+
+    return app, lifespan
+
+
+def make_server_app(config=None):
+    app, lifespan = make_app(config=config)
+
+    # includes the /v1 prefix
+    server_app = FastAPI(
+        lifespan=lifespan,
+        **app_kwargs,
+    )
+
+    @server_app.get("/", include_in_schema=False)
+    async def docs_redirect():
+        return RedirectResponse(url="/v1/docs")
+
+    server_app.mount("/v1", app)
+
+    return server_app

bbot_server/api/app.py

@@ -1,3 +1,3 @@
-from . import make_server_app
+from ._fastapi import make_server_app
 
 server_app = make_server_app()

bbot_server/api/mcp.py

@@ -0,0 +1,14 @@
+import logging
+from fastapi_mcp import FastApiMCP
+
+MCP_ENDPOINTS = {}
+
+log = logging.getLogger("bbot_server.api.mcp")
+
+
+def make_mcp_server(fastapi_app, config, mcp_endpoints=None):
+    if mcp_endpoints is None:
+        mcp_endpoints = MCP_ENDPOINTS
+    log.debug(f"Creating MCP server with endpoints: {','.join(mcp_endpoints)}")
+    mcp = FastApiMCP(fastapi_app, include_operations=list(mcp_endpoints))
+    mcp.mount()

bbot_server/applets/_base.py

@@ -2,16 +2,18 @@
 import asyncio
 import inspect
 import logging
+from fastapi import APIRouter
 from omegaconf import OmegaConf
 from typing import Annotated, Any  # noqa
 from functools import cached_property
 from pydantic import BaseModel, Field  # noqa
 from pymongo import WriteConcern, ASCENDING
-from fastapi import APIRouter
+from pymongo.errors import OperationFailure
 
 from bbot.models.pydantic import Event
-from bbot_server.utils.misc import utc_now
+from bbot.core.helpers import misc as bbot_misc
 from bbot_server.applets._routing import ROUTE_TYPES
+from bbot_server.utils import misc as bbot_server_misc
 from bbot_server.models.activity_models import Activity
 
 word_regex = re.compile(r"\W+")
@@ -82,6 +84,10 @@ class BaseApplet:
     # optionally override route prefix
     _route_prefix = None
 
+    # BBOT helpers
+    helpers = bbot_server_misc
+    bbot_helpers = bbot_misc
+
     def __init__(self, parent=None):
         # TODO: we need to collect all the child applets before doing any fastapi setup
 
@@ -212,14 +218,70 @@ async def _native_setup(self):
             await self.setup()
 
     async def build_indexes(self, model):
+        """
+        Builds MongoDB indexes for the given model.
+
+        Pydantic annotations on the model determine the type of indexes:
+
+        - indexed - basic ascending index
+        - indexed-text - text index (for quick searching of partial strings - ideal for technology/vuln descriptions etc.)
+        - indexed-compound:field1,field2 - compound index on multiple fields (useful for preventing duplicates)
+        """
         if not model:
             return
         for fieldname, field in model.model_fields.items():
+            unique = "unique" in field.metadata
+            # normal indexes
             if "indexed" in field.metadata:
-                unique = "unique" in field.metadata
-                await self.collection.create_index([(fieldname, ASCENDING)], unique=unique)
-            elif "indexed_text" in field.metadata:
-                await self.collection.create_index([(fieldname, "text")])
+                index = [(fieldname, ASCENDING)]
+                self.log.debug(f"Creating index: {index}")
+                await self.collection.create_index(index, unique=unique)
+            # text indexes
+            if "indexed-text" in field.metadata:
+                index = [(fieldname, "text")]
+                self.log.debug(f"Creating text index: {index}")
+                try:
+                    await self.collection.create_index(index)
+                except OperationFailure as e:
+                    # the only purpose of the below code is to add fields to an existing text index
+                    # if there's an easier way to do this, please delete this mess
+                    # ideally we should not have to delete and recreate the index
+                    if "index already exists" in str(e):
+                        # Get existing text index
+                        existing_indexes = await self.collection.list_indexes().to_list()
+                        text_index = next((idx for idx in existing_indexes if "text" in idx["key"].values()), None)
+                        if text_index:
+                            self.log.debug(f"Found existing text index: {text_index}")
+                            # Check if the field is already in the index
+                            existing_fields = text_index["weights"].keys()
+                            if fieldname in existing_fields:
+                                self.log.debug(f"Field {fieldname} already exists in text index, skipping")
+                                continue
+
+                            # Drop the existing text index
+                            index_name = text_index["name"]
+                            self.log.debug(f"Dropping existing text index {index_name}")
+                            await self.collection.drop_index(index_name)
+
+                            # Create new index with all fields from weights
+                            existing_fields = [(f, "text") for f in text_index["weights"].keys()]
+                            self.log.debug(f"Existing fields from weights: {existing_fields}")
+                            new_index = existing_fields + [(fieldname, "text")]
+                            self.log.debug(f"Creating new text index with fields: {new_index}")
+                            await self.collection.create_index(new_index)
+                        else:
+                            self.log.error(f"Failed to find existing text index: {e}")
+                    else:
+                        self.log.error(f"Error creating text index: {e}")
+            # compound indexes
+            for metadata in field.metadata:
+                if isinstance(metadata, str) and metadata.startswith("indexed-compound:"):
+                    # create a compound index
+                    fields = metadata.split(":")[-1].split(",")
+                    fields = [fieldname] + fields
+                    index = [(fieldname, ASCENDING) for fieldname in fields]
+                    self.log.debug(f"Creating compound index: {index}")
+                    await self.collection.create_index(index, unique=unique)
 
     async def register_watchdog_tasks(self, broker):
         # register watchdog tasks
@@ -344,6 +406,9 @@ def watches_activity(self, activity_type):
             return True
         return activity_type in self.watched_activities
 
+    async def compute_stats(self, asset, stats):
+        pass
+
     @property
     def is_main_server(self):
         return self.root._is_main_server
@@ -442,9 +507,6 @@ def is_native(self):
         """
         return self.interface_type == "python"
 
-    def utc_now(self):
-        return utc_now()
-
     def __getattr__(self, name):
         # try getting the attribute from all the child applets
         for child_applet in getattr(self, "child_applets", []):