Merge pull request #41 from blacklanternsecurity/dev

Dev -> Stable

Author: TheTechromancer

.github/workflows/docker-tests.yml

@@ -11,7 +11,7 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  test:
+  docker-test:
     runs-on: self-hosted
     steps:
       - uses: actions/checkout@v4

README.md

@@ -29,13 +29,13 @@ BBOT Server is a database and multiplayer hub for all your [BBOT](https://github
 
 ```bash
 # clone the repo and cd into it
-git clone [email protected]:blacklanternsecurity/bbot-server.git && cd bbot_server
+git clone [email protected]:blacklanternsecurity/bbot-server.git && cd bbot-server
 
 # Install in editable mode
 pipx install -e .
 ```
 
-Note: to update to the latest version, run `git pull` in the `bbot_server` directory.
+Note: to update to the latest version, run `git pull` in the `bbot-server` directory.
 
 ## Start the server
 
@@ -63,7 +63,7 @@ The first time you start BBOT Server, an API key will be auto generated and put
 
 # list of API keys to be considered valid
 api_keys:
-  - 4aa8b3c2-9b4d-4208-890c-4ce9ad3b4710
+  - deadbeef-9b4d-4208-890c-4ce9ad3b4710
 ```
 
 The `api_keys` value in `config.yml` is used by both the server (as a database of valid API keys), and by the client (it will pick one from the list and use it). Normally it just works and you don't have to mess with it. But to access BBOT Server remotely, you'll need to copy the API key from the server onto your local system, along with its URL:
@@ -77,6 +77,8 @@ api_keys:
 
 This tells `bbctl` (the client) where the server is, and gives it the means to authenticate.
 
+To utilise the API key and interact with the BBOT Server via the HTTP API, set the `X-API-Key` HTTP header to the value of a valid API key.
+
 ### Adding and Revoking API Keys
 
 API keys can be added and removed if you are on the server machine:
@@ -94,13 +96,29 @@ bbctl server apikey delete deadbeef-9b4d-4208-890c-4ce9ad3b4710
 
 ## Send a BBOT Scan to the Server
 
-You can output a BBOT scan directly to BBOT server:
+You can output a BBOT scan directly to BBOT server with the following preset:
+
+```yaml
+# bbot-server.yml
+
+output_modules:
+  - http
+
+config:
+  modules:
+    http:
+      # URL of BBOT Server
+      url: http://localhost:8807/v1/events/
+      # API Key header
+      headers:
+        x-api-key: deadbeef-9b4d-4208-890c-4ce9ad3b4710
+```
 
 Note that this requires BBOT 3.0 or later (install with `pipx install git+https://github.com/blacklanternsecurity/[email protected]`)
 
 ```bash
 # Start a BBOT scan, sending output to BBOT server
-bbot -t evilcorp.com -p subdomain-enum -om http -c modules.http.url=http://localhost:8807/v1/events/
+bbot -t evilcorp.com -p subdomain-enum ./bbot-server.yml
 ```
 
 ## Ingest events from past BBOT scans
@@ -200,10 +218,10 @@ You can list targets like so:
 
 ```bash
 # List targets
-bbctl target list
+bbctl scan target list
 
 # Create a new target
-bbctl target create --seeds seeds.txt --blacklist blacklist.txt --name custom_target
+bbctl scan target create --seeds seeds.txt --blacklist blacklist.txt --name custom_target
 
 # List only the assets that match your new target
 bbctl asset list --target custom_target
@@ -331,4 +349,6 @@ After connecting your AI client to BBOT Server, you can ask it sensible question
 
 *REST API*
 
+Connect to the default URL at [http://localhost:8807](http://localhost:8807/) to view and use the interactive API documentation.
+
 ![rest-api](https://github.com/user-attachments/assets/567bd266-b047-4005-bc0b-22d5bfd2a12b)

bbot_server/agent/__init__.py

@@ -1,9 +1,10 @@
 import logging
 from contextlib import asynccontextmanager
 from fastapi.openapi.utils import get_openapi
-from fastapi import FastAPI, HTTPException, Depends, Request, WebSocket
 from fastapi.responses import RedirectResponse, ORJSONResponse
+from fastapi import FastAPI, HTTPException, Depends, Request, WebSocket
 
+from bbot_server import modules  # noqa: F401
 import bbot_server.config as bbcfg
 from bbot_server.errors import BBOTServerError, handle_bbot_server_error
 

bbot_server/api/_fastapi.py

@@ -1,20 +1,13 @@
 import bbot_server.config as bbcfg
 
-from bbot_server.applets._base import BaseApplet
-
-# applet imports
-from bbot_server.applets.stats import StatsApplet
-from bbot_server.applets.assets import AssetsApplet
-from bbot_server.applets.events import EventsApplet
-from bbot_server.applets.scans import ScansApplet
-from bbot_server.applets.activity import ActivityApplet
+from bbot_server.applets.base import BaseApplet
 
 
 class RootApplet(BaseApplet):
-    include_apps = [AssetsApplet, EventsApplet, ScansApplet, ActivityApplet, StatsApplet]
-
     name = "Root Applet"
 
+    attach_to = ""
+
     _nested = False
 
     _route_prefix = ""

bbot_server/applets/_root.py

@@ -2,6 +2,7 @@
 import asyncio
 import inspect
 import logging
+import traceback
 from fastapi import APIRouter
 from omegaconf import OmegaConf
 from typing import Annotated, Any  # noqa
@@ -10,12 +11,14 @@
 from pymongo import WriteConcern, ASCENDING
 from pymongo.errors import OperationFailure
 
+from bbot_server.assets import Asset
 from bbot.models.pydantic import Event
+from bbot_server.modules import API_MODULES
 from bbot_server.errors import BBOTServerError
 from bbot.core.helpers import misc as bbot_misc
 from bbot_server.applets._routing import ROUTE_TYPES
 from bbot_server.utils import misc as bbot_server_misc
-from bbot_server.models.activity_models import Activity
+from bbot_server.modules.activity.activity_models import Activity
 
 word_regex = re.compile(r"\W+")
 
@@ -75,8 +78,8 @@ class BaseApplet:
     # the pydantic model this applet uses
     model = None
 
-    # optionally you can include other applets
-    include_apps = []
+    # which other applet should include this one
+    attach_to = ""
 
     # whether to nest this applet under its parent
     # this is typically true for every applet except the root
@@ -85,6 +88,16 @@ class BaseApplet:
     # optionally override route prefix
     _route_prefix = None
 
+    # priority of this applet's handle_activity method, between 1 and 5, inclusive
+    # higher numbers are higher priority
+    # this is used to determine the order in which applets' .handle_activity methods are called
+    _activity_priority = 3
+
+    # priority of this applet's handle_event method, between 1 and 5, inclusive
+    # higher numbers are higher priority
+    # this is used to determine the order in which applets' .handle_event methods are called
+    _event_priority = 3
+
     # BBOT helpers
     helpers = bbot_server_misc
     bbot_helpers = bbot_misc
@@ -113,14 +126,13 @@ def __init__(self, parent=None):
 
         self._add_custom_routes()
 
-        for app in self.include_apps:
+        applets_to_include = API_MODULES.get(self.name_lowercase, {})
+        for included_app_name in sorted(applets_to_include):
             try:
-                self.include_app(app)
+                self.include_app(applets_to_include[included_app_name])
             except Exception as e:
-                self.log.error(f"Error including app {app}: {e}")
-                import traceback
-
-                traceback.print_exc()
+                self.log.error(f"Error including app {included_app_name}: {e}")
+                self.log.error(traceback.format_exc())
 
         self._setup_finished = False
 
@@ -346,7 +358,7 @@ async def _cleanup(self):
     async def cleanup(self):
         pass
 
-    async def handle_activity(self, activity: Activity):
+    async def handle_activity(self, activity: Activity, asset: Asset = None):
         pass
 
     async def handle_event(self, event: Event, asset=None):
@@ -367,7 +379,8 @@ async def _emit_activity(self, activity: Activity):
         await self.root.message_queue.publish_asset(activity)
 
     def include_app(self, app_class):
-        self.log.debug(f"{self.__class__.__name__} including {app_class.__name__}")
+        self.log.debug(f"{self.name_lowercase} including applet {app_class.name_lowercase}")
+
         # instantiate it
         applet = app_class(parent=self)
         # set it as an attribute on self
@@ -401,10 +414,17 @@ async def _put_obj(self, obj):
             {"host": obj.host, "type": self.model.__name__}, {"$set": obj.model_dump()}, upsert=True
         )
 
-    @cached_property
-    def name_lowercase(self):
-        # Replace non-alphanumeric characters with an underscore
-        return word_regex.sub("_", self.name.lower())
+    class NameLowercaseDescriptor:
+        def __init__(self):
+            self._cache = {}
+
+        def __get__(self, obj, owner):
+            cache_key = owner if obj is None else obj
+            if cache_key not in self._cache:
+                self._cache[cache_key] = word_regex.sub("_", cache_key.name.lower())
+            return self._cache[cache_key]
+
+    name_lowercase = NameLowercaseDescriptor()
 
     def all_child_applets(self, include_self=False):
         applets = []