Merge branch 'dev' into mcp_additions

Author: TheTechromancer

.github/workflows/docker-tests.yml

@@ -12,7 +12,7 @@ concurrency:
 
 jobs:
   docker-test:
-    runs-on: self-hosted
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
       - name: Set up Python

.github/workflows/tests.yml

@@ -12,7 +12,7 @@ concurrency:
 
 jobs:
   test:
-    runs-on: self-hosted
+    runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
@@ -42,7 +42,7 @@ jobs:
           poetry run ruff format --check
       - name: Run tests
         run: |
-          NO_COLOR=true poetry run pytest --disable-warnings --log-cli-level=INFO --cov-report xml:cov.xml --cov=bbot_server .
+          NO_COLOR=true poetry run pytest --reruns 2 --disable-warnings --log-cli-level=INFO --cov-report xml:cov.xml --cov=bbot_server .
       - name: Upload Code Coverage
         uses: codecov/codecov-action@v3
         with:

README.md

@@ -136,45 +136,48 @@ To start a scan in BBOT server, you need to first create a **Preset** and **Targ
 
 1. Create Preset
 
-The preset defines which flags, modules, API keys, etc. will be used for the scan. It typically looks something like this:
+    The preset defines which flags, modules, API keys, etc. will be used for the scan. It typically looks something like this:
 
-**`my_preset.yml`**:
-```yaml
-include:
-  - subdomain-enum
-  - cloud-enum
-  - code-enum
+    **`my_preset.yml`**:
+    ```yaml
+    include:
+      - subdomain-enum
+      - cloud-enum
+      - code-enum
 
-modules:
-  - nuclei
+    modules:
+      - nuclei
 
-config:
-  - virustotal:
-    api_key: deadbeef
-```
+    config:
+      modules:
+        virustotal:
+          api_key: deadbeef
+    ```
 
-```bash
-# create a new scan preset
-bbctl scan preset create my_preset.yml
-```
+    ```bash
+    # create a new scan preset
+    bbctl scan preset create my_preset.yml
+    ```
+
+    For more guidance and examples on presets, check out the [bbot docs](https://www.blacklanternsecurity.com/bbot/Stable/scanning/presets/).
 
 2. Create Target
 
-A target defines what's in-scope for the scan. They can also be used when filtering assets.
+    A target defines what's in-scope for the scan. They can also be used when filtering assets.
 
-```bash
-# create a new scan target
-bbctl scan target create --seeds evilcorp.txt --name "my_target"
-```
+    ```bash
+    # create a new scan target
+    bbctl scan target create --seeds evilcorp.txt --name "my_target"
+    ```
 
 3. Start Scan
 
-Now that we've created a preset and target, we can start the scan:
+    Now that we've created a preset and target, we can start the scan:
 
-```bash
-# start the scan
-bbctl scan start --preset my_preset --target my_target --name "demonic_jimmy"
-```
+    ```bash
+    # start the scan
+    bbctl scan start --preset my_preset --target my_target --name "demonic_jimmy"
+    ```
 
 ## Monitor scan progress
 
@@ -325,6 +328,71 @@ The SSE server listens at `http://localhost:8807/v1/mcp/`
 
 After connecting your AI client to BBOT Server, you can ask it sensible questions like, "Use MCP to get all the bbot findings", "what are the top open ports?", "what else can you do with BBOT MCP?", etc.
 
+## As a Python Library
+
+You can interact fully with BBOT Server as a Python library. It supports either local or remote connections, and the interface to both is identical:
+
+### Asynchronous
+
+```python
+import asyncio
+from bbot_server import BBOTServer
+
+async def main():
+    # talk directly to local MongoDB + Redis
+    bbot_server = BBOTServer(interface="python")
+
+    # or to a remote BBOT Server instance (config must contain a valid API key)
+    bbot_server = BBOTServer(interface="http", url="http://bbot:8807/v1/")
+
+    # one-time setup
+    await bbot_server.setup()
+
+    hosts = await bbot_server.get_hosts()
+    print(f"hosts: {hosts}")
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+### Synchronous
+
+```python
+from bbot_server import BBOTServer
+
+if __name__ == "__main__":
+    # talk directly to local MongoDB + Redis
+    bbot_server = BBOTServer(interface="python", synchronous=True)
+
+    # or to a remote BBOT Server instance (config must contain a valid API key)
+    bbot_server = BBOTServer(interface="http", url="http://bbot:8807/v1/", synchronous=True)
+
+    # one-time setup
+    bbot_server.setup()
+
+    hosts = bbot_server.get_hosts()
+    print(f"hosts: {hosts}")
+```
+
+## Running Tests
+
+When running tests, first start MongoDB and Redis via Docker:
+
+```bash
+docker run --rm -p 27017:27017 mongo
+docker run --rm -p 6379:6379 redis
+```
+
+Then execute `pytest`:
+
+```bash
+# run all tests
+poetry run pytest -v
+
+# run specific tests
+poetry run pytest -v -k test_applet_scans
+```
+
 ## Screenshots
 
 *Tailing activities in real time*

bbot_server/api/mcp.py

@@ -1,12 +1,13 @@
 import logging
-from fastapi_mcp import FastApiMCP
 
 MCP_ENDPOINTS = {}
 
 log = logging.getLogger("bbot_server.api.mcp")
 
 
 def make_mcp_server(fastapi_app, config, mcp_endpoints=None):
+    from fastapi_mcp import FastApiMCP
+
     if mcp_endpoints is None:
         mcp_endpoints = MCP_ENDPOINTS
     log.debug(f"Creating MCP server with endpoints: {','.join(mcp_endpoints)}")

bbot_server/applets/_root.py

@@ -6,8 +6,6 @@
 class RootApplet(BaseApplet):
     name = "Root Applet"
 
-    attach_to = ""
-
     _nested = False
 
     _route_prefix = ""
@@ -28,26 +26,18 @@ async def setup(self):
         if self.is_native:
             # set up asset store, user store, and gridfs buckets
             if self.asset_store is None:
-                from bbot_server.store.user_store import UserStore
-                from bbot_server.store.asset_store import AssetStore
+                from bbot_server.store import UserStore, AssetStore, EventStore
 
                 self.asset_store = AssetStore()
                 await self.asset_store.setup()
-                self.asset_db = self.asset_store.db
-                self.asset_fs = self.asset_store.fs
 
                 self.user_store = UserStore()
                 await self.user_store.setup()
-                self.user_db = self.user_store.db
-                self.user_fs = self.user_store.fs
-
-            # set up event store
-            from bbot_server.event_store import EventStore
 
-            self.event_store = EventStore()
-            await self.event_store.setup()
+                self.event_store = EventStore()
+                await self.event_store.setup()
 
-            # set up NATS client
+            # set up message queue
             from bbot_server.message_queue import MessageQueue
 
             self.message_queue = MessageQueue()