diff --git a/docs/reference/cli-reference.md b/docs/agent/cli-reference.md
similarity index 94%
rename from docs/reference/cli-reference.md
rename to docs/agent/cli-reference.md
index 17f9a1e..fd979cb 100644
--- a/docs/reference/cli-reference.md
+++ b/docs/agent/cli-reference.md
@@ -16,7 +16,7 @@ The CLI is organised into the following command groups:
 
 ## Global Options
 
-All commands support the following verbosity options:
+The following verbosity options are supported by the `watch` command:
 
 | Option | Short | Type | Default | Description |
 |--------|-------|------|---------|-------------|
@@ -27,6 +27,8 @@ All commands support the following verbosity options:
 - **1 (-v)**: INFO level - General information and warnings
 - **2 (-vv)**: DEBUG level - Detailed debugging information
 
+> **Note**: The `-v`/`--verbose` flag currently only works with the `watch` command. Parse and validate commands use default logging levels. This is a known limitation.
+
 ## Parse Commands
 
 Parse commands extract and process data from EPU files without persisting to the backend API. These are primarily used for development, debugging, and data validation purposes.
@@ -44,7 +46,6 @@ python -m smartem_agent parse dir [OPTIONS] EPU_OUTPUT_DIR
 | Parameter | Type | Required | Description |
 |-----------|------|----------|-------------|
 | `epu_output_dir` | str | Yes | Path to EPU output directory containing multiple grid directories |
-| `--verbose` | count | No | Verbosity level (see [Global Options](#global-options)) |
 
 **Example:**
 ```bash
@@ -69,7 +70,6 @@ python -m smartem_agent parse grid [OPTIONS] GRID_DATA_DIR
 | Parameter | Type | Required | Description |
 |-----------|------|----------|-------------|
 | `grid_data_dir` | str | Yes | Path to individual grid data directory |
-| `--verbose` | count | No | Verbosity level (see [Global Options](#global-options)) |
 
 **Example:**
 ```bash
@@ -94,7 +94,6 @@ python -m smartem_agent parse session [OPTIONS] PATH
 | Parameter | Type | Required | Description |
 |-----------|------|----------|-------------|
 | `path` | str | Yes | Path to EPU session manifest file |
-| `--verbose` | count | No | Verbosity level (see [Global Options](#global-options)) |
 
 **Example:**
 ```bash
@@ -116,7 +115,6 @@ python -m smartem_agent parse atlas [OPTIONS] PATH
 | Parameter | Type | Required | Description |
 |-----------|------|----------|-------------|
 | `path` | str | Yes | Path to atlas manifest file |
-| `--verbose` | count | No | Verbosity level (see [Global Options](#global-options)) |
 
 **Example:**
 ```bash
@@ -138,7 +136,6 @@ python -m smartem_agent parse gridsquare-metadata [OPTIONS] PATH
 | Parameter | Type | Required | Description |
 |-----------|------|----------|-------------|
 | `path` | str | Yes | Path to grid square metadata file |
-| `--verbose` | count | No | Verbosity level (see [Global Options](#global-options)) |
 
 **Example:**
 ```bash
@@ -160,7 +157,6 @@ python -m smartem_agent parse gridsquare [OPTIONS] PATH
 | Parameter | Type | Required | Description |
 |-----------|------|----------|-------------|
 | `path` | str | Yes | Path to grid square manifest file |
-| `--verbose` | count | No | Verbosity level (see [Global Options](#global-options)) |
 
 **Example:**
 ```bash
@@ -182,7 +178,6 @@ python -m smartem_agent parse foilhole [OPTIONS] PATH
 | Parameter | Type | Required | Description |
 |-----------|------|----------|-------------|
 | `path` | str | Yes | Path to foil hole manifest file |
-| `--verbose` | count | No | Verbosity level (see [Global Options](#global-options)) |
 
 **Example:**
 ```bash
@@ -204,7 +199,6 @@ python -m smartem_agent parse micrograph [OPTIONS] PATH
 | Parameter | Type | Required | Description |
 |-----------|------|----------|-------------|
 | `path` | str | Yes | Path to micrograph manifest file |
-| `--verbose` | count | No | Verbosity level (see [Global Options](#global-options)) |
 
 **Example:**
 ```bash
@@ -226,7 +220,6 @@ python -m smartem_agent validate [OPTIONS] PATH
 | Parameter | Type | Required | Default | Description |
 |-----------|------|----------|---------|-------------|
 | `path` | str | Yes | - | Path to EPU project directory to validate |
-| `--verbose` | count | No | 0 | Verbosity level (see [Global Options](#global-options)) |
 
 **Exit Codes:**
 - **0**: Directory structure is valid
@@ -487,4 +480,4 @@ The watch command requires a compatible SmartEM backend API with the following e
 - Consider `--heartbeat-interval` tuning based on network stability
 - Use `--dry-run` for local testing and development
 
-This CLI reference provides comprehensive coverage of all SmartEM Agent command-line functionality. For additional help with specific use cases or troubleshooting, refer to the [CLI Troubleshooting Guide](troubleshoot-cli.md).
+This CLI reference provides comprehensive coverage of all SmartEM Agent command-line functionality. For additional help with specific use cases or troubleshooting, refer to the [CLI Troubleshooting Guide](troubleshooting.md).
diff --git a/docs/how-to/run-agent.md b/docs/agent/deployment.md
similarity index 99%
rename from docs/how-to/run-agent.md
rename to docs/agent/deployment.md
index 6c5c571..225fa95 100644
--- a/docs/how-to/run-agent.md
+++ b/docs/agent/deployment.md
@@ -27,7 +27,7 @@ The agent operates in several modes depending on the timing of EPU data acquisit
 
 ## Quick Start
 
-For comprehensive parameter documentation, see the [CLI Reference](../reference/cli-reference.md). For troubleshooting, see the [CLI Troubleshooting Guide](troubleshoot-cli.md).
+For comprehensive parameter documentation, see the [CLI Reference](cli-reference.md). For troubleshooting, see the [CLI Troubleshooting Guide](troubleshooting.md).
 
 ### Basic Directory Monitoring
 
diff --git a/docs/agent/index.md b/docs/agent/index.md
new file mode 100644
index 0000000..16d7b16
--- /dev/null
+++ b/docs/agent/index.md
@@ -0,0 +1,22 @@
+# Agent
+
+Documentation for the SmartEM Agent - the EPU filesystem watcher that runs on Windows workstations near the microscope.
+
+```{toctree}
+:maxdepth: 1
+
+cli-reference
+deployment
+troubleshooting
+```
+
+## Topics
+
+### Usage
+
+- [CLI Reference](cli-reference.md) - Command-line interface documentation and options
+- [Troubleshooting](troubleshooting.md) - Common issues and solutions
+
+### Deployment
+
+- [Agent Deployment](deployment.md) - Running the agent on Windows workstations
diff --git a/docs/how-to/troubleshoot-cli.md b/docs/agent/troubleshooting.md
similarity index 99%
rename from docs/how-to/troubleshoot-cli.md
rename to docs/agent/troubleshooting.md
index 1a2f568..2d6db26 100644
--- a/docs/how-to/troubleshoot-cli.md
+++ b/docs/agent/troubleshooting.md
@@ -1,6 +1,6 @@
 # SmartEM CLI Troubleshooting Guide
 
-This guide provides solutions for common issues encountered when using the SmartEM Agent command-line interface. For comprehensive parameter documentation, see the [CLI Reference](../reference/cli-reference.md).
+This guide provides solutions for common issues encountered when using the SmartEM Agent command-line interface. For comprehensive parameter documentation, see the [CLI Reference](cli-reference.md).
 
 ## Quick Diagnostics
 
diff --git a/docs/architecture/index.md b/docs/architecture/index.md
new file mode 100644
index 0000000..4fff2ab
--- /dev/null
+++ b/docs/architecture/index.md
@@ -0,0 +1,18 @@
+# Architecture
+
+System design documentation and architectural decision records.
+
+## Decision Records
+
+Architectural Decision Records (ADRs) document significant technical decisions.
+
+- [Decision Records](../decision-records/index.md) - Browse all ADRs
+
+## System Design
+
+Architecture documentation is maintained in the main smartem-decisions repository:
+
+- System overview and component interactions
+- Backend-Agent communication protocols (SSE, REST)
+- Data flow and message queue patterns
+- Database schema design
diff --git a/docs/athena/index.md b/docs/athena/index.md
new file mode 100644
index 0000000..813d8cf
--- /dev/null
+++ b/docs/athena/index.md
@@ -0,0 +1,30 @@
+# Athena Integration
+
+Documentation for ThermoFisher Athena API integration.
+
+## API Reference
+
+The Athena Decision Service API specification is available in the API documentation:
+
+- [Athena API Docs](../api/athena/index.html) - Interactive Swagger UI
+- [API Documentation Guide](../backend/api-documentation.md) - How to use the interactive documentation
+
+## Mock Server
+
+For local development without access to a real Athena service:
+
+```bash
+# Install mock dependencies
+pip install -e ".[mock]"
+
+# Start the Athena mock server
+python -c "
+from athena_api.mock import AthenaAPIServer
+server = AthenaAPIServer()
+server.run()
+"
+
+# Visit http://localhost:8000/docs
+```
+
+See the [API Documentation](../backend/api-documentation.md#mock-server-for-development) guide for more details.
diff --git a/docs/how-to/use-api-documentation.md b/docs/backend/api-documentation.md
similarity index 100%
rename from docs/how-to/use-api-documentation.md
rename to docs/backend/api-documentation.md
diff --git a/docs/how-to/run-backend.md b/docs/backend/api-server.md
similarity index 93%
rename from docs/how-to/run-backend.md
rename to docs/backend/api-server.md
index 5a1223d..b19516c 100644
--- a/docs/how-to/run-backend.md
+++ b/docs/backend/api-server.md
@@ -10,10 +10,10 @@ The core backend service providing HTTP API, database operations, and message qu
 
 # launch RabbitMQ worker (consumer)
 python -m smartem_backend.consumer              # ERROR level (default)
-python -m smartem_backend.consumer -v           # INFO level  
+python -m smartem_backend.consumer -v           # INFO level
 python -m smartem_backend.consumer -vv          # DEBUG level
 
-# simulating an system event: 
+# simulating an system event:
 python -m smartem_backend.simulate_msg --help # to see a list of options
 ./tools/simulate-messages.sh # run a simulation, triggering system events in sequence
 
diff --git a/docs/how-to/database-migrations.md b/docs/backend/database.md
similarity index 89%
rename from docs/how-to/database-migrations.md
rename to docs/backend/database.md
index 33dd5ee..f5a1600 100644
--- a/docs/how-to/database-migrations.md
+++ b/docs/backend/database.md
@@ -33,10 +33,10 @@ python -m alembic upgrade head
 
 This will:
 1. Create the Alembic version tracking table
-2. Apply migration 001: Create user preferences table
-3. Apply baseline migration (6e6302f1ccb6): Create all core SmartEM schema tables (acquisition, grid, gridsquare, micrograph, foilhole, atlas, quality prediction models, etc.)
-4. Apply migration 002: Add performance indexes for acquisition datetime queries
-5. Apply remaining migrations including agent communication tables
+2. Apply migration 001: Create core SmartEM schema baseline (acquisition, grid, gridsquare, micrograph, foilhole, atlas, etc.)
+3. Apply migration 002: Add performance indexes for acquisition datetime queries
+4. Apply migration 003: Add prediction model tables
+5. Apply remaining migrations for quality metrics, training tables, and schema fixes
 
 ### Running Migrations
 
@@ -140,7 +140,7 @@ def upgrade() -> None:
         column('preference_key', sa.String),
         column('preference_value', sa.JSON)
     )
-    
+
     op.bulk_insert(user_preferences, [
         {
             'user_id': 'system',
@@ -268,11 +268,17 @@ Ensure your environment variables are set:
 │   ├── env.py                              # Migration environment
 │   ├── script.py.mako                      # Migration template
 │   └── versions/                           # Individual migration files
-│       ├── 001_add_user_preferences.py
-│       ├── 002_add_acquisition_time_index.py
-│       └── 003_seed_system_config.py
+│       ├── 2025_09_18_1042-001_create_core_smartem_schema_baseline.py
+│       ├── 2025_01_11_1431-002_add_acquisition_time_index.py
+│       ├── 2025_09_18_1630-003_add_scifi_robot_prediction_models.py
+│       └── ...                              # Additional migrations
 ```
 
+**Migration file naming convention**: `YYYY_MM_DD_HHMM-NNN_description.py` where:
+- `YYYY_MM_DD_HHMM` is the creation timestamp
+- `NNN` is the sequential migration number
+- `description` describes the change
+
 ## Further Reading
 
 - [Alembic Documentation](https://alembic.sqlalchemy.org/)
diff --git a/docs/how-to/use-core-http-api-client.md b/docs/backend/http-api-client.md
similarity index 92%
rename from docs/how-to/use-core-http-api-client.md
rename to docs/backend/http-api-client.md
index e685414..1d399c8 100644
--- a/docs/how-to/use-core-http-api-client.md
+++ b/docs/backend/http-api-client.md
@@ -71,17 +71,16 @@ acquisitions = await client.aget_acquisitions()
 ### Creating an Acquisition
 
 ```python
-from smartem_agent.model.schemas import EpuSessionData
+from smartem_common.schemas import AcquisitionData
 from datetime import datetime
 
-# Create an acquisition from EPU session data
-session = EpuSessionData(
-    id="my-acquisition-id",
+# Create an acquisition from acquisition data
+acquisition_data = AcquisitionData(
     name="My Acquisition",
     start_time=datetime.now(),
     storage_path="/path/to/storage"
 )
-response = client.create_acquisition(session)
+response = client.create_acquisition(acquisition_data)
 print(f"Created acquisition with ID: {response.id}")
 
 # Alternatively, create directly with an API request model
@@ -138,29 +137,28 @@ The client supports the full hierarchy of entities:
 Here's an example of creating entities at each level:
 
 ```python
+from smartem_common.schemas import (
+    AcquisitionData, GridData, GridSquareData, FoilHoleData,
+    MicrographData, MicrographManifest
+)
+
 # Create an acquisition
-acquisition = client.create_acquisition(EpuSessionData(id="acq-1", name="Test Acquisition"))
+acquisition_data = AcquisitionData(name="Test Acquisition")
+acquisition = client.create_acquisition(acquisition_data)
 
 # Create a grid for the acquisition
-from smartem_agent.model.schemas import GridData
-
 grid = GridData(data_dir="/path/to/grid")
 grid_response = client.create_acquisition_grid(acquisition.id, grid)
 
 # Create a grid square for the grid
-from smartem_agent.model.schemas import GridSquareData
-
 gridsquare = GridSquareData(id="gs-1", data_dir="/path/to/gridsquare")
 gridsquare_response = client.create_grid_gridsquare(grid_response.id, gridsquare)
 
 # Create a foil hole for the grid square
-from smartem_agent.model.schemas import FoilHoleData
-
 foilhole = FoilHoleData(id="fh-1", gridsquare_id=gridsquare.id)
 foilhole_response = client.create_gridsquare_foilhole(gridsquare_response.id, foilhole)
 
 # Create a micrograph for the foil hole
-from smartem_agent.model.schemas import MicrographData, MicrographManifest
 
 manifest = MicrographManifest(
     unique_id="mic-1",
diff --git a/docs/backend/index.md b/docs/backend/index.md
new file mode 100644
index 0000000..0348bfa
--- /dev/null
+++ b/docs/backend/index.md
@@ -0,0 +1,24 @@
+# Backend
+
+Documentation for the SmartEM backend services: API server, database, and message queue.
+
+```{toctree}
+:maxdepth: 1
+
+api-server
+database
+api-documentation
+http-api-client
+```
+
+## Topics
+
+### Running the Backend
+
+- [API Server](api-server.md) - Running and configuring the FastAPI server locally
+- [Database](database.md) - PostgreSQL setup, Alembic migrations, schema management
+
+### API Usage
+
+- [API Documentation](api-documentation.md) - Interactive Swagger UI documentation for SmartEM and Athena APIs
+- [HTTP API Client](http-api-client.md) - Python client for programmatic API access
diff --git a/docs/explanations/backend-agent-communication-system-design.md b/docs/decision-records/backend-agent-communication-system-design.md
similarity index 100%
rename from docs/explanations/backend-agent-communication-system-design.md
rename to docs/decision-records/backend-agent-communication-system-design.md
diff --git a/docs/explanations/decisions.md b/docs/decision-records/decisions.md
similarity index 100%
rename from docs/explanations/decisions.md
rename to docs/decision-records/decisions.md
diff --git a/docs/explanations/decisions/0001-record-architecture-decisions.md b/docs/decision-records/decisions/0001-record-architecture-decisions.md
similarity index 100%
rename from docs/explanations/decisions/0001-record-architecture-decisions.md
rename to docs/decision-records/decisions/0001-record-architecture-decisions.md
diff --git a/docs/explanations/decisions/0002-switched-to-python-copier-template.md b/docs/decision-records/decisions/0002-switched-to-python-copier-template.md
similarity index 100%
rename from docs/explanations/decisions/0002-switched-to-python-copier-template.md
rename to docs/decision-records/decisions/0002-switched-to-python-copier-template.md
diff --git a/docs/explanations/decisions/0003-message-queue-message-grouping.md b/docs/decision-records/decisions/0003-message-queue-message-grouping.md
similarity index 100%
rename from docs/explanations/decisions/0003-message-queue-message-grouping.md
rename to docs/decision-records/decisions/0003-message-queue-message-grouping.md
diff --git a/docs/explanations/decisions/0004-zocalo-dependency-free.md b/docs/decision-records/decisions/0004-zocalo-dependency-free.md
similarity index 100%
rename from docs/explanations/decisions/0004-zocalo-dependency-free.md
rename to docs/decision-records/decisions/0004-zocalo-dependency-free.md
diff --git a/docs/explanations/decisions/0005-detect-secrets-for-secret-scanning.md b/docs/decision-records/decisions/0005-detect-secrets-for-secret-scanning.md
similarity index 100%
rename from docs/explanations/decisions/0005-detect-secrets-for-secret-scanning.md
rename to docs/decision-records/decisions/0005-detect-secrets-for-secret-scanning.md
diff --git a/docs/explanations/decisions/0006-sealed-secrets-kubernetes-secrets-management.md b/docs/decision-records/decisions/0006-sealed-secrets-kubernetes-secrets-management.md
similarity index 100%
rename from docs/explanations/decisions/0006-sealed-secrets-kubernetes-secrets-management.md
rename to docs/decision-records/decisions/0006-sealed-secrets-kubernetes-secrets-management.md
diff --git a/docs/explanations/decisions/0007-eliminate-smartem-api-circular-dependency.md b/docs/decision-records/decisions/0007-eliminate-smartem-api-circular-dependency.md
similarity index 100%
rename from docs/explanations/decisions/0007-eliminate-smartem-api-circular-dependency.md
rename to docs/decision-records/decisions/0007-eliminate-smartem-api-circular-dependency.md
diff --git a/docs/explanations/decisions/0008-backend-to-agent-communication-architecture.md b/docs/decision-records/decisions/0008-backend-to-agent-communication-architecture.md
similarity index 100%
rename from docs/explanations/decisions/0008-backend-to-agent-communication-architecture.md
rename to docs/decision-records/decisions/0008-backend-to-agent-communication-architecture.md
diff --git a/docs/explanations/decisions/0009-commit-generated-route-tree.md b/docs/decision-records/decisions/0009-commit-generated-route-tree.md
similarity index 100%
rename from docs/explanations/decisions/0009-commit-generated-route-tree.md
rename to docs/decision-records/decisions/0009-commit-generated-route-tree.md
diff --git a/docs/explanations/decisions/0011-remove-python-copier-template.md b/docs/decision-records/decisions/0011-remove-python-copier-template.md
similarity index 100%
rename from docs/explanations/decisions/0011-remove-python-copier-template.md
rename to docs/decision-records/decisions/0011-remove-python-copier-template.md
diff --git a/docs/decision-records/decisions/0012-audience-based-documentation-structure.md b/docs/decision-records/decisions/0012-audience-based-documentation-structure.md
new file mode 100644
index 0000000..5fc10d9
--- /dev/null
+++ b/docs/decision-records/decisions/0012-audience-based-documentation-structure.md
@@ -0,0 +1,95 @@
+# 12. Audience-Based Documentation Structure
+
+Date: 2026-01-06
+
+## Status
+
+Accepted
+
+## Context
+
+The SmartEM documentation inherited a Diataxis framework (Tutorials, How-To, Explanations, Reference) from the Python Copier template. While Diataxis is well-regarded for general-purpose documentation, it presents challenges for our specific context:
+
+### Problems with Diataxis for SmartEM
+
+1. **Audience ambiguity**: A "How-To" guide doesn't indicate whether it's for developers, operators, or integrators. Users must read content to determine relevance.
+
+2. **Component fragmentation**: Information about the Agent is scattered across how-to guides, explanations, and reference sections, requiring navigation across multiple directories.
+
+3. **Use-case disconnect**: Common tasks like "deploy to production" or "debug agent issues" span multiple Diataxis categories, creating friction for users with specific goals.
+
+4. **Growing complexity**: As the system expands to multiple components (Backend, Agent, Athena, Frontend), generic categories become insufficient.
+
+### Requirements
+
+- Clear entry points for different audiences (developers, operators, integrators)
+- Component-focused documentation that groups related information
+- Use-case driven organisation that matches how users approach the system
+- Scalable structure as new components are added
+
+## Decision
+
+We will restructure documentation around three axes:
+
+1. **Audience**: Developers, Operators, Integrators
+2. **Component**: Backend, Agent, Athena (and future components)
+3. **Use-Case**: Getting Started, Operations, Development, Architecture
+
+### Target Structure
+
+```
+docs/
+├── getting-started/       # Entry points by audience
+│   ├── for-developers.md
+│   ├── for-operators.md
+│   └── for-integrators.md
+├── backend/               # Backend component
+│   ├── api-server.md
+│   ├── database.md
+│   └── messaging.md
+├── agent/                 # Agent component
+│   ├── cli-reference.md
+│   ├── deployment.md
+│   └── epu-integration.md
+├── athena/                # Athena integration
+│   ├── api-reference.md
+│   └── mock-server.md
+├── operations/            # Cross-cutting operations
+│   ├── kubernetes.md
+│   └── environment-variables.md
+├── development/           # Contributing, testing
+│   ├── contributing.md
+│   └── testing.md
+├── architecture/          # Design docs and ADRs
+│   ├── system-design.md
+│   └── decisions/
+└── api/                   # Generated API docs
+```
+
+### Structure Principles
+
+- Top-level directories reflect components and cross-cutting concerns
+- Each section includes audience indicators where content differs by role
+- ADRs remain in `/decision-records/decisions/` as the authoritative location
+- API reference docs (Swagger/OpenAPI) remain generated and auto-published
+
+## Consequences
+
+### Positive
+
+- **Clear navigation**: Users can quickly find content relevant to their role
+- **Component coherence**: All Agent docs in one place, all Backend docs together
+- **Scalable**: New components get new directories without restructuring
+- **Reduced duplication**: Shared concepts documented once, linked from context
+
+### Negative
+
+- **Migration effort**: Existing docs must be reorganised and some rewritten
+- **Link breakage**: External links to old structure will break (redirects needed)
+- **Learning curve**: Contributors must understand new structure
+
+### Migration Notes
+
+- Existing content will be reorganised, not discarded
+- Sphinx configuration will be updated for new structure
+- WebUI will require corresponding updates (separate task)
diff --git a/docs/explanations/decisions/COPYME b/docs/decision-records/decisions/COPYME
similarity index 100%
rename from docs/explanations/decisions/COPYME
rename to docs/decision-records/decisions/COPYME
diff --git a/docs/explanations/epu-data-structures.md b/docs/decision-records/epu-data-structures.md
similarity index 100%
rename from docs/explanations/epu-data-structures.md
rename to docs/decision-records/epu-data-structures.md
diff --git a/docs/decision-records/index.md b/docs/decision-records/index.md
new file mode 100644
index 0000000..50806e6
--- /dev/null
+++ b/docs/decision-records/index.md
@@ -0,0 +1,26 @@
+# Decision Records
+
+This section contains architectural decision records (ADRs) and design documentation for SmartEM.
+
+```{toctree}
+:maxdepth: 2
+
+decisions
+backend-agent-communication-system-design
+smartem-agent-design
+epu-data-structures
+technical-notes
+```
+
+## Architecture Decision Records
+
+ADRs document significant technical decisions made during development. See [ADR-0001](decisions/0001-record-architecture-decisions.md) for the decision to use this format.
+
+- [All ADRs](decisions.md) - Complete list of architecture decision records
+
+## Design Documents
+
+- [Backend-Agent Communication](backend-agent-communication-system-design.md) - System design for SSE-based communication
+- [SmartEM Agent Design](smartem-agent-design.md) - Agent architecture and EPU integration
+- [EPU Data Structures](epu-data-structures.md) - EPU XML schemas and data models
+- [Technical Notes](technical-notes.md) - Implementation notes and research
diff --git a/docs/explanations/smartem-agent-design.md b/docs/decision-records/smartem-agent-design.md
similarity index 100%
rename from docs/explanations/smartem-agent-design.md
rename to docs/decision-records/smartem-agent-design.md
diff --git a/docs/explanations/smartem-workspace-developer-guide.md b/docs/decision-records/smartem-workspace-developer-guide.md
similarity index 100%
rename from docs/explanations/smartem-workspace-developer-guide.md
rename to docs/decision-records/smartem-workspace-developer-guide.md
diff --git a/docs/explanations/technical-notes.md b/docs/decision-records/technical-notes.md
similarity index 100%
rename from docs/explanations/technical-notes.md
rename to docs/decision-records/technical-notes.md
diff --git a/docs/how-to/contribute.md b/docs/development/contributing.md
similarity index 100%
rename from docs/how-to/contribute.md
rename to docs/development/contributing.md
diff --git a/docs/how-to/run-e2e-dev-simulation.md b/docs/development/e2e-simulation.md
similarity index 95%
rename from docs/how-to/run-e2e-dev-simulation.md
rename to docs/development/e2e-simulation.md
index 8c82833..bbbbf7d 100644
--- a/docs/how-to/run-e2e-dev-simulation.md
+++ b/docs/development/e2e-simulation.md
@@ -342,7 +342,7 @@ Access RabbitMQ management UI:
 ```bash
 # RabbitMQ management interface (if enabled in k3s)
 # Check k8s service for management UI port mapping
-# Default credentials: guest/guest  # pragma: allowlist secret
+# Default credentials: guest/guest
 ```
 
 ### Cleaning Queue Between Tests
@@ -407,7 +407,7 @@ python -m fastapi dev src/smartem_backend/api_server.py \
 ### Backend Worker/Consumer
 ```bash
 source .venv/bin/activate
-export RABBITMQ_URL=amqp://guest:guest@localhost:30672/  # pragma: allowlist secret
+export RABBITMQ_URL=amqp://guest:guest@localhost:30672/
 python -m smartem_backend.consumer -vv 2>&1 | tee logs/e2e-tests/TIMESTAMP/logs/consumer.log
 ```
 **Verbosity flags**: `-v` for INFO, `-vv` for DEBUG
@@ -633,20 +633,20 @@ python -m smartem_agent watch \
 **ALWAYS check the agent log file first if things aren't working!**
 
 1. **Wrong agent command**: Use `python -m smartem_agent watch`, NOT:
-   - ❌ `python -m smartem_agent.main`
-   - ❌ `python -m smartem_agent`
-   - ✅ `python -m smartem_agent watch`
+   - `python -m smartem_agent.main`
+   - `python -m smartem_agent`
+   - `python -m smartem_agent watch`
 
 2. **Wrong recording path**: Must use .tar.gz file, NOT directory:
-   - ❌ `../../testdata/recordings/pre-acquisition`
-   - ✅ `../../testdata/recordings/bi37708-42_fsrecord.tar.gz`
+   - `../../testdata/recordings/pre-acquisition`
+   - `../../testdata/recordings/bi37708-42_fsrecord.tar.gz`
 
 3. **FastAPI dev server**: Use uvicorn for e2e tests, NOT `python -m fastapi dev`:
-   - ❌ `python -m fastapi dev src/smartem_backend/api_server.py`
-   - ✅ `python -m uvicorn smartem_backend.api_server:app`
+   - `python -m fastapi dev src/smartem_backend/api_server.py`
+   - `python -m uvicorn smartem_backend.api_server:app`
 
 4. **Forgot to activate venv**: All commands must run with venv activated:
-   - ✅ `source .venv/bin/activate` at the start
+   - `source .venv/bin/activate` at the start
 
 5. **Agent log is tiny (< 1KB)**: Agent failed to start - check the error message in agent.log
 
@@ -741,17 +741,17 @@ tail -f "$TEST_DIR/logs/playback.log"
 ### Success Criteria
 
 A successful test should demonstrate:
-- ✅ All backend services start without errors
-- ✅ RabbitMQ queue is empty at test start
-- ✅ Database is freshly created with no leftover data
-- ✅ Playback creates expected files in output directory
-- ✅ Agent detects file changes and sends data to API
-- ✅ API receives and stores data in database
-- ✅ Worker processes messages from queue
-- ✅ No errors in any service logs
-- ✅ Database contains expected entities after test
-- ✅ **Filesystem counts match database counts**: EPU sessions → grids, GridSquares → gridsquare rows, MRC files → micrograph rows
-- ✅ **No or minimal "No grid found" messages** in agent.log (indicates proper grid association)
+- All backend services start without errors
+- RabbitMQ queue is empty at test start
+- Database is freshly created with no leftover data
+- Playback creates expected files in output directory
+- Agent detects file changes and sends data to API
+- API receives and stores data in database
+- Worker processes messages from queue
+- No errors in any service logs
+- Database contains expected entities after test
+- **Filesystem counts match database counts**: EPU sessions -> grids, GridSquares -> gridsquare rows, MRC files -> micrograph rows
+- **No or minimal "No grid found" messages** in agent.log (indicates proper grid association)
 
 ## Cleanup and Teardown
 
@@ -795,7 +795,7 @@ kubectl get pods -n smartem-decisions
 kubectl logs deployment/rabbitmq -n smartem-decisions
 
 # Verify connection from host
-curl -u guest:guest http://localhost:30673  # pragma: allowlist secret
+curl -u guest:guest http://localhost:30673
 ```
 
 ### Agent Not Detecting Files
diff --git a/docs/how-to/generate-docs.md b/docs/development/generate-docs.md
similarity index 100%
rename from docs/how-to/generate-docs.md
rename to docs/development/generate-docs.md
diff --git a/docs/how-to/sync-github-labels.md b/docs/development/github-labels.md
similarity index 100%
rename from docs/how-to/sync-github-labels.md
rename to docs/development/github-labels.md
diff --git a/docs/development/index.md b/docs/development/index.md
new file mode 100644
index 0000000..3b2e7a0
--- /dev/null
+++ b/docs/development/index.md
@@ -0,0 +1,31 @@
+# Development
+
+Documentation for contributing to the SmartEM codebase.
+
+```{toctree}
+:maxdepth: 1
+
+contributing
+tools
+generate-docs
+e2e-simulation
+github-labels
+mcp-interface
+```
+
+## Topics
+
+### Contributing
+
+- [Contributing Guide](contributing.md) - Code style, pull requests, reviews
+
+### Tools and Testing
+
+- [Development Tools](tools.md) - Utility tools for development, testing, and maintenance
+- [E2E Simulation](e2e-simulation.md) - End-to-end development simulation setup
+- [MCP Interface](mcp-interface.md) - SmartEM MCP server for Claude Code integration
+
+### Documentation and CI/CD
+
+- [Generate Documentation](generate-docs.md) - Building the documentation locally
+- [GitHub Labels](github-labels.md) - Synchronising GitHub labels across repositories
diff --git a/docs/how-to/use-mcp-interface.md b/docs/development/mcp-interface.md
similarity index 98%
rename from docs/how-to/use-mcp-interface.md
rename to docs/development/mcp-interface.md
index a0f11bb..49e4b46 100644
--- a/docs/how-to/use-mcp-interface.md
+++ b/docs/development/mcp-interface.md
@@ -48,7 +48,7 @@ python -m smartem_mcp interactive
 
 Example questions:
 - "Show me a summary of session /path/to/epu/directory"
-- "Find low quality items in /path/to/epu with threshold 0.3"  
+- "Find low quality items in /path/to/epu with threshold 0.3"
 - "What are the recent acquisitions?"
 
 ### 2. Command Line Usage
@@ -78,13 +78,13 @@ async with SmartEMMCPClient() as client:
     result = await client.parse_epu_directory("/path/to/epu")
     if result.get("success"):
         print(f"Found {result['grid_count']} grids")
-    
+
     # Find low quality items
     quality_result = await client.find_low_quality_items(
-        "/path/to/epu", 
+        "/path/to/epu",
         threshold=0.3
     )
-    
+
     # Query API data
     acquisitions = await client.query_recent_acquisitions(limit=5)
 ```
@@ -233,7 +233,7 @@ Start a new Claude Code session and verify the SmartEM tools are available:
 
 You should see the following tools available:
 - `parse_epu_directory` - Parse EPU microscopy directories
-- `query_quality_metrics` - Find low-quality images and foil holes  
+- `query_quality_metrics` - Find low-quality images and foil holes
 - `query_acquisitions` - Query recent acquisition sessions
 - `query_grid_status` - Get grid processing status
 
@@ -373,7 +373,7 @@ client = SmartEMMCPClient(api_token="your_token_here")
 #### 1. MCP Server Registration Problems
 
 **"SmartEM MCP server not found"**
-- Ensure MCP dependencies are installed: `pip install -e .[mcp]` (includes FastMCP 2.0)  
+- Ensure MCP dependencies are installed: `pip install -e .[mcp]` (includes FastMCP 2.0)
 - Verify Python path includes smartem_mcp module: `python -c "import smartem_mcp; print('OK')"`
 - Check registration with `claude mcp list`
 
@@ -391,7 +391,7 @@ client = SmartEMMCPClient(api_token="your_token_here")
 
 **"Invalid EPU directory"**
 - Verify directory contains EpuSession.dm file
-- Check Metadata/ and Images-Disc*/ subdirectories exist  
+- Check Metadata/ and Images-Disc*/ subdirectories exist
 - Ensure proper file permissions for reading EPU data
 
 **"Permission denied accessing EPU files"**
@@ -461,7 +461,7 @@ python -m smartem_mcp interactive
 - Ensure virtual environment is activated when registering
 - Use absolute Python paths: `claude mcp add smartem -- /path/to/.venv/bin/python -m smartem_mcp server`
 
-#### Production Environment  
+#### Production Environment
 - Verify all dependencies are installed in production Python environment
 - Check firewall rules for API connectivity
 - Ensure proper logging configuration for production
@@ -482,7 +482,7 @@ If issues persist:
    pip list | grep -E "(smartem|mcp)"
    claude mcp list
    claude mcp status smartem
-   
+
    # Test basic functionality
    python -m smartem_mcp --version
    python -c "import smartem_mcp; print('Import successful')"
diff --git a/docs/how-to/development-tools.md b/docs/development/tools.md
similarity index 100%
rename from docs/how-to/development-tools.md
rename to docs/development/tools.md
diff --git a/docs/explanations.md b/docs/explanations.md
deleted file mode 100644
index 50612ad..0000000
--- a/docs/explanations.md
+++ /dev/null
@@ -1,12 +0,0 @@
-# Explanations
-
-Explanations of how it works and why it works that way.
-
-## Architecture & Design
-- [SmartEM Agent Design](/docs/explanations/smartem-agent-design) - Agent architecture
-- [Backend-Agent Communication](/docs/explanations/backend-agent-communication-system-design) - Communication system design
-- [EPU Data Structures](/docs/explanations/epu-data-structures) - EPU data format documentation
-- [Technical Notes](/docs/explanations/technical-notes) - Technical implementation notes
-
-## Architecture Decision Records
-- [Architecture Decisions](/docs/explanations/decisions) - All ADRs
diff --git a/docs/tutorials/installation.md b/docs/getting-started/for-developers.md
similarity index 80%
rename from docs/tutorials/installation.md
rename to docs/getting-started/for-developers.md
index 77eda4b..651d0ea 100644
--- a/docs/tutorials/installation.md
+++ b/docs/getting-started/for-developers.md
@@ -61,13 +61,7 @@ pip install -e .[all]
 
 ## Verify Installation
 
-The library should now be installed with the command-line interface available on your system path. Verify the installation by checking the installed version:
-
-```bash
-smartem-decisions --version
-```
-
-You can also verify that all components are correctly installed by running:
+Verify that all components are correctly installed by running:
 
 ```bash
 # Test core functionality
@@ -77,11 +71,18 @@ python -c "import smartem_backend, smartem_agent, smartem_common; print('Core co
 python -c "import smartem_mcp; print('MCP components available')"
 ```
 
+You can also verify the CLI tools are available:
+
+```bash
+# Check agent CLI is accessible
+python -m smartem_agent --help
+```
+
 ## Next Steps
 
 Once installation is complete, you can proceed to:
 
-- [Run the backend service](../how-to/run-backend.md)
-- [Deploy using containers](../how-to/run-container.md)  
-- [Set up the development environment](../how-to/development-tools.md)
-- [Configure logging](../how-to/configure-logging.md)
+- [Run the backend service](../backend/api-server.md)
+- [Deploy using containers](../operations/run-container.md)
+- [Set up the development environment](../development/tools.md)
+- [Configure logging](../operations/logging.md)
diff --git a/docs/getting-started/index.md b/docs/getting-started/index.md
new file mode 100644
index 0000000..e9bb5b6
--- /dev/null
+++ b/docs/getting-started/index.md
@@ -0,0 +1,30 @@
+# Getting Started
+
+This section provides entry points for different audiences working with SmartEM.
+
+## By Audience
+
+```{toctree}
+:maxdepth: 1
+
+for-developers
+```
+
+### For Developers
+
+Set up a local development environment to contribute to SmartEM.
+
+- [Development Environment Setup](for-developers.md) - Install dependencies, configure tools, run services locally
+
+### For Operators
+
+Deploy and operate SmartEM in production environments.
+
+- See [Operations](../operations/index.md) for Kubernetes deployment, container configuration, and environment setup
+
+### For Integrators
+
+Use SmartEM APIs to build integrations.
+
+- See [Backend API Documentation](../backend/api-documentation.md) for interactive API documentation
+- See [HTTP API Client](../backend/http-api-client.md) for programmatic access
diff --git a/docs/terms-and-defs.md b/docs/glossary.md
similarity index 96%
rename from docs/terms-and-defs.md
rename to docs/glossary.md
index f5f7c92..afe6cea 100644
--- a/docs/terms-and-defs.md
+++ b/docs/glossary.md
@@ -1,4 +1,4 @@
-# Terms and Definitions
+# Glossary
 
 ## EPU
 
@@ -25,7 +25,7 @@ https://instruct-eric.org/help/about-aria
 
 ## Atlas
 
-A comprehensive overview image of the entire grid, composed of individual tiles arranged in a 5×5 grid pattern. The
+A comprehensive overview image of the entire grid, composed of individual tiles arranged in a 5x5 grid pattern. The
 atlas provides spatial context and navigation reference for subsequent high-magnification data collection across the
 specimen grid.
 
diff --git a/docs/how-to.md b/docs/how-to.md
deleted file mode 100644
index 99ec4d4..0000000
--- a/docs/how-to.md
+++ /dev/null
@@ -1,36 +0,0 @@
-# How-to Guides
-
-Practical step-by-step guides for the more experienced user.
-
-## Getting Started
-- [Run Backend](/docs/how-to/run-backend) - Start the SmartEM backend services
-- [Run Agent](/docs/how-to/run-agent) - Run the SmartEM agent
-- [Run Container](/docs/how-to/run-container) - Run services in containers
-
-## Development
-- [Run E2E Dev Simulation](/docs/how-to/run-e2e-dev-simulation) - End-to-end development testing
-- [Development Tools](/docs/how-to/development-tools) - Available development tooling
-- [Troubleshoot CLI](/docs/how-to/troubleshoot-cli) - CLI troubleshooting guide
-- [Generate Docs](/docs/how-to/generate-docs) - Build documentation locally
-
-## Configuration
-- [Configure Environment Variables](/docs/how-to/configure-environment-variables) - Environment setup
-- [Configure Logging](/docs/how-to/configure-logging) - Logging configuration
-- [Container User Configuration](/docs/how-to/container-user-configuration) - Container user setup
-
-## Deployment
-- [Containerization](/docs/how-to/containerization) - Build container images
-- [Deploy Kubernetes](/docs/how-to/deploy-kubernetes) - Kubernetes deployment guide
-- [Manage Kubernetes Secrets](/docs/how-to/manage-kubernetes-secrets) - Secrets management
-
-## Database
-- [Database Migrations](/docs/how-to/database-migrations) - Alembic migration guide
-
-## APIs & Integration
-- [Use API Documentation](/docs/how-to/use-api-documentation) - Interactive API docs
-- [Use HTTP API Client](/docs/how-to/use-core-http-api-client) - Core HTTP client usage
-- [Use MCP Interface](/docs/how-to/use-mcp-interface) - MCP interface guide
-
-## Contributing
-- [Contribute](/docs/how-to/contribute) - Contribution guidelines
-- [Sync GitHub Labels](/docs/how-to/sync-github-labels) - Label synchronization
diff --git a/docs/how-to/container-user-configuration.md b/docs/operations/container-user-configuration.md
similarity index 98%
rename from docs/how-to/container-user-configuration.md
rename to docs/operations/container-user-configuration.md
index 374d8b1..e6c5ead 100644
--- a/docs/how-to/container-user-configuration.md
+++ b/docs/operations/container-user-configuration.md
@@ -414,9 +414,9 @@ docker build --build-arg groupid=5000 --build-arg userid=5000 \
 ## Related Documentation
 
 - [Run in a Container](run-container.md) - Basic container usage
-- [Kubernetes Deployment](deploy-kubernetes.md) - Full deployment guide
+- [Kubernetes Deployment](kubernetes.md) - Full deployment guide
 - [Containerization](containerization.md) - Building and pushing images
-- [Managing Kubernetes Secrets](manage-kubernetes-secrets.md) - Secure credential management
+- [Managing Kubernetes Secrets](kubernetes-secrets.md) - Secure credential management
 
 ## References
 
diff --git a/docs/how-to/containerization.md b/docs/operations/containerization.md
similarity index 97%
rename from docs/how-to/containerization.md
rename to docs/operations/containerization.md
index c7bb3da..a7c0ab6 100644
--- a/docs/how-to/containerization.md
+++ b/docs/operations/containerization.md
@@ -114,7 +114,7 @@ This approach:
 
 - [Container User Configuration](container-user-configuration.md) - Detailed guide on UID/GID configuration
 - [Run in a Container](run-container.md) - Using pre-built containers
-- [Kubernetes Deployment](deploy-kubernetes.md) - Deploying to Kubernetes
+- [Kubernetes Deployment](kubernetes.md) - Deploying to Kubernetes
 
 ## References
 
diff --git a/docs/how-to/configure-environment-variables.md b/docs/operations/environment-variables.md
similarity index 93%
rename from docs/how-to/configure-environment-variables.md
rename to docs/operations/environment-variables.md
index 5ae3503..6a13caf 100644
--- a/docs/how-to/configure-environment-variables.md
+++ b/docs/operations/environment-variables.md
@@ -125,7 +125,7 @@ cp .env.example.k8s.production .env.k8s.production
 - Should use Sealed Secrets (encrypted) instead of plain `.env` files
 - Credentials should be rotated regularly
 - Access to this file should be strictly controlled
-- See [Manage Kubernetes Secrets](manage-kubernetes-secrets.md) for best practices
+- See [Manage Kubernetes Secrets](kubernetes-secrets.md) for best practices
 
 **Used By**:
 - Production CI/CD pipelines
@@ -151,7 +151,7 @@ SMARTEM_MCP_LOG_LEVEL=INFO
 
 **Used By**:
 - MCP server when run via Claude Code
-- See [Use MCP Interface](use-mcp-interface.md) for details
+- See [Use MCP Interface](../development/mcp-interface.md) for details
 
 ## Quick Start: First-Time Setup
 
@@ -339,9 +339,9 @@ env | grep POSTGRES
 
 ## Related Documentation
 
-- [Run Backend Services](run-backend.md) - Starting backend API and consumer
-- [Run E2E Development Simulation](run-e2e-dev-simulation.md) - E2E testing workflows
-- [Deploy to Kubernetes](deploy-kubernetes.md) - K8s deployment guide
-- [Manage Kubernetes Secrets](manage-kubernetes-secrets.md) - Sealed Secrets and security
-- [Database Migrations](database-migrations.md) - Alembic migration workflow
-- [Use MCP Interface](use-mcp-interface.md) - Claude Code MCP integration
+- [Run Backend Services](../backend/api-server.md) - Starting backend API and consumer
+- [Run E2E Development Simulation](../development/e2e-simulation.md) - E2E testing workflows
+- [Deploy to Kubernetes](kubernetes.md) - K8s deployment guide
+- [Manage Kubernetes Secrets](kubernetes-secrets.md) - Sealed Secrets and security
+- [Database Migrations](../backend/database.md) - Alembic migration workflow
+- [Use MCP Interface](../development/mcp-interface.md) - Claude Code MCP integration
diff --git a/docs/operations/index.md b/docs/operations/index.md
new file mode 100644
index 0000000..d64c374
--- /dev/null
+++ b/docs/operations/index.md
@@ -0,0 +1,33 @@
+# Operations
+
+Cross-cutting operational documentation for deploying and running SmartEM in production.
+
+```{toctree}
+:maxdepth: 1
+
+kubernetes
+kubernetes-secrets
+containerization
+run-container
+container-user-configuration
+environment-variables
+logging
+```
+
+## Topics
+
+### Kubernetes
+
+- [Kubernetes Deployment](kubernetes.md) - Deploying SmartEM to Kubernetes clusters
+- [Kubernetes Secrets](kubernetes-secrets.md) - Managing secrets and sensitive configuration
+
+### Containers
+
+- [Containerisation](containerization.md) - Building container images
+- [Running Containers](run-container.md) - Container execution and debugging
+- [Container User Configuration](container-user-configuration.md) - User and permission configuration
+
+### Configuration
+
+- [Environment Variables](environment-variables.md) - Complete configuration reference
+- [Logging](logging.md) - Log configuration and structured logging
diff --git a/docs/how-to/manage-kubernetes-secrets.md b/docs/operations/kubernetes-secrets.md
similarity index 94%
rename from docs/how-to/manage-kubernetes-secrets.md
rename to docs/operations/kubernetes-secrets.md
index b7e36fb..b38156d 100644
--- a/docs/how-to/manage-kubernetes-secrets.md
+++ b/docs/operations/kubernetes-secrets.md
@@ -4,8 +4,8 @@ This guide explains how to securely manage secrets in the SmartEM Decisions proj
 
 ## Overview
 
-The SmartEM Decisions project uses [Bitnami Sealed Secrets](https://sealed-secrets.netlify.app/) to securely manage sensitive 
-configuration data such as database credentials and message queue passwords. Sealed Secrets provide a secure alternative to 
+The SmartEM Decisions project uses [Bitnami Sealed Secrets](https://sealed-secrets.netlify.app/) to securely manage sensitive
+configuration data such as database credentials and message queue passwords. Sealed Secrets provide a secure alternative to
 storing plain-text secrets in version control.
 
 ### Why Sealed Secrets?
@@ -13,7 +13,7 @@ storing plain-text secrets in version control.
 Sealed Secrets offer several security advantages over traditional Kubernetes secrets:
 
 - **Version Control Safe**: Sealed secrets are encrypted and safe to commit to Git repositories
-- **Asymmetric Encryption**: Uses public/private key cryptography for maximum security  
+- **Asymmetric Encryption**: Uses public/private key cryptography for maximum security
 - **Cluster-Specific**: Secrets are encrypted for a specific cluster and cannot be used elsewhere
 - **Automatic Decryption**: The sealed-secrets controller automatically decrypts secrets in the cluster
 - **Audit Trail**: All secret changes are tracked in version control with proper attribution
@@ -49,7 +49,7 @@ kubeseal --version
 
 ### Cluster Requirements
 
-The sealed-secrets controller must be installed in your Kubernetes cluster. For Diamond Light Source clusters, this is 
+The sealed-secrets controller must be installed in your Kubernetes cluster. For Diamond Light Source clusters, this is
 typically pre-installed. Verify the controller is running:
 
 ```bash
@@ -83,7 +83,7 @@ Development environments use automatically generated secure passwords for conven
 
 This will:
 - Generate cryptographically secure random passwords
-- Create sealed secrets for the `smartem-decisions` namespace  
+- Create sealed secrets for the `smartem-decisions` namespace
 - Display a summary of generated usernames (passwords remain sealed)
 - Update `k8s/environments/development/secrets.yaml`
 
@@ -112,9 +112,9 @@ For advanced use cases, you can manually create sealed secrets:
 kubectl create secret generic smartem-secrets \
     --namespace=smartem-decisions-production \
     --from-literal=POSTGRES_USER="secure_postgres_user" \
-    --from-literal=POSTGRES_PASSWORD="secure_postgres_password" \  # pragma: allowlist secret
+    --from-literal=POSTGRES_PASSWORD="secure_postgres_password" \
     --from-literal=RABBITMQ_USER="secure_rabbitmq_user" \
-    --from-literal=RABBITMQ_PASSWORD="secure_rabbitmq_password" \  # pragma: allowlist secret
+    --from-literal=RABBITMQ_PASSWORD="secure_rabbitmq_password" \
     --dry-run=client \
     --output=yaml > temp-secret.yaml
 ```
@@ -191,7 +191,7 @@ k8s/environments/
 │   ├── kustomization.yaml    # References secrets.yaml
 │   └── secrets.yaml          # Sealed secret for development
 ├── staging/
-│   ├── kustomization.yaml    # References secrets.yaml  
+│   ├── kustomization.yaml    # References secrets.yaml
 │   └── secrets.yaml          # Sealed secret for staging
 └── production/
     ├── kustomization.yaml    # References secrets.yaml
@@ -315,7 +315,7 @@ To use different secret names:
 # Modify script variables or create custom sealed secret
 kubectl create secret generic custom-secret-name \
     --namespace=smartem-decisions \
-    --from-literal=API_KEY="secure_api_key" \  # pragma: allowlist secret
+    --from-literal=API_KEY="secure_api_key" \
     --dry-run=client -o yaml | \
 kubeseal --format=yaml --namespace=smartem-decisions > custom-sealed-secret.yaml
 ```
@@ -331,7 +331,7 @@ For complex applications requiring multiple secret sources:
 # Additional API secrets
 kubectl create secret generic api-secrets \
     --namespace=smartem-decisions-production \
-    --from-literal=EXTERNAL_API_KEY="api_key_here" \  # pragma: allowlist secret
+    --from-literal=EXTERNAL_API_KEY="api_key_here" \
     --dry-run=client -o yaml | \
 kubeseal --format=yaml --namespace=smartem-decisions-production > api-sealed-secrets.yaml
 ```
@@ -341,7 +341,7 @@ kubeseal --format=yaml --namespace=smartem-decisions-production > api-sealed-sec
 When moving between clusters, sealed secrets must be regenerated:
 
 ```bash
-# Export existing secret from source cluster  
+# Export existing secret from source cluster
 kubectl get secret smartem-secrets -n smartem-decisions -o yaml > plain-secret.yaml
 
 # Remove cluster-specific metadata
@@ -361,7 +361,7 @@ rm plain-secret.yaml
 │   │   ├── secrets.yaml                     # Development sealed secrets
 │   │   └── kustomization.yaml               # References secrets.yaml
 │   ├── staging/
-│   │   ├── secrets.yaml                     # Staging sealed secrets  
+│   │   ├── secrets.yaml                     # Staging sealed secrets
 │   │   └── kustomization.yaml               # References secrets.yaml
 │   └── production/
 │       ├── secrets.yaml                     # Production sealed secrets
diff --git a/docs/how-to/deploy-kubernetes.md b/docs/operations/kubernetes.md
similarity index 93%
rename from docs/how-to/deploy-kubernetes.md
rename to docs/operations/kubernetes.md
index 80accdb..8e482c2 100644
--- a/docs/how-to/deploy-kubernetes.md
+++ b/docs/operations/kubernetes.md
@@ -48,7 +48,7 @@ k8s/
 
 ## Security: Sealed Secrets
 
-The project uses [Bitnami Sealed Secrets](https://sealed-secrets.netlify.app/) for secure credential management. Before 
+The project uses [Bitnami Sealed Secrets](https://sealed-secrets.netlify.app/) for secure credential management. Before
 deploying to any environment, you must generate the appropriate sealed secrets:
 
 ### Generate Secrets for Development
@@ -65,12 +65,12 @@ deploying to any environment, you must generate the appropriate sealed secrets:
 ./tools/k8s/generate-sealed-secrets.sh production
 ```
 
-Sealed secrets are encrypted with the cluster's public key and safe to commit to version control. The sealed-secrets 
+Sealed secrets are encrypted with the cluster's public key and safe to commit to version control. The sealed-secrets
 controller automatically decrypts them into regular Kubernetes secrets that applications can use.
 
 > **Security Note**: Never commit plain-text secrets to version control. Always use sealed secrets for credential management.
 
-For comprehensive secret management documentation, see [Managing Kubernetes Secrets](manage-kubernetes-secrets.md).
+For comprehensive secret management documentation, see [Managing Kubernetes Secrets](kubernetes-secrets.md).
 
 ## Detailed Documentation
 
diff --git a/docs/how-to/configure-logging.md b/docs/operations/logging.md
similarity index 98%
rename from docs/how-to/configure-logging.md
rename to docs/operations/logging.md
index 4f47016..22e0947 100644
--- a/docs/how-to/configure-logging.md
+++ b/docs/operations/logging.md
@@ -66,7 +66,7 @@ For the HTTP API, you can also control logging via environment variables:
 ```bash
 # Set log level via environment variable (equivalent to -v/-vv flags)
 SMARTEM_LOG_LEVEL=ERROR python -m smartem_backend.api_server
-SMARTEM_LOG_LEVEL=INFO python -m smartem_backend.api_server 
+SMARTEM_LOG_LEVEL=INFO python -m smartem_backend.api_server
 SMARTEM_LOG_LEVEL=DEBUG python -m smartem_backend.api_server
 ```
 
diff --git a/docs/how-to/publish-smartem-workspace-to-pypi.md b/docs/operations/publish-smartem-workspace-to-pypi.md
similarity index 100%
rename from docs/how-to/publish-smartem-workspace-to-pypi.md
rename to docs/operations/publish-smartem-workspace-to-pypi.md
diff --git a/docs/how-to/run-container.md b/docs/operations/run-container.md
similarity index 95%
rename from docs/how-to/run-container.md
rename to docs/operations/run-container.md
index 058ec84..2b8c687 100644
--- a/docs/how-to/run-container.md
+++ b/docs/operations/run-container.md
@@ -57,7 +57,7 @@ $ docker run -e ROLE=worker -e SMARTEM_LOG_LEVEL=DEBUG ghcr.io/DiamondLightSourc
 
 ## Complete Development Stack
 
-For a complete development environment with database and message queue, see the [Kubernetes deployment guide](../../k8s/README.md) which provides a docker-compose-like experience.
+For a complete development environment with database and message queue, see the [Kubernetes deployment guide](kubernetes.md) which provides a docker-compose-like experience.
 
 ## Advanced Configuration
 
diff --git a/docs/how-to/setup-smartem-workspace.md b/docs/operations/setup-smartem-workspace.md
similarity index 100%
rename from docs/how-to/setup-smartem-workspace.md
rename to docs/operations/setup-smartem-workspace.md
diff --git a/docs/reference.md b/docs/reference.md
deleted file mode 100644
index 19a3600..0000000
--- a/docs/reference.md
+++ /dev/null
@@ -1,13 +0,0 @@
-# Reference
-
-Technical reference material including APIs and release notes.
-
-## CLI
-- [CLI Reference](/docs/reference/cli-reference) - Command-line interface documentation
-
-## API Documentation
-- [Athena Decision Service API](/api/athena/index.html) - Decision service API docs
-- [SmartEM Core API](/api/smartem/index.html) - Core API docs
-
-## External Links
-- [Release Notes](https://github.com/DiamondLightSource/smartem-decisions/releases) - GitHub releases
diff --git a/docs/tutorials.md b/docs/tutorials.md
deleted file mode 100644
index 17a0052..0000000
--- a/docs/tutorials.md
+++ /dev/null
@@ -1,5 +0,0 @@
-# Tutorials
-
-Tutorials for installation and typical usage. New users start here.
-
-- [Installation](/docs/tutorials/installation) - Get started with SmartEM
diff --git a/webui/scripts/generate-mdx-docs.ts b/webui/scripts/generate-mdx-docs.ts
index b903072..9450583 100644
--- a/webui/scripts/generate-mdx-docs.ts
+++ b/webui/scripts/generate-mdx-docs.ts
@@ -17,13 +17,17 @@ const projectRoot = resolve(__dirname, '..')
 const docsSource = resolve(projectRoot, '..', 'docs')
 const docsTarget = resolve(projectRoot, 'src', 'docs')
 
-// Files to exclude from sync (Sphinx-specific or handled separately)
+// Files to exclude from sync (by filename only)
 const EXCLUDED_FILES = new Set([
-  'index.md', // Uses React components in webui version
   'genindex.md', // Sphinx-specific
   'athena-decision-service-api-spec.README.md', // API spec file
 ])
 
+// Paths to exclude from sync (relative to docs root)
+const EXCLUDED_PATHS = new Set([
+  'index.md', // Root index uses React components in webui version
+])
+
 // Directories to exclude from sync
 const EXCLUDED_DIRS = new Set(['_templates', 'images', 'api', '_api', '_static'])
 
@@ -54,8 +58,9 @@ function collectMdFiles(dir: string, baseDir: string = dir): string[] {
         files.push(...collectMdFiles(fullPath, baseDir))
       }
     } else if (entry.isFile() && entry.name.endsWith('.md')) {
-      if (!EXCLUDED_FILES.has(entry.name)) {
-        files.push(relative(baseDir, fullPath))
+      const relativePath = relative(baseDir, fullPath)
+      if (!EXCLUDED_FILES.has(entry.name) && !EXCLUDED_PATHS.has(relativePath)) {
+        files.push(relativePath)
       }
     }
   }