Add plugin configuration file support (#204)

* Add plugin configuration structures

* Define PluginConfig with 7 categories: authentication, authorization, rate_limiting, validation, content, observability, audit
* Add PluginEntry with name, commit hash, required flag, and flows
* Implement validation with set semantics for flows
* Add CRUD methods for plugin management
* Add Plugins field to Config struct

* Integrate plugin support into Config

* Add Plugin, UpsertPlugin, DeletePlugin, ListPlugins methods to Config
* Add plugin validation to Config.validate()

* Add tests for plugin configuration

* Add unit tests for PluginEntry validation, equality, and flow checks
* Add unit tests for PluginConfig CRUD operations
* Add integration tests for plugin persistence
* Add test for invalid plugin config validation

* Add plugin configuration testdata files

* Add valid configuration examples: basic_plugins.toml, multiple_plugins.toml, minimal_plugins.toml
* Add invalid configuration examples for validation testing

* Add tests for loading static plugin configuration files

* Add tests for loading valid plugin configurations from testdata
* Add tests for loading invalid plugin configurations
* Add round-trip test for loading and saving plugin configurations

* Add plugin configuration documentation

* Document plugin categories and execution order
* Explain required plugins and content mutation behavior
* Document observability plugin parallel execution
* Add complete configuration examples
* Update mkdocs navigation

* Allow plugin entries to be 'equal' with duplicate flows, and ignoring order

Author: peteski22

docs/plugin-configuration.md

@@ -0,0 +1,278 @@
+# Plugin Configuration
+
+## Overview
+
+The `mcpd` daemon supports a plugin subsystem for extending request/response processing.
+
+---
+
+## Plugin Categories
+
+!!! info Plugin execution order
+    Within each category, plugins execute in the order they appear in the configuration file.
+
+Plugins are organized into categories and execute during specific phases of the request lifecycle.
+
+Categories execute in the order shown below for both request and response phases.
+
+| Order | Category         | Purpose                                      | Execution  |
+|-------|------------------|----------------------------------------------|------------|
+| 1     | `observability`  | Collect metrics and traces (non-blocking)    | Parallel   |
+| 2     | `authentication` | Validate client identity                     | Sequential |
+| 3     | `authorization`  | Verify permissions after authentication      | Sequential |
+| 4     | `rate_limiting`  | Enforce request rate limits                  | Sequential |
+| 5     | `validation`     | Check request/response structure and content | Sequential |
+| 6     | `content`        | Transform request/response payloads          | Sequential |
+| 7     | `audit`          | Log compliance and security events           | Sequential |
+
+---
+
+## Plugin Execution Flows
+
+Plugins can execute during one or both flows/phases:
+
+* `request`: Executes during the request phase
+* `response`: Executes during the response phase
+
+---
+
+## Configuration Format
+
+```toml
+[[servers]]
+  name = "api-server"
+  package = "uvx::[email protected]"
+  tools = ["create", "read", "update", "delete"]
+
+[[plugins.authentication]]
+  name = "jwt-auth"
+  commit_hash = "abc123"
+  required = true
+  flows = ["request"]
+
+[[plugins.authentication]]
+  name = "api-key-auth"
+  flows = ["request", "response"]
+
+[[plugins.authorization]]
+  name = "rbac"
+  required = true
+  flows = ["request"]
+
+[[plugins.observability]]
+  name = "metrics"
+  flows = ["request", "response"]
+```
+
+---
+
+## Plugin Fields
+
+| Field         | Type    | Required | Description                                          |
+|---------------|---------|----------|------------------------------------------------------|
+| `name`        | string  | Yes      | Name of the plugin binary in the plugins directory   |
+| `commit_hash` | string  | No       | SHA/hash for validating plugin version               |
+| `required`    | boolean | No       | Whether plugin failure should block the request      |
+| `flows`       | array   | Yes      | Execution phases: ["request"], ["response"], or both |
+
+---
+
+## Execution Order
+
+Plugins execute in the order they appear in the configuration file within their category.
+
+```toml
+[[plugins.authentication]]
+  name = "jwt-auth"
+  flows = ["request"]
+
+[[plugins.authentication]]
+  name = "api-key-auth"
+  flows = ["request"]
+```
+
+During the request phase, `jwt-auth` executes first, followed by `api-key-auth`.
+
+---
+
+## Required Plugins
+
+!!! warning "Required Plugin Failures"
+    If a required (serial) plugin fails or rejects a request/response, the overall request is rejected immediately.
+
+Mark plugins as required when their successful execution is critical:
+
+```toml
+[[plugins.authentication]]
+  name = "jwt-auth"
+  required = true
+  flows = ["request"]
+```
+
+When `required` is not specified or set to `false`, plugin failures are logged but do not block the request.
+
+---
+
+## Content Mutation
+
+!!! info "Content Plugin Behavior"
+    Only plugins in the `content` category may mutate requests or responses. Modified content is passed to the next plugin in the chain.
+
+Content plugins modify the request by setting the modified request in their response. Other plugin categories can only observe or reject requests.
+
+### Example Content Plugin Flow
+
+```toml
+[[plugins.content]]
+  name = "encryption"
+  flows = ["request"]
+
+[[plugins.content]]
+  name = "compression"
+  flows = ["request"]
+```
+
+The `encryption` plugin processes the request first and may modify it. The modified request is then passed to the `compression` plugin.
+
+---
+
+## Observability Plugin Execution
+
+!!! note "Parallel Execution"
+    Observability plugins run in *parallel* and cannot modify requests or responses.
+
+Observability plugins are designed for metrics collection, tracing, and monitoring. They execute concurrently for performance.
+
+### Required Observability Plugins
+
+If any observability plugin is marked as `required`, request processing waits for all observability plugins to complete before aggregating results. 
+If any required observability plugin fails, the request is rejected after all have completed.
+
+```toml
+[[plugins.observability]]
+  name = "metrics"
+  required = true
+  flows = ["request", "response"]
+
+[[plugins.observability]]
+  name = "tracing"
+  flows = ["request", "response"]
+```
+
+In this example, both `metrics` and `tracing` run in parallel, but the request will be rejected if `metrics` fails 
+(once `metrics` and `tracing` have completed).
+
+---
+
+## Multiple Plugins Per Category
+
+You can configure multiple plugins within the same category. They execute in the order defined:
+
+```toml
+[[plugins.authentication]]
+  name = "jwt-auth"
+  required = true
+  flows = ["request"]
+
+[[plugins.authentication]]
+  name = "api-key-auth"
+  flows = ["request"]
+
+[[plugins.authentication]]
+  name = "oauth2"
+  flows = ["request"]
+```
+
+Request processing order: `jwt-auth` → `api-key-auth` → `oauth2`
+
+---
+
+## Minimal Configuration
+
+Plugins are optional. A configuration file without plugins is valid:
+
+```toml
+[[servers]]
+  name = "simple-server"
+  package = "uvx::[email protected]"
+  tools = ["tool1"]
+```
+
+---
+
+## Complete Example
+
+```toml
+[[servers]]
+  name = "production-api"
+  package = "uvx::[email protected]"
+  tools = ["create_user", "get_user", "update_user", "delete_user"]
+
+[[plugins.authentication]]
+  name = "jwt-auth"
+  commit_hash = "a1b2c3d4"
+  required = true
+  flows = ["request"]
+
+[[plugins.authorization]]
+  name = "rbac"
+  commit_hash = "e5f6g7h8"
+  required = true
+  flows = ["request"]
+
+[[plugins.rate_limiting]]
+  name = "token-bucket"
+  flows = ["request"]
+
+[[plugins.validation]]
+  name = "schema-validator"
+  required = true
+  flows = ["request", "response"]
+
+[[plugins.content]]
+  name = "encryption"
+  flows = ["request", "response"]
+
+[[plugins.observability]]
+  name = "prometheus-metrics"
+  required = true
+  flows = ["request", "response"]
+
+[[plugins.observability]]
+  name = "distributed-tracing"
+  flows = ["request", "response"]
+
+[[plugins.audit]]
+  name = "compliance-logger"
+  required = true
+  flows = ["response"]
+```
+
+### Execution Flow
+
+#### Request Phase
+
+1. `jwt-auth` (authentication) - sequential
+2. `rbac` (authorization) - sequential
+3. `token-bucket` (rate_limiting) - sequential
+4. `schema-validator` (validation) - sequential
+5. `encryption` (content) - sequential
+6. `prometheus-metrics` + `distributed-tracing` (observability) - parallel
+
+#### Response Phase
+
+1. `schema-validator` (validation) - sequential
+2. `encryption` (content) - sequential
+3. `prometheus-metrics` + `distributed-tracing` (observability) - parallel
+4. `compliance-logger` (audit) - sequential
+
+---
+
+## Validation
+
+Plugin configurations are validated when the daemon starts or during hot reload. Common validation errors:
+
+* Empty plugin name
+* Missing or empty `flows` array
+* Invalid flow values (must be `request` or `response`)
+* Duplicate flow values

internal/config/config.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/BurntSushi/toml"
 
+	"github.com/mozilla-ai/mcpd/v2/internal/context"
 	"github.com/mozilla-ai/mcpd/v2/internal/flags"
 	"github.com/mozilla-ai/mcpd/v2/internal/perms"
 )
@@ -128,6 +129,74 @@ func (c *Config) SaveConfig() error {
 	return c.saveConfig()
 }
 
+// Plugin retrieves a plugin by category and name.
+func (c *Config) Plugin(category string, name string) (PluginEntry, bool) {
+	if c.Plugins == nil {
+		return PluginEntry{}, false
+	}
+	return c.Plugins.plugin(category, name)
+}
+
+// UpsertPlugin creates or updates a plugin entry and saves the configuration.
+func (c *Config) UpsertPlugin(category string, entry PluginEntry) (context.UpsertResult, error) {
+	if c.Plugins == nil {
+		c.Plugins = &PluginConfig{}
+	}
+
+	result, err := c.Plugins.upsertPlugin(category, entry)
+	if err != nil {
+		return result, err
+	}
+
+	if result == context.Noop {
+		return result, nil
+	}
+
+	if err := c.validate(); err != nil {
+		return context.Noop, err
+	}
+
+	if err := c.saveConfig(); err != nil {
+		return context.Noop, fmt.Errorf("failed to save updated config: %w", err)
+	}
+
+	return result, nil
+}
+
+// DeletePlugin removes a plugin entry and saves the configuration.
+func (c *Config) DeletePlugin(category string, name string) (context.UpsertResult, error) {
+	if c.Plugins == nil {
+		return context.Noop, fmt.Errorf("no plugins configured")
+	}
+
+	result, err := c.Plugins.deletePlugin(category, name)
+	if err != nil {
+		return result, err
+	}
+
+	if result == context.Noop {
+		return result, err
+	}
+
+	if err := c.validate(); err != nil {
+		return context.Noop, err
+	}
+
+	if err := c.saveConfig(); err != nil {
+		return context.Noop, fmt.Errorf("failed to save updated config: %w", err)
+	}
+
+	return result, nil
+}
+
+// ListPlugins returns all plugins in a category.
+func (c *Config) ListPlugins(category string) []PluginEntry {
+	if c.Plugins == nil {
+		return nil
+	}
+	return c.Plugins.listPlugins(category)
+}
+
 // keyFor generates a temporary version of the ServerEntry to be used as a composite key.
 // It consists of the name of the server and the package without version information.
 func keyFor(entry ServerEntry) serverKey {
@@ -173,7 +242,17 @@ func (c *Config) validate() error {
 		return err
 	}
 
-	// TODO: Add more sub-validation as we add more parts to the config file.
+	if c.Daemon != nil {
+		if err := c.Daemon.Validate(); err != nil {
+			return fmt.Errorf("daemon configuration error: %w", err)
+		}
+	}
+
+	if c.Plugins != nil {
+		if err := c.Plugins.Validate(); err != nil {
+			return fmt.Errorf("plugin configuration error: %w", err)
+		}
+	}
 
 	return nil
 }