feat: add configurable HTTP header policies via new headers middleware

- Introduces a new `headers` configuration section where admins can define
 arbitrary HTTP headers per endpoint using regex path patterns.
- Policies are evaluated in order; the first match wins, allowing fine-grained
 control over Cache-Control, Vary, security and custom headers.
- Removes hard-coded Cache-Control headers from API config handler, proxy
 responses and frontend static serving so they are now driven by policy.
- Adds comprehensive tests for policy loading, matching logic, middleware
 integration and concurrent usage.

Author: mattevans

config.example.yaml

@@ -133,3 +133,59 @@ features:
 
   # Example: Attestation performance - enabled for all networks (disabled_networks omitted)
   - path: "/ethereum/attestation-performance"
+
+# HTTP Headers Configuration
+# Allows setting arbitrary HTTP headers per endpoint based on path patterns
+# Policies are evaluated in order - first match wins
+# Use this for Cache-Control, Vary, security headers, custom headers, etc.
+headers:
+  policies:
+    # Static assets (JS, CSS, images, fonts) - aggressive caching
+    # Pattern matches file extensions for immutable assets with content hashing
+    - name: "static_assets"
+      path_pattern: "\\.(js|css|png|jpg|jpeg|gif|svg|woff|woff2|ttf|eot|ico)$"
+      headers:
+        Cache-Control: "public, max-age=31536000, immutable"
+        # Vary: "Accept-Encoding"  # Example: vary cache on encoding
+
+    # HTML pages (index.html) - minimal caching with stale-while-revalidate
+    # Pattern ensures dynamic content gets refreshed quickly
+    - name: "html_pages"
+      path_pattern: "\\.html$"
+      headers:
+        Cache-Control: "public, max-age=1, s-maxage=5, stale-while-revalidate=1"
+
+    # API config endpoint - moderate CDN caching with long stale window
+    # Config changes infrequently so longer stale serving is acceptable
+    - name: "api_config"
+      path_pattern: "^/api/v1/config$"
+      headers:
+        Cache-Control: "public, max-age=1, s-maxage=60, stale-while-revalidate=300"
+
+    # API proxy responses - short caching with stale-while-revalidate
+    # Proxied API responses should be relatively fresh
+    - name: "api_proxy"
+      path_pattern: "^/api/v1/.+/.+"
+      headers:
+        Cache-Control: "public, max-age=1, s-maxage=5, stale-while-revalidate=1"
+
+    # Health and metrics - no caching
+    # Monitoring endpoints must always be fresh
+    - name: "health_metrics"
+      path_pattern: "^/(health|metrics)$"
+      headers:
+        Cache-Control: "no-cache, no-store, must-revalidate"
+
+    # Example: Custom headers for specific endpoint
+    # - name: "custom_endpoint"
+    #   path_pattern: "^/api/v1/special$"
+    #   headers:
+    #     Cache-Control: "public, max-age=300"
+    #     X-Custom-Header: "my-value"
+    #     Vary: "Accept-Encoding, Accept-Language"
+
+    # Default fallback - minimal caching for everything else
+    - name: "default"
+      path_pattern: ".*"
+      headers:
+        Cache-Control: "public, max-age=1"

internal/api/config.go

@@ -84,7 +84,6 @@ func (h *ConfigHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 
 	// Set headers.
 	w.Header().Set("Content-Type", "application/json")
-	w.Header().Set("Cache-Control", "public, max-age=1, s-maxage=60, stale-while-revalidate=300")
 
 	// Encode response
 	if err := json.NewEncoder(w).Encode(response); err != nil {

internal/config/config.go

@@ -23,6 +23,7 @@ type Config struct {
 	Cartographoor cartographoor.Config `yaml:"cartographoor"`
 	Bounds        BoundsConfig         `yaml:"bounds"`
 	RateLimiting  RateLimitingConfig   `yaml:"rate_limiting"`
+	Headers       HeadersConfig        `yaml:"headers"`
 }
 
 // ServerConfig contains HTTP server settings.
@@ -77,6 +78,18 @@ type RateLimitRule struct {
 	Window      time.Duration `yaml:"window"`       // Time window
 }
 
+// HeadersConfig holds HTTP headers configuration.
+type HeadersConfig struct {
+	Policies []HeaderPolicy `yaml:"policies"`
+}
+
+// HeaderPolicy defines headers to set for matching request paths.
+type HeaderPolicy struct {
+	Name        string            `yaml:"name"`         // Policy name for logging/debugging
+	PathPattern string            `yaml:"path_pattern"` // Regex pattern to match request paths
+	Headers     map[string]string `yaml:"headers"`      // Headers to set (key: value)
+}
+
 // Validate validates the configuration and sets defaults.
 func (c *BoundsConfig) Validate() error {
 	// Set defaults

internal/config/headers_test.go

@@ -0,0 +1,209 @@
+package config
+
+import (
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"gopkg.in/yaml.v3"
+)
+
+func TestHeadersConfig_Loading(t *testing.T) {
+	yamlConfig := `
+server:
+  port: 8080
+  host: "0.0.0.0"
+  read_timeout: 30s
+  write_timeout: 30s
+  shutdown_timeout: 10s
+  log_level: "info"
+
+redis:
+  address: "localhost:6379"
+  password: ""
+  db: 0
+  dial_timeout: 5s
+  read_timeout: 3s
+  write_timeout: 3s
+  pool_size: 10
+
+leader:
+  lock_key: "lab:leader:lock"
+  lock_ttl: 30s
+  renew_interval: 10s
+  retry_interval: 5s
+
+cartographoor:
+  source_url: "https://example.com/networks.json"
+  refresh_interval: 5m
+  request_timeout: 30s
+  networks_ttl: 0s
+
+bounds:
+  refresh_interval: 10s
+  request_timeout: 30s
+  bounds_ttl: 0s
+
+rate_limiting:
+  enabled: false
+
+headers:
+  policies:
+    - name: "static_assets"
+      path_pattern: "\\.(js|css|png)$"
+      headers:
+        Cache-Control: "public, max-age=31536000, immutable"
+        Vary: "Accept-Encoding"
+
+    - name: "api_config"
+      path_pattern: "^/api/v1/config$"
+      headers:
+        Cache-Control: "public, max-age=1, s-maxage=60"
+
+    - name: "default"
+      path_pattern: ".*"
+      headers:
+        Cache-Control: "public, max-age=1"
+`
+
+	var cfg Config
+
+	err := yaml.Unmarshal([]byte(yamlConfig), &cfg)
+	require.NoError(t, err)
+
+	// Verify headers config loaded
+	assert.Len(t, cfg.Headers.Policies, 3)
+
+	// Verify first policy
+	assert.Equal(t, "static_assets", cfg.Headers.Policies[0].Name)
+	assert.Equal(t, `\.(js|css|png)$`, cfg.Headers.Policies[0].PathPattern)
+	assert.Len(t, cfg.Headers.Policies[0].Headers, 2)
+	assert.Equal(t, "public, max-age=31536000, immutable", cfg.Headers.Policies[0].Headers["Cache-Control"])
+	assert.Equal(t, "Accept-Encoding", cfg.Headers.Policies[0].Headers["Vary"])
+
+	// Verify second policy
+	assert.Equal(t, "api_config", cfg.Headers.Policies[1].Name)
+	assert.Equal(t, "^/api/v1/config$", cfg.Headers.Policies[1].PathPattern)
+	assert.Len(t, cfg.Headers.Policies[1].Headers, 1)
+	assert.Equal(t, "public, max-age=1, s-maxage=60", cfg.Headers.Policies[1].Headers["Cache-Control"])
+
+	// Verify third policy
+	assert.Equal(t, "default", cfg.Headers.Policies[2].Name)
+	assert.Equal(t, ".*", cfg.Headers.Policies[2].PathPattern)
+}
+
+func TestExampleConfig_LoadsSuccessfully(t *testing.T) {
+	// Load the actual config.example.yaml file
+	cfg, err := Load("../../config.example.yaml")
+	require.NoError(t, err)
+
+	// Verify headers section exists and has policies
+	assert.NotEmpty(t, cfg.Headers.Policies, "config.example.yaml should have header policies")
+
+	// Log what we found
+	t.Logf("Loaded %d header policies from config.example.yaml", len(cfg.Headers.Policies))
+
+	for i, policy := range cfg.Headers.Policies {
+		t.Logf("  Policy %d: %s with %d headers", i+1, policy.Name, len(policy.Headers))
+	}
+}
+
+func TestHeadersConfig_EmptyPolicies(t *testing.T) {
+	yamlConfig := `
+server:
+  port: 8080
+  host: "0.0.0.0"
+  read_timeout: 30s
+  write_timeout: 30s
+  shutdown_timeout: 10s
+  log_level: "info"
+
+redis:
+  address: "localhost:6379"
+  dial_timeout: 5s
+  pool_size: 10
+
+leader:
+  lock_key: "lab:leader:lock"
+  lock_ttl: 30s
+  renew_interval: 10s
+  retry_interval: 5s
+
+cartographoor:
+  source_url: "https://example.com/networks.json"
+  refresh_interval: 5m
+  request_timeout: 30s
+
+bounds:
+  refresh_interval: 10s
+  request_timeout: 30s
+
+rate_limiting:
+  enabled: false
+
+headers:
+  policies: []
+`
+
+	var cfg Config
+
+	err := yaml.Unmarshal([]byte(yamlConfig), &cfg)
+	require.NoError(t, err)
+
+	// Empty policies should be fine
+	assert.Empty(t, cfg.Headers.Policies)
+}
+
+func TestHeadersConfig_NoHeadersSection(t *testing.T) {
+	// Create minimal config without headers section
+	tmpFile, err := os.CreateTemp("", "config-*.yaml")
+	require.NoError(t, err)
+
+	defer os.Remove(tmpFile.Name())
+
+	yamlConfig := `
+server:
+  port: 8080
+  host: "0.0.0.0"
+  read_timeout: 30s
+  write_timeout: 30s
+  shutdown_timeout: 10s
+  log_level: "info"
+
+redis:
+  address: "localhost:6379"
+  dial_timeout: 5s
+  read_timeout: 3s
+  write_timeout: 3s
+  pool_size: 10
+
+leader:
+  lock_key: "lab:leader:lock"
+  lock_ttl: 30s
+  renew_interval: 10s
+  retry_interval: 5s
+
+cartographoor:
+  source_url: "https://example.com/networks.json"
+  refresh_interval: 5m
+  request_timeout: 30s
+
+bounds:
+  refresh_interval: 10s
+  request_timeout: 30s
+
+rate_limiting:
+  enabled: false
+`
+
+	_, err = tmpFile.WriteString(yamlConfig)
+	require.NoError(t, err)
+	tmpFile.Close()
+
+	cfg, err := Load(tmpFile.Name())
+	require.NoError(t, err)
+
+	// Missing headers section should result in empty policies
+	assert.Empty(t, cfg.Headers.Policies)
+}

internal/frontend/frontend.go

@@ -173,8 +173,7 @@ func (f *Frontend) serveIndex(w http.ResponseWriter, r *http.Request) {
 		"content_length": len(html),
 	}).Debug("Serving route-specific cached index.html")
 
-	// Set cache headers for index.html (contains config and bounds data)
-	w.Header().Set("Cache-Control", "public, max-age=1, s-maxage=5, stale-while-revalidate=1")
+	// Set content type for index.html
 	w.Header().Set("Content-Type", "text/html; charset=utf-8")
 	w.WriteHeader(http.StatusOK)
 
@@ -219,12 +218,6 @@ func (f *Frontend) setCacheHeaders(w http.ResponseWriter, filePath string) {
 	}
 
 	w.Header().Set("Content-Type", contentType)
-
-	// Static assets get long cache (1 year)
-	// index.html gets no-cache (handled in serveIndex)
-	if !strings.HasSuffix(filePath, "index.html") {
-		w.Header().Set("Cache-Control", "public, max-age=31536000, immutable")
-	}
 }
 
 // refreshLoop listens for bounds and cartographoor update notifications and refreshes the cached index.html.