Add docs for remote MCP servers (#413)

* Add docs for remote MCP servers

* Update modules/ai-agents/pages/mcp/remote/developer-guide.adoc

Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>

* Add period

* Apply suggestions from code review

* Update modules/ai-agents/pages/mcp/remote/developer-guide.adoc

* Update nav

* Update nav

* Apply suggestions from code review

* Simplify

* Update configuration.adoc

* Update overview.adoc

* Update headings to be task oriented

* Require 25.2.5

* Improve overview

* Improvements

* Split examples

* Clarify mcp client

* Pipeline -> configuation

* Use templates for quickstart

* Use templates for quickstart

* Fix grammar

* Update modules/ai-agents/pages/mcp/remote/overview.adoc

Co-authored-by: Paulo Borges <[email protected]>

* Apply suggestions from code review

Co-authored-by: Paulo Borges <[email protected]>

* Add lint details

* Update whats new

* Clarify docs MCP

* Document property restrictions

* Remove private beta

* Remove private beta

* Fix formatting

---------

Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>
Co-authored-by: Paulo Borges <[email protected]>

Author: 3 people

modules/ROOT/nav.adoc

@@ -69,10 +69,18 @@
 ** xref:security:secrets.adoc[Secrets]
 ** xref:security:cloud-safety-reliability.adoc[Safety and Reliability]
 
-* xref:ai-agents:mcp/local/index.adoc[]
-** xref:ai-agents:mcp/local/overview.adoc[Overview]
-** xref:ai-agents:mcp/local/quickstart.adoc[Quickstart]
-** xref:ai-agents:mcp/local/configuration.adoc[Configure]
+* xref:ai-agents:index.adoc[AI Agents]
+** xref:ai-agents:mcp/overview.adoc[MCP Overview]
+** xref:ai-agents:mcp/local/index.adoc[Local Redpanda Cloud MCP]
+*** xref:ai-agents:mcp/local/overview.adoc[Overview]
+*** xref:ai-agents:mcp/local/quickstart.adoc[Quickstart]
+*** xref:ai-agents:mcp/local/configuration.adoc[Configure]
+** xref:ai-agents:mcp/remote/index.adoc[Remote MCP]
+*** xref:ai-agents:mcp/remote/overview.adoc[Overview]
+*** xref:ai-agents:mcp/remote/quickstart.adoc[Quickstart]
+*** xref:ai-agents:mcp/remote/developer-guide.adoc[Developer Guide]
+*** xref:ai-agents:mcp/remote/admin-guide.adoc[Admin Guide]
+*** xref:ai-agents:mcp/remote/pipeline-patterns.adoc[MCP Server Patterns]
 
 * xref:develop:connect/about.adoc[Redpanda Connect]
 ** xref:develop:connect/connect-quickstart.adoc[Quickstart]
@@ -490,8 +498,9 @@
 ***** xref:reference:rpk/rpk-cloud/rpk-cloud-cluster-select.adoc[]
 **** xref:reference:rpk/rpk-cloud/rpk-cloud-login.adoc[]
 **** xref:reference:rpk/rpk-cloud/rpk-cloud-logout.adoc[]
-**** rpk cloud mcp
+**** xref:reference:rpk/rpk-cloud/rpk-cloud-mcp.adoc[]
 ***** xref:reference:rpk/rpk-cloud/rpk-cloud-mcp-install.adoc[]
+***** xref:reference:rpk/rpk-cloud/rpk-cloud-mcp-proxy.adoc[]
 ***** xref:reference:rpk/rpk-cloud/rpk-cloud-mcp-stdio.adoc[]
 *** xref:reference:rpk/rpk-cluster/rpk-cluster.adoc[]
 **** xref:reference:rpk/rpk-cluster/rpk-cluster-config.adoc[]

modules/ai-agents/examples/customer_enrichment.yaml

@@ -0,0 +1,39 @@
+label: customer_enrichment
+processors:
+  - label: fetch_customer_base
+    sql_select:
+      driver: "postgres"
+      dsn: "${secrets.POSTGRES_DSN}"
+      table: "customers"
+      where: "customer_id = ?"
+      args_mapping: 'root = [this.customer_id]'
+
+  - label: enrich_with_orders
+    sql_select:
+      driver: "postgres"
+      dsn: "${secrets.POSTGRES_DSN}"
+      table: "orders"
+      where: "customer_id = ? AND created_at >= NOW() - INTERVAL '30 days'"
+      args_mapping: 'root = [this.customer_id]'
+
+  - label: combine_data
+    mutation: |
+      root = {
+        "customer": this.customers.index(0),
+        "recent_orders": this.orders,
+        "metrics": {
+          "total_orders": this.orders.length(),
+          "total_spent": this.orders.map_each(o -> o.total).sum(),
+          "avg_order_value": this.orders.map_each(o -> o.total).mean()
+        }
+      }
+
+meta:
+  mcp:
+    enabled: true
+    description: "Get comprehensive customer profile with recent order history and metrics"
+    properties:
+      - name: customer_id
+        type: string
+        description: "Customer ID to analyze"
+        required: true

modules/ai-agents/examples/gcp_bigquery_select_processor.yaml

@@ -0,0 +1,45 @@
+label: gcp_bigquery_select_processor
+processors:
+  - label: prepare_parameters
+    mutation: |
+      meta customer_id = this.customer_id.string().catch("12345")
+      meta limit = this.limit.number().catch(10)
+  - label: query_bigquery
+    gcp_bigquery_select:
+      project: my-gcp-project
+      credentials_json: |
+        ${secrets.BIGQUERY_CREDENTIALS}
+      table: my_dataset.customer_orders
+      columns:
+        - "order_id"
+        - "customer_id"
+        - "order_date"
+        - "total_amount"
+        - "status"
+      where: customer_id = ? AND order_date >= DATE_SUB(CURRENT_DATE(), INTERVAL 30 DAY)
+      suffix: "ORDER BY order_date DESC LIMIT ?"
+      args_mapping: root = [ @customer_id, @limit ]
+  - label: format_response
+    mutation: |
+      root = {
+        "orders": this,
+        "metadata": {
+          "source": "BigQuery",
+          "customer_id": @customer_id,
+          "fetched_at": now().ts_format("2006-01-02T15:04:05.000Z")
+        }
+      }
+
+meta:
+  mcp:
+    enabled: true
+    description: "Query customer orders from BigQuery"
+    properties:
+      - name: customer_id
+        type: string
+        description: "Customer ID to filter orders"
+        required: true
+      - name: limit
+        type: number
+        description: "Maximum number of orders to return"
+        required: false

modules/ai-agents/examples/generate_input.yaml

@@ -0,0 +1,22 @@
+label: generate_input
+generate:
+  mapping: |
+    let event_type = ["login", "logout", "purchase", "view_page", "click_button"].index(random_int(max:4))
+    root = {
+      "id": uuid_v4(),
+      "timestamp": now().ts_format("2006-01-02T15:04:05.000Z"),
+      "user_id": random_int(min:1, max:10000),
+      "event_type": $event_type,
+      "data": {
+        "session_id": ksuid(),
+        "ip_address": "192.168.%v.%v".format(random_int(max:255), random_int(min:1, max:254)),
+        "user_agent": ["Chrome", "Firefox", "Safari", "Edge"].index(random_int(max:3)),
+        "amount": if $event_type == "purchase" { random_int(min:10, max:500) } else { null }
+      }
+    }
+
+meta:
+  mcp:
+    enabled: true
+    description: "Generate an example user event message with realistic data"
+    properties: []

modules/ai-agents/examples/http_processor.yaml

@@ -0,0 +1,37 @@
+label: fetch-weather
+processors:
+  - label: prepare_parameters
+    mutation: |
+      meta city_name = this.city_name
+  - label: fetch_weather
+    http:
+      url: 'https://wttr.in/${! @city_name }?format=j1'
+      verb: GET
+      headers:
+        Accept: "application/json"
+        User-Agent: "redpanda-mcp-server/1.0"
+  - label: format_response
+    mutation: |
+      root = {
+        "city": @city_name,
+        "temperature": this.current_condition.0.temp_C.number(),
+        "feels_like": this.current_condition.0.FeelsLikeC.number(),
+        "humidity": this.current_condition.0.humidity.number(),
+        "pressure": this.current_condition.0.pressure.number(),
+        "description": this.current_condition.0.weatherDesc.0.value,
+        "wind_speed": this.current_condition.0.windspeedKmph.number(),
+        "metadata": {
+          "source": "wttr.in",
+          "fetched_at": now().ts_format("2006-01-02T15:04:05.000Z")
+        }
+      }
+
+meta:
+  mcp:
+    enabled: true
+    description: "Fetch current weather information for a specified city"
+    properties:
+      - name: city_name
+        type: string
+        description: "Name of the city to get weather information for"
+        required: true

modules/ai-agents/examples/memory_cache.yaml

@@ -0,0 +1,14 @@
+label: memory_cache
+memory:
+  default_ttl: "5m"
+  init_values:
+    "user:1001": '{"name": "Alice", "role": "admin"}'
+    "user:1002": '{"name": "Bob", "role": "user"}'
+    "config:theme": "dark"
+    "config:language": "en"
+  shards: 4
+
+meta:
+  mcp:
+    enabled: true
+    description: "In-memory cache for storing user data, configuration, and temporary values"

modules/ai-agents/examples/observable_tool.yaml

@@ -0,0 +1,60 @@
+label: observable_tool
+processors:
+  - label: init_tracing
+    mutation: |
+      # Generate correlation ID for request tracing
+      meta req_id = uuid_v7()
+      meta start_time = now()
+
+      # Log request start with structured data
+      root.trace = {
+        "request_id": @req_id,
+        "timestamp": @start_time.ts_format("2006-01-02T15:04:05.000Z"),
+        "tool": "observable_tool",
+        "version": "1.0.0"
+      }
+
+  - label: log_request_start
+    log:
+      message: "MCP tool request started"
+      fields:
+        request_id: "${! @req_id }"
+        tool_name: "observable_tool"
+        input_params: "${! this.without(\"trace\") }"
+        user_agent: "${! meta(\"User-Agent\").catch(\"unknown\") }"
+      level: "INFO"
+
+  - label: finalize_response
+    mutation: |
+      # Calculate total execution time
+      meta duration = (now().ts_unix_nano() - @start_time.ts_unix_nano()) / 1000000
+
+      # Add trace information to response
+      root.metadata = {
+        "request_id": @req_id,
+        "execution_time_ms": @duration,
+        "timestamp": now().ts_format("2006-01-02T15:04:05.000Z"),
+        "tool": "observable_tool",
+        "success": !this.exists("error")
+      }
+
+  - label: log_completion
+    log:
+      message: "MCP tool request completed"
+      fields:
+        request_id: "${! @req_id }"
+        duration_ms: "${! this.metadata.execution_time_ms }"
+        success: "${! this.metadata.success }"
+        result_size: "${! content().length() }"
+      level: "INFO"
+
+meta:
+  tags: [ example ]
+  mcp:
+    enabled: true
+    description: "Example tool with comprehensive observability and error handling"
+    properties:
+      - name: user_id
+        type: string
+        description: "User ID to fetch data for"
+        required: true

modules/ai-agents/examples/order_workflow.yaml

@@ -0,0 +1,107 @@
+label: order_workflow
+processors:
+  - label: validate_order
+    mutation: |
+      # Validation logic
+      root = if this.total <= 0 {
+        throw("Invalid order total")
+      } else { this }
+
+  - label: mock_inventory_check
+    mutation: |
+      # Mock inventory data for testing
+      let inventory = {
+        "widget-001": {"quantity": 100, "name": "Standard Widget"},
+        "widget-premium": {"quantity": 25, "name": "Premium Widget"},
+        "widget-limited": {"quantity": 2, "name": "Limited Edition Widget"}
+      }
+
+      let product = $inventory.get(this.product_id)
+      root = if $product == null {
+        throw("Product not found: " + this.product_id)
+      } else if $product.quantity < this.quantity {
+        throw("Insufficient inventory. Available: " + $product.quantity.string())
+      } else {
+        this.merge({
+          "inventory_check": "passed",
+          "available_quantity": $product.quantity,
+          "product_name": $product.name
+        })
+      }
+
+  - label: route_by_priority
+    switch:
+      - check: 'this.total > 1000'
+        processors:
+          - label: mock_high_value_processing
+            mutation: |
+              # Mock premium processing
+              root = this.merge({
+                "processing_tier": "premium",
+                "processing_time_estimate": "2-4 hours",
+                "assigned_rep": "[email protected]",
+                "priority_score": 95
+              })
+
+      - check: 'this.customer_tier == "vip"'
+        processors:
+          - label: mock_vip_processing
+            mutation: |
+              # Mock VIP processing
+              root = this.merge({
+                "processing_tier": "vip",
+                "processing_time_estimate": "1-2 hours",
+                "assigned_rep": "[email protected]",
+                "priority_score": 90,
+                "perks": ["expedited_shipping", "white_glove_service"]
+              })
+
+      - processors:
+          - label: mock_standard_processing
+            mutation: |
+              # Mock standard processing
+              root = this.merge({
+                "processing_tier": "standard",
+                "processing_time_estimate": "24-48 hours",
+                "assigned_rep": "[email protected]",
+                "priority_score": 50
+              })
+
+  - label: finalize_order
+    mutation: |
+      # Add final processing metadata
+      # Calculate estimated fulfillment by parsing processing time
+      let max_hours = this.processing_time_estimate.split("-").index(1).split(" ").index(0).number()
+
+      root = this.merge({
+        "order_status": "processed",
+        "processed_at": now().ts_format("2006-01-02T15:04:05.000Z"),
+        "estimated_fulfillment": "TBD - calculated based on processing tier",
+        "processing_time_hours": $max_hours
+      })
+
+meta:
+  mcp:
+    enabled: true
+    description: "Process orders with validation, inventory check, and tiered routing (with mocks for testing)"
+    properties:
+      - name: order_id
+        type: string
+        description: "Unique order identifier"
+        required: true
+      - name: product_id
+        type: string
+        description: "Product ID (try: widget-001, widget-premium, widget-limited)"
+        required: true
+      - name: quantity
+        type: number
+        description: "Quantity to order"
+        required: true
+      - name: total
+        type: number
+        description: "Order total in dollars"
+        required: true
+      - name: customer_tier
+        type: string
+        description: "Customer tier (optional: vip, standard)"
+        required: false

modules/ai-agents/examples/redpanda_cache.yaml

@@ -0,0 +1,15 @@
+label: redpanda_cache
+redpanda:
+  seed_brokers: ["${REDPANDA_BROKERS}"]
+  topic: "mcp-cache-topic"
+  tls:
+    enabled: true
+  sasl:
+    - mechanism: "SCRAM-SHA-512"
+      username: "${secrets.MCP_REDPANDA_CREDENTIALS.username}"
+      password: "${secrets.MCP_REDPANDA_CREDENTIALS.password}"
+
+meta:
+  mcp:
+    enabled: true
+    description: "Redpanda-backed distributed cache using Kafka topics for persistence"