feat: improve HAProxy buffer config and add debug tooling

- Add tune.bufsize 65536 to haproxy.cfg and haproxy-upstreamproxy.cfg for 64KB body support
- Add stats socket for runtime debugging (/tmp/haproxy.sock)
- Lower body_within_limit to 50KB to stay safely under SPOE frame limit
- Remove redundant max-frame-size from crowdsec.cfg (tune.bufsize handles it)
- Add docker-compose.dev.yaml with debug container (tcpdump, socat, curl, etc.)
- Move per-message logging from Debug to Trace for quieter Debug mode
- Add Debug log for AppSec validation results

Author: LaurenceJJones

config/crowdsec.cfg

@@ -9,8 +9,8 @@ spoe-agent crowdsec-agent
 
     option      var-prefix      crowdsec
     option      set-on-error    error
-    timeout     hello           100ms
-    timeout     idle            30s
+    timeout     hello           200ms
+    timeout     idle            55s
     timeout     processing      500ms
     use-backend crowdsec-spoa
     log         global
@@ -22,12 +22,14 @@ spoe-message crowdsec-tcp
     event on-client-session
 
 ## HTTP message with body - used when body size is within limit for AppSec
+## Note: Host and captcha cookie are extracted from headers=req.hdrs, no need to send separately
 spoe-message crowdsec-http-body
-    args remediation=var(txn.crowdsec.remediation) crowdsec_captcha_cookie=req.cook(crowdsec_captcha_cookie) id=unique-id host=hdr(Host) method=method path=path query=query version=req.ver headers=req.hdrs body=req.body url=url ssl=ssl_fc src-ip=src src-port=src_port
+    args remediation=var(txn.crowdsec.remediation) id=unique-id method=method path=path query=query version=req.ver headers=req.hdrs body=req.body url=url ssl=ssl_fc src-ip=src src-port=src_port
 
 ## HTTP message without body - used when body is too large or not needed
+## Note: Host and captcha cookie are extracted from headers=req.hdrs, no need to send separately
 spoe-message crowdsec-http-no-body
-    args remediation=var(txn.crowdsec.remediation) crowdsec_captcha_cookie=req.cook(crowdsec_captcha_cookie) id=unique-id host=hdr(Host) method=method path=path query=query version=req.ver headers=req.hdrs url=url ssl=ssl_fc src-ip=src src-port=src_port
+    args remediation=var(txn.crowdsec.remediation) id=unique-id method=method path=path query=query version=req.ver headers=req.hdrs url=url ssl=ssl_fc src-ip=src src-port=src_port
 
 ## Group for HTTP message with body - used when body size is within limit for AppSec
 spoe-group crowdsec-http-body

config/haproxy-upstreamproxy.cfg

@@ -4,6 +4,9 @@
 
 global
     log stdout format raw local0
+    tune.bufsize 65536  # 64KB - increased for WAF body inspection
+    stats socket /tmp/haproxy.sock mode 660 level admin
+    stats timeout 30s
     lua-prepend-path /usr/lib/crowdsec-haproxy-spoa-bouncer/lua/?.lua
     lua-load /usr/lib/crowdsec-haproxy-spoa-bouncer/lua/crowdsec.lua
     setenv CROWDSEC_BAN_TEMPLATE_PATH /var/lib/crowdsec-haproxy-spoa-bouncer/html/ban.html
@@ -39,7 +42,7 @@ frontend test
     # ACL for body size limit (100000 bytes = ~100KB) - adjust here to change limit globally
     # Note spop protocol has limitations on the size of the message, so altering this value will not ensure the whole
     # body is sent for processing but should be enough to prevent overwhelming the SPOA bouncer.
-    acl body_within_limit req.body_size -m int le 100000
+    acl body_within_limit req.body_size -m int le 51200  # 50KB - stay safely under SPOE frame limit
 
     # Debug headers to verify IP extraction
     http-request set-header X-Debug-Direct-IP %[src]

config/haproxy.cfg

@@ -2,6 +2,8 @@
 global
     log stdout format raw local0
     tune.bufsize 65536  # 64KB - increased for WAF body inspection
+    stats socket /tmp/haproxy.sock mode 660 level admin
+    stats timeout 30s
     lua-prepend-path /usr/lib/crowdsec-haproxy-spoa-bouncer/lua/?.lua
     lua-load /usr/lib/crowdsec-haproxy-spoa-bouncer/lua/crowdsec.lua
     setenv CROWDSEC_BAN_TEMPLATE_PATH /var/lib/crowdsec-haproxy-spoa-bouncer/html/ban.html
@@ -29,7 +31,7 @@ frontend test
     # ACL for body size limit (100000 bytes = ~100KB) - adjust here to change limit globally
     # Note spop protocol has limitations on the size of the message, so altering this value will not ensure the whole
     # body is sent for processing but should be enough to prevent overwhelming the SPOA bouncer.
-    acl body_within_limit req.body_size -m int le 100000
+    acl body_within_limit req.body_size -m int le 51200  # 50KB - stay safely under SPOE frame limit
 
     ## If you don't want to render any content, you can use the following line
     # tcp-request content reject if !{ var(txn.crowdsec.remediation) -m str "allow" }

docker-compose.dev.yaml

@@ -0,0 +1,58 @@
+# Development/Debug overlay for docker-compose.yaml
+# Usage: podman compose -f docker-compose.yaml -f docker-compose.dev.yaml up -d
+#
+# This adds a debug container with network tools for troubleshooting
+
+services:
+  # Debug sidecar with network tools
+  debug:
+    image: alpine:latest
+    container_name: debug
+    command: >
+      sh -c "apk add --no-cache tcpdump socat curl bind-tools netcat-openbsd strace && sleep infinity"
+    networks:
+      - crowdsec
+    cap_add:
+      - NET_RAW      # Required for tcpdump
+      - NET_ADMIN    # Required for some network debugging
+    volumes:
+      # Mount shared sockets volume
+      - sockets:/run:ro
+      # Mount HAProxy tmp for stats socket access
+      - haproxy-tmp:/haproxy-tmp:ro
+      # Mount configs for inspection
+      - ./config:/config:ro
+    depends_on:
+      - haproxy
+      - spoa
+
+  # Override HAProxy to share /tmp via named volume
+  haproxy:
+    volumes:
+      - haproxy-tmp:/tmp
+
+volumes:
+  haproxy-tmp:
+
+# Example debug commands:
+# 
+# Enter debug container:
+#   podman compose -f docker-compose.yaml -f docker-compose.dev.yaml exec debug ash
+#
+# Install tools (once inside):
+#   apk add --no-cache tcpdump socat curl bind-tools netcat-openbsd strace
+#
+# Capture SPOE traffic:
+#   tcpdump -i any -X port 9000
+#
+# Test HAProxy stats socket:
+#   echo "show info" | socat /haproxy-tmp/haproxy.sock stdio
+#
+# DNS debugging:
+#   dig crowdsec
+#   nslookup spoa
+#
+# Test connectivity:
+#   curl -v http://haproxy:8080/
+#   nc -zv spoa 9000
+

pkg/spoa/root.go

@@ -162,7 +162,7 @@ func New(config *SpoaConfig) (*Spoa, error) {
 func (s *Spoa) HandleSPOE(ctx context.Context, writer *encoding.ActionWriter, message *encoding.Message) {
 	messageNameBytes := message.NameBytes()
 
-	s.logger.Debugf("Received message: %s", messageNameBytes)
+	s.logger.Tracef("Received message: %s", messageNameBytes)
 
 	switch {
 	case bytes.Equal(messageNameBytes, messageCrowdsecTCP): //TCP message type always runs so match it first
@@ -174,7 +174,7 @@ func (s *Spoa) HandleSPOE(ctx context.Context, writer *encoding.ActionWriter, me
 		s.handleHTTPRequest(ctx, writer, message)
 	default:
 		// Unknown message type
-		s.logger.Debugf("Unknown message type: %s", messageNameBytes)
+		s.logger.Tracef("Unknown message type: %s", messageNameBytes)
 	}
 }
 
@@ -431,6 +431,9 @@ func (s *Spoa) handleHTTPRequest(ctx context.Context, writer *encoding.ActionWri
 		httpMessageDataPool.Put(msgData)
 	}()
 
+	//log body length here
+	s.logger.Tracef("body length: %d", len(msgData.BodyCopied))
+
 	var tcpRemediation remediation.Remediation
 
 	// Get remediation passed from crowdsec-tcp handler (if any)
@@ -621,6 +624,8 @@ func (s *Spoa) validateWithAppSec(
 		return currentRemediation
 	}
 
+	logger.WithField("remediation", appSecRemediation.String()).Debug("AppSec validation result")
+
 	// Track AppSec block metrics
 	if appSecRemediation > remediation.Allow && appSecReq.RemoteIP != "" {
 		if ipAddr, parseErr := netip.ParseAddr(appSecReq.RemoteIP); parseErr == nil {