CF PCAP

jobs/rep/templates/bpm.yml.erb

@@ -2,6 +2,9 @@
 processes:
 - name: rep
   executable: /var/vcap/jobs/rep/bin/rep
+  # This is required to preserve the file capabilities of cf-pcap to allow the vcap user to capture
+  # packets inside the app container without root.
+  capabilities: [ "SETFCAP" ]
   limits:
     open_files: 100000
   hooks:

packages/buildpack_app_lifecycle/packaging

@@ -27,12 +27,16 @@ ldd $DEST/launcher && echo "launcher must be statically linked" && false
 cp /var/vcap/packages/diego-sshd/diego-sshd ${DEST}/diego-sshd
 cp /var/vcap/packages/diego-sshd/*.exe ${DEST}
 cp /var/vcap/packages/diego-sshd/winpty.dll ${DEST}/winpty.dll
+cp /var/vcap/packages/cf-pcap/cf-pcap ${DEST}/cf-pcap
 cp /var/vcap/packages/healthcheck/healthcheck ${DEST}/healthcheck
 cp /var/vcap/packages/healthcheck/healthcheck.exe ${DEST}/healthcheck.exe
 
+setcap cap_net_raw+ep ${DEST}/cf-pcap
+
 tar -czf ${BOSH_INSTALL_TARGET}/buildpack_app_lifecycle.tgz \
+  --xattrs --xattrs-include='*' \
   -C ${DEST} \
-  builder launcher shell healthcheck diego-sshd  \
+  builder launcher shell healthcheck diego-sshd cf-pcap  \
   builder.exe launcher.exe getenv.exe healthcheck.exe diego-sshd.exe \
   winpty-agent.exe winpty.dll
 

packages/buildpack_app_lifecycle/spec

@@ -5,6 +5,7 @@ dependencies:
   - golang-1.25-linux
   - healthcheck
   - diego-sshd
+  - cf-pcap
 
 files:
   - code.cloudfoundry.org/go.mod

packages/cf-pcap/packaging

@@ -0,0 +1,18 @@
+set -e
+
+pushd libpcap-1.10.5
+  ./configure
+  make
+popd
+
+source /var/vcap/packages/golang-*-linux/bosh/compile.env
+
+pushd code.cloudfoundry.org
+  # -I adds the local libpcap to the search path for the compiler
+  export CGO_CFLAGS="-I${BOSH_COMPILE_TARGET}/libpcap-1.10.5"
+  # -L adds the local libpcap to the search path for the linker
+  # -linkmode external: use an external linker: https://cs.opensource.google/go/go/+/refs/tags/go1.18:src/cmd/cgo/doc.go;l=794
+  # -extldflags -static: pass -static to ld, see man page for details
+  export CGO_LDFLAGS="-L${BOSH_COMPILE_TARGET}/libpcap-1.10.5 -static"
+  go build -ldflags '-linkmode external' -o ${BOSH_INSTALL_TARGET}/cf-pcap -a -installsuffix static code.cloudfoundry.org/cf-pcap
+popd

packages/cf-pcap/spec

@@ -0,0 +1,20 @@
+---
+name: cf-pcap
+dependencies:
+  - golang-1.25-linux
+
+files:
+  - libpcap-1.10.5/**/*
+  - code.cloudfoundry.org/go.mod
+  - code.cloudfoundry.org/go.sum
+  - code.cloudfoundry.org/vendor/modules.txt
+  - code.cloudfoundry.org/cf-pcap/*.go # gosub
+  - code.cloudfoundry.org/vendor/github.com/gopacket/gopacket/*.go # gosub
+  - code.cloudfoundry.org/vendor/github.com/gopacket/gopacket/endian/*.go # gosub
+  - code.cloudfoundry.org/vendor/github.com/gopacket/gopacket/layers/*.go # gosub
+  - code.cloudfoundry.org/vendor/github.com/gopacket/gopacket/pcap/*.go # gosub
+  - code.cloudfoundry.org/vendor/github.com/gopacket/gopacket/pcapgo/*.go # gosub
+  - code.cloudfoundry.org/vendor/golang.org/x/net/bpf/*.go # gosub
+  - code.cloudfoundry.org/vendor/golang.org/x/sys/unix/*.go # gosub
+  - code.cloudfoundry.org/vendor/golang.org/x/sys/unix/*.s # gosub
+  - code.cloudfoundry.org/vendor/golang.org/x/sys/windows/*.go # gosub

packages/cnb_app_lifecycle/packaging

@@ -20,8 +20,12 @@ ldd $DEST/builder && echo "builder must be statically linked" && false
 ldd $DEST/launcher && echo "launcher must be statically linked" && false
 
 cp /var/vcap/packages/diego-sshd/diego-sshd ${DEST}/diego-sshd
+cp /var/vcap/packages/cf-pcap/cf-pcap ${DEST}/cf-pcap
 cp /var/vcap/packages/healthcheck/healthcheck ${DEST}/healthcheck
 
+setcap cap_net_raw+ep ${DEST}/cf-pcap
+
 tar -czf ${BOSH_INSTALL_TARGET}/cnb_app_lifecycle.tgz \
+  --xattrs --xattrs-include='*' \
   -C ${DEST} \
-  builder launcher healthcheck diego-sshd
+  builder launcher healthcheck diego-sshd cf-pcap

packages/cnb_app_lifecycle/spec

@@ -5,6 +5,7 @@ dependencies:
   - golang-1.25-linux
   - healthcheck
   - diego-sshd
+  - cf-pcap
 
 files:
   - cnbapplifecycle/go.mod

packages/docker_app_lifecycle/packaging

@@ -19,6 +19,11 @@ ldd ${DEST}/builder && echo "builder must be statically linked" && false
 ldd ${DEST}/launcher && echo "launcher must be statically linked" && false
 
 cp /var/vcap/packages/diego-sshd/diego-sshd ${DEST}/diego-sshd
+cp /var/vcap/packages/cf-pcap/cf-pcap ${DEST}/cf-pcap
 cp /var/vcap/packages/healthcheck/healthcheck ${DEST}/healthcheck
 
-tar -czf ${BOSH_INSTALL_TARGET}/docker_app_lifecycle.tgz -C ${DEST} builder launcher healthcheck diego-sshd
+setcap cap_net_raw+ep ${DEST}/cf-pcap
+
+tar -czf ${BOSH_INSTALL_TARGET}/docker_app_lifecycle.tgz \
+  --xattrs --xattrs-include='*' \
+  -C ${DEST} builder launcher healthcheck diego-sshd cf-pcap

packages/docker_app_lifecycle/spec

@@ -5,6 +5,7 @@ dependencies:
   - golang-1.25-linux
   - healthcheck
   - diego-sshd
+  - cf-pcap
 
 files:
   - code.cloudfoundry.org/go.mod

src/code.cloudfoundry.org/cf-pcap/pcap.go

@@ -0,0 +1,137 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"log/slog"
+	"os"
+	"os/signal"
+	"syscall"
+
+	"github.com/gopacket/gopacket"
+	"github.com/gopacket/gopacket/pcap"
+	"github.com/gopacket/gopacket/pcapgo"
+)
+
+var (
+	interfaceName = flag.String("interface", "", "Network interface to capture from (e.g. eth0, any)")
+	snaplen       = flag.Int("snaplen", 65535, "Snapshot length - max bytes to capture per packet")
+	filter        = flag.String("filter", "", "BPF filter expression (e.g. 'tcp port 80')")
+	verbose       = flag.Bool("v", false, "Verbose output")
+)
+
+var (
+	logLevel = &slog.LevelVar{}
+)
+
+func init() {
+	slog.SetDefault(slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: logLevel})))
+}
+
+func main() {
+	os.Exit(Main())
+}
+
+func Main() int {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	flag.Parse()
+
+	if *interfaceName == "" {
+		slog.Error("interface flag is required")
+		return 1
+	}
+
+	if *verbose {
+		logLevel.Set(slog.LevelDebug)
+	}
+
+	slog.Debug("parsed flags",
+		"interface", *interfaceName,
+		"snaplen", *snaplen,
+		"filter", *filter,
+	)
+
+	errC := Capture(ctx, *interfaceName, *snaplen, *filter)
+
+	sigChan := make(chan os.Signal, 1)
+	signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
+
+	select {
+	case err := <-errC:
+		if err != nil {
+			slog.Error("capture failed", "error", err)
+			return 1
+		}
+	case sig := <-sigChan:
+		slog.Info("received signal, stopping capture", "signal", sig.String())
+		cancel()
+	}
+
+	// drain channel to ensure clean shutdown
+	for range errC {
+	}
+
+	return 0
+}
+
+func Capture(ctx context.Context, interfaceName string, snaplen int, filter string) <-chan error {
+	errC := make(chan error, 1)
+
+	handle, err := pcap.OpenLive(interfaceName, int32(snaplen), true, pcap.BlockForever)
+	if err != nil {
+		errC <- fmt.Errorf("failed to open device %s: %w", interfaceName, err)
+		return errC
+	}
+
+	if filter != "" {
+		// TODO: we should somehow filter out our own SSH traffic. This is not as easy because the
+		// connection is broken off more than once and we'd have to know the diego-sshd server port
+		// which is probably configurable.
+		err = handle.SetBPFFilter(filter)
+		if err != nil {
+			handle.Close()
+			errC <- fmt.Errorf("failed to set BPF filter '%s': %w", filter, err)
+			return errC
+		}
+	}
+
+	pcapWriter := pcapgo.NewWriter(os.Stdout)
+	err = pcapWriter.WriteFileHeader(uint32(snaplen), handle.LinkType())
+	if err != nil {
+		handle.Close()
+		errC <- fmt.Errorf("failed to write pcap header: %w", err)
+		return errC
+	}
+
+	go func(ctx context.Context, h *pcap.Handle, w *pcapgo.Writer, errC chan error) {
+		defer h.Close()
+		s := gopacket.NewPacketSource(h, h.LinkType())
+
+		var err error
+	packetLoop:
+		for {
+			select {
+			case <-ctx.Done():
+				h.Close()
+			case packet, ok := <-s.Packets():
+				if !ok {
+					break packetLoop
+				} else if packet == nil {
+					continue
+				}
+
+				err = w.WritePacket(packet.Metadata().CaptureInfo, packet.Data())
+				if err != nil {
+					break packetLoop
+				}
+			}
+		}
+
+		errC <- err
+	}(ctx, handle, pcapWriter, errC)
+
+	return errC
+}

src/code.cloudfoundry.org/go.mod

@@ -53,6 +53,7 @@ require (
 	github.com/golang-jwt/jwt/v4 v4.5.2
 	github.com/golang/protobuf v1.5.4
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
+	github.com/gopacket/gopacket v1.5.0
 	github.com/hashicorp/errwrap v1.1.0
 	github.com/hashicorp/go-multierror v1.1.1
 	github.com/jackc/pgx/v5 v5.8.0

src/code.cloudfoundry.org/go.sum

@@ -980,6 +980,8 @@ github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57Q
 github.com/googleapis/gax-go/v2 v2.7.1/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI=
 github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4=
 github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g=
+github.com/gopacket/gopacket v1.5.0 h1:9s9fcSUVKFlRV97B77Bq9XNV3ly2gvvsneFMQUGjc+M=
+github.com/gopacket/gopacket v1.5.0/go.mod h1:i3NaGaqfoWKAr1+g7qxEdWsmfT+MXuWkAe9+THv8LME=
 github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
 github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
 github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=