Merge pull request #465 from hookdeck/chore/azure-managed-redis

feat: support redis clusters

Author: alexluong

.env.example

@@ -17,6 +17,8 @@ REDIS_HOST="redis"
 REDIS_PORT="6379"
 REDIS_PASSWORD="password"
 REDIS_DATABASE="0"
+REDIS_TLS_ENABLED="false"         # Enable TLS encryption (required for Azure Managed Redis)
+REDIS_CLUSTER_ENABLED="false"     # Enable cluster mode (required for Azure Managed Redis)
 
 # ============================== Log Store ==============================
 

.gitignore

@@ -30,3 +30,4 @@ cmd/dev
 .cursor/
 .claude/
 *.local.*
+bin/

Makefile

@@ -1,5 +1,36 @@
 TEST?=$$(go list ./...)
 
+# Build targets
+.PHONY: build
+build:
+	@echo "Building all binaries..."
+	go build -o bin/outpost ./cmd/outpost
+	go build -o bin/outpost-server ./cmd/outpost-server
+	go build -o bin/outpost-migrate-redis ./cmd/outpost-migrate-redis
+	@echo "Binaries built in ./bin/"
+
+build/goreleaser:
+	goreleaser release -f ./build/.goreleaser.yaml --snapshot --clean
+
+build/outpost:
+	go build -o bin/outpost ./cmd/outpost
+
+build/server:
+	go build -o bin/outpost-server ./cmd/outpost-server
+
+build/migrate-redis:
+	go build -o bin/outpost-migrate-redis ./cmd/outpost-migrate-redis
+
+install:
+	@echo "Installing binaries to GOPATH/bin..."
+	go install ./cmd/outpost
+	go install ./cmd/outpost-server
+	go install ./cmd/outpost-migrate-redis
+	@echo "Installation complete"
+
+clean:
+	rm -f bin/outpost bin/outpost-server bin/outpost-migrate-redis
+
 up:
 	make up/deps
 	make up/outpost
@@ -53,8 +84,42 @@ up/test:
 down/test:
 	docker-compose -f build/test/compose.yml down --volumes
 
+up/test/rediscluster:
+	@echo "Ensuring test network exists..."
+	@docker network create outpost-test_default 2>/dev/null || true
+	@UNAME_S=$$(uname -s); \
+	if [ "$$UNAME_S" = "Darwin" ]; then \
+		REDIS_IMAGE=neohq/redis-cluster:latest; \
+		echo "Detected macOS, using neohq/redis-cluster image..."; \
+	else \
+		REDIS_IMAGE=grokzen/redis-cluster:7.2.4; \
+		echo "Using grokzen/redis-cluster image..."; \
+	fi; \
+	REDIS_IMAGE=$$REDIS_IMAGE docker-compose -f build/test/redis-cluster-compose.yml up -d
+	@echo "Starting Redis cluster and test runner containers..."
+	@echo "  - Redis cluster: 6 nodes (3 masters + 3 replicas)"
+	@echo "  - Test runner: Alpine container with Go code mounted"
+	@echo ""
+	@echo "Waiting for cluster to initialize..."
+	@sleep 10
+	@echo "Checking Redis cluster status:"
+	@docker exec redis-cluster redis-cli -p 7000 cluster info | grep cluster_state || echo "Failed to check cluster status"
+	@echo ""
+	@echo "Test environment ready. Run: make test/e2e/rediscluster"
+
+down/test/rediscluster:
+	@echo "Stopping Redis cluster test environment..."
+	@docker-compose -f build/test/redis-cluster-compose.yml down --volumes
+	@echo "Redis cluster test environment stopped."
+
 test/setup:
-	bash scripts/test-setup-info.sh
+	@echo "To setup the test environment, run the following command:"
+	@echo "$$ make up/test"
+	@echo "$$ make up/azure"
+	@echo ""
+	@echo "Before running the tests, make sure to:"
+	@echo "$$ export TESTINFRA=1 TESTAZURE=1"
+	@echo ""
 
 test:
 	go test $(TEST) $(TESTARGS)
@@ -65,6 +130,15 @@ test/unit:
 test/integration:
 	go test $(TEST) $(TESTARGS) -run "Integration"
 
+test/e2e/rediscluster:
+	@echo "Running Redis cluster e2e tests in Docker container..."
+	@if ! docker ps | grep -q test-runner; then \
+		echo "Error: test-runner container not running. Run 'make up/test/rediscluster' first."; \
+		exit 1; \
+	fi
+	@docker exec test-runner sh -c "cd /app && go test ./cmd/e2e -v -run TestRedisClusterBasicSuite"
+	@echo "Redis cluster e2e tests completed."
+
 test/race:
 	TESTRACE=1 go test $(TEST) $(TESTARGS) -race
 
@@ -77,6 +151,9 @@ test/coverage/html:
 docs/generate/config:
 	go run cmd/configdocsgen/main.go
 
+redis/debug:
+	go run cmd/redis-debug/main.go $(ARGS)
+
 network:
 	docker network create outpost
 

README.md

@@ -47,7 +47,7 @@ SDKs:
 
 Outpost is a self-hosted and open-source infrastructure that enables event producers to add outbound webhooks and [Event Destinations](https://eventdestinations.org?ref=github-outpost) to their platform with support for destination types such as Webhooks, Hookdeck Event Gateway, Amazon EventBridge, AWS SQS, AWS S3, GCP Pub/Sub, RabbitMQ, and Kafka.
 
-The Outpost runtime has minimal dependencies (Redis, PostgreSQL or Clickhouse, and one of the supported message queues), is 100% backward compatible with your existing webhooks implementation and is optimized for high-throughput, low-cost operation.
+The Outpost runtime has minimal dependencies (Redis or Redis cluster, PostgreSQL, and one of the supported message queues), is 100% backward compatible with your existing webhooks implementation, and is optimized for high-throughput, low-cost operation.
 
 Outpost is built and maintained by [Hookdeck](https://hookdeck.com?ref=github-outpost). It's written in Go and distributed as a binary and Docker container under the Apache-2.0 license.
 
@@ -118,6 +118,34 @@ cp .env.example .env
 
 Update the `$API_KEY` value within the new `.env` file.
 
+#### Redis Configuration
+
+Outpost supports both standard Redis and cluster Redis configurations:
+
+**Standard Redis** (default, for local development and single-node Redis):
+```env
+REDIS_HOST="redis"
+REDIS_PORT="6379" 
+REDIS_TLS_ENABLED="false"
+REDIS_CLUSTER_ENABLED="false"
+```
+
+**Redis Cluster** (for Redis Enterprise and managed Redis services):
+```env
+REDIS_HOST="your-redis-cluster.example.com"
+REDIS_PORT="10000"
+REDIS_TLS_ENABLED="true"
+REDIS_CLUSTER_ENABLED="true"
+```
+
+For other cloud Redis services or self-hosted Redis clusters, set `REDIS_CLUSTER_ENABLED="true"` if using Redis clustering.
+
+**Troubleshooting Redis connectivity**: Use the built-in diagnostic tool to test your Redis connection:
+```sh
+go run cmd/redis-debug/main.go your-redis-host 6379 password 0 [tls] [cluster]
+```
+See the [Redis Troubleshooting Guide](https://docs.outpost.hookdeck.com/references/troubleshooting-redis) for detailed guidance.
+
 Start the Outpost dependencies and services:
 
 ```sh

build/.goreleaser.yaml

@@ -7,7 +7,13 @@ before:
 
 project_name: outpost
 
+git:
+  # Ignore SDK tags when determining version
+  ignore_tags:
+    - 'sdks/*'
+
 builds:
+  # Orchestrator binary
   - id: outpost
     ldflags:
       - -s -w
@@ -16,7 +22,7 @@ builds:
     binary: outpost
     env:
       - CGO_ENABLED=0
-    main: ./cmd/outpost/main.go
+    main: ./cmd/outpost
     goos:
       - linux
     goarch:
@@ -29,7 +35,63 @@ builds:
     binary: outpost
     env:
       - CGO_ENABLED=0
-    main: ./cmd/outpost/main.go
+    main: ./cmd/outpost
+    goos:
+      - linux
+    goarch:
+      - arm64
+
+  # Server binary
+  - id: outpost-server
+    ldflags:
+      - -s -w
+      - -X github.com/hookdeck/outpost/internal/version.version={{.Version}}
+      - -X github.com/hookdeck/outpost/internal/version.commit={{.FullCommit}}
+    binary: outpost-server
+    env:
+      - CGO_ENABLED=0
+    main: ./cmd/outpost-server
+    goos:
+      - linux
+    goarch:
+      - amd64
+  - id: outpost-server-arm64
+    ldflags:
+      - -s -w
+      - -X github.com/hookdeck/outpost/internal/version.version={{.Version}}
+      - -X github.com/hookdeck/outpost/internal/version.commit={{.FullCommit}}
+    binary: outpost-server
+    env:
+      - CGO_ENABLED=0
+    main: ./cmd/outpost-server
+    goos:
+      - linux
+    goarch:
+      - arm64
+
+  # Migration binary
+  - id: outpost-migrate-redis
+    ldflags:
+      - -s -w
+      - -X github.com/hookdeck/outpost/internal/version.version={{.Version}}
+      - -X github.com/hookdeck/outpost/internal/version.commit={{.FullCommit}}
+    binary: outpost-migrate-redis
+    env:
+      - CGO_ENABLED=0
+    main: ./cmd/outpost-migrate-redis
+    goos:
+      - linux
+    goarch:
+      - amd64
+  - id: outpost-migrate-redis-arm64
+    ldflags:
+      - -s -w
+      - -X github.com/hookdeck/outpost/internal/version.version={{.Version}}
+      - -X github.com/hookdeck/outpost/internal/version.commit={{.FullCommit}}
+    binary: outpost-migrate-redis
+    env:
+      - CGO_ENABLED=0
+    main: ./cmd/outpost-migrate-redis
     goos:
       - linux
     goarch:
@@ -63,6 +125,10 @@ dockers:
     dockerfile: ./build/Dockerfile.goreleaser
     ids:
       - outpost
+      - outpost-server
+      - outpost-migrate-redis
+    extra_files:
+      - build/entrypoint.sh
     image_templates:
       - "hookdeck/outpost:latest-amd64"
       - "hookdeck/outpost:{{ .Tag }}-amd64"
@@ -80,6 +146,10 @@ dockers:
     dockerfile: ./build/Dockerfile.goreleaser
     ids:
       - outpost-arm64
+      - outpost-server-arm64
+      - outpost-migrate-redis-arm64
+    extra_files:
+      - build/entrypoint.sh
     image_templates:
       - "hookdeck/outpost:latest-arm64"
       - "hookdeck/outpost:{{ .Tag }}-arm64"

build/Dockerfile

@@ -1,13 +1,20 @@
 # Stage 0
-# Build the binary
+# Build the binaries
 FROM golang:1.23-alpine
 WORKDIR /app
 COPY . .
 
-RUN go build -o ./bin/outpost ./cmd/outpost/main.go
+# Build all binaries
+RUN go build -o ./bin/outpost ./cmd/outpost/main.go && \
+    go build -o ./bin/outpost-server ./cmd/outpost-server/main.go && \
+    go build -o ./bin/outpost-migrate-redis ./cmd/outpost-migrate-redis/main.go
 
 # Stage 1
-# Copy binary to a new image
+# Copy binaries to a new image
 FROM scratch
 COPY --from=0 /app/bin/outpost /bin/outpost
-ENTRYPOINT ["/bin/outpost"]
+COPY --from=0 /app/bin/outpost-server /bin/outpost-server
+COPY --from=0 /app/bin/outpost-migrate-redis /bin/outpost-migrate-redis
+
+# Default to running the server
+ENTRYPOINT ["/bin/outpost", "serve"]

build/Dockerfile.goreleaser

@@ -1,3 +1,17 @@
-FROM gcr.io/distroless/base
-COPY outpost outpost
-ENTRYPOINT ["./outpost"]
+FROM busybox:1.36-musl AS busybox
+
+FROM gcr.io/distroless/base-debian12:nonroot
+
+# Copy statically linked shell from busybox for entrypoint script
+COPY --from=busybox /bin/sh /bin/sh
+
+# Copy all binaries
+COPY outpost /usr/local/bin/outpost
+COPY outpost-server /usr/local/bin/outpost-server
+COPY outpost-migrate-redis /usr/local/bin/outpost-migrate-redis
+
+# Copy entrypoint script
+COPY build/entrypoint.sh /usr/local/bin/entrypoint.sh
+
+# Default entrypoint runs migrations and starts server
+ENTRYPOINT ["/bin/sh", "/usr/local/bin/entrypoint.sh"]

build/dev/Dockerfile

@@ -2,4 +2,4 @@ FROM golang:1.23-alpine AS fetch
 RUN go install github.com/air-verse/[email protected]
 WORKDIR /app
 COPY . .
-CMD ["air"]
+ENTRYPOINT ["sh", "./build/dev/entrypoint.sh"]

build/dev/entrypoint.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+set -e
+
+echo "- Running: go run ./cmd/outpost-migrate-redis init --current"
+
+go run ./cmd/outpost-migrate-redis init --current
+
+echo "- Running: air serve"
+
+exec air serve

build/entrypoint.sh

@@ -0,0 +1,32 @@
+#!/bin/sh
+set -e
+
+# If any arguments are passed, delegate to outpost binary
+# Examples: docker run hookdeck/outpost serve
+#           docker run hookdeck/outpost migrate plan
+#           docker run hookdeck/outpost --version
+#           docker run hookdeck/outpost --help
+if [ "$#" -gt 0 ]; then
+    exec /usr/local/bin/outpost "$@"
+fi
+
+# Default behavior when no args: check migrations, then start server
+
+echo "Checking database migrations..."
+if ! /usr/local/bin/outpost migrate init --current --log-format=json; then
+    echo ""
+    echo "ERROR: Database migrations are pending."
+    echo "Please run migrations before starting the server:"
+    echo "  docker run --rm -it hookdeck/outpost migrate apply"
+    echo ""
+    echo "For help with migration commands and configuration:"
+    echo "  docker run --rm hookdeck/outpost migrate --help"
+    echo ""
+    echo "Learn more about Outpost migration workflow at:"
+    echo "  https://outpost.hookdeck.com/docs/guides/migration"
+    echo ""
+    exit 1
+fi
+
+echo "Starting Outpost server..."
+exec /usr/local/bin/outpost serve