ROCSP: Replace Redis Cluster with a consistently sharded all-primary nodes (#6516)

Author: beautifulentropy

cmd/ocsp-responder/main.go

@@ -179,10 +179,10 @@ as generated by Boulder's ceremony command.
 		cmd.FailOnError(err, "While initializing dbMap")
 
 		// Set up the redis source and the combined multiplex source.
-		rocspReader, err := rocsp_config.MakeClient(&c.OCSPResponder.Redis, clk, scope)
+		rocspRWClient, err := rocsp_config.MakeClient(&c.OCSPResponder.Redis, clk, scope)
 		cmd.FailOnError(err, "Could not make redis client")
 
-		err = rocspReader.Ping(context.Background())
+		err = rocspRWClient.Ping(context.Background())
 		cmd.FailOnError(err, "pinging Redis")
 
 		liveSigningPeriod := c.OCSPResponder.LiveSigningPeriod.Duration
@@ -203,7 +203,7 @@ as generated by Boulder's ceremony command.
 		}
 		liveSource := live.New(rac, int64(maxInflight), c.OCSPResponder.MaxSigningWaiters)
 
-		rocspSource, err := redis_responder.NewRedisSource(rocspReader, liveSource, liveSigningPeriod, clk, scope, logger)
+		rocspSource, err := redis_responder.NewRedisSource(rocspRWClient, liveSource, liveSigningPeriod, clk, scope, logger)
 		cmd.FailOnError(err, "Could not create redis source")
 
 		var sac sapb.StorageAuthorityReadOnlyClient

cmd/rocsp-tool/client.go

@@ -20,7 +20,7 @@ import (
 )
 
 type client struct {
-	redis         *rocsp.WritingClient
+	redis         rocsp.Writer
 	db            *db.WrappedMap // optional
 	ocspGenerator capb.OCSPGeneratorClient
 	clk           clock.Clock

cmd/rocsp-tool/client_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"math/big"
+	"os"
 	"testing"
 	"time"
 
@@ -22,7 +23,35 @@ import (
 	"google.golang.org/grpc"
 )
 
-func makeClient() (*rocsp.WritingClient, clock.Clock) {
+func makeClient() (*rocsp.RWClient, clock.Clock) {
+	CACertFile := "../../test/redis-tls/minica.pem"
+	CertFile := "../../test/redis-tls/boulder/cert.pem"
+	KeyFile := "../../test/redis-tls/boulder/key.pem"
+	tlsConfig := cmd.TLSConfig{
+		CACertFile: &CACertFile,
+		CertFile:   &CertFile,
+		KeyFile:    &KeyFile,
+	}
+	tlsConfig2, err := tlsConfig.Load()
+	if err != nil {
+		panic(err)
+	}
+
+	rdb := redis.NewRing(&redis.RingOptions{
+		Addrs: map[string]string{
+			"shard1": "10.33.33.8:4218",
+			"shard2": "10.33.33.9:4218",
+		},
+		Username:  "unittest-rw",
+		Password:  "824968fa490f4ecec1e52d5e34916bdb60d45f8d",
+		TLSConfig: tlsConfig2,
+	})
+	clk := clock.NewFake()
+	return rocsp.NewWritingClient(rdb, 500*time.Millisecond, clk, metrics.NoopRegisterer), clk
+}
+
+// TODO(#6517) remove this helper.
+func makeClusterClient() (*rocsp.CRWClient, clock.Clock) {
 	CACertFile := "../../test/redis-tls/minica.pem"
 	CertFile := "../../test/redis-tls/boulder/cert.pem"
 	KeyFile := "../../test/redis-tls/boulder/key.pem"
@@ -43,7 +72,8 @@ func makeClient() (*rocsp.WritingClient, clock.Clock) {
 		TLSConfig: tlsConfig2,
 	})
 	clk := clock.NewFake()
-	return rocsp.NewWritingClient(rdb, 500*time.Millisecond, clk, metrics.NoopRegisterer), clk
+
+	return rocsp.NewClusterWritingClient(rdb, 5*time.Second, clk, metrics.NoopRegisterer), clk
 }
 
 func TestGetStartingID(t *testing.T) {
@@ -79,7 +109,14 @@ func TestGetStartingID(t *testing.T) {
 }
 
 func TestStoreResponse(t *testing.T) {
-	redisClient, clk := makeClient()
+	// TODO(#6517) remove this block.
+	var redisClient rocsp.Writer
+	var clk clock.Clock
+	if os.Getenv("BOULDER_CONFIG_DIR") == "test/config" {
+		redisClient, clk = makeClusterClient()
+	} else {
+		redisClient, clk = makeClient()
+	}
 
 	issuer, err := core.LoadCert("../../test/hierarchy/int-e1.cert.pem")
 	test.AssertNotError(t, err, "loading int-e1")
@@ -116,7 +153,14 @@ func (mog mockOCSPGenerator) GenerateOCSP(ctx context.Context, in *capb.Generate
 }
 
 func TestLoadFromDB(t *testing.T) {
-	redisClient, clk := makeClient()
+	// TODO(#6517) remove this block.
+	var redisClient rocsp.Writer
+	var clk clock.Clock
+	if os.Getenv("BOULDER_CONFIG_DIR") == "test/config" {
+		redisClient, clk = makeClusterClient()
+	} else {
+		redisClient, clk = makeClient()
+	}
 
 	dbMap, err := sa.NewDbMap(vars.DBConnSA, sa.DbSettings{})
 	if err != nil {

docker-compose.next.yml

@@ -3,5 +3,28 @@ services:
   boulder:
     environment:
       FAKE_DNS: 10.77.77.77
-      BOULDER_CONFIG_DIR: test/config-next
+      BOULDER_CONFIG_DIR: &boulder_config_dir test/config-next
       GOFLAGS: -mod=vendor
+  # TODO(#6517): remove bredis_clusterer
+  bredis_clusterer:
+    depends_on:
+      - bredis_7
+      - bredis_8
+  # TODO(#6517): move both nodes to docker-compose.yml
+  bredis_7:
+    image: redis:6.2.7
+    volumes:
+      - ./test/:/test/:cached
+    command: redis-server /test/redis.config
+    networks:
+      redisnet:
+        ipv4_address: 10.33.33.8
+
+  bredis_8:
+    image: redis:6.2.7
+    volumes:
+      - ./test/:/test/:cached
+    command: redis-server /test/redis.config
+    networks:
+      redisnet:
+        ipv4_address: 10.33.33.9

docker-compose.yml

@@ -8,7 +8,7 @@ services:
     image: &boulder_image letsencrypt/boulder-tools:${BOULDER_TOOLS_TAG:-go1.19.2_2022-10-05}
     environment:
       FAKE_DNS: 10.77.77.77
-      BOULDER_CONFIG_DIR: test/config
+      BOULDER_CONFIG_DIR: &boulder_config_dir test/config
       GOFLAGS: -mod=vendor
       # Go 1.18 turned off SHA-1 validation on CSRs (and certs, but that doesn't
       # affect us) by default, but it can be turned back on with the x509sha1
@@ -63,64 +63,67 @@ services:
     # small.
     command: mysqld --bind-address=0.0.0.0 --slow-query-log --log-output=TABLE --log-queries-not-using-indexes=ON
     logging:
-        driver: none
-
+      driver: none
+  # TODO(#6517): replace all bredis_ services with those from
+  # docker-compose.next.yml.
   bredis_1:
     image: redis:6.2.7
     volumes:
       - ./test/:/test/:cached
-    command: redis-server /test/redis.config
+    command: redis-server /test/redis-cluster.config
     networks:
       redisnet:
-          ipv4_address: 10.33.33.2
+        ipv4_address: 10.33.33.2
 
   bredis_2:
     image: redis:6.2.7
     volumes:
       - ./test/:/test/:cached
-    command: redis-server /test/redis.config
+    command: redis-server /test/redis-cluster.config
     networks:
       redisnet:
-          ipv4_address: 10.33.33.3
+        ipv4_address: 10.33.33.3
 
   bredis_3:
     image: redis:6.2.7
     volumes:
       - ./test/:/test/:cached
-    command: redis-server /test/redis.config
+    command: redis-server /test/redis-cluster.config
     networks:
       redisnet:
-          ipv4_address: 10.33.33.4
+        ipv4_address: 10.33.33.4
 
   bredis_4:
     image: redis:6.2.7
     volumes:
       - ./test/:/test/:cached
-    command: redis-server /test/redis.config
+    command: redis-server /test/redis-cluster.config
     networks:
       redisnet:
-          ipv4_address: 10.33.33.5
+        ipv4_address: 10.33.33.5
 
   bredis_5:
     image: redis:6.2.7
     volumes:
       - ./test/:/test/:cached
-    command: redis-server /test/redis.config
+    command: redis-server /test/redis-cluster.config
     networks:
       redisnet:
-          ipv4_address: 10.33.33.6
+        ipv4_address: 10.33.33.6
 
   bredis_6:
     image: redis:6.2.7
     volumes:
       - ./test/:/test/:cached
-    command: redis-server /test/redis.config
+    command: redis-server /test/redis-cluster.config
     networks:
       redisnet:
-          ipv4_address: 10.33.33.7
-
+        ipv4_address: 10.33.33.7
+  # TODO(#6517): remove bredis_clusterer.
   bredis_clusterer:
     image: redis:6.2.7
+    environment:
+      BOULDER_CONFIG_DIR: *boulder_config_dir
     volumes:
       - ./test/:/test/:cached
       - ./cluster/:/cluster/:cached
@@ -134,9 +137,9 @@ services:
       - bredis_6
     networks:
       redisnet:
-          ipv4_address: 10.33.33.10
-          aliases:
-            - boulder-redis-clusterer
+        ipv4_address: 10.33.33.10
+        aliases:
+          - boulder-redis-clusterer
 
   bconsul:
     image: hashicorp/consul:1.13.1
@@ -152,7 +155,7 @@ services:
     environment:
       GO111MODULE: "on"
       GOFLAGS: -mod=vendor
-      BOULDER_CONFIG_DIR: test/config
+      BOULDER_CONFIG_DIR: *boulder_config_dir
     networks:
       - bluenet
     volumes:

docs/redis.md

@@ -1,5 +1,7 @@
 # Redis
 
+TODO(#6517): Update this to reflect the use of Redis Ring.
+
 We use Redis Cluster for OCSP. The Boulder dev environment stands up a cluster
 of 6 nodes, with 3 primaries and 3 replicas. Check docker-compose.yml for
 details of those.

ocsp/responder/redis/redis_source.go

@@ -49,7 +49,7 @@ type redisSource struct {
 // NewRedisSource returns a responder.Source which will look up OCSP responses in a
 // Redis table.
 func NewRedisSource(
-	client *rocsp.WritingClient,
+	client rocsp.Writer,
 	signer responder.Source,
 	liveSigningPeriod time.Duration,
 	clk clock.Clock,