ING-1329: Add testing for client cert auth

.github/actions/start-couchbase-cluster/action.yml

@@ -24,6 +24,7 @@ runs:
             kv-memory: 2048
             index-memory: 1024
             fts-memory: 1024
+            use-dino-certs: true
       run: |
         CBDC_ID=$(cbdinocluster -v alloc --def="${CLUSTERCONFIG}")
         cbdinocluster -v buckets add ${CBDC_ID} default --ram-quota-mb=100 --flush-enabled=true --num-replicas=2

.github/workflows/client_cert_auth_test.yml

@@ -0,0 +1,70 @@
+name: Run Client Cert Auth Tests
+permissions:
+  contents: read
+  packages: read
+
+on:
+  push:
+    tags:
+      - v*
+    branches:
+      - master
+  pull_request:
+jobs:
+  test:
+    name: Test
+    strategy:
+      matrix:
+        server:
+          - 8.0.0-3534
+          - 7.6.5
+          - 7.2.2
+
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+      - name: Install cbdinocluster
+        uses: ./.github/actions/install-cbdinocluster
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Start couchbase cluster
+        id: start-cluster
+        uses: ./.github/actions/start-couchbase-cluster
+      - uses: actions/setup-go@v5
+        with:
+          go-version: 1.24
+      - uses: arduino/setup-protoc@v3
+        with:
+          version: 31.1
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Install Tools
+        run: |
+          go install google.golang.org/protobuf/cmd/[email protected]
+          go install google.golang.org/grpc/cmd/[email protected]
+          go install github.com/matryer/[email protected]
+      - name: Install Dependencies
+        run: go get ./...
+      - name: Generate Files
+        run: |
+          go generate ./...
+      - name: Run Test
+        env:
+          SGTEST_CBCONNSTR: ${{ steps.start-cluster.outputs.node-ip }}
+          SGTEST_DINOID: ${{ steps.start-cluster.outputs.dino-id }}
+        run: go test ./gateway/test -run TestGatewayOps -v -testify.m TestClientCertAuth
+
+      - name: Collect couchbase logs
+        timeout-minutes: 10
+        if: failure()
+        run: |
+          mkdir -p ./client-cert-auth-logs
+          cbdinocluster -v collect-logs ${{ steps.start-cluster.outputs.dino-id }} ./client-cert-auth-logs
+      - name: Upload couchbase logs
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: cbcollect-logs-${{ matrix.server }}
+          path: ./client-cert-auth-logs/*
+          retention-days: 5

cmd/gateway/main.go

@@ -446,13 +446,19 @@ func startGateway() {
 
 	var selfSignedCert *tls.Certificate
 	if config.selfSign {
-		generatedCert, err := selfsignedcert.GenerateCertificate()
+		generatedCert, generatedKey, err := selfsignedcert.GenerateCertificate()
 		if err != nil {
 			logger.Error("failed to generate a self-signed certificate")
 			os.Exit(1)
 		}
 
-		selfSignedCert = generatedCert
+		tlsCert, err := selfsignedcert.ConstructTlsCert(generatedCert, generatedKey)
+		if err != nil {
+			logger.Error("failed to generate a self-signed certificate")
+			os.Exit(1)
+		}
+
+		selfSignedCert = tlsCert
 	}
 
 	var grpcCertificate tls.Certificate

gateway/test/dapi_graceful_shutdown_test.go

@@ -19,11 +19,16 @@ import (
 func (s *GatewayOpsTestSuite) TestGracefulShutdown() {
 	s.T().Logf("setting up new instance of stellar gateway...")
 
-	gwCert, err := selfsignedcert.GenerateCertificate()
+	cert, key, err := selfsignedcert.GenerateCertificate()
 	if err != nil {
 		s.T().Fatalf("failed to create testing certificate: %s", err)
 	}
 
+	gwCert, err := selfsignedcert.ConstructTlsCert(cert, key)
+	if err != nil {
+		s.T().Fatalf("failed to construct testing certificate: %s", err)
+	}
+
 	logger, err := zap.NewDevelopment()
 	if err != nil {
 		s.T().Fatalf("failed to initialize test logging: %s", err)

gateway/test/mtls_test.go

@@ -0,0 +1,127 @@
+package test
+
+import (
+	"context"
+	"crypto/tls"
+	"time"
+
+	"github.com/couchbase/goprotostellar/genproto/kv_v1"
+	"github.com/couchbase/stellar-gateway/testutils"
+	"github.com/couchbase/stellar-gateway/utils/certificates"
+	"github.com/stretchr/testify/assert"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/credentials"
+)
+
+func (s *GatewayOpsTestSuite) TestClientCertAuth() {
+	testutils.SkipIfNoDinoCluster(s.T())
+
+	s.Run("KvService", s.KvService)
+}
+
+func (s *GatewayOpsTestSuite) KvService() {
+	dino := testutils.StartDinoTesting(s.T(), false)
+
+	username := "kvUser"
+
+	clientCert, err := certificates.GenerateSignedClientCert(s.caCert, s.caKey, username)
+	assert.NoError(s.T(), err)
+
+	conn, err := grpc.NewClient(s.gwConnAddr,
+		grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{
+			RootCAs:            s.clientCaCertPool,
+			Certificates:       []tls.Certificate{*clientCert},
+			InsecureSkipVerify: false,
+		})))
+	if err != nil {
+		s.T().Fatalf("failed to connect to test gateway: %s", err)
+	}
+
+	kvClient := kv_v1.NewKvServiceClient(conn)
+
+	s.Run("UserMissing", func() {
+		_, err := kvClient.Get(context.Background(), &kv_v1.GetRequest{
+			BucketName:     s.bucketName,
+			ScopeName:      s.scopeName,
+			CollectionName: s.collectionName,
+			Key:            s.testDocId(),
+		})
+		assertRpcStatus(s.T(), err, codes.PermissionDenied)
+		assert.Contains(s.T(), err.Error(), "Your certificate is invalid")
+	})
+
+	// Create a user without any permissions
+	dino.AddUser(username, false, false)
+
+	s.T().Cleanup(func() {
+		dino.RemoveUser(username)
+	})
+
+	s.Run("NoReadPermission", func() {
+		_, err := kvClient.Get(context.Background(), &kv_v1.GetRequest{
+			BucketName:     s.bucketName,
+			ScopeName:      s.scopeName,
+			CollectionName: s.collectionName,
+			Key:            s.testDocId(),
+		})
+		assertRpcStatus(s.T(), err, codes.PermissionDenied)
+		assert.Contains(s.T(), err.Error(), "No permissions to read documents")
+	})
+
+	// Add read permissions to the user and wait for this to take effect
+	dino.AddUser(username, true, false)
+	time.Sleep(time.Second * 5)
+
+	s.Run("ReadSuccess", func() {
+		resp, err := kvClient.Get(context.Background(), &kv_v1.GetRequest{
+			BucketName:     s.bucketName,
+			ScopeName:      s.scopeName,
+			CollectionName: s.collectionName,
+			Key:            s.testDocId(),
+		})
+		requireRpcSuccess(s.T(), resp, err)
+		assertValidCas(s.T(), resp.Cas)
+		assert.Equal(s.T(), TEST_CONTENT, resp.GetContentUncompressed())
+		assert.Nil(s.T(), resp.GetContentCompressed())
+		assert.Equal(s.T(), TEST_CONTENT_FLAGS, resp.ContentFlags)
+		assert.Nil(s.T(), resp.Expiry)
+	})
+
+	s.Run("NoWritePermission", func() {
+		docId := s.randomDocId()
+		_, err := kvClient.Upsert(context.Background(), &kv_v1.UpsertRequest{
+			BucketName:     s.bucketName,
+			ScopeName:      s.scopeName,
+			CollectionName: s.collectionName,
+			Key:            docId,
+			Content: &kv_v1.UpsertRequest_ContentUncompressed{
+				ContentUncompressed: TEST_CONTENT,
+			},
+			ContentFlags: TEST_CONTENT_FLAGS,
+		})
+		assertRpcStatus(s.T(), err, codes.PermissionDenied)
+		assert.Contains(s.T(), err.Error(), "No permissions to write documents")
+	})
+
+	// Add write permissions to the user
+	dino.AddUser(username, false, true)
+	time.Sleep(time.Second * 5)
+
+	s.Run("WriteSuccess", func() {
+		docId := s.randomDocId()
+		resp, err := kvClient.Upsert(context.Background(), &kv_v1.UpsertRequest{
+			BucketName:     s.bucketName,
+			ScopeName:      s.scopeName,
+			CollectionName: s.collectionName,
+			Key:            docId,
+			Content: &kv_v1.UpsertRequest_ContentUncompressed{
+				ContentUncompressed: TEST_CONTENT,
+			},
+			ContentFlags: TEST_CONTENT_FLAGS,
+		})
+		requireRpcSuccess(s.T(), resp, err)
+		assertValidCas(s.T(), resp.Cas)
+		assertValidMutationToken(s.T(), resp.MutationToken, s.bucketName)
+	})
+}

gateway/test/suite_test.go

@@ -2,7 +2,9 @@ package test
 
 import (
 	"context"
+	"crypto/ecdsa"
 	"crypto/tls"
+	"crypto/x509"
 	"encoding/base64"
 	"encoding/json"
 	"fmt"
@@ -23,6 +25,7 @@ import (
 	"github.com/couchbase/stellar-gateway/contrib/grpcheaderauth"
 	"github.com/couchbase/stellar-gateway/gateway"
 	"github.com/couchbase/stellar-gateway/testutils"
+	"github.com/couchbase/stellar-gateway/utils/certificates"
 	"github.com/couchbase/stellar-gateway/utils/selfsignedcert"
 	"github.com/google/uuid"
 	"github.com/stretchr/testify/assert"
@@ -40,6 +43,7 @@ type GatewayOpsTestSuite struct {
 	testClusterInfo  *testutils.CanonicalTestCluster
 	gatewayCloseFunc func()
 	gatewayConn      *grpc.ClientConn
+	gwConnAddr       string
 	gatewayClosedCh  chan struct{}
 	dapiCli          *http.Client
 	dapiAddr         string
@@ -55,6 +59,10 @@ type GatewayOpsTestSuite struct {
 	basicRestCreds string
 	readRestCreds  string
 
+	caCert           *x509.Certificate
+	caKey            *ecdsa.PrivateKey
+	clientCaCertPool *x509.CertPool
+
 	clusterVersion *NodeVersion
 	features       []TestFeature
 
@@ -249,11 +257,23 @@ func (s *GatewayOpsTestSuite) SetupSuite() {
 			s.T().Fatalf("failed to initialize test logging: %s", err)
 		}
 
-		gwCert, err := selfsignedcert.GenerateCertificate()
+		caCert, caKey, err := selfsignedcert.GenerateCertificate()
 		if err != nil {
 			s.T().Fatalf("failed to create testing certificate: %s", err)
 		}
 
+		gwCert, err := certificates.GenerateSignedServerCert(caCert, caKey)
+		if err != nil {
+			s.T().Fatalf("failed to create signed gateway certificate: %s", err)
+		}
+
+		clientCaCert := x509.NewCertPool()
+		clientCaCert.AddCert(caCert)
+
+		s.caCert = caCert
+		s.caKey = caKey
+		s.clientCaCertPool = clientCaCert
+
 		gwStartInfoCh := make(chan *gateway.StartupInfo, 1)
 		gwCtx, gwCtxCancel := context.WithCancel(context.Background())
 		gw, err := gateway.NewGateway(&gateway.Config{
@@ -265,6 +285,7 @@ func (s *GatewayOpsTestSuite) SetupSuite() {
 			BindDapiPort:    0,
 			GrpcCertificate: *gwCert,
 			DapiCertificate: *gwCert,
+			ClientCaCert:    clientCaCert,
 			AlphaEndpoints:  true,
 			NumInstances:    1,
 			ProxyServices:   []string{"query", "analytics", "mgmt", "search"},
@@ -308,6 +329,7 @@ func (s *GatewayOpsTestSuite) SetupSuite() {
 		}
 
 		s.gatewayConn = conn
+		s.gwConnAddr = connAddr
 		s.gatewayCloseFunc = gwCtxCancel
 		s.gatewayClosedCh = gwClosedCh
 		s.dapiCli = dapiCli
@@ -330,6 +352,7 @@ func (s *GatewayOpsTestSuite) SetupSuite() {
 		}
 
 		s.gatewayConn = conn
+		s.gwConnAddr = connAddr
 		s.dapiCli = dapiCli
 		s.dapiAddr = dapiAddr
 	}

testutils/dinocluster.go

@@ -3,6 +3,7 @@ package testutils
 import (
 	"bufio"
 	"context"
+	"fmt"
 	"io"
 	"log"
 	"net/http"
@@ -93,6 +94,22 @@ func runDinoRemoveNode(node string) error {
 	return runNoResDinoCmd([]string{"nodes", "rm", globalTestConfig.DinoId, node})
 }
 
+func runDinoAddUser(username string, canRead, canWrite bool) error {
+	return runNoResDinoCmd([]string{
+		"users",
+		"add",
+		globalTestConfig.DinoId,
+		username,
+		"--password=password",
+		fmt.Sprintf("--can-read=%v", canRead),
+		fmt.Sprintf("--can-write=%v", canWrite),
+	})
+}
+
+func runDinoRemoveUser(username string) error {
+	return runNoResDinoCmd([]string{"users", "remove", globalTestConfig.DinoId, username})
+}
+
 type DinoController struct {
 	t             *testing.T
 	oldFoSettings *cbmgmtx.GetAutoFailoverSettingsResponse
@@ -182,3 +199,13 @@ func (c *DinoController) RemoveNode(node string) {
 	err := runDinoRemoveNode(node)
 	require.NoError(c.t, err)
 }
+
+func (c *DinoController) AddUser(username string, canRead, canWrite bool) {
+	err := runDinoAddUser(username, canRead, canWrite)
+	require.NoError(c.t, err)
+}
+
+func (c *DinoController) RemoveUser(username string) {
+	err := runDinoRemoveUser(username)
+	require.NoError(c.t, err)
+}