ING-1329: Add support for testing client cert auth

Author: Westwooo

.github/actions/start-couchbase-cluster/action.yml

@@ -24,6 +24,7 @@ runs:
             kv-memory: 2048
             index-memory: 1024
             fts-memory: 1024
+            use-dino-certs: true
       run: |
         CBDC_ID=$(cbdinocluster -v alloc --def="${CLUSTERCONFIG}")
         cbdinocluster -v buckets add ${CBDC_ID} default --ram-quota-mb=100 --flush-enabled=true --num-replicas=2

cmd/gateway/main.go

@@ -446,13 +446,19 @@ func startGateway() {
 
 	var selfSignedCert *tls.Certificate
 	if config.selfSign {
-		generatedCert, err := selfsignedcert.GenerateCertificate()
+		generatedCert, generatedKey, err := selfsignedcert.GenerateCertificate()
 		if err != nil {
 			logger.Error("failed to generate a self-signed certificate")
 			os.Exit(1)
 		}
 
-		selfSignedCert = generatedCert
+		tlsCert, err := selfsignedcert.ConstructTlsCert(generatedCert, generatedKey)
+		if err != nil {
+			logger.Error("failed to generate a self-signed certificate")
+			os.Exit(1)
+		}
+
+		selfSignedCert = tlsCert
 	}
 
 	var grpcCertificate tls.Certificate

gateway/test/dapi_graceful_shutdown_test.go

@@ -19,11 +19,16 @@ import (
 func (s *GatewayOpsTestSuite) TestGracefulShutdown() {
 	s.T().Logf("setting up new instance of stellar gateway...")
 
-	gwCert, err := selfsignedcert.GenerateCertificate()
+	cert, key, err := selfsignedcert.GenerateCertificate()
 	if err != nil {
 		s.T().Fatalf("failed to create testing certificate: %s", err)
 	}
 
+	gwCert, err := selfsignedcert.ConstructTlsCert(cert, key)
+	if err != nil {
+		s.T().Fatalf("failed to construct testing certificate: %s", err)
+	}
+
 	logger, err := zap.NewDevelopment()
 	if err != nil {
 		s.T().Fatalf("failed to initialize test logging: %s", err)

gateway/test/mtls_test.go

@@ -0,0 +1,112 @@
+package test
+
+import (
+	"context"
+	"crypto/tls"
+	"time"
+
+	"github.com/couchbase/goprotostellar/genproto/kv_v1"
+	"github.com/couchbase/stellar-gateway/testutils"
+	"github.com/couchbase/stellar-gateway/utils/certificates"
+	"github.com/stretchr/testify/assert"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/credentials"
+)
+
+func (s *GatewayOpsTestSuite) TestKvServiceMtls() {
+	// TO DO - is there a way to skip if client cert auth is not enabled
+	testutils.SkipIfNoDinoCluster(s.T())
+
+	dino := testutils.StartDinoTesting(s.T(), false)
+
+	// Create a user without any permissions
+	username := "kvUser"
+	dino.AddUser(username, false, false)
+
+	s.T().Cleanup(func() {
+		dino.RemoveUser(username)
+	})
+
+	clientCert, err := certificates.GenerateSignedClientCert(s.caCert, s.caKey, username)
+	assert.NoError(s.T(), err)
+
+	conn, err := grpc.NewClient(s.gwConnAddr,
+		grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{
+			RootCAs:            s.clientCaCertPool,
+			Certificates:       []tls.Certificate{*clientCert},
+			InsecureSkipVerify: false,
+		})))
+	if err != nil {
+		s.T().Fatalf("failed to connect to test gateway: %s", err)
+	}
+
+	kvClient := kv_v1.NewKvServiceClient(conn)
+
+	s.Run("ReadFailure", func() {
+		_, err := kvClient.Get(context.Background(), &kv_v1.GetRequest{
+			BucketName:     s.bucketName,
+			ScopeName:      s.scopeName,
+			CollectionName: s.collectionName,
+			Key:            s.testDocId(),
+		})
+		assertRpcStatus(s.T(), err, codes.PermissionDenied)
+		assert.Contains(s.T(), err.Error(), "No permissions to read documents")
+	})
+
+	// Add read permissions to the user and wait for this to take effect
+	dino.AddUser(username, true, false)
+	time.Sleep(time.Second * 5)
+
+	s.Run("ReadSuccess", func() {
+		resp, err := kvClient.Get(context.Background(), &kv_v1.GetRequest{
+			BucketName:     s.bucketName,
+			ScopeName:      s.scopeName,
+			CollectionName: s.collectionName,
+			Key:            s.testDocId(),
+		})
+		requireRpcSuccess(s.T(), resp, err)
+		assertValidCas(s.T(), resp.Cas)
+		assert.Equal(s.T(), TEST_CONTENT, resp.GetContentUncompressed())
+		assert.Nil(s.T(), resp.GetContentCompressed())
+		assert.Equal(s.T(), TEST_CONTENT_FLAGS, resp.ContentFlags)
+		assert.Nil(s.T(), resp.Expiry)
+	})
+
+	s.Run("WriteFailure", func() {
+		docId := s.randomDocId()
+		_, err := kvClient.Upsert(context.Background(), &kv_v1.UpsertRequest{
+			BucketName:     s.bucketName,
+			ScopeName:      s.scopeName,
+			CollectionName: s.collectionName,
+			Key:            docId,
+			Content: &kv_v1.UpsertRequest_ContentUncompressed{
+				ContentUncompressed: TEST_CONTENT,
+			},
+			ContentFlags: TEST_CONTENT_FLAGS,
+		})
+		assertRpcStatus(s.T(), err, codes.PermissionDenied)
+		assert.Contains(s.T(), err.Error(), "No permissions to write documents")
+	})
+
+	// Add write permissions to the user
+	dino.AddUser(username, false, true)
+	time.Sleep(time.Second * 5)
+
+	s.Run("WriteSuccess", func() {
+		docId := s.randomDocId()
+		resp, err := kvClient.Upsert(context.Background(), &kv_v1.UpsertRequest{
+			BucketName:     s.bucketName,
+			ScopeName:      s.scopeName,
+			CollectionName: s.collectionName,
+			Key:            docId,
+			Content: &kv_v1.UpsertRequest_ContentUncompressed{
+				ContentUncompressed: TEST_CONTENT,
+			},
+			ContentFlags: TEST_CONTENT_FLAGS,
+		})
+		requireRpcSuccess(s.T(), resp, err)
+		assertValidCas(s.T(), resp.Cas)
+		assertValidMutationToken(s.T(), resp.MutationToken, s.bucketName)
+	})
+}

gateway/test/suite_test.go

@@ -2,7 +2,9 @@ package test
 
 import (
 	"context"
+	"crypto/ecdsa"
 	"crypto/tls"
+	"crypto/x509"
 	"encoding/base64"
 	"encoding/json"
 	"fmt"
@@ -23,6 +25,7 @@ import (
 	"github.com/couchbase/stellar-gateway/contrib/grpcheaderauth"
 	"github.com/couchbase/stellar-gateway/gateway"
 	"github.com/couchbase/stellar-gateway/testutils"
+	"github.com/couchbase/stellar-gateway/utils/certificates"
 	"github.com/couchbase/stellar-gateway/utils/selfsignedcert"
 	"github.com/google/uuid"
 	"github.com/stretchr/testify/assert"
@@ -40,6 +43,7 @@ type GatewayOpsTestSuite struct {
 	testClusterInfo  *testutils.CanonicalTestCluster
 	gatewayCloseFunc func()
 	gatewayConn      *grpc.ClientConn
+	gwConnAddr       string
 	gatewayClosedCh  chan struct{}
 	dapiCli          *http.Client
 	dapiAddr         string
@@ -55,6 +59,10 @@ type GatewayOpsTestSuite struct {
 	basicRestCreds string
 	readRestCreds  string
 
+	caCert           *x509.Certificate
+	caKey            *ecdsa.PrivateKey
+	clientCaCertPool *x509.CertPool
+
 	clusterVersion *NodeVersion
 	features       []TestFeature
 
@@ -249,11 +257,23 @@ func (s *GatewayOpsTestSuite) SetupSuite() {
 			s.T().Fatalf("failed to initialize test logging: %s", err)
 		}
 
-		gwCert, err := selfsignedcert.GenerateCertificate()
+		caCert, caKey, err := selfsignedcert.GenerateCertificate()
 		if err != nil {
 			s.T().Fatalf("failed to create testing certificate: %s", err)
 		}
 
+		gwCert, err := certificates.GenerateSignedServerCert(caCert, caKey)
+		if err != nil {
+			s.T().Fatalf("failed to create signed gateway certificate: %s", err)
+		}
+
+		clientCaCert := x509.NewCertPool()
+		clientCaCert.AddCert(caCert)
+
+		s.caCert = caCert
+		s.caKey = caKey
+		s.clientCaCertPool = clientCaCert
+
 		gwStartInfoCh := make(chan *gateway.StartupInfo, 1)
 		gwCtx, gwCtxCancel := context.WithCancel(context.Background())
 		gw, err := gateway.NewGateway(&gateway.Config{
@@ -265,6 +285,7 @@ func (s *GatewayOpsTestSuite) SetupSuite() {
 			BindDapiPort:    0,
 			GrpcCertificate: *gwCert,
 			DapiCertificate: *gwCert,
+			ClientCaCert:    clientCaCert,
 			AlphaEndpoints:  true,
 			NumInstances:    1,
 			ProxyServices:   []string{"query", "analytics", "mgmt", "search"},
@@ -308,6 +329,7 @@ func (s *GatewayOpsTestSuite) SetupSuite() {
 		}
 
 		s.gatewayConn = conn
+		s.gwConnAddr = connAddr
 		s.gatewayCloseFunc = gwCtxCancel
 		s.gatewayClosedCh = gwClosedCh
 		s.dapiCli = dapiCli
@@ -330,6 +352,7 @@ func (s *GatewayOpsTestSuite) SetupSuite() {
 		}
 
 		s.gatewayConn = conn
+		s.gwConnAddr = connAddr
 		s.dapiCli = dapiCli
 		s.dapiAddr = dapiAddr
 	}

testutils/dinocluster.go

@@ -3,6 +3,7 @@ package testutils
 import (
 	"bufio"
 	"context"
+	"fmt"
 	"io"
 	"log"
 	"net/http"
@@ -93,6 +94,22 @@ func runDinoRemoveNode(node string) error {
 	return runNoResDinoCmd([]string{"nodes", "rm", globalTestConfig.DinoId, node})
 }
 
+func runDinoAddUser(username string, canRead, canWrite bool) error {
+	return runNoResDinoCmd([]string{
+		"users",
+		"add",
+		globalTestConfig.DinoId,
+		username,
+		"--password=password",
+		fmt.Sprintf("--can-read=%v", canRead),
+		fmt.Sprintf("--can-write=%v", canWrite),
+	})
+}
+
+func runDinoRemoveUser(username string) error {
+	return runNoResDinoCmd([]string{"users", "remove", globalTestConfig.DinoId, username})
+}
+
 type DinoController struct {
 	t             *testing.T
 	oldFoSettings *cbmgmtx.GetAutoFailoverSettingsResponse
@@ -182,3 +199,13 @@ func (c *DinoController) RemoveNode(node string) {
 	err := runDinoRemoveNode(node)
 	require.NoError(c.t, err)
 }
+
+func (c *DinoController) AddUser(username string, canRead, canWrite bool) {
+	err := runDinoAddUser(username, canRead, canWrite)
+	require.NoError(c.t, err)
+}
+
+func (c *DinoController) RemoveUser(username string) {
+	err := runDinoRemoveUser(username)
+	require.NoError(c.t, err)
+}

utils/certificates/certificates.go

@@ -0,0 +1,73 @@
+package certificates
+
+import (
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/tls"
+	"crypto/x509"
+	"fmt"
+	"math/big"
+	"net"
+	"time"
+
+	"github.com/couchbase/stellar-gateway/utils/selfsignedcert"
+	"github.com/pkg/errors"
+)
+
+func GenerateSignedServerCert(caCert *x509.Certificate, caKey *ecdsa.PrivateKey) (*tls.Certificate, error) {
+	priv, err := ecdsa.GenerateKey(elliptic.P384(), rand.Reader)
+	if err != nil {
+		panic(err)
+	}
+
+	template := x509.Certificate{
+		SerialNumber:          big.NewInt(1),
+		NotBefore:             time.Now(),
+		NotAfter:              time.Now().Add(7 * 24 * time.Hour),
+		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
+		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+		BasicConstraintsValid: true,
+		IPAddresses:           []net.IP{net.ParseIP("127.0.0.1")},
+	}
+
+	derBytes, err := x509.CreateCertificate(rand.Reader, &template, caCert, &priv.PublicKey, caKey)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to create signed server certificate")
+	}
+
+	cert, err := x509.ParseCertificate(derBytes)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to parse cert from bytes")
+	}
+
+	return selfsignedcert.ConstructTlsCert(cert, priv)
+}
+
+func GenerateSignedClientCert(caCert *x509.Certificate, caKey *ecdsa.PrivateKey, username string) (*tls.Certificate, error) {
+	priv, err := ecdsa.GenerateKey(elliptic.P384(), rand.Reader)
+	if err != nil {
+		panic(err)
+	}
+
+	template := x509.Certificate{
+		NotBefore:             time.Now(),
+		NotAfter:              time.Now().Add(7 * 24 * time.Hour),
+		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
+		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
+		BasicConstraintsValid: true,
+		EmailAddresses:        []string{fmt.Sprintf("%[email protected]", username)},
+	}
+
+	derBytes, err := x509.CreateCertificate(rand.Reader, &template, caCert, &priv.PublicKey, caKey)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to create singed server certificate")
+	}
+
+	cert, err := x509.ParseCertificate(derBytes)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to parse cert from bytes")
+	}
+
+	return selfsignedcert.ConstructTlsCert(cert, priv)
+}