From 63f8a0cc395d34b1ef843c06f104cc7f4b16193f Mon Sep 17 00:00:00 2001 From: Jack Westwood Date: Tue, 11 Nov 2025 13:38:43 +0000 Subject: [PATCH] ING-1329: Add support for testing client cert auth --- .../start-couchbase-cluster/action.yml | 1 + .github/workflows/client_cert_auth_test.yml | 70 +++++++++++ cmd/gateway/main.go | 10 +- gateway/test/dapi_graceful_shutdown_test.go | 7 +- gateway/test/mtls_test.go | 115 ++++++++++++++++++ gateway/test/suite_test.go | 25 +++- testutils/dinocluster.go | 37 ++++++ utils/certificates/certificates.go | 73 +++++++++++ utils/selfsignedcert/selfsignedcert.go | 42 ++++--- 9 files changed, 361 insertions(+), 19 deletions(-) create mode 100644 .github/workflows/client_cert_auth_test.yml create mode 100644 gateway/test/mtls_test.go create mode 100644 utils/certificates/certificates.go diff --git a/.github/actions/start-couchbase-cluster/action.yml b/.github/actions/start-couchbase-cluster/action.yml index 57059690..23485ccb 100644 --- a/.github/actions/start-couchbase-cluster/action.yml +++ b/.github/actions/start-couchbase-cluster/action.yml @@ -24,6 +24,7 @@ runs: kv-memory: 2048 index-memory: 1024 fts-memory: 1024 + use-dino-certs: true run: | CBDC_ID=$(cbdinocluster -v alloc --def="${CLUSTERCONFIG}") cbdinocluster -v buckets add ${CBDC_ID} default --ram-quota-mb=100 --flush-enabled=true --num-replicas=2 diff --git a/.github/workflows/client_cert_auth_test.yml b/.github/workflows/client_cert_auth_test.yml new file mode 100644 index 00000000..20c43332 --- /dev/null +++ b/.github/workflows/client_cert_auth_test.yml @@ -0,0 +1,70 @@ +name: Run Client Cert Auth Tests +permissions: + contents: read + packages: read + +on: + push: + tags: + - v* + branches: + - master + pull_request: +jobs: + test: + name: Test + strategy: + matrix: + server: + - 8.0.0-3534 + - 7.6.5 + - 7.2.2 + + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + - name: Install cbdinocluster + uses: ./.github/actions/install-cbdinocluster + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + - name: Start couchbase cluster + id: start-cluster + uses: ./.github/actions/start-couchbase-cluster + - uses: actions/setup-go@v5 + with: + go-version: 1.24 + - uses: arduino/setup-protoc@v3 + with: + version: 31.1 + repo-token: ${{ secrets.GITHUB_TOKEN }} + - name: Install Tools + run: | + go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.36 + go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.5 + go install github.com/matryer/moq@v0.5 + - name: Install Dependencies + run: go get ./... + - name: Generate Files + run: | + go generate ./... + - name: Run Test + env: + SGTEST_CBCONNSTR: ${{ steps.start-cluster.outputs.node-ip }} + SGTEST_DINOID: ${{ steps.start-cluster.outputs.dino-id }} + run: go test ./gateway/test -run TestGatewayOps -v -testify.m TestClientCertAuth + + - name: Collect couchbase logs + timeout-minutes: 10 + if: failure() + run: | + mkdir -p ./client-cert-auth-logs + cbdinocluster -v collect-logs ${{ steps.start-cluster.outputs.dino-id }} ./client-cert-auth-logs + - name: Upload couchbase logs + if: failure() + uses: actions/upload-artifact@v4 + with: + name: cbcollect-logs-${{ matrix.server }} + path: ./client-cert-auth-logs/* + retention-days: 5 \ No newline at end of file diff --git a/cmd/gateway/main.go b/cmd/gateway/main.go index a59eda27..05286b6f 100644 --- a/cmd/gateway/main.go +++ b/cmd/gateway/main.go @@ -446,13 +446,19 @@ func startGateway() { var selfSignedCert *tls.Certificate if config.selfSign { - generatedCert, err := selfsignedcert.GenerateCertificate() + generatedCert, generatedKey, err := selfsignedcert.GenerateCertificate() if err != nil { logger.Error("failed to generate a self-signed certificate") os.Exit(1) } - selfSignedCert = generatedCert + tlsCert, err := selfsignedcert.ConstructTlsCert(generatedCert, generatedKey) + if err != nil { + logger.Error("failed to generate a self-signed certificate") + os.Exit(1) + } + + selfSignedCert = tlsCert } var grpcCertificate tls.Certificate diff --git a/gateway/test/dapi_graceful_shutdown_test.go b/gateway/test/dapi_graceful_shutdown_test.go index 9d025d0a..1b2dd929 100644 --- a/gateway/test/dapi_graceful_shutdown_test.go +++ b/gateway/test/dapi_graceful_shutdown_test.go @@ -19,11 +19,16 @@ import ( func (s *GatewayOpsTestSuite) TestGracefulShutdown() { s.T().Logf("setting up new instance of stellar gateway...") - gwCert, err := selfsignedcert.GenerateCertificate() + cert, key, err := selfsignedcert.GenerateCertificate() if err != nil { s.T().Fatalf("failed to create testing certificate: %s", err) } + gwCert, err := selfsignedcert.ConstructTlsCert(cert, key) + if err != nil { + s.T().Fatalf("failed to construct testing certificate: %s", err) + } + logger, err := zap.NewDevelopment() if err != nil { s.T().Fatalf("failed to initialize test logging: %s", err) diff --git a/gateway/test/mtls_test.go b/gateway/test/mtls_test.go new file mode 100644 index 00000000..4ba94221 --- /dev/null +++ b/gateway/test/mtls_test.go @@ -0,0 +1,115 @@ +package test + +import ( + "context" + "crypto/tls" + "time" + + "github.com/couchbase/goprotostellar/genproto/kv_v1" + "github.com/couchbase/stellar-gateway/testutils" + "github.com/couchbase/stellar-gateway/utils/certificates" + "github.com/stretchr/testify/assert" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" +) + +func (s *GatewayOpsTestSuite) TestClientCertAuth() { + testutils.SkipIfNoDinoCluster(s.T()) + + s.Run("KvService", s.KvService) +} + +func (s *GatewayOpsTestSuite) KvService() { + dino := testutils.StartDinoTesting(s.T(), false) + username := "kvUser" + conn := s.newClientCertConn(username) + kvClient := kv_v1.NewKvServiceClient(conn) + getFn := func() (*kv_v1.GetResponse, error) { + return kvClient.Get(context.Background(), &kv_v1.GetRequest{ + BucketName: s.bucketName, + ScopeName: s.scopeName, + CollectionName: s.collectionName, + Key: s.testDocId(), + }) + } + + s.Run("UserMissing", func() { + _, err := getFn() + assertRpcStatus(s.T(), err, codes.PermissionDenied) + assert.Contains(s.T(), err.Error(), "Your certificate is invalid") + }) + + dino.AddUnprivilegedUser(username) + time.Sleep(time.Second * 5) + s.T().Cleanup(func() { + dino.RemoveUser(username) + }) + + s.Run("NoUserPermissions", func() { + _, err := getFn() + assertRpcStatus(s.T(), err, codes.PermissionDenied) + assert.Contains(s.T(), err.Error(), "No permissions to read documents") + }) + + dino.AddReadOnlyUser(username) + time.Sleep(time.Second * 5) + + s.Run("ReadSuccess", func() { + resp, err := getFn() + requireRpcSuccess(s.T(), resp, err) + }) + + s.Run("NoWritePermission", func() { + docId := s.randomDocId() + _, err := kvClient.Upsert(context.Background(), &kv_v1.UpsertRequest{ + BucketName: s.bucketName, + ScopeName: s.scopeName, + CollectionName: s.collectionName, + Key: docId, + Content: &kv_v1.UpsertRequest_ContentUncompressed{ + ContentUncompressed: TEST_CONTENT, + }, + ContentFlags: TEST_CONTENT_FLAGS, + }) + assertRpcStatus(s.T(), err, codes.PermissionDenied) + assert.Contains(s.T(), err.Error(), "No permissions to write documents") + }) + + dino.AddWriteUser(username) + time.Sleep(time.Second * 5) + + s.Run("WriteSuccess", func() { + docId := s.randomDocId() + resp, err := kvClient.Upsert(context.Background(), &kv_v1.UpsertRequest{ + BucketName: s.bucketName, + ScopeName: s.scopeName, + CollectionName: s.collectionName, + Key: docId, + Content: &kv_v1.UpsertRequest_ContentUncompressed{ + ContentUncompressed: TEST_CONTENT, + }, + ContentFlags: TEST_CONTENT_FLAGS, + }) + requireRpcSuccess(s.T(), resp, err) + assertValidCas(s.T(), resp.Cas) + assertValidMutationToken(s.T(), resp.MutationToken, s.bucketName) + }) +} + +func (s *GatewayOpsTestSuite) newClientCertConn(username string) *grpc.ClientConn { + cert, err := certificates.GenerateSignedClientCert(s.caCert, s.caKey, username) + assert.NoError(s.T(), err) + + conn, err := grpc.NewClient(s.gwConnAddr, + grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{ + RootCAs: s.clientCaCertPool, + Certificates: []tls.Certificate{*cert}, + InsecureSkipVerify: false, + }))) + if err != nil { + s.T().Fatalf("failed to connect to test gateway: %s", err) + } + + return conn +} diff --git a/gateway/test/suite_test.go b/gateway/test/suite_test.go index 74302249..78f7f154 100644 --- a/gateway/test/suite_test.go +++ b/gateway/test/suite_test.go @@ -2,7 +2,9 @@ package test import ( "context" + "crypto/ecdsa" "crypto/tls" + "crypto/x509" "encoding/base64" "encoding/json" "fmt" @@ -24,6 +26,7 @@ import ( "github.com/couchbase/stellar-gateway/contrib/grpcheaderauth" "github.com/couchbase/stellar-gateway/gateway" "github.com/couchbase/stellar-gateway/testutils" + "github.com/couchbase/stellar-gateway/utils/certificates" "github.com/couchbase/stellar-gateway/utils/selfsignedcert" "github.com/google/uuid" "github.com/stretchr/testify/assert" @@ -41,6 +44,7 @@ type GatewayOpsTestSuite struct { testClusterInfo *testutils.CanonicalTestCluster gatewayCloseFunc func() gatewayConn *grpc.ClientConn + gwConnAddr string gatewayClosedCh chan struct{} dapiCli *http.Client dapiAddr string @@ -56,6 +60,10 @@ type GatewayOpsTestSuite struct { basicRestCreds string readRestCreds string + caCert *x509.Certificate + caKey *ecdsa.PrivateKey + clientCaCertPool *x509.CertPool + clusterVersion *NodeVersion features []TestFeature @@ -264,11 +272,23 @@ func (s *GatewayOpsTestSuite) SetupSuite() { s.T().Fatalf("failed to initialize test logging: %s", err) } - gwCert, err := selfsignedcert.GenerateCertificate() + caCert, caKey, err := selfsignedcert.GenerateCertificate() if err != nil { s.T().Fatalf("failed to create testing certificate: %s", err) } + gwCert, err := certificates.GenerateSignedServerCert(caCert, caKey) + if err != nil { + s.T().Fatalf("failed to create signed gateway certificate: %s", err) + } + + clientCaCert := x509.NewCertPool() + clientCaCert.AddCert(caCert) + + s.caCert = caCert + s.caKey = caKey + s.clientCaCertPool = clientCaCert + gwStartInfoCh := make(chan *gateway.StartupInfo, 1) gwCtx, gwCtxCancel := context.WithCancel(context.Background()) gw, err := gateway.NewGateway(&gateway.Config{ @@ -280,6 +300,7 @@ func (s *GatewayOpsTestSuite) SetupSuite() { BindDapiPort: 0, GrpcCertificate: *gwCert, DapiCertificate: *gwCert, + ClientCaCert: clientCaCert, AlphaEndpoints: true, NumInstances: 1, ProxyServices: []string{"query", "analytics", "mgmt", "search"}, @@ -324,6 +345,7 @@ func (s *GatewayOpsTestSuite) SetupSuite() { } s.gatewayConn = conn + s.gwConnAddr = connAddr s.gatewayCloseFunc = gwCtxCancel s.gatewayClosedCh = gwClosedCh s.dapiCli = dapiCli @@ -346,6 +368,7 @@ func (s *GatewayOpsTestSuite) SetupSuite() { } s.gatewayConn = conn + s.gwConnAddr = connAddr s.dapiCli = dapiCli s.dapiAddr = dapiAddr } diff --git a/testutils/dinocluster.go b/testutils/dinocluster.go index b08abb68..e5cceb38 100644 --- a/testutils/dinocluster.go +++ b/testutils/dinocluster.go @@ -3,6 +3,7 @@ package testutils import ( "bufio" "context" + "fmt" "io" "log" "net/http" @@ -93,6 +94,22 @@ func runDinoRemoveNode(node string) error { return runNoResDinoCmd([]string{"nodes", "rm", globalTestConfig.DinoId, node}) } +func runDinoAddUser(username string, canRead, canWrite bool) error { + return runNoResDinoCmd([]string{ + "users", + "add", + globalTestConfig.DinoId, + username, + "--password=password", + fmt.Sprintf("--can-read=%v", canRead), + fmt.Sprintf("--can-write=%v", canWrite), + }) +} + +func runDinoRemoveUser(username string) error { + return runNoResDinoCmd([]string{"users", "remove", globalTestConfig.DinoId, username}) +} + type DinoController struct { t *testing.T oldFoSettings *cbmgmtx.GetAutoFailoverSettingsResponse @@ -182,3 +199,23 @@ func (c *DinoController) RemoveNode(node string) { err := runDinoRemoveNode(node) require.NoError(c.t, err) } + +func (c *DinoController) AddUnprivilegedUser(username string) { + err := runDinoAddUser(username, false, false) + require.NoError(c.t, err) +} + +func (c *DinoController) AddReadOnlyUser(username string) { + err := runDinoAddUser(username, true, false) + require.NoError(c.t, err) +} + +func (c *DinoController) AddWriteUser(username string) { + err := runDinoAddUser(username, true, true) + require.NoError(c.t, err) +} + +func (c *DinoController) RemoveUser(username string) { + err := runDinoRemoveUser(username) + require.NoError(c.t, err) +} diff --git a/utils/certificates/certificates.go b/utils/certificates/certificates.go new file mode 100644 index 00000000..90de7ce6 --- /dev/null +++ b/utils/certificates/certificates.go @@ -0,0 +1,73 @@ +package certificates + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "fmt" + "math/big" + "net" + "time" + + "github.com/couchbase/stellar-gateway/utils/selfsignedcert" + "github.com/pkg/errors" +) + +func GenerateSignedServerCert(caCert *x509.Certificate, caKey *ecdsa.PrivateKey) (*tls.Certificate, error) { + priv, err := ecdsa.GenerateKey(elliptic.P384(), rand.Reader) + if err != nil { + panic(err) + } + + template := x509.Certificate{ + SerialNumber: big.NewInt(1), + NotBefore: time.Now(), + NotAfter: time.Now().Add(7 * 24 * time.Hour), + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + IPAddresses: []net.IP{net.ParseIP("127.0.0.1")}, + } + + derBytes, err := x509.CreateCertificate(rand.Reader, &template, caCert, &priv.PublicKey, caKey) + if err != nil { + return nil, errors.Wrap(err, "failed to create signed server certificate") + } + + cert, err := x509.ParseCertificate(derBytes) + if err != nil { + return nil, errors.Wrap(err, "failed to parse cert from bytes") + } + + return selfsignedcert.ConstructTlsCert(cert, priv) +} + +func GenerateSignedClientCert(caCert *x509.Certificate, caKey *ecdsa.PrivateKey, username string) (*tls.Certificate, error) { + priv, err := ecdsa.GenerateKey(elliptic.P384(), rand.Reader) + if err != nil { + panic(err) + } + + template := x509.Certificate{ + NotBefore: time.Now(), + NotAfter: time.Now().Add(7 * 24 * time.Hour), + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, + BasicConstraintsValid: true, + EmailAddresses: []string{fmt.Sprintf("%s@couchbase.com", username)}, + } + + derBytes, err := x509.CreateCertificate(rand.Reader, &template, caCert, &priv.PublicKey, caKey) + if err != nil { + return nil, errors.Wrap(err, "failed to create singed server certificate") + } + + cert, err := x509.ParseCertificate(derBytes) + if err != nil { + return nil, errors.Wrap(err, "failed to parse cert from bytes") + } + + return selfsignedcert.ConstructTlsCert(cert, priv) +} diff --git a/utils/selfsignedcert/selfsignedcert.go b/utils/selfsignedcert/selfsignedcert.go index f19f9ca6..fb3aee6a 100644 --- a/utils/selfsignedcert/selfsignedcert.go +++ b/utils/selfsignedcert/selfsignedcert.go @@ -15,21 +15,16 @@ import ( "github.com/pkg/errors" ) -func GenerateCertificate() (*tls.Certificate, error) { +func GenerateCertificate() (*x509.Certificate, *ecdsa.PrivateKey, error) { priv, err := ecdsa.GenerateKey(elliptic.P384(), rand.Reader) if err != nil { - return nil, errors.Wrap(err, "failed to generate private key") - } - - privBytes, err := x509.MarshalPKCS8PrivateKey(priv) - if err != nil { - return nil, errors.Wrap(err, "unable to marshal private key") + return nil, nil, errors.New("failed to generate private key") } serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128) serialNumber, err := rand.Int(rand.Reader, serialNumberLimit) if err != nil { - return nil, errors.Wrap(err, "failed to generate serial number") + return nil, nil, errors.Wrap(err, "failed to generate serial number") } template := x509.Certificate{ @@ -41,32 +36,49 @@ func GenerateCertificate() (*tls.Certificate, error) { NotBefore: time.Now(), NotAfter: time.Now().Add(7 * 24 * time.Hour), - KeyUsage: x509.KeyUsageDigitalSignature, - ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign, BasicConstraintsValid: true, + IsCA: true, } derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv) if err != nil { - return nil, errors.Wrap(err, "failed to create certificate") + return nil, nil, errors.Wrap(err, "failed to create certificate") + } + + cert, err := x509.ParseCertificate(derBytes) + if err != nil { + return nil, nil, errors.Wrap(err, "failed to parse cert from bytes") + } + + return cert, priv, nil +} + +func ConstructTlsCert(cert *x509.Certificate, key *ecdsa.PrivateKey) (*tls.Certificate, error) { + keyBytes, err := x509.MarshalPKCS8PrivateKey(key) + if err != nil { + return nil, errors.Wrap(err, "unable to marshal private key") } keyBuf := bytes.NewBuffer(nil) - err = pem.Encode(keyBuf, &pem.Block{Type: "PRIVATE KEY", Bytes: privBytes}) + err = pem.Encode(keyBuf, &pem.Block{ + Type: "EC PRIVATE KEY", + Bytes: keyBytes, + }) if err != nil { return nil, errors.Wrap(err, "failed to write key pem data") } certBuf := bytes.NewBuffer(nil) - err = pem.Encode(certBuf, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}) + err = pem.Encode(certBuf, &pem.Block{Type: "CERTIFICATE", Bytes: cert.Raw}) if err != nil { return nil, errors.Wrap(err, "failed to write cert pem data") } - cert, err := tls.X509KeyPair(certBuf.Bytes(), keyBuf.Bytes()) + tlsCert, err := tls.X509KeyPair(certBuf.Bytes(), keyBuf.Bytes()) if err != nil { return nil, errors.Wrap(err, "failed to produce tls certificate") } - return &cert, nil + return &tlsCert, nil }