wip: Failing test - error on read after dial.

Author: hessjcg

dialer_test.go

@@ -1174,3 +1174,77 @@ func TestDialerChecksSubjectAlternativeNameAndFails(t *testing.T) {
 		t.Fatal("want error containing `tls: failed to verify certificate`. Got: ", err)
 	}
 }
+
+func TestDialerRefreshesAfterClientCertificateError(t *testing.T) {
+	inst := mock.NewFakeCSQLInstanceWithSan(
+		"my-project", "my-region", "my-instance", []string{"db.example.com"},
+		mock.WithDNS("db.example.com"),
+		mock.WithServerCAMode("GOOGLE_MANAGED_CAS_CA"),
+	)
+
+	d := setupDialer(t, setupConfig{
+		skipServer:   true,
+		testInstance: inst,
+		reqs: []*mock.Request{
+			mock.InstanceGetSuccess(inst, 2),
+			mock.CreateEphemeralSuccess(inst, 2),
+		},
+		dialerOptions: []Option{
+			WithTokenSource(mock.EmptyTokenSource{}),
+			WithDebugLogger(&dialerTestLogger{t: t}),
+			//WithLazyRefresh(),
+			// Note: this succeeds with lazy refresh, but fails with lazy.
+			// because dialer.ForceRefresh does not block connections while the
+			// refresh is in progress.
+		},
+	})
+	cancel1 := mock.StartServerProxy(t, inst)
+	t.Log("First attempt...")
+	testSuccessfulDial(
+		context.Background(), t, d,
+		"my-project:my-region:my-instance",
+	)
+	t.Log("First attempt OK. Resetting client cert.")
+
+	// Close the server
+	cancel1()
+
+	mock.RotateClientCA(inst)
+	time.Sleep(2 * time.Second)
+
+	// Recreate the instance, which generates new server certificates
+	// Start the server with new certificates
+	cancel2 := mock.StartServerProxy(t, inst)
+	defer cancel2()
+
+	// Dial a second time. We expect no error on dial, but TLS error on read.
+	conn, err := d.Dial(context.Background(), "my-project:my-region:my-instance")
+	if err != nil {
+		t.Fatal("Should be no certificate error after, got ", err)
+	}
+
+	// Expect an error on read. This should trigger the dialer to refresh.
+	_, err = io.ReadAll(conn)
+	if err != nil {
+		t.Log("Got error on read as expected.", err)
+	} else {
+		t.Fatal("Want read error, got no error")
+	}
+
+	time.Sleep(2 * time.Second)
+	// Dial again. This should occur after the refresh has completed.
+	t.Log("Third attempt...")
+	testSuccessfulDial(
+		context.Background(), t, d,
+		"my-project:my-region:my-instance",
+	)
+	t.Log("Third attempt OK.")
+}
+
+type dialerTestLogger struct {
+	t *testing.T
+}
+
+func (l *dialerTestLogger) Debugf(f string, args ...interface{}) {
+	l.t.Logf(f, args...)
+}

internal/mock/certs.go

@@ -48,22 +48,22 @@ var instanceWithCnSubject = name("myProject:myInstance")
 // From the cloud-sql-jdbc-socket-factory project:
 // core/src/test/java/com/google/cloud/sql/core/TestCertificateGenerator.java
 type TLSCertificates struct {
-	ServerCaKeyPair             *rsa.PrivateKey
-	SigningCaKeyPair            *rsa.PrivateKey
-	ServerKeyPair               *rsa.PrivateKey
-	ServerIntermediateCaKeyPair *rsa.PrivateKey
-	ServerSigningCaKeyPair      *rsa.PrivateKey
-	ClientKeyPair               *rsa.PrivateKey
-	DomainServerKeyPair         *rsa.PrivateKey
-
-	ServerCaCert              *x509.Certificate
-	SigningCaCert             *x509.Certificate
-	ServerCert                *x509.Certificate
-	ServerIntermediateCaCert  *x509.Certificate
-	CasServerCertificate      *x509.Certificate
-	CasServerCertificateChain []*x509.Certificate
-	DomainServerCertificate   *x509.Certificate
-	clientCertExpires         time.Time
+	clientCertExpires time.Time
+	projectName       string
+	instanceName      string
+	sans              []string
+
+	serverCaKeyPair             *rsa.PrivateKey
+	serverIntermediateCaKeyPair *rsa.PrivateKey
+	clientSigningCaKeyPair      *rsa.PrivateKey
+
+	serverCaCert               *x509.Certificate
+	serverIntermediateCaCert   *x509.Certificate
+	clientSigningCACertificate *x509.Certificate
+
+	serverKeyPair        *rsa.PrivateKey
+	serverCert           *x509.Certificate
+	casServerCertificate *x509.Certificate
 }
 
 func mustGenerateKey() *rsa.PrivateKey {
@@ -76,55 +76,13 @@ func mustGenerateKey() *rsa.PrivateKey {
 
 // newTLSCertificates creates a new instance of the TLSCertificates.
 func newTLSCertificates(projectName, instanceName string, sans []string, clientCertExpires time.Time) *TLSCertificates {
-	oneYear := time.Now().AddDate(1, 0, 0)
-
 	c := &TLSCertificates{
-		clientCertExpires:           clientCertExpires,
-		ServerCaKeyPair:             mustGenerateKey(),
-		SigningCaKeyPair:            mustGenerateKey(),
-		ServerKeyPair:               mustGenerateKey(),
-		ServerIntermediateCaKeyPair: mustGenerateKey(),
-		ServerSigningCaKeyPair:      mustGenerateKey(),
-		ClientKeyPair:               mustGenerateKey(),
-		DomainServerKeyPair:         mustGenerateKey(),
+		clientCertExpires: clientCertExpires,
+		projectName:       projectName,
+		instanceName:      instanceName,
+		sans:              sans,
 	}
-
-	c.ServerCaCert = mustBuildRootCertificate(serverCaSubject, c.ServerCaKeyPair)
-	c.SigningCaCert = mustBuildRootCertificate(signingCaSubject, c.SigningCaKeyPair)
-
-	c.ServerCert = mustBuildSignedCertificate(
-		false,
-		name(projectName+":"+instanceName),
-		c.ServerKeyPair,
-		serverCaSubject,
-		c.ServerCaKeyPair,
-		oneYear,
-		nil)
-
-	c.ServerIntermediateCaCert =
-		mustBuildSignedCertificate(
-			true,
-			intermediateCaSubject,
-			c.ServerIntermediateCaKeyPair,
-			serverCaSubject,
-			c.ServerCaKeyPair,
-			oneYear,
-			nil)
-
-	c.CasServerCertificate =
-		mustBuildSignedCertificate(
-			false,
-			name(""),
-			c.ServerKeyPair,
-			intermediateCaSubject,
-			c.ServerIntermediateCaKeyPair,
-			oneYear,
-			sans)
-
-	c.CasServerCertificateChain =
-		[]*x509.Certificate{
-			c.CasServerCertificate, c.ServerIntermediateCaCert, c.ServerCaCert}
-
+	c.RotateCA()
 	return c
 }
 
@@ -252,7 +210,7 @@ func (ct *TLSCertificates) signWithClientKey(clientKey *rsa.PublicKey) ([]byte,
 		KeyUsage:              x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
 		BasicConstraintsValid: true,
 	}
-	certBytes, err := x509.CreateCertificate(rand.Reader, cert, ct.SigningCaCert, clientKey, ct.SigningCaKeyPair)
+	certBytes, err := x509.CreateCertificate(rand.Reader, cert, ct.clientSigningCACertificate, clientKey, ct.clientSigningCaKeyPair)
 	if err != nil {
 		return nil, err
 	}
@@ -273,9 +231,9 @@ func (ct *TLSCertificates) generateServerCertWithCn(cn string) *x509.Certificate
 	return mustBuildSignedCertificate(
 		false,
 		name(cn),
-		ct.ServerKeyPair,
+		ct.serverKeyPair,
 		serverCaSubject,
-		ct.ServerCaKeyPair,
+		ct.serverCaKeyPair,
 		time.Now().Add(1*time.Hour), nil)
 }
 
@@ -286,16 +244,67 @@ func (ct *TLSCertificates) serverChain(serverCAMode string) []tls.Certificate {
 	// if this server is running in legacy mode
 	if serverCAMode == "" || serverCAMode == "GOOGLE_MANAGED_INTERNAL_CA" {
 		return []tls.Certificate{{
-			Certificate: [][]byte{ct.ServerCert.Raw, ct.ServerCaCert.Raw},
-			PrivateKey:  ct.ServerKeyPair,
-			Leaf:        ct.ServerCert,
+			Certificate: [][]byte{ct.serverCert.Raw, ct.serverCaCert.Raw},
+			PrivateKey:  ct.serverKeyPair,
+			Leaf:        ct.serverCert,
 		}}
 	}
 
 	return []tls.Certificate{{
-		Certificate: [][]byte{ct.CasServerCertificate.Raw, ct.ServerIntermediateCaCert.Raw, ct.ServerCaCert.Raw},
-		PrivateKey:  ct.ServerKeyPair,
-		Leaf:        ct.CasServerCertificate,
+		Certificate: [][]byte{ct.casServerCertificate.Raw, ct.serverIntermediateCaCert.Raw, ct.serverCaCert.Raw},
+		PrivateKey:  ct.serverKeyPair,
+		Leaf:        ct.casServerCertificate,
 	}}
 
 }
+func (ct *TLSCertificates) ClientCAPool() *x509.CertPool {
+	clientCa := x509.NewCertPool()
+	clientCa.AddCert(ct.clientSigningCACertificate)
+	return clientCa
+}
+
+func (ct *TLSCertificates) RotateClientCA() {
+	ct.clientSigningCaKeyPair = mustGenerateKey()
+	ct.clientSigningCACertificate = mustBuildRootCertificate(signingCaSubject, ct.clientSigningCaKeyPair)
+}
+
+func (ct *TLSCertificates) RotateCA() {
+	oneYear := time.Now().AddDate(1, 0, 0)
+	ct.serverCaKeyPair = mustGenerateKey()
+	ct.clientSigningCaKeyPair = mustGenerateKey()
+	ct.serverKeyPair = mustGenerateKey()
+	ct.serverIntermediateCaKeyPair = mustGenerateKey()
+
+	ct.serverCaCert = mustBuildRootCertificate(serverCaSubject, ct.serverCaKeyPair)
+
+	ct.serverIntermediateCaCert =
+		mustBuildSignedCertificate(
+			true,
+			intermediateCaSubject,
+			ct.serverIntermediateCaKeyPair,
+			serverCaSubject,
+			ct.serverCaKeyPair,
+			oneYear,
+			nil)
+
+	ct.casServerCertificate =
+		mustBuildSignedCertificate(
+			false,
+			name(""),
+			ct.serverKeyPair,
+			intermediateCaSubject,
+			ct.serverIntermediateCaKeyPair,
+			oneYear,
+			ct.sans)
+
+	ct.serverCert = mustBuildSignedCertificate(
+		false,
+		name(ct.projectName+":"+ct.instanceName),
+		ct.serverKeyPair,
+		serverCaSubject,
+		ct.serverCaKeyPair,
+		oneYear,
+		nil)
+
+	ct.RotateClientCA()
+}

internal/mock/cloudsql.go

@@ -61,8 +61,7 @@ type FakeCSQLInstance struct {
 	// Cert is the server's certificate
 	Cert *x509.Certificate
 	// certs holds all of the certificates for this instance
-	certs    *TLSCertificates
-	clientCa *x509.CertPool
+	certs *TLSCertificates
 }
 
 // String returns the instance connection name for the
@@ -75,12 +74,13 @@ func (f FakeCSQLInstance) String() string {
 func (f FakeCSQLInstance) serverCACert() ([]byte, error) {
 	if f.signer != nil {
 		return f.signer(f.Cert, f.Key)
+	} else {
+		if f.serverCAMode == "" || f.serverCAMode == "GOOGLE_MANAGED_INTERNAL_CA" {
+			// legacy server mode, return only the server cert
+			return toPEMFormat(f.certs.serverCert)
+		}
+		return toPEMFormat(f.certs.casServerCertificate, f.certs.serverIntermediateCaCert, f.certs.serverCaCert)
 	}
-	if f.serverCAMode == "" || f.serverCAMode == "GOOGLE_MANAGED_INTERNAL_CA" {
-		// clasic server mode, return cert with CA
-		return toPEMFormat(f.certs.ServerCert)
-	}
-	return toPEMFormat(f.certs.CasServerCertificateChain...)
 }
 
 // ClientCert creates an ephemeral client certificate signed with the Cloud SQL
@@ -219,17 +219,10 @@ func NewFakeCSQLInstanceWithSan(project, region, name string, sanDNSNames []stri
 
 	certs := newTLSCertificates(project, name, sanDNSNames, f.certExpiry)
 
-	f.Key = certs.ServerKeyPair
-	f.Cert = certs.ServerCert
+	f.Key = certs.serverKeyPair
+	f.Cert = certs.serverCert
 	f.certs = certs
 
-	clientCa, err := x509.SystemCertPool()
-	clientCa.AddCert(certs.SigningCaCert)
-	if err != nil {
-		panic(err)
-	}
-	f.clientCa = clientCa
-
 	return f
 }
 
@@ -263,7 +256,7 @@ func StartServerProxy(t *testing.T, i FakeCSQLInstance) func() {
 
 	ln, err := tls.Listen("tcp", ":3307", &tls.Config{
 		Certificates: i.certs.serverChain(i.serverCAMode),
-		ClientCAs:    i.clientCa,
+		ClientCAs:    i.certs.ClientCAPool(),
 		ClientAuth:   tls.RequireAndVerifyClientCert,
 	})
 	if err != nil {
@@ -303,3 +296,11 @@ func StartServerProxy(t *testing.T, i FakeCSQLInstance) func() {
 		_ = ln.Close()
 	}
 }
+
+func RotateCA(inst FakeCSQLInstance) {
+	inst.certs.RotateCA()
+}
+
+func RotateClientCA(inst FakeCSQLInstance) {
+	inst.certs.RotateClientCA()
+}