Test connecting tenants via password over TLS through proxy

This commit adds infrastructure for testing secure connections through
the multi-tenancy proxy to a standalone SQL server. This works "out of
the box" for essentially those tests for which we previously had client
certificates working.

The tests that fail here are GoPG, Hibernate, Sequelize, Django and they
all fail for the basic reason that they don't support secure query
strings, at least not in the way they are supplied right now. To
*really* find out whether they have issues with multi-tenancy (or even
client certs), we need to customize the way they receive the connection
parameters. This is not something I plan to investigate.

Closes cockroachdb/cockroach#52413.

Author: tbg

testing/main_test.go

@@ -47,14 +47,19 @@ type tenantServer interface {
 }
 
 // newServer creates a new cockroachDB server.
-func newServer(t *testing.T, insecure bool) testserver.TestServer {
+func newServer(t *testing.T, auth authMode) testserver.TestServer {
 	t.Helper()
 	var ts testserver.TestServer
 	var err error
-	if insecure {
-		ts, err = testserver.NewTestServer()
-	} else {
+	switch auth {
+	case authClientCert:
 		ts, err = testserver.NewTestServer(testserver.SecureOpt())
+	case authPassword:
+		ts, err = testserver.NewTestServer(testserver.SecureOpt(), testserver.RootPasswordOpt("hunter2"))
+	case authInsecure:
+		ts, err = testserver.NewTestServer()
+	default:
+		err = fmt.Errorf("unknown authMode %d", auth)
 	}
 	if err != nil {
 		t.Fatal(err)
@@ -64,9 +69,9 @@ func newServer(t *testing.T, insecure bool) testserver.TestServer {
 
 // newTenant creates a new SQL Tenant pointed at the given TestServer. See
 // TestServer.NewTenantServer for more information.
-func newTenant(t *testing.T, ts testserver.TestServer) testserver.TestServer {
+func newTenant(t *testing.T, ts testserver.TestServer, proxy bool) testserver.TestServer {
 	t.Helper()
-	tenant, err := ts.(tenantServer).NewTenantServer(false /* proxy */)
+	tenant, err := ts.(tenantServer).NewTenantServer(proxy)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -193,17 +198,39 @@ var minRequiredVersionsByORMName = map[string]struct {
 	},
 }
 
+type authMode byte
+
+const (
+	// Use client certs. When testing tenants, does not use the proxy (as the proxy does not support client certs).
+	authClientCert authMode = iota
+	// Use password auth. When testing tenants, tests through a proxy.
+	authPassword
+	// Use --insecure. When testing tenants, does not use the proxy (as the proxy does not support insecure connections).
+	authInsecure
+
+	authModeSentinel // sentinel to iterate over all modes
+)
+
+func (mode authMode) String() string {
+	switch mode {
+	case authClientCert:
+		return "client-cert"
+	case authPassword:
+		return "password"
+	case authInsecure:
+		return "insecure"
+	default:
+		return "unknown"
+	}
+}
+
 type testInfo struct {
 	language, orm string
 	tableNames    testTableNames  // defaults to defaultTestTableNames
 	columnNames   testColumnNames // defaults to defaultTestColumnNames
-	// insecure is set if ORM does not handle secure servers (client certs).
-	// In that case, we start an insecure server (and don't test in tenant
-	// mode).
-	insecure bool
 }
 
-func testORM(t *testing.T, info testInfo) {
+func testORM(t *testing.T, info testInfo, auth authMode) {
 	if info.tableNames == (testTableNames{}) {
 		info.tableNames = defaultTestTableNames
 	}
@@ -222,7 +249,7 @@ func testORM(t *testing.T, info testInfo) {
 	}
 	var testCases []testCase
 	{
-		ts := newServer(t, info.insecure)
+		ts := newServer(t, auth)
 		db, dbURL, stopDB := startServerWithApplication(t, ts, app)
 		defer stopDB()
 
@@ -268,11 +295,19 @@ FROM
 			t.Fatalf("unable to read cluster version: %s", err)
 		}
 		if tenantsSupported {
-			tenant := newTenant(t, ts)
+			// Connect to the tenant through the SQL proxy, which is only supported
+			// when using secure+password auth. (The proxy does not support client
+			// certs or insecure connections).
+			proxySupported := auth == authPassword
+			name := "RegularTenant"
+			if proxySupported {
+				name += "ThroughProxy"
+			}
+			tenant := newTenant(t, ts, proxySupported)
 			db, dbURL, stopDB := startServerWithApplication(t, tenant, app)
 			defer stopDB()
 			testCases = append(testCases, testCase{
-				name:  "RegularTenant",
+				name:  name,
 				db:    db,
 				dbURL: dbURL,
 			})
@@ -368,62 +403,108 @@ FROM
 	}
 }
 
+func testORMForAuthModesExcept(t *testing.T, info testInfo, skips map[authMode]string /* mode -> reason */) {
+	for auth := authMode(0); auth < authModeSentinel; auth++ {
+		t.Run(fmt.Sprint(auth), func(t *testing.T) {
+			if msg := skips[auth]; msg != "" {
+				t.Skip(msg)
+			}
+			testORM(t, info, auth)
+		})
+	}
+}
+
+func nothingSkipped() map[authMode]string { return nil }
+
 func TestGORM(t *testing.T) {
-	testORM(t, testInfo{language: "go", orm: "gorm"})
+	testORMForAuthModesExcept(t, testInfo{language: "go", orm: "gorm"}, nothingSkipped())
 }
 
 func TestGOPG(t *testing.T) {
-	testORM(t, testInfo{
-		language: "go",
-		orm:      "gopg",
-		// GoPG does not support client certs:
-		// https://github.com/go-pg/pg/blob/v10/options.go
-		// If we set up a secure deployment and went through the proxy, it would work (or should anyway), but only
-		// via the 'database' parameter; GoPG also does not support the 'options' parameter.
-		insecure: true,
-	})
+	testORMForAuthModesExcept(t,
+		testInfo{language: "go", orm: "gopg"},
+		map[authMode]string{
+			// https://github.com/go-pg/pg/blob/v10/options.go
+			// If we set up a secure deployment and went through the proxy, it would work (or should anyway), but only
+			// via the 'database' parameter; GoPG also does not support the 'options' parameter.
+			//
+			// pg: options other than 'sslmode', 'application_name' and 'connect_timeout' are not supported
+			authClientCert: "GoPG does not support custom root cert",
+			authPassword:   "GoPG does not support custom root cert",
+		})
 }
 
 func TestHibernate(t *testing.T) {
-	testORM(t, testInfo{
-		language: "java",
-		orm:      "hibernate",
-		// Possibly does not unescape the path correctly:
-		// Caused by: java.io.FileNotFoundException:
-		//	%2Ftmp%2Fcockroach-testserver913095208%2Fcerts%2Fca.crt (No such file or directory)
-		insecure: true,
-	})
+	testORMForAuthModesExcept(t,
+		testInfo{language: "java", orm: "hibernate"},
+		map[authMode]string{
+			// Driver does not unescape the path correctly:
+			// Caused by: java.io.FileNotFoundException:
+			//	%2Ftmp%2Fcockroach-testserver913095208%2Fcerts%2Fca.crt (No such file or directory)
+			//
+			// Furthermore, if we preprocess the query string via
+			//
+			//    tc.dbURL.RawQuery, err = url.QueryUnescape(tc.dbURL.RawQuery)
+			//
+			// then we run into
+			//   https://github.com/dbeaver/dbeaver/issues/1835
+			// because hibernate expects the key in DER format, but it is PEM.
+			authClientCert: "needs DER format and unescaped query string",
+			// Doesn't seem to understand connection strings.
+			// Caused by: java.net.UnknownHostException: root:hunter2@localhost
+			authPassword: "needs custom setup for password support",
+		},
+	)
 }
 
 func TestSequelize(t *testing.T) {
-	testORM(t, testInfo{
-		language: "node",
-		orm:      "sequelize",
-		// Requires bespoke code to actually use SSL, see:
-		// https://github.com/sequelize/sequelize/issues/10015
-		insecure: true,
-	})
+	testORMForAuthModesExcept(t,
+		testInfo{language: "node", orm: "sequelize"},
+		map[authMode]string{
+			// Requires bespoke code to actually use SSL, see:
+			// https://github.com/sequelize/sequelize/issues/10015
+			authClientCert: "needs custom SSL setup",
+			authPassword:   "needs custom SSL setup",
+		},
+	)
 }
 
 func TestSQLAlchemy(t *testing.T) {
-	testORM(t, testInfo{
-		language: "python",
-		orm:      "sqlalchemy",
-	})
+	testORMForAuthModesExcept(t, testInfo{language: "python", orm: "sqlalchemy"}, nothingSkipped())
 }
 
 func TestDjango(t *testing.T) {
-	testORM(t, testInfo{
+	testORMForAuthModesExcept(t, testInfo{
 		language:    "python",
 		orm:         "django",
 		tableNames:  djangoTestTableNames,
 		columnNames: djangoTestColumnNames,
-		// No support for client certs (at least not via the query string).
-		// psycopg2.OperationalError: fe_sendauth: no password supplied
-		insecure: true,
-	})
+	},
+		map[authMode]string{
+			// No support for client certs (at least not via the query string).
+			// psycopg2.OperationalError: fe_sendauth: no password supplied
+			authClientCert: "client certs via query string unsupported",
+			// Ditto,
+			// psycopg2.OperationalError: fe_sendauth: no password supplied
+			authPassword: "password via query string unsupported",
+		},
+	)
 }
 
+// TODO(rafiss): why can't I run the ActiveRecords tests manually
+// with this invocation?
+//
+//   ./docker.sh make deps
+//   ./docker.sh go test -v -run ActiveRecord ./testing
+//
+// It always fails opaquely like this (after some normal-looking output):
+//
+//   => Run `rails server -h` for more startup options
+//   Exiting
+//   make: *** [Makefile:23: start] Error 1
+//
+
 func TestActiveRecord(t *testing.T) {
-	testORM(t, testInfo{language: "ruby", orm: "activerecord"})
+	testORMForAuthModesExcept(t, testInfo{language: "ruby", orm: "activerecord"}, nothingSkipped())
 }
+