Merge #159010 #159787

159010: roachtest: randomize volume type and XFS r=nameisbhaskar,williamchoe3,herkolategan a=golgeek

Prior to this patch, tests had three possiblities for volume types:
1. Specify `PreferLocalSSD()` `DisableLocalSSD()` options.
2. Force a specific volume type via `VolumeType()` (volume type being provider specific, this was only possible for tests that target a single provider).
3. Rely on the default behavior for `PreferLocalSSD()` and use the default volume type in roachprod for the provider in case local SSD was not selected.

The way `PreferLocalSSD()` was implemented meant that most of the tests were actually running on local SSDs (when available on the machine type), leading to a gap in the testing strategy with regards to volume types.

This patch brings the new ClusterSpec option `RandomizeVolumeType()`. In case volume type is not forced and neither `PreferLocalSSD()` or `DisableLocalSSD()` are specified, specifying `RandomizeVolumeType()` will take precendence over the `--local-ssd` argument and will randomly select a volume type from those available in the targeted provider:
- AWS: gp3, io2, local SSD
- GCE: pd-ssd, local SSD
- Azure: premium-ssd, premium-ssd-v2, ultra-disk, local SSD
- IBM: 10iops-tier

Note: volume type randomization gives the same weight to all options.

This patch also introduces a random chance of provisioning XFS as a filesystem via the option `RandomlyUseXfs()`.
The option is built in the same way as `RandomlyUseZfs()`, granting a 20% chance of XFS if present.
If both `RandomlyUseZfs()` and `RandomlyUseXfs()` are used, they both get a 20% chance, leaving 60% chance for the default ext4.

All/most of the KV tests, and a few admission-control are switched to volume type/filesystem randomization as requested in [this issue](#146661). 

Epic: none
Fixes: 146661
Release note: None

159787: oidcccl,provisioning: add user provisioning for OIDC authentication r=souravcrl a=souravcrl

Previously, the OIDC authentication flow in the DB Console only supported logging in with existing user accounts. There was no built-in mechanism to automatically create a new database user when a user authenticated via an OIDC provider for the first time.

This was inadequate because administrators would need to manually create a database user for every first‑time OIDC login, adding friction and overhead to onboarding.

To address this, this patch introduces automatic user provisioning for the OIDC authentication flow. When a new user successfully authenticates via an OIDC provider, a corresponding CockroachDB user is now automatically created if one does not already exist. This functionality is controlled by a new cluster setting, `security.provisioning.oidc.enabled`, which is disabled by default to maintain backward compatibility and ensure administrators can opt-in to this behavior.

Note: The `security.provisioning.oidc.enabled` cluster setting requires checking user existence before provisioning. This may introduce latency when concurrent OIDC authentication attempts from browsers generate high read request load on the user table.

Fixes: #126680
Epic: CRDB-48764

Release note (enterprise change): A new cluster setting, `security.provisioning.oidc.enabled`, has been added to allow for the automatic provisioning of users when they log in for the first time via OIDC. When enabled, a new user will be created in CockroachDB upon their first successful OIDC authentication. This feature is disabled by default.

On enabling the setting, user gets created on oidc login and can be validated using the `SHOW users` command.

```
   > SELECT * FROM [SHOW USERS] WHERE username = 'testuser';

       username    |                     options                     | member_of | estimated_last_login_time
  -----------------+-------------------------------------------------+-----------+----------------------------
    testuser       | {PROVISIONSRC=oidc:https://accounts.google.com} | {}        | NULL
  (1 row)

  NOTICE: estimated_last_login_time is computed on a best effort basis; it is not guaranteed to capture every login event
```

Co-authored-by: Ludovic Leroux <ludo.leroux@cockroachlabs.com>
Co-authored-by: Shriram Ravindranathan <shriram.ravindranathan@cockroachlabs.com>

Author: 3 people

pkg/ccl/logictestccl/testdata/logic_test/provisioning

@@ -3,7 +3,7 @@
 statement error role "root" cannot have a PROVISIONSRC
 ALTER ROLE root PROVISIONSRC 'ldap:ldap.example.com'
 
-statement error pq: PROVISIONSRC "ldap.example.com" was not prefixed with any valid auth methods \["ldap" "jwt_token"\]
+statement error pq: PROVISIONSRC "ldap.example.com" was not prefixed with any valid auth methods \["ldap" "jwt_token" "oidc"\]
 CREATE ROLE role_with_provisioning PROVISIONSRC 'ldap.example.com'
 
 statement error pq: conflicting role options

pkg/ccl/oidcccl/BUILD.bazel

@@ -16,6 +16,7 @@ go_library(
         "//pkg/ccl/securityccl/jwthelper",
         "//pkg/ccl/utilccl",
         "//pkg/roachpb",
+        "//pkg/security/provisioning",
         "//pkg/security/username",
         "//pkg/server",
         "//pkg/server/authserver",
@@ -55,6 +56,7 @@ go_test(
         "//pkg/ccl",
         "//pkg/roachpb",
         "//pkg/security/certnames",
+        "//pkg/security/provisioning",
         "//pkg/security/securityassets",
         "//pkg/security/securitytest",
         "//pkg/security/username",

pkg/ccl/oidcccl/authentication_oidc.go

@@ -19,6 +19,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/ccl/jwtauthccl"
 	"github.com/cockroachdb/cockroach/pkg/ccl/utilccl"
 	"github.com/cockroachdb/cockroach/pkg/roachpb"
+	"github.com/cockroachdb/cockroach/pkg/security/provisioning"
 	secuser "github.com/cockroachdb/cockroach/pkg/security/username"
 	"github.com/cockroachdb/cockroach/pkg/server"
 	"github.com/cockroachdb/cockroach/pkg/server/authserver"
@@ -44,6 +45,7 @@ const (
 	codeKey                  = "code"
 	stateKey                 = "state"
 	secretCookieName         = "oidc_secret"
+	oidcProvisioningKey      = "oidc"
 	oidcLoginPath            = "/oidc/v1/login"
 	oidcCallbackPath         = "/oidc/v1/callback"
 	oidcJWTPath              = "/oidc/v1/jwt"
@@ -158,6 +160,7 @@ type oidcAuthenticationConf struct {
 	authZEnabled                 bool
 	groupClaim                   string
 	userinfoGroupKey             string
+	provisioningEnabled          bool
 }
 
 // GetOIDCConf is used to extract certain parts of the OIDC
@@ -420,9 +423,10 @@ func reloadConfigLocked(
 			httputil.WithDialerTimeout(clientTimeout),
 			httputil.WithCustomCAPEM(OIDCProviderCustomCA.Get(&st.SV)),
 		),
-		authZEnabled:     OIDCAuthZEnabled.Get(&st.SV),
-		groupClaim:       OIDCAuthGroupClaim.Get(&st.SV),
-		userinfoGroupKey: OIDCAuthUserinfoGroupKey.Get(&st.SV),
+		authZEnabled:        OIDCAuthZEnabled.Get(&st.SV),
+		groupClaim:          OIDCAuthGroupClaim.Get(&st.SV),
+		userinfoGroupKey:    OIDCAuthUserinfoGroupKey.Get(&st.SV),
+		provisioningEnabled: provisioning.ClusterProvisioningConfig(st).Enabled("oidc"),
 	}
 
 	if !oidcAuthServer.conf.enabled && conf.enabled {
@@ -490,6 +494,69 @@ func getRegionSpecificRedirectURL(locality roachpb.Locality, conf redirectURLCon
 	return s, nil
 }
 
+// maybeProvisionUserLocked checks the cached OIDC configuration to see whether
+// automatic user provisioning is enabled. If so, it attempts to create a SQL
+// user linked to the OIDC identity provider.
+//
+// This function is called after a successful OIDC token exchange and
+// verification. Its execution relies on the success of the underlying OIDC
+// library, which operates on the following assumptions:
+//
+//  1. OIDC Discovery: The library uses the OIDC discovery protocol to fetch the
+//     provider's configuration from "/.well-known/openid-configuration". This
+//     assumes the provider has discovery enabled and accessible.
+//
+//  2. Issuer Validation: The go-oidc library's verifier ensures the 'iss' claim
+//     in the ID Token matches the issuer URL from the discovery document. There
+//     is also an exception made to this in go-oidc for accounts.google.com
+//
+// Errors during username validation, provisioning source parsing, or user creation
+// are logged with detailed context and result in an HTTP 500 response with a
+// generic error message to the client.
+func maybeProvisionUserLocked(
+	ctx context.Context,
+	authConf oidcAuthenticationConf,
+	execCfg *sql.ExecutorConfig,
+	username string,
+	w http.ResponseWriter,
+) (err error) {
+	if !authConf.provisioningEnabled {
+		return
+	}
+
+	log.Dev.Infof(ctx, "OIDC: attempting user provisioning for %s", username)
+	telemetry.Inc(provisioning.BeginOIDCProvisionUseCounter)
+
+	// Convert the extracted username string to a username.SQLUsername type.
+	sqlUsername, err := secuser.MakeSQLUsernameFromUserInput(username, secuser.PurposeCreation)
+	if err != nil {
+		log.Dev.Errorf(ctx, "OIDC provisioning: invalid username format for %s: %v", username, err)
+		http.Error(w, "OIDC: invalid username format", http.StatusInternalServerError)
+		return err
+	}
+
+	// Create the provisioning source identifier string, e.g., "oidc:https://accounts.example.com".
+	idpString := oidcProvisioningKey + ":" + authConf.providerURL
+	provisioningSource, err := provisioning.ParseProvisioningSource(idpString)
+	if err != nil {
+		// This error occurs if the provisioning package doesn't recognize the "oidc:" prefix.
+		log.Dev.Errorf(ctx, "OIDC provisioning: error parsing provisioning source IDP %s: %v", idpString, err)
+		http.Error(w, "OIDC: provisioning error", http.StatusInternalServerError)
+		return err
+	}
+
+	// Call the core provisioning function using the execCfg.
+	if err = sql.CreateRoleForProvisioning(ctx, execCfg, sqlUsername, provisioningSource.String()); err != nil {
+		log.Dev.Errorf(ctx, "OIDC provisioning: error provisioning user %s: %v", sqlUsername, err)
+		http.Error(w, "OIDC: provisioning error", http.StatusInternalServerError)
+		return err
+	}
+
+	log.Dev.Infof(ctx, "OIDC: successfully provisioned user %s", sqlUsername)
+	telemetry.Inc(provisioning.ProvisionOIDCSuccessCounter)
+	return
+}
+
 // ConfigureOIDC attaches handlers to the server `mux` that
 // can initiate and complete an OIDC authentication flow.
 // This flow consists of an initial login request that triggers
@@ -608,6 +675,12 @@ var ConfigureOIDC = func(
 			return
 		}
 
+		// OIDC user provisioning
+		if err := maybeProvisionUserLocked(ctx, oidcAuthentication.conf, oidcAuthentication.execCfg, username, w); err != nil {
+			log.Dev.Errorf(ctx, "OIDC provisioning failed with error: %v", err)
+			return
+		}
+
 		// OIDC authorization
 		if err := oidcAuthentication.authorize(ctx, rawIDToken, rawAccessToken, username); err != nil {
 			log.Dev.Errorf(ctx, "OIDC authorization failed with error: %v", err)
@@ -938,6 +1011,9 @@ var ConfigureOIDC = func(
 	OIDCAuthUserinfoGroupKey.SetOnChange(&st.SV, func(ctx context.Context) {
 		reloadConfig(ambientCtx.AnnotateCtx(ctx), oidcAuthentication, locality, st)
 	})
+	provisioning.OIDCProvisioningEnabled.SetOnChange(&st.SV, func(ctx context.Context) {
+		reloadConfig(ambientCtx.AnnotateCtx(ctx), oidcAuthentication, locality, st)
+	})
 
 	return oidcAuthentication, nil
 }