GODRIVER-3175 Add Kubernetes support for OIDC.

matthewdale · matthewdale · commit 63b1504dc03b · 2025-03-05T13:49:29.000-08:00
diff --git a/.evergreen/config.yml b/.evergreen/config.yml
@@ -162,17 +162,18 @@ functions:
         args: 
           - ${DRIVERS_TOOLS}/.evergreen/teardown.sh
 
-  assume-ec2-role:
+  assume-test-secrets-ec2-role:
     - command: ec2.assume_role
       params:
         role_arn: ${aws_test_secrets_role}
+        duration_seconds: 1800
 
   run-oidc-auth-test-with-test-credentials:
     - command: subprocess.exec
       type: test
       params:
         binary: bash
-        env: 
+        env:
           OIDC: oidc
         include_expansions_in_env: [DRIVERS_TOOLS, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN]
         args: [*task-runner, test-oidc]
@@ -396,6 +397,16 @@ functions:
         include_expansions_in_env: ["MONGODB_URI"]
         args: [*task-runner, test-goleak]
 
+  "run oidc k8s test":
+    - command: subprocess.exec
+      type: test
+      params:
+        binary: bash
+        include_expansions_in_env: [AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN, VARIANT, DRIVERS_TOOLS]
+        env:
+          OIDC_ENV: k8s
+        args: [*task-runner, test-oidc-remote]
+
   run-ocsp-server:
     - command: subprocess.exec
       params:
@@ -1658,6 +1669,19 @@ tasks:
           OIDC_ENV: gcp
         args: [*task-runner, test-oidc-remote]
 
+  - name: "oidc-auth-test-k8s"
+    commands:
+    - func: assume-test-secrets-ec2-role
+    - func: "run oidc k8s test"
+      vars:
+        VARIANT: eks
+    - func: "run oidc k8s test"
+      vars:
+        VARIANT: gke
+    - func: "run oidc k8s test"
+      vars:
+        VARIANT: aks
+
   - name: "test-search-index"
     commands:
       - func: "bootstrap-mongo-orchestration"
@@ -1902,14 +1926,17 @@ task_groups:
            - ${DRIVERS_TOOLS}/.evergreen/csfle/azurekms/teardown.sh
       - func: teardown
       - func: handle-test-artifacts
-      
     tasks:
       - testazurekms-task
 
   - name: testoidc_task_group
+    setup_group_can_fail_task: true
+    setup_group_timeout_secs: 1800
+    teardown_task_can_fail_task: true
+    teardown_group_timeout_secs: 180 # 3 minutes (max allowed time)
     setup_group:
       - func: setup-system
-      - func: assume-ec2-role
+      - func: assume-test-secrets-ec2-role
       - command: subprocess.exec
         params:
           binary: bash
@@ -1926,12 +1953,14 @@ task_groups:
             - ${DRIVERS_TOOLS}/.evergreen/auth_oidc/teardown.sh
       - func: teardown
       - func: handle-test-artifacts
-    setup_group_can_fail_task: true
-    setup_group_timeout_secs: 1800
     tasks:
       - oidc-auth-test
 
   - name: testazureoidc_task_group
+    setup_group_can_fail_task: true
+    setup_group_timeout_secs: 1800
+    teardown_task_can_fail_task: true
+    teardown_group_timeout_secs: 180 # 3 minutes (max allowed time)
     setup_group:
       - func: setup-system
       - command: subprocess.exec
@@ -1949,12 +1978,14 @@ task_groups:
             - ${DRIVERS_TOOLS}/.evergreen/auth_oidc/azure/teardown.sh
       - func: teardown
       - func: handle-test-artifacts
-    setup_group_can_fail_task: true
-    setup_group_timeout_secs: 1800
     tasks:
       - oidc-auth-test-azure
 
   - name: testgcpoidc_task_group
+    setup_group_can_fail_task: true
+    setup_group_timeout_secs: 1800
+    teardown_task_can_fail_task: true
+    teardown_group_timeout_secs: 180 # 3 minutes (max allowed time)
     setup_group:
       - func: setup-system
       - command: subprocess.exec
@@ -1972,10 +2003,32 @@ task_groups:
             - ${DRIVERS_TOOLS}/.evergreen/auth_oidc/gcp/teardown.sh
       - func: teardown
       - func: handle-test-artifacts
+    tasks:
+      - oidc-auth-test-gcp
+
+  - name: testk8soidc_task_group
     setup_group_can_fail_task: true
     setup_group_timeout_secs: 1800
+    teardown_task_can_fail_task: true
+    teardown_group_timeout_secs: 180 # 3 minutes (max allowed time)
+    setup_group:
+      - func: setup-system
+      - command: subprocess.exec
+        params:
+          binary: bash
+          include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"]
+          args:
+            - ${DRIVERS_TOOLS}/.evergreen/auth_oidc/k8s/setup.sh
+    teardown_group:
+      - command: subprocess.exec
+        params:
+          binary: bash
+          args:
+            - ${DRIVERS_TOOLS}/.evergreen/auth_oidc/k8s/teardown.sh
+      - func: teardown
+      - func: handle-test-artifacts
     tasks:
-      - oidc-auth-test-gcp
+      - oidc-auth-test-k8s
 
   - name: test-aws-lambda-task-group
     setup_group:
@@ -2322,8 +2375,6 @@ buildvariants:
       GO_DIST: "/opt/golang/go1.22"
     tasks:
       - name: testoidc_task_group
-        batchtime: 20160 # Use a batchtime of 14 days as suggested by the CSFLE test README
       - name: testazureoidc_task_group
-        batchtime: 20160 # Use a batchtime of 14 days as suggested by the CSFLE test README
       - name: testgcpoidc_task_group
-        batchtime: 20160 # Use a batchtime of 14 days as suggested by the CSFLE test README
+      - name: testk8soidc_task_group
diff --git a/etc/run-oidc-remote-test.sh b/etc/run-oidc-remote-test.sh
@@ -31,6 +31,14 @@ elif [ $OIDC_ENV == "gcp" ]; then
     export GCPOIDC_TEST_CMD="PROJECT_DIRECTORY='.' OIDC_ENV=gcp OIDC=oidc ./etc/run-oidc-test.sh ./test"
     bash ${DRIVERS_TOOLS}/.evergreen/auth_oidc/gcp/run-driver-test.sh
 
+elif [ $OIDC_ENV == "k8s" ]; then
+    export K8S_VARIANT=${VARIANT}
+    export K8S_DRIVERS_TAR_FILE=$DRIVERS_TAR_FILE
+    export K8S_TEST_CMD="PROJECT_DIRECTORY='.' OIDC_ENV=k8s OIDC=oidc ./etc/run-oidc-test.sh ./test"
+    bash ${DRIVERS_TOOLS}/.evergreen/auth_oidc/k8s/setup-pod.sh
+    bash ${DRIVERS_TOOLS}/.evergreen/auth_oidc/k8s/run-driver-test.sh
+    bash ${DRIVERS_TOOLS}/.evergreen/auth_oidc/k8s/teardown-pod.sh
+
 else
     echo "Unrecognized OIDC_ENV $OIDC_ENV"
     exit 1
diff --git a/etc/run-oidc-test.sh b/etc/run-oidc-test.sh
@@ -21,6 +21,12 @@ elif [ $OIDC_ENV == "azure" ]; then
 elif [ $OIDC_ENV == "gcp" ]; then
     source ./secrets-export.sh
 
+elif [ $OIDC_ENV == "k8s" ]; then
+    # "run-driver-test.sh" in drivers-evergreen-tools takes care of sourcing
+    # "secrets-export.sh". Nothing to do in this block, but we still need a
+    # command to be syntactically valid, so use no-op command ":".
+    :
+
 else
     echo "Unrecognized OIDC_ENV $OIDC_ENV"
     exit 1
diff --git a/internal/cmd/testoidcauth/main.go b/internal/cmd/testoidcauth/main.go
@@ -13,6 +13,7 @@ import (
 	"os"
 	"path"
 	"reflect"
+	"strings"
 	"sync"
 	"time"
 	"unsafe"
@@ -101,7 +102,7 @@ func main() {
 
 	hasError := false
 	aux := func(test_name string, f func() error) {
-		fmt.Printf("%s...", test_name)
+		fmt.Printf("%s...\n", test_name)
 		err := f()
 		if err != nil {
 			fmt.Println("Test Error: ", err)
@@ -148,6 +149,8 @@ func main() {
 		aux("machine_5_2_azureWithNoUsername", machine52azureWithBadUsername)
 	case "gcp":
 		aux("machine_6_1_gcpWithNoUsername", machine61gcpWithNoUsername)
+	case "k8s":
+		aux("machine_k8s", machinek8s)
 	default:
 		log.Fatal("Unknown OIDC_ENV: ", env)
 	}
@@ -282,7 +285,7 @@ func machine21validCallbackInputs() error {
 		tokenFile := tokenFile("test_user1")
 		accessToken, err := os.ReadFile(tokenFile)
 		if err != nil {
-			fmt.Printf("machine_2_1: failed reading token file: %v", err)
+			return nil, fmt.Errorf("machine_2_1: failed reading token file: %w", err)
 		}
 		return &options.OIDCCredential{
 			AccessToken:  string(accessToken),
@@ -1817,3 +1820,27 @@ func machine61gcpWithNoUsername() error {
 	}
 	return nil
 }
+
+// machinek8s tests the "k8s" Kubernetes OIDC environment. There is no specified
+// prose test for "k8s", so this test simply checks that you can run a "find"
+// with the provided conn string.
+func machinek8s() error {
+	if !strings.Contains(uriSingle, "ENVIRONMENT:k8s") {
+		return fmt.Errorf("expected MONGODB_URI_SINGLE to specify ENVIRONMENT:k8s for Kubernetes test")
+	}
+
+	opts := options.Client().ApplyURI(uriSingle)
+	client, err := mongo.Connect(opts)
+	if err != nil {
+		return fmt.Errorf("machine_k8s: failed connecting client: %v", err)
+	}
+	defer func() { _ = client.Disconnect(context.Background()) }()
+
+	coll := client.Database("test").Collection("test")
+
+	_, err = coll.Find(context.Background(), bson.D{})
+	if err != nil {
+		return fmt.Errorf("machine_k8s: failed executing Find: %v", err)
+	}
+	return nil
+}
diff --git a/mongo/options/clientoptions.go b/mongo/options/clientoptions.go
@@ -593,20 +593,22 @@ func (c *ClientOptions) Validate() error {
 			return fmt.Errorf("cannot set both OIDCMachineCallback and OIDCHumanCallback, only one may be specified")
 		}
 		if c.Auth.OIDCHumanCallback == nil && c.Auth.AuthMechanismProperties[auth.AllowedHostsProp] != "" {
-			return fmt.Errorf("Cannot specify ALLOWED_HOSTS without an OIDCHumanCallback")
+			return fmt.Errorf("cannot specify ALLOWED_HOSTS without an OIDCHumanCallback")
 		}
 		if env, ok := c.Auth.AuthMechanismProperties[auth.EnvironmentProp]; ok {
 			switch env {
 			case auth.GCPEnvironmentValue, auth.AzureEnvironmentValue:
+				if c.Auth.AuthMechanismProperties[auth.ResourceProp] == "" {
+					return fmt.Errorf("%q must be set for the %s %q", auth.ResourceProp, env, auth.EnvironmentProp)
+				}
+				fallthrough
+			case auth.K8SEnvironmentValue:
 				if c.Auth.OIDCMachineCallback != nil {
 					return fmt.Errorf("OIDCMachineCallback cannot be specified with the %s %q", env, auth.EnvironmentProp)
 				}
 				if c.Auth.OIDCHumanCallback != nil {
 					return fmt.Errorf("OIDCHumanCallback cannot be specified with the %s %q", env, auth.EnvironmentProp)
 				}
-				if c.Auth.AuthMechanismProperties[auth.ResourceProp] == "" {
-					return fmt.Errorf("%q must be set for the %s %q", auth.ResourceProp, env, auth.EnvironmentProp)
-				}
 			default:
 				if c.Auth.AuthMechanismProperties[auth.ResourceProp] != "" {
 					return fmt.Errorf("%q must not be set for the %s %q", auth.ResourceProp, env, auth.EnvironmentProp)
diff --git a/mongo/options/clientoptions_test.go b/mongo/options/clientoptions_test.go
@@ -467,107 +467,14 @@ func TestClientOptions(t *testing.T) {
 					OIDCMachineCallback:     emptyCb,
 					AuthMechanismProperties: map[string]string{"ALLOWED_HOSTS": "www.example.com"},
 				}),
-				err: fmt.Errorf("Cannot specify ALLOWED_HOSTS without an OIDCHumanCallback"),
+				err: fmt.Errorf("cannot specify ALLOWED_HOSTS without an OIDCHumanCallback"),
 			},
 			{
 				name: "cannot set OIDCMachineCallback in GCP Environment",
 				opts: Client().SetAuth(Credential{
 					AuthMechanism:           "MONGODB-OIDC",
 					OIDCMachineCallback:     emptyCb,
-					AuthMechanismProperties: map[string]string{"ENVIRONMENT": "gcp"},
-				}),
-				err: fmt.Errorf(`OIDCMachineCallback cannot be specified with the gcp "ENVIRONMENT"`),
-			},
-			{
-				name: "cannot set OIDCMachineCallback in AZURE Environment",
-				opts: Client().SetAuth(Credential{
-					AuthMechanism:           "MONGODB-OIDC",
-					OIDCMachineCallback:     emptyCb,
-					AuthMechanismProperties: map[string]string{"ENVIRONMENT": "azure"},
-				}),
-				err: fmt.Errorf(`OIDCMachineCallback cannot be specified with the azure "ENVIRONMENT"`),
-			},
-			{
-				name: "TOKEN_RESOURCE must be set in GCP Environment",
-				opts: Client().SetAuth(Credential{
-					AuthMechanism:           "MONGODB-OIDC",
-					AuthMechanismProperties: map[string]string{"ENVIRONMENT": "gcp"},
-				}),
-				err: fmt.Errorf(`"TOKEN_RESOURCE" must be set for the gcp "ENVIRONMENT"`),
-			},
-			{
-				name: "TOKEN_RESOURCE must be set in AZURE Environment",
-				opts: Client().SetAuth(Credential{
-					AuthMechanism:           "MONGODB-OIDC",
-					AuthMechanismProperties: map[string]string{"ENVIRONMENT": "azure"},
-				}),
-				err: fmt.Errorf(`"TOKEN_RESOURCE" must be set for the azure "ENVIRONMENT"`),
-			},
-			{
-				name: "TOKEN_RESOURCE must not be set in TEST Environment",
-				opts: Client().SetAuth(Credential{
-					AuthMechanism:           "MONGODB-OIDC",
-					AuthMechanismProperties: map[string]string{"ENVIRONMENT": "test", "TOKEN_RESOURCE": "stuff"},
-				}),
-				err: fmt.Errorf(`"TOKEN_RESOURCE" must not be set for the test "ENVIRONMENT"`),
-			},
-			{
-				name: "TOKEN_RESOURCE must not be set in any other Environment",
-				opts: Client().SetAuth(Credential{
-					AuthMechanism:           "MONGODB-OIDC",
-					AuthMechanismProperties: map[string]string{"ENVIRONMENT": "random env!", "TOKEN_RESOURCE": "stuff"},
-				}),
-				err: fmt.Errorf(`"TOKEN_RESOURCE" must not be set for the random env! "ENVIRONMENT"`),
-			},
-		}
-		for _, tc := range testCases {
-			tc := tc // Capture range variable.
-
-			t.Run(tc.name, func(t *testing.T) {
-				t.Parallel()
-
-				err := tc.opts.Validate()
-				assert.Equal(t, tc.err, err, "want error %v, got error %v", tc.err, err)
-			})
-		}
-	})
-	t.Run("OIDC auth configuration validation", func(t *testing.T) {
-		t.Parallel()
-
-		emptyCb := func(_ context.Context, _ *OIDCArgs) (*OIDCCredential, error) {
-			return nil, nil
-		}
-
-		testCases := []struct {
-			name string
-			opts *ClientOptions
-			err  error
-		}{
-			{
-				name: "password must not be set",
-				opts: Client().SetAuth(Credential{AuthMechanism: "MONGODB-OIDC", Password: "password"}),
-				err:  fmt.Errorf("password must not be set for the MONGODB-OIDC auth mechanism"),
-			},
-			{
-				name: "cannot set both OIDCMachineCallback and OIDCHumanCallback simultaneously",
-				opts: Client().SetAuth(Credential{AuthMechanism: "MONGODB-OIDC",
-					OIDCMachineCallback: emptyCb, OIDCHumanCallback: emptyCb}),
-				err: fmt.Errorf("cannot set both OIDCMachineCallback and OIDCHumanCallback, only one may be specified"),
-			},
-			{
-				name: "cannot set ALLOWED_HOSTS without OIDCHumanCallback",
-				opts: Client().SetAuth(Credential{AuthMechanism: "MONGODB-OIDC",
-					OIDCMachineCallback:     emptyCb,
-					AuthMechanismProperties: map[string]string{"ALLOWED_HOSTS": "www.example.com"},
-				}),
-				err: fmt.Errorf("Cannot specify ALLOWED_HOSTS without an OIDCHumanCallback"),
-			},
-			{
-				name: "cannot set OIDCMachineCallback in GCP Environment",
-				opts: Client().SetAuth(Credential{
-					AuthMechanism:           "MONGODB-OIDC",
-					OIDCMachineCallback:     emptyCb,
-					AuthMechanismProperties: map[string]string{"ENVIRONMENT": "gcp"},
+					AuthMechanismProperties: map[string]string{"ENVIRONMENT": "gcp", "TOKEN_RESOURCE": "stuff"},
 				}),
 				err: fmt.Errorf(`OIDCMachineCallback cannot be specified with the gcp "ENVIRONMENT"`),
 			},
@@ -576,7 +483,7 @@ func TestClientOptions(t *testing.T) {
 				opts: Client().SetAuth(Credential{
 					AuthMechanism:           "MONGODB-OIDC",
 					OIDCMachineCallback:     emptyCb,
-					AuthMechanismProperties: map[string]string{"ENVIRONMENT": "azure"},
+					AuthMechanismProperties: map[string]string{"ENVIRONMENT": "azure", "TOKEN_RESOURCE": "stuff"},
 				}),
 				err: fmt.Errorf(`OIDCMachineCallback cannot be specified with the azure "ENVIRONMENT"`),
 			},
diff --git a/testdata/auth/connection-string.json b/testdata/auth/connection-string.json
diff --git a/testdata/auth/connection-string.yml b/testdata/auth/connection-string.yml
diff --git a/x/mongo/driver/auth/oidc.go b/x/mongo/driver/auth/oidc.go
