fix(auth): validate sandbox pod owner references

Require Kubernetes ServiceAccount bootstrap to validate the live pod's controlling Sandbox ownerReference before minting a gateway sandbox JWT.

The K8s resolver now verifies the pod sandbox-id annotation, the controlling Sandbox CR UID, and the Sandbox CR sandbox-id label in addition to TokenReview and live pod UID checks.

Update gateway architecture and user-facing auth docs to describe the additional Kubernetes bootstrap binding checks.

Tested with focused k8s_sa tests, full pre-commit, and a fresh Helm dev cluster sandbox create.

Author: TaylorMutch

architecture/gateway.md

@@ -56,10 +56,11 @@ Podman, and VM drivers deliver the initial token through supervisor-only
 runtime material; Kubernetes supervisors exchange a projected ServiceAccount
 token through `IssueSandboxToken`. The gateway validates that projected token
 with Kubernetes `TokenReview`, requires the configured sandbox service account,
-checks the returned pod binding against the live pod UID, and reads the pod's
-sandbox annotation before minting the gateway JWT. Supervisors renew gateway
-JWTs in memory before expiry only while the sandbox record still exists. Older
-tokens are not server-revoked; deployments bound replay exposure with short
+checks the returned pod binding against the live pod UID, and verifies the pod's
+controlling `Sandbox` ownerReference against the live Sandbox CR UID and
+sandbox-id label before minting the gateway JWT. Supervisors renew gateway JWTs
+in memory before expiry only while the sandbox record still exists. Older tokens
+are not server-revoked; deployments bound replay exposure with short
 `gateway_jwt.ttl_secs` lifetimes.
 
 Gateway JWT signing-key rotation is currently an offline operator action. The

crates/openshell-driver-kubernetes/src/driver.rs

@@ -1177,9 +1177,10 @@ fn sandbox_template_to_k8s(
     }
     // Carry the sandbox UUID as a pod annotation so the gateway can resolve
     // a projected SA token claim (pod name + uid) back to a sandbox identity
-    // when the supervisor calls `IssueSandboxToken` at startup. The gateway's
-    // K8s Role does NOT grant `patch pods`, so this annotation is
-    // effectively immutable post-create (see plan §11.8).
+    // when the supervisor calls `IssueSandboxToken` at startup. The gateway
+    // also verifies the pod's controlling Sandbox ownerReference against the
+    // live CR before accepting this annotation. Its K8s Role does NOT grant
+    // `patch pods`, so this annotation is effectively immutable post-create.
     let mut pod_annotations = platform_config_struct(template, "annotations")
         .and_then(|v| match v {
             serde_json::Value::Object(map) => Some(map),

crates/openshell-server/src/auth/k8s_sa.rs

@@ -5,10 +5,11 @@
 //!
 //! Path-scoped to `IssueSandboxToken`. Validates a projected SA token
 //! presented by a sandbox pod, reads the pod's `openshell.io/sandbox-id`
-//! annotation, and returns a [`Principal::Sandbox`] with
-//! [`SandboxIdentitySource::K8sServiceAccount`]. The `IssueSandboxToken`
-//! handler then mints a gateway-signed JWT for that sandbox id; subsequent
-//! gRPC calls from the supervisor use the gateway-minted JWT validated by
+//! annotation, verifies the pod is controlled by the corresponding Sandbox CR,
+//! and returns a [`Principal::Sandbox`] with
+//! [`SandboxIdentitySource::K8sServiceAccount`]. The `IssueSandboxToken` handler
+//! then mints a gateway-signed JWT for that sandbox id; subsequent gRPC calls
+//! from the supervisor use the gateway-minted JWT validated by
 //! [`super::sandbox_jwt::SandboxJwtAuthenticator`].
 //!
 //! This is the only authenticator that talks to the K8s apiserver. It is
@@ -22,7 +23,8 @@ use k8s_openapi::api::{
     core::v1::Pod,
 };
 use k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta;
-use kube::api::{Api, PostParams};
+use kube::api::{Api, ApiResource, PostParams};
+use kube::core::{DynamicObject, gvk::GroupVersionKind};
 use std::sync::Arc;
 use tonic::Status;
 use tracing::{debug, info, warn};
@@ -32,10 +34,16 @@ use tracing::{debug, info, warn};
 pub const ISSUE_SANDBOX_TOKEN_PATH: &str = "/openshell.v1.OpenShell/IssueSandboxToken";
 
 /// Pod annotation that binds a sandbox pod to its UUID. Set by the
-/// Kubernetes compute driver at pod-create time. The gateway treats this
-/// annotation as authoritative; the K8s `Role` granted to the gateway must
-/// not include `patch pods` (see plan §11.8).
+/// Kubernetes compute driver at pod-create time. The gateway accepts this
+/// annotation only after validating the pod's `TokenReview` binding, live UID,
+/// and owning Sandbox CR. The K8s `Role` granted to the gateway must not
+/// include `patch pods` (see plan §11.8).
 pub const SANDBOX_ID_ANNOTATION: &str = "openshell.io/sandbox-id";
+const SANDBOX_API_GROUP: &str = "agents.x-k8s.io";
+const SANDBOX_API_VERSION: &str = "v1alpha1";
+const SANDBOX_API_VERSION_FULL: &str = "agents.x-k8s.io/v1alpha1";
+const SANDBOX_KIND: &str = "Sandbox";
+const SANDBOX_ID_LABEL: &str = "openshell.ai/sandbox-id";
 const POD_NAME_EXTRA: &str = "authentication.kubernetes.io/pod-name";
 const POD_UID_EXTRA: &str = "authentication.kubernetes.io/pod-uid";
 
@@ -130,11 +138,18 @@ struct TokenReviewIdentity {
     pod_uid: String,
 }
 
+#[derive(Debug, Clone, PartialEq, Eq)]
+struct SandboxOwnerReference {
+    name: String,
+    uid: String,
+}
+
 /// Resolver backed by the apiserver's `TokenReview` API and `kube::Client`
 /// for the per-pod annotation lookup.
 pub struct LiveK8sResolver {
     token_reviews_api: Api<TokenReview>,
     pods_api: Api<Pod>,
+    sandboxes_api: Api<DynamicObject>,
     expected_audience: String,
     sandbox_namespace: String,
     expected_service_account: String,
@@ -148,10 +163,16 @@ impl LiveK8sResolver {
         expected_service_account: String,
     ) -> Self {
         let token_reviews_api: Api<TokenReview> = Api::all(client.clone());
-        let pods_api: Api<Pod> = Api::namespaced(client, namespace);
+        let pods_api: Api<Pod> = Api::namespaced(client.clone(), namespace);
+        let sandbox_gvk =
+            GroupVersionKind::gvk(SANDBOX_API_GROUP, SANDBOX_API_VERSION, SANDBOX_KIND);
+        let sandbox_resource = ApiResource::from_gvk(&sandbox_gvk);
+        let sandboxes_api: Api<DynamicObject> =
+            Api::namespaced_with(client, namespace, &sandbox_resource);
         Self {
             token_reviews_api,
             pods_api,
+            sandboxes_api,
             expected_audience,
             sandbox_namespace: namespace.to_string(),
             expected_service_account,
@@ -234,13 +255,27 @@ impl K8sIdentityResolver for LiveK8sResolver {
             return Err(Status::permission_denied("SA token pod UID mismatch"));
         }
 
-        let sandbox_id = pod
-            .metadata
-            .annotations
-            .as_ref()
-            .and_then(|a| a.get(SANDBOX_ID_ANNOTATION))
-            .cloned()
-            .unwrap_or_default();
+        let sandbox_id = pod_sandbox_id(&pod)?;
+
+        let owner = sandbox_owner_reference(&pod)?;
+        let sandbox_cr = self.sandboxes_api.get_opt(&owner.name).await.map_err(|e| {
+            warn!(
+                pod = %identity.pod_name,
+                sandbox_owner = %owner.name,
+                error = %e,
+                "failed to fetch owning Sandbox CR for pod identity validation"
+            );
+            Status::internal(format!("sandbox GET failed: {e}"))
+        })?;
+        let Some(sandbox_cr) = sandbox_cr else {
+            warn!(
+                pod = %identity.pod_name,
+                sandbox_owner = %owner.name,
+                "pod ownerReference points to a Sandbox CR that does not exist"
+            );
+            return Err(Status::permission_denied("sandbox owner not found"));
+        };
+        validate_sandbox_owner_reference(&owner, &sandbox_id, &sandbox_cr)?;
 
         Ok(Some(ResolvedK8sIdentity {
             sandbox_id,
@@ -315,6 +350,93 @@ fn user_extra_one(user: &UserInfo, key: &str) -> Result<String, Status> {
     Ok(values[0].clone())
 }
 
+#[allow(clippy::result_large_err)]
+fn pod_sandbox_id(pod: &Pod) -> Result<String, Status> {
+    let sandbox_id = pod
+        .metadata
+        .annotations
+        .as_ref()
+        .and_then(|a| a.get(SANDBOX_ID_ANNOTATION))
+        .cloned()
+        .unwrap_or_default();
+    if sandbox_id.is_empty() {
+        return Err(Status::permission_denied(
+            "pod is not bound to a sandbox identity",
+        ));
+    }
+    Ok(sandbox_id)
+}
+
+#[allow(clippy::result_large_err)]
+fn sandbox_owner_reference(pod: &Pod) -> Result<SandboxOwnerReference, Status> {
+    let owner_refs = pod.metadata.owner_references.as_deref().unwrap_or_default();
+    let mut sandbox_refs = owner_refs.iter().filter(|owner| {
+        owner.api_version == SANDBOX_API_VERSION_FULL && owner.kind == SANDBOX_KIND
+    });
+    let Some(owner) = sandbox_refs.next() else {
+        return Err(Status::permission_denied(
+            "pod is not controlled by an OpenShell Sandbox",
+        ));
+    };
+    if sandbox_refs.next().is_some() {
+        return Err(Status::permission_denied(
+            "pod has multiple OpenShell Sandbox owners",
+        ));
+    }
+    if owner.controller != Some(true) {
+        return Err(Status::permission_denied(
+            "pod Sandbox ownerReference is not controlling",
+        ));
+    }
+    if owner.name.is_empty() || owner.uid.is_empty() {
+        return Err(Status::permission_denied(
+            "pod Sandbox ownerReference is incomplete",
+        ));
+    }
+    Ok(SandboxOwnerReference {
+        name: owner.name.clone(),
+        uid: owner.uid.clone(),
+    })
+}
+
+#[allow(clippy::result_large_err)]
+fn validate_sandbox_owner_reference(
+    owner: &SandboxOwnerReference,
+    sandbox_id: &str,
+    sandbox_cr: &DynamicObject,
+) -> Result<(), Status> {
+    let actual_uid = sandbox_cr.metadata.uid.as_deref().unwrap_or_default();
+    if actual_uid != owner.uid {
+        warn!(
+            sandbox_owner = %owner.name,
+            owner_uid = %owner.uid,
+            actual_uid = %actual_uid,
+            "pod Sandbox ownerReference UID does not match live Sandbox CR"
+        );
+        return Err(Status::permission_denied("sandbox owner UID mismatch"));
+    }
+
+    let actual_sandbox_id = sandbox_cr
+        .metadata
+        .labels
+        .as_ref()
+        .and_then(|labels| labels.get(SANDBOX_ID_LABEL))
+        .map(String::as_str)
+        .unwrap_or_default();
+    if actual_sandbox_id != sandbox_id {
+        warn!(
+            sandbox_owner = %owner.name,
+            owner_uid = %owner.uid,
+            pod_sandbox_id = %sandbox_id,
+            cr_sandbox_id = %actual_sandbox_id,
+            "pod sandbox annotation does not match owning Sandbox CR label"
+        );
+        return Err(Status::permission_denied("sandbox owner ID mismatch"));
+    }
+
+    Ok(())
+}
+
 #[cfg(test)]
 pub mod test_support {
     use super::*;
@@ -352,6 +474,7 @@ pub mod test_support {
 mod tests {
     use super::test_support::FakeResolver;
     use super::*;
+    use k8s_openapi::apimachinery::pkg::apis::meta::v1::OwnerReference;
     use std::collections::BTreeMap;
 
     fn bearer_headers(token: &str) -> http::HeaderMap {
@@ -391,6 +514,52 @@ mod tests {
         }
     }
 
+    fn sandbox_owner(name: &str, uid: &str) -> OwnerReference {
+        OwnerReference {
+            api_version: SANDBOX_API_VERSION_FULL.to_string(),
+            block_owner_deletion: None,
+            controller: Some(true),
+            kind: SANDBOX_KIND.to_string(),
+            name: name.to_string(),
+            uid: uid.to_string(),
+        }
+    }
+
+    fn pod_with_owner_refs(owner_references: Vec<OwnerReference>) -> Pod {
+        Pod {
+            metadata: ObjectMeta {
+                owner_references: Some(owner_references),
+                ..Default::default()
+            },
+            ..Default::default()
+        }
+    }
+
+    fn pod_with_sandbox_id(sandbox_id: Option<&str>) -> Pod {
+        Pod {
+            metadata: ObjectMeta {
+                annotations: sandbox_id.map(|id| {
+                    BTreeMap::from([(SANDBOX_ID_ANNOTATION.to_string(), id.to_string())])
+                }),
+                ..Default::default()
+            },
+            ..Default::default()
+        }
+    }
+
+    fn sandbox_cr(name: &str, uid: &str, sandbox_id: &str) -> DynamicObject {
+        let sandbox_gvk =
+            GroupVersionKind::gvk(SANDBOX_API_GROUP, SANDBOX_API_VERSION, SANDBOX_KIND);
+        let sandbox_resource = ApiResource::from_gvk(&sandbox_gvk);
+        let mut cr = DynamicObject::new(name, &sandbox_resource);
+        cr.metadata.uid = Some(uid.to_string());
+        cr.metadata.labels = Some(BTreeMap::from([(
+            SANDBOX_ID_LABEL.to_string(),
+            sandbox_id.to_string(),
+        )]));
+        cr
+    }
+
     #[test]
     fn token_review_identity_extracts_pod_binding() {
         let status = token_review_status(
@@ -491,6 +660,86 @@ mod tests {
         assert_eq!(err.code(), tonic::Code::PermissionDenied);
     }
 
+    #[test]
+    fn pod_sandbox_id_requires_annotation() {
+        assert_eq!(
+            pod_sandbox_id(&pod_with_sandbox_id(Some("sandbox-id-a"))).unwrap(),
+            "sandbox-id-a"
+        );
+
+        let err = pod_sandbox_id(&pod_with_sandbox_id(None))
+            .expect_err("missing sandbox-id annotation must fail");
+        assert_eq!(err.code(), tonic::Code::PermissionDenied);
+    }
+
+    #[test]
+    fn sandbox_owner_reference_extracts_controlling_sandbox_owner() {
+        let pod = pod_with_owner_refs(vec![sandbox_owner("sandbox-a", "cr-uid-a")]);
+
+        let owner = sandbox_owner_reference(&pod).expect("expected Sandbox owner");
+
+        assert_eq!(
+            owner,
+            SandboxOwnerReference {
+                name: "sandbox-a".to_string(),
+                uid: "cr-uid-a".to_string(),
+            }
+        );
+    }
+
+    #[test]
+    fn sandbox_owner_reference_rejects_missing_owner() {
+        let pod = pod_with_owner_refs(vec![]);
+
+        let err = sandbox_owner_reference(&pod).expect_err("missing owner must fail");
+
+        assert_eq!(err.code(), tonic::Code::PermissionDenied);
+    }
+
+    #[test]
+    fn sandbox_owner_reference_requires_controlling_owner() {
+        let mut owner = sandbox_owner("sandbox-a", "cr-uid-a");
+        owner.controller = Some(false);
+        let pod = pod_with_owner_refs(vec![owner]);
+
+        let err = sandbox_owner_reference(&pod).expect_err("non-controller owner must fail");
+
+        assert_eq!(err.code(), tonic::Code::PermissionDenied);
+    }
+
+    #[test]
+    fn sandbox_owner_reference_rejects_ambiguous_sandbox_owners() {
+        let pod = pod_with_owner_refs(vec![
+            sandbox_owner("sandbox-a", "cr-uid-a"),
+            sandbox_owner("sandbox-b", "cr-uid-b"),
+        ]);
+
+        let err = sandbox_owner_reference(&pod).expect_err("multiple owners must fail");
+
+        assert_eq!(err.code(), tonic::Code::PermissionDenied);
+    }
+
+    #[test]
+    fn validate_sandbox_owner_reference_requires_matching_cr_uid_and_label() {
+        let owner = SandboxOwnerReference {
+            name: "sandbox-a".to_string(),
+            uid: "cr-uid-a".to_string(),
+        };
+        let cr = sandbox_cr("sandbox-a", "cr-uid-a", "sandbox-id-a");
+        validate_sandbox_owner_reference(&owner, "sandbox-id-a", &cr)
+            .expect("matching CR should be accepted");
+
+        let wrong_uid = sandbox_cr("sandbox-a", "cr-uid-b", "sandbox-id-a");
+        let err = validate_sandbox_owner_reference(&owner, "sandbox-id-a", &wrong_uid)
+            .expect_err("wrong CR UID must fail");
+        assert_eq!(err.code(), tonic::Code::PermissionDenied);
+
+        let wrong_label = sandbox_cr("sandbox-a", "cr-uid-a", "sandbox-id-b");
+        let err = validate_sandbox_owner_reference(&owner, "sandbox-id-a", &wrong_label)
+            .expect_err("wrong sandbox-id label must fail");
+        assert_eq!(err.code(), tonic::Code::PermissionDenied);
+    }
+
     #[tokio::test]
     async fn authenticates_on_issue_path_only() {
         let resolved = ResolvedK8sIdentity {