Skip to content

Commit 674af1d

Browse files
committed
fix(auth): stabilize sandbox token bootstrap
Serialize the supervisor's first sandbox JWT acquisition so concurrent startup clients reuse the process-wide token slot instead of racing into duplicate K8s ServiceAccount bootstrap exchanges. Set XDG_STATE_HOME inside Docker e2e's host-visible workdir. GitHub Actions container jobs talk to the host Docker daemon, so driver-owned sandbox JWT bind mounts must resolve from a path visible on both sides. Verification: mise run pre-commit; OPENSHELL_E2E_DOCKER_TEST=bypass_detection OPENSHELL_SUPERVISOR_IMAGE=openshell/supervisor:dev-25-g15a2a59fb-dirty e2e/rust/e2e-docker.sh; local helm dev deploy plus sandbox log check showed one K8s token exchange line.
1 parent 15a2a59 commit 674af1d

2 files changed

Lines changed: 30 additions & 10 deletions

File tree

crates/openshell-sandbox/src/grpc_client.rs

Lines changed: 25 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,11 @@ static TOKEN_SLOT: OnceLock<TokenSlot> = OnceLock::new();
6666
/// Source used to acquire the process-wide token slot.
6767
static TOKEN_SOURCE: OnceLock<TokenSource> = OnceLock::new();
6868

69+
/// Serializes the first token acquisition. Several supervisor subsystems
70+
/// connect during startup; without this guard they can all observe an empty
71+
/// [`TOKEN_SLOT`] and perform duplicate K8s bootstrap exchanges.
72+
static TOKEN_INIT_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(());
73+
6974
/// One-shot guard so the renewal loop spawns at most once per process.
7075
static REFRESH_SPAWNED: OnceLock<()> = OnceLock::new();
7176

@@ -183,15 +188,7 @@ async fn build_plain_channel(endpoint: &str) -> Result<Channel> {
183188
/// spawned once per process via [`REFRESH_SPAWNED`].
184189
async fn connect_channel(endpoint: &str) -> Result<AuthedChannel> {
185190
let channel = build_plain_channel(endpoint).await?;
186-
let (slot, source) = if let Some(existing) = TOKEN_SLOT.get() {
187-
let source = TOKEN_SOURCE.get().copied().unwrap_or(TokenSource::Env);
188-
(existing.clone(), source)
189-
} else {
190-
let acquired = acquire_sandbox_token(endpoint, &channel).await?;
191-
let slot = install_token_slot(&acquired.token)?;
192-
let _ = TOKEN_SOURCE.set(acquired.source);
193-
(slot, acquired.source)
194-
};
191+
let (slot, source) = token_slot(endpoint, &channel).await?;
195192
let plain_channel = channel.clone();
196193
let intercepted = InterceptedService::new(channel, AuthInterceptor::new(slot.clone()));
197194
if REFRESH_SPAWNED.set(()).is_ok() {
@@ -204,6 +201,25 @@ async fn connect_channel(endpoint: &str) -> Result<AuthedChannel> {
204201
Ok(intercepted)
205202
}
206203

204+
async fn token_slot(endpoint: &str, plain_channel: &Channel) -> Result<(TokenSlot, TokenSource)> {
205+
if let Some(existing) = TOKEN_SLOT.get() {
206+
let source = TOKEN_SOURCE.get().copied().unwrap_or(TokenSource::Env);
207+
return Ok((existing.clone(), source));
208+
}
209+
210+
let _guard = TOKEN_INIT_LOCK.lock().await;
211+
212+
if let Some(existing) = TOKEN_SLOT.get() {
213+
let source = TOKEN_SOURCE.get().copied().unwrap_or(TokenSource::Env);
214+
return Ok((existing.clone(), source));
215+
}
216+
217+
let acquired = acquire_sandbox_token(endpoint, plain_channel).await?;
218+
let slot = install_token_slot(&acquired.token)?;
219+
let _ = TOKEN_SOURCE.set(acquired.source);
220+
Ok((slot, acquired.source))
221+
}
222+
207223
/// Resolve the sandbox JWT used to authenticate every outbound RPC.
208224
///
209225
/// `endpoint` is logged on errors but never used for transport here; the

e2e/with-docker-gateway.sh

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,10 @@ DOCKER_SUPERVISOR_ARGS=()
7878
# Isolate CLI/SDK gateway metadata from the developer's real config.
7979
export XDG_CONFIG_HOME="${WORKDIR}/config"
8080
export XDG_DATA_HOME="${WORKDIR}/data"
81+
# Docker e2e runs in a GitHub Actions container while talking to the host
82+
# Docker daemon. Keep gateway state in the host-visible workdir so driver-owned
83+
# bind mounts, including sandbox JWT files, resolve on both sides.
84+
export XDG_STATE_HOME="${WORKDIR}/state"
8185

8286
cleanup() {
8387
local exit_code=$?
@@ -394,7 +398,7 @@ PKI_DIR="${WORKDIR}/pki"
394398
e2e_generate_pki "${GATEWAY_BIN}" "${PKI_DIR}"
395399

396400
HOST_PORT=$(e2e_pick_port)
397-
STATE_DIR="${WORKDIR}/state"
401+
STATE_DIR="${XDG_STATE_HOME}"
398402
mkdir -p "${STATE_DIR}"
399403
JWT_DIR="${STATE_DIR}/jwt"
400404

0 commit comments

Comments
 (0)