fix: prevent data races in concurrent token introspection

- Clone result templates in gRPC introspection to prevent shared state across concurrent calls
- Clone singleflight results before returning to prevent caller modifications affecting concurrent requests
- Add comprehensive concurrency tests covering gRPC introspection, cache paths, and singleflight scenarios

This ensures each introspection request receives an isolated result instance, preventing data races when multiple goroutines perform concurrent introspections with custom result templates.

Author: vasayxtx

idptoken/grpc_client.go

@@ -165,10 +165,17 @@ func (c *GRPCClient) IntrospectToken(
 		c.setSessionID(sessionIDMeta[0])
 	}
 
-	result := c.resultTemplate
-	if result == nil {
+	// Create a new result instance for this introspection request.
+	// If a custom result template is configured, clone it to get a fresh instance.
+	// This is critical for thread-safety: without cloning, concurrent gRPC introspection
+	// calls would share the same resultTemplate instance, leading to data races.
+	var result IntrospectionResult
+	if c.resultTemplate != nil {
+		result = c.resultTemplate.Clone()
+	} else {
 		result = &DefaultIntrospectionResult{}
 	}
+
 	result.SetIsActive(resp.GetActive())
 	if !result.IsActive() {
 		return result, nil

idptoken/grpc_client_test.go

@@ -348,6 +348,168 @@ func TestGRPCClient_IntrospectToken_CustomClaims(t *gotesting.T) {
 	require.Equal(t, 42, clonedCustomResult.CustomIntField)
 }
 
+// TestGRPCClient_IntrospectToken_ConcurrentCloning tests that concurrent
+// introspection calls don't cause data races with result template cloning.
+func TestGRPCClient_IntrospectToken_ConcurrentCloning(t *gotesting.T) {
+	const validAccessToken = "access-token-with-introspection-permission"
+	const numGoroutines = 100
+
+	opaqueToken := "opaque-token-" + uuid.NewString()
+	opaqueTokenScope := []jwt.AccessPolicy{{
+		TenantUUID:        uuid.NewString(),
+		ResourceNamespace: "account-server",
+		Role:              "admin",
+		ResourcePath:      "resource-" + uuid.NewString(),
+	}}
+	opaqueTokenRegClaims := jwtgo.RegisteredClaims{
+		ExpiresAt: jwtgo.NewNumericDate(time.Now().Add(time.Hour)),
+	}
+
+	jwtScopeToGRPC := func(jwtScope []jwt.AccessPolicy) []*pb.AccessTokenScope {
+		grpcScope := make([]*pb.AccessTokenScope, len(jwtScope))
+		for i, scope := range jwtScope {
+			grpcScope[i] = &pb.AccessTokenScope{
+				TenantUuid:        scope.TenantUUID,
+				ResourceNamespace: scope.ResourceNamespace,
+				RoleName:          scope.Role,
+				ResourcePath:      scope.ResourcePath,
+			}
+		}
+		return grpcScope
+	}
+
+	t.Run("custom result template", func(t *gotesting.T) {
+		grpcServerTokenIntrospector := testing.NewGRPCServerTokenIntrospectorMock()
+		grpcServerTokenIntrospector.SetAccessTokenForIntrospection(validAccessToken)
+		grpcServerTokenIntrospector.SetResultForToken(opaqueToken, &pb.IntrospectTokenResponse{
+			Active:           true,
+			TokenType:        idputil.TokenTypeBearer,
+			Aud:              opaqueTokenRegClaims.Audience,
+			Exp:              opaqueTokenRegClaims.ExpiresAt.Unix(),
+			Scope:            jwtScopeToGRPC(opaqueTokenScope),
+			CustomClaimsJson: `{"custom_string_field":"custom_value","custom_int_field":42}`,
+		}, nil)
+
+		grpcIDPSrv := idptest.NewGRPCServer(idptest.WithGRPCTokenIntrospector(grpcServerTokenIntrospector))
+		require.NoError(t, grpcIDPSrv.StartAndWaitForReady(time.Second))
+		defer func() { grpcIDPSrv.GracefulStop() }()
+
+		grpcClient, err := idptoken.NewGRPCClientWithOpts(
+			grpcIDPSrv.Addr(),
+			insecure.NewCredentials(),
+			idptoken.GRPCClientOpts{
+				ResultTemplate: &CustomIntrospectionResult{},
+			},
+		)
+		require.NoError(t, err)
+		defer func() { require.NoError(t, grpcClient.Close()) }()
+
+		// Run concurrent introspection requests
+		errChan := make(chan error, numGoroutines)
+		startCh := make(chan struct{})
+		for i := 0; i < numGoroutines; i++ {
+			go func(index int) {
+				<-startCh // Wait for signal to start all goroutines simultaneously
+				result, introspectErr := grpcClient.IntrospectToken(context.Background(), opaqueToken, nil, validAccessToken)
+				if introspectErr != nil {
+					errChan <- introspectErr
+					return
+				}
+
+				customResult, ok := result.(*CustomIntrospectionResult)
+				if !ok {
+					errChan <- fmt.Errorf("goroutine %d: result should be of type CustomIntrospectionResult", index)
+					return
+				}
+
+				if customResult.CustomStringField != "custom_value" {
+					errChan <- fmt.Errorf("goroutine %d: expected CustomStringField='custom_value', got '%s'",
+						index, customResult.CustomStringField)
+					return
+				}
+
+				if customResult.CustomIntField != 42 {
+					errChan <- fmt.Errorf("goroutine %d: expected CustomIntField=42, got %d",
+						index, customResult.CustomIntField)
+					return
+				}
+
+				// Modify the result - this would cause a race if the same object is shared
+				customResult.CustomStringField = fmt.Sprintf("modified_by_goroutine_%d", index)
+				customResult.CustomIntField = index
+
+				errChan <- nil
+			}(i)
+		}
+
+		close(startCh) // Start all goroutines at once
+
+		// Collect results
+		for i := 0; i < numGoroutines; i++ {
+			err := <-errChan
+			require.NoError(t, err)
+		}
+	})
+
+	t.Run("default result template", func(t *gotesting.T) {
+		grpcServerTokenIntrospector := testing.NewGRPCServerTokenIntrospectorMock()
+		grpcServerTokenIntrospector.SetAccessTokenForIntrospection(validAccessToken)
+		grpcServerTokenIntrospector.SetResultForToken(opaqueToken, &pb.IntrospectTokenResponse{
+			Active:    true,
+			TokenType: idputil.TokenTypeBearer,
+			Aud:       opaqueTokenRegClaims.Audience,
+			Exp:       opaqueTokenRegClaims.ExpiresAt.Unix(),
+			Scope:     jwtScopeToGRPC(opaqueTokenScope),
+		}, nil)
+
+		grpcIDPSrv := idptest.NewGRPCServer(idptest.WithGRPCTokenIntrospector(grpcServerTokenIntrospector))
+		require.NoError(t, grpcIDPSrv.StartAndWaitForReady(time.Second))
+		defer func() { grpcIDPSrv.GracefulStop() }()
+
+		grpcClient, err := idptoken.NewGRPCClient(grpcIDPSrv.Addr(), insecure.NewCredentials())
+		require.NoError(t, err)
+		defer func() { require.NoError(t, grpcClient.Close()) }()
+
+		// Run concurrent introspection requests
+		errChan := make(chan error, numGoroutines)
+		startCh := make(chan struct{})
+		for i := 0; i < numGoroutines; i++ {
+			go func(index int) {
+				<-startCh // Wait for signal to start all goroutines simultaneously
+				result, introspectErr := grpcClient.IntrospectToken(context.Background(), opaqueToken, nil, validAccessToken)
+				if introspectErr != nil {
+					errChan <- introspectErr
+					return
+				}
+
+				if !result.IsActive() {
+					errChan <- fmt.Errorf("goroutine %d: expected active token", index)
+					return
+				}
+
+				if result.GetTokenType() != idputil.TokenTypeBearer {
+					errChan <- fmt.Errorf("goroutine %d: expected token type %s, got %s",
+						index, idputil.TokenTypeBearer, result.GetTokenType())
+					return
+				}
+
+				// Modify the result - this would cause a race if the same object is shared
+				result.SetTokenType(fmt.Sprintf("modified_by_goroutine_%d", index))
+
+				errChan <- nil
+			}(i)
+		}
+
+		close(startCh) // Start all goroutines at once
+
+		// Collect results
+		for i := 0; i < numGoroutines; i++ {
+			err := <-errChan
+			require.NoError(t, err)
+		}
+	})
+}
+
 func TestGRPCClient_ExchangeToken(t *gotesting.T) {
 	tokenExpiresIn := time.Hour
 	tokenExpiresAt := time.Now().Add(time.Hour)

idptoken/introspector.go

@@ -369,10 +369,14 @@ func (i *Introspector) IntrospectToken(ctx context.Context, token string) (Intro
 		if err := i.validateClaims(cachedItem.IntrospectionResult.GetClaims()); err != nil {
 			return nil, fmt.Errorf("validate claims of cached introspection result: %w", err)
 		}
+		// Clone the cached result to prevent data races. Without cloning, concurrent calls
+		// would receive the same cached instance, and modifications by one goroutine could
+		// affect other goroutines reading the same object.
 		return cachedItem.IntrospectionResult.Clone(), nil
 	}
 
 	if cachedItem, ok := i.NegativeCache.Get(ctx, cacheKey); ok {
+		// Clone the cached result for thread-safety (same reason as above).
 		return cachedItem.IntrospectionResult.Clone(), nil
 	}
 
@@ -382,11 +386,11 @@ func (i *Introspector) IntrospectToken(ctx context.Context, token string) (Intro
 			return nil, err
 		}
 		if !result.IsActive() {
-			i.NegativeCache.Add(ctx, cacheKey, IntrospectionCacheItem{IntrospectionResult: result.Clone(), CreatedAt: time.Now()})
+			i.NegativeCache.Add(ctx, cacheKey, IntrospectionCacheItem{IntrospectionResult: result, CreatedAt: time.Now()})
 			return result, nil
 		}
 		result.GetClaims().ApplyScopeFilter(i.scopeFilter)
-		i.ClaimsCache.Add(ctx, cacheKey, IntrospectionCacheItem{IntrospectionResult: result.Clone(), CreatedAt: time.Now()})
+		i.ClaimsCache.Add(ctx, cacheKey, IntrospectionCacheItem{IntrospectionResult: result, CreatedAt: time.Now()})
 		if err = i.validateClaims(result.GetClaims()); err != nil {
 			return nil, fmt.Errorf("validate claims of received introspection result: %w", err)
 		}
@@ -395,7 +399,10 @@ func (i *Introspector) IntrospectToken(ctx context.Context, token string) (Intro
 	if err != nil {
 		return nil, err // already wrapped
 	}
-	return resultVal.(IntrospectionResult), nil
+	// Clone the result before returning to ensure thread-safety.
+	// This prevents the caller from modifying the singleflight-shared result object,
+	// which could affect other concurrent callers waiting for the same singleflight operation.
+	return resultVal.(IntrospectionResult).Clone(), nil
 }
 
 // AddTrustedIssuer adds trusted issuer with specified name and URL.
@@ -633,6 +640,10 @@ func (i *Introspector) makeIntrospectFuncHTTP(introspectionEndpointURL string) i
 			return nil, unexpectedCodeErr
 		}
 
+		// Create a new result instance for this HTTP introspection request.
+		// If a custom result template is configured, clone it to get a fresh instance.
+		// This is critical for thread-safety: without cloning, concurrent HTTP introspection
+		// calls would share the same resultTemplate instance, leading to data races.
 		var res IntrospectionResult
 		if i.resultTemplate != nil {
 			res = i.resultTemplate.Clone()