diff --git a/cmd/sharded-test-server/main.go b/cmd/sharded-test-server/main.go index e76e087be90..a3bcd3ca67a 100644 --- a/cmd/sharded-test-server/main.go +++ b/cmd/sharded-test-server/main.go @@ -231,7 +231,26 @@ func start(proxyFlags, shardFlags []string, logDirPath, workDirPath string, numb shards[i] = shard } - // Start virtual-workspace servers + // Wait for shards to be ready before starting virtual workspaces. + // Virtual workspaces depend on shards for token authentication (TokenReview), + // so shards must be fully ready before VWs can accept authenticated requests. + shardsErrCh := make(chan indexErrTuple) + for i, s := range shards { + terminatedCh, err := s.WaitForReady(ctx) + if err != nil { + return err + } + err = testshard.ScrapeMetrics(ctx, s, workDirPath) + if err != nil { + return err + } + go func(i int, terminatedCh <-chan error) { + err := <-terminatedCh + shardsErrCh <- indexErrTuple{i, err} + }(i, terminatedCh) + } + + // Start virtual-workspace servers after shards are ready vwPort := "6444" var virtualWorkspaces []*VirtualWorkspace if standaloneVW { @@ -250,23 +269,6 @@ func start(proxyFlags, shardFlags []string, logDirPath, workDirPath string, numb } } - // Wait for shards to be ready - shardsErrCh := make(chan indexErrTuple) - for i, s := range shards { - terminatedCh, err := s.WaitForReady(ctx) - if err != nil { - return err - } - err = testshard.ScrapeMetrics(ctx, s, workDirPath) - if err != nil { - return err - } - go func(i int, terminatedCh <-chan error) { - err := <-terminatedCh - shardsErrCh <- indexErrTuple{i, err} - }(i, terminatedCh) - } - // Wait for virtual workspaces to be ready virtualWorkspacesErrCh := make(chan indexErrTuple) if standaloneVW { diff --git a/go.mod b/go.mod index 48496fcde61..9e998035a31 100644 --- a/go.mod +++ b/go.mod @@ -52,38 +52,38 @@ require ( ) replace ( - k8s.io/api => github.com/kcp-dev/kubernetes/staging/src/k8s.io/api v0.0.0-20251216144411-4b3495fdcb9d - k8s.io/apiextensions-apiserver => github.com/kcp-dev/kubernetes/staging/src/k8s.io/apiextensions-apiserver v0.0.0-20251216144411-4b3495fdcb9d - k8s.io/apimachinery => github.com/kcp-dev/kubernetes/staging/src/k8s.io/apimachinery v0.0.0-20251216144411-4b3495fdcb9d - k8s.io/apiserver => github.com/kcp-dev/kubernetes/staging/src/k8s.io/apiserver v0.0.0-20251216144411-4b3495fdcb9d - k8s.io/cli-runtime => github.com/kcp-dev/kubernetes/staging/src/k8s.io/cli-runtime v0.0.0-20251216144411-4b3495fdcb9d - k8s.io/client-go => github.com/kcp-dev/kubernetes/staging/src/k8s.io/client-go v0.0.0-20251216144411-4b3495fdcb9d - k8s.io/cloud-provider => github.com/kcp-dev/kubernetes/staging/src/k8s.io/cloud-provider v0.0.0-20251216144411-4b3495fdcb9d - k8s.io/cluster-bootstrap => github.com/kcp-dev/kubernetes/staging/src/k8s.io/cluster-bootstrap v0.0.0-20251216144411-4b3495fdcb9d - k8s.io/code-generator => github.com/kcp-dev/kubernetes/staging/src/k8s.io/code-generator v0.0.0-20251216144411-4b3495fdcb9d - k8s.io/component-base => github.com/kcp-dev/kubernetes/staging/src/k8s.io/component-base v0.0.0-20251216144411-4b3495fdcb9d - k8s.io/component-helpers => github.com/kcp-dev/kubernetes/staging/src/k8s.io/component-helpers v0.0.0-20251216144411-4b3495fdcb9d - k8s.io/controller-manager => github.com/kcp-dev/kubernetes/staging/src/k8s.io/controller-manager v0.0.0-20251216144411-4b3495fdcb9d - k8s.io/cri-api => github.com/kcp-dev/kubernetes/staging/src/k8s.io/cri-api v0.0.0-20251216144411-4b3495fdcb9d - k8s.io/cri-client => github.com/kcp-dev/kubernetes/staging/src/k8s.io/cri-client v0.0.0-20251216144411-4b3495fdcb9d - k8s.io/csi-translation-lib => github.com/kcp-dev/kubernetes/staging/src/k8s.io/csi-translation-lib v0.0.0-20251216144411-4b3495fdcb9d - k8s.io/dynamic-resource-allocation => github.com/kcp-dev/kubernetes/staging/src/k8s.io/dynamic-resource-allocation v0.0.0-20251216144411-4b3495fdcb9d - k8s.io/endpointslice => github.com/kcp-dev/kubernetes/staging/src/k8s.io/endpointslice v0.0.0-20251216144411-4b3495fdcb9d - k8s.io/externaljwt => github.com/kcp-dev/kubernetes/staging/src/k8s.io/externaljwt v0.0.0-20251216144411-4b3495fdcb9d - k8s.io/kms => github.com/kcp-dev/kubernetes/staging/src/k8s.io/kms v0.0.0-20251216144411-4b3495fdcb9d - k8s.io/kube-aggregator => github.com/kcp-dev/kubernetes/staging/src/k8s.io/kube-aggregator v0.0.0-20251216144411-4b3495fdcb9d - k8s.io/kube-controller-manager => github.com/kcp-dev/kubernetes/staging/src/k8s.io/kube-controller-manager v0.0.0-20251216144411-4b3495fdcb9d - k8s.io/kube-proxy => github.com/kcp-dev/kubernetes/staging/src/k8s.io/kube-proxy v0.0.0-20251216144411-4b3495fdcb9d - k8s.io/kube-scheduler => github.com/kcp-dev/kubernetes/staging/src/k8s.io/kube-scheduler v0.0.0-20251216144411-4b3495fdcb9d - k8s.io/kubectl => github.com/kcp-dev/kubernetes/staging/src/k8s.io/kubectl v0.0.0-20251216144411-4b3495fdcb9d - k8s.io/kubelet => github.com/kcp-dev/kubernetes/staging/src/k8s.io/kubelet v0.0.0-20251216144411-4b3495fdcb9d - k8s.io/kubernetes => github.com/kcp-dev/kubernetes v0.0.0-20251216144411-4b3495fdcb9d - k8s.io/metrics => github.com/kcp-dev/kubernetes/staging/src/k8s.io/metrics v0.0.0-20251216144411-4b3495fdcb9d - k8s.io/mount-utils => github.com/kcp-dev/kubernetes/staging/src/k8s.io/mount-utils v0.0.0-20251216144411-4b3495fdcb9d - k8s.io/pod-security-admission => github.com/kcp-dev/kubernetes/staging/src/k8s.io/pod-security-admission v0.0.0-20251216144411-4b3495fdcb9d - k8s.io/sample-apiserver => github.com/kcp-dev/kubernetes/staging/src/k8s.io/sample-apiserver v0.0.0-20251216144411-4b3495fdcb9d - k8s.io/sample-cli-plugin => github.com/kcp-dev/kubernetes/staging/src/k8s.io/sample-cli-plugin v0.0.0-20251216144411-4b3495fdcb9d - k8s.io/sample-controller => github.com/kcp-dev/kubernetes/staging/src/k8s.io/sample-controller v0.0.0-20251216144411-4b3495fdcb9d + k8s.io/api => github.com/kcp-dev/kubernetes/staging/src/k8s.io/api v0.0.0-20260302103047-1393ab398c11 + k8s.io/apiextensions-apiserver => github.com/kcp-dev/kubernetes/staging/src/k8s.io/apiextensions-apiserver v0.0.0-20260302103047-1393ab398c11 + k8s.io/apimachinery => github.com/kcp-dev/kubernetes/staging/src/k8s.io/apimachinery v0.0.0-20260302103047-1393ab398c11 + k8s.io/apiserver => github.com/kcp-dev/kubernetes/staging/src/k8s.io/apiserver v0.0.0-20260302103047-1393ab398c11 + k8s.io/cli-runtime => github.com/kcp-dev/kubernetes/staging/src/k8s.io/cli-runtime v0.0.0-20260302103047-1393ab398c11 + k8s.io/client-go => github.com/kcp-dev/kubernetes/staging/src/k8s.io/client-go v0.0.0-20260302103047-1393ab398c11 + k8s.io/cloud-provider => github.com/kcp-dev/kubernetes/staging/src/k8s.io/cloud-provider v0.0.0-20260302103047-1393ab398c11 + k8s.io/cluster-bootstrap => github.com/kcp-dev/kubernetes/staging/src/k8s.io/cluster-bootstrap v0.0.0-20260302103047-1393ab398c11 + k8s.io/code-generator => github.com/kcp-dev/kubernetes/staging/src/k8s.io/code-generator v0.0.0-20260302103047-1393ab398c11 + k8s.io/component-base => github.com/kcp-dev/kubernetes/staging/src/k8s.io/component-base v0.0.0-20260302103047-1393ab398c11 + k8s.io/component-helpers => github.com/kcp-dev/kubernetes/staging/src/k8s.io/component-helpers v0.0.0-20260302103047-1393ab398c11 + k8s.io/controller-manager => github.com/kcp-dev/kubernetes/staging/src/k8s.io/controller-manager v0.0.0-20260302103047-1393ab398c11 + k8s.io/cri-api => github.com/kcp-dev/kubernetes/staging/src/k8s.io/cri-api v0.0.0-20260302103047-1393ab398c11 + k8s.io/cri-client => github.com/kcp-dev/kubernetes/staging/src/k8s.io/cri-client v0.0.0-20260302103047-1393ab398c11 + k8s.io/csi-translation-lib => github.com/kcp-dev/kubernetes/staging/src/k8s.io/csi-translation-lib v0.0.0-20260302103047-1393ab398c11 + k8s.io/dynamic-resource-allocation => github.com/kcp-dev/kubernetes/staging/src/k8s.io/dynamic-resource-allocation v0.0.0-20260302103047-1393ab398c11 + k8s.io/endpointslice => github.com/kcp-dev/kubernetes/staging/src/k8s.io/endpointslice v0.0.0-20260302103047-1393ab398c11 + k8s.io/externaljwt => github.com/kcp-dev/kubernetes/staging/src/k8s.io/externaljwt v0.0.0-20260302103047-1393ab398c11 + k8s.io/kms => github.com/kcp-dev/kubernetes/staging/src/k8s.io/kms v0.0.0-20260302103047-1393ab398c11 + k8s.io/kube-aggregator => github.com/kcp-dev/kubernetes/staging/src/k8s.io/kube-aggregator v0.0.0-20260302103047-1393ab398c11 + k8s.io/kube-controller-manager => github.com/kcp-dev/kubernetes/staging/src/k8s.io/kube-controller-manager v0.0.0-20260302103047-1393ab398c11 + k8s.io/kube-proxy => github.com/kcp-dev/kubernetes/staging/src/k8s.io/kube-proxy v0.0.0-20260302103047-1393ab398c11 + k8s.io/kube-scheduler => github.com/kcp-dev/kubernetes/staging/src/k8s.io/kube-scheduler v0.0.0-20260302103047-1393ab398c11 + k8s.io/kubectl => github.com/kcp-dev/kubernetes/staging/src/k8s.io/kubectl v0.0.0-20260302103047-1393ab398c11 + k8s.io/kubelet => github.com/kcp-dev/kubernetes/staging/src/k8s.io/kubelet v0.0.0-20260302103047-1393ab398c11 + k8s.io/kubernetes => github.com/kcp-dev/kubernetes v0.0.0-20260302103047-1393ab398c11 + k8s.io/metrics => github.com/kcp-dev/kubernetes/staging/src/k8s.io/metrics v0.0.0-20260302103047-1393ab398c11 + k8s.io/mount-utils => github.com/kcp-dev/kubernetes/staging/src/k8s.io/mount-utils v0.0.0-20260302103047-1393ab398c11 + k8s.io/pod-security-admission => github.com/kcp-dev/kubernetes/staging/src/k8s.io/pod-security-admission v0.0.0-20260302103047-1393ab398c11 + k8s.io/sample-apiserver => github.com/kcp-dev/kubernetes/staging/src/k8s.io/sample-apiserver v0.0.0-20260302103047-1393ab398c11 + k8s.io/sample-cli-plugin => github.com/kcp-dev/kubernetes/staging/src/k8s.io/sample-cli-plugin v0.0.0-20260302103047-1393ab398c11 + k8s.io/sample-controller => github.com/kcp-dev/kubernetes/staging/src/k8s.io/sample-controller v0.0.0-20260302103047-1393ab398c11 ) replace ( diff --git a/go.sum b/go.sum index d7e9c78a835..ecf2b9edc82 100644 --- a/go.sum +++ b/go.sum @@ -117,54 +117,54 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kcp-dev/embeddedetcd v1.1.0 h1:4u5BwZdD43rMnZc3VOpj/VS/+WgJDknv1wuyy4rVkzM= github.com/kcp-dev/embeddedetcd v1.1.0/go.mod h1:KNR9s+3UcXtNamapwLH7M/8ZMZCHpIU5YalOAEjOwJg= -github.com/kcp-dev/kubernetes v0.0.0-20251216144411-4b3495fdcb9d h1:yuX74pYzl7juxEYv8o90uTMX64KEQitb5kiq3XxLh1Y= -github.com/kcp-dev/kubernetes v0.0.0-20251216144411-4b3495fdcb9d/go.mod h1:5abjPkqQJKTbeBJpHD8ssZvASKmoYs0WN2WPyyhkwrw= -github.com/kcp-dev/kubernetes/staging/src/k8s.io/api v0.0.0-20251216144411-4b3495fdcb9d h1:ADbmSRHVYRyJa07ulqPyeCC4dMuuh24N1IjCHdfeeOg= -github.com/kcp-dev/kubernetes/staging/src/k8s.io/api v0.0.0-20251216144411-4b3495fdcb9d/go.mod h1:3Y5V97lz2MrKYzHlUaXejkj+coCmqde9E9WwVFuWRXE= -github.com/kcp-dev/kubernetes/staging/src/k8s.io/apiextensions-apiserver v0.0.0-20251216144411-4b3495fdcb9d h1:8IVhiVfslBHX5XTRocRHn5DisT0peZANdAK+O1oeWXI= -github.com/kcp-dev/kubernetes/staging/src/k8s.io/apiextensions-apiserver v0.0.0-20251216144411-4b3495fdcb9d/go.mod h1:NL2CyapDmJ+5XVVY8qr6niVA3UHVF17kPl0zh6ohkVM= -github.com/kcp-dev/kubernetes/staging/src/k8s.io/apimachinery v0.0.0-20251216144411-4b3495fdcb9d h1:4vqAooOXCKj/Vs6lD5LXVOHxzP7GQfGKwJPMgHo9X9A= -github.com/kcp-dev/kubernetes/staging/src/k8s.io/apimachinery v0.0.0-20251216144411-4b3495fdcb9d/go.mod h1:/GwIlEcWuTX9zKIg2mbw0LRFIsXwrfoVxn+ef0X13lw= -github.com/kcp-dev/kubernetes/staging/src/k8s.io/apiserver v0.0.0-20251216144411-4b3495fdcb9d h1:Sr+/M7xxy9lfVlDbwD1o3lx/cSYujKHyThlCEnX/EVY= -github.com/kcp-dev/kubernetes/staging/src/k8s.io/apiserver v0.0.0-20251216144411-4b3495fdcb9d/go.mod h1:msyjTyI8TyfhYybEkao5LA8bUrVqz1xhic5zxsfejoM= -github.com/kcp-dev/kubernetes/staging/src/k8s.io/client-go v0.0.0-20251216144411-4b3495fdcb9d h1:+DEeqjkVZOv+d640zObWXQS/IwW4tcoAQAcDAI6rfX4= -github.com/kcp-dev/kubernetes/staging/src/k8s.io/client-go v0.0.0-20251216144411-4b3495fdcb9d/go.mod h1:EA6EHLX97x5H59hA02pKPLlZBMQEYnYMsIMglrufpFo= -github.com/kcp-dev/kubernetes/staging/src/k8s.io/cloud-provider v0.0.0-20251216144411-4b3495fdcb9d h1:nlSYadXu0V4UCLwDNCFhgc4hWIn0jD66E3q4EdWalBE= -github.com/kcp-dev/kubernetes/staging/src/k8s.io/cloud-provider v0.0.0-20251216144411-4b3495fdcb9d/go.mod h1:ZZzwSqYu465kx/03+L6Axo9WQxQxiJJuR7kx8i+km6o= -github.com/kcp-dev/kubernetes/staging/src/k8s.io/cluster-bootstrap v0.0.0-20251216144411-4b3495fdcb9d h1:Lax08JrDyD+c981lKaJzbHY2k8kN/QqV512SNbNznOA= -github.com/kcp-dev/kubernetes/staging/src/k8s.io/cluster-bootstrap v0.0.0-20251216144411-4b3495fdcb9d/go.mod h1:L+CgxxDLy//AhiEyqsCdiTs4TphPxXI1IQArc9jwMO8= -github.com/kcp-dev/kubernetes/staging/src/k8s.io/code-generator v0.0.0-20251216144411-4b3495fdcb9d h1:6YJF/LFkbE2fthm2YEhRc2bvzi0aZxvChOMHspRQJSs= -github.com/kcp-dev/kubernetes/staging/src/k8s.io/code-generator v0.0.0-20251216144411-4b3495fdcb9d/go.mod h1:WO4jix1ghA2Qv/tXyNSQl6JXSm6G7jR2AVaOz4w6Z5M= -github.com/kcp-dev/kubernetes/staging/src/k8s.io/component-base v0.0.0-20251216144411-4b3495fdcb9d h1:azNXy4sDXqLvH+t1De6dut0zwMsy45WzuGXURzam1x4= -github.com/kcp-dev/kubernetes/staging/src/k8s.io/component-base v0.0.0-20251216144411-4b3495fdcb9d/go.mod h1:ji6LERznCQ/l4DKb74Vd/5W8Q89b7SvBen7B4AMY0Oo= -github.com/kcp-dev/kubernetes/staging/src/k8s.io/component-helpers v0.0.0-20251216144411-4b3495fdcb9d h1:CWw6cyX74P3e2PMVEjdtYwWz8WP9bWaJvhgirOE9YOM= -github.com/kcp-dev/kubernetes/staging/src/k8s.io/component-helpers v0.0.0-20251216144411-4b3495fdcb9d/go.mod h1:l7hECO13A52pGPFD+SlZJ+EGCuRoVOm0wtdDCMQQkf4= -github.com/kcp-dev/kubernetes/staging/src/k8s.io/controller-manager v0.0.0-20251216144411-4b3495fdcb9d h1:VdmE8ON6Z1Vr0IqNnWGXHESmu1LXzx/GwzQyEOW+mME= -github.com/kcp-dev/kubernetes/staging/src/k8s.io/controller-manager v0.0.0-20251216144411-4b3495fdcb9d/go.mod h1:r87/fGxIfARYiYYbMjR8AIOqf25GMlSSqL/F05+OIzI= -github.com/kcp-dev/kubernetes/staging/src/k8s.io/cri-api v0.0.0-20251216144411-4b3495fdcb9d h1:YjKN1PiXFn3shkrxcH+oIjZ7EBzoXJS7rKFeNBoMy+Q= -github.com/kcp-dev/kubernetes/staging/src/k8s.io/cri-api v0.0.0-20251216144411-4b3495fdcb9d/go.mod h1:4qVUjidMg7/Z9YGZpqIDygbkPWkg3mkS1PvOx/kpHTE= -github.com/kcp-dev/kubernetes/staging/src/k8s.io/cri-client v0.0.0-20251216144411-4b3495fdcb9d h1:hvRoCFg68wLQ06H2eyOwYf+H/FWgMgwFcdS+ki7VLB4= -github.com/kcp-dev/kubernetes/staging/src/k8s.io/cri-client v0.0.0-20251216144411-4b3495fdcb9d/go.mod h1:WVLptW3AzZFX1zSlSxuN9oUO/R6A1+6I4XFZBjhOV4A= -github.com/kcp-dev/kubernetes/staging/src/k8s.io/csi-translation-lib v0.0.0-20251216144411-4b3495fdcb9d h1:tO2J32wp1uxGitM3z5PW1KowCudo5fMyCTXySG/Dxdk= -github.com/kcp-dev/kubernetes/staging/src/k8s.io/csi-translation-lib v0.0.0-20251216144411-4b3495fdcb9d/go.mod h1:yIEfKOC6kDBKmPljSFyWf3OJl6qndPa1Z8CZnvM1zAk= -github.com/kcp-dev/kubernetes/staging/src/k8s.io/dynamic-resource-allocation v0.0.0-20251216144411-4b3495fdcb9d h1:sKN7PrYHnAp8xO8FQVD6gRFbU0DyA+PJJHFuWOa4kuk= -github.com/kcp-dev/kubernetes/staging/src/k8s.io/dynamic-resource-allocation v0.0.0-20251216144411-4b3495fdcb9d/go.mod h1:ScEkH9zs6QurmNvJvTPTbs1BbrUH6PBMrvT1qKee6U4= -github.com/kcp-dev/kubernetes/staging/src/k8s.io/endpointslice v0.0.0-20251216144411-4b3495fdcb9d h1:7djQqCV5Hy/MN2YjcdbFe++ontvwLbr0CKPwqBaP3v4= -github.com/kcp-dev/kubernetes/staging/src/k8s.io/endpointslice v0.0.0-20251216144411-4b3495fdcb9d/go.mod h1:RilV1eQyuARqygtpZNyB+JG59w6kCKMzLeYYQfMqZqM= -github.com/kcp-dev/kubernetes/staging/src/k8s.io/externaljwt v0.0.0-20251216144411-4b3495fdcb9d h1:h0G3tF4WlJIV6sCjsd/uKdEYLxsFUWoJRfyqRDpGw4w= -github.com/kcp-dev/kubernetes/staging/src/k8s.io/externaljwt v0.0.0-20251216144411-4b3495fdcb9d/go.mod h1:LIqFAVwSkcWVlP3c78wxe2VGmgDySxfqX/wwXzVrV/Q= -github.com/kcp-dev/kubernetes/staging/src/k8s.io/kms v0.0.0-20251216144411-4b3495fdcb9d h1:T/dgbsjumuv8u787QEPgxo73mLJMppKCDYinr9oTL9M= -github.com/kcp-dev/kubernetes/staging/src/k8s.io/kms v0.0.0-20251216144411-4b3495fdcb9d/go.mod h1:s1CFkLG7w9eaTYvctOxosx88fl4spqmixnNpys0JAtM= -github.com/kcp-dev/kubernetes/staging/src/k8s.io/kube-aggregator v0.0.0-20251216144411-4b3495fdcb9d h1:MuSqL8tqlEpBQjDnkHIhKxfC6mE7KAVlywXi6hSRUK4= -github.com/kcp-dev/kubernetes/staging/src/k8s.io/kube-aggregator v0.0.0-20251216144411-4b3495fdcb9d/go.mod h1:GOPdnpyxb2xGzTzBC7NOr0rpnWcGH8/pY/tPHX0Ou44= -github.com/kcp-dev/kubernetes/staging/src/k8s.io/kube-controller-manager v0.0.0-20251216144411-4b3495fdcb9d h1:cmim9RUMpSDQATqal63w7+puOsnlT5W7HKh2srqARe0= -github.com/kcp-dev/kubernetes/staging/src/k8s.io/kube-controller-manager v0.0.0-20251216144411-4b3495fdcb9d/go.mod h1:IDd35yynkN5S0lWnaz/xF+/fZSf668aSIVe+GQpR5tI= -github.com/kcp-dev/kubernetes/staging/src/k8s.io/kubelet v0.0.0-20251216144411-4b3495fdcb9d h1:V695w7ZtHCdyM01jl5BU2frkzdJNH5GF2Ce3yM3vLt0= -github.com/kcp-dev/kubernetes/staging/src/k8s.io/kubelet v0.0.0-20251216144411-4b3495fdcb9d/go.mod h1:5xnzJEi0iAetJLsqhsO5yMAnW3yPZ+zs32oh4VAKgc0= -github.com/kcp-dev/kubernetes/staging/src/k8s.io/mount-utils v0.0.0-20251216144411-4b3495fdcb9d h1:IyKGBxh0tjYuDdigR99DTLeDDPw02jkLQ/lVwLd2pkY= -github.com/kcp-dev/kubernetes/staging/src/k8s.io/mount-utils v0.0.0-20251216144411-4b3495fdcb9d/go.mod h1:MIjjYlqJ0ziYQg0MO09kc9S96GIcMkhF/ay9MncF0GA= -github.com/kcp-dev/kubernetes/staging/src/k8s.io/pod-security-admission v0.0.0-20251216144411-4b3495fdcb9d h1:prTuFQawKSFps5jC9k3ckE/hLPqz7iKWBPpZhhndv9Y= -github.com/kcp-dev/kubernetes/staging/src/k8s.io/pod-security-admission v0.0.0-20251216144411-4b3495fdcb9d/go.mod h1:3bwMqCfzs5exVFZydu9eBJkw5UbmNzDvxCYT7JWYVAo= +github.com/kcp-dev/kubernetes v0.0.0-20260302103047-1393ab398c11 h1:6GxhLQ+APQeuFSzqqvzHPewj9ZQhszng7aEI/NwI/WY= +github.com/kcp-dev/kubernetes v0.0.0-20260302103047-1393ab398c11/go.mod h1:5abjPkqQJKTbeBJpHD8ssZvASKmoYs0WN2WPyyhkwrw= +github.com/kcp-dev/kubernetes/staging/src/k8s.io/api v0.0.0-20260302103047-1393ab398c11 h1:AOWlM7cBv/rTjseQhvD4NRr9fCuaq9a5+GWC3yOz+zY= +github.com/kcp-dev/kubernetes/staging/src/k8s.io/api v0.0.0-20260302103047-1393ab398c11/go.mod h1:3Y5V97lz2MrKYzHlUaXejkj+coCmqde9E9WwVFuWRXE= +github.com/kcp-dev/kubernetes/staging/src/k8s.io/apiextensions-apiserver v0.0.0-20260302103047-1393ab398c11 h1:I5/wMs2LC0YKRSLQxxffyKlz4yjKEohqnYP+D0Vcu0U= +github.com/kcp-dev/kubernetes/staging/src/k8s.io/apiextensions-apiserver v0.0.0-20260302103047-1393ab398c11/go.mod h1:NL2CyapDmJ+5XVVY8qr6niVA3UHVF17kPl0zh6ohkVM= +github.com/kcp-dev/kubernetes/staging/src/k8s.io/apimachinery v0.0.0-20260302103047-1393ab398c11 h1:BpIuN+mJXhKEc2kDt6/NkQKCeIdcdMu1ZsO7jWX/mLE= +github.com/kcp-dev/kubernetes/staging/src/k8s.io/apimachinery v0.0.0-20260302103047-1393ab398c11/go.mod h1:/GwIlEcWuTX9zKIg2mbw0LRFIsXwrfoVxn+ef0X13lw= +github.com/kcp-dev/kubernetes/staging/src/k8s.io/apiserver v0.0.0-20260302103047-1393ab398c11 h1:ooK5jWN4TPqW8nrvlAwP7Mh/fb5ll6SXWJF9u9ttD34= +github.com/kcp-dev/kubernetes/staging/src/k8s.io/apiserver v0.0.0-20260302103047-1393ab398c11/go.mod h1:msyjTyI8TyfhYybEkao5LA8bUrVqz1xhic5zxsfejoM= +github.com/kcp-dev/kubernetes/staging/src/k8s.io/client-go v0.0.0-20260302103047-1393ab398c11 h1:NvwrA5KfGlgS0gPxRxGmGrA3X+0NQ2zPhOxFmB21Lnk= +github.com/kcp-dev/kubernetes/staging/src/k8s.io/client-go v0.0.0-20260302103047-1393ab398c11/go.mod h1:EA6EHLX97x5H59hA02pKPLlZBMQEYnYMsIMglrufpFo= +github.com/kcp-dev/kubernetes/staging/src/k8s.io/cloud-provider v0.0.0-20260302103047-1393ab398c11 h1:3ZONHeUU0goyOoCjovF8uGLb58lyV2xc5Ev9ygem+/M= +github.com/kcp-dev/kubernetes/staging/src/k8s.io/cloud-provider v0.0.0-20260302103047-1393ab398c11/go.mod h1:ZZzwSqYu465kx/03+L6Axo9WQxQxiJJuR7kx8i+km6o= +github.com/kcp-dev/kubernetes/staging/src/k8s.io/cluster-bootstrap v0.0.0-20260302103047-1393ab398c11 h1:P2mru3Gg7T6u8OxePl0PbOjEf+ucBGteIOKjLTJwipw= +github.com/kcp-dev/kubernetes/staging/src/k8s.io/cluster-bootstrap v0.0.0-20260302103047-1393ab398c11/go.mod h1:L+CgxxDLy//AhiEyqsCdiTs4TphPxXI1IQArc9jwMO8= +github.com/kcp-dev/kubernetes/staging/src/k8s.io/code-generator v0.0.0-20260302103047-1393ab398c11 h1:U/+YL/+FhEbtcSqswhC9cXkSLr6Ul7VN0D37i829ViY= +github.com/kcp-dev/kubernetes/staging/src/k8s.io/code-generator v0.0.0-20260302103047-1393ab398c11/go.mod h1:WO4jix1ghA2Qv/tXyNSQl6JXSm6G7jR2AVaOz4w6Z5M= +github.com/kcp-dev/kubernetes/staging/src/k8s.io/component-base v0.0.0-20260302103047-1393ab398c11 h1:MORn1qCuEF7mRgV/tsGZX3ApfydVldjU2v47IbOSEQA= +github.com/kcp-dev/kubernetes/staging/src/k8s.io/component-base v0.0.0-20260302103047-1393ab398c11/go.mod h1:ji6LERznCQ/l4DKb74Vd/5W8Q89b7SvBen7B4AMY0Oo= +github.com/kcp-dev/kubernetes/staging/src/k8s.io/component-helpers v0.0.0-20260302103047-1393ab398c11 h1:Ha4M1J0Oo+59YLevM6hLhNCqnTqqVIBDyihR9zRyWCY= +github.com/kcp-dev/kubernetes/staging/src/k8s.io/component-helpers v0.0.0-20260302103047-1393ab398c11/go.mod h1:l7hECO13A52pGPFD+SlZJ+EGCuRoVOm0wtdDCMQQkf4= +github.com/kcp-dev/kubernetes/staging/src/k8s.io/controller-manager v0.0.0-20260302103047-1393ab398c11 h1:3v8owO9eqHBQUczO0Rxh2H9lPyTHVbsB8HnsXmQv4wA= +github.com/kcp-dev/kubernetes/staging/src/k8s.io/controller-manager v0.0.0-20260302103047-1393ab398c11/go.mod h1:r87/fGxIfARYiYYbMjR8AIOqf25GMlSSqL/F05+OIzI= +github.com/kcp-dev/kubernetes/staging/src/k8s.io/cri-api v0.0.0-20260302103047-1393ab398c11 h1:jUUpQx1A73sUMMMAIZm9OXdt0L9z/2FgYF5PTzErJrs= +github.com/kcp-dev/kubernetes/staging/src/k8s.io/cri-api v0.0.0-20260302103047-1393ab398c11/go.mod h1:4qVUjidMg7/Z9YGZpqIDygbkPWkg3mkS1PvOx/kpHTE= +github.com/kcp-dev/kubernetes/staging/src/k8s.io/cri-client v0.0.0-20260302103047-1393ab398c11 h1:OobJlZ4RqQ2iCJZIk96B+faHtLoHhJzdj0eR8vqHSik= +github.com/kcp-dev/kubernetes/staging/src/k8s.io/cri-client v0.0.0-20260302103047-1393ab398c11/go.mod h1:WVLptW3AzZFX1zSlSxuN9oUO/R6A1+6I4XFZBjhOV4A= +github.com/kcp-dev/kubernetes/staging/src/k8s.io/csi-translation-lib v0.0.0-20260302103047-1393ab398c11 h1:MNkhI6yepnd+43miQQj0XgOLn+OBJrnz/g23k3jAuvw= +github.com/kcp-dev/kubernetes/staging/src/k8s.io/csi-translation-lib v0.0.0-20260302103047-1393ab398c11/go.mod h1:yIEfKOC6kDBKmPljSFyWf3OJl6qndPa1Z8CZnvM1zAk= +github.com/kcp-dev/kubernetes/staging/src/k8s.io/dynamic-resource-allocation v0.0.0-20260302103047-1393ab398c11 h1:vnsG2GOdzE7r4e74fn1gf7IdK7pDJE4YU/bppHniMuY= +github.com/kcp-dev/kubernetes/staging/src/k8s.io/dynamic-resource-allocation v0.0.0-20260302103047-1393ab398c11/go.mod h1:ScEkH9zs6QurmNvJvTPTbs1BbrUH6PBMrvT1qKee6U4= +github.com/kcp-dev/kubernetes/staging/src/k8s.io/endpointslice v0.0.0-20260302103047-1393ab398c11 h1:lePOQPC4WiCpY43viOEd+JR9ITO6ARMfKxK/WhANZ/I= +github.com/kcp-dev/kubernetes/staging/src/k8s.io/endpointslice v0.0.0-20260302103047-1393ab398c11/go.mod h1:RilV1eQyuARqygtpZNyB+JG59w6kCKMzLeYYQfMqZqM= +github.com/kcp-dev/kubernetes/staging/src/k8s.io/externaljwt v0.0.0-20260302103047-1393ab398c11 h1:lmLGC7pp+VSVT3MqC8Ru2Jr5Ix2JQfMOZnLmKj+/z0Q= +github.com/kcp-dev/kubernetes/staging/src/k8s.io/externaljwt v0.0.0-20260302103047-1393ab398c11/go.mod h1:LIqFAVwSkcWVlP3c78wxe2VGmgDySxfqX/wwXzVrV/Q= +github.com/kcp-dev/kubernetes/staging/src/k8s.io/kms v0.0.0-20260302103047-1393ab398c11 h1:isok89RLj5BGBQloFdOtbLGGHXxH83TVQNUOgJlwTXU= +github.com/kcp-dev/kubernetes/staging/src/k8s.io/kms v0.0.0-20260302103047-1393ab398c11/go.mod h1:s1CFkLG7w9eaTYvctOxosx88fl4spqmixnNpys0JAtM= +github.com/kcp-dev/kubernetes/staging/src/k8s.io/kube-aggregator v0.0.0-20260302103047-1393ab398c11 h1:fvXDUhGqE6IFHAQWSfP6mp+AR2NzwScT0OQN48Fv/64= +github.com/kcp-dev/kubernetes/staging/src/k8s.io/kube-aggregator v0.0.0-20260302103047-1393ab398c11/go.mod h1:GOPdnpyxb2xGzTzBC7NOr0rpnWcGH8/pY/tPHX0Ou44= +github.com/kcp-dev/kubernetes/staging/src/k8s.io/kube-controller-manager v0.0.0-20260302103047-1393ab398c11 h1:PMGdZOsvGrRx/Z/7F+UTA6S3UwKWqAMqItNTLnu790s= +github.com/kcp-dev/kubernetes/staging/src/k8s.io/kube-controller-manager v0.0.0-20260302103047-1393ab398c11/go.mod h1:IDd35yynkN5S0lWnaz/xF+/fZSf668aSIVe+GQpR5tI= +github.com/kcp-dev/kubernetes/staging/src/k8s.io/kubelet v0.0.0-20260302103047-1393ab398c11 h1:aYMr+w22+MQkBd0h5G/T0+fhzEimlw5P4jEXDHIamQ8= +github.com/kcp-dev/kubernetes/staging/src/k8s.io/kubelet v0.0.0-20260302103047-1393ab398c11/go.mod h1:5xnzJEi0iAetJLsqhsO5yMAnW3yPZ+zs32oh4VAKgc0= +github.com/kcp-dev/kubernetes/staging/src/k8s.io/mount-utils v0.0.0-20260302103047-1393ab398c11 h1:euSf1nraM66nS0nDR9LC2tfsWTVp9u+mJynja43Hdic= +github.com/kcp-dev/kubernetes/staging/src/k8s.io/mount-utils v0.0.0-20260302103047-1393ab398c11/go.mod h1:MIjjYlqJ0ziYQg0MO09kc9S96GIcMkhF/ay9MncF0GA= +github.com/kcp-dev/kubernetes/staging/src/k8s.io/pod-security-admission v0.0.0-20260302103047-1393ab398c11 h1:pwvQmzz+cwV6rt2HonVAgM26ABqVHATfXT4tx1sBp9w= +github.com/kcp-dev/kubernetes/staging/src/k8s.io/pod-security-admission v0.0.0-20260302103047-1393ab398c11/go.mod h1:3bwMqCfzs5exVFZydu9eBJkw5UbmNzDvxCYT7JWYVAo= github.com/kcp-dev/logicalcluster/v3 v3.0.5 h1:JbYakokb+5Uinz09oTXomSUJVQsqfxEvU4RyHUYxHOU= github.com/kcp-dev/logicalcluster/v3 v3.0.5/go.mod h1:EWBUBxdr49fUB1cLMO4nOdBWmYifLbP1LfoL20KkXYY= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= diff --git a/pkg/virtual/apiexport/authorizer/maximal_permission_policy.go b/pkg/virtual/apiexport/authorizer/maximal_permission_policy.go index abab95aef3a..7701b3da8be 100644 --- a/pkg/virtual/apiexport/authorizer/maximal_permission_policy.go +++ b/pkg/virtual/apiexport/authorizer/maximal_permission_policy.go @@ -24,6 +24,7 @@ import ( kerrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apiserver/pkg/authentication/user" "k8s.io/apiserver/pkg/authorization/authorizer" + rbacregistryvalidation "k8s.io/kubernetes/pkg/registry/rbac/validation" kcpkubernetesclientset "github.com/kcp-dev/client-go/kubernetes" "github.com/kcp-dev/logicalcluster/v3" @@ -151,18 +152,23 @@ func getClaimedIdentity(apiExport *apisv1alpha2.APIExport, attr authorizer.Attri } func prefixAttributes(attr authorizer.Attributes) *authorizer.AttributesRecord { - prefixedUser := &user.DefaultInfo{ - Name: apisv1alpha1.MaximalPermissionPolicyRBACUserGroupPrefix + attr.GetUser().GetName(), - UID: attr.GetUser().GetUID(), - Extra: attr.GetUser().GetExtra(), + // Use rbacregistryvalidation.PrefixUser to properly handle ServiceAccount + // rewriting. For SAs with a cluster scope, this rewrites the user name from + // "system:serviceaccount::" to + // "apis.kcp.io:binding:system:kcp:serviceaccount:::" + // which allows RBAC to be set up for cross-cluster SA access. + prefixedUser := rbacregistryvalidation.PrefixUser(attr.GetUser(), apisv1alpha1.MaximalPermissionPolicyRBACUserGroupPrefix) + + // Strip scope-related Extra keys from ServiceAccounts only. The maximal permission + // policy check runs in the workspace where the claimed APIExport lives (e.g., root + // for tenancy.kcp.io), but ServiceAccount tokens are scoped to their originating workspace. + // Without stripping scopes, the deep SAR would be denied due to scope mismatch. + // Regular users don't have this scoping issue, so we preserve their Extra fields. + if rbacregistryvalidation.IsServiceAccount(attr.GetUser()) { + prefixedUser = stripScopesFromUser(prefixedUser) } - prefixedUser.Groups = make([]string, 0, len(attr.GetUser().GetGroups())) - for _, g := range attr.GetUser().GetGroups() { - prefixedUser.Groups = append(prefixedUser.Groups, apisv1alpha1.MaximalPermissionPolicyRBACUserGroupPrefix+g) - } - - return &authorizer.AttributesRecord{ + attr = &authorizer.AttributesRecord{ User: prefixedUser, Verb: attr.GetVerb(), Namespace: attr.GetNamespace(), @@ -174,4 +180,37 @@ func prefixAttributes(attr authorizer.Attributes) *authorizer.AttributesRecord { ResourceRequest: attr.IsResourceRequest(), Path: attr.GetPath(), } + + return attr.(*authorizer.AttributesRecord) +} + +// stripScopesFromUser returns a copy of the user with scope-related Extra keys removed. +func stripScopesFromUser(u user.Info) user.Info { + extra := u.GetExtra() + if extra == nil { + return u + } + + // Check if we need to strip anything + _, hasScopes := extra[rbacregistryvalidation.ScopeExtraKey] + _, hasClusterName := extra["authentication.kcp.io/cluster-name"] + if !hasScopes && !hasClusterName { + return u + } + + // Create new Extra map without scope keys + newExtra := make(map[string][]string, len(extra)) + for k, v := range extra { + if k == rbacregistryvalidation.ScopeExtraKey || k == "authentication.kcp.io/cluster-name" { + continue + } + newExtra[k] = v + } + + return &user.DefaultInfo{ + Name: u.GetName(), + UID: u.GetUID(), + Groups: u.GetGroups(), + Extra: newExtra, + } } diff --git a/test/e2e/apibinding/maximalpermissionpolicy_authorizer_test.go b/test/e2e/apibinding/maximalpermissionpolicy_authorizer_test.go index a5528c33d55..51f829e61ce 100644 --- a/test/e2e/apibinding/maximalpermissionpolicy_authorizer_test.go +++ b/test/e2e/apibinding/maximalpermissionpolicy_authorizer_test.go @@ -25,6 +25,8 @@ import ( "github.com/stretchr/testify/require" "sigs.k8s.io/yaml" + authenticationv1 "k8s.io/api/authentication/v1" + corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -33,6 +35,7 @@ import ( "k8s.io/client-go/discovery/cached/memory" "k8s.io/client-go/rest" "k8s.io/client-go/restmapper" + "k8s.io/utils/ptr" kcpdynamic "github.com/kcp-dev/client-go/dynamic" kcpkubernetesclientset "github.com/kcp-dev/client-go/kubernetes" @@ -40,6 +43,7 @@ import ( apisv1alpha2 "github.com/kcp-dev/sdk/apis/apis/v1alpha2" "github.com/kcp-dev/sdk/apis/core" tenancyv1alpha1 "github.com/kcp-dev/sdk/apis/tenancy/v1alpha1" + "github.com/kcp-dev/sdk/apis/third_party/conditions/util/conditions" kcpclientset "github.com/kcp-dev/sdk/client/clientset/versioned/cluster" kcptesting "github.com/kcp-dev/sdk/testing" kcptestinghelpers "github.com/kcp-dev/sdk/testing/helpers" @@ -512,3 +516,383 @@ func toYAML(t *testing.T, binding interface{}) string { require.NoError(t, err) return string(bs) } + +// TestMaximalPermissionPolicyServiceAccountClaimedTenancyResources tests that a ServiceAccount +// in a provider workspace can access claimed tenancy.kcp.io resources via the APIExport virtual +// workspace. This is a regression test for https://github.com/kcp-dev/kcp/issues/3840. +// +// The issue was that SA tokens are scoped to their originating workspace, but the maximal +// permission policy check runs in a different workspace (root for tenancy.kcp.io). Without +// stripping scope-related Extra fields from the user, the deep SAR would fail due to scope mismatch. +func TestMaximalPermissionPolicyServiceAccountClaimedTenancyResources(t *testing.T) { + t.Parallel() + framework.Suite(t, "control-plane") + + server := kcptesting.SharedKcpServer(t) + + cfg := server.BaseConfig(t) + + kubeClusterClient, err := kcpkubernetesclientset.NewForConfig(cfg) + require.NoError(t, err) + + kcpClusterClient, err := kcpclientset.NewForConfig(cfg) + require.NoError(t, err) + + dynamicClusterClient, err := kcpdynamic.NewForConfig(cfg) + require.NoError(t, err) + + // Create workspaces under root for this test + orgPath, _ := kcptesting.NewWorkspaceFixture(t, server, core.RootCluster.Path(), kcptesting.WithType(core.RootCluster.Path(), "organization")) + providerPath, _ := kcptesting.NewWorkspaceFixture(t, server, orgPath, kcptesting.WithName("provider")) + consumerPath, consumerWorkspace := kcptesting.NewWorkspaceFixture(t, server, orgPath, kcptesting.WithName("consumer")) + + t.Logf("Get the tenancy.kcp.io APIExport identity hash from root") + kcptestinghelpers.EventuallyCondition(t, func() (conditions.Getter, error) { + return kcpClusterClient.Cluster(core.RootCluster.Path()).ApisV1alpha2().APIExports().Get(t.Context(), "tenancy.kcp.io", metav1.GetOptions{}) + }, kcptestinghelpers.Is(apisv1alpha2.APIExportIdentityValid)) + + tenancyAPIExport, err := kcpClusterClient.Cluster(core.RootCluster.Path()).ApisV1alpha2().APIExports().Get(t.Context(), "tenancy.kcp.io", metav1.GetOptions{}) + require.NoError(t, err) + tenancyIdentityHash := tenancyAPIExport.Status.IdentityHash + require.NotEmpty(t, tenancyIdentityHash, "tenancy.kcp.io identity hash should not be empty") + t.Logf("Found tenancy.kcp.io identity hash: %s", tenancyIdentityHash) + + t.Logf("Install cowboys APIResourceSchema in provider workspace %q", providerPath) + mapper := restmapper.NewDeferredDiscoveryRESTMapper(memory.NewMemCacheClient(kcpClusterClient.Cluster(providerPath).Discovery())) + err = helpers.CreateResourceFromFS(t.Context(), dynamicClusterClient.Cluster(providerPath), mapper, nil, "apiresourceschema_cowboys.yaml", testFiles) + require.NoError(t, err) + + t.Logf("Create an APIExport with permissionClaim on tenancy.kcp.io/workspaces in provider workspace %q", providerPath) + apiExport := &apisv1alpha2.APIExport{ + ObjectMeta: metav1.ObjectMeta{ + Name: "wildwest.dev", + }, + Spec: apisv1alpha2.APIExportSpec{ + Resources: []apisv1alpha2.ResourceSchema{ + { + Name: "cowboys", + Group: "wildwest.dev", + Schema: "today.cowboys.wildwest.dev", + Storage: apisv1alpha2.ResourceSchemaStorage{ + CRD: &apisv1alpha2.ResourceSchemaStorageCRD{}, + }, + }, + }, + PermissionClaims: []apisv1alpha2.PermissionClaim{ + { + GroupResource: apisv1alpha2.GroupResource{ + Group: "tenancy.kcp.io", + Resource: "workspaces", + }, + IdentityHash: tenancyIdentityHash, + Verbs: []string{"get", "list", "watch"}, + }, + }, + }, + } + _, err = kcpClusterClient.Cluster(providerPath).ApisV1alpha2().APIExports().Create(t.Context(), apiExport, metav1.CreateOptions{}) + require.NoError(t, err) + + t.Logf("Wait for APIExport to be ready with identity hash") + kcptestinghelpers.EventuallyCondition(t, func() (conditions.Getter, error) { + return kcpClusterClient.Cluster(providerPath).ApisV1alpha2().APIExports().Get(t.Context(), "wildwest.dev", metav1.GetOptions{}) + }, kcptestinghelpers.Is(apisv1alpha2.APIExportIdentityValid)) + + t.Logf("Create namespace and ServiceAccount in provider workspace %q", providerPath) + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "provider-ns", + }, + } + _, err = kubeClusterClient.Cluster(providerPath).CoreV1().Namespaces().Create(t.Context(), ns, metav1.CreateOptions{}) + require.NoError(t, err) + + sa := &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: "provider-sa", + Namespace: "provider-ns", + }, + } + _, err = kubeClusterClient.Cluster(providerPath).CoreV1().ServiceAccounts("provider-ns").Create(t.Context(), sa, metav1.CreateOptions{}) + require.NoError(t, err) + + t.Logf("Grant ServiceAccount apiexports/content access in provider workspace %q", providerPath) + contentRole := &rbacv1.ClusterRole{ + ObjectMeta: metav1.ObjectMeta{ + Name: "provider-sa-apiexport-content", + }, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{"apis.kcp.io"}, + Resources: []string{"apiexports/content"}, + ResourceNames: []string{"wildwest.dev"}, + Verbs: []string{"*"}, + }, + }, + } + _, err = kubeClusterClient.Cluster(providerPath).RbacV1().ClusterRoles().Create(t.Context(), contentRole, metav1.CreateOptions{}) + require.NoError(t, err) + + contentRoleBinding := &rbacv1.ClusterRoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: "provider-sa-apiexport-content", + }, + Subjects: []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: "provider-sa", + Namespace: "provider-ns", + }, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: rbacv1.SchemeGroupVersion.Group, + Kind: "ClusterRole", + Name: "provider-sa-apiexport-content", + }, + } + _, err = kubeClusterClient.Cluster(providerPath).RbacV1().ClusterRoleBindings().Create(t.Context(), contentRoleBinding, metav1.CreateOptions{}) + require.NoError(t, err) + + t.Logf("Grant bind permission on APIExport for consumer binding") + bindRole := &rbacv1.ClusterRole{ + ObjectMeta: metav1.ObjectMeta{ + Name: "apiexport-bind", + }, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{"apis.kcp.io"}, + Resources: []string{"apiexports"}, + ResourceNames: []string{"wildwest.dev"}, + Verbs: []string{"bind"}, + }, + }, + } + _, err = kubeClusterClient.Cluster(providerPath).RbacV1().ClusterRoles().Create(t.Context(), bindRole, metav1.CreateOptions{}) + require.NoError(t, err) + + bindRoleBinding := &rbacv1.ClusterRoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: "apiexport-bind-authenticated", + }, + Subjects: []rbacv1.Subject{ + { + Kind: "Group", + Name: "system:authenticated", + APIGroup: rbacv1.SchemeGroupVersion.Group, + }, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: rbacv1.SchemeGroupVersion.Group, + Kind: "ClusterRole", + Name: "apiexport-bind", + }, + } + _, err = kubeClusterClient.Cluster(providerPath).RbacV1().ClusterRoleBindings().Create(t.Context(), bindRoleBinding, metav1.CreateOptions{}) + require.NoError(t, err) + + t.Logf("Create APIBinding in consumer workspace %q accepting the workspaces claim", consumerPath) + apiBinding := &apisv1alpha2.APIBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: "wildwest", + }, + Spec: apisv1alpha2.APIBindingSpec{ + Reference: apisv1alpha2.BindingReference{ + Export: &apisv1alpha2.ExportBindingReference{ + Path: providerPath.String(), + Name: "wildwest.dev", + }, + }, + PermissionClaims: []apisv1alpha2.AcceptablePermissionClaim{ + { + ScopedPermissionClaim: apisv1alpha2.ScopedPermissionClaim{ + PermissionClaim: apisv1alpha2.PermissionClaim{ + GroupResource: apisv1alpha2.GroupResource{ + Group: "tenancy.kcp.io", + Resource: "workspaces", + }, + IdentityHash: tenancyIdentityHash, + Verbs: []string{"get", "list", "watch"}, + }, + Selector: apisv1alpha2.PermissionClaimSelector{ + MatchAll: true, + }, + }, + State: apisv1alpha2.ClaimAccepted, + }, + }, + }, + } + kcptestinghelpers.Eventually(t, func() (bool, string) { + _, err := kcpClusterClient.Cluster(consumerPath).ApisV1alpha2().APIBindings().Create(t.Context(), apiBinding, metav1.CreateOptions{}) + if err != nil { + return false, fmt.Sprintf("error creating APIBinding: %v", err) + } + return true, "" + }, wait.ForeverTestTimeout, 100*time.Millisecond) + + t.Logf("Wait for APIBinding to be ready") + kcptestinghelpers.EventuallyCondition(t, func() (conditions.Getter, error) { + return kcpClusterClient.Cluster(consumerPath).ApisV1alpha2().APIBindings().Get(t.Context(), "wildwest", metav1.GetOptions{}) + }, kcptestinghelpers.Is(apisv1alpha2.InitialBindingCompleted)) + + t.Logf("Create a token for the ServiceAccount") + tokenReq := &authenticationv1.TokenRequest{ + Spec: authenticationv1.TokenRequestSpec{ + ExpirationSeconds: ptr.To(int64(3600)), + }, + } + tokenResp, err := kubeClusterClient.Cluster(providerPath).CoreV1().ServiceAccounts("provider-ns").CreateToken(t.Context(), "provider-sa", tokenReq, metav1.CreateOptions{}) + require.NoError(t, err) + saToken := tokenResp.Status.Token + require.NotEmpty(t, saToken) + t.Logf("Got SA token") + + t.Logf("Get the APIExport virtual workspace URL") + var apiExportVWURL string + kcptestinghelpers.Eventually(t, func() (bool, string) { + apiExportEndpointSlice, err := kcpClusterClient.Cluster(providerPath).ApisV1alpha1().APIExportEndpointSlices().Get(t.Context(), "wildwest.dev", metav1.GetOptions{}) + if err != nil { + return false, fmt.Sprintf("error getting APIExportEndpointSlice: %v", err) + } + urls := framework.ExportVirtualWorkspaceURLs(apiExportEndpointSlice) + var found bool + apiExportVWURL, found, err = framework.VirtualWorkspaceURL(t.Context(), kcpClusterClient, consumerWorkspace, urls) + if err != nil { + return false, fmt.Sprintf("error getting virtual workspace URL: %v", err) + } + return found, fmt.Sprintf("waiting for virtual workspace URL to be available: %v", urls) + }, wait.ForeverTestTimeout, 100*time.Millisecond) + t.Logf("Got APIExport virtual workspace URL: %s", apiExportVWURL) + + t.Logf("Create a client using the ServiceAccount token to access the virtual workspace") + saConfig := rest.CopyConfig(cfg) + saConfig.Host = apiExportVWURL + saConfig.BearerToken = saToken + // Clear cert-based auth since we're using token + saConfig.CertData = nil + saConfig.KeyData = nil + saConfig.CertFile = "" + saConfig.KeyFile = "" + + saDynamicClient, err := kcpdynamic.NewForConfig(saConfig) + require.NoError(t, err) + + t.Logf("Verify that the ServiceAccount can list workspaces via the virtual workspace (claimed tenancy.kcp.io resource)") + // This is the key test - before the fix, this would return 403 because the SA token + // is scoped to the provider workspace, but the maximal permission policy check runs + // in the root workspace where tenancy.kcp.io lives. + kcptestinghelpers.Eventually(t, func() (bool, string) { + _, err := saDynamicClient.Resource(tenancyv1alpha1.SchemeGroupVersion.WithResource("workspaces")).List(t.Context(), metav1.ListOptions{}) + if err != nil { + return false, fmt.Sprintf("error listing workspaces: %v", err) + } + return true, "" + }, wait.ForeverTestTimeout, 100*time.Millisecond, "ServiceAccount should be able to list claimed workspaces resource") + + t.Logf("Test passed - ServiceAccount can access claimed tenancy.kcp.io resources via virtual workspace") + + // Now test the negative case: create another ServiceAccount WITHOUT apiexports/content permission + // and verify it cannot access the claimed resources + t.Logf("Create another ServiceAccount WITHOUT apiexports/content access") + unauthorizedSA := &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: "unauthorized-sa", + Namespace: "provider-ns", + }, + } + _, err = kubeClusterClient.Cluster(providerPath).CoreV1().ServiceAccounts("provider-ns").Create(t.Context(), unauthorizedSA, metav1.CreateOptions{}) + require.NoError(t, err) + + t.Logf("Create a token for the unauthorized ServiceAccount") + unauthorizedTokenReq := &authenticationv1.TokenRequest{ + Spec: authenticationv1.TokenRequestSpec{ + ExpirationSeconds: ptr.To(int64(3600)), + }, + } + unauthorizedTokenResp, err := kubeClusterClient.Cluster(providerPath).CoreV1().ServiceAccounts("provider-ns").CreateToken(t.Context(), "unauthorized-sa", unauthorizedTokenReq, metav1.CreateOptions{}) + require.NoError(t, err) + unauthorizedToken := unauthorizedTokenResp.Status.Token + require.NotEmpty(t, unauthorizedToken) + + t.Logf("Create a client using the unauthorized ServiceAccount token") + unauthorizedConfig := rest.CopyConfig(cfg) + unauthorizedConfig.Host = apiExportVWURL + unauthorizedConfig.BearerToken = unauthorizedToken + unauthorizedConfig.CertData = nil + unauthorizedConfig.KeyData = nil + unauthorizedConfig.CertFile = "" + unauthorizedConfig.KeyFile = "" + + unauthorizedDynamicClient, err := kcpdynamic.NewForConfig(unauthorizedConfig) + require.NoError(t, err) + + t.Logf("Verify that the unauthorized ServiceAccount CANNOT list workspaces via the virtual workspace") + // Without apiexports/content permission, the SA should get a 403 Forbidden + _, err = unauthorizedDynamicClient.Resource(tenancyv1alpha1.SchemeGroupVersion.WithResource("workspaces")).List(t.Context(), metav1.ListOptions{}) + require.Error(t, err, "unauthorized ServiceAccount should not be able to list workspaces") + require.True(t, apierrors.IsForbidden(err), "expected Forbidden error, got: %v", err) + + t.Logf("Negative test passed - unauthorized ServiceAccount correctly denied access to claimed resources") + + // IMPORTANT: This behaviour is intentional. We might want to change this in the future, but now - no. + // Test with a real user (not a ServiceAccount) to verify they CANNOT access claimed resources + // because real users also have scopes attached when accessing via virtual workspace, and we + // only strip scopes for ServiceAccounts (not regular users). + // This is the expected behavior - regular users should not be able to bypass scope restrictions. + t.Logf("Test with a real user (not ServiceAccount) accessing claimed resources") + + t.Logf("Grant user-1 apiexports/content access in provider workspace %q", providerPath) + userContentRole := &rbacv1.ClusterRole{ + ObjectMeta: metav1.ObjectMeta{ + Name: "user-1-apiexport-content", + }, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{"apis.kcp.io"}, + Resources: []string{"apiexports/content"}, + ResourceNames: []string{"wildwest.dev"}, + Verbs: []string{"*"}, + }, + }, + } + _, err = kubeClusterClient.Cluster(providerPath).RbacV1().ClusterRoles().Create(t.Context(), userContentRole, metav1.CreateOptions{}) + require.NoError(t, err) + + userContentRoleBinding := &rbacv1.ClusterRoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: "user-1-apiexport-content", + }, + Subjects: []rbacv1.Subject{ + { + Kind: "User", + Name: "user-1", + APIGroup: rbacv1.SchemeGroupVersion.Group, + }, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: rbacv1.SchemeGroupVersion.Group, + Kind: "ClusterRole", + Name: "user-1-apiexport-content", + }, + } + _, err = kubeClusterClient.Cluster(providerPath).RbacV1().ClusterRoleBindings().Create(t.Context(), userContentRoleBinding, metav1.CreateOptions{}) + require.NoError(t, err) + + t.Logf("Create a client for user-1 to access the virtual workspace") + user1Config := framework.StaticTokenUserConfig("user-1", rest.CopyConfig(cfg)) + user1Config.Host = apiExportVWURL + + user1DynamicClient, err := kcpdynamic.NewForConfig(user1Config) + require.NoError(t, err) + + t.Logf("Verify that user-1 CANNOT list workspaces via the virtual workspace (claimed tenancy.kcp.io resource)") + // Real users also have scope restrictions when accessing via the virtual workspace. + // Unlike ServiceAccounts, we do NOT strip scopes from regular users, so they cannot + // access claimed resources from APIExports in workspaces they don't have scope on. + // This is the expected and secure behavior. + _, err = user1DynamicClient.Resource(tenancyv1alpha1.SchemeGroupVersion.WithResource("workspaces")).List(t.Context(), metav1.ListOptions{}) + require.Error(t, err, "user-1 should not be able to list workspaces due to scope restrictions") + require.True(t, apierrors.IsForbidden(err), "expected Forbidden error for user-1 due to scope restrictions, got: %v", err) + + t.Logf("Real user test passed - user-1 correctly denied access due to scope restrictions (scopes not stripped for regular users)") +}