Skip to content

Commit b9ecc50

Browse files
irbekrmirbekrm
authored
cmd/k8s-operator,k8s-operator,kube/kubetypes: add an option to configure app connector via Connector spec (tailscale#13950)
* cmd/k8s-operator,k8s-operator,kube/kubetypes: add an option to configure app connector via Connector spec Updates tailscale#11113 Signed-off-by: Irbe Krumina <[email protected]>
1 parent 6ff8584 commit b9ecc50

File tree

11 files changed

+381
-46
lines changed

11 files changed

+381
-46
lines changed

‎cmd/k8s-operator/connector.go‎

Lines changed: 52 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,8 @@ import (
1313
"sync"
1414
"time"
1515

16-
"github.com/pkg/errors"
16+
"errors"
17+
1718
"go.uber.org/zap"
1819
xslices "golang.org/x/exp/slices"
1920
corev1 "k8s.io/api/core/v1"
@@ -58,6 +59,7 @@ type ConnectorReconciler struct {
5859

5960
subnetRouters set.Slice[types.UID] // for subnet routers gauge
6061
exitNodes set.Slice[types.UID] // for exit nodes gauge
62+
appConnectors set.Slice[types.UID] // for app connectors gauge
6163
}
6264

6365
var (
@@ -67,6 +69,8 @@ var (
6769
gaugeConnectorSubnetRouterResources = clientmetric.NewGauge(kubetypes.MetricConnectorWithSubnetRouterCount)
6870
// gaugeConnectorExitNodeResources tracks the number of Connectors currently managed by this operator instance that are exit nodes.
6971
gaugeConnectorExitNodeResources = clientmetric.NewGauge(kubetypes.MetricConnectorWithExitNodeCount)
72+
// gaugeConnectorAppConnectorResources tracks the number of Connectors currently managed by this operator instance that are app connectors.
73+
gaugeConnectorAppConnectorResources = clientmetric.NewGauge(kubetypes.MetricConnectorWithAppConnectorCount)
7074
)
7175

7276
func (a *ConnectorReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) {
@@ -108,13 +112,12 @@ func (a *ConnectorReconciler) Reconcile(ctx context.Context, req reconcile.Reque
108112
oldCnStatus := cn.Status.DeepCopy()
109113
setStatus := func(cn *tsapi.Connector, _ tsapi.ConditionType, status metav1.ConditionStatus, reason, message string) (reconcile.Result, error) {
110114
tsoperator.SetConnectorCondition(cn, tsapi.ConnectorReady, status, reason, message, cn.Generation, a.clock, logger)
115+
var updateErr error
111116
if !apiequality.Semantic.DeepEqual(oldCnStatus, cn.Status) {
112117
// An error encountered here should get returned by the Reconcile function.
113-
if updateErr := a.Client.Status().Update(ctx, cn); updateErr != nil {
114-
err = errors.Wrap(err, updateErr.Error())
115-
}
118+
updateErr = a.Client.Status().Update(ctx, cn)
116119
}
117-
return res, err
120+
return res, errors.Join(err, updateErr)
118121
}
119122

120123
if !slices.Contains(cn.Finalizers, FinalizerName) {
@@ -150,6 +153,9 @@ func (a *ConnectorReconciler) Reconcile(ctx context.Context, req reconcile.Reque
150153
cn.Status.SubnetRoutes = cn.Spec.SubnetRouter.AdvertiseRoutes.Stringify()
151154
return setStatus(cn, tsapi.ConnectorReady, metav1.ConditionTrue, reasonConnectorCreated, reasonConnectorCreated)
152155
}
156+
if cn.Spec.AppConnector != nil {
157+
cn.Status.IsAppConnector = true
158+
}
153159
cn.Status.SubnetRoutes = ""
154160
return setStatus(cn, tsapi.ConnectorReady, metav1.ConditionTrue, reasonConnectorCreated, reasonConnectorCreated)
155161
}
@@ -189,23 +195,37 @@ func (a *ConnectorReconciler) maybeProvisionConnector(ctx context.Context, logge
189195
sts.Connector.routes = cn.Spec.SubnetRouter.AdvertiseRoutes.Stringify()
190196
}
191197

198+
if cn.Spec.AppConnector != nil {
199+
sts.Connector.isAppConnector = true
200+
if len(cn.Spec.AppConnector.Routes) != 0 {
201+
sts.Connector.routes = cn.Spec.AppConnector.Routes.Stringify()
202+
}
203+
}
204+
192205
a.mu.Lock()
193-
if sts.Connector.isExitNode {
206+
if cn.Spec.ExitNode {
194207
a.exitNodes.Add(cn.UID)
195208
} else {
196209
a.exitNodes.Remove(cn.UID)
197210
}
198-
if sts.Connector.routes != "" {
211+
if cn.Spec.SubnetRouter != nil {
199212
a.subnetRouters.Add(cn.GetUID())
200213
} else {
201214
a.subnetRouters.Remove(cn.GetUID())
202215
}
216+
if cn.Spec.AppConnector != nil {
217+
a.appConnectors.Add(cn.GetUID())
218+
} else {
219+
a.appConnectors.Remove(cn.GetUID())
220+
}
203221
a.mu.Unlock()
204222
gaugeConnectorSubnetRouterResources.Set(int64(a.subnetRouters.Len()))
205223
gaugeConnectorExitNodeResources.Set(int64(a.exitNodes.Len()))
224+
gaugeConnectorAppConnectorResources.Set(int64(a.appConnectors.Len()))
206225
var connectors set.Slice[types.UID]
207226
connectors.AddSlice(a.exitNodes.Slice())
208227
connectors.AddSlice(a.subnetRouters.Slice())
228+
connectors.AddSlice(a.appConnectors.Slice())
209229
gaugeConnectorResources.Set(int64(connectors.Len()))
210230

211231
_, err := a.ssr.Provision(ctx, logger, sts)
@@ -248,12 +268,15 @@ func (a *ConnectorReconciler) maybeCleanupConnector(ctx context.Context, logger
248268
a.mu.Lock()
249269
a.subnetRouters.Remove(cn.UID)
250270
a.exitNodes.Remove(cn.UID)
271+
a.appConnectors.Remove(cn.UID)
251272
a.mu.Unlock()
252273
gaugeConnectorExitNodeResources.Set(int64(a.exitNodes.Len()))
253274
gaugeConnectorSubnetRouterResources.Set(int64(a.subnetRouters.Len()))
275+
gaugeConnectorAppConnectorResources.Set(int64(a.appConnectors.Len()))
254276
var connectors set.Slice[types.UID]
255277
connectors.AddSlice(a.exitNodes.Slice())
256278
connectors.AddSlice(a.subnetRouters.Slice())
279+
connectors.AddSlice(a.appConnectors.Slice())
257280
gaugeConnectorResources.Set(int64(connectors.Len()))
258281
return true, nil
259282
}
@@ -262,8 +285,14 @@ func (a *ConnectorReconciler) validate(cn *tsapi.Connector) error {
262285
// Connector fields are already validated at apply time with CEL validation
263286
// on custom resource fields. The checks here are a backup in case the
264287
// CEL validation breaks without us noticing.
265-
if !(cn.Spec.SubnetRouter != nil || cn.Spec.ExitNode) {
266-
return errors.New("invalid spec: a Connector must expose subnet routes or act as an exit node (or both)")
288+
if cn.Spec.SubnetRouter == nil && !cn.Spec.ExitNode && cn.Spec.AppConnector == nil {
289+
return errors.New("invalid spec: a Connector must be configured as at least one of subnet router, exit node or app connector")
290+
}
291+
if (cn.Spec.SubnetRouter != nil || cn.Spec.ExitNode) && cn.Spec.AppConnector != nil {
292+
return errors.New("invalid spec: a Connector that is configured as an app connector must not be also configured as a subnet router or exit node")
293+
}
294+
if cn.Spec.AppConnector != nil {
295+
return validateAppConnector(cn.Spec.AppConnector)
267296
}
268297
if cn.Spec.SubnetRouter == nil {
269298
return nil
@@ -272,19 +301,27 @@ func (a *ConnectorReconciler) validate(cn *tsapi.Connector) error {
272301
}
273302

274303
func validateSubnetRouter(sb *tsapi.SubnetRouter) error {
275-
if len(sb.AdvertiseRoutes) < 1 {
304+
if len(sb.AdvertiseRoutes) == 0 {
276305
return errors.New("invalid subnet router spec: no routes defined")
277306
}
278-
var err error
279-
for _, route := range sb.AdvertiseRoutes {
307+
return validateRoutes(sb.AdvertiseRoutes)
308+
}
309+
310+
func validateAppConnector(ac *tsapi.AppConnector) error {
311+
return validateRoutes(ac.Routes)
312+
}
313+
314+
func validateRoutes(routes tsapi.Routes) error {
315+
var errs []error
316+
for _, route := range routes {
280317
pfx, e := netip.ParsePrefix(string(route))
281318
if e != nil {
282-
err = errors.Wrap(err, fmt.Sprintf("route %s is invalid: %v", route, err))
319+
errs = append(errs, fmt.Errorf("route %v is invalid: %v", route, e))
283320
continue
284321
}
285322
if pfx.Masked() != pfx {
286-
err = errors.Wrap(err, fmt.Sprintf("route %s has non-address bits set; expected %s", pfx, pfx.Masked()))
323+
errs = append(errs, fmt.Errorf("route %s has non-address bits set; expected %s", pfx, pfx.Masked()))
287324
}
288325
}
289-
return err
326+
return errors.Join(errs...)
290327
}

‎cmd/k8s-operator/connector_test.go‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,12 +8,14 @@ package main
88
import (
99
"context"
1010
"testing"
11+
"time"
1112

1213
"go.uber.org/zap"
1314
appsv1 "k8s.io/api/apps/v1"
1415
corev1 "k8s.io/api/core/v1"
1516
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1617
"k8s.io/apimachinery/pkg/types"
18+
"k8s.io/client-go/tools/record"
1719
"sigs.k8s.io/controller-runtime/pkg/client/fake"
1820
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
1921
"tailscale.com/kube/kubetypes"
@@ -296,3 +298,100 @@ func TestConnectorWithProxyClass(t *testing.T) {
296298
expectReconciled(t, cr, "", "test")
297299
expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
298300
}
301+
302+
func TestConnectorWithAppConnector(t *testing.T) {
303+
// Setup
304+
cn := &tsapi.Connector{
305+
ObjectMeta: metav1.ObjectMeta{
306+
Name: "test",
307+
UID: types.UID("1234-UID"),
308+
},
309+
TypeMeta: metav1.TypeMeta{
310+
Kind: tsapi.ConnectorKind,
311+
APIVersion: "tailscale.io/v1alpha1",
312+
},
313+
Spec: tsapi.ConnectorSpec{
314+
AppConnector: &tsapi.AppConnector{},
315+
},
316+
}
317+
fc := fake.NewClientBuilder().
318+
WithScheme(tsapi.GlobalScheme).
319+
WithObjects(cn).
320+
WithStatusSubresource(cn).
321+
Build()
322+
ft := &fakeTSClient{}
323+
zl, err := zap.NewDevelopment()
324+
if err != nil {
325+
t.Fatal(err)
326+
}
327+
cl := tstest.NewClock(tstest.ClockOpts{})
328+
fr := record.NewFakeRecorder(1)
329+
cr := &ConnectorReconciler{
330+
Client: fc,
331+
clock: cl,
332+
ssr: &tailscaleSTSReconciler{
333+
Client: fc,
334+
tsClient: ft,
335+
defaultTags: []string{"tag:k8s"},
336+
operatorNamespace: "operator-ns",
337+
proxyImage: "tailscale/tailscale",
338+
},
339+
logger: zl.Sugar(),
340+
recorder: fr,
341+
}
342+
343+
// 1. Connector with app connnector is created and becomes ready
344+
expectReconciled(t, cr, "", "test")
345+
fullName, shortName := findGenName(t, fc, "", "test", "connector")
346+
opts := configOpts{
347+
stsName: shortName,
348+
secretName: fullName,
349+
parentType: "connector",
350+
hostname: "test-connector",
351+
app: kubetypes.AppConnector,
352+
isAppConnector: true,
353+
}
354+
expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
355+
expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
356+
// Connector's ready condition should be set to true
357+
358+
cn.ObjectMeta.Finalizers = append(cn.ObjectMeta.Finalizers, "tailscale.com/finalizer")
359+
cn.Status.IsAppConnector = true
360+
cn.Status.Conditions = []metav1.Condition{{
361+
Type: string(tsapi.ConnectorReady),
362+
Status: metav1.ConditionTrue,
363+
LastTransitionTime: metav1.Time{Time: cl.Now().Truncate(time.Second)},
364+
Reason: reasonConnectorCreated,
365+
Message: reasonConnectorCreated,
366+
}}
367+
expectEqual(t, fc, cn, nil)
368+
369+
// 2. Connector with invalid app connector routes has status set to invalid
370+
mustUpdate[tsapi.Connector](t, fc, "", "test", func(conn *tsapi.Connector) {
371+
conn.Spec.AppConnector.Routes = tsapi.Routes{tsapi.Route("1.2.3.4/5")}
372+
})
373+
cn.Spec.AppConnector.Routes = tsapi.Routes{tsapi.Route("1.2.3.4/5")}
374+
expectReconciled(t, cr, "", "test")
375+
cn.Status.Conditions = []metav1.Condition{{
376+
Type: string(tsapi.ConnectorReady),
377+
Status: metav1.ConditionFalse,
378+
LastTransitionTime: metav1.Time{Time: cl.Now().Truncate(time.Second)},
379+
Reason: reasonConnectorInvalid,
380+
Message: "Connector is invalid: route 1.2.3.4/5 has non-address bits set; expected 0.0.0.0/5",
381+
}}
382+
expectEqual(t, fc, cn, nil)
383+
384+
// 3. Connector with valid app connnector routes becomes ready
385+
mustUpdate[tsapi.Connector](t, fc, "", "test", func(conn *tsapi.Connector) {
386+
conn.Spec.AppConnector.Routes = tsapi.Routes{tsapi.Route("10.88.2.21/32")}
387+
})
388+
cn.Spec.AppConnector.Routes = tsapi.Routes{tsapi.Route("10.88.2.21/32")}
389+
cn.Status.Conditions = []metav1.Condition{{
390+
Type: string(tsapi.ConnectorReady),
391+
Status: metav1.ConditionTrue,
392+
LastTransitionTime: metav1.Time{Time: cl.Now().Truncate(time.Second)},
393+
Reason: reasonConnectorCreated,
394+
Message: reasonConnectorCreated,
395+
}}
396+
expectReconciled(t, cr, "", "test")
397+
}

‎cmd/k8s-operator/deploy/crds/tailscale.com_connectors.yaml‎

Lines changed: 47 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,10 @@ spec:
2424
jsonPath: .status.isExitNode
2525
name: IsExitNode
2626
type: string
27+
- description: Whether this Connector instance is an app connector.
28+
jsonPath: .status.isAppConnector
29+
name: IsAppConnector
30+
type: string
2731
- description: Status of the deployed Connector resources.
2832
jsonPath: .status.conditions[?(@.type == "ConnectorReady")].reason
2933
name: Status
@@ -66,10 +70,40 @@ spec:
6670
https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status
6771
type: object
6872
properties:
73+
appConnector:
74+
description: |-
75+
AppConnector defines whether the Connector device should act as a Tailscale app connector. A Connector that is
76+
configured as an app connector cannot be a subnet router or an exit node. If this field is unset, the
77+
Connector does not act as an app connector.
78+
Note that you will need to manually configure the permissions and the domains for the app connector via the
79+
Admin panel.
80+
Note also that the main tested and supported use case of this config option is to deploy an app connector on
81+
Kubernetes to access SaaS applications available on the public internet. Using the app connector to expose
82+
cluster workloads or other internal workloads to tailnet might work, but this is not a use case that we have
83+
tested or optimised for.
84+
If you are using the app connector to access SaaS applications because you need a predictable egress IP that
85+
can be whitelisted, it is also your responsibility to ensure that cluster traffic from the connector flows
86+
via that predictable IP, for example by enforcing that cluster egress traffic is routed via an egress NAT
87+
device with a static IP address.
88+
https://tailscale.com/kb/1281/app-connectors
89+
type: object
90+
properties:
91+
routes:
92+
description: |-
93+
Routes are optional preconfigured routes for the domains routed via the app connector.
94+
If not set, routes for the domains will be discovered dynamically.
95+
If set, the app connector will immediately be able to route traffic using the preconfigured routes, but may
96+
also dynamically discover other routes.
97+
https://tailscale.com/kb/1332/apps-best-practices#preconfiguration
98+
type: array
99+
minItems: 1
100+
items:
101+
type: string
102+
format: cidr
69103
exitNode:
70104
description: |-
71-
ExitNode defines whether the Connector node should act as a
72-
Tailscale exit node. Defaults to false.
105+
ExitNode defines whether the Connector device should act as a Tailscale exit node. Defaults to false.
106+
This field is mutually exclusive with the appConnector field.
73107
https://tailscale.com/kb/1103/exit-nodes
74108
type: boolean
75109
hostname:
@@ -90,9 +124,11 @@ spec:
90124
type: string
91125
subnetRouter:
92126
description: |-
93-
SubnetRouter defines subnet routes that the Connector node should
94-
expose to tailnet. If unset, none are exposed.
127+
SubnetRouter defines subnet routes that the Connector device should
128+
expose to tailnet as a Tailscale subnet router.
95129
https://tailscale.com/kb/1019/subnets/
130+
If this field is unset, the device does not get configured as a Tailscale subnet router.
131+
This field is mutually exclusive with the appConnector field.
96132
type: object
97133
required:
98134
- advertiseRoutes
@@ -125,8 +161,10 @@ spec:
125161
type: string
126162
pattern: ^tag:[a-zA-Z][a-zA-Z0-9-]*$
127163
x-kubernetes-validations:
128-
- rule: has(self.subnetRouter) || self.exitNode == true
129-
message: A Connector needs to be either an exit node or a subnet router, or both.
164+
- rule: has(self.subnetRouter) || (has(self.exitNode) && self.exitNode == true) || has(self.appConnector)
165+
message: A Connector needs to have at least one of exit node, subnet router or app connector configured.
166+
- rule: '!((has(self.subnetRouter) || (has(self.exitNode) && self.exitNode == true)) && has(self.appConnector))'
167+
message: The appConnector field is mutually exclusive with exitNode and subnetRouter fields.
130168
status:
131169
description: |-
132170
ConnectorStatus describes the status of the Connector. This is set
@@ -200,6 +238,9 @@ spec:
200238
If MagicDNS is enabled in your tailnet, it is the MagicDNS name of the
201239
node.
202240
type: string
241+
isAppConnector:
242+
description: IsAppConnector is set to true if the Connector acts as an app connector.
243+
type: boolean
203244
isExitNode:
204245
description: IsExitNode is set to true if the Connector acts as an exit node.
205246
type: boolean

0 commit comments

Comments
 (0)