cmd/{containerboot,k8s-operator},kube/kubetypes: kube Ingress L7 proxies only advertise HTTPS endpoint when ready (tailscale#14171)

cmd/containerboot,kube/kubetypes,cmd/k8s-operator: detect if Ingress is created in a tailnet that has no HTTPS

This attempts to make Kubernetes Operator L7 Ingress setup failures more explicit:
- the Ingress resource now only advertises HTTPS endpoint via status.ingress.loadBalancer.hostname when/if the proxy has succesfully loaded serve config
- the proxy attempts to catch cases where HTTPS is disabled for the tailnet and logs a warning

Updates tailscale#12079
Updates tailscale#10407

Signed-off-by: Irbe Krumina <[email protected]>

Author: irbekrm

cmd/containerboot/kube.go

@@ -9,30 +9,55 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"log"
 	"net/http"
 	"net/netip"
 	"os"
 
 	"tailscale.com/kube/kubeapi"
 	"tailscale.com/kube/kubeclient"
+	"tailscale.com/kube/kubetypes"
 	"tailscale.com/tailcfg"
 )
 
-// storeDeviceID writes deviceID to 'device_id' data field of the named
-// Kubernetes Secret.
-func storeDeviceID(ctx context.Context, secretName string, deviceID tailcfg.StableNodeID) error {
+// kubeClient is a wrapper around Tailscale's internal kube client that knows how to talk to the kube API server. We use
+// this rather than any of the upstream Kubernetes client libaries to avoid extra imports.
+type kubeClient struct {
+	kubeclient.Client
+	stateSecret string
+}
+
+func newKubeClient(root string, stateSecret string) (*kubeClient, error) {
+	if root != "/" {
+		// If we are running in a test, we need to set the root path to the fake
+		// service account directory.
+		kubeclient.SetRootPathForTesting(root)
+	}
+	var err error
+	kc, err := kubeclient.New("tailscale-container")
+	if err != nil {
+		return nil, fmt.Errorf("Error creating kube client: %w", err)
+	}
+	if (root != "/") || os.Getenv("TS_KUBERNETES_READ_API_SERVER_ADDRESS_FROM_ENV") == "true" {
+		// Derive the API server address from the environment variables
+		// Used to set http server in tests, or optionally enabled by flag
+		kc.SetURL(fmt.Sprintf("https://%s:%s", os.Getenv("KUBERNETES_SERVICE_HOST"), os.Getenv("KUBERNETES_SERVICE_PORT_HTTPS")))
+	}
+	return &kubeClient{Client: kc, stateSecret: stateSecret}, nil
+}
+
+// storeDeviceID writes deviceID to 'device_id' data field of the client's state Secret.
+func (kc *kubeClient) storeDeviceID(ctx context.Context, deviceID tailcfg.StableNodeID) error {
 	s := &kubeapi.Secret{
 		Data: map[string][]byte{
-			"device_id": []byte(deviceID),
+			kubetypes.KeyDeviceID: []byte(deviceID),
 		},
 	}
-	return kc.StrategicMergePatchSecret(ctx, secretName, s, "tailscale-container")
+	return kc.StrategicMergePatchSecret(ctx, kc.stateSecret, s, "tailscale-container")
 }
 
-// storeDeviceEndpoints writes device's tailnet IPs and MagicDNS name to fields
-// 'device_ips', 'device_fqdn' of the named Kubernetes Secret.
-func storeDeviceEndpoints(ctx context.Context, secretName string, fqdn string, addresses []netip.Prefix) error {
+// storeDeviceEndpoints writes device's tailnet IPs and MagicDNS name to fields 'device_ips', 'device_fqdn' of client's
+// state Secret.
+func (kc *kubeClient) storeDeviceEndpoints(ctx context.Context, fqdn string, addresses []netip.Prefix) error {
 	var ips []string
 	for _, addr := range addresses {
 		ips = append(ips, addr.Addr().String())
@@ -44,24 +69,36 @@ func storeDeviceEndpoints(ctx context.Context, secretName string, fqdn string, a
 
 	s := &kubeapi.Secret{
 		Data: map[string][]byte{
-			"device_fqdn": []byte(fqdn),
-			"device_ips":  deviceIPs,
+			kubetypes.KeyDeviceFQDN: []byte(fqdn),
+			kubetypes.KeyDeviceIPs:  deviceIPs,
 		},
 	}
-	return kc.StrategicMergePatchSecret(ctx, secretName, s, "tailscale-container")
+	return kc.StrategicMergePatchSecret(ctx, kc.stateSecret, s, "tailscale-container")
+}
+
+// storeHTTPSEndpoint writes an HTTPS endpoint exposed by this device via 'tailscale serve' to the client's state
+// Secret. In practice this will be the same value that gets written to 'device_fqdn', but this should only be called
+// when the serve config has been successfully set up.
+func (kc *kubeClient) storeHTTPSEndpoint(ctx context.Context, ep string) error {
+	s := &kubeapi.Secret{
+		Data: map[string][]byte{
+			kubetypes.KeyHTTPSEndpoint: []byte(ep),
+		},
+	}
+	return kc.StrategicMergePatchSecret(ctx, kc.stateSecret, s, "tailscale-container")
 }
 
 // deleteAuthKey deletes the 'authkey' field of the given kube
 // secret. No-op if there is no authkey in the secret.
-func deleteAuthKey(ctx context.Context, secretName string) error {
+func (kc *kubeClient) deleteAuthKey(ctx context.Context) error {
 	// m is a JSON Patch data structure, see https://jsonpatch.com/ or RFC 6902.
 	m := []kubeclient.JSONPatch{
 		{
 			Op:   "remove",
 			Path: "/data/authkey",
 		},
 	}
-	if err := kc.JSONPatchResource(ctx, secretName, kubeclient.TypeSecrets, m); err != nil {
+	if err := kc.JSONPatchResource(ctx, kc.stateSecret, kubeclient.TypeSecrets, m); err != nil {
 		if s, ok := err.(*kubeapi.Status); ok && s.Code == http.StatusUnprocessableEntity {
 			// This is kubernetes-ese for "the field you asked to
 			// delete already doesn't exist", aka no-op.
@@ -72,22 +109,19 @@ func deleteAuthKey(ctx context.Context, secretName string) error {
 	return nil
 }
 
-var kc kubeclient.Client
-
-func initKubeClient(root string) {
-	if root != "/" {
-		// If we are running in a test, we need to set the root path to the fake
-		// service account directory.
-		kubeclient.SetRootPathForTesting(root)
+// storeCapVerUID stores the current capability version of tailscale and, if provided, UID of the Pod in the tailscale
+// state Secret.
+// These two fields are used by the Kubernetes Operator to observe the current capability version of tailscaled running in this container.
+func (kc *kubeClient) storeCapVerUID(ctx context.Context, podUID string) error {
+	capVerS := fmt.Sprintf("%d", tailcfg.CurrentCapabilityVersion)
+	d := map[string][]byte{
+		kubetypes.KeyCapVer: []byte(capVerS),
 	}
-	var err error
-	kc, err = kubeclient.New("tailscale-container")
-	if err != nil {
-		log.Fatalf("Error creating kube client: %v", err)
+	if podUID != "" {
+		d[kubetypes.KeyPodUID] = []byte(podUID)
 	}
-	if (root != "/") || os.Getenv("TS_KUBERNETES_READ_API_SERVER_ADDRESS_FROM_ENV") == "true" {
-		// Derive the API server address from the environment variables
-		// Used to set http server in tests, or optionally enabled by flag
-		kc.SetURL(fmt.Sprintf("https://%s:%s", os.Getenv("KUBERNETES_SERVICE_HOST"), os.Getenv("KUBERNETES_SERVICE_PORT_HTTPS")))
+	s := &kubeapi.Secret{
+		Data: d,
 	}
+	return kc.StrategicMergePatchSecret(ctx, kc.stateSecret, s, "tailscale-container")
 }

cmd/containerboot/kube_test.go

@@ -21,22 +21,22 @@ func TestSetupKube(t *testing.T) {
 		cfg     *settings
 		wantErr bool
 		wantCfg *settings
-		kc      kubeclient.Client
+		kc      *kubeClient
 	}{
 		{
 			name: "TS_AUTHKEY set, state Secret exists",
 			cfg: &settings{
 				AuthKey:    "foo",
 				KubeSecret: "foo",
 			},
-			kc: &kubeclient.FakeClient{
+			kc: &kubeClient{stateSecret: "foo", Client: &kubeclient.FakeClient{
 				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
 					return false, false, nil
 				},
 				GetSecretImpl: func(context.Context, string) (*kubeapi.Secret, error) {
 					return nil, nil
 				},
-			},
+			}},
 			wantCfg: &settings{
 				AuthKey:    "foo",
 				KubeSecret: "foo",
@@ -48,14 +48,14 @@ func TestSetupKube(t *testing.T) {
 				AuthKey:    "foo",
 				KubeSecret: "foo",
 			},
-			kc: &kubeclient.FakeClient{
+			kc: &kubeClient{stateSecret: "foo", Client: &kubeclient.FakeClient{
 				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
 					return false, true, nil
 				},
 				GetSecretImpl: func(context.Context, string) (*kubeapi.Secret, error) {
 					return nil, &kubeapi.Status{Code: 404}
 				},
-			},
+			}},
 			wantCfg: &settings{
 				AuthKey:    "foo",
 				KubeSecret: "foo",
@@ -67,14 +67,14 @@ func TestSetupKube(t *testing.T) {
 				AuthKey:    "foo",
 				KubeSecret: "foo",
 			},
-			kc: &kubeclient.FakeClient{
+			kc: &kubeClient{stateSecret: "foo", Client: &kubeclient.FakeClient{
 				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
 					return false, false, nil
 				},
 				GetSecretImpl: func(context.Context, string) (*kubeapi.Secret, error) {
 					return nil, &kubeapi.Status{Code: 404}
 				},
-			},
+			}},
 			wantCfg: &settings{
 				AuthKey:    "foo",
 				KubeSecret: "foo",
@@ -87,14 +87,14 @@ func TestSetupKube(t *testing.T) {
 				AuthKey:    "foo",
 				KubeSecret: "foo",
 			},
-			kc: &kubeclient.FakeClient{
+			kc: &kubeClient{stateSecret: "foo", Client: &kubeclient.FakeClient{
 				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
 					return false, false, nil
 				},
 				GetSecretImpl: func(context.Context, string) (*kubeapi.Secret, error) {
 					return nil, &kubeapi.Status{Code: 403}
 				},
-			},
+			}},
 			wantCfg: &settings{
 				AuthKey:    "foo",
 				KubeSecret: "foo",
@@ -111,11 +111,11 @@ func TestSetupKube(t *testing.T) {
 				AuthKey:    "foo",
 				KubeSecret: "foo",
 			},
-			kc: &kubeclient.FakeClient{
+			kc: &kubeClient{stateSecret: "foo", Client: &kubeclient.FakeClient{
 				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
 					return false, false, errors.New("broken")
 				},
-			},
+			}},
 			wantErr: true,
 		},
 		{
@@ -127,14 +127,14 @@ func TestSetupKube(t *testing.T) {
 			wantCfg: &settings{
 				KubeSecret: "foo",
 			},
-			kc: &kubeclient.FakeClient{
+			kc: &kubeClient{stateSecret: "foo", Client: &kubeclient.FakeClient{
 				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
 					return false, true, nil
 				},
 				GetSecretImpl: func(context.Context, string) (*kubeapi.Secret, error) {
 					return nil, &kubeapi.Status{Code: 404}
 				},
-			},
+			}},
 		},
 		{
 			// Interactive login using URL in Pod logs
@@ -145,28 +145,28 @@ func TestSetupKube(t *testing.T) {
 			wantCfg: &settings{
 				KubeSecret: "foo",
 			},
-			kc: &kubeclient.FakeClient{
+			kc: &kubeClient{stateSecret: "foo", Client: &kubeclient.FakeClient{
 				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
 					return false, false, nil
 				},
 				GetSecretImpl: func(context.Context, string) (*kubeapi.Secret, error) {
 					return &kubeapi.Secret{}, nil
 				},
-			},
+			}},
 		},
 		{
 			name: "TS_AUTHKEY not set, state Secret contains auth key, we do not have RBAC to patch it",
 			cfg: &settings{
 				KubeSecret: "foo",
 			},
-			kc: &kubeclient.FakeClient{
+			kc: &kubeClient{stateSecret: "foo", Client: &kubeclient.FakeClient{
 				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
 					return false, false, nil
 				},
 				GetSecretImpl: func(context.Context, string) (*kubeapi.Secret, error) {
 					return &kubeapi.Secret{Data: map[string][]byte{"authkey": []byte("foo")}}, nil
 				},
-			},
+			}},
 			wantCfg: &settings{
 				KubeSecret: "foo",
 			},
@@ -177,14 +177,14 @@ func TestSetupKube(t *testing.T) {
 			cfg: &settings{
 				KubeSecret: "foo",
 			},
-			kc: &kubeclient.FakeClient{
+			kc: &kubeClient{stateSecret: "foo", Client: &kubeclient.FakeClient{
 				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
 					return true, false, nil
 				},
 				GetSecretImpl: func(context.Context, string) (*kubeapi.Secret, error) {
 					return &kubeapi.Secret{Data: map[string][]byte{"authkey": []byte("foo")}}, nil
 				},
-			},
+			}},
 			wantCfg: &settings{
 				KubeSecret:         "foo",
 				AuthKey:            "foo",
@@ -194,9 +194,9 @@ func TestSetupKube(t *testing.T) {
 	}
 
 	for _, tt := range tests {
-		kc = tt.kc
+		kc := tt.kc
 		t.Run(tt.name, func(t *testing.T) {
-			if err := tt.cfg.setupKube(context.Background()); (err != nil) != tt.wantErr {
+			if err := tt.cfg.setupKube(context.Background(), kc); (err != nil) != tt.wantErr {
 				t.Errorf("settings.setupKube() error = %v, wantErr %v", err, tt.wantErr)
 			}
 			if diff := cmp.Diff(*tt.cfg, *tt.wantCfg); diff != "" {