Add proxy protocol support for tunnel servers

Add support for configuring proxy protocol on inlets tunnel servers
provisioned by the operator. When enabled, the tunnel server is started
with the --proxy-proto flag so that the original client IP address is
preserved and forwarded to upstream services.

The proxy protocol can be set per-service using the
operator.inlets.dev/proxy-proto annotation.

Configuration options:
- Annotation: operator.inlets.dev/proxy-proto (per-service)

Note: CRD has been updated with a new field.

Signed-off-by: Han Verstraete (OpenFaaS Ltd) <han@openfaas.com>
Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>

Author: welteki

Dockerfile

@@ -30,6 +30,8 @@ COPY validate.go validate.go
 COPY validate_test.go validate_test.go
 COPY config.go  config.go
 COPY config_test.go  config_test.go
+COPY userdata.go userdata.go
+COPY userdata_test.go userdata_test.go
 
 RUN gofmt -l -d $(find . -type f -name '*.go' -not -path "./vendor/*")
 

README.md

@@ -63,6 +63,18 @@ Install the chart with `annotatedOnly: true`, then run:
 kubectl annotate service nginx-1 operator.inlets.dev/manage=1
 ```
 
+## Proxy Protocol support
+
+Proxy protocol can be enabled on tunnel exit servers so that the original client IP address is preserved and forwarded to your services. This is controlled by an annotation.
+
+Allowed values are `v1`, `v2`, or `""` (disabled).
+
+```bash
+kubectl annotate service nginx-1 operator.inlets.dev/proxy-proto=v2
+```
+
+> **Important**: The proxy protocol configuration is applied when the tunnel exit server VM is provisioned and **cannot be changed afterwards**. If you need to change the proxy protocol setting for an existing service, you must delete the service (which will delete the tunnel and VM), then recreate it with the new annotation.
+
 ## Using IPVS for your Kubernetes networking?
 
 For IPVS, you need to declare a Tunnel Custom Resource instead of using the LoadBalancer field.

chart/inlets-operator/crds/operator.inlets.dev_tunnels.yaml

@@ -2,8 +2,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.11.1
-  creationTimestamp: null
+    controller-gen.kubebuilder.io/version: v0.14.0
   name: tunnels.operator.inlets.dev
 spec:
   group: operator.inlets.dev
@@ -29,7 +28,7 @@ spec:
           name: HostIP
           type: string
         - jsonPath: .metadata.creationTimestamp
-          name: Created
+          name: Age
           type: date
         - jsonPath: .status.hostId
           name: HostID
@@ -50,10 +49,19 @@ spec:
           type: object
           properties:
             apiVersion:
-              description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              description: |-
+                APIVersion defines the versioned schema of this representation of an object.
+                Servers should convert recognized schemas to the latest internal value, and
+                may reject unrecognized values.
+                More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
               type: string
             kind:
-              description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              description: |-
+                Kind is a string value representing the REST resource this object represents.
+                Servers may infer this from the endpoint the client submits requests to.
+                Cannot be updated.
+                In CamelCase.
+                More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
               type: string
             metadata:
               type: object
@@ -71,14 +79,23 @@ spec:
                       type: string
                   nullable: true
                 licenseRef:
-                  description: LicenseRef is the secret used to load the inlets-client license, and is the same for each tunnel within the cluster
+                  description: |-
+                    LicenseRef is the secret used to load the inlets-client
+                    license, and is the same for each tunnel within the cluster
                   type: object
                   properties:
                     name:
                       type: string
                     namespace:
                       type: string
                   nullable: true
+                proxyProto:
+                  description: |-
+                    ProxyProto when set, is passed onto the tunnel server
+                    in order to have it send the original source IP.
+                    Note: any upstream must be able to read the Proxy Protocol header
+                  type: string
+                  nullable: true
                 serviceRef:
                   description: ServiceRef is the internal service to tunnel to the remote host
                   type: object
@@ -112,7 +129,9 @@ spec:
                       type: string
                   nullable: true
                 generated:
-                  description: Generated is set to true when the tunnel is created by the operator and false when a user creates the Tunnel via YAML
+                  description: |-
+                    Generated is set to true when the tunnel is created by the operator and false
+                    when a user creates the Tunnel via YAML
                   type: boolean
                 hostIP:
                   type: string

config.go

@@ -24,17 +24,17 @@ type InfraConfig struct {
 	AnnotatedOnly   bool
 	MaxClientMemory string
 	Plan            string
-	ProConfig       InletsProConfig
+	TunnelConfig    TunnelConfig
 }
 
-type InletsProConfig struct {
+type TunnelConfig struct {
 	License       string
 	LicenseFile   string
 	ClientImage   string
 	InletsRelease string
 }
 
-func (c InletsProConfig) GetLicenseKey() (string, error) {
+func (c TunnelConfig) GetLicenseKey() (string, error) {
 	val := ""
 	if len(c.License) > 0 {
 		val = c.License

config_test.go

@@ -9,7 +9,7 @@ import (
 func Test_GetLicenseKey_FromLiteral(t *testing.T) {
 	want := "static.key.text"
 
-	c := InletsProConfig{
+	c := TunnelConfig{
 		License: want,
 	}
 
@@ -38,7 +38,7 @@ func Test_GetLicenseKey_FromFile(t *testing.T) {
 	f.Close()
 	defer os.Remove(name)
 
-	c := InletsProConfig{
+	c := TunnelConfig{
 		LicenseFile: name,
 	}
 
@@ -70,7 +70,7 @@ func Test_GetLicenseKey_FromFileTrimsWhitespace_JWT(t *testing.T) {
 	f.Close()
 	defer os.Remove(name)
 
-	c := InletsProConfig{
+	c := TunnelConfig{
 		LicenseFile: name,
 	}
 
@@ -88,7 +88,7 @@ func Test_GetLicenseKey_FromFileTrimsWhitespace_JWT(t *testing.T) {
 func Test_GetLicenseKey_FromLiteral_WithDashes(t *testing.T) {
 	want := `static-dashes-key-text`
 
-	c := InletsProConfig{
+	c := TunnelConfig{
 		License: want,
 	}
 

controller.go

@@ -49,6 +49,7 @@ import (
 const controllerAgentName = "inlets-operator"
 const inletsPROControlPort = 8123
 const inletsPortsAnnotation = "inlets.dev/ports"
+const proxyProtoAnnotation = "operator.inlets.dev/proxy-proto"
 const licenseSecretName = "inlets-license"
 
 const (
@@ -396,8 +397,7 @@ func (c *Controller) syncHandler(key string) error {
 		// No pre-created secret ref, and no generated secret name either
 		// so create one.
 		if getSecretName(tunnel) == "" {
-			_, err = createTunnelAuthTokenSecret(tunnel, c)
-			if err != nil {
+			if _, err = createTunnelAuthTokenSecret(tunnel, c); err != nil {
 				klog.Infof("Error creating tunnel auth token: %s", err)
 				return fmt.Errorf("error creating tunnel auth token: %s", err)
 			}
@@ -605,6 +605,13 @@ func createTunnelResource(service *corev1.Service, c *Controller) error {
 			return nil
 		}
 
+		var proxyProto string
+		if v, ok := service.Annotations[proxyProtoAnnotation]; ok && v != "" && v != "v1" && v != "v2" {
+			return fmt.Errorf("%s annotation must be 'v1', 'v2', or empty string, got: %s", proxyProtoAnnotation, v)
+		} else {
+			proxyProto = v
+		}
+
 		klog.Infof("Creating Tunnel: %s.%s\n", name, namespace)
 
 		tunnel := &inletsv1alpha1.Tunnel{
@@ -614,6 +621,7 @@ func createTunnelResource(service *corev1.Service, c *Controller) error {
 					Namespace: service.Namespace,
 				},
 				UpdateServiceIP: true,
+				ProxyProto:      proxyProto,
 			},
 			ObjectMeta: metav1.ObjectMeta{
 				Name:      name,
@@ -670,7 +678,7 @@ func createClientDeployment(tunnel *inletsv1alpha1.Tunnel, c *Controller) error
 		return err
 	}
 
-	licenseKey, _ := c.infraConfig.ProConfig.GetLicenseKey()
+	licenseKey, _ := c.infraConfig.TunnelConfig.GetLicenseKey()
 
 	ports := getPortsString(service)
 
@@ -733,7 +741,7 @@ func updateClientDeploymentRef(tunnel *inletsv1alpha1.Tunnel, c *Controller) err
 	if deployment.ObjectMeta.Annotations != nil &&
 		deployment.ObjectMeta.Annotations[inletsPortsAnnotation] != getPortsString(service) {
 
-		licenseKey, _ := c.infraConfig.ProConfig.GetLicenseKey()
+		licenseKey, _ := c.infraConfig.TunnelConfig.GetLicenseKey()
 
 		ports := getPortsString(service)
 		clientDeployment := makeClientDeployment(tunnel,
@@ -761,7 +769,12 @@ func getHostConfig(c *Controller, tunnel *inletsv1alpha1.Tunnel, service *corev1
 		return provision.BasicHost{}, err
 	}
 
-	userData := provision.MakeExitServerUserdata(tokenValue, inletsVersion)
+	proxyProto := ""
+	if v, ok := service.Annotations[proxyProtoAnnotation]; ok {
+		proxyProto = v
+	}
+
+	userData := makeExitServerUserdata(tokenValue, inletsVersion, proxyProto)
 
 	var host provision.BasicHost
 

image_test.go

@@ -5,7 +5,7 @@ import "testing"
 func Test_GetInletsReleaseDefault(t *testing.T) {
 
 	c := InfraConfig{
-		ProConfig: InletsProConfig{
+		TunnelConfig: TunnelConfig{
 			License: "non-empty",
 		},
 		AccessKey: "key",
@@ -22,7 +22,7 @@ func Test_GetInletsReleaseDefault(t *testing.T) {
 func Test_GetInletsReleaseOverride(t *testing.T) {
 
 	c := InfraConfig{
-		ProConfig: InletsProConfig{
+		TunnelConfig: TunnelConfig{
 			License:       "non-empty",
 			InletsRelease: "0.9.40",
 		},
@@ -41,7 +41,7 @@ func Test_GetInletsReleaseOverride(t *testing.T) {
 func Test_InletsClientImageDefault(t *testing.T) {
 
 	c := InfraConfig{
-		ProConfig: InletsProConfig{
+		TunnelConfig: TunnelConfig{
 			License: "non-empty",
 		},
 		AccessKey: "key",
@@ -57,7 +57,7 @@ func Test_InletsClientImageDefault(t *testing.T) {
 func Test_InletsClientImageOverride(t *testing.T) {
 
 	c := InfraConfig{
-		ProConfig: InletsProConfig{
+		TunnelConfig: TunnelConfig{
 			License:     "non-empty",
 			ClientImage: "alexellis2/inlets-pro:0.9.40",
 		},

main.go

@@ -42,7 +42,7 @@ const defaultRelease = "0.9.40"
 
 func main() {
 	infra := &InfraConfig{
-		ProConfig: InletsProConfig{},
+		TunnelConfig: TunnelConfig{},
 	}
 
 	flag.StringVar(&infra.Provider, "provider", "", "Your infrastructure provider - 'equinix-metal', 'digitalocean', 'scaleway', 'gce', 'linode', 'azure', 'ec2' or 'hetzner'")
@@ -57,10 +57,10 @@ func main() {
 	flag.StringVar(&infra.VpcID, "vpc-id", "", "The VPC ID to create the exit-server in (ec2)")
 	flag.StringVar(&infra.SubnetID, "subnet-id", "", "The Subnet ID where the exit-server should be placed (ec2)")
 	flag.StringVar(&infra.ProjectID, "project-id", "", "The project ID if using equinix-metal, or gce as the provider")
-	flag.StringVar(&infra.ProConfig.License, "license", "", "Supply a license for use with inlets-pro")
-	flag.StringVar(&infra.ProConfig.LicenseFile, "license-file", "", "Supply a file to read for the inlets-pro license")
-	flag.StringVar(&infra.ProConfig.ClientImage, "client-image", "ghcr.io/inlets/inlets-pro:"+defaultRelease, "Container image for inlets tunnel clients run in the cluster")
-	flag.StringVar(&infra.ProConfig.InletsRelease, "inlets-release", defaultRelease, "Inlets version to use to create tunnel servers")
+	flag.StringVar(&infra.TunnelConfig.License, "license", "", "Supply a license for use with inlets-pro")
+	flag.StringVar(&infra.TunnelConfig.LicenseFile, "license-file", "", "Supply a file to read for the inlets-pro license")
+	flag.StringVar(&infra.TunnelConfig.ClientImage, "client-image", "ghcr.io/inlets/inlets-pro:"+defaultRelease, "Container image for inlets tunnel clients run in the cluster")
+	flag.StringVar(&infra.TunnelConfig.InletsRelease, "inlets-release", defaultRelease, "Inlets version to use to create tunnel servers")
 
 	flag.StringVar(&infra.MaxClientMemory, "max-client-memory", "128Mi", "Maximum memory limit for the tunnel clients")
 
@@ -89,7 +89,7 @@ func main() {
 		infra.GetInletsClientImage(),
 		infra.GetInletsRelease())
 
-	if _, err := infra.ProConfig.GetLicenseKey(); err != nil {
+	if _, err := infra.TunnelConfig.GetLicenseKey(); err != nil {
 		fmt.Fprintf(os.Stderr, "%s\n", err.Error())
 		os.Exit(1)
 	}
@@ -133,18 +133,18 @@ func main() {
 
 // GetInletsClientImage returns the image for the client-side tunnel
 func (i *InfraConfig) GetInletsClientImage() string {
-	if i.ProConfig.ClientImage == "" {
+	if i.TunnelConfig.ClientImage == "" {
 		return fmt.Sprintf("ghcr.io/inlets/inlets-pro:%s", defaultRelease)
 	}
-	return strings.TrimSpace(i.ProConfig.ClientImage)
+	return strings.TrimSpace(i.TunnelConfig.ClientImage)
 }
 
 func (i *InfraConfig) GetInletsRelease() string {
-	if i.ProConfig.InletsRelease == "" {
+	if i.TunnelConfig.InletsRelease == "" {
 		return defaultRelease
 	}
 
-	return strings.TrimSpace(i.ProConfig.InletsRelease)
+	return strings.TrimSpace(i.TunnelConfig.InletsRelease)
 }
 
 // GetAccessKey from parameter or file trimming

pkg/apis/inletsoperator/v1alpha1/types.go

@@ -63,6 +63,13 @@ type TunnelSpec struct {
 	// +nullable
 	// +kubebuilder:validation:Optional
 	UpdateServiceIP bool `json:"updateServiceIP,omitempty"`
+
+	// +nullable
+	// +kubebuilder:validation:Optional
+	// ProxyProto when set, is passed onto the tunnel server
+	// in order to have it send the original source IP.
+	// Note: any upstream must be able to read the Proxy Protocol header
+	ProxyProto string `json:"proxyProto,omitempty"`
 }
 
 // TunnelStatus is the status for a Tunnel resource

userdata.go

@@ -0,0 +1,47 @@
+// Copyright (c) inlets Author(s) 2019. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+package main
+
+import "fmt"
+
+// makeExitServerUserdata makes a user-data script in bash to setup inlets
+//
+//	with a systemd service and the given version. If proxyProto is non-empty,
+//
+// the PROXY_PROTO environment variable is set in the service configuration.
+func makeExitServerUserdata(authToken, version, proxyProto string) string {
+	return fmt.Sprintf(`#!/bin/bash
+export AUTHTOKEN="%s"
+export IP=$(curl -sfSL https://checkip.amazonaws.com)
+export PROXY_PROTO="%s"
+
+curl -SLsf https://github.com/inlets/inlets-pro/releases/download/%s/inlets-pro -o /tmp/inlets-pro && \
+  chmod +x /tmp/inlets-pro  && \
+  mv /tmp/inlets-pro /usr/local/bin/inlets-pro
+
+cat > /etc/systemd/system/inlets-pro.service <<EOF
+[Unit]
+Description=inlets TCP server
+After=network.target
+
+[Service]
+Type=simple
+Restart=always
+RestartSec=2
+StartLimitInterval=0
+EnvironmentFile=/etc/default/inlets-pro
+ExecStart=/usr/local/bin/inlets-pro tcp server --auto-tls --auto-tls-san="\${IP}" --token="\${AUTHTOKEN}" --proxy-protocol="\${PROXY_PROTO}"
+
+[Install]
+WantedBy=multi-user.target
+EOF
+
+echo "AUTHTOKEN=$AUTHTOKEN" >> /etc/default/inlets-pro && \
+  echo "IP=$IP" >> /etc/default/inlets-pro && \
+  echo "PROXY_PROTO=$PROXY_PROTO" >> /etc/default/inlets-pro && \
+  systemctl daemon-reload && \
+  systemctl start inlets-pro && \
+  systemctl enable inlets-pro
+`, authToken, proxyProto, version)
+}