From d153fe663f77187c2846d62a0a2b326ca9612244 Mon Sep 17 00:00:00 2001 From: Lucas Lara Date: Fri, 25 Nov 2022 23:11:44 +0100 Subject: [PATCH] Make controller network optional in distributed mode --- README.md | 6 +- cmd.go | 8 +- examples/distributed-controller-network.yaml | 106 ++++++++++++++++++ examples/distributed.yaml | 10 -- .../compose.yaml | 62 ++++++++++ tests/distributed-controller-network/run.sh | 14 +++ tests/distributed/compose.yaml | 56 +-------- tests/distributed/run.sh | 7 +- 8 files changed, 198 insertions(+), 71 deletions(-) create mode 100644 examples/distributed-controller-network.yaml create mode 100644 tests/distributed-controller-network/compose.yaml create mode 100755 tests/distributed-controller-network/run.sh diff --git a/README.md b/README.md index 456f711e..497b6e41 100644 --- a/README.md +++ b/README.md @@ -449,12 +449,16 @@ Each caddy docker proxy instance can be executed in one of the following modes. Acts as a proxy to your Docker resources. The server starts without any configuration, and will not serve anything until it is configured by a "controller". -In order to make a server discoverable and configurable by controllers, you need to mark it with label `caddy_controlled_server` and define the controller network via CLI option `controller-network` or environment variable `CADDY_CONTROLLER_NETWORK`. +In order to make a server discoverable and configurable by controllers, you need to mark it with label `caddy_controlled_server`. Server instances doesn't need access to Docker host socket and you can run it in manager or worker nodes. [Configuration example](examples/distributed.yaml#L5) +Optionally, you can do a more secure setup by restricting the server to only be configurable from a specific network. To do that you need to set the controller network via CLI option `controller-network` or environment variable `CADDY_CONTROLLER_NETWORK` on both server and controller instances. + +[Configuration example](examples/distributed-controller-network.yaml#L5) + ### Controller Controller monitors your Docker cluster, generates Caddy configuration and pushes to all servers it finds in your Docker cluster. diff --git a/cmd.go b/cmd.go index 65d99e7f..2394862d 100644 --- a/cmd.go +++ b/cmd.go @@ -120,17 +120,19 @@ func getAdminListen(options *config.Options) string { if options.ControllerNetwork.Contains(v.IP) { return "tcp/" + v.IP.String() + ":2019" } - break case *net.IPNet: if options.ControllerNetwork.Contains(v.IP) { return "tcp/" + v.IP.String() + ":2019" } - break } } } } - return "tcp/localhost:2019" + if options.Mode&config.Controller == config.Controller { + return "tcp/localhost:2019" + } else { + return "tcp/0.0.0.0:2019" + } } func createOptions(flags caddycmd.Flags) *config.Options { diff --git a/examples/distributed-controller-network.yaml b/examples/distributed-controller-network.yaml new file mode 100644 index 00000000..f03b1df8 --- /dev/null +++ b/examples/distributed-controller-network.yaml @@ -0,0 +1,106 @@ +version: '3.7' + +services: + + caddy_server: + image: lucaslorentz/caddy-docker-proxy:ci-alpine + ports: + - 80:80 + - 443:443 + networks: + - caddy_controller + - caddy + environment: + - CADDY_DOCKER_MODE=server + - CADDY_CONTROLLER_NETWORK=10.200.200.0/24 + volumes: + # this volume is needed to keep the certificates + # otherwise, new ones will be re-issued upon restart + - caddy_data:/data + deploy: + replicas: 3 + labels: + caddy_controlled_server: + + caddy_controller: + image: lucaslorentz/caddy-docker-proxy:ci-alpine + networks: + - caddy_controller + - caddy + environment: + - CADDY_DOCKER_MODE=controller + - CADDY_CONTROLLER_NETWORK=10.200.200.0/24 + volumes: + - /var/run/docker.sock:/var/run/docker.sock + + # Proxy to service + whoami0: + image: jwilder/whoami + networks: + - caddy + deploy: + labels: + caddy: whoami0.example.com + caddy.reverse_proxy: "{{upstreams 8000}}" + # remove the following line when you have verified your setup + # Otherwise you risk being rate limited by let's encrypt + caddy.tls.ca: https://acme-staging-v02.api.letsencrypt.org/directory + + # Proxy to service + whoami1: + image: jwilder/whoami + networks: + - caddy + deploy: + labels: + caddy: whoami1.example.com + caddy.reverse_proxy: "{{upstreams 8000}}" + caddy.tls: "internal" + + # Proxy to container + whoami2: + image: jwilder/whoami + networks: + - caddy + labels: + caddy: whoami2.example.com + caddy.reverse_proxy: "{{upstreams 8000}}" + caddy.tls: "internal" + + # Proxy to container + whoami3: + image: jwilder/whoami + networks: + - caddy + labels: + caddy: whoami3.example.com + caddy.reverse_proxy: "{{upstreams 8000}}" + caddy.tls: "internal" + + # Proxy with matches and route + echo_0: + image: brndnmtthws/nginx-echo-headers + networks: + - caddy + deploy: + labels: + caddy: echo0.example.com + caddy.@match.path: "/sourcepath /sourcepath/*" + caddy.route: "@match" + caddy.route.0_uri: "strip_prefix /sourcepath" + caddy.route.1_rewrite: "* /targetpath{path}" + caddy.route.2_reverse_proxy: "{{upstreams 8080}}" + caddy.tls: "internal" + +networks: + caddy: + driver: overlay + caddy_controller: + driver: overlay + ipam: + driver: default + config: + - subnet: "10.200.200.0/24" + +volumes: + caddy_data: {} diff --git a/examples/distributed.yaml b/examples/distributed.yaml index f03b1df8..9df016f9 100644 --- a/examples/distributed.yaml +++ b/examples/distributed.yaml @@ -8,11 +8,9 @@ services: - 80:80 - 443:443 networks: - - caddy_controller - caddy environment: - CADDY_DOCKER_MODE=server - - CADDY_CONTROLLER_NETWORK=10.200.200.0/24 volumes: # this volume is needed to keep the certificates # otherwise, new ones will be re-issued upon restart @@ -25,11 +23,9 @@ services: caddy_controller: image: lucaslorentz/caddy-docker-proxy:ci-alpine networks: - - caddy_controller - caddy environment: - CADDY_DOCKER_MODE=controller - - CADDY_CONTROLLER_NETWORK=10.200.200.0/24 volumes: - /var/run/docker.sock:/var/run/docker.sock @@ -95,12 +91,6 @@ services: networks: caddy: driver: overlay - caddy_controller: - driver: overlay - ipam: - driver: default - config: - - subnet: "10.200.200.0/24" volumes: caddy_data: {} diff --git a/tests/distributed-controller-network/compose.yaml b/tests/distributed-controller-network/compose.yaml new file mode 100644 index 00000000..bae906de --- /dev/null +++ b/tests/distributed-controller-network/compose.yaml @@ -0,0 +1,62 @@ +version: '3.7' + +services: + + caddy_server: + image: caddy-docker-proxy:local + ports: + - 80:80 + - 443:443 + networks: + - caddy_controller + - caddy + environment: + - CADDY_DOCKER_MODE=server + - CADDY_CONTROLLER_NETWORK=10.200.200.0/24 + deploy: + replicas: 3 + labels: + caddy_controlled_server: + + caddy_controller: + image: caddy-docker-proxy:local + networks: + - caddy_controller + - caddy + environment: + - CADDY_DOCKER_MODE=controller + - CADDY_CONTROLLER_NETWORK=10.200.200.0/24 + volumes: + - source: "${DOCKER_SOCKET_PATH}" + target: "${DOCKER_SOCKET_PATH}" + type: ${DOCKER_SOCKET_TYPE} + + whoami_service: + image: jwilder/whoami + networks: + - caddy + deploy: + labels: + caddy: whoami_service.example.com + caddy.reverse_proxy: "{{upstreams 8000}}" + caddy.tls: "internal" + + whoami_container: + image: jwilder/whoami + networks: + - caddy + labels: + caddy: whoami_container.example.com + caddy.reverse_proxy: "{{upstreams 8000}}" + caddy.tls: "internal" + +networks: + caddy: + name: caddy_test + external: true + caddy_controller: + driver: overlay + ipam: + driver: default + config: + - subnet: "10.200.200.0/24" \ No newline at end of file diff --git a/tests/distributed-controller-network/run.sh b/tests/distributed-controller-network/run.sh new file mode 100755 index 00000000..2856957f --- /dev/null +++ b/tests/distributed-controller-network/run.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +set -e + +. ../functions.sh + +docker stack deploy -c compose.yaml --prune caddy_test + +retry curl --show-error -s -k -f --resolve whoami_service.example.com:443:127.0.0.1 https://whoami_service.example.com && +retry curl --show-error -s -k -f --resolve whoami_container.example.com:443:127.0.0.1 https://whoami_container.example.com || { + docker service logs caddy_test_caddy_controller + docker service logs caddy_test_caddy_server + exit 1 +} diff --git a/tests/distributed/compose.yaml b/tests/distributed/compose.yaml index d9e3520f..0e84cc9e 100644 --- a/tests/distributed/compose.yaml +++ b/tests/distributed/compose.yaml @@ -8,11 +8,9 @@ services: - 80:80 - 443:443 networks: - - caddy_controller - caddy environment: - CADDY_DOCKER_MODE=server - - CADDY_CONTROLLER_NETWORK=10.200.200.0/24 deploy: replicas: 3 labels: @@ -21,80 +19,34 @@ services: caddy_controller: image: caddy-docker-proxy:local networks: - - caddy_controller - caddy environment: - CADDY_DOCKER_MODE=controller - - CADDY_CONTROLLER_NETWORK=10.200.200.0/24 volumes: - source: "${DOCKER_SOCKET_PATH}" target: "${DOCKER_SOCKET_PATH}" type: ${DOCKER_SOCKET_TYPE} - # Proxy to service - whoami0: + whoami_service: image: jwilder/whoami networks: - caddy deploy: labels: - caddy: whoami0.example.com + caddy: whoami_service.example.com caddy.reverse_proxy: "{{upstreams 8000}}" caddy.tls: "internal" - # Proxy to service - whoami1: - image: jwilder/whoami - networks: - - caddy - deploy: - labels: - caddy: whoami1.example.com - caddy.reverse_proxy: "{{upstreams 8000}}" - caddy.tls: "internal" - - # Proxy to container - whoami2: - image: jwilder/whoami - networks: - - caddy - labels: - caddy: whoami2.example.com - caddy.reverse_proxy: "{{upstreams 8000}}" - caddy.tls: "internal" - - # Proxy to container - whoami3: + whoami_container: image: jwilder/whoami networks: - caddy labels: - caddy: whoami3.example.com + caddy: whoami_container.example.com caddy.reverse_proxy: "{{upstreams 8000}}" caddy.tls: "internal" - # Proxy with matches and route - echo_0: - image: brndnmtthws/nginx-echo-headers - networks: - - caddy - deploy: - labels: - caddy: echo0.example.com - caddy.@match.path: "/sourcepath /sourcepath/*" - caddy.route: "@match" - caddy.route.0_uri: "strip_prefix /sourcepath" - caddy.route.1_rewrite: "* /targetpath{path}" - caddy.route.2_reverse_proxy: "{{upstreams 8080}}" - caddy.tls: "internal" - networks: caddy: name: caddy_test external: true - caddy_controller: - driver: overlay - ipam: - driver: default - config: - - subnet: "10.200.200.0/24" \ No newline at end of file diff --git a/tests/distributed/run.sh b/tests/distributed/run.sh index a8fa02cb..2856957f 100755 --- a/tests/distributed/run.sh +++ b/tests/distributed/run.sh @@ -6,11 +6,8 @@ set -e docker stack deploy -c compose.yaml --prune caddy_test -retry curl --show-error -s -k -f --resolve whoami0.example.com:443:127.0.0.1 https://whoami0.example.com && -retry curl --show-error -s -k -f --resolve whoami1.example.com:443:127.0.0.1 https://whoami1.example.com && -retry curl --show-error -s -k -f --resolve whoami2.example.com:443:127.0.0.1 https://whoami2.example.com && -retry curl --show-error -s -k -f --resolve whoami3.example.com:443:127.0.0.1 https://whoami3.example.com && -retry curl --show-error -s -k -f --resolve echo0.example.com:443:127.0.0.1 https://echo0.example.com/sourcepath/something || { +retry curl --show-error -s -k -f --resolve whoami_service.example.com:443:127.0.0.1 https://whoami_service.example.com && +retry curl --show-error -s -k -f --resolve whoami_container.example.com:443:127.0.0.1 https://whoami_container.example.com || { docker service logs caddy_test_caddy_controller docker service logs caddy_test_caddy_server exit 1