Native haproxy consul integration (#486)

* haproxy: use native dns instead of consul-template

* drop consul-template!

* feat(loadbalancer): working on noble

* move local dev to noble

* configurable backend count

* remove demo backends counts

* add some notes to docs

---------

Co-authored-by: Jacob Coffee <[email protected]>

Author: ewdurbin

Vagrantfile

@@ -11,7 +11,7 @@ SERVERS = [
   {:name => "docs", :codename => "noble"},
   {:name => "downloads", :codename => "noble"},
   {:name => "hg", :codename => "noble"},
-  {:name => "loadbalancer", :ports => [20000, 20001, 20002, 20003, 20004, 20005, 20010, 20011]},
+  {:name => "loadbalancer", :codename => "noble", :ports => [20000, 20001, 20002, 20003, 20004, 20005, 20010, 20011]},
   "mail",
   "moin",
   "planet",

docs/guides/haproxy-registration-guide.md

@@ -6,10 +6,12 @@ Register a service with haproxy
     vagrant up salt-master
     vagrant up loadbalancer
     ```
+
 2.  In the local repository, create a new state/directory to manage files for your service:
     ```console
     touch salt/base/salt.sls
     ```
+
 3.  Additionally, add an `nginx` configuration state and `consul` service state that exposes that directory over HTTP:
     - This configuration might look similar to an existing haproxy service like `letsencrypt`
     ```yaml
@@ -55,21 +57,48 @@ Register a service with haproxy
     }
     ~
     ```
-5.  Prepare an SSH configuration file to access the host with native ssh commands:
+
+5. Add an entry in `pillar/base/haproxy.sls` to create the haproxy configuration:
+    ```
+    letsencrypt-well-known:
+      domains: []
+      verify_host: salt.psf.io
+      check: "GET /.well-known/acme-challenge/sentinel HTTP/1.1\\r\\nHost:\\ salt.psf.io"
+    ```
+
+    This will render given the template in `salt/haproxy/config/haproxy.cfg.jinja` to create
+    a service which has two "slots" that will be filled based on the DNS resolution of the consul
+    service registered in step 3.
+
+6.  Prepare an SSH configuration file to access the host with native ssh commands:
     ```console
     vagrant ssh-config salt-master loadbalancer >> vagrant-ssh
     ```
-6.  Open an SSH session with port forwarding to the haproxy status page:
+
+7.  Open an SSH session with port forwarding to the haproxy status page:
     ```console
     ssh -L 4646:127.0.0.1:4646 -F vagrant-ssh loadbalancer
     ```
     - Open [`http://localhost:4646/haproxy?stats`][loadbalancer] to see ``haproxy`` status
-7.  In a new window run:
+
+    ![](images/haproxy-service.png)
+
+    You will see the two "slots" registered in haproxy, with one host found via Consul DNS.
+
+    - Green indicates the host exists, was resolved, and is passing health check.
+    - Brown indicates that a slot does not have a host, in other words not enough hosts were resolved, so it is reserved in "maintenance" state.
+    - Red would indicate that a host exists, was resolved, and is failing health check.
+
+8.  In a new window run:
     ```console
     ssh -F sshconfig -L 8500:127.0.0.1:8500 salt-master
     ```
     - Open [`http://localhost:8500/ui/vagrant/services`][consul] to see what ``consul`` services are registered
 
+    ![](images/consul-service.png)
+
+    You can browse to see what services have been registered, and what nodes are advertising that service.
+
 [//]: # (Quicklink targets)
 [loadbalancer]: <http://localhost:4646/haproxy?stats>
-[consul]: <http://localhost:8500/ui/vagrant/services>
+[consul]: <http://localhost:8500/ui/vagrant/services>

docs/guides/images/consul-service.png

@@ -99,12 +99,14 @@ haproxy:
       domains: []
       verify_host: salt.psf.io
       check: "GET /.well-known/acme-challenge/sentinel HTTP/1.1\\r\\nHost:\\ salt.psf.io"
+      backends: 1
 
     publish-files:
       domains:
         - salt-public.psf.io
       verify_host: salt.psf.io
       check: "GET /salt-server-list.rst HTTP/1.1\\r\\nHost:\\ salt-public.psf.io"
+      backends: 1
 
   redirects:
     cheeseshop.python.org:

docs/guides/images/haproxy-service.png

@@ -7,7 +7,6 @@ consul-pkgs:
   pkg.installed:
     - pkgs:
       - consul
-      - consul-template
 
 consul:
   file.managed:
@@ -115,50 +114,6 @@ consul:
     - group: consul
     - require:
       - pkg: consul-pkgs
-
-
-consul-template:
-  pkg.installed: []
-
-  cmd.run:
-    - name: consul-template -config /etc/consul-template.d -once
-    - require:
-      - pkg: consul-pkgs
-      - service: consul
-    - onchanges:
-      - file: /etc/consul-template.d/*.json
-      - file: /usr/share/consul-template/templates/*
-
-  file.managed:
-    - name: /lib/systemd/system/consul-template.service
-    - source: salt://consul/init/consul-template.service
-    - mode: "0644"
-
-  service.running:
-    - enable: True
-    - restart: True
-    - require:
-      - pkg: consul-pkgs
-      - service: consul
-    - watch:
-      - file: consul-template
-      - file: /etc/consul-template.d/*.json
-      - file: /usr/share/consul-template/templates/*
-
-
-/etc/consul-template.d/base.json:
-  file.managed:
-    - source: salt://consul/etc/consul-template/base.json
-    - user: root
-    - group: root
-    - mode: "0644"
-
-
-/usr/share/consul-template/templates/:
-  file.directory:
-    - user: root
-    - group: consul
-
 {% endif %}
 
 

pillar/base/haproxy.sls

@@ -1,6 +1,11 @@
 {% set haproxy = salt["pillar.get"]("haproxy", {}) -%}
 {% set psf_internal = salt["pillar.get"]("psf_internal_network") -%}
 
+resolvers consul
+    nameserver consul 127.0.0.1:8600
+    accepted_payload_size 8192
+    hold valid 5s
+
 global
     log /dev/log    local0
     log /dev/log    local1 notice
@@ -215,7 +220,25 @@ backend redirect
 {% for service, config in haproxy.services.items() %}
 backend {{ service }}
     {% if config.get("check") -%}
+    {% if grains["oscodename"] != "noble" -%}
     option httpchk {{ config.check }}
+    {%- else -%}
+    # Noble Config using the newer http-check syntax
+    # We need to split the check into parts to handle the extra things
+    # ...maybe there is a better way to do this?
+    {% set check_parts = config.check.split(' ', 2) -%}
+    {% set method = check_parts[0] -%}
+    {% set path = check_parts[1] -%}
+    {% if check_parts|length > 2 -%}
+        {% set extra = check_parts[2].split('\r\n') -%}
+        {% set version = extra[0] -%}
+        {% set headers = extra[1:] -%}
+    {% endif -%}
+    http-check send meth {{ method }} uri {{ path }} ver {{ version }}
+    {%- for header in headers %}
+    http-check send hdr {{ header.replace(':\\ ', ': ') }}
+    {%- endfor %}
+    {%- endif %}
     {%- endif %}
 
     # http://gnuterrypratchett.com/
@@ -230,8 +253,7 @@ backend {{ service }}
     {{ item }}
     {% endfor -%}
 
-    {{ "{{" }}range service "{{ service }}@{{ pillar.dc }}" "any"}}
-    {% raw %}server {{.Node}} {{.Address}}:{{.Port}}{% endraw %}{% if config.get("check", True) %} check{% if config.get("sni", False)%} check-sni {{ config.get("sni") }}{% endif %}{% if config.get("sni", False)%} sni str({{ config.get("sni") }}){% endif %}{% endif %}{% if config.get("tls", True) %} ssl force-tlsv12 verifyhost {{ config.get("verify_host", service + ".psf.io") }} ca-file {{ config.get("ca-file", "PSF_CA.pem") }}{% endif %}{{ "{{end}}" }}
+    server-template backend {{ config.get("backends", 2) }} _{{ service }}._tcp.service.{{ pillar.dc }}.consul resolvers consul resolve-opts allow-dup-ip resolve-prefer ipv4 {% if config.get("check", True) %} check{% if config.get("sni", False)%} check-sni {{ config.get("sni") }}{% endif %}{% if config.get("sni", False)%} sni str({{ config.get("sni") }}){% endif %}{% endif %}{% if config.get("tls", True) %} ssl force-tlsv12 verifyhost {{ config.get("verify_host", service + ".psf.io") }} ca-file {{ config.get("ca-file", "PSF_CA.pem") }}{% endif %}
 
 {% endfor %}
 
@@ -248,8 +270,7 @@ listen {{ name }}
     {{ line }}
     {% endfor %}
 
-    {{ "{{" }}range service "{{ config.service }}@{{ pillar.dc }}"}}
-    {% raw %}server {{.Node}} {{.Address}}:{{.Port}} check{{end}}{% endraw %}{% if config.get("send_proxy", False) %} send-proxy{% endif %}
+    server-template backend {{ config.get("backends", 2) }} _{{ config.service }}._tcp.service.{{ pillar.dc }}.consul resolvers consul resolve-opts allow-dup-ip resolve-prefer ipv4 check {% if config.get("send_proxy", False) %} send-proxy{% endif %}
 
 {% endfor %}
 

salt/consul/init.sls

@@ -27,12 +27,12 @@ haproxy:
     - reload: True
     - require:
       - pkg: haproxy
-      - cmd: consul-template
       - service: rsyslog
     - watch:
       - file: /etc/ssl/private/*.pem
       - file: /etc/haproxy/fastly_token
       - file: /etc/haproxy/our_domains
+      - file: /etc/haproxy/haproxy.cfg
 
 
 /etc/haproxy/fastly_token:
@@ -56,31 +56,13 @@ haproxy:
       - pkg: haproxy
 
 
-/usr/share/consul-template/templates/haproxy.cfg:
+/etc/haproxy/haproxy.cfg:
   file.managed:
     - source: salt://haproxy/config/haproxy.cfg.jinja
     - template: jinja
     - user: root
     - group: root
     - mode: "0644"
-    - require:
-      - pkg: consul-pkgs
-
-
-/etc/consul-template.d/haproxy.json:
-  file.managed:
-    - source: salt://consul/etc/consul-template/template.json.jinja
-    - template: jinja
-    - context:
-        source: /usr/share/consul-template/templates/haproxy.cfg
-        destination: /etc/haproxy/haproxy.cfg
-        command: service haproxy reload
-    - user: root
-    - group: root
-    - mode: "0640"
-    - require:
-      - pkg: consul-pkgs
-
 
 /usr/local/bin/haproxy-ocsp:
   {% if ocsp %}

salt/haproxy/config/haproxy.cfg.jinja

@@ -62,9 +62,6 @@ postgresql-server:
     - require:
       - cmd: postgresql-psf-cluster
       - file: {{ postgresql.config_dir }}/conf.d
-      {% if salt["match.compound"](pillar["roles"]["postgresql-replica"]["pattern"]) %}
-      - cmd: consul-template
-      {% endif %}
     - watch:
       - file: {{ postgresql.config_file }}
       - file: {{ postgresql.ident_file }}
@@ -95,7 +92,6 @@ postgresql-psf-cluster:
       - file: postgresql-data
       {% if salt["match.compound"](pillar["roles"]["postgresql-replica"]["pattern"]) %}
       - file: /etc/ssl/certs/PSF_CA.pem
-      - file: /etc/consul.d/service-postgresql.json
       - service: consul
       {% endif %}
 
@@ -222,36 +218,3 @@ replicator:
     - mode: "0644"
     - require:
       - pkg: consul-pkgs
-
-
-{% if salt["match.compound"](pillar["roles"]["postgresql-replica"]["pattern"]) %}
-
-/usr/share/consul-template/templates/recovery.conf:
-  file.managed:
-    - source: salt://postgresql/server/configs/recovery.conf.jinja
-    - template: jinja
-    - user: postgres
-    - group: postgres
-    - mode: "0640"
-    - show_diff: False
-    - require:
-      - pkg: consul-template
-      - cmd: postgresql-psf-cluster
-      - file: {{ postgresql.config_dir }}
-
-
-/etc/consul-template.d/postgresql-recovery.json:
-  file.managed:
-    - source: salt://consul/etc/consul-template/template.json.jinja
-    - template: jinja
-    - context:
-        source: /usr/share/consul-template/templates/recovery.conf
-        destination: {{ postgresql.recovery_file }}
-        command: "chgrp postgres {{ postgresql.recovery_file }} && chmod 640 {{ postgresql.recovery_file }} && service postgresql restart"
-    - user: root
-    - group: root
-    - mode: "0640"
-    - require:
-      - pkg: consul-template
-
-{% endif %}