Merge branch 'internetstandards:main' into main

apio-sys · web-flow · commit 46e878a4961d · 2026-02-26T20:26:36.000+01:00
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
@@ -19,11 +19,12 @@ env:
   # should be used to transfer images between jobs. Forked and dependabot builds don't
   # have permission to push to registry.
   use_registry: ${{ ! (github.event_name == 'pull_request' && (github.event.pull_request.head.repo.full_name != github.repository || startsWith(github.head_ref, 'dependabot/'))) }}
+  COMPOSE_VERSION: 2.40.3
 
 jobs:
   # builds all docker images in parallel
   build-docker:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
 
     strategy:
       matrix:
@@ -227,7 +228,7 @@ jobs:
           retention-days: 1
 
   docs:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     needs: [build-docker]
     steps:
       - name: Branch deployment docs
@@ -272,7 +273,7 @@ jobs:
           sudo apt-get update
 
           # upgrade Docker
-          sudo apt install --upgrade docker-ce docker-compose-plugin=2.33.0\*
+          sudo apt install --upgrade docker-ce docker-compose-plugin=$COMPOSE_VERSION\*
 
       - name: Debug info
         run: |
@@ -374,7 +375,7 @@ jobs:
   lintcheck:
     name: lint/check
     needs: [build-docker]
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
 
     env:
       # used in `compose.yaml` files to determine version of images to pull
@@ -452,7 +453,7 @@ jobs:
           sudo apt-get update
 
           # upgrade Docker
-          sudo apt install --upgrade docker-ce docker-compose-plugin=2.33.0\*
+          sudo apt install --upgrade docker-ce docker-compose-plugin=$COMPOSE_VERSION\*
 
       - name: Debug info
         run: |
@@ -539,7 +540,7 @@ jobs:
           sudo apt-get update
 
           # upgrade Docker
-          sudo apt install --upgrade docker-ce docker-compose-plugin=2.33.0\*
+          sudo apt install --upgrade docker-ce docker-compose-plugin=$COMPOSE_VERSION\*
 
       - name: Debug info
         run: |
@@ -657,7 +658,7 @@ jobs:
           sudo apt-get update
 
           # upgrade Docker
-          sudo apt install --upgrade docker-ce docker-compose-plugin=2.33.0\*
+          sudo apt install --upgrade docker-ce docker-compose-plugin=$COMPOSE_VERSION\*
 
       - name: Debug info
         run: |
diff --git a/checks/caa/parser.py b/checks/caa/parser.py
@@ -38,11 +38,11 @@ def node_get_named_child_value(node: Node, name: str) -> Optional[str]:
 ACME_VALIDATION_METHODS = {
     "http-01",
     "dns-01",
-    "http-01",
-    "tls-alpn-01",
     "tls-alpn-01",
     "email-reply-00",
     "tkauth-01",
+    "onion-csr-01",
+    "bp-nodeid-00",
 }
 
 # RFC 8657 4
diff --git a/docker/batch-test.env b/docker/batch-test.env
@@ -21,20 +21,20 @@ IPV6_SUBNET_PUBLIC=fd00:43:1::/48
 IPV6_GATEWAY_PUBLIC=fd00:43:1::1
 IPV6_IP_PUBLIC=fd00:43:1::100
 # use internal IPv4 subnet and IP's
-IPV4_SUBNET_PUBLIC=172.43.0.0/16
-IPV4_IP_PUBLIC=172.43.0.100
-IPV4_WEBSERVER_IP_PUBLIC=172.43.0.100
-IPV4_UNBOUND_IP_PUBLIC=172.43.0.101
+IPV4_SUBNET_PUBLIC=172.16.43.0/24
+IPV4_IP_PUBLIC=172.16.43.100
+IPV4_WEBSERVER_IP_PUBLIC=172.16.43.100
+IPV4_UNBOUND_IP_PUBLIC=172.16.43.101
 IPV6_UNBOUND_IP_PUBLIC=fd00:43:1::101
 
 # use fixed IPv4 addresses for internal networking to prevent resolving cache issues when recreating containers
 IPV4_SUBNET_INTERNAL=192.168.43.0/24
 
-IPV4_IP_MOCK_RESOLVER_PUBLIC=172.43.0.114
+IPV4_IP_MOCK_RESOLVER_PUBLIC=172.16.43.114
 IPV6_IP_MOCK_RESOLVER_PUBLIC=fd00:43:1::114
 
-IPV4_IP_TEST_TARGET_PUBLIC=172.43.0.51
-IPV4_IP_TEST_TARGET_MAIL_PUBLIC=172.43.0.52
+IPV4_IP_TEST_TARGET_PUBLIC=172.16.43.51
+IPV4_IP_TEST_TARGET_MAIL_PUBLIC=172.16.43.52
 IPV6_IP_TEST_TARGET_PUBLIC=fd00:43:1::51
 IPV6_IP_TEST_TARGET_MAIL_PUBLIC=fd00:43:1::52
 
diff --git a/docker/compose-dist.sh b/docker/compose-dist.sh
@@ -0,0 +1,16 @@
+#!/usr/bin/env sh
+
+# wrapper to run the right compose command with the right environment variables from the util container
+
+set -e
+
+# determine install base for multi environment deployments (parent of directory containing this file)
+INTERNETNL_INSTALL_BASE=$(dirname "$(dirname "$(readlink -f "$0")")")
+
+exec docker run -ti --rm --pull=never \
+  --volume /var/run/docker.sock:/var/run/docker.sock \
+  --volume "$INTERNETNL_INSTALL_BASE:/opt/Internet.nl" \
+  --workdir /opt/Internet.nl \
+  --network none \
+  "ghcr.io/internetstandards/util:$RELEASE" \
+  docker compose --env-file=docker/defaults.env --env-file=docker/host.env --env-file=docker/local.env "$@"
diff --git a/docker/compose.development.yaml b/docker/compose.development.yaml
@@ -70,3 +70,10 @@ services:
       - ../checks:/app/checks
       - ../interface:/app/interface
       - ../internetnl:/app/internetnl
+
+networks:
+  public-internet:
+    driver_opts:
+      # development environments break connectivity when outgoing IP is set to the internal public IP
+      com.docker.network.host_ipv4: !reset null
+      com.docker.network.host_ipv6: !reset null
diff --git a/docker/compose.yaml b/docker/compose.yaml
@@ -1111,6 +1111,9 @@ networks:
       com.docker.network.enable_ipv6: "true"
       # network for internal communication between services
       com.docker.network.bridge.enable_icc: "true"
+      # set NAT source IPs to the configured public IPs
+      com.docker.network.host_ipv4: $IPV4_IP_PUBLIC
+      com.docker.network.host_ipv6: $IPV6_IP_PUBLIC
     ipam:
       driver: default
       config:
diff --git a/docker/defaults.env b/docker/defaults.env
@@ -16,7 +16,7 @@ AUTO_UPDATE_TO=
 # Container images/versions to use for external containers
 DOCKER_IMAGE_REDIS=redis:7.4.1-alpine
 DOCKER_IMAGE_RABBITMQ=rabbitmq:3.13.7-management-alpine
-DOCKER_IMAGE_POSTGRES=postgres:15.14-alpine3.22
+DOCKER_IMAGE_POSTGRES=postgres:15.16-alpine3.23
 DOCKER_IMAGE_ROUTINATOR=nlnetlabs/routinator:v0.14.0
 DOCKER_IMAGE_PROMETHEUS=prom/prometheus:v3.0.1
 DOCKER_IMAGE_PROMETHEUS_ALERTMANAGER=prom/alertmanager:v0.27.0
@@ -25,7 +25,7 @@ DOCKER_IMAGE_REDIS_EXPORTER=oliver006/redis_exporter:v1.66.0
 DOCKER_IMAGE_STATSD_EXPORTER=prom/statsd-exporter:v0.28.0
 DOCKER_IMAGE_CELERY_EXPORTER=danihodovic/celery-exporter:0.10.14
 DOCKER_IMAGE_NODE_EXPORTER=quay.io/prometheus/node-exporter:v1.8.2
-DOCKER_IMAGE_DOCKER_STATSD_EXPORTER=aequitas/docker-stats-exporter:0.1.0
+DOCKER_IMAGE_DOCKER_STATSD_EXPORTER=aequitas/docker-stats-exporter:0.2.0
 DOCKER_IMAGE_NGINX_LOGS_EXPORTER=ghcr.io/martin-helmich/prometheus-nginxlog-exporter/exporter:v1.11.0-amd64
 
 # see: documentation/Docker-deployment.yml for information about the network settings
diff --git a/docker/deploy.sh b/docker/deploy.sh
@@ -8,6 +8,9 @@ echo "Deploying release: $RELEASE"
 
 # copy release specific support files
 cp -v /dist/docker/* docker
+# put $RELEASE into the compose.sh file
+envsubst '$RELEASE' < docker/compose-dist.sh > docker/compose.sh
+chmod a+x docker/compose.sh
 
 # set release version in local.env config
 echo "RELEASE='$RELEASE' # deploy $(date)" >> docker/local.env
diff --git a/docker/integration-tests/mock-resolver/bad-ipv6-ns.test.zone b/docker/integration-tests/mock-resolver/bad-ipv6-ns.test.zone
@@ -11,11 +11,11 @@
     NS bad-ns1
     NS bad-ns2
 
-good-ns1  A 172.43.0.114
+good-ns1  A 172.16.43.114
 good-ns1  AAAA fd00:43:1::114
 bad-ns1  AAAA fd00:90::1
 bad-ns2  AAAA fd00:90::2
 
 ; IPV4_IP_TEST_TARGET_PUBLIC
-* A 172.43.0.51
+* A 172.16.43.51
   AAAA fd00:43:1::51
diff --git a/docker/integration-tests/mock-resolver/test.zone b/docker/integration-tests/mock-resolver/test.zone
@@ -2,7 +2,7 @@
 
 ; configure DNS for the internet.nl application instance that is running in test
 ; IPV4_WEBSERVER_IP_PUBLIC
-internet                    A                    172.43.0.100
+internet                    A                    172.16.43.100
 ; IPV6_IP_PUBLIC
                             AAAA                 fd00:43:1::100
 ; all it's subdomains
@@ -30,7 +30,7 @@ conn.www.ipv6.internet      CNAME                ipv6.internet
 ; nameserver setup for the connection test
 test-ns-signed.internet     NS                   ns.test-ns-signed.internet
 ; IPV4_UNBOUND_IP_PUBLIC
-ns.test-ns-signed.internet  A                    172.43.0.101
+ns.test-ns-signed.internet  A                    172.16.43.101
 ; IPV6_UNBOUND_IP_PUBLIC
                             AAAA                 fd00:43:1::101
 
@@ -44,15 +44,15 @@ platforminternet            CNAME                internet
 ; configure DNS for targets that are tested by the internet.test instance
 ; normal 100% website test target
 ; IPV4_IP_TEST_TARGET_PUBLIC
-target                      A                    172.43.0.51
+target                      A                    172.16.43.51
 ; IPV6_IP_TEST_TARGET_PUBLIC
                             AAAA                 fd00:43:1::51
 *.target                    CNAME                target
 
 ; normal 100% email test target
 *.mail-target           MX                   10 mx.mail-target.test.
 ; IPV4_IP_TEST_TARGET_MAIL_PUBLIC
-mx.mail-target          A                    172.43.0.52
+mx.mail-target          A                    172.16.43.52
 ; IPV6_IP_TEST_TARGET_MAIL_PUBLIC
                         AAAA                 fd00:43:1::52
 
@@ -62,7 +62,7 @@ bad-ipv6-ns             NS                   good-ns1.bad-ipv6-ns
                         NS                   bad-ns2.bad-ipv6-ns
                         NS                   bad-ns3.bad-ipv6-ns
 ; IPV4_IP_MOCK_RESOLVER_PUBLIC
-good-ns1.bad-ipv6-ns    A                    172.43.0.114
+good-ns1.bad-ipv6-ns    A                    172.16.43.114
 ; IPV6_IP_MOCK_RESOLVER_PUBLIC
 good-ns1.bad-ipv6-ns    AAAA                 fd00:43:1::114
 bad-ns2.bad-ipv6-ns     AAAA                 fd00:90::1
diff --git a/docker/test.env b/docker/test.env
@@ -20,20 +20,20 @@ IPV6_SUBNET_PUBLIC=fd00:43:1::/48
 IPV6_GATEWAY_PUBLIC=fd00:43:1::1
 IPV6_IP_PUBLIC=fd00:43:1::100
 # use internal IPv4 subnet and IP's
-IPV4_SUBNET_PUBLIC=172.43.0.0/16
-IPV4_IP_PUBLIC=172.43.0.100
-IPV4_WEBSERVER_IP_PUBLIC=172.43.0.100
-IPV4_UNBOUND_IP_PUBLIC=172.43.0.101
+IPV4_SUBNET_PUBLIC=172.16.43.0/24
+IPV4_IP_PUBLIC=172.16.43.100
+IPV4_WEBSERVER_IP_PUBLIC=172.16.43.100
+IPV4_UNBOUND_IP_PUBLIC=172.16.43.101
 IPV6_UNBOUND_IP_PUBLIC=fd00:43:1::101
 
 # use fixed IPv4 addresses for internal networking to prevent resolving cache issues when recreating containers
 IPV4_SUBNET_INTERNAL=192.168.43.0/24
 
-IPV4_IP_MOCK_RESOLVER_PUBLIC=172.43.0.114
+IPV4_IP_MOCK_RESOLVER_PUBLIC=172.16.43.114
 IPV6_IP_MOCK_RESOLVER_PUBLIC=fd00:43:1::114
 
-IPV4_IP_TEST_TARGET_PUBLIC=172.43.0.51
-IPV4_IP_TEST_TARGET_MAIL_PUBLIC=172.43.0.52
+IPV4_IP_TEST_TARGET_PUBLIC=172.16.43.51
+IPV4_IP_TEST_TARGET_MAIL_PUBLIC=172.16.43.52
 IPV6_IP_TEST_TARGET_PUBLIC=fd00:43:1::51
 IPV6_IP_TEST_TARGET_MAIL_PUBLIC=fd00:43:1::52
 
diff --git a/docker/util.Dockerfile b/docker/util.Dockerfile
@@ -1,6 +1,6 @@
-FROM alpine:3.20.3
+FROM alpine:3.22
 
-RUN apk add --no-cache curl postgresql15 python3 py3-prometheus-client py3-requests jq docker-cli docker-cli-compose pigz jq
+RUN apk add --no-cache curl postgresql15 python3 py3-prometheus-client py3-requests jq docker-cli docker-cli-compose pigz jq envsubst
 
 # install cron tasks
 COPY docker/cron/periodic /etc/periodic/
@@ -25,6 +25,7 @@ COPY docker/host-dist.env /dist/docker/
 COPY docker/host-multi-dist.env /dist/docker/
 COPY docker/compose.yaml /dist/docker/
 COPY docker/user_manage.sh /dist/docker/
+COPY docker/compose-dist.sh /dist/docker/
 RUN chmod a-w /dist/docker/*
 
 # add release as label for auto_update feature
diff --git a/docker/webserver/tls_init.sh b/docker/webserver/tls_init.sh
@@ -1,3 +1,5 @@
+#!/bin/sh
+
 # solve chickenegg problem with letsencrypt
 
 # create initial dummy TLS certificate to allow nginx to start
@@ -7,4 +9,4 @@ if ! test -f "/etc/letsencrypt/live/$INTERNETNL_DOMAINNAME/privkey.pem";then
     -keyout "/etc/letsencrypt/live/$INTERNETNL_DOMAINNAME/privkey.pem" \
     -out "/etc/letsencrypt/live/$INTERNETNL_DOMAINNAME/fullchain.pem" \
     -subj "/CN=$INTERNETNL_DOMAINNAME"
-fi
+fi
diff --git a/documentation/Docker-DNS.md b/documentation/Docker-DNS.md
@@ -114,11 +114,11 @@ Letsencrypt account ID and private key are stored in a Docker volume for persist
 
 When deploying a new instance, first complete the full setup. After that perform the following steps to restore the account:
 
-    docker compose --project-name=internetnl-prod stop webserver
+    /opt/Internet.nl/docker/compose.sh stop webserver
     rm -rf /var/lib/docker/volumes/internetnl-prod_certbot-config/_data/*
     cp -r <location of backed up _data directory> /var/lib/docker/volumes/internetnl-prod_certbot-config/_data/
-    docker compose --project-name=internetnl-prod start webserver
+    /opt/Internet.nl/docker/compose.sh start webserver
 
 The certbot instance in the webserver container should start requesting a certificate for the domain after at most 1 minute. You can check the progress using:
 
-    docker compose --project-name=internetnl-prod exec webserver cat /var/log/letsencrypt/letsencrypt.log
+    /opt/Internet.nl/docker/compose.sh exec webserver cat /var/log/letsencrypt/letsencrypt.log
diff --git a/documentation/Docker-deployment-batch.md b/documentation/Docker-deployment-batch.md
@@ -35,29 +35,21 @@ A domain name is recommended to access the server and have Letsencrypt TLS be se
 
 After installation and basic configuration of the OS switch to `root` user.
 
-Run the following command to install required dependencies:
-
-    apt update
-    apt install -yqq ca-certificates curl gnupg
-
-Setup Docker Apt repository:
-
-    install -m 0755 -d /etc/apt/keyrings
-    curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg
-    chmod a+r /etc/apt/keyrings/docker.gpg
-    echo "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \
-      "$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" > /etc/apt/sources.list.d/docker.list
-    apt update
-
-Install Docker:
-
-    apt install -yqq docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
-
-Configure Docker for IPv6 and Live restore:
-
-    echo '{"experimental": true, "ip6tables": true, "live-restore": true}' > /etc/docker/daemon.json
-    systemctl stop docker
-    systemctl start docker
+Run the following commands to configure Docker for IPv6 and Live restore and to install required dependencies, setup Docker Apt repository, and install Docker:
+
+    mkdir -p /etc/docker && \
+    echo '{"ip6tables": true, "live-restore": true}' > /etc/docker/daemon.json && \
+    apt update && \
+    apt install -yqq --no-install-recommends --no-install-suggests ca-certificates curl jq gnupg && \
+    install -m 0755 -d /etc/apt/keyrings && \
+    curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o - > /etc/apt/keyrings/docker.gpg && \
+    chmod a+r /etc/apt/keyrings/docker.gpg && \
+    echo -e "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] \
+    https://download.docker.com/linux/"$(. /etc/os-release && echo "$ID $VERSION_CODENAME")" stable\n \
+    deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] \
+    https://download.docker.com/linux/"$(. /etc/os-release && echo "$ID $VERSION_CODENAME")" test" \
+    > /etc/apt/sources.list.d/docker.list && apt update && \
+    apt install -yqq --no-install-recommends --no-install-suggests docker-ce
 
 ## Application setup
 
@@ -127,7 +119,7 @@ This command should complete without an error, indicating the application stack
 
 Create database indexes:
 
-    docker compose --project-name=internetnl-prod exec app ./manage.py api_create_db_indexes
+    /opt/Internet.nl/docker/compose.sh exec app ./manage.py api_create_db_indexes
 
 ## DNS setup
 
@@ -169,9 +161,9 @@ Please see the generic troubleshooting documentation in [Deployment#Troubleshoot
 If batch jobs seem to be stuck, check the `Batch` dashboard in Grafana (see [Deployment#Grafana](Docker-deployment.md#metrics-grafanaprometheus)). The `Individual task completion rate` graph should show a variaty of tasks being completed per second. If this graph is empty or shows gaps this might indicate the batch jobs got stuck in a deadlock. In this case bring down the environment, remove the Redis and RabbitMQ volumes and start the environment again:
 
     cd /opt/Internet.nl
-    docker compose --project-name=internetnl-prod down
+    /opt/Internet.nl/docker/compose.sh down
     docker volume rm internetnl-prod_rabbitmq
     docker volume rm internetnl-prod_redis
-    env -i docker compose --env-file=docker/defaults.env --env-file=docker/host.env --env-file=docker/local.env up --wait --no-build
+    env -i /opt/Internet.nl/docker/compose.sh up --wait --no-build
 
 This issue can happen on rare occasions, is you encounter this persistently please create an issue in the Internet.nl repository: https://github.com/internetstandards/Internet.nl/issues/new
diff --git a/documentation/Docker-deployment.md b/documentation/Docker-deployment.md
diff --git a/documentation/Docker-getting-started.md b/documentation/Docker-getting-started.md
diff --git a/documentation/Docker-integration-tests.md b/documentation/Docker-integration-tests.md
diff --git a/documentation/Docker-multi-deployment.md b/documentation/Docker-multi-deployment.md
diff --git a/integration_tests/batch/results.py b/integration_tests/batch/results.py
diff --git a/internetnl/settings.py b/internetnl/settings.py
diff --git a/requirements.txt b/requirements.txt
