Merge branch 'internetstandards:main' into main

apio-sys · web-flow · commit 73e85747fbe5 · 2026-02-02T14:09:13.000+01:00
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
@@ -725,7 +725,7 @@ jobs:
 
       - name: Collect Docker Compose logs
         if: always()
-        run: make logs-all-dump env=test > docker-compose.log
+        run: make logs-all-dump env=batch-test > docker-compose.log
 
       - uses: test-summary/action@v2.3
         with:
diff --git a/docker/compose.development.yaml b/docker/compose.development.yaml
@@ -1,7 +1,7 @@
 services:
   # terminate tls so we don't need to have exceptions in the nginx config file for development
   port-expose:
-    image: nginx:1.27.3-alpine
+    image: nginx:1.29.1-alpine3.22
     networks:
       - public-internet
       - internal
diff --git a/docker/compose.integration-tests.yaml b/docker/compose.integration-tests.yaml
@@ -4,7 +4,7 @@ services:
   # from the internal network to the outside
   # also terminate tls so we don't need to have exceptions in the nginx config file for development
   port-expose:
-    image: nginx:1.27.3-alpine
+    image: nginx:1.29.1-alpine3.22
     networks:
       - public-internet
       - port-expose
@@ -96,7 +96,7 @@ services:
       - $RABBITMQ_GUI
 
   test-target:
-    image: nginx:1.27.3-alpine
+    image: nginx:1.29.1-alpine3.22
 
     networks:
       public-internet:
@@ -137,7 +137,7 @@ services:
       MH_SMTP_BIND_ADDR: 0.0.0.0:25
 
   static:
-    image: nginx:1.27.3-alpine
+    image: nginx:1.29.1-alpine3.22
 
     restart: unless-stopped
 
diff --git a/docker/compose.yaml b/docker/compose.yaml
@@ -59,7 +59,7 @@ services:
       - nginx-logs-exporter:/var/log/nginx/prometheus-nginxlog-exporter/
 
     healthcheck:
-      test: ["CMD", "service", "nginx", "status"]
+      test: ["CMD", "curl", "-ksSo/dev/null", "https://$INTERNETNL_DOMAINNAME", "--resolve", "$INTERNETNL_DOMAINNAME:443:127.0.0.1"]
       interval: $HEALTHCHECK_INTERVAL
       start_interval: $HEALTHCHECK_START_INTERVAL
       start_period: 1m
diff --git a/docker/webserver.Dockerfile b/docker/webserver.Dockerfile
@@ -1,11 +1,12 @@
-FROM nginx:1.27.3
+FROM nginx:1.29.1-alpine3.22
 
-RUN apt-get update && apt-get install -y \
+RUN apk add --no-cache \
+  # for random quic host key
+  openssl \
   # for htpasswd
   apache2-utils \
-  # for gixy install
-  python3-venv \
-  && rm -rf /var/lib/apt/lists/*
+  # for gixy and certbot install
+  python3
 
 # install nginx config static analysis tool
 RUN python3 -m venv /opt/gixy
diff --git a/docker/webserver/nginx_templates/default.conf.template b/docker/webserver/nginx_templates/default.conf.template
@@ -32,12 +32,6 @@ resolver 127.0.0.11 ipv6=off valid=5s;
 
 root /var/www/internet.nl;
 
-# enable OSCP stapling
-ssl_stapling on;
-ssl_stapling_verify on;
-ssl_protocols TLSv1.2 TLSv1.3;
-ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305;
-
 http2 on;
 http3 on;
 quic_gso on;
diff --git a/docker/webserver/nginx_templates/letsencrypt.conf.template b/docker/webserver/nginx_templates/letsencrypt.conf.template
diff --git a/docker/webserver/nginx_templates/tls.conf.template b/docker/webserver/nginx_templates/tls.conf.template
@@ -0,0 +1,8 @@
+# If certificate has OCSP, enable the ssl_stapling
+#ssl_stapling on;
+#ssl_stapling_verify on;
+ssl_protocols TLSv1.2 TLSv1.3;
+ssl_ciphers TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384:TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384:TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256:TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256:TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256:TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256;
+ssl_ecdh_curve SecP384r1MLKEM1024:X25519MLKEM768:SecP256r1MLKEM768:secp521r1:brainpoolP512r1:x448:brainpoolP384r1:secp384r1:x25519:secp256r1:brainpoolP256r1;
+ssl_certificate /etc/letsencrypt/live/${INTERNETNL_DOMAINNAME}/fullchain.pem;
+ssl_certificate_key /etc/letsencrypt/live/${INTERNETNL_DOMAINNAME}/privkey.pem;
diff --git a/integration_tests/batch/test_batch.py b/integration_tests/batch/test_batch.py
@@ -64,7 +64,7 @@ def test_batch_openapi(page):
     expect(response).to_be_ok()
 
 
-def test_batch_request(unique_id, register_test_user, test_domain):
+def test_batch_request(unique_id, register_test_user, test_domain, docker_compose_exec):
     """A test via the Batch API should succeed."""
     request_data = {"type": "web", "domains": [test_domain], "name": unique_id}
 
@@ -129,6 +129,24 @@ def test_batch_request(unique_id, register_test_user, test_domain):
     response = requests.get(report_url, verify=False)
     assert response.status_code == 200, "test results should be publicly accessible without authentication"
 
+    # delete result files (this can happen in production when cleanup is done by a cron job)
+    docker_compose_exec("app", "sh -c 'rm -v /app/batch_results/*'")
+
+    # try to get batch results again after files have been deleted (this should trigger a new generation task)
+    results_response = requests.get(INTERNETNL_API + "requests/" + test_id + "/results", auth=auth, verify=False)
+    # expect error because result need to be generated again
+    assert results_response.status_code == 400
+    assert "Report is being generated." in results_response.text
+
+    # wait for report generation and batch to be done
+    wait_for_request_status(INTERNETNL_API + "requests/" + test_id, "done", timeout=60, auth=auth)
+
+    # get batch results again after starting generation
+    results_response = requests.get(INTERNETNL_API + "requests/" + test_id + "/results", auth=auth, verify=False)
+    print(f"{results_response.text=}")
+    results_response.raise_for_status()
+    print("api results JSON:", results_response.text)
+
 
 def test_batch_static_requires_no_auth():
     """Static files should be available without authentication for viewing batch results."""
diff --git a/integration_tests/conftest.py b/integration_tests/conftest.py
@@ -187,8 +187,7 @@ def register_test_user(unique_id):
 
         # reload nginx
         command = (
-            f'docker compose --ansi=never --project-name "{COMPOSE_PROJECT_NAME}"'
-            " exec webserver service nginx reload"
+            f'docker compose --ansi=never --project-name "{COMPOSE_PROJECT_NAME}"' " exec webserver nginx -s reload"
         )
         subprocess.check_call(command, shell=True, universal_newlines=True)
 
diff --git a/interface/batch/util.py b/interface/batch/util.py
@@ -273,6 +273,9 @@ def on_failure(exc, task_id, args, kwargs, einfo):
                 results = gather_batch_results_technical(user, batch_request, site_url)
                 save_batch_results_to_file(user, batch_request, results, technical=True)
 
+    request_lock_id = redis_id.batch_results_request_lock.id.format(batch_request.request_id)
+    cache.delete(request_lock_id)
+
 
 def gather_batch_results(user, batch_request, site_url):
     """
diff --git a/interface/batch/views.py b/interface/batch/views.py
@@ -26,6 +26,7 @@
     register_request,
 )
 from internetnl import log
+from .util import request_already_generating
 
 
 @require_http_methods(["GET", "POST"])
@@ -70,9 +71,10 @@ def results(request, request_id, *args, technical=False, **kwargs):
         return bad_client_request_response("The request is not yet `done`.")
     else:
         if not batch_request.has_report_file():
+            if request_already_generating(batch_request.request_id):
+                return bad_client_request_response("Report is already being generated.")
             batch_async_generate_results.delay(user=user, batch_request=batch_request, site_url=get_site_url(request))
-            return bad_client_request_response("The request is not yet `done`.")
-
+            return bad_client_request_response("Report is being generated.")
         else:
             report_file = batch_request.get_report_file(technical)
             try:
diff --git a/interface/test/batch/test_batch.py b/interface/test/batch/test_batch.py
@@ -3,6 +3,7 @@
 from django.core.cache import cache
 from checks.models import BatchRequest, BatchRequestType, BatchUser, BatchRequestStatus
 from interface.batch.util import get_request, register_request
+from interface.batch.views import results
 from django.test.client import RequestFactory
 import pytest
 from interface.batch.util import batch_async_generate_results
@@ -68,15 +69,27 @@ def test_batch_request_result_generation(db, client, mocker):
     batch_request.save()
 
     # if batch request is done, a batch_async_generate_results task should be put on the queue to generate the results
-    get_request(request, batch_request, test_user)
+    response = get_request(request, batch_request, test_user)
+    assert response.status_code == 200
+    assert json.loads(response.content)["request"]["status"] == "generating"
     assert batch_async_generate_results.delay.call_count == 1
 
     # there should not be an additional task put on the queue when one is already present
-    get_request(request, batch_request, test_user)
+    response = get_request(request, batch_request, test_user)
+    assert response.status_code == 200
+    assert json.loads(response.content)["request"]["status"] == "generating"
+    assert batch_async_generate_results.delay.call_count == 1
+
+    # when directly accessing results there should also not be a extra task added to the queue
+    assert (
+        results(request, batch_request.request_id, batch_user=test_user).status_code == 400
+    ), "should return `bad_client_request_response`"
     assert batch_async_generate_results.delay.call_count == 1
 
     # if the cache expires a new batch_async_generate_results task can be added
     lock_id = redis_id.batch_results_request_lock.id.format(batch_request.request_id)
     cache.delete(lock_id)
-    get_request(request, batch_request, test_user)
+    response = get_request(request, batch_request, test_user)
+    assert response.status_code == 200
+    assert json.loads(response.content)["request"]["status"] == "generating"
     assert batch_async_generate_results.delay.call_count == 2

Original file line number	Diff line number	Diff line change
`@@ -187,8 +187,7 @@ def register_test_user(unique_id):`
`187`	`187`
`188`	`188`	`# reload nginx`
`189`	`189`	`command = (`
`190`		`- f'docker compose --ansi=never --project-name "{COMPOSE_PROJECT_NAME}"'`
`191`		`- " exec webserver service nginx reload"`
	`190`	`+ f'docker compose --ansi=never --project-name "{COMPOSE_PROJECT_NAME}"' " exec webserver nginx -s reload"`
`192`	`191`	`)`
`193`	`192`	`subprocess.check_call(command, shell=True, universal_newlines=True)`
`194`	`193`