Merge branch 'master' into fix-memory-issues

Author: alexanderstephan

.cirrus.yml

@@ -1,15 +1,15 @@
 FreeBSD_task:
   freebsd_instance:
     matrix:
-      image_family: freebsd-13-2
+      image_family: freebsd-14-2
   only_if: $CIRRUS_BRANCH =~ 'master|next'
   install_script:
-    - pkg update -f && pkg upgrade -y && pkg install -y openssl git gmake lua53 socat pcre
+    - pkg update -f && pkg upgrade -y && pkg install -y openssl git gmake lua54 socat pcre2
   script:
     - sudo sysctl kern.corefile=/tmp/%N.%P.core
     - sudo sysctl kern.sugid_coredump=1
     - scripts/build-vtest.sh
-    - gmake CC=clang V=1 ERR=1 TARGET=freebsd USE_ZLIB=1 USE_PCRE=1 USE_OPENSSL=1 USE_LUA=1 LUA_INC=/usr/local/include/lua53 LUA_LIB=/usr/local/lib LUA_LIB_NAME=lua-5.3
+    - gmake CC=clang V=1 ERR=1 TARGET=freebsd USE_ZLIB=1 USE_PCRE2=1 USE_PCRE2_JIT=1 USE_OPENSSL=1 USE_LUA=1 LUA_INC=/usr/local/include/lua54 LUA_LIB=/usr/local/lib LUA_LIB_NAME=lua-5.4
     - ./haproxy -vv
     - ldd haproxy
   test_script:

.github/h2spec.config

@@ -20,8 +20,8 @@ defaults
 frontend h2
     mode http
     bind 127.0.0.1:8443 ssl crt reg-tests/ssl/common.pem alpn h2,http/1.1
-    default_backend h2
+    default_backend h2b
 
-backend h2
+backend h2b
     errorfile 200 .github/errorfile
     http-request deny deny_status 200

.github/matrix.py

@@ -14,6 +14,7 @@
 import sys
 import urllib.request
 from os import environ
+from packaging import version
 
 #
 # this CI is used for both development and stable branches of HAProxy
@@ -47,7 +48,7 @@ def determine_latest_openssl(ssl):
     latest_tag = ""
     for tag in tags:
         if "openssl-" in tag:
-            if tag > latest_tag:
+            if (not latest_tag) or (version.parse(tag[8:]) > version.parse(latest_tag[8:])):
                 latest_tag = tag
     return "OPENSSL_VERSION={}".format(latest_tag[8:])
 
@@ -66,6 +67,37 @@ def determine_latest_aws_lc(ssl):
     latest_tag = max(valid_tags, key=aws_lc_version_string_to_num)
     return "AWS_LC_VERSION={}".format(latest_tag[1:])
 
+def aws_lc_fips_version_string_to_num(version_string):
+    return tuple(map(int, version_string[12:].split('.')))
+
+def aws_lc_fips_version_valid(version_string):
+    return re.match('^AWS-LC-FIPS-[0-9]+(\.[0-9]+)*$', version_string)
+
+@functools.lru_cache(5)
+def determine_latest_aws_lc_fips(ssl):
+    # the AWS-LC-FIPS tags are at the end of the list, so let's get a lot
+    tags = get_all_github_tags("https://api.github.com/repos/aws/aws-lc/tags?per_page=200")
+    if not tags:
+        return "AWS_LC_FIPS_VERSION=failed_to_detect"
+    valid_tags = list(filter(aws_lc_fips_version_valid, tags))
+    latest_tag = max(valid_tags, key=aws_lc_fips_version_string_to_num)
+    return "AWS_LC_FIPS_VERSION={}".format(latest_tag[12:])
+
+def wolfssl_version_string_to_num(version_string):
+    return tuple(map(int, version_string[1:].removesuffix('-stable').split('.')))
+
+def wolfssl_version_valid(version_string):
+    return re.match('^v[0-9]+(\.[0-9]+)*-stable$', version_string)
+
+@functools.lru_cache(5)
+def determine_latest_wolfssl(ssl):
+    tags = get_all_github_tags("https://api.github.com/repos/wolfssl/wolfssl/tags")
+    if not tags:
+        return "WOLFSSL_VERSION=failed_to_detect"
+    valid_tags = list(filter(wolfssl_version_valid, tags))
+    latest_tag = max(valid_tags, key=wolfssl_version_string_to_num)
+    return "WOLFSSL_VERSION={}".format(latest_tag[1:].removesuffix('-stable'))
+
 @functools.lru_cache(5)
 def determine_latest_libressl(ssl):
     try:
@@ -85,14 +117,6 @@ def clean_compression(compression):
     return compression.replace("USE_", "").lower()
 
 
-def get_asan_flags(cc):
-    return [
-        "USE_OBSOLETE_LINKER=1",
-        'DEBUG_CFLAGS="-g -fsanitize=address"',
-        'LDFLAGS="-fsanitize=address"',
-        'CPU_CFLAGS.generic="-O1"',
-    ]
-
 def main(ref_name):
     print("Generating matrix for branch '{}'.".format(ref_name))
 
@@ -101,9 +125,11 @@ def main(ref_name):
     # Ubuntu
 
     if "haproxy-" in ref_name:
-        os = "ubuntu-22.04" # stable branch
+        os = "ubuntu-24.04"         # stable branch
+        os_arm = "ubuntu-24.04-arm" # stable branch
     else:
-        os = "ubuntu-latest" # development branch
+        os = "ubuntu-24.04"         # development branch
+        os_arm = "ubuntu-24.04-arm" # development branch
 
     TARGET = "linux-glibc"
     for CC in ["gcc", "clang"]:
@@ -124,16 +150,16 @@ def main(ref_name):
                 "TARGET": TARGET,
                 "CC": CC,
                 "FLAGS": [
+                    'DEBUG="-DDEBUG_LIST"',
                     "USE_ZLIB=1",
                     "USE_OT=1",
                     "OT_INC=${HOME}/opt-ot/include",
                     "OT_LIB=${HOME}/opt-ot/lib",
                     "OT_RUNPATH=1",
-                    "USE_PCRE=1",
-                    "USE_PCRE_JIT=1",
+                    "USE_PCRE2=1",
+                    "USE_PCRE2_JIT=1",
                     "USE_LUA=1",
                     "USE_OPENSSL=1",
-                    "USE_SYSTEMD=1",
                     "USE_WURFL=1",
                     "WURFL_INC=addons/wurfl/dummy",
                     "WURFL_LIB=addons/wurfl/dummy",
@@ -148,35 +174,37 @@ def main(ref_name):
 
         # ASAN
 
-        matrix.append(
-            {
-                "name": "{}, {}, ASAN, all features".format(os, CC),
-                "os": os,
-                "TARGET": TARGET,
-                "CC": CC,
-                "FLAGS": get_asan_flags(CC)
-                + [
-                    "USE_ZLIB=1",
-                    "USE_OT=1",
-                    "OT_INC=${HOME}/opt-ot/include",
-                    "OT_LIB=${HOME}/opt-ot/lib",
-                    "OT_RUNPATH=1",
-                    "USE_PCRE=1",
-                    "USE_PCRE_JIT=1",
-                    "USE_LUA=1",
-                    "USE_OPENSSL=1",
-                    "USE_SYSTEMD=1",
-                    "USE_WURFL=1",
-                    "WURFL_INC=addons/wurfl/dummy",
-                    "WURFL_LIB=addons/wurfl/dummy",
-                    "USE_DEVICEATLAS=1",
-                    "DEVICEATLAS_SRC=addons/deviceatlas/dummy",
-                    "USE_PROMEX=1",
-                    "USE_51DEGREES=1",
-                    "51DEGREES_SRC=addons/51degrees/dummy/pattern",
-                ],
-            }
-        )
+        for os_asan in [os, os_arm]:
+            matrix.append(
+                {
+                    "name": "{}, {}, ASAN, all features".format(os_asan, CC),
+                    "os": os_asan,
+                    "TARGET": TARGET,
+                    "CC": CC,
+                    "FLAGS": [
+                        "USE_OBSOLETE_LINKER=1",
+                        'ARCH_FLAGS="-g -fsanitize=address"',
+                        'OPT_CFLAGS="-O1"',
+                        "USE_ZLIB=1",
+                        "USE_OT=1",
+                        "OT_INC=${HOME}/opt-ot/include",
+                        "OT_LIB=${HOME}/opt-ot/lib",
+                        "OT_RUNPATH=1",
+                        "USE_PCRE2=1",
+                        "USE_PCRE2_JIT=1",
+                        "USE_LUA=1",
+                        "USE_OPENSSL=1",
+                        "USE_WURFL=1",
+                        "WURFL_INC=addons/wurfl/dummy",
+                        "WURFL_LIB=addons/wurfl/dummy",
+                        "USE_DEVICEATLAS=1",
+                        "DEVICEATLAS_SRC=addons/deviceatlas/dummy",
+                        "USE_PROMEX=1",
+                        "USE_51DEGREES=1",
+                        "51DEGREES_SRC=addons/51degrees/dummy/pattern",
+                    ],
+                }
+            )
 
         for compression in ["USE_ZLIB=1"]:
             matrix.append(
@@ -193,9 +221,10 @@ def main(ref_name):
             "stock",
             "OPENSSL_VERSION=1.0.2u",
             "OPENSSL_VERSION=1.1.1s",
+            "OPENSSL_VERSION=3.5.1",
             "QUICTLS=yes",
-            "WOLFSSL_VERSION=git-d83f2fa",
-            "AWS_LC_VERSION=1.16.0",
+            "WOLFSSL_VERSION=5.7.0",
+            "AWS_LC_VERSION=1.39.0",
             # "BORINGSSL=yes",
         ]
 
@@ -207,8 +236,7 @@ def main(ref_name):
 
         for ssl in ssl_versions:
             flags = ["USE_OPENSSL=1"]
-            if ssl == "BORINGSSL=yes" or ssl == "QUICTLS=yes" or "LIBRESSL" in ssl or "WOLFSSL" in ssl or "AWS_LC" in ssl:
-                flags.append("USE_QUIC=1")
+            skipdup=0
             if "WOLFSSL" in ssl:
                 flags.append("USE_OPENSSL_WOLFSSL=1")
             if "AWS_LC" in ssl:
@@ -218,8 +246,23 @@ def main(ref_name):
                 flags.append("SSL_INC=${HOME}/opt/include")
             if "LIBRESSL" in ssl and "latest" in ssl:
                 ssl = determine_latest_libressl(ssl)
+                skipdup=1
             if "OPENSSL" in ssl and "latest" in ssl:
                 ssl = determine_latest_openssl(ssl)
+                skipdup=1
+
+            # if "latest" equals a version already in the list
+            if ssl in ssl_versions and skipdup == 1:
+                continue
+
+            openssl_supports_quic = False
+            try:
+              openssl_supports_quic = version.Version(ssl.split("OPENSSL_VERSION=",1)[1]) >= version.Version("3.5.0")
+            except:
+              pass
+
+            if ssl == "BORINGSSL=yes" or ssl == "QUICTLS=yes" or "LIBRESSL" in ssl or "WOLFSSL" in ssl or "AWS_LC" in ssl or openssl_supports_quic:
+                flags.append("USE_QUIC=1")
 
             matrix.append(
                 {
@@ -235,9 +278,9 @@ def main(ref_name):
     # macOS
 
     if "haproxy-" in ref_name:
-        os = "macos-12"     # stable branch
+        os = "macos-13"     # stable branch
     else:
-        os = "macos-latest" # development branch
+        os = "macos-15"     # development branch
 
     TARGET = "osx"
     for CC in ["clang"]:

.github/workflows/aws-lc-fips.yml

@@ -0,0 +1,12 @@
+name: AWS-LC-FIPS
+
+on:
+  schedule:
+    - cron: "0 0 * * 4"
+  workflow_dispatch:
+
+jobs:
+  test:
+    uses: ./.github/workflows/aws-lc-template.yml
+    with:
+      command: "from matrix import determine_latest_aws_lc_fips; print(determine_latest_aws_lc_fips(''))"

.github/workflows/aws-lc-template.yml

@@ -0,0 +1,103 @@
+name: AWS-LC template
+
+on:
+  workflow_call:
+    inputs:
+      command:
+        required: true
+        type: string
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    if: ${{ github.repository_owner == 'haproxy' || github.event_name == 'workflow_dispatch' }}
+    steps:
+      - uses: actions/checkout@v5
+      - name: Install VTest
+        run: |
+          scripts/build-vtest.sh
+      - name: Determine latest AWS-LC release
+        id: get_aws_lc_release
+        run: |
+          result=$(cd .github && python3  -c "${{ inputs.command }}")
+          echo $result
+          echo "result=$result" >> $GITHUB_OUTPUT
+      - name: Cache AWS-LC
+        id: cache_aws_lc
+        uses: actions/cache@v4
+        with:
+          path: '~/opt/'
+          key: ssl-${{ steps.get_aws_lc_release.outputs.result }}-Ubuntu-latest-gcc
+      - name: Install apt dependencies
+        run: |
+          sudo apt-get update -o Acquire::Languages=none -o Acquire::Translation=none
+          sudo apt-get --no-install-recommends -y install socat gdb jose
+      - name: Install AWS-LC
+        if: ${{ steps.cache_ssl.outputs.cache-hit != 'true' }}
+        run: env ${{ steps.get_aws_lc_release.outputs.result }} scripts/build-ssl.sh
+      - name: Compile HAProxy
+        run: |
+          make -j$(nproc) ERR=1 CC=gcc TARGET=linux-glibc \
+            USE_OPENSSL_AWSLC=1 USE_QUIC=1 \
+            SSL_LIB=${HOME}/opt/lib SSL_INC=${HOME}/opt/include \
+            DEBUG="-DDEBUG_POOL_INTEGRITY -DDEBUG_UNIT" \
+            ADDLIB="-Wl,-rpath,/usr/local/lib/ -Wl,-rpath,$HOME/opt/lib/"
+          sudo make install
+      - name: Show HAProxy version
+        id: show-version
+        run: |
+          ldd $(which haproxy)
+          haproxy -vv
+          echo "version=$(haproxy -v |awk 'NR==1{print $3}')" >> $GITHUB_OUTPUT
+      - name: Install problem matcher for VTest
+        run: echo "::add-matcher::.github/vtest.json"
+      - name: Run VTest for HAProxy
+        id: vtest
+        run: |
+          # This is required for macOS which does not actually allow to increase
+          # the '-n' soft limit to the hard limit, thus failing to run.
+          ulimit -n 65536
+          # allow to catch coredumps
+          ulimit -c unlimited
+          make reg-tests VTEST_PROGRAM=../vtest/vtest REGTESTS_TYPES=default,bug,devel
+      - name: Run Unit tests
+        id: unittests
+        run: |
+          make unit-tests
+      - name: Show VTest results
+        if: ${{ failure() && steps.vtest.outcome == 'failure' }}
+        run: |
+          for folder in ${TMPDIR:-/tmp}/haregtests-*/vtc.*; do
+            printf "::group::"
+            cat $folder/INFO
+            cat $folder/LOG
+            echo "::endgroup::"
+          done
+          exit 1
+      - name: Show coredumps
+        if: ${{ failure() && steps.vtest.outcome == 'failure' }}
+        run: |
+          failed=false
+          shopt -s nullglob
+          for file in /tmp/core.*; do
+            failed=true
+            printf "::group::"
+            gdb -ex 'thread apply all bt full' ./haproxy $file
+            echo "::endgroup::"
+          done
+          if [ "$failed" = true ]; then
+            exit 1;
+          fi
+      - name: Show Unit-Tests results
+        if: ${{ failure() && steps.unittests.outcome == 'failure' }}
+        run: |
+          for result in ${TMPDIR:-/tmp}/ha-unittests-*/results/res.*; do
+            printf "::group::"
+            cat $result
+            echo "::endgroup::"
+          done
+          exit 1
+