Merge /httpd/httpd/trunk:r1920747,1920751

  *) mod_md: update to version 2.4.28
     - When the server starts, it looks for new, staged certificates to
       activate. If the staged set of files in 'md/staging/<domain>' is messed
       up, this could prevent further renewals to happen. Now, when the staging
       set is present, but could not be activated due to an error, purge the
       whole directory. [icing]
     - Fix certificate retrieval on ACME renewal to not require a 'Location:'
       header returned by the ACME CA. This was the way it was done in ACME
       before it became an IETF standard. Let's Encrypt still supports this,
       but other CAs do not. [icing]
     - Restore compatibility with OpenSSL < 1.1. [ylavic]



git-svn-id: https://svn.apache.org/repos/asf/httpd/httpd/branches/2.4.x@1920753 13f79535-47bb-0310-9956-ffa450edef68

Author: icing

changes-entries/md_v2.4.28.txt

@@ -0,0 +1,11 @@
+  *) mod_md: update to version 2.4.28
+     - When the server starts, it looks for new, staged certificates to
+       activate. If the staged set of files in 'md/staging/<domain>' is messed
+       up, this could prevent further renewals to happen. Now, when the staging
+       set is present, but could not be activated due to an error, purge the
+       whole directory. [icing]
+     - Fix certificate retrieval on ACME renewal to not require a 'Location:'
+       header returned by the ACME CA. This was the way it was done in ACME
+       before it became an IETF standard. Let's Encrypt still supports this,
+       but other CAs do not. [icing]
+     - Restore compatibility with OpenSSL < 1.1. [ylavic]

modules/md/md_acme_drive.c

@@ -305,11 +305,11 @@ static apr_status_t csr_req(md_acme_t *acme, const md_http_response_t *res, void
 
     (void)acme;
     location = apr_table_get(res->headers, "location");
-    if (!location) {
-        md_log_perror(MD_LOG_MARK, MD_LOG_ERR, APR_EINVAL, d->p, 
-                      "cert created without giving its location header");
-        return APR_EINVAL;
-    }
+    if (!location)
+      return rv;
+
+    md_log_perror(MD_LOG_MARK, MD_LOG_DEBUG, 0, d->p,
+                  "cert created with location header (old ACMEv1 style)");
     ad->order->certificate = apr_pstrdup(d->p, location);
     if (APR_SUCCESS != (rv = md_acme_order_save(d->store, d->p, MD_SG_STAGING, 
                                                 d->md->name, ad->order, 0))) { 

modules/md/md_reg.c

@@ -1194,7 +1194,7 @@ static apr_status_t run_load_staging(void *baton, apr_pool_t *p, apr_pool_t *pte
     result =  va_arg(ap, md_result_t*);
 
     if (APR_STATUS_IS_ENOENT(rv = md_load(reg->store, MD_SG_STAGING, md->name, NULL, ptemp))) {
-        md_log_perror(MD_LOG_MARK, MD_LOG_TRACE2, 0, ptemp, "%s: nothing staged", md->name);
+        md_log_perror(MD_LOG_MARK, MD_LOG_DEBUG, 0, ptemp, "%s: nothing staged", md->name);
         goto out;
     }
 
@@ -1259,7 +1259,9 @@ apr_status_t md_reg_load_stagings(md_reg_t *reg, apr_array_header_t *mds,
         }
         else if (!APR_STATUS_IS_ENOENT(rv)) {
             md_log_perror(MD_LOG_MARK, MD_LOG_ERR, rv, p, APLOGNO(10069)
-                          "%s: error loading staged set", md->name);
+                          "%s: error loading staged set, purging it", md->name);
+            md_store_purge(reg->store, p, MD_SG_STAGING, md->name);
+            md_store_purge(reg->store, p, MD_SG_CHALLENGES, md->name);
         }
     }
 

modules/md/md_version.h

@@ -27,15 +27,15 @@
  * @macro
  * Version number of the md module as c string
  */
-#define MOD_MD_VERSION "2.4.26"
+#define MOD_MD_VERSION "2.4.28"
 
 /**
  * @macro
  * Numerical representation of the version number of the md module
  * release. This is a 24 bit number with 8 bits for major number, 8 bits
  * for minor and 8 bits for patch. Version 1.2.3 becomes 0x010203.
  */
-#define MOD_MD_VERSION_NUM 0x02041a
+#define MOD_MD_VERSION_NUM 0x02041c
 
 #define MD_ACME_DEF_URL         "https://acme-v02.api.letsencrypt.org/directory"
 #define MD_TAILSCALE_DEF_URL    "file://localhost/var/run/tailscale/tailscaled.sock"

test/modules/md/conftest.py

@@ -39,9 +39,7 @@ def env(pytestconfig) -> MDTestEnv:
 @pytest.fixture(autouse=True, scope="package")
 def _md_package_scope(env):
     env.httpd_error_log.add_ignored_lognos([
-        "AH10085",   # There are no SSL certificates configured and no other module contributed any
-        "AH10045",   # No VirtualHost matches Managed Domain
-        "AH10105",   # MDomain does not match any VirtualHost with 'SSLEngine on'
+        "AH10085"   # There are no SSL certificates configured and no other module contributed any
     ])
 
 
@@ -59,7 +57,3 @@ def acme(env):
     if acme_server is not None:
         acme_server.stop()
 
-@pytest.fixture(autouse=True, scope="package")
-def _stop_package_scope(env):
-    yield
-    assert env.apache_stop() == 0

test/modules/md/md_cert_util.py

@@ -1,6 +1,5 @@
 import logging
 import re
-import os
 import socket
 import OpenSSL
 import time
@@ -12,6 +11,7 @@
 from http.client import HTTPConnection
 from urllib.parse import urlparse
 
+from cryptography import x509
 
 SEC_PER_DAY = 24 * 60 * 60
 
@@ -23,45 +23,6 @@ class MDCertUtil(object):
     # Utility class for inspecting certificates in test cases
     # Uses PyOpenSSL: https://pyopenssl.org/en/stable/index.html
 
-    @classmethod
-    def create_self_signed_cert(cls, path, name_list, valid_days, serial=1000):
-        domain = name_list[0]
-        if not os.path.exists(path):
-            os.makedirs(path)
-
-        cert_file = os.path.join(path, 'pubcert.pem')
-        pkey_file = os.path.join(path, 'privkey.pem')
-        # create a key pair
-        if os.path.exists(pkey_file):
-            key_buffer = open(pkey_file, 'rt').read()
-            k = OpenSSL.crypto.load_privatekey(OpenSSL.crypto.FILETYPE_PEM, key_buffer)
-        else:
-            k = OpenSSL.crypto.PKey()
-            k.generate_key(OpenSSL.crypto.TYPE_RSA, 2048)
-
-        # create a self-signed cert
-        cert = OpenSSL.crypto.X509()
-        cert.get_subject().C = "DE"
-        cert.get_subject().ST = "NRW"
-        cert.get_subject().L = "Muenster"
-        cert.get_subject().O = "greenbytes GmbH"
-        cert.get_subject().CN = domain
-        cert.set_serial_number(serial)
-        cert.gmtime_adj_notBefore(valid_days["notBefore"] * SEC_PER_DAY)
-        cert.gmtime_adj_notAfter(valid_days["notAfter"] * SEC_PER_DAY)
-        cert.set_issuer(cert.get_subject())
-
-        cert.add_extensions([OpenSSL.crypto.X509Extension(
-            b"subjectAltName", False, b", ".join(map(lambda n: b"DNS:" + n.encode(), name_list))
-        )])
-        cert.set_pubkey(k)
-        cert.sign(k, 'sha1')
-
-        open(cert_file, "wt").write(
-            OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_PEM, cert).decode('utf-8'))
-        open(pkey_file, "wt").write(
-            OpenSSL.crypto.dump_privatekey(OpenSSL.crypto.FILETYPE_PEM, k).decode('utf-8'))
-
     @classmethod
     def load_server_cert(cls, host_ip, host_port, host_name, tls=None, ciphers=None):
         ctx = OpenSSL.SSL.Context(OpenSSL.SSL.SSLv23_METHOD)
@@ -138,17 +99,26 @@ def get_serial(self):
         # add leading 0s to align with word boundaries.
         return ("%lx" % (self.cert.get_serial_number())).upper()
 
-    def same_serial_as(self, other):
-        if isinstance(other, MDCertUtil):
-            return self.cert.get_serial_number() == other.cert.get_serial_number()
-        elif isinstance(other, OpenSSL.crypto.X509):
-            return self.cert.get_serial_number() == other.get_serial_number()
-        elif isinstance(other, str):
+    @staticmethod
+    def _get_serial(cert) -> int:
+        if isinstance(cert, x509.Certificate):
+            return cert.serial_number
+        if isinstance(cert, MDCertUtil):
+            return cert.get_serial_number()
+        elif isinstance(cert, OpenSSL.crypto.X509):
+            return cert.get_serial_number()
+        elif isinstance(cert, str):
             # assume a hex number
-            return self.cert.get_serial_number() == int(other, 16)
-        elif isinstance(other, int):
-            return self.cert.get_serial_number() == other
-        return False
+            return int(cert, 16)
+        elif isinstance(cert, int):
+            return cert
+        return 0
+
+    def get_serial_number(self):
+        return self._get_serial(self.cert)
+
+    def same_serial_as(self, other):
+        return self._get_serial(self.cert) == self._get_serial(other)
 
     def get_not_before(self):
         tsp = self.cert.get_notBefore()

test/modules/md/md_env.py

@@ -12,9 +12,9 @@
 import time
 
 from datetime import datetime, timedelta
-from typing import Dict, Optional
+from typing import Dict, Optional, Any
 
-from pyhttpd.certs import CertificateSpec
+from pyhttpd.certs import CertificateSpec, Credentials, HttpdTestCA
 from .md_cert_util import MDCertUtil
 from pyhttpd.env import HttpdTestSetup, HttpdTestEnv
 from pyhttpd.result import ExecResult
@@ -73,10 +73,10 @@ def has_acme_server(cls):
 
     @classmethod
     def has_acme_eab(cls):
-        # Pebble v2.5.0 and v2.5.1 do not support HS256 for EAB, which
-        # is the only thing mod_md supports.
-        # Should work for pebble until v2.4.0 and v2.5.2+.
-        # Reference: https://github.com/letsencrypt/pebble/issues/455
+        # Pebble, in v2.5.0 no longer supported HS256 for EAB, which
+        # is the only thing mod_md supports. Issue opened at pebble:
+        # https://github.com/letsencrypt/pebble/issues/455
+        # is fixed in v2.6.0
         return cls.get_acme_server() == 'pebble'
 
     @classmethod
@@ -611,8 +611,13 @@ def await_ocsp_status(self, domain, timeout=10, ca_file=None):
             time.sleep(0.1)
         raise TimeoutError(f"ocsp respopnse not available: {domain}")
 
-    def create_self_signed_cert(self, name_list, valid_days, serial=1000, path=None):
-        dirpath = path
-        if not path:
-            dirpath = os.path.join(self.store_domains(), name_list[0])
-        return MDCertUtil.create_self_signed_cert(dirpath, name_list, valid_days, serial)
+    def create_self_signed_cert(self, spec: CertificateSpec,
+                                valid_from: timedelta = timedelta(days=-1),
+                                valid_to: timedelta = timedelta(days=89),
+                                serial: Optional[int] = None) -> Credentials:
+        key_type = spec.key_type if spec.key_type else 'rsa4096'
+        return HttpdTestCA.create_credentials(spec=spec, issuer=None,
+                                              key_type=key_type,
+                                              valid_from=valid_from,
+                                              valid_to=valid_to,
+                                              serial=serial)

test/modules/md/test_502_acmev2_drive.py

@@ -4,11 +4,12 @@
 import json
 import os.path
 import re
-import time
+from datetime import timedelta
 
 import pytest
+from pyhttpd.certs import CertificateSpec
 
-from .md_conf import MDConf, MDConf
+from .md_conf import MDConf
 from .md_cert_util import MDCertUtil
 from .md_env import MDTestEnv
 
@@ -430,9 +431,12 @@ def test_md_502_201(self, env, renew_window, test_data_list):
         print("TRACE: start testing renew window: %s" % renew_window)
         for tc in test_data_list:
             print("TRACE: create self-signed cert: %s" % tc["valid"])
-            env.create_self_signed_cert([name], tc["valid"])
-            cert2 = MDCertUtil(env.store_domain_file(name, 'pubcert.pem'))
-            assert not cert2.same_serial_as(cert1)
+            creds = env.create_self_signed_cert(CertificateSpec(domains=[name]),
+                                                valid_from=timedelta(days=tc["valid"]["notBefore"]),
+                                                valid_to=timedelta(days=tc["valid"]["notAfter"]))
+            assert creds.certificate.serial_number != cert1.get_serial_number()
+            # copy it over, assess status again
+            creds.save_cert_pem(env.store_domain_file(name, 'pubcert.pem'))
             md = env.a2md(["list", name]).json['output'][0]
             assert md["renew"] == tc["renew"], \
                 "Expected renew == {} indicator in {}, test case {}".format(tc["renew"], md, tc)

test/modules/md/test_702_auto.py

@@ -1,9 +1,9 @@
 import os
-import time
+from datetime import timedelta
 
 import pytest
+from pyhttpd.certs import CertificateSpec
 
-from pyhttpd.conf import HttpdConf
 from pyhttpd.env import HttpdTestEnv
 from .md_cert_util import MDCertUtil
 from .md_env import MDTestEnv
@@ -320,18 +320,22 @@ def test_md_702_009(self, env):
         assert cert1.same_serial_as(stat['rsa']['serial'])
         #
         # create self-signed cert, with critical remaining valid duration -> drive again
-        env.create_self_signed_cert([domain], {"notBefore": -120, "notAfter": 2}, serial=7029)
-        cert3 = MDCertUtil(env.store_domain_file(domain, 'pubcert.pem'))
-        assert cert3.same_serial_as('1B75')
+        creds = env.create_self_signed_cert(CertificateSpec(domains=[domain]),
+                                            valid_from=timedelta(days=-120),
+                                            valid_to=timedelta(days=2),
+                                            serial=7029)
+        creds.save_cert_pem(env.store_domain_file(domain, 'pubcert.pem'))
+        creds.save_pkey_pem(env.store_domain_file(domain, 'privkey.pem'))
+        assert creds.certificate.serial_number == 7029
         assert env.apache_restart() == 0
         stat = env.get_certificate_status(domain)
-        assert cert3.same_serial_as(stat['rsa']['serial'])
+        assert creds.certificate.serial_number == int(stat['rsa']['serial'], 16)
         #
         # cert should renew and be different afterwards
         assert env.await_completion([domain], must_renew=True)
         stat = env.get_certificate_status(domain)
-        assert not cert3.same_serial_as(stat['rsa']['serial'])
-        
+        creds.certificate.serial_number != int(stat['rsa']['serial'], 16)
+
     # test case: drive with an unsupported challenge due to port availability 
     def test_md_702_010(self, env):
         domain = self.test_domain
@@ -543,6 +547,40 @@ def test_md_702_032(self, env):
         assert name2 in cert1b.get_san_list()
         assert not cert1.same_serial_as(cert1b)
 
+    # test case: one MD on a vhost with ServerAlias. Renew.
+    # Exchange ServerName and ServerAlias. Is the rename detected?
+    # See: https://github.com/icing/mod_md/issues/338
+    def test_md_702_033(self, env):
+        domain = self.test_domain
+        name_x = "test-x." + domain
+        name_a = "test-a." + domain
+        domains1 = [name_x, name_a]
+        #
+        # generate 1 MD and 2 vhosts
+        conf = MDConf(env, admin="admin@" + domain)
+        conf.add_md(domains=[name_x])
+        conf.add_vhost(domains=domains1)
+        conf.install()
+        #
+        # restart (-> drive), check that MD was synched and completes
+        assert env.apache_restart() == 0
+        env.check_md(domains1)
+        assert env.await_completion([name_x])
+        env.check_md_complete(name_x)
+        cert_x = env.get_cert(name_x)
+        #
+        # reverse ServerName and ServerAlias
+        domains2 = [name_a, name_x]
+        conf = MDConf(env, admin="admin@" + domain)
+        conf.add_md(domains=[name_a])
+        conf.add_vhost(domains=domains2)
+        conf.install()
+        # restart, check that host still works and kept the cert
+        assert env.apache_restart() == 0
+        status = env.get_certificate_status(name_a)
+        assert cert_x.same_serial_as(status['rsa']['serial'])
+
+
     # test case: test "tls-alpn-01" challenge handling
     def test_md_702_040(self, env):
         domain = self.test_domain