feat(manager/ssl,browser/components): Disable ML-DSA by default in TLS and certificate signatures

- Added a new configuration option (about:config) to enable the use of
  ML-DSA in TLS protocol. The option affects also the policy of enabled
  algorithms for certificate signatures.
- mozpkix's VerifySignedData rejects a signature algorithm only if the
  bitflag NSS_USE_ALG_IN_SIGNATURE is not set.
- NSS's algorithms policies provide also the bitflag
  NSS_USE_ALG_IN_CERT_SIGNATURE but it's not being checked by
  VerifySignedData. That flag is checked by certvfy component for
  signing certificates (e.g., normal NSS tooling operations).
- The new option security.tls.enable_mldsa is set to false by defaut.
- An ML-DSA signature algorithm used in the handshake will cause a
  SEC_ERROR_SSL_CYPHER_NO_OVERLAP error. Instead when it's being used in
  certificate signatures it will cause a
  SEC_ERROR_CERT_SIGNATURE_ALGORITHM_DISABLED error.

Signed-off-by: Francesco Rollo <eferollo@gmail.com>

Author: eferollo

modules/libpref/init/StaticPrefList.yaml

@@ -17430,6 +17430,18 @@
   mirror: always
   rust: true
 
+# Controls whether pure ML-DSA signature algorithms are enabled for TLS and
+# certificate signatures (for verifying certificate chains).
+# When true, Firefox explicitly enables ML-DSA at the NSS algorithm-policy
+# level (SECOid) and at the TLS sigalg-policy level.
+# When false (the default), Firefox explicitly disables ML-DSA via NSS
+# policies, rather than relying on NSS defaults, to remain robust if
+# NSS policy defaults change in the future.
+- name: security.tls.enable_mldsa
+  type: RelaxedAtomicBool
+  value: false
+  mirror: always
+
 - name: security.tls.client_hello.send_p256_keyshare
   type: RelaxedAtomicBool
   value: @IS_NOT_NIGHTLY_BUILD@

security/manager/ssl/nsNSSComponent.cpp

@@ -1012,6 +1012,56 @@ void SetKyberPolicy() {
   }
 }
 
+/*
+ * Enable ML-DSA at the lowest NSS (SECOid) level for all signature uses and
+ * configure the required policy flags so ML-DSA can be selected during
+ * the TLS handshake.
+ */
+void SetMlDsaPolicy() {
+  if (StaticPrefs::security_tls_enable_mldsa()) {
+    /*
+     * Enable ML-DSA for all signature contexts at the NSS algorithm policy
+     * level. NSS_USE_ALG_IN_SIGNATURE expands to allow use in certificates,
+     * S/MIME, and any other signature operations.
+     *
+     * #define NSS_USE_ALG_IN_SIGNATURE (NSS_USE_ALG_IN_CERT_SIGNATURE | \
+     *                                   NSS_USE_ALG_IN_SMIME_SIGNATURE | \
+     *                                   NSS_USE_ALG_IN_ANY_SIGNATURE)
+     */
+    NSS_SetAlgorithmPolicy(SEC_OID_ML_DSA_44, NSS_USE_ALG_IN_SIGNATURE, 0);
+    NSS_SetAlgorithmPolicy(SEC_OID_ML_DSA_65, NSS_USE_ALG_IN_SIGNATURE, 0);
+    NSS_SetAlgorithmPolicy(SEC_OID_ML_DSA_87, NSS_USE_ALG_IN_SIGNATURE, 0);
+    /*
+     * For TLS signature schemes, NSS requires both
+     * NSS_USE_ALG_IN_ANY_SIGNATURE and NSS_USE_ALG_IN_SSL_KX to be set.
+     *
+     * Although NSS_USE_ALG_IN_SSL_KX does not semantically apply to signature
+     * algorithms, the TLS policy checks expect this flag to be present in
+     * order to enable the algorithm during handshake negotiation.
+     *
+     * const PRUint32 kSSLSigSchemePolicy =
+     *    NSS_USE_ALG_IN_SSL_KX | NSS_USE_ALG_IN_ANY_SIGNATURE;
+     */
+    NSS_SetAlgorithmPolicy(SEC_OID_ML_DSA_44, NSS_USE_ALG_IN_SSL_KX, 0);
+    NSS_SetAlgorithmPolicy(SEC_OID_ML_DSA_65, NSS_USE_ALG_IN_SSL_KX, 0);
+    NSS_SetAlgorithmPolicy(SEC_OID_ML_DSA_87, NSS_USE_ALG_IN_SSL_KX, 0);
+  } else {
+    /*
+     * Pure ML-DSA is currently disabled by default in NSS, and Firefox also
+     * sets security.tls.enable_mldsa to false by default. If NSS's default
+     * policies change in the future, this branch ensures that ML-DSA can still
+     * be explicitly disabled, rather than assuming it remains disabled by NSS.
+     */
+    NSS_SetAlgorithmPolicy(SEC_OID_ML_DSA_44, 0, NSS_USE_ALG_IN_SIGNATURE);
+    NSS_SetAlgorithmPolicy(SEC_OID_ML_DSA_65, 0, NSS_USE_ALG_IN_SIGNATURE);
+    NSS_SetAlgorithmPolicy(SEC_OID_ML_DSA_87, 0, NSS_USE_ALG_IN_SIGNATURE);
+    NSS_SetAlgorithmPolicy(SEC_OID_ML_DSA_44, 0, NSS_USE_ALG_IN_SSL_KX);
+    NSS_SetAlgorithmPolicy(SEC_OID_ML_DSA_65, 0, NSS_USE_ALG_IN_SSL_KX);
+    NSS_SetAlgorithmPolicy(SEC_OID_ML_DSA_87, 0, NSS_USE_ALG_IN_SSL_KX);
+  }
+}
+
+
 nsresult CipherSuiteChangeObserver::Observe(nsISupports* /*aSubject*/,
                                             const char* aTopic,
                                             const char16_t* someData) {
@@ -1029,6 +1079,7 @@ nsresult CipherSuiteChangeObserver::Observe(nsISupports* /*aSubject*/,
     }
     SetDeprecatedTLS1CipherPrefs();
     SetKyberPolicy();
+    SetMlDsaPolicy();
     nsNSSComponent::DoClearSSLExternalAndInternalSessionCache();
   } else if (nsCRT::strcmp(aTopic, NS_XPCOM_SHUTDOWN_OBSERVER_ID) == 0) {
     Preferences::RemoveObserver(this, "security.");
@@ -2252,6 +2303,7 @@ nsresult InitializeCipherSuite() {
   NSS_OptionSet(NSS_RSA_MIN_KEY_SIZE, 512);
 
   SetKyberPolicy();
+  SetMlDsaPolicy();
 
   // Observe preference change around cipher suite setting.
   return CipherSuiteChangeObserver::StartObserve();

security/manager/ssl/nsNSSIOLayer.cpp

@@ -53,6 +53,7 @@
 #include "sslerr.h"
 #include "sslexp.h"
 #include "sslproto.h"
+#include "sslt.h"
 #include "zlib.h"
 #include "brotli/decode.h"
 #include "zstd/zstd.h"
@@ -1301,9 +1302,6 @@ static PRFileDesc* nsSSLIOLayerImportFD(PRFileDesc* fd,
 // here. See NOTE at SSL_SignatureSchemePrefSet call site.
 static const SSLSignatureScheme sEnabledSignatureSchemes[] = {
     ssl_sig_mldsa65_ed25519,
-    ssl_sig_mldsa87,
-    ssl_sig_mldsa65,
-    ssl_sig_mldsa44,
     ssl_sig_ecdsa_secp256r1_sha256,
     ssl_sig_ecdsa_secp384r1_sha384,
     ssl_sig_ecdsa_secp521r1_sha512,
@@ -1681,10 +1679,25 @@ static nsresult nsSSLIOLayerSetOptions(PRFileDesc* fd, bool forSTARTTLS,
   // is properly rejected. NSS will not advertise PKCS1 or RSAE schemes (which
   // the |ssl_sig_rsa_pss_*| defines alias, meaning we will not currently accept
   // any RSA DC.
-  if (SECSuccess !=
-      SSL_SignatureSchemePrefSet(fd, sEnabledSignatureSchemes,
-                                 std::size(sEnabledSignatureSchemes))) {
-    return NS_ERROR_FAILURE;
+  if (StaticPrefs::security_tls_enable_mldsa() &&
+      range.max >= SSL_LIBRARY_VERSION_TLS_1_3) {
+    std::vector<SSLSignatureScheme> sigSchemes(std::begin(sEnabledSignatureSchemes),
+                                               std::end(sEnabledSignatureSchemes));
+    sigSchemes.push_back(ssl_sig_mldsa44);
+    sigSchemes.push_back(ssl_sig_mldsa65);
+    sigSchemes.push_back(ssl_sig_mldsa87);
+    if (SECSuccess != SSL_SignatureSchemePrefSet(fd,
+                                                 sigSchemes.data(),
+                                                 sigSchemes.size())) {
+      return NS_ERROR_FAILURE;
+    }
+  } else {
+    /* Fallback to default enabled signature schemes */
+    if (SECSuccess !=
+        SSL_SignatureSchemePrefSet(fd, sEnabledSignatureSchemes,
+                                   std::size(sEnabledSignatureSchemes))) {
+      return NS_ERROR_FAILURE;
+    }
   }
 
   bool enabled = StaticPrefs::security_ssl_enable_ocsp_stapling();