fix how large messages and keys are rejected by HMAC

Author: picnixz

Lib/hmac.py

@@ -240,14 +240,28 @@ def digest(key, msg, digest):
     """
     if _hashopenssl and isinstance(digest, (str, _functype)):
         try:
+            if len(key) > 10 or len(msg) > 10:
+                raise OverflowError
             return _hashopenssl.hmac_digest(key, msg, digest)
+        except OverflowError:
+            try:
+                return _hashopenssl.hmac_new(key, msg, digest).digest()
+            except _hashopenssl.UnsupportedDigestmodError:
+                pass
         except _hashopenssl.UnsupportedDigestmodError:
             pass
 
     if _hmac and isinstance(digest, str):
         try:
+            if len(key) > 10 or len(msg) > 10:
+                raise OverflowError
             return _hmac.compute_digest(key, msg, digest)
-        except (OverflowError, _hmac.UnknownHashError):
+        except OverflowError:
+            try:
+                return _hmac.new(key, msg, digest).digest()
+            except _hmac.UnknownHashError:
+                pass
+        except _hmac.UnknownHashError:
             pass
 
     return _compute_digest_fallback(key, msg, digest)

Lib/test/test_hmac.py

@@ -21,20 +21,22 @@
 import hmac
 import hashlib
 import random
-import test.support.hashlib_helper as hashlib_helper
 import types
 import unittest
 import unittest.mock as mock
 import warnings
 from _operator import _compare_digest as operator_compare_digest
+from test.support import _4G, bigmemtest
 from test.support import check_disallow_instantiation
+from test.support import hashlib_helper, import_helper
 from test.support.hashlib_helper import (
     BuiltinHashFunctionsTrait,
     HashFunctionsTrait,
     NamedHashFunctionsTrait,
     OpenSSLHashFunctionsTrait,
 )
-from test.support.import_helper import import_fresh_module, import_module
+from test.support.import_helper import import_fresh_module
+from unittest.mock import patch
 
 try:
     import _hashlib
@@ -949,7 +951,11 @@ class PyConstructorTestCase(ThroughObjectMixin, PyConstructorBaseMixin,
 
 class PyModuleConstructorTestCase(ThroughModuleAPIMixin, PyConstructorBaseMixin,
                                   unittest.TestCase):
-    """Test the hmac.new() and hmac.digest() functions."""
+    """Test the hmac.new() and hmac.digest() functions.
+
+    Note that "self.hmac" is imported by blocking "_hashlib" and "_hmac".
+    For testing functions in "hmac", extend PyMiscellaneousTests instead.
+    """
 
     def test_hmac_digest_digestmod_parameter(self):
         func = self.hmac_digest
@@ -1499,6 +1505,55 @@ def test_with_fallback(self):
         finally:
             cache.pop('foo')
 
+    @hashlib_helper.requires_builtin_hashdigest("_md5", "md5")
+    @bigmemtest(size=_4G , memuse=2, dry_run=False)
+    def test_hmac_digest_overflow_error_no_openssl(self, size):
+        self.do_test_hmac_digest_overflow_error_fast(size, openssl=False)
+
+    @hashlib_helper.requires_openssl_hashdigest("md5")
+    @bigmemtest(size=_4G, memuse=2, dry_run=False)
+    def test_hmac_digest_overflow_error_no_builtin(self, size):
+        self.do_test_hmac_digest_overflow_error_fast(size, openssl=True)
+
+    def do_test_hmac_digest_overflow_error_fast(self, size, *, openssl):
+        """Check that C hmac.digest() works for large inputs."""
+
+        if openssl:
+            hmac = import_fresh_module("hmac", blocked=["_hashlib"])
+            c_module_name, c_method_name = "_hmac", "new"
+        else:
+            hmac = import_fresh_module("hmac", blocked=["_hmac"])
+            c_module_name, c_method_name = "_hashlib", "hmac_new"
+
+        cext = import_helper.import_module(c_module_name)
+        cnew = getattr(cext, c_method_name)
+
+        bigkey = b'K' * size
+        bigmsg = b'M' * size
+
+        with patch.object(hmac, "_compute_digest_fallback") as slow:
+            with patch.object(cext, c_method_name, wraps=cnew) as new:
+                self.assertIsInstance(hmac.digest(bigkey, b'm', "md5"), bytes)
+            new.assert_called_once()
+            with patch.object(cext, c_method_name, wraps=cnew) as new:
+                self.assertIsInstance(hmac.digest(b'k', bigmsg, "md5"), bytes)
+            new.assert_called_once()
+        slow.assert_not_called()
+
+    @hashlib_helper.requires_hashdigest("md5", openssl=True)
+    @bigmemtest(size=_4G, memuse=2, dry_run=False)
+    def test_hmac_digest_no_overflow_error_in_fallback(self, size):
+        hmac = import_fresh_module("hmac", blocked=["_hashlib", "_hmac"])
+
+        for key, msg in [(b'K' * size, b'm'), (b'k', b'M' * size)]:
+            with self.subTest(keysize=len(key), msgsize=len(msg)):
+                with unittest.mock.patch.object(
+                    hmac, "_compute_digest_fallback",
+                    wraps=hmac._compute_digest_fallback,
+                ) as f:
+                    self.assertIsInstance(hmac.digest(key, msg, "md5"), bytes)
+                    f.assert_called_once()
+
 
 class BuiltinMiscellaneousTests(BuiltinModuleMixin, unittest.TestCase):
     """HMAC-BLAKE2 is not standardized as BLAKE2 is a keyed hash function.
@@ -1511,7 +1566,7 @@ class BuiltinMiscellaneousTests(BuiltinModuleMixin, unittest.TestCase):
     @classmethod
     def setUpClass(cls):
         super().setUpClass()
-        cls.blake2 = import_module("_blake2")
+        cls.blake2 = import_helper.import_module("_blake2")
         cls.blake2b = cls.blake2.blake2b
         cls.blake2s = cls.blake2.blake2s
 

Misc/NEWS.d/next/Library/2025-07-21-11-56-47.gh-issue-136912.zWosAL.rst

@@ -0,0 +1,3 @@
+:func:`hmac.digest` now properly rejects keys and messages whose length
+exceeds its capabililties. The exact limit is implementation-defined. Patch
+by Bénédikt Tran.