Bluetooth: Host: fix aes ccm authentication

Bluetooth Host calculated authentication value correctly only
for data smaller than 255 bytes. If data is larger then
authentication transformation used wrong flags.
Since the issue was symmetric two Zephyr
based devices were able to understand each other. Hence,
other devices like Android or IOS smartphones weren't able
to authenticate large frames and broke communication.

Signed-off-by: Aleksandr Khromykh <[email protected]>

Author: alxelax

subsys/bluetooth/host/aes_ccm.c

@@ -35,9 +35,9 @@ static inline void xor16(uint8_t *dst, const uint8_t *a, const uint8_t *b)
 	dst[15] = a[15] ^ b[15];
 }
 
-/* pmsg is assumed to have the nonce already present in bytes 1-13 */
+/* b field is assumed to have the nonce already present in bytes 1-13 */
 static int ccm_calculate_X0(const uint8_t key[16], const uint8_t *aad, uint8_t aad_len,
-			    size_t mic_size, uint8_t msg_len, uint8_t b[16],
+			    size_t mic_size, uint16_t msg_len, uint8_t b[16],
 			    uint8_t X0[16])
 {
 	int i, j, err;
@@ -95,7 +95,7 @@ static int ccm_calculate_X0(const uint8_t key[16], const uint8_t *aad, uint8_t a
 }
 
 static int ccm_auth(const uint8_t key[16], uint8_t nonce[13],
-		    const uint8_t *cleartext_msg, size_t msg_len, const uint8_t *aad,
+		    const uint8_t *cleartext_msg, uint16_t msg_len, const uint8_t *aad,
 		    size_t aad_len, uint8_t *mic, size_t mic_size)
 {
 	uint8_t b[16], Xn[16], s0[16];
@@ -148,7 +148,7 @@ static int ccm_auth(const uint8_t key[16], uint8_t nonce[13],
 }
 
 static int ccm_crypt(const uint8_t key[16], const uint8_t nonce[13],
-		     const uint8_t *in_msg, uint8_t *out_msg, size_t msg_len)
+		     const uint8_t *in_msg, uint8_t *out_msg, uint16_t msg_len)
 {
 	uint8_t a_i[16], s_i[16];
 	uint16_t last_blk, blk_cnt;
@@ -192,7 +192,7 @@ int bt_ccm_decrypt(const uint8_t key[16], uint8_t nonce[13],
 {
 	uint8_t mic[16];
 
-	if (aad_len >= 0xff00 || mic_size > sizeof(mic)) {
+	if (aad_len >= 0xff00 || mic_size > sizeof(mic) || len > UINT16_MAX) {
 		return -EINVAL;
 	}
 
@@ -219,7 +219,7 @@ int bt_ccm_encrypt(const uint8_t key[16], uint8_t nonce[13],
 	BT_DBG("aad_len %zu mic_size %zu", aad_len, mic_size);
 
 	/* Unsupported AAD size */
-	if (aad_len >= 0xff00 || mic_size > 16) {
+	if (aad_len >= 0xff00 || mic_size > 16 || len > UINT16_MAX) {
 		return -EINVAL;
 	}