[nrf fromtree] Bluetooth: Host: Classic: Fix allocated PSM comparison

The code for checking for allocated BR/EDR PSMs was potentially resulting
in an integer overflow, due to doing a <= UINT16_MAX (0xffff) comparison
on a uint16_t variable. To avoid this, use a uint32_t variable internally.

Signed-off-by: Johan Hedberg <[email protected]>
(cherry picked from commit 90367c8)

Author: jhedberg

subsys/bluetooth/host/classic/l2cap_br.c

@@ -1154,6 +1154,45 @@ static void l2cap_br_conf_rsp(struct bt_l2cap_br *l2cap, uint8_t ident,
 }
 
 static int bt_l2cap_br_allocate_psm(uint16_t *psm)
+{
+	/* DYN_END is UINT16_MAX, so to be able to do a psm <= DYN_END comparison
+	 * we need to use uint32_t as the type.
+	 */
+	static uint32_t allocated_psm = L2CAP_BR_PSM_DYN_START;
+
+	if (allocated_psm < L2CAP_BR_PSM_DYN_END) {
+		allocated_psm = allocated_psm + 1;
+	} else {
+		goto failed;
+	}
+
+	for (; allocated_psm <= L2CAP_BR_PSM_DYN_END; allocated_psm++) {
+		/* Bluetooth Core Specification Version 6.0 | Vol 3, Part A, section 4.2
+		 *
+		 * The PSM field is at least two octets in length. All PSM values shall have
+		 * the least significant bit of the most significant octet equal to 0 and the
+		 * least significant bit of all other octets equal to 1.
+		 */
+		if ((allocated_psm & 0x0101) != 0x0001) {
+			continue;
+		}
+
+		if (l2cap_br_server_lookup_psm((uint16_t)allocated_psm)) {
+			LOG_DBG("PSM 0x%04x has been used", allocated_psm);
+			continue;
+		}
+
+		LOG_DBG("Allocated PSM 0x%04x for new server", allocated_psm);
+		*psm = (uint16_t)allocated_psm;
+		return 0;
+	}
+
+failed:
+	LOG_WRN("No free dynamic PSMs available");
+	return -EADDRNOTAVAIL;
+}
+
+int bt_l2cap_br_server_register(struct bt_l2cap_server *server)
 {
 	static uint16_t allocated_psm = L2CAP_BR_PSM_DYN_START;