[nrf fromtree] Bluetooth: Host: Fix SMP Pairing failed code on invalid Public Key

sjanc · HaavardRei · commit 58571f9d9c4f · 2025-08-05T10:16:45.000+02:00
Core Specification 6.1 clarified exptected erro code in case peer sending invalid Public Key. In case pairing is aborted during or immediately after Public Key Exchange phase has completed (which is the case here) expected reason code is set to "DHKey Check Failed". This was affecting SM/CEN/KDU/BI-04-C and SM/PER/KDU/BI-04-C qualification test cases. Signed-off-by: Szymon Janc <szymon.janc@codecoup.pl> (cherry picked from commit 9b27a47) Signed-off-by: Håvard Reierstad <haavard.reierstad@nordicsemi.no>
diff --git a/subsys/bluetooth/host/smp.c b/subsys/bluetooth/host/smp.c
@@ -4499,7 +4499,7 @@ static uint8_t smp_public_key(struct bt_smp *smp, struct net_buf *buf)
 		}
 	} else if (!bt_pub_key_is_valid(smp->pkey)) {
 		LOG_WRN("Received invalid public key");
-		return BT_SMP_ERR_INVALID_PARAMS;
+		return BT_SMP_ERR_DHKEY_CHECK_FAILED;
 	}
 
 	if (IS_ENABLED(CONFIG_BT_CENTRAL) &&

Original file line number	Diff line number	Diff line change
`@@ -4499,7 +4499,7 @@ static uint8_t smp_public_key(struct bt_smp smp, struct net_buf buf)`
`4499`	`4499`	`}`
`4500`	`4500`	`} else if (!bt_pub_key_is_valid(smp->pkey)) {`
`4501`	`4501`	`LOG_WRN("Received invalid public key");`
`4502`		`- return BT_SMP_ERR_INVALID_PARAMS;`
	`4502`	`+ return BT_SMP_ERR_DHKEY_CHECK_FAILED;`
`4503`	`4503`	`}`
`4504`	`4504`
`4505`	`4505`	`if (IS_ENABLED(CONFIG_BT_CENTRAL) &&`