From 8a19e26ba02c3440d9372524c1369ecc7e40bcc2 Mon Sep 17 00:00:00 2001 From: Keith Packard Date: Wed, 27 Apr 2022 22:39:36 -0700 Subject: [PATCH] vl53l0x: Perform 'abs' operation before scaling and dividing mcps diff value If the computed value is negative, then it's important to flip the sign before doing the division as that will not have the same result on an unsigned value as it would on a signed value in the same bits. Signed-off-by: Keith Packard --- sensor/vl53l0x/api/core/src/vl53l0x_api_core.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sensor/vl53l0x/api/core/src/vl53l0x_api_core.c b/sensor/vl53l0x/api/core/src/vl53l0x_api_core.c index 8f0632a1..679ff2ab 100644 --- a/sensor/vl53l0x/api/core/src/vl53l0x_api_core.c +++ b/sensor/vl53l0x/api/core/src/vl53l0x_api_core.c @@ -1861,8 +1861,8 @@ VL53L0X_Error VL53L0X_calc_sigma_estimate(VL53L0X_DEV Dev, * 500 is added to ensure rounding when integer division * truncates. */ - diff1_mcps = (((peakSignalRate_kcps << 16) - - 2 * xTalkCompRate_kcps) + 500)/1000; + diff1_mcps = abs((int32_t) ((((peakSignalRate_kcps << 16) - + 2 * xTalkCompRate_kcps) + 500)/1000)); /* vcselRate + xtalkCompRate */ diff2_mcps = ((peakSignalRate_kcps << 16) + 500)/1000; @@ -1872,7 +1872,7 @@ VL53L0X_Error VL53L0X_calc_sigma_estimate(VL53L0X_DEV Dev, diff1_mcps <<= 8; /* FixPoint0824/FixPoint1616 = FixPoint2408 */ - xTalkCorrection = abs(diff1_mcps/diff2_mcps); + xTalkCorrection = diff1_mcps/diff2_mcps; /* FixPoint2408 << 8 = FixPoint1616 */ xTalkCorrection <<= 8;