math: Use unsigned ints when checking for 64-bit float == 1.0 or 2.0

A common pattern in the math code is to want to check for equality to
1.0, but using the raw bits of the value stored in two ints. That's
done with ((hi - 0x3ff00000) | lo) == 0, but 'hi' is often stored as a
signed value to make checking the sign bit easier. Do the subtraction
using unsigned values to avoid undefined behavior.

Signed-off-by: Keith Packard <[email protected]>

Author: keith-packard

newlib/libm/math/s_acos.c

@@ -65,7 +65,7 @@ acos64(__float64 x)
     if (ix >= 0x3ff00000) { /* |x| >= 1 */
         __uint32_t lx;
         GET_LOW_WORD(lx, x);
-        if (((ix - 0x3ff00000) | lx) == 0) { /* |x|==1 */
+        if ((((__uint32_t) ix - 0x3ff00000) | lx) == 0) { /* |x|==1 */
             if (hx > 0)
                 return _F_64(0.0); /* acos(1) = 0  */
             else

newlib/libm/math/s_acosh.c

@@ -48,7 +48,7 @@ acosh64(__float64 x)
             return x + x;
         } else
             return log64(x) + ln2; /* acosh(huge)=log(2x) */
-    } else if (((hx - 0x3ff00000) | lx) == 0) {
+    } else if ((((__uint32_t) hx - 0x3ff00000) | lx) == 0) {
         return _F_64(0.0); /* acosh(1) = 0 */
     } else if (hx > 0x40000000) { /* 2**28 > x > 2 */
         t = x * x;

newlib/libm/math/s_asin.c

@@ -73,7 +73,7 @@ asin64(__float64 x)
     if (ix >= 0x3ff00000) { /* |x|>= 1 */
         __uint32_t lx;
         GET_LOW_WORD(lx, x);
-        if (((ix - 0x3ff00000) | lx) == 0)
+        if ((((__uint32_t) ix - 0x3ff00000) | lx) == 0)
             /* asin(1)=+-pi/2 with inexact */
             return x * pio2_hi + x * pio2_lo;
         return __math_invalid(x); /* asin(|x|>1) is NaN */

newlib/libm/math/s_pow.c

@@ -118,7 +118,7 @@ pow64(__float64 x, __float64 y)
     /* x|y==NaN return NaN unless x==1 then return 1 */
     if (ix > 0x7ff00000 || ((ix == 0x7ff00000) && (lx != 0)) ||
         iy > 0x7ff00000 || ((iy == 0x7ff00000) && (ly != 0))) {
-        if (((hx - 0x3ff00000) | lx) == 0 && !issignaling64_inline(y))
+        if ((((__uint32_t) hx - 0x3ff00000) | lx) == 0 && !issignaling64_inline(y))
             return one;
         else
             return x + y;
@@ -150,7 +150,7 @@ pow64(__float64 x, __float64 y)
     /* special value of y */
     if (ly == 0) {
         if (iy == 0x7ff00000) { /* y is +-inf */
-            if (((ix - 0x3ff00000) | lx) == 0)
+            if ((((__uint32_t) ix - 0x3ff00000) | lx) == 0)
                 return one; /* +-1**+-inf = 1 */
             else if (ix >= 0x3ff00000) /* (|x|>1)**+-inf = inf,0 */
                 return (hy >= 0) ? y : zero;
@@ -181,7 +181,7 @@ pow64(__float64 x, __float64 y)
             if (hy < 0)
                 z = one / z; /* z = (1/|x|) */
             if (hx < 0) {
-                if (((ix - 0x3ff00000) | yisint) == 0) {
+                if ((((__uint32_t) ix - 0x3ff00000) | yisint) == 0) {
                     return __math_invalid(x); /* (-1)**non-int is NaN */
                 } else if (yisint == 1)
                     z = -z; /* (x<0)**odd = -(|x|**odd) */

newlib/libm/math/sr_lgamma.c

@@ -256,7 +256,7 @@ __math_lgamma_r(__float64 x, int *signgamp, int *divzero)
     }
 
     /* purge off 1 and 2 */
-    if ((((ix - 0x3ff00000) | lx) == 0) || (((ix - 0x40000000) | lx) == 0))
+    if (((((__uint32_t) ix - 0x3ff00000) | lx) == 0) || ((((__uint32_t) ix - 0x40000000) | lx) == 0))
         r = 0;
     /* for x < 2.0 */
     else if (ix < 0x40000000) {