cmov: use masksel to impl CmovEq (#1343)

Uses the `masksel` function added in #1342 to impl `CmovEq`. This should
be less circuitious than the previous implementation.

This approach means we can get rid of all of the `test*` functions like
`testeq*`, `testne*`, and `testnz*` and implement everything in terms of
mask generation.

The pure Rust versions of `masknz32` and `masknz64` still internally use
`black_box` to prevent optimizations around the non-zero bit extracted
before computing the mask.

Author: tarcieri

cmov/src/portable.rs

@@ -54,12 +54,12 @@ impl Cmov for u32 {
 impl CmovEq for u32 {
     #[inline]
     fn cmovne(&self, rhs: &Self, input: Condition, output: &mut Condition) {
-        output.cmovnz(&input, testne32(*self, *rhs));
+        *output = masksel(*output, input, (maskne32(*self, *rhs) & 0xFF) as u8);
     }
 
     #[inline]
     fn cmoveq(&self, rhs: &Self, input: Condition, output: &mut Condition) {
-        output.cmovnz(&input, testeq32(*self, *rhs));
+        *output = masksel(*output, input, (maskeq32(*self, *rhs) & 0xFF) as u8);
     }
 }
 
@@ -78,25 +78,49 @@ impl Cmov for u64 {
 impl CmovEq for u64 {
     #[inline]
     fn cmovne(&self, rhs: &Self, input: Condition, output: &mut Condition) {
-        output.cmovnz(&input, testne64(*self, *rhs));
+        *output = masksel(*output, input, (maskne64(*self, *rhs) & 0xFF) as u8);
     }
 
     #[inline]
     fn cmoveq(&self, rhs: &Self, input: Condition, output: &mut Condition) {
-        output.cmovnz(&input, testeq64(*self, *rhs));
+        *output = masksel(*output, input, (maskeq64(*self, *rhs) & 0xFF) as u8);
     }
 }
 
+/// Returns `u32::MAX` if `x` is equal to `y`, otherwise returns `0` (32-bit version)
+fn maskeq32(x: u32, y: u32) -> u32 {
+    !maskne32(x, y)
+}
+
+/// Returns `u32::MAX` if `x` is equal to `y`, otherwise returns `0` (64-bit version)
+fn maskeq64(x: u64, y: u64) -> u64 {
+    !maskne64(x, y)
+}
+
+/// Returns `0` if `x` is equal to `y`, otherwise returns `1` (32-bit version)
+fn maskne32(x: u32, y: u32) -> u32 {
+    masknz32(x ^ y)
+}
+
+/// Returns `0` if `x` is equal to `y`, otherwise returns `1` (64-bit version)
+fn maskne64(x: u64, y: u64) -> u64 {
+    masknz64(x ^ y)
+}
+
 /// Return a [`u32::MAX`] mask if `condition` is non-zero, otherwise return zero for a zero input.
 #[cfg(not(target_arch = "arm"))]
 fn masknz32(condition: u32) -> u32 {
-    testnz32(condition).wrapping_neg()
+    let x = condition | condition.wrapping_neg(); // MSB of `x` now `1` if non-zero
+    let nz = core::hint::black_box(x >> (u32::BITS - 1)); // Extract MSB
+    nz.wrapping_neg()
 }
 
 /// Return a [`u64::MAX`] mask if `condition` is non-zero, otherwise return zero for a zero input.
 #[cfg(not(target_arch = "arm"))]
 fn masknz64(condition: u64) -> u64 {
-    testnz64(condition).wrapping_neg()
+    let x = condition | condition.wrapping_neg(); // MSB of `x` now `1` if non-zero
+    let nz = core::hint::black_box(x >> (u64::BITS - 1)); // Extract MSB
+    nz.wrapping_neg()
 }
 
 /// Optimized mask generation for ARM32 targets.
@@ -139,53 +163,76 @@ where
     (a & !mask) | (b & mask)
 }
 
-/// Returns `1` if `x` is equal to `y`, otherwise returns `0` (32-bit version)
-fn testeq32(x: u32, y: u32) -> Condition {
-    testne32(x, y) ^ 1
-}
-
-/// Returns `1` if `x` is equal to `y`, otherwise returns `0` (64-bit version)
-fn testeq64(x: u64, y: u64) -> Condition {
-    testne64(x, y) ^ 1
-}
+#[cfg(test)]
+mod tests {
+    // Spot check up to a given limit
+    const TEST_LIMIT: u32 = 65536;
 
-/// Returns `0` if `x` is equal to `y`, otherwise returns `1` (32-bit version)
-fn testne32(x: u32, y: u32) -> Condition {
-    (testnz32(x ^ y) & 0xFF) as Condition
-}
+    #[test]
+    fn maskeq32() {
+        assert_eq!(super::maskeq32(0, 0), u32::MAX);
+        assert_eq!(super::maskeq32(1, 0), 0);
+        assert_eq!(super::maskeq32(0, 1), 0);
+        assert_eq!(super::maskeq32(1, 1), u32::MAX);
+        assert_eq!(super::maskeq32(u32::MAX, 1), 0);
+        assert_eq!(super::maskeq32(1, u32::MAX), 0);
+        assert_eq!(super::maskeq32(u32::MAX, u32::MAX), u32::MAX);
+    }
 
-/// Returns `0` if `x` is equal to `y`, otherwise returns `1` (64-bit version)
-fn testne64(x: u64, y: u64) -> Condition {
-    (testnz64(x ^ y) & 0xFF) as Condition
-}
+    #[test]
+    fn maskeq64() {
+        assert_eq!(super::maskeq64(0, 0), u64::MAX);
+        assert_eq!(super::maskeq64(1, 0), 0);
+        assert_eq!(super::maskeq64(0, 1), 0);
+        assert_eq!(super::maskeq64(1, 1), u64::MAX);
+        assert_eq!(super::maskeq64(u64::MAX, 1), 0);
+        assert_eq!(super::maskeq64(1, u64::MAX), 0);
+        assert_eq!(super::maskeq64(u64::MAX, u64::MAX), u64::MAX);
+    }
 
-/// Returns `0` if `x` is `0`, otherwise returns `1` (32-bit version)
-fn testnz32(mut x: u32) -> u32 {
-    x |= x.wrapping_neg(); // MSB now set if non-zero
-    core::hint::black_box(x >> (u32::BITS - 1)) // Extract MSB
-}
+    #[test]
+    fn maskne32() {
+        assert_eq!(super::maskne32(0, 0), 0);
+        assert_eq!(super::maskne32(1, 0), u32::MAX);
+        assert_eq!(super::maskne32(0, 1), u32::MAX);
+        assert_eq!(super::maskne32(1, 1), 0);
+        assert_eq!(super::maskne32(u32::MAX, 1), u32::MAX);
+        assert_eq!(super::maskne32(1, u32::MAX), u32::MAX);
+        assert_eq!(super::maskne32(u32::MAX, u32::MAX), 0);
+    }
 
-/// Returns `0` if `x` is `0`, otherwise returns `1` (64-bit version)
-fn testnz64(mut x: u64) -> u64 {
-    x |= x.wrapping_neg(); // MSB now set if non-zero (or unset if zero)
-    core::hint::black_box(x >> (u64::BITS - 1)) // Extract MSB
-}
+    #[test]
+    fn maskne64() {
+        assert_eq!(super::maskne64(0, 0), 0);
+        assert_eq!(super::maskne64(1, 0), u64::MAX);
+        assert_eq!(super::maskne64(0, 1), u64::MAX);
+        assert_eq!(super::maskne64(1, 1), 0);
+        assert_eq!(super::maskne64(u64::MAX, 1), u64::MAX);
+        assert_eq!(super::maskne64(1, u64::MAX), u64::MAX);
+        assert_eq!(super::maskne64(u64::MAX, u64::MAX), 0);
+    }
 
-#[cfg(test)]
-mod tests {
     #[test]
     fn masknz32() {
         assert_eq!(super::masknz32(0), 0);
-        for i in 1..=u8::MAX {
-            assert_eq!(super::masknz32(i.into()), u32::MAX);
+        for i in 1..=TEST_LIMIT {
+            assert_eq!(super::masknz32(i), u32::MAX);
+        }
+
+        for i in (u32::MAX - TEST_LIMIT)..=u32::MAX {
+            assert_eq!(super::masknz32(i), u32::MAX);
         }
     }
 
     #[test]
     fn masknz64() {
         assert_eq!(super::masknz64(0), 0);
-        for i in 1..=u8::MAX {
-            assert_eq!(super::masknz64(i.into()), u64::MAX);
+        for i in 1..=(TEST_LIMIT as u64) {
+            assert_eq!(super::masknz64(i), u64::MAX);
+        }
+
+        for i in (u64::MAX - TEST_LIMIT as u64)..=u64::MAX {
+            assert_eq!(super::masknz64(i), u64::MAX);
         }
     }
 
@@ -196,65 +243,11 @@ mod tests {
 
         assert_eq!(super::masksel(17u32, 101077u32, 0u32), 17u32);
         assert_eq!(super::masksel(17u32, 101077u32, u32::MAX), 101077u32);
-    }
-
-    #[test]
-    fn testeq32() {
-        assert_eq!(super::testeq32(0, 0), 1);
-        assert_eq!(super::testeq32(1, 0), 0);
-        assert_eq!(super::testeq32(0, 1), 0);
-        assert_eq!(super::testeq32(1, 1), 1);
-        assert_eq!(super::testeq32(u32::MAX, 1), 0);
-        assert_eq!(super::testeq32(1, u32::MAX), 0);
-        assert_eq!(super::testeq32(u32::MAX, u32::MAX), 1);
-    }
-
-    #[test]
-    fn testeq64() {
-        assert_eq!(super::testeq64(0, 0), 1);
-        assert_eq!(super::testeq64(1, 0), 0);
-        assert_eq!(super::testeq64(0, 1), 0);
-        assert_eq!(super::testeq64(1, 1), 1);
-        assert_eq!(super::testeq64(u64::MAX, 1), 0);
-        assert_eq!(super::testeq64(1, u64::MAX), 0);
-        assert_eq!(super::testeq64(u64::MAX, u64::MAX), 1);
-    }
 
-    #[test]
-    fn testne32() {
-        assert_eq!(super::testne32(0, 0), 0);
-        assert_eq!(super::testne32(1, 0), 1);
-        assert_eq!(super::testne32(0, 1), 1);
-        assert_eq!(super::testne32(1, 1), 0);
-        assert_eq!(super::testne32(u32::MAX, 1), 1);
-        assert_eq!(super::testne32(1, u32::MAX), 1);
-        assert_eq!(super::testne32(u32::MAX, u32::MAX), 0);
-    }
-
-    #[test]
-    fn testne64() {
-        assert_eq!(super::testne64(0, 0), 0);
-        assert_eq!(super::testne64(1, 0), 1);
-        assert_eq!(super::testne64(0, 1), 1);
-        assert_eq!(super::testne64(1, 1), 0);
-        assert_eq!(super::testne64(u64::MAX, 1), 1);
-        assert_eq!(super::testne64(1, u64::MAX), 1);
-        assert_eq!(super::testne64(u64::MAX, u64::MAX), 0);
-    }
-
-    #[test]
-    fn testnz32() {
-        assert_eq!(super::testnz32(0), 0);
-        for i in 1..=u8::MAX {
-            assert_eq!(super::testnz32(i as u32), 1);
-        }
-    }
-
-    #[test]
-    fn testnz64() {
-        assert_eq!(super::testnz64(0), 0);
-        for i in 1..=u8::MAX {
-            assert_eq!(super::testnz64(i as u64), 1);
-        }
+        assert_eq!(super::masksel(129u64, 0xFFEEDDCCBBAA9988u64, 0u64), 129u64);
+        assert_eq!(
+            super::masksel(129u64, 0xFFEEDDCCBBAA9988u64, u64::MAX),
+            0xFFEEDDCCBBAA9988u64
+        );
     }
 }