cmov: extract maskgen32 and maskgen64 in portable backend (#1335)

Extracts some functions in the portable backend that can both be easily
tested and easily substituted with some platform-specific idealized
assembly for emulating CMOVNZ that everything else can build on top of.

Also adds tests in CI for an `armv7-unknown-linux-gnueabi` target we can
use for verifying correct behavior on 32-bit platforms.

Author: tarcieri

.github/workflows/cmov.yml

@@ -101,6 +101,11 @@ jobs:
     strategy:
       matrix:
         include:
+          # ARM32
+          - target: armv7-unknown-linux-gnueabi
+            rust: 1.85.0 # MSRV
+          - target: armv7-unknown-linux-gnueabi
+            rust: stable # MSRV
           # ARM64
           - target: aarch64-unknown-linux-gnu
             rust: 1.85.0 # MSRV
@@ -130,6 +135,7 @@ jobs:
     strategy:
       matrix:
         target:
+          - armv7-unknown-linux-gnueabi
           - powerpc-unknown-linux-gnu
           - s390x-unknown-linux-gnu
           - x86_64-unknown-linux-gnu

cmov/src/portable.rs

@@ -7,81 +7,80 @@
 // TODO(tarcieri): more optimized implementation for small integers
 
 use crate::{Cmov, CmovEq, Condition};
-use core::hint::black_box;
 
 /// Bitwise non-zero: returns `1` if `x != 0`, and otherwise returns `0`.
 macro_rules! bitnz {
     ($value:expr, $bits:expr) => {
-        black_box(($value | $value.wrapping_neg()) >> ($bits - 1))
+        core::hint::black_box(($value | $value.wrapping_neg()) >> ($bits - 1))
     };
 }
 
 impl Cmov for u16 {
     #[inline]
     fn cmovnz(&mut self, value: &Self, condition: Condition) {
-        let mut tmp = *self as u64;
-        tmp.cmovnz(&(*value as u64), condition);
+        let mut tmp = *self as u32;
+        tmp.cmovnz(&(*value as u32), condition);
         *self = tmp as u16;
     }
 
     #[inline]
     fn cmovz(&mut self, value: &Self, condition: Condition) {
-        let mut tmp = *self as u64;
-        tmp.cmovz(&(*value as u64), condition);
+        let mut tmp = *self as u32;
+        tmp.cmovz(&(*value as u32), condition);
         *self = tmp as u16;
     }
 }
 
 impl CmovEq for u16 {
     #[inline]
     fn cmovne(&self, rhs: &Self, input: Condition, output: &mut Condition) {
-        (*self as u64).cmovne(&(*rhs as u64), input, output);
+        (*self as u32).cmovne(&(*rhs as u32), input, output);
     }
 
     #[inline]
     fn cmoveq(&self, rhs: &Self, input: Condition, output: &mut Condition) {
-        (*self as u64).cmoveq(&(*rhs as u64), input, output);
+        (*self as u32).cmoveq(&(*rhs as u32), input, output);
     }
 }
 
 impl Cmov for u32 {
     #[inline]
     fn cmovnz(&mut self, value: &Self, condition: Condition) {
-        let mut tmp = *self as u64;
-        tmp.cmovnz(&(*value as u64), condition);
-        *self = tmp as u32;
+        let mask = nzmask32(condition);
+        *self = (*self & !mask) | (*value & mask);
     }
 
     #[inline]
     fn cmovz(&mut self, value: &Self, condition: Condition) {
-        let mut tmp = *self as u64;
-        tmp.cmovz(&(*value as u64), condition);
-        *self = tmp as u32;
+        let mask = nzmask32(condition);
+        *self = (*self & mask) | (*value & !mask);
     }
 }
 
 impl CmovEq for u32 {
     #[inline]
     fn cmovne(&self, rhs: &Self, input: Condition, output: &mut Condition) {
-        (*self as u64).cmovne(&(*rhs as u64), input, output);
+        let ne = bitnz!(self ^ rhs, u32::BITS) as u8;
+        output.cmovnz(&input, ne);
     }
 
     #[inline]
     fn cmoveq(&self, rhs: &Self, input: Condition, output: &mut Condition) {
-        (*self as u64).cmoveq(&(*rhs as u64), input, output);
+        let ne = bitnz!(self ^ rhs, u32::BITS) as u8;
+        output.cmovnz(&input, ne ^ 1);
     }
 }
 
 impl Cmov for u64 {
     #[inline]
     fn cmovnz(&mut self, value: &Self, condition: Condition) {
-        let mask = (bitnz!(condition, u8::BITS) as u64).wrapping_sub(1);
-        *self = (*self & mask) | (*value & !mask);
+        let mask = nzmask64(condition);
+        *self = (*self & !mask) | (*value & mask);
     }
 
     #[inline]
     fn cmovz(&mut self, value: &Self, condition: Condition) {
-        let mask = (1 ^ bitnz!(condition, u8::BITS) as u64).wrapping_sub(1);
+        let mask = nzmask64(condition);
         *self = (*self & mask) | (*value & !mask);
     }
 }
@@ -99,3 +98,40 @@ impl CmovEq for u64 {
         output.cmovnz(&input, ne ^ 1);
     }
 }
+
+/// Return a [`u32::MAX`] mask if `condition` is non-zero, otherwise return zero for a zero input.
+pub fn nzmask32(condition: Condition) -> u32 {
+    bitnz!(condition as u32, u32::BITS).wrapping_neg()
+}
+
+/// Return a [`u64::MAX`] mask if `condition` is non-zero, otherwise return zero for a zero input.
+pub fn nzmask64(condition: Condition) -> u64 {
+    bitnz!(condition as u64, u64::BITS).wrapping_neg()
+}
+
+#[cfg(test)]
+mod tests {
+    #[test]
+    fn bitnz() {
+        assert_eq!(bitnz!(0u8, u8::BITS), 0);
+        for i in 1..=u8::MAX {
+            assert_eq!(bitnz!(i, u8::BITS), 1);
+        }
+    }
+
+    #[test]
+    fn nzmask32() {
+        assert_eq!(super::nzmask32(0), 0);
+        for i in 1..=u8::MAX {
+            assert_eq!(super::nzmask32(i), u32::MAX);
+        }
+    }
+
+    #[test]
+    fn nzmask64() {
+        assert_eq!(super::nzmask64(0), 0);
+        for i in 1..=u8::MAX {
+            assert_eq!(super::nzmask64(i), u64::MAX);
+        }
+    }
+}