28099 feat unittest assertclose

modules/packages/UnitTest.chpl

@@ -177,6 +177,7 @@ module UnitTest {
   use TestError;
   use List, Map;
   private use IO, IO.FormattedIO;
+  private use Math;
 
   @chpldoc.nodoc
   config const testNames: string = "None";
@@ -474,6 +475,195 @@ module UnitTest {
       checkAssertEquality(first, second);
     }
 
+    proc defaultRelTol param : real do return 1e-5;
+    proc defaultAbsTol param : real do return 0.0;
+
+    /*
+    Check if a scalar or array is within tolerance of the expected value.
+
+    :arg actual: actual value
+    :type actual: ?T or [] ?T
+
+    :arg expected: expected value
+    :type expected: T or [] T
+
+    :arg relTol: relative tolerance
+    :type relTol: real
+
+    :arg absTol: absolute tolerance
+    :type absTol: real
+
+    :arg equalNan: whether or not NaN should compare equal, defaults to true
+    :type equalNan: bool
+
+    :returns: True if within tolerance, else false
+    :rtype: bool or [] bool
+
+    :throws IllegalArgumentError: either tolerance is negative
+    */
+    pragma "insert line file info"
+    pragma "always propagate line file info"
+    @chpldoc.nodoc
+    proc withinTol(const actual: ?T, const expected: T,
+                   const in relTol: real=defaultRelTol,
+                   const in absTol: real=defaultAbsTol,
+                   const in equalNan: bool=true) throws {
+      if relTol < 0.0 || absTol < 0.0 {
+        throw new owned IllegalArgumentError(
+          "relTol and absTol must both be nonnegative.");
+      }
+
+      proc compareNanFalse(const actual_, const expected_) {
+        // Compare two values, returning false if at least one is NaN
+        return abs(actual_ - expected_) <= absTol + relTol * abs(expected_);
+      }
+
+      proc compareNanTrue(const actual_: real, const expected_: real): bool {
+        // Compare two values, returning true if both are NaN
+        return compareNanFalse(actual_, expected_) || (isNan(actual_) &&
+                                                       isNan(expected_));
+      }
+
+      proc compareComplex(const actual_: complex,
+                          const expected_: complex): bool {
+        // if real/imag part of either complex value is NaN, compare real
+        // and imaginary parts separately, such that, e.g., nan+0i == nan+0i
+        if isNan(actual_.re) || isNan(actual_.im)
+        || isNan(expected_.re) || isNan(expected_.im) {
+          return compareNanTrue(actual_.re, expected_.re)
+              && compareNanTrue(actual_.im, expected_.im);
+        }
+        // else compare complex numbers standard way, with absolute values
+        return compareNanFalse(actual_, expected_);
+      }
+
+      if equalNan {
+        if isComplexType(T) ||
+          (isArrayType(T) && isComplexType(actual.eltType)) {
+          return compareComplex(actual, expected);
+        }
+        if isImagType(T) || (isArrayType(T) && isImagType(actual.eltType)) {
+          // no imaginary NaN, must cast to real
+          return compareNanTrue(actual:real, expected:real);
+        }
+        return compareNanTrue(actual, expected);
+      }
+      return compareNanFalse(actual, expected);
+    }
+
+    /*
+      Assert that two arrays are, element-wise, approximately equal to within
+      the specified tolerances.
+
+      It asserts that actual <= absTol + relTol*expected for each element.
+
+      :arg actual: actual, user-computed value
+      :type actual: [] ?T
+
+      :arg expected: expected or desired value
+      :type expected: [] T
+
+      :arg relTol: relative tolerance
+      :type relTol: real
+
+      :arg absTol: absolute tolerance
+      :type absTol: real
+
+      :arg equalNan: whether or not NaN should compare equal, defaults to true
+      :type equalNan: bool
+
+      :throws IllegalArgumentError: If `actual` and `expected` are of different
+      shape.
+
+      :throws AssertionError: If `actual` is not approximately equal to
+      `expected` within the specified tolerance.
+    */
+    pragma "insert line file info"
+    pragma "always propagate line file info"
+    proc assertClose(const actual: [] ?T, const expected: [] T,
+                     const in relTol: real=defaultRelTol,
+                     const in absTol: real=defaultAbsTol,
+                     const in equalNan: bool=true): void throws
+                     where (isNumericType(T) && !isIntegralType(T)) {
+      if actual.shape != expected.shape {
+        throw new owned IllegalArgumentError(
+          "Actual array shape " + actual.shape:string
+        + " does not match expected: " + expected.shape:string
+        );
+      }
+      var isWithinTol = withinTol(actual=actual, expected=expected,
+                                  relTol=relTol, absTol=absTol,
+                                  equalNan=equalNan);
+      const passes = && reduce isWithinTol;
+      if !passes {
+        const numFailed: int = isWithinTol.count(false);
+        const numTotal: int = isWithinTol.size;
+        const percFailed: real = 100.0 * numFailed / numTotal;
+
+        var maxAbsErr = min(real);
+        var maxRelErr = min(real);
+        forall (a, e) in zip(actual, expected) with (max reduce maxAbsErr,
+                                                     max reduce maxRelErr) {
+          const absErr = abs(a - e);
+          const relErr = absErr / abs(e);
+          maxAbsErr reduce= absErr;
+          maxRelErr reduce= relErr;
+        }
+        throw new owned AssertionError(
+          "assert failed for %i/%i elements (%r%%)\n".format(numFailed,
+            numTotal, percFailed)
+          + "Max Rel. Error: %r%%\n".format(maxRelErr*100.0)
+          + "Max Abs. Error: %r".format(maxAbsErr)
+        );
+      }
+    }
+
+    /*
+      Assert that two numeric types are approximately equal to within
+      the specified tolerances.
+
+      It asserts that actual <= absTol + relTol*expected.
+
+      :arg actual: actual, user-computed value
+      :type actual: T
+
+      :arg expected: expected or desired value
+      :type expected: T
+
+      :arg relTol: relative tolerance
+      :type relTol: real
+
+      :arg absTol: absolute tolerance
+      :type absTol: real
+
+      :arg equalNan: whether or not NaN should compare equal, defaults to true
+      :type equalNan: bool
+
+      :throws AssertionError: If `actual` is not approximately equal to
+      `expected` within the specified tolerance.
+    */
+    pragma "insert line file info"
+    pragma "always propagate line file info"
+    proc assertClose(const in actual: ?T, const in expected: T,
+                     const in relTol: real=defaultRelTol,
+                     const in absTol: real=defaultAbsTol,
+                     const in equalNan: bool=true):void throws
+                     where (isNumericType(T) && !isIntegralType(T)) {
+      var isWithinTol = withinTol(actual=actual, expected=expected,
+                                  relTol=relTol, absTol=absTol,
+                                  equalNan=equalNan);
+      if !isWithinTol {
+        const absErr: real = abs(actual - expected);
+        const relErr = absErr / abs(expected);
+        throw new owned UnitTest.TestError.AssertionError(
+          "assert failed for actual=%n, expected=%n\n".format(actual, expected)
+          + "Rel. Error: %r %%\n".format(relErr*100.0)
+          + "Abs. Error: %r".format(absErr)
+        );
+      }
+    }
+
+
     /*
       Assert that x matches the regular expression pattern.
 

test/library/packages/UnitTest/AssertClose/AssertCloseTest.chpl

@@ -0,0 +1,129 @@
+use UnitTest;
+
+proc testAssert(actual, expected, absTol=0.0, relTol=1e-15,
+                equalNan=true, reason="") {
+  var sep = "=="*40;
+  var test = new Test();
+  try {
+    test.assertClose(actual=actual, expected=expected, absTol=absTol,
+                     relTol=relTol, equalNan=equalNan);
+  } catch e {
+    writeln("Error Caught in "+reason);
+    writeln(e);
+    writeln(sep);
+  }
+}
+
+
+proc main() {
+  equalReal();
+  equalRealArray();
+  equalImag();
+  equalComplex();
+  equalComplexArray();
+
+  equalNanReal();
+  equalNanImag();
+  equalNanComplex();
+
+  negativeRelTol();
+  negativeAbsTol();
+  mismatchedArraySize();
+
+  unequalReal();
+  unequalComplex();
+  unequalComplexArray();
+
+  unequalNanReal();
+  unequalNanImag();
+  unequalNanComplex();
+}
+
+proc equalReal() {
+  testAssert(1.8, 2.0, relTol=0.1, reason="Equal real numbers");
+}
+
+proc equalRealArray() {
+  var a: [0..4] real = [.3, .31, .35, -.1, -1e-9];
+  var e: [1..5] real = [.3, .32, .6, -.2, -2e-9];
+  testAssert(a, e, absTol=0.3, relTol=0.0, reason="Equal real 1D arrays");
+}
+
+proc equalImag() {
+  testAssert(1.8i, 2.0i, relTol=0.1, reason="Equal imag numbers");
+}
+proc equalComplex() {
+  testAssert(-13+2.5i, -13+2.4i, relTol=8e-3, reason="Equal complex numbers");
+}
+proc equalComplexArray() {
+  var a: [0..2, 0..1] complex = [2+0.99i, 1+1i; -3+1i, 5+1i; 6+1i, 7+1i];
+  var e: [1..3, 0..1] complex = [2+1i, 1+1i; -2.99+1i, 5+1i; 6+1i, 7+1i];
+  testAssert(a, e, relTol=0.01, reason="Equal complex 2D arrays");
+}
+
+proc equalNanReal() {
+  testAssert(nan, nan, reason="Equal real NaN");
+  testAssert([3.1, nan, 6.5], [4.1, nan, 6.5], absTol=1.0, relTol=0.0,
+             reason="Equal real NaN array");
+}
+proc equalNanImag() {
+  testAssert(nan*1i, nan*1i, reason="Equal imag NaN");
+  testAssert([nan*1i, 3.5i], [nan*1i, 3.9i], absTol=0.5, relTol=0.0,
+             reason="Equal imag NaN array");
+}
+proc equalNanComplex() {
+  testAssert(nan+nan*1i, nan+nan*1i, reason="Equal complex NaN");
+  testAssert(1.0+nan*1i, 1.0+nan*1i, reason="Equal complex partial NaN array");
+  testAssert([nan+nan*1i, 14+9i], [nan+nan*1i, 16+9i],
+             absTol=2.0, relTol=0.0, reason="Equal complex NaN array");
+}
+
+proc negativeRelTol() {
+  testAssert(1.5, 1.6, absTol=-0.1, reason="Negative relative tolerance");
+}
+
+proc negativeAbsTol() {
+  testAssert(1.5, 1.6, absTol=0.1, relTol=-1e-15,
+             reason="Negative absolute tolerance");
+}
+
+proc mismatchedArraySize() {
+  var a = [.3, .31, .35, -.1, -1e-9];
+  var e = [.3, .32, .35, -.1];
+  testAssert(a, e, reason="Mismatched array sizes");
+}
+
+proc unequalReal() {
+  testAssert(0.0, 1e-6, absTol=9.9e-7, reason="Unequal real numbers");
+}
+proc unequalComplex() {
+  testAssert(1.8+3i, 1.8+3.000001i, reason="Unequal complex numbers");
+}
+
+proc unequalComplexArray() {
+  var a = [.3+1i, 2-3i];
+  var e = [.3+1i, 3-2i];
+  testAssert(a, e, absTol=1.0, relTol=0.0, reason="Unequal complex 1D arrays");
+}
+
+proc unequalNanReal() {
+  testAssert(nan, 2.5, reason="Unequal real NaN, equalNan true");
+  testAssert(nan, 2.5, equalNan=false,
+             reason="Unequal real NaN, equalNan false");
+  testAssert([3.1, nan, 6.5], [3.1, nan, 6.5], equalNan=false,
+             reason="Unequal real array NaN");
+}
+
+proc unequalNanImag() {
+  testAssert(nan*1i, nan*1i, equalNan=false,
+             reason="Unequal imag NaN");
+  testAssert([3.1i, 14i, 6.5i], [4.1i, nan*1i, 6.5i],
+             absTol=1.0, relTol=0.0, reason="Unequal imag array NaN");
+}
+
+proc unequalNanComplex() {
+  testAssert(nan+2.0i, 2.5+2.0i, reason="Unequal complex partial NaN, real");
+  testAssert(1.0+nan*1i, 1.0+2.5i, reason="Unequal complex partial NaN, imag");
+  testAssert(nan+nan*1i, nan+nan*1i, equalNan=false,
+             reason="Unequal complex NaN");
+}

test/library/packages/UnitTest/AssertClose/AssertCloseTest.good

@@ -0,0 +1,64 @@
+Error Caught in Negative relative tolerance
+IllegalArgumentError: relTol and absTol must both be nonnegative.
+================================================================================
+Error Caught in Negative absolute tolerance
+IllegalArgumentError: relTol and absTol must both be nonnegative.
+================================================================================
+Error Caught in Mismatched array sizes
+IllegalArgumentError: Actual array shape (5,) does not match expected: (4,)
+================================================================================
+Error Caught in Unequal real numbers
+AssertionError: in AssertCloseTest.chpl:8 - assert failed for actual=0, expected=1e-06
+Rel. Error: 100 %
+Abs. Error: 1e-06
+================================================================================
+Error Caught in Unequal complex numbers
+AssertionError: in AssertCloseTest.chpl:8 - assert failed for actual=1.8 + 3i, expected=1.8 + 3i
+Rel. Error: 2.85831e-05 %
+Abs. Error: 1e-06
+================================================================================
+Error Caught in Unequal complex 1D arrays
+AssertionError: in AssertCloseTest.chpl:8 - assert failed for 1/2 elements (50%)
+Max Rel. Error: 39.2232%
+Max Abs. Error: 1.41421
+================================================================================
+Error Caught in Unequal real NaN, equalNan true
+AssertionError: in AssertCloseTest.chpl:8 - assert failed for actual=nan, expected=2.5
+Rel. Error: nan %
+Abs. Error: nan
+================================================================================
+Error Caught in Unequal real NaN, equalNan false
+AssertionError: in AssertCloseTest.chpl:8 - assert failed for actual=nan, expected=2.5
+Rel. Error: nan %
+Abs. Error: nan
+================================================================================
+Error Caught in Unequal real array NaN
+AssertionError: in AssertCloseTest.chpl:8 - assert failed for 1/3 elements (33.3333%)
+Max Rel. Error: nan%
+Max Abs. Error: nan
+================================================================================
+Error Caught in Unequal imag NaN
+AssertionError: in AssertCloseTest.chpl:8 - assert failed for actual=nani, expected=nani
+Rel. Error: nan %
+Abs. Error: nan
+================================================================================
+Error Caught in Unequal imag array NaN
+AssertionError: in AssertCloseTest.chpl:8 - assert failed for 1/3 elements (33.3333%)
+Max Rel. Error: nan%
+Max Abs. Error: nan
+================================================================================
+Error Caught in Unequal complex partial NaN, real
+AssertionError: in AssertCloseTest.chpl:8 - assert failed for actual=nan, expected=2.5 + 2i
+Rel. Error: nan %
+Abs. Error: nan
+================================================================================
+Error Caught in Unequal complex partial NaN, imag
+AssertionError: in AssertCloseTest.chpl:8 - assert failed for actual=nan, expected=1 + 2.5i
+Rel. Error: nan %
+Abs. Error: nan
+================================================================================
+Error Caught in Unequal complex NaN
+AssertionError: in AssertCloseTest.chpl:8 - assert failed for actual=nan, expected=nan
+Rel. Error: nan %
+Abs. Error: nan
+================================================================================