Raise error if random(ZZ) called w/ nonpositive height

Previously, we silently ignored the negative sign when height was
negative, but this was undocumented, and crashed when the height was
zero.

Author: d-torrance

M2/Macaulay2/d/interface.dd

@@ -47,8 +47,8 @@ setupfun("testCatch",testCatch);
 
 export rawRandomZZ(e:Expr):Expr := (
      when e
-     is Nothing do toExpr(Ccode(ZZ, "rawRandomInteger(", "NULL)"))
-     is maxN:ZZcell do toExpr(Ccode(ZZ, "rawRandomInteger(", maxN.v, ")"))
+     is Nothing do toExpr(Ccode(ZZorNull, "rawRandomInteger(", "NULL)"))
+     is maxN:ZZcell do toExpr(Ccode(ZZorNull, "rawRandomInteger(", maxN.v, ")"))
      else WrongArgZZ());
 setupfun("rawRandomZZ",rawRandomZZ);
 export rawFareyApproximation(e:Expr):Expr := (

M2/Macaulay2/e/interface/random.cpp

@@ -56,14 +56,11 @@ int32_t rawRandomInt(int32_t max)
 void rawSetRandomInteger(mpz_ptr result, gmp_ZZ maxN)
 /* if height is the null pointer, use the default height */
 {
-  if (maxN == nullptr)
-    mpz_urandomm(result, state, maxHeight);
-  else if (1 != mpz_sgn(maxN))
-    {
-      mpz_set_si(result, 0);
-    }
-  else
-    mpz_urandomm(result, state, maxN);
+  if (maxN == nullptr) maxN = maxHeight;
+  if (mpz_cmp_si(maxN, 0) <= 0)
+    throw exc::engine_error("expected a positive height");
+
+  mpz_urandomm(result, state, maxN);
 }
 
 gmp_ZZ rawRandomInteger(gmp_ZZ maxN)
@@ -72,7 +69,12 @@ gmp_ZZ rawRandomInteger(gmp_ZZ maxN)
   mpz_ptr result = getmemstructtype(mpz_ptr);
   mpz_init(result);
 
-  rawSetRandomInteger(result, maxN);
+  try {
+    rawSetRandomInteger(result, maxN);
+  } catch (const exc::engine_error& e) {
+    ERROR(e.what());
+    return nullptr;
+  }
 
   mpz_reallocate_limbs(result);
   return result;

M2/Macaulay2/tests/normal/randommat.m2

@@ -34,3 +34,6 @@ assert isSurjective random(R^3,R^6,MaximalRank=>true)
 assert(random(ZZ^2, ZZ^2, MaximalRank => true) - id_(ZZ^2) != 0)
 assert(random(QQ^2, QQ^2, MaximalRank => true) - id_(QQ^2) != 0)
 assert(random(R^2,  R^2,  MaximalRank => true) - id_(R^2)  != 0)
+
+-- used to crash M2 (#2089)
+assert try random(ZZ^2, ZZ^2, Height => 0) then false else true