[Bug #20654] Fix floor and ceil when ndigits is large (ruby#11277)

* Fix floor when ndigits is large

[Bug #20654]

This commit fixes Integer#floor and Float#floor when the number is
negative and ndigits is large such that 10**ndigits is a bignum.

Previously, it would return 0 in such cases. However, this would cause
unexpected behaviour such as:

    puts -1.floor(-5) # => -100000
    puts -1.floor(-10) # => -10000000000
    puts -1.floor(-20) # => 0

This commit changes the last result so that it will return
-100000000000000000000.

* Fix ceil when ndigits is large

[Bug #20654]

This commit fixes Integer#ceil and Float#ceil when the number is
negative and ndigits is large such that 10**ndigits is a bignum.

Previously, it would return 0 in such cases. However, this would cause
unexpected behaviour such as:

    puts 1.ceil(-5) # => 100000
    puts 1.ceil(-10) # => 10000000000
    puts 1.ceil(-20) # => 0

This commit changes the last result so that it will return
100000000000000000000.

Author: peterzhu2118

numeric.c

@@ -2373,11 +2373,7 @@ rb_int_round(VALUE num, int ndigits, enum ruby_num_rounding_mode mode)
 static VALUE
 rb_int_floor(VALUE num, int ndigits)
 {
-    VALUE f;
-
-    if (int_round_zero_p(num, ndigits))
-        return INT2FIX(0);
-    f = int_pow(10, -ndigits);
+    VALUE f = int_pow(10, -ndigits);
     if (FIXNUM_P(num) && FIXNUM_P(f)) {
         SIGNED_VALUE x = FIX2LONG(num), y = FIX2LONG(f);
         int neg = x < 0;
@@ -2386,21 +2382,19 @@ rb_int_floor(VALUE num, int ndigits)
         if (neg) x = -x;
         return LONG2NUM(x);
     }
-    if (RB_FLOAT_TYPE_P(f)) {
-        /* then int_pow overflow */
-        return INT2FIX(0);
+    else {
+        bool neg = int_neg_p(num);
+        if (neg) num = rb_int_minus(rb_int_plus(rb_int_uminus(num), f), INT2FIX(1));
+        num = rb_int_mul(rb_int_div(num, f), f);
+        if (neg) num = rb_int_uminus(num);
+        return num;
     }
-    return rb_int_minus(num, rb_int_modulo(num, f));
 }
 
 static VALUE
 rb_int_ceil(VALUE num, int ndigits)
 {
-    VALUE f;
-
-    if (int_round_zero_p(num, ndigits))
-        return INT2FIX(0);
-    f = int_pow(10, -ndigits);
+    VALUE f = int_pow(10, -ndigits);
     if (FIXNUM_P(num) && FIXNUM_P(f)) {
         SIGNED_VALUE x = FIX2LONG(num), y = FIX2LONG(f);
         int neg = x < 0;
@@ -2410,11 +2404,16 @@ rb_int_ceil(VALUE num, int ndigits)
         if (neg) x = -x;
         return LONG2NUM(x);
     }
-    if (RB_FLOAT_TYPE_P(f)) {
-        /* then int_pow overflow */
-        return INT2FIX(0);
+    else {
+        bool neg = int_neg_p(num);
+        if (neg)
+            num = rb_int_uminus(num);
+        else
+            num = rb_int_plus(num, rb_int_minus(f, INT2FIX(1)));
+        num = rb_int_mul(rb_int_div(num, f), f);
+        if (neg) num = rb_int_uminus(num);
+        return num;
     }
-    return rb_int_plus(num, rb_int_minus(f, rb_int_modulo(num, f)));
 }
 
 VALUE

test/ruby/test_float.rb

@@ -530,6 +530,10 @@ def test_floor_with_precision
     assert_raise(TypeError) {1.0.floor(nil)}
     def (prec = Object.new).to_int; 2; end
     assert_equal(0.99, 0.998.floor(prec))
+
+    assert_equal(-10000000000, -1.0.floor(-10), "[Bug #20654]")
+    assert_equal(-100000000000000000000, -1.0.floor(-20), "[Bug #20654]")
+    assert_equal(-100000000000000000000000000000000000000000000000000, -1.0.floor(-50), "[Bug #20654]")
   end
 
   def test_ceil_with_precision
@@ -557,6 +561,10 @@ def test_ceil_with_precision
     assert_raise(TypeError) {1.0.ceil(nil)}
     def (prec = Object.new).to_int; 2; end
     assert_equal(0.99, 0.981.ceil(prec))
+
+    assert_equal(10000000000, 1.0.ceil(-10), "[Bug #20654]")
+    assert_equal(100000000000000000000, 1.0.ceil(-20), "[Bug #20654]")
+    assert_equal(100000000000000000000000000000000000000000000000000, 1.0.ceil(-50), "[Bug #20654]")
   end
 
   def test_truncate_with_precision

test/ruby/test_integer.rb

@@ -465,6 +465,10 @@ def test_floor
 
     assert_int_equal(1111_1111_1111_1111_1111_1111_1111_1111, 1111_1111_1111_1111_1111_1111_1111_1111.floor(1))
     assert_int_equal(10**400, (10**400).floor(1))
+
+    assert_int_equal(-10000000000, -1.floor(-10), "[Bug #20654]")
+    assert_int_equal(-100000000000000000000, -1.floor(-20), "[Bug #20654]")
+    assert_int_equal(-100000000000000000000000000000000000000000000000000, -1.floor(-50), "[Bug #20654]")
   end
 
   def test_ceil
@@ -493,6 +497,10 @@ def test_ceil
 
     assert_int_equal(1111_1111_1111_1111_1111_1111_1111_1111, 1111_1111_1111_1111_1111_1111_1111_1111.ceil(1))
     assert_int_equal(10**400, (10**400).ceil(1))
+
+    assert_int_equal(10000000000, 1.ceil(-10), "[Bug #20654]")
+    assert_int_equal(100000000000000000000, 1.ceil(-20), "[Bug #20654]")
+    assert_int_equal(100000000000000000000000000000000000000000000000000, 1.ceil(-50), "[Bug #20654]")
   end
 
   def test_truncate