get BN.new("...", 0) working as OpenSSL does - using MPI format

Author: kares

src/main/java/org/jruby/ext/openssl/BN.java

@@ -115,8 +115,7 @@ public final BigInteger getValue() {
     }
 
     @JRubyMethod(name="initialize", required=1, optional=1, visibility = Visibility.PRIVATE)
-    public IRubyObject initialize(final ThreadContext context,
-        final IRubyObject[] args) {
+    public IRubyObject initialize(final ThreadContext context, final IRubyObject[] args) {
         final Ruby runtime = context.runtime;
         if (this.value != BigInteger.ZERO) { // already initialized
             throw newBNError(runtime, "illegal initialization");
@@ -126,14 +125,7 @@ public IRubyObject initialize(final ThreadContext context,
         final RubyString str = args[0].asString();
         switch (base) {
         case 0:
-            final byte[] bytes = str.getBytes(); final int signum;
-            if ( ( bytes[0] & 0x80 ) != 0 ) {
-                bytes[0] &= 0x7f; signum = -1;
-            }
-            else {
-                signum = 1;
-            }
-            this.value = new BigInteger(signum, bytes);
+            this.value = initBigIntegerMPI(runtime, str);
             break;
         case 2:
             // this seems wrong to me, but is the behavior of the
@@ -147,18 +139,44 @@ public IRubyObject initialize(final ThreadContext context,
         case 10:
         case 16:
             // here, the ASCII-encoded decimal or hex string is used
-            try {
-                this.value = new BigInteger(str.toString(), base);
-                break;
-            } catch (NumberFormatException e) {
-                throw runtime.newArgumentError("value " + str + " is not legal for radix " + base);
-            }
+            this.value = initBigIntegerBase(runtime, str, base);
+            break;
         default:
             throw runtime.newArgumentError("illegal radix: " + base);
         }
         return this;
     }
 
+    private static BigInteger initBigIntegerMPI(final Ruby runtime, RubyString str) {
+        final ByteList byteList = str.getByteList();
+        final int off = byteList.getBegin();
+        final byte[] b = byteList.getUnsafeBytes();
+
+        long len = ((b[off] & 0xFFl) << 24) | ((b[off+1] & 0xFF) << 16) | ((b[off+2] & 0xFF) << 8) | (b[off+3] & 0xFF);
+        final byte[] bytes = new byte[(int) len];
+        System.arraycopy(b, off + 4, bytes, 0, bytes.length);
+
+        final int signum;
+        if ( (bytes[0] & 0x80) == 0x80 ) {
+            signum = -1; bytes[0] &= 0x7f;
+        }
+        else {
+            signum = +1;
+        }
+        return new BigInteger(signum, bytes);
+
+    }
+
+    private static BigInteger initBigIntegerBase(final Ruby runtime, RubyString str, final int base) {
+        // here, the ASCII-encoded decimal or hex string is used
+        try {
+            return new BigInteger(str.toString(), base);
+        }
+        catch (NumberFormatException e) {
+            throw runtime.newArgumentError("value " + str + " is not legal for radix " + base);
+        }
+    }
+
     @Override
     public IRubyObject initialize_copy(final IRubyObject that) {
         super.initialize_copy(that);

src/test/ruby/test_bn.rb

@@ -76,4 +76,43 @@ def test_to_java
     assert_equal java.math.BigInteger.valueOf(24), OpenSSL::BN.new('24').to_java
   end if defined? JRUBY_VERSION
 
+  def test_new_str
+    e1 = OpenSSL::BN.new(999.to_s(16), 16) # OpenSSL::BN.new(str, 16) must be most stable
+    e2 = OpenSSL::BN.new((2**107-1).to_s(16), 16)
+    assert_equal(e1, OpenSSL::BN.new("999"))
+    assert_equal(e2, OpenSSL::BN.new((2**107-1).to_s))
+    assert_equal(e1, OpenSSL::BN.new("999", 10))
+    assert_equal(e2, OpenSSL::BN.new((2**107-1).to_s, 10))
+    assert_equal(e1, OpenSSL::BN.new("\x03\xE7", 2))
+    assert_equal(e2, OpenSSL::BN.new("\a\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF", 2))
+    assert_equal(e1, OpenSSL::BN.new("\x00\x00\x00\x02\x03\xE7", 0))
+    assert_equal(e2, OpenSSL::BN.new("\x00\x00\x00\x0E\a\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF", 0))
+  end
+
+  def test_new_bn
+    e1 = OpenSSL::BN.new(999.to_s(16), 16)
+    e2 = OpenSSL::BN.new((2**107-1).to_s(16), 16)
+    assert_equal(e1, OpenSSL::BN.new(e1))
+    assert_equal(e2, OpenSSL::BN.new(e2))
+  end
+
+  def test_new_integer
+    assert_equal(999.to_bn, OpenSSL::BN.new(999))
+    assert_equal((2 ** 107 - 1).to_bn, OpenSSL::BN.new(2 ** 107 - 1))
+    assert_equal(-999.to_bn, OpenSSL::BN.new(-999))
+    assert_equal((-(2 ** 107 - 1)).to_bn, OpenSSL::BN.new(-(2 ** 107 - 1)))
+  end
+
+  def test_to_bn
+    e1 = OpenSSL::BN.new(999.to_s(16), 16)
+    e2 = OpenSSL::BN.new((2**107-1).to_s(16), 16)
+    assert_equal(e1, 999.to_bn)
+    assert_equal(e2, (2**107-1).to_bn)
+  end
+
+  def test_prime_p
+    assert_equal(true, OpenSSL::BN.new((2 ** 107 - 1).to_s(16), 16).prime?)
+    assert_equal(true, OpenSSL::BN.new((2 ** 127 - 1).to_s(16), 16).prime?(1))
+  end
+
 end