Improve ARM64 compatibility (#3447)

oleksiyskononenko · samukweku · commit 2fad60d51c83 · 2023-03-31T12:52:45.000+11:00
With changes on this PR datatable builds and passes all the tests on ARM platform. Since on this platform `long double` is identical to `double`, we had to - use tolerance `1e-15` for float tests; - improve handling of very small and very large floats when casting/freading similar to Rdatatable/data.table@ff3e7d4. WIP for #3222
diff --git a/docs/releases/v1.1.0.rst b/docs/releases/v1.1.0.rst
@@ -170,6 +170,8 @@
     -[enh] Header detection heuristics has been improved in the case when
         some of the column names are missing. [#3363]
 
+    -[enh] Improved handling of very small and very large float values. [#3447]
+
     -[fix] :func:`fread()` will no longer fail while reading mostly empty
         files. [#3055]
 
diff --git a/src/core/column/cast_date32.cc b/src/core/column/cast_date32.cc
@@ -1,5 +1,5 @@
 //------------------------------------------------------------------------------
-// Copyright 2021 H2O.ai
+// Copyright 2021-2023 H2O.ai
 //
 // Permission is hereby granted, free of charge, to any person obtaining a
 // copy of this software and associated documentation files (the "Software"),
@@ -25,7 +25,6 @@
 #include "column/cast.h"
 #include "python/date.h"
 #include "python/string.h"
-#include "read/constants.h"  // dt::read::pow10lookup
 namespace dt {
 
 
diff --git a/src/core/column/cast_string.cc b/src/core/column/cast_string.cc
@@ -1,5 +1,5 @@
 //------------------------------------------------------------------------------
-// Copyright 2020-2021 H2O.ai
+// Copyright 2020-2023 H2O.ai
 //
 // Permission is hereby granted, free of charge, to any person obtaining a
 // copy of this software and associated documentation files (the "Software"),
@@ -192,12 +192,23 @@ static bool parse_double(const char* ch, const char* end, double* out) {
     }
     e += Eneg? -exp : exp;
   }
-  e += 350; // lookup table is arranged from -350 (0) to +350 (700)
-  if (e < 0 || e > 700 || ch != end) return false;
 
-  double r = static_cast<double>(static_cast<long double>(mantissa) *
-                                 dt::read::pow10lookup[e]);
-  *out = negative? -r : r;
+  if (e < -350 || e > 350 || ch != end) return false;
+  auto r = static_cast<long double>(mantissa);
+
+  // For very small and very large floats do a separate lookup
+  // for extra exponent, i.e. anything above 300 or below -300.
+  // Note that the lookup array indices go from 0 (e == -300)
+  // to 600 (e == 300). The approach is based on
+  // https://github.com/Rdatatable/data.table/pull/4165
+  if (e < -300 || e > 300) {
+    auto extra = static_cast<int_fast8_t>(e - copysign(300, e));
+    r *= dt::read::pow10lookup[extra + 300];
+    e -= extra;
+  }
+  r *= dt::read::pow10lookup[e + 300];
+  *out = static_cast<double>(negative? -r : r);
+
   return true;
 }
 
diff --git a/src/core/read/constants.cc b/src/core/read/constants.cc
@@ -3,7 +3,7 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this
 // file, You can obtain one at http://mozilla.org/MPL/2.0/.
 //
-// © H2O.ai 2018
+// © H2O.ai 2018-2023
 //------------------------------------------------------------------------------
 #include <limits>
 #include "read/constants.h"
@@ -66,18 +66,7 @@ const uint8_t allowedseps[128] = {
 };
 
 
-#if DT_OS_WINDOWS
-  #define LDBL(value) std::numeric_limits<double>::max()
-#else
-  #define LDBL(value) value
-#endif
-
-const long double pow10lookup[701] = {
-  1.0E-350L, 1.0E-349L, 1.0E-348L, 1.0E-347L, 1.0E-346L, 1.0E-345L, 1.0E-344L, 1.0E-343L, 1.0E-342L, 1.0E-341L,
-  1.0E-340L, 1.0E-339L, 1.0E-338L, 1.0E-337L, 1.0E-336L, 1.0E-335L, 1.0E-334L, 1.0E-333L, 1.0E-332L, 1.0E-331L,
-  1.0E-330L, 1.0E-329L, 1.0E-328L, 1.0E-327L, 1.0E-326L, 1.0E-325L, 1.0E-324L, 1.0E-323L, 1.0E-322L, 1.0E-321L,
-  1.0E-320L, 1.0E-319L, 1.0E-318L, 1.0E-317L, 1.0E-316L, 1.0E-315L, 1.0E-314L, 1.0E-313L, 1.0E-312L, 1.0E-311L,
-  1.0E-310L, 1.0E-309L, 1.0E-308L, 1.0E-307L, 1.0E-306L, 1.0E-305L, 1.0E-304L, 1.0E-303L, 1.0E-302L, 1.0E-301L,
+const long double pow10lookup[601] = {
   1.0E-300L, 1.0E-299L, 1.0E-298L, 1.0E-297L, 1.0E-296L, 1.0E-295L, 1.0E-294L, 1.0E-293L, 1.0E-292L, 1.0E-291L,
   1.0E-290L, 1.0E-289L, 1.0E-288L, 1.0E-287L, 1.0E-286L, 1.0E-285L, 1.0E-284L, 1.0E-283L, 1.0E-282L, 1.0E-281L,
   1.0E-280L, 1.0E-279L, 1.0E-278L, 1.0E-277L, 1.0E-276L, 1.0E-275L, 1.0E-274L, 1.0E-273L, 1.0E-272L, 1.0E-271L,
@@ -138,13 +127,8 @@ const long double pow10lookup[701] = {
   1.0E+270L, 1.0E+271L, 1.0E+272L, 1.0E+273L, 1.0E+274L, 1.0E+275L, 1.0E+276L, 1.0E+277L, 1.0E+278L, 1.0E+279L,
   1.0E+280L, 1.0E+281L, 1.0E+282L, 1.0E+283L, 1.0E+284L, 1.0E+285L, 1.0E+286L, 1.0E+287L, 1.0E+288L, 1.0E+289L,
   1.0E+290L, 1.0E+291L, 1.0E+292L, 1.0E+293L, 1.0E+294L, 1.0E+295L, 1.0E+296L, 1.0E+297L, 1.0E+298L, 1.0E+299L,
-  1.0E+300L, 1.0E+301L, 1.0E+302L, 1.0E+303L, 1.0E+304L, 1.0E+305L, 1.0E+306L, 1.0E+307L, 1.0E+308L, LDBL(1.0E+309L),
-  LDBL(1.0E+310L), LDBL(1.0E+311L), LDBL(1.0E+312L), LDBL(1.0E+313L), LDBL(1.0E+314L), LDBL(1.0E+315L), LDBL(1.0E+316L), LDBL(1.0E+317L), LDBL(1.0E+318L), LDBL(1.0E+319L),
-  LDBL(1.0E+320L), LDBL(1.0E+321L), LDBL(1.0E+322L), LDBL(1.0E+323L), LDBL(1.0E+324L), LDBL(1.0E+325L), LDBL(1.0E+326L), LDBL(1.0E+327L), LDBL(1.0E+328L), LDBL(1.0E+329L),
-  LDBL(1.0E+330L), LDBL(1.0E+331L), LDBL(1.0E+332L), LDBL(1.0E+333L), LDBL(1.0E+334L), LDBL(1.0E+335L), LDBL(1.0E+336L), LDBL(1.0E+337L), LDBL(1.0E+338L), LDBL(1.0E+339L),
-  LDBL(1.0E+340L), LDBL(1.0E+341L), LDBL(1.0E+342L), LDBL(1.0E+343L), LDBL(1.0E+344L), LDBL(1.0E+345L), LDBL(1.0E+346L), LDBL(1.0E+347L), LDBL(1.0E+348L), LDBL(1.0E+349L),
-  LDBL(1.0E+350L)
+  1.0E+300L
 };
 
 
-}}  // namespace dt::read::
+}}  // namespace dt::read
diff --git a/src/core/read/constants.h b/src/core/read/constants.h
@@ -13,7 +13,7 @@ namespace read {
 
 extern const uint8_t hexdigits[256];
 extern const uint8_t allowedseps[128];
-extern const long double pow10lookup[701];
+extern const long double pow10lookup[601];
 
 }} // namespace dt::read::
 #endif
diff --git a/src/core/read/parsers/parse_float.cc b/src/core/read/parsers/parse_float.cc
@@ -1,5 +1,5 @@
 //------------------------------------------------------------------------------
-// Copyright 2018-2021 H2O.ai
+// Copyright 2018-2023 H2O.ai
 //
 // Permission is hereby granted, free of charge, to any person obtaining a
 // copy of this software and associated documentation files (the "Software"),
@@ -131,7 +131,7 @@ static void parse_float64_simple(const ParseContext& ctx) {
   const char* ch = ctx.ch;
 
   bool neg = 0, Eneg = 0;
-  double r;
+  long double r;
 
   if (ch < ctx.eof) {
     ch += (neg = *ch=='-') + (*ch=='+');
@@ -227,11 +227,23 @@ static void parse_float64_simple(const ParseContext& ctx) {
     }
     e += Eneg? -exp : exp;
   }
-  e += 350; // lookup table is arranged from -350 (0) to +350 (700)
-  if (e < 0 || e > 700) goto fail;
 
-  r = static_cast<double>(static_cast<long double>(acc) * dt::read::pow10lookup[e]);
-  ctx.target->float64 = neg? -r : r;
+  if (e < -350 || e > 350) goto fail;
+  r = static_cast<long double>(acc);
+
+  // For very small and very large floats do a separate lookup
+  // for extra exponent, i.e. anything above 300 or below -300.
+  // Note that the lookup array indices go from 0 (e == -300)
+  // to 600 (e == 300). The approach is based on
+  // https://github.com/Rdatatable/data.table/pull/4165
+  if (e < -300 || e > 300) {
+    auto extra = static_cast<int_fast8_t>(e - copysign(300, e));
+    r *= dt::read::pow10lookup[extra + 300];
+    e -= extra;
+  }
+  r *= dt::read::pow10lookup[e + 300];
+  ctx.target->float64 = static_cast<double>(neg? -r : r);
+
   ctx.ch = ch;
   return;
 
diff --git a/tests/conftest.py b/tests/conftest.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 #-------------------------------------------------------------------------------
-# Copyright 2018-2020 H2O.ai
+# Copyright 2018-2023 H2O.ai
 #
 # Permission is hereby granted, free of charge, to any person obtaining a
 # copy of this software and associated documentation files (the "Software"),
@@ -47,6 +47,11 @@ def is_ppc64():
     return platform.system() == "Linux" and "ppc64le" in platform_hardware
 
 
+def is_arm():
+    """Helper function to determine ARM platform"""
+    return platform.processor() == "arm"
+
+
 @pytest.fixture(scope="session")
 def noppc64():
     """ Skip the test if running in PowerPC64 """
@@ -83,10 +88,11 @@ def tol():
     long double type, resulting in a loss of precision when fread converts
     double literals into double numbers.
     """
-    platform_tols = {"Windows": 1e-15, "PowerPC64": 1e-16}
-    platform_system = "PowerPC64" if is_ppc64() else platform.system()
-
-    return platform_tols.get(platform_system, 0)
+    tols = {"Windows": 1e-15, "PowerPC64": 1e-16, "ARM": 1e-15} 
+    p = "PowerPC64" if is_ppc64() else \
+        "ARM" if is_arm() else \
+        platform.system()
+    return tols.get(p, 0)
 
 
 @pytest.fixture(scope="session")
diff --git a/tests/fread/test-fread-small.py b/tests/fread/test-fread-small.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 #-------------------------------------------------------------------------------
-# Copyright 2018-2021 H2O.ai
+# Copyright 2018-2023 H2O.ai
 #
 # Permission is hereby granted, free of charge, to any person obtaining a
 # copy of this software and associated documentation files (the "Software"),
@@ -163,15 +163,23 @@ def test_float_hex_invalid():
 
 
 def test_float_decimal0(tol):
-    assert list_equals(dt.fread("1.3485701e-303\n").to_list(),
-                       [[1.3485701e-303]],
-                       rel_tol = tol)
-    assert list_equals(dt.fread("1.46761e-313\n").to_list(),
-                       [[1.46761e-313]],
-                       rel_tol = tol)
-    assert (dt.fread("A\n1.23456789123456789123456999\n")[0, 0] ==
-            1.23456789123456789123456999)
-
+    src_str = ["1.23456789123456789123456999",
+               "1.3485701e+303",
+               "1.3485701e-303",
+               "1.46761e+343",
+               "1.46761e-343",
+              ]
+
+    src_float = [1.23456789123456789123456999,
+                 1.3485701e+303,
+                 1.3485701e-303,
+                 1.46761e+343,
+                 1.46761e-343,
+                ]
+
+    DT = dt.fread(text="\n".join(src_str))
+    assert list_equals(DT.to_list(), [src_float], rel_tol = tol)
+    assert DT[0, 0] == src_float[0]
 
 
 def test_float_precision():
diff --git a/tests/munging/test-cast.py b/tests/munging/test-cast.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 #-------------------------------------------------------------------------------
-# Copyright 2018-2021 H2O.ai
+# Copyright 2018-2023 H2O.ai
 #
 # Permission is hereby granted, free of charge, to any person obtaining a
 # copy of this software and associated documentation files (the "Software"),
@@ -186,9 +186,9 @@ def test_cast_double_to_float():
 
 @pytest.mark.parametrize("target_stype", ltype.real.stypes)
 def test_cast_str_to_double(target_stype):
-    DT = dt.Frame(A=["2.45", "-3.333", "0.13e+29", "boo", None, "-4e-4"])
+    DT = dt.Frame(A=["2.45", "-3.333", "0.13e+29", "boo", None, "-4e-4", "3.14e+323"])
     DT["A"] = target_stype
-    assert_equals(DT, dt.Frame(A=[2.45, -3.333, 1.3e28, None, None, -0.0004],
+    assert_equals(DT, dt.Frame(A=[2.45, -3.333, 1.3e28, None, None, -0.0004, 3.14e+323],
                                stype=target_stype))
 
 
