properly handle decimal exponents

lib/explorer/series.ex

@@ -32,6 +32,11 @@ defmodule Explorer.Series do
       >
       > This functionality is considered unstable. It is a work-in-progress feature
       > and may not always work as expected. It may be changed at any point.
+      >
+      > Note: The underlying Polars decimal type is a fixed point decimal with optional
+      > precision and non-negative scale, backed by a signed 128-bit integer which allows
+      > for up to 38 significant digits (max precision is 38). Elixir's `Decimal` can handle
+      > higher precision, which may cause errors when converting very large decimal values.
     * `:null` - `nil`s exclusively
     * `:string` - UTF-8 encoded binary
     * `:time` - Time type that unwraps to `Elixir.Time`

native/explorer/src/datatypes.rs

@@ -629,31 +629,40 @@ impl ExDecimal {
         }
     }
 
-    pub fn signed_coef(self) -> i128 {
-        self.sign as i128 * self.coef
+    pub fn signed_coef(self) -> Result<i128, ExplorerError> {
+        let base = self.sign as i128 * self.coef;
+        if self.exp > 0 {
+            base.checked_mul(10_i128.pow(self.exp as u32))
+                .ok_or_else(|| {
+                    ExplorerError::Other(
+                        "cannot decode a valid decimal from term; check that `coef` fits into an `i128`. error: throw(<term>)".to_string()
+                    )
+                })
+        } else {
+            Ok(base)
+        }
     }
 
     pub fn scale(self) -> usize {
-        self.exp
-            .abs()
-            .try_into()
-            .expect("cannot convert exponent (Elixir) to scale (Rust)")
+        if self.exp > 0 {
+            0
+        } else {
+            self.exp
+                .abs()
+                .try_into()
+                .expect("cannot convert exponent (Elixir) to scale (Rust)")
+        }
     }
 }
 
 impl Literal for ExDecimal {
     fn lit(self) -> Expr {
-        let size = usize::try_from(-(self.exp)).expect("exponent should fit an usize");
+        let coef = self.signed_coef().unwrap();
+        let scale = self.scale();
+
         Expr::Literal(LiteralValue::Scalar(Scalar::new(
-            DataType::Decimal(Some(size), Some(size)),
-            AnyValue::Decimal(
-                if self.sign.is_positive() {
-                    self.coef
-                } else {
-                    -self.coef
-                },
-                size,
-            ),
+            DataType::Decimal(Some(scale), Some(scale)),
+            AnyValue::Decimal(coef, scale),
         )))
     }
 }
@@ -716,6 +725,7 @@ impl<'a> rustler::Decoder<'a> for ExValidValue<'a> {
                 } else if let Ok(duration) = term.decode::<ExDuration>() {
                     Ok(ExValidValue::Duration(duration))
                 } else if let Ok(decimal) = term.decode::<ExDecimal>() {
+                    decimal.signed_coef().map_err(|_| rustler::Error::BadArg)?;
                     Ok(ExValidValue::Decimal(decimal))
                 } else {
                     Err(rustler::Error::BadArg)

native/explorer/src/series/from_list.rs

@@ -240,14 +240,16 @@ pub fn s_from_list_decimal(
                     })
             }
 
-            TermType::Map => item
-                .decode::<ExDecimal>()
-                .map(|ex_decimal| AnyValue::Decimal(ex_decimal.signed_coef(), ex_decimal.scale()))
-                .map_err(|error| {
-                    ExplorerError::Other(format!(
+            TermType::Map => {
+                match item.decode::<ExDecimal>() {
+                    Ok(ex_decimal) => ex_decimal
+                        .signed_coef()
+                        .map(|coef| AnyValue::Decimal(coef, ex_decimal.scale())),
+                    Err(error) => Err(ExplorerError::Other(format!(
                         "cannot decode a valid decimal from term; check that `coef` fits into an `i128`. error: {error:?}"
-                    ))
-                }),
+                    ))),
+                }
+            }
             TermType::Atom => Ok(AnyValue::Null),
 
             TermType::Float => item

test/explorer/series_test.exs

@@ -1997,6 +1997,39 @@ defmodule Explorer.SeriesTest do
       assert s3.dtype == {:decimal, 38, 1}
       assert Series.to_list(s3) === [Decimal.new("2.2"), Decimal.new("4.0"), Decimal.new("6.1")]
     end
+
+    test "adding decimal series with positive and negative exponents" do
+      s1 =
+        Series.from_list([
+          Decimal.new("2.1e20"),
+          Decimal.new("-3.5e10"),
+          Decimal.new("1.5e-15"),
+          Decimal.new("1.0e2")
+        ])
+
+      s2 =
+        Series.from_list([
+          Decimal.new("1.0e20"),
+          Decimal.new("2.0e10"),
+          Decimal.new("2.5e-15"),
+          Decimal.new("5.0e-2")
+        ])
+
+      s3 = Series.add(s1, s2)
+      [v1, v2, v3, v4] = Series.to_list(s3)
+
+      assert Decimal.eq?(v1, Decimal.new("3.1e20"))
+      assert Decimal.eq?(v2, Decimal.new("-1.5e10"))
+      assert Decimal.eq?(v3, Decimal.new("4.0e-15"))
+      assert Decimal.eq?(v4, Decimal.new("100.05"))
+    end
+
+    test "overflow with values exceeding i128 limits" do
+      assert_raise RuntimeError,
+                   "Generic Error: cannot decode a valid decimal from term;" <>
+                     " check that `coef` fits into an `i128`. error: throw(<term>)",
+                   fn -> Series.from_list([Decimal.new("3.4e38")]) end
+    end
   end
 
   describe "subtract/2" do