lists_to_integer() & friends: Don't crash for overlong lists

Let `list_to_integer/1` and `list_to_integer/2` raise a `system_limit`
exception when the result doesn't fit in a bignum. Let
`string:to_integer/1` return `{error,system_limit}` when the result
doesn't fit.

Author: bjorng

erts/emulator/beam/bif.c

@@ -3357,6 +3357,9 @@ BIF_RETTYPE string_list_to_integer_1(BIF_ALIST_1)
      case LTI_NO_INTEGER:
 	 hp = HAlloc(BIF_P,3);
 	 BIF_RET(TUPLE2(hp, am_error, am_no_integer));
+     case LTI_SYSTEM_LIMIT:
+	 hp = HAlloc(BIF_P,3);
+	 BIF_RET(TUPLE2(hp, am_error, am_system_limit));
      default:
 	 hp = HAlloc(BIF_P,3);
 	 BIF_RET(TUPLE2(hp, res, tail));
@@ -3365,25 +3368,27 @@ BIF_RETTYPE string_list_to_integer_1(BIF_ALIST_1)
 
 BIF_RETTYPE list_to_integer_1(BIF_ALIST_1)
  {
-   /* Using erts_list_to_integer is about twice as fast as using
-      erts_chars_to_integer because we do not have to copy the 
-      entire list */
+     /* Using erts_list_to_integer() is about twice as fast as using
+      * erts_chars_to_integer() because we do not have to copy the
+      * entire list. */
      Eterm res;
      Eterm dummy;
      /* must be a list */
-     if (erts_list_to_integer(BIF_P, BIF_ARG_1, 10,
-                              &res, &dummy) != LTI_ALL_INTEGER) {
-	 BIF_ERROR(BIF_P,BADARG);
+     switch (erts_list_to_integer(BIF_P, BIF_ARG_1, 10, &res, &dummy)) {
+     case LTI_ALL_INTEGER:
+         BIF_RET(res);
+     case LTI_SYSTEM_LIMIT:
+	 BIF_ERROR(BIF_P, SYSTEM_LIMIT);
+     default:
+	 BIF_ERROR(BIF_P, BADARG);
      }
-     BIF_RET(res);
  }
 
 BIF_RETTYPE list_to_integer_2(BIF_ALIST_2)
 {
-  /* Bif implementation is about 50% faster than pure erlang,
-     and since we have erts_chars_to_integer now it is simpler
-     as well. This could be optimized further if we did not have to
-     copy the list to buf. */
+    /* The BIF implementation is about 50% faster than pure Erlang,
+     * and since we now have erts_list_to_integer() it is simpler as
+     * well. */
     Sint i;
     Eterm res, dummy;
     int base;
@@ -3399,11 +3404,14 @@ BIF_RETTYPE list_to_integer_2(BIF_ALIST_2)
         BIF_ERROR(BIF_P, BADARG);
     }
 
-    if (erts_list_to_integer(BIF_P, BIF_ARG_1, base,
-                             &res, &dummy) != LTI_ALL_INTEGER) {
-        BIF_ERROR(BIF_P,BADARG);
+    switch (erts_list_to_integer(BIF_P, BIF_ARG_1, base, &res, &dummy)) {
+    case LTI_ALL_INTEGER:
+        BIF_RET(res);
+    case LTI_SYSTEM_LIMIT:
+        BIF_ERROR(BIF_P, SYSTEM_LIMIT);
+    default:
+        BIF_ERROR(BIF_P, BADARG);
     }
-    BIF_RET(res);
 }
 
 /**********************************************************************/

erts/emulator/beam/big.c

@@ -3008,6 +3008,11 @@ LTI_result_t erts_list_to_integer(Process *BIF_P, Eterm orig_list,
          m  = (lg2+D_EXP-1)/D_EXP; /* number of digits */
          m  = BIG_NEED_SIZE(m);    /* number of words + thing */
 
+         if (m > BIG_ARITY_MAX) {
+             error_res = LTI_SYSTEM_LIMIT;
+             goto error;
+         }
+
          hp = HAlloc(BIF_P, m);
          hp_end = hp + m;
 

erts/emulator/beam/big.h

@@ -188,7 +188,8 @@ typedef enum {
     LTI_BAD_STRUCTURE = 0,
     LTI_NO_INTEGER    = 1,
     LTI_SOME_INTEGER  = 2,
-    LTI_ALL_INTEGER   = 3
+    LTI_ALL_INTEGER   = 3,
+    LTI_SYSTEM_LIMIT  = 4,
 } LTI_result_t;
 
 LTI_result_t erts_list_to_integer(Process *BIF_P, Eterm orig_list,

erts/emulator/test/list_bif_SUITE.erl

@@ -50,14 +50,21 @@ t_list_to_integer(Config) when is_list(Config) ->
     12373 = (catch list_to_integer("12373")),
     -12373 =  (catch list_to_integer("-12373")),
     12373 = (catch list_to_integer("+12373")),
-    {'EXIT',{badarg,_}} = ( catch list_to_integer(abc)),
+    {'EXIT',{badarg,_}} = (catch list_to_integer(abc)),
     {'EXIT',{badarg,_}} = (catch list_to_integer("")),
     {12373281903728109372810937209817320981321,"ABC"} = string:to_integer("12373281903728109372810937209817320981321ABC"),
     {-12373281903728109372810937209817320981321,"ABC"} = string:to_integer("-12373281903728109372810937209817320981321ABC"),
     {12,[345]} = string:to_integer([$1,$2,345]),
-    {error, badarg} = string:to_integer([$1,$2,a]),
+    {error,badarg} = string:to_integer([$1,$2,a]),
     {error,no_integer} = string:to_integer([$A]),
     {error,badarg} = string:to_integer($A),
+
+    %% System limit.
+    Digits = lists:duplicate(11_000_000, $9),
+    {'EXIT',{system_limit,_}} = catch list_to_integer(Digits),
+    {'EXIT',{system_limit,_}} = catch list_to_integer(Digits, 16),
+    {error,system_limit} = string:to_integer(Digits),
+
     ok.
 
 %% Test hd/1 with correct and incorrect arguments.

erts/emulator/test/num_bif_SUITE.erl

@@ -670,9 +670,11 @@ t_string_to_integer(Config) when is_list(Config) ->
                    {"10",''}                    %Base 20
                   ]),
 
-    %% log2 calculation overflow bug in do_integer_to_list (OTP-12624).
-    %% Would crash with segmentation fault.
-    0 = list_to_integer(lists:duplicate(10000000,$0)),
+    %% System limit
+    Digits = lists:duplicate(11_000_000, $9),
+    {'EXIT',{system_limit,_}} = catch list_to_integer(Digits),
+    {'EXIT',{system_limit,_}} = catch list_to_integer(Digits, 16),
+    {error,system_limit} = string:to_integer(Digits),
 
     ok.