Lua library Integer overflow can cause the DragonFly crash. (CVE-2020-14147) (#5421)

* CVE-2020-14147

* test: fix LuaIntOverflow expectation (should fail on overflow)

Author: KIMDONGYEON00

src/core/interpreter_test.cc

@@ -522,4 +522,8 @@ TEST_F(InterpreterTest, AvoidIntOverflow) {
   EXPECT_EQ("str(0000FFFF)", ser_.res);
 }
 
+TEST_F(InterpreterTest, LuaIntOverflow) {
+  EXPECT_FALSE(Execute("EVAL \"struct.pack('>I2147483648', '10')\" 0"));
+}
+
 }  // namespace dfly

src/redis/lua/struct/lua_struct.c

@@ -82,19 +82,20 @@ typedef struct Header {
 } Header;
 
 
-static int getnum (const char **fmt, int df) {
+static int getnum (lua_State *L, const char **fmt, int df) {
   if (!isdigit(**fmt))  /* no number? */
     return df;  /* return default value */
   else {
     int a = 0;
     do {
+      if (a > (INT_MAX / 10) || a * 10 > (INT_MAX - (**fmt - '0')))
+        luaL_error(L, "integral size overflow");
       a = a*10 + *((*fmt)++) - '0';
     } while (isdigit(**fmt));
     return a;
   }
 }
 
-
 #define defaultoptions(h)	((h)->endian = native.endian, (h)->align = 1)
 
 
@@ -108,9 +109,9 @@ static size_t optsize (lua_State *L, char opt, const char **fmt) {
     case 'f':  return sizeof(float);
     case 'd':  return sizeof(double);
     case 'x': return 1;
-    case 'c': return  getnum(fmt, 1);
+    case 'c': return  getnum(L, fmt, 1);
     case 'i': case 'I': {
-      int sz = getnum(fmt, sizeof(int));
+      int sz = getnum(L, fmt, sizeof(int));
       if (sz > MAXINTSIZE)
         luaL_error(L, "integral size %d is larger than limit of %d",
                        sz, MAXINTSIZE);
@@ -143,7 +144,7 @@ static void controloptions (lua_State *L, int opt, const char **fmt,
     case '>': h->endian = BIG; return;
     case '<': h->endian = LITTLE; return;
     case '!': {
-      int a = getnum(fmt, MAXALIGN);
+      int a = getnum(L, fmt, MAXALIGN);
       if (!isp2(a))
         luaL_error(L, "alignment %d is not a power of 2", a);
       h->align = a;