Merge pull request #733 from tbeu/add-int-overflow-check-to-vrefbuffer

Adding int overflow checks to vrefbuffer

Author: redboltz

include/msgpack/v1/vrefbuffer.hpp

@@ -58,6 +58,10 @@ class vrefbuffer {
         :m_ref_size(std::max(ref_size, detail::packer_max_buffer_size + 1)),
          m_chunk_size(chunk_size)
     {
+        if((sizeof(chunk) + chunk_size) < chunk_size) {
+            throw std::bad_alloc();
+        }
+
         size_t nfirst = (sizeof(iovec) < 72/2) ?
             72 / sizeof(iovec) : 8;
 
@@ -141,7 +145,11 @@ class vrefbuffer {
             if(sz < len) {
                 sz = len;
             }
-
+             
+            if(sizeof(chunk) + sz < sz){
+                throw std::bad_alloc();
+            }
+            
             chunk* c = static_cast<chunk*>(::malloc(sizeof(chunk) + sz));
             if(!c) {
                 throw std::bad_alloc();
@@ -183,6 +191,10 @@ class vrefbuffer {
     {
         size_t sz = m_chunk_size;
 
+        if((sizeof(chunk) + sz) < sz){
+            throw std::bad_alloc();
+        }
+
         chunk* empty = static_cast<chunk*>(::malloc(sizeof(chunk) + sz));
         if(!empty) {
             throw std::bad_alloc();

src/vrefbuffer.c

@@ -30,6 +30,10 @@ bool msgpack_vrefbuffer_init(msgpack_vrefbuffer* vbuf,
         ref_size > MSGPACK_PACKER_MAX_BUFFER_SIZE + 1 ?
         ref_size : MSGPACK_PACKER_MAX_BUFFER_SIZE + 1 ;
 
+    if((sizeof(msgpack_vrefbuffer_chunk) + chunk_size) < chunk_size) {
+        return false;
+    }
+
     nfirst = (sizeof(struct iovec) < 72/2) ?
             72 / sizeof(struct iovec) : 8;
 
@@ -135,6 +139,9 @@ int msgpack_vrefbuffer_append_copy(msgpack_vrefbuffer* vbuf,
             sz = len;
         }
 
+        if((sizeof(msgpack_vrefbuffer_chunk) + sz) < sz){
+            return -1;
+        }
         chunk = (msgpack_vrefbuffer_chunk*)malloc(
                 sizeof(msgpack_vrefbuffer_chunk) + sz);
         if(chunk == NULL) {
@@ -164,8 +171,13 @@ int msgpack_vrefbuffer_append_copy(msgpack_vrefbuffer* vbuf,
 int msgpack_vrefbuffer_migrate(msgpack_vrefbuffer* vbuf, msgpack_vrefbuffer* to)
 {
     size_t sz = vbuf->chunk_size;
+    msgpack_vrefbuffer_chunk* empty;
+
+    if((sizeof(msgpack_vrefbuffer_chunk) + sz) < sz){
+        return -1;
+    }
 
-    msgpack_vrefbuffer_chunk* empty = (msgpack_vrefbuffer_chunk*)malloc(
+    empty = (msgpack_vrefbuffer_chunk*)malloc(
             sizeof(msgpack_vrefbuffer_chunk) + sz);
     if(empty == NULL) {
         return -1;

test/msgpack_c.cpp

@@ -1352,3 +1352,14 @@ TEST(MSGPACKC, unpack_array_uint64)
     EXPECT_EQ(0xFFF0000000000001LL, obj.via.array.ptr[0].via.u64);
     msgpack_zone_destroy(&z);
 }
+
+
+TEST(MSGPACKC, vref_buffer_overflow)
+{
+    msgpack_vrefbuffer vbuf;
+    msgpack_vrefbuffer to;
+    size_t ref_size = 0;
+    size_t chunk_size = std::numeric_limits<size_t>::max();
+    EXPECT_FALSE(msgpack_vrefbuffer_init(&vbuf, ref_size, chunk_size));
+    EXPECT_EQ(-1, msgpack_vrefbuffer_migrate(&vbuf, &to));
+}

test/msgpack_vref.cpp

@@ -264,3 +264,14 @@ TEST(MSGPACK, vrefbuffer_small_int64)
     msgpack::vrefbuffer vbuf(0, 0);
     GEN_TEST_VREF(int64_t, vbuf);
 }
+
+TEST(MSGPACK, vref_buffer_overflow)
+{
+    size_t ref_size = 0;
+    size_t chunk_size = std::numeric_limits<size_t>::max();
+    char *buf = (char *)malloc(0x1000);
+    ASSERT_THROW(msgpack::vrefbuffer vbuf(ref_size, chunk_size), std::bad_alloc);
+    msgpack::vrefbuffer vbuf2(0, 0x1000);
+    ASSERT_THROW(vbuf2.append_copy(buf, chunk_size), std::bad_alloc);
+    free(buf);
+}