Fix potential multiplication overflows

Author: tobydox

libvncclient/cursor.c

@@ -63,7 +63,7 @@ rfbBool HandleCursorShape(rfbClient* client,int xhot, int yhot, int width, int h
   if(client->rcSource)
     free(client->rcSource);
 
-  client->rcSource = malloc(width * height * bytesPerPixel);
+  client->rcSource = malloc((size_t)width * height * bytesPerPixel);
   if (client->rcSource == NULL)
     return FALSE;
 
@@ -146,7 +146,7 @@ rfbBool HandleCursorShape(rfbClient* client,int xhot, int yhot, int width, int h
     return FALSE;
   }
 
-  client->rcMask = malloc(width * height);
+  client->rcMask = malloc((size_t)width * height);
   if (client->rcMask == NULL) {
     free(client->rcSource);
     client->rcSource = NULL;

libvncserver/cursor.c

@@ -477,7 +477,7 @@ void rfbMakeRichCursorFromXCursor(rfbScreenInfoPtr rfbScreen,rfbCursorPtr cursor
 
    if(cursor->richSource && cursor->cleanupRichSource)
        free(cursor->richSource);
-   cp=cursor->richSource=(unsigned char*)calloc(cursor->width*bpp,cursor->height);
+   cp=cursor->richSource=(unsigned char*)calloc((size_t)cursor->width*bpp,cursor->height);
    if(!cp)
        return;
    cursor->cleanupRichSource=TRUE;
@@ -534,7 +534,7 @@ void rfbHideCursor(rfbClientPtr cl)
    for(j=0;j<y2;j++)
      memcpy(s->frameBuffer+(y1+j)*rowstride+x1*bpp,
 	    s->underCursorBuffer+j*x2*bpp,
-	    x2*bpp);
+	    (size_t)x2*bpp);
 
    /* Copy to all scaled versions */
    rfbScaledScreenUpdate(s, x1, y1, x1+x2, y1+y2);

libvncserver/hextile.c

@@ -179,7 +179,7 @@ sendHextiles##bpp(rfbClientPtr cl, int rx, int ry, int rw, int rh) {
                                    cl->scaledScreen->paddedWidthInBytes, w, h); \
                                                                                 \
                 memcpy(&cl->updateBuf[cl->ublen], (char *)clientPixelData,      \
-                       w * h * (bpp/8));                                        \
+                       (size_t)w * h * (bpp/8));                                \
                                                                                 \
                 cl->ublen += w * h * (bpp/8);                                   \
                 rfbStatRecordEncodingSentAdd(cl, rfbEncodingHextile,            \

libvncserver/selbox.c

@@ -244,7 +244,7 @@ int rfbSelectBox(rfbScreenInfoPtr rfbScreen,rfbFontDataPtr font,
    selData.cancelX = selData.cancelBX+(k-j)/2;
    selData.okY = y2-border;
 
-   frameBufferBackup = (char*)malloc(bpp*(x2-x1)*(y2-y1));
+   frameBufferBackup = (char*)malloc((size_t)bpp*(x2-x1)*(y2-y1));
    if (!frameBufferBackup)
        return(-1);
 
@@ -271,7 +271,7 @@ int rfbSelectBox(rfbScreenInfoPtr rfbScreen,rfbFontDataPtr font,
    for(j=0;j<y2-y1;j++)
      memcpy(frameBufferBackup+j*(x2-x1)*bpp,
 	    rfbScreen->frameBuffer+j*rfbScreen->paddedWidthInBytes+x1*bpp,
-	    (x2-x1)*bpp);
+	    (size_t)(x2-x1)*bpp);
 
    /* paint list and buttons */
    rfbFillRect(rfbScreen,x1,y1,x2,y2,colour);
@@ -286,7 +286,7 @@ int rfbSelectBox(rfbScreenInfoPtr rfbScreen,rfbFontDataPtr font,
    for(j=0;j<y2-y1;j++)
      memcpy(rfbScreen->frameBuffer+j*rfbScreen->paddedWidthInBytes+x1*bpp,
 	    frameBufferBackup+j*(x2-x1)*bpp,
-	    (x2-x1)*bpp);
+	    (size_t)(x2-x1)*bpp);
    free(frameBufferBackup);
    rfbMarkRectAsModified(rfbScreen,x1,y1,x2,y2);
    rfbScreen->screenData = screenDataBackup;

libvncserver/tight.c

@@ -975,7 +975,7 @@ SendIndexedRect(rfbClientPtr cl,
             entryLen = 4;
 
         memcpy(&cl->updateBuf[cl->ublen], tightAfterBuf,
-               paletteNumColors * entryLen);
+               (size_t)paletteNumColors * entryLen);
         cl->ublen += paletteNumColors * entryLen;
         rfbStatRecordEncodingSentAdd(cl, cl->tightEncoding,
                                      3 + paletteNumColors * entryLen);
@@ -1617,7 +1617,7 @@ SendJpegRect(rfbClientPtr cl, int x, int y, int w, int h, int quality)
         unsigned char *dst;
         int inRed, inGreen, inBlue, i, j;
 
-        if((tmpbuf = (unsigned char *)malloc(w * h * 3)) == NULL)
+        if((tmpbuf = (unsigned char *)malloc((size_t)w * h * 3)) == NULL)
             rfbLog("Memory allocation failure!\n");
         srcptr = (uint16_t *)&cl->scaledScreen->frameBuffer
             [y * cl->scaledScreen->paddedWidthInBytes + x * ps];