Fix bad pointer math

In SSE, we were incrementing a pointer as if it were a uint16_t, not an
si128, causing out of bounds accesses on non-block aligned chunks.

Similarly, simplify the alignment logic and check return values and
pointers in related accesses (non-block aligned linear lut lookup).

Signed-off-by: Kimball Thurston <kdt3rd@gmail.com>

Author: kdt3rd

src/lib/OpenEXRCore/internal_dwa_compressor.h

@@ -854,6 +854,7 @@ DwaCompressor_uncompress (
     if (version > 2) { return EXR_ERR_BAD_CHUNK_LEADER; }
 
     rv = DwaCompressor_setupChannelData (me);
+    if (rv != EXR_ERR_SUCCESS) { return rv; }
 
     //
     // Uncompress the UNKNOWN data into _planarUncBuffer[UNKNOWN]
@@ -1079,6 +1080,8 @@ DwaCompressor_uncompress (
         packedAcBufferEnd += decoder._packedAcCount * sizeof (uint16_t);
 
         packedDcBufferEnd += decoder._packedDcCount * sizeof (uint16_t);
+
+        totalAcUncompressedCount -= decoder._packedAcCount;
         totalDcUncompressedCount -= decoder._packedDcCount;
 
         me->_channelData[rChan].processed = 1;
@@ -1101,6 +1104,12 @@ DwaCompressor_uncompress (
 
         if (cd->processed) continue;
 
+        if (chan->width == 0 || chan->height == 0)
+        {
+            cd->processed = 1;
+            continue;
+        }
+
         switch (cd->compression)
         {
             case LOSSY_DCT:
@@ -1138,6 +1147,7 @@ DwaCompressor_uncompress (
                     packedDcBufferEnd +=
                         (size_t) decoder._packedDcCount * sizeof (uint16_t);
 
+                    totalAcUncompressedCount -= decoder._packedAcCount;
                     totalDcUncompressedCount -= decoder._packedDcCount;
                     if (rv != EXR_ERR_SUCCESS) { return rv; }
                 }

src/lib/OpenEXRCore/internal_dwa_decoder.h

@@ -320,19 +320,12 @@ LossyDctDecoder_execute (
     // Allocate a temp aligned buffer to hold a rows worth of full
     // 8x8 half-float blocks
     //
-
     rowBlockHandle = alloc_fn (
         (size_t) numComp * (size_t) numBlocksX * 64 * sizeof (uint16_t) +
         _SSE_ALIGNMENT);
     if (!rowBlockHandle) return EXR_ERR_OUT_OF_MEMORY;
 
-    rowBlock[0] = (uint16_t*) rowBlockHandle;
-
-    for (int i = 0; i < _SSE_ALIGNMENT; ++i)
-    {
-        if (((uintptr_t) (rowBlockHandle + i) & _SSE_ALIGNMENT_MASK) == 0)
-            rowBlock[0] = (uint16_t*) (rowBlockHandle + i);
-    }
+    rowBlock[0] = (uint16_t*) simd_align_pointer (rowBlockHandle);
 
     for (int comp = 1; comp < numComp; ++comp)
         rowBlock[comp] = rowBlock[comp - 1] + numBlocksX * 64;
@@ -649,8 +642,8 @@ LossyDctDecoder_execute (
                     {
                         _mm_storeu_si128 (dst, _mm_loadu_si128 (src));
 
-                        src += 8 * 8;
-                        dst += 8;
+                        ++dst;
+                        src += 8;
                     }
                 }
             }
@@ -720,9 +713,16 @@ LossyDctDecoder_execute (
 
                     dst += 8 * numFullBlocksX;
 
-                    for (int x = 0; x < maxX; ++x)
+                    if (d->_toLinear)
+                    {
+                        for (int x = 0; x < maxX; ++x)
+                        {
+                            *dst++ = d->_toLinear[*src++];
+                        }
+                    }
+                    else
                     {
-                        *dst++ = d->_toLinear[*src++];
+                        memcpy (dst, src, maxX * sizeof(uint16_t));
                     }
                 }
             }

src/lib/OpenEXRCore/internal_dwa_simd.h

@@ -68,6 +68,14 @@ __extension__ extern __inline float32x4x2_t
 }
 #endif
 
+static inline uint8_t *simd_align_pointer (uint8_t* ptr)
+{
+    return ptr +
+        ((_SSE_ALIGNMENT - (((uintptr_t)ptr) & _SSE_ALIGNMENT_MASK)) &
+         _SSE_ALIGNMENT_MASK);
+}
+
+
 //
 // Color space conversion, Inverse 709 CSC, Y'CbCr -> R'G'B'
 //