fix(rla): More robust to corrupted RLA files that could overrun buffers (#4624)

Signed-off-by: Larry Gritz <[email protected]>

Author: lgritz

src/rla.imageio/rlainput.cpp

@@ -604,7 +604,15 @@ RLAInput::decode_channel_group(int first_channel, short num_channels,
     // OIIO conventions.
     if (num_bits == 8 || num_bits == 16 || num_bits == 32) {
         // ok -- no rescaling needed
-    } else if (num_bits == 10) {
+    }
+    int bytes_per_chan = ceil2(std::max(int(num_bits), 8)) / 8;
+    if (size_t(offset + (m_spec.width - 1) * pixelsize
+               + num_channels * bytes_per_chan)
+        > m_buf.size()) {
+        errorfmt("Probably corrupt file (buffer overrun avoided)");
+        return false;  // Probably corrupt? Would have overrun
+    }
+    if (num_bits == 10) {
         // fast, common case -- use templated hard-code
         for (int x = 0; x < m_spec.width; ++x) {
             uint16_t* b = (uint16_t*)(&m_buf[offset + x * pixelsize]);

testsuite/rla/ref/out.txt

@@ -319,5 +319,8 @@ Full command line was:
 oiiotool ERROR: read : "src/crash-3951.rla": Read error: couldn't read RLE data span
 Full command line was:
 > oiiotool src/crash-3951.rla -o crash4.exr
+oiiotool ERROR: read : "src/crash-1.rla": Probably corrupt file (buffer overrun avoided)
+Full command line was:
+> oiiotool src/crash-1.rla -o crash5.exr
 Comparing "rlacrop.rla" and "ref/rlacrop.rla"
 PASS

testsuite/rla/run.py

@@ -22,5 +22,6 @@
 command += oiiotool(OIIO_TESTSUITE_IMAGEDIR + "/crash2.rla -o crash2.exr", failureok = True)
 command += oiiotool("src/crash-1629.rla -o crash3.exr", failureok = True)
 command += oiiotool("src/crash-3951.rla -o crash4.exr", failureok = True)
+command += oiiotool("src/crash-1.rla -o crash5.exr", failureok = True)
 
 outputs = [ "rlacrop.rla", 'out.txt' ]