fix(exr): Avoid integer overflow for large deep exr slice strides (#4542)

Fixes #4540

---------

Signed-off-by: Larry Gritz <[email protected]>

Author: lgritz

src/openexr.imageio/exrinput.cpp

@@ -1485,7 +1485,7 @@ OpenEXRInput::read_native_deep_scanlines(int subimage, int miplevel, int ybegin,
     try {
         size_t npixels = (yend - ybegin) * m_spec.width;
         chend          = clamp(chend, chbegin + 1, m_spec.nchannels);
-        int nchans     = chend - chbegin;
+        size_t nchans  = chend - chbegin;
 
         // Set up the count and pointers arrays and the Imf framebuffer
         std::vector<TypeDesc> channeltypes;
@@ -1502,16 +1502,17 @@ OpenEXRInput::read_native_deep_scanlines(int subimage, int miplevel, int ybegin,
                               sizeof(unsigned int),
                               sizeof(unsigned int) * m_spec.width);
         frameBuffer.insertSampleCountSlice(countslice);
+        size_t slchans      = m_spec.width * nchans;
+        size_t xstride      = sizeof(void*) * nchans;
+        size_t ystride      = sizeof(void*) * slchans;
+        size_t samplestride = deepdata.samplesize();
 
         for (int c = chbegin; c < chend; ++c) {
-            Imf::DeepSlice slice(
-                part.pixeltype[c],
-                (char*)(&pointerbuf[0] + (c - chbegin) - m_spec.x * nchans
-                        - ybegin * m_spec.width * nchans),
-                sizeof(void*) * nchans,  // xstride of pointer array
-                sizeof(void*) * nchans
-                    * m_spec.width,      // ystride of pointer array
-                deepdata.samplesize());  // stride of data sample
+            Imf::DeepSlice slice(part.pixeltype[c],
+                                 (char*)(&pointerbuf[0] + (c - chbegin)
+                                         - m_spec.x * nchans
+                                         - ybegin * slchans),
+                                 xstride, ystride, samplestride);
             frameBuffer.insert(m_spec.channelnames[c].c_str(), slice);
         }
         m_deep_scanline_input_part->setFrameBuffer(frameBuffer);
@@ -1564,7 +1565,7 @@ OpenEXRInput::read_native_deep_tiles(int subimage, int miplevel, int xbegin,
         size_t height  = yend - ybegin;
         size_t npixels = width * height;
         chend          = clamp(chend, chbegin + 1, m_spec.nchannels);
-        int nchans     = chend - chbegin;
+        size_t nchans  = chend - chbegin;
 
         // Set up the count and pointers arrays and the Imf framebuffer
         std::vector<TypeDesc> channeltypes;
@@ -1579,14 +1580,15 @@ OpenEXRInput::read_native_deep_tiles(int subimage, int miplevel, int xbegin,
             Imf::UINT, (char*)(&all_samples[0] - xbegin - ybegin * width),
             sizeof(unsigned int), sizeof(unsigned int) * width);
         frameBuffer.insertSampleCountSlice(countslice);
+        size_t slchans      = width * nchans;
+        size_t xstride      = sizeof(void*) * nchans;
+        size_t ystride      = sizeof(void*) * slchans;
+        size_t samplestride = deepdata.samplesize();
         for (int c = chbegin; c < chend; ++c) {
-            Imf::DeepSlice slice(
-                part.pixeltype[c],
-                (char*)(&pointerbuf[0] + (c - chbegin) - xbegin * nchans
-                        - ybegin * width * nchans),
-                sizeof(void*) * nchans,          // xstride of pointer array
-                sizeof(void*) * nchans * width,  // ystride of pointer array
-                deepdata.samplesize());          // stride of data sample
+            Imf::DeepSlice slice(part.pixeltype[c],
+                                 (char*)(&pointerbuf[0] + (c - chbegin)
+                                         - xbegin * nchans - ybegin * slchans),
+                                 xstride, ystride, samplestride);
             frameBuffer.insert(m_spec.channelnames[c].c_str(), slice);
         }
         m_deep_tiled_input_part->setFrameBuffer(frameBuffer);

src/openexr.imageio/exroutput.cpp

@@ -1700,17 +1700,17 @@ OpenEXROutput::write_deep_scanlines(int ybegin, int yend, int /*z*/,
         return false;
     }
 
-    int nchans         = m_spec.nchannels;
+    size_t nchans(m_spec.nchannels);
     const DeepData* dd = &deepdata;
     std::unique_ptr<DeepData> dd_local;  // In case we need a copy
     bool same_chantypes = true;
-    for (int c = 0; c < nchans; ++c)
+    for (size_t c = 0; c < nchans; ++c)
         same_chantypes &= (m_spec.channelformat(c) == deepdata.channeltype(c));
     if (!same_chantypes) {
         // If the channel types don't match, we need to make a copy of the
         // DeepData and convert the channels to the spec's channel types.
         std::vector<TypeDesc> chantypes;
-        if (m_spec.channelformats.size() == size_t(nchans))
+        if (m_spec.channelformats.size() == nchans)
             chantypes = m_spec.channelformats;
         else
             chantypes.resize(nchans, m_spec.format);
@@ -1729,15 +1729,15 @@ OpenEXROutput::write_deep_scanlines(int ybegin, int yend, int /*z*/,
         frameBuffer.insertSampleCountSlice(countslice);
         std::vector<void*> pointerbuf;
         dd->get_pointers(pointerbuf);
-        for (int c = 0; c < nchans; ++c) {
+        size_t slchans      = size_t(m_spec.width) * nchans;
+        size_t xstride      = sizeof(void*) * nchans;
+        size_t ystride      = sizeof(void*) * slchans;
+        size_t samplestride = dd->samplesize();
+        for (size_t c = 0; c < nchans; ++c) {
             Imf::DeepSlice slice(m_pixeltype[c],
                                  (char*)(&pointerbuf[c] - m_spec.x * nchans
-                                         - ybegin * m_spec.width * nchans),
-                                 sizeof(void*)
-                                     * nchans,  // xstride of pointer array
-                                 sizeof(void*) * nchans
-                                     * m_spec.width,  // ystride of pointer array
-                                 dd->samplesize());   // stride of data sample
+                                         - ybegin * slchans),
+                                 xstride, ystride, samplestride);
             frameBuffer.insert(m_spec.channelnames[c].c_str(), slice);
         }
         m_deep_scanline_output_part->setFrameBuffer(frameBuffer);
@@ -1772,17 +1772,17 @@ OpenEXROutput::write_deep_tiles(int xbegin, int xend, int ybegin, int yend,
         return false;
     }
 
-    int nchans         = m_spec.nchannels;
+    size_t nchans      = size_t(m_spec.nchannels);
     const DeepData* dd = &deepdata;
     std::unique_ptr<DeepData> dd_local;  // In case we need a copy
     bool same_chantypes = true;
-    for (int c = 0; c < nchans; ++c)
+    for (size_t c = 0; c < nchans; ++c)
         same_chantypes &= (m_spec.channelformat(c) == deepdata.channeltype(c));
     if (!same_chantypes) {
         // If the channel types don't match, we need to make a copy of the
         // DeepData and convert the channels to the spec's channel types.
         std::vector<TypeDesc> chantypes;
-        if (m_spec.channelformats.size() == size_t(nchans))
+        if (m_spec.channelformats.size() == nchans)
             chantypes = m_spec.channelformats;
         else
             chantypes.resize(nchans, m_spec.format);
@@ -1803,15 +1803,15 @@ OpenEXROutput::write_deep_tiles(int xbegin, int xend, int ybegin, int yend,
         frameBuffer.insertSampleCountSlice(countslice);
         std::vector<void*> pointerbuf;
         dd->get_pointers(pointerbuf);
-        for (int c = 0; c < nchans; ++c) {
+        size_t slchans      = width * nchans;
+        size_t xstride      = sizeof(void*) * nchans;
+        size_t ystride      = sizeof(void*) * slchans;
+        size_t samplestride = dd->samplesize();
+        for (size_t c = 0; c < nchans; ++c) {
             Imf::DeepSlice slice(m_pixeltype[c],
                                  (char*)(&pointerbuf[c] - xbegin * nchans
-                                         - ybegin * width * nchans),
-                                 sizeof(void*)
-                                     * nchans,  // xstride of pointer array
-                                 sizeof(void*) * nchans
-                                     * width,        // ystride of pointer array
-                                 dd->samplesize());  // stride of data sample
+                                         - ybegin * slchans),
+                                 xstride, ystride, samplestride);
             frameBuffer.insert(m_spec.channelnames[c].c_str(), slice);
         }
         m_deep_tiled_output_part->setFrameBuffer(frameBuffer);