Backported of:

From 5a0adf1aba7d41c6b94ba167c0c4308d2eecfd17 Mon Sep 17 00:00:00 2001
From: peterhillman <peterh@wetafx.co.nz>
Date: Wed, 22 Sep 2021 16:13:34 +1200
Subject: [PATCH] prevent overflow in bytesPerDeepLineTable (#1152)

* prevent overflow in bytesPerDeepLineTable

Signed-off-by: Peter Hillman <peterh@wetafx.co.nz>

* restore zapped 'const' from ImfMisc

Signed-off-by: Peter Hillman <peterh@wetafx.co.nz>
diff --git a/IlmImf/ImfMisc.cpp b/IlmImf/ImfMisc.cpp
index b091015..d3a21b7 100644
--- a/IlmImf/ImfMisc.cpp
+++ b/IlmImf/ImfMisc.cpp
@@ -167,16 +167,28 @@ bytesPerDeepLineTable (const Header &header,
          c != channels.end();
          ++c)
     {
+    	const uint64_t pixelSize = pixelTypeSize (c.channel().type);
+
         for (int y = minY; y <= maxY; ++y)
             if (modp (y, c.channel().ySampling) == 0)
             {
-                int nBytes = 0;
+                uint64_t nBytes = 0;
                 for (int x = dataWindow.min.x; x <= dataWindow.max.x; x++)
                 {
                     if (modp (x, c.channel().xSampling) == 0)
-                        nBytes += pixelTypeSize (c.channel().type) *
-                                  sampleCount(base, xStride, yStride, x, y);
+                        nBytes += pixelSize *
+                                  static_cast<uint64_t>(sampleCount(base, xStride, yStride, x, y));
                 }
+
+            //
+            // architectures where size_t is smaller than 64 bits may overflow
+            // (scanlines with more than 2^32 bytes are not currently supported so this should not occur with valid files)
+            //
+            if( static_cast<uint64_t>(bytesPerLine[y - dataWindow.min.y]) + nBytes > SIZE_MAX)
+            {
+                throw IEX_NAMESPACE::IoExc("Scanline size too large");
+            }
+
                 bytesPerLine[y - dataWindow.min.y] += nBytes;
             }
     }
@@ -184,9 +196,12 @@ bytesPerDeepLineTable (const Header &header,
     size_t maxBytesPerLine = 0;
 
     for (int y = minY; y <= maxY; ++y)
+    {
         if (maxBytesPerLine < bytesPerLine[y - dataWindow.min.y])
+	{
             maxBytesPerLine = bytesPerLine[y - dataWindow.min.y];
-
+	}
+    }
     return maxBytesPerLine;
 }