From 688012dca2c39033aa2dc7bcea9796787cfd1b44 Mon Sep 17 00:00:00 2001
From: Su_Laus <sulau@freenet.de>
Date: Sat, 4 Feb 2023 23:24:21 +0100
Subject: [PATCH] tiffcrop correctly update buffersize after rotateImage()
 fix#520  -- enlarge buffsize and check integer overflow within rotateImage().

Reference:https://gitlab.com/libtiff/libtiff/-/commit/688012dca2c39033aa2dc7bcea9796787cfd1b44
Conflict:Adaptation Context

---
 tools/tiffcrop.c | 38 +++++++++++++++++++++++++++++++++++---
 1 file changed, 35 insertions(+), 3 deletions(-)

diff --git a/tools/tiffcrop.c b/tools/tiffcrop.c
index 3f839d1..e2f8b83 100644
--- a/tools/tiffcrop.c
+++ b/tools/tiffcrop.c
@@ -8680,7 +8680,8 @@ rotateImage(uint16_t rotation, struct image_data *image, uint32_t *img_width,
   uint32_t   bytes_per_pixel, bytes_per_sample;
   uint32_t   row, rowsize, src_offset, dst_offset;
   uint32_t   i, col, width, length;
-  uint32_t   colsize, buffsize, col_offset, pix_offset;
+  uint32_t   colsize, col_offset, pix_offset;
+  tmsize_t   buffsize;
   unsigned char *ibuff;
   unsigned char *src;
   unsigned char *dst;
@@ -8693,12 +8694,40 @@ rotateImage(uint16_t rotation, struct image_data *image, uint32_t *img_width,
   spp = image->spp;
   bps = image->bps;
 
+  if ((spp != 0 && bps != 0 &&
+       width > (uint32_t)((UINT32_MAX - 7) / spp / bps)) ||
+      (spp != 0 && bps != 0 &&
+       length > (uint32_t)((UINT32_MAX - 7) / spp / bps)))
+  {
+    TIFFError("rotateImage", "Integer overflow detected.");
+    return (-1);
+  }
   rowsize = ((bps * spp * width) + 7) / 8;
   colsize = ((bps * spp * length) + 7) / 8;
   if ((colsize * width) > (rowsize * length))
-    buffsize = (colsize + 1) * width;
+  {
+    if (((tmsize_t)colsize + 1) != 0 &&
+        (tmsize_t)width > ((TIFF_TMSIZE_T_MAX - NUM_BUFF_OVERSIZE_BYTES) /
+                           ((tmsize_t)colsize + 1)))
+    {
+        TIFFError("rotateImage",
+                  "Integer overflow when calculating buffer size.");
+        return (-1);
+    }
+    buffsize = ((tmsize_t)colsize + 1) * width;
+  }
   else
+  {
+    if (((tmsize_t)rowsize + 1) != 0 &&
+        (tmsize_t)length > ((TIFF_TMSIZE_T_MAX - NUM_BUFF_OVERSIZE_BYTES) /
+                            ((tmsize_t)rowsize + 1)))
+    {
+        TIFFError("rotateImage",
+                  "Integer overflow when calculating buffer size.");
+        return (-1);
+    }
     buffsize = (rowsize + 1) * length;
+  }
 
   bytes_per_sample = (bps + 7) / 8;
   bytes_per_pixel  = ((bps * spp) + 7) / 8;
@@ -8721,7 +8750,10 @@ rotateImage(uint16_t rotation, struct image_data *image, uint32_t *img_width,
   /* Add 3 padding bytes for extractContigSamplesShifted32bits */
   if (!(rbuff = (unsigned char *)limitMalloc(buffsize + NUM_BUFF_OVERSIZE_BYTES)))
     {
-    TIFFError("rotateImage", "Unable to allocate rotation buffer of %1u bytes", buffsize + NUM_BUFF_OVERSIZE_BYTES);
+    TIFFError("rotateImage",
+              "Unable to allocate rotation buffer of %" TIFF_SSIZE_FORMAT
+              " bytes ",
+              buffsize + NUM_BUFF_OVERSIZE_BYTES);
     return (-1);
     }
   _TIFFmemset(rbuff, '\0', buffsize + NUM_BUFF_OVERSIZE_BYTES);
-- 
2.27.0

