Blame SOURCES/libtiff-CVE-2012-2088.patch

0d8ff3
Do strip and tile size calculations in unsigned arithmetic, and then
0d8ff3
complain if the result overflows signed int32, because callers of these
0d8ff3
functions expect signed results (tsize_t is signed).  CVE-2012-2088
0d8ff3
0d8ff3
NB: must be applied after libtiff-subsampling.patch to avoid fuzz issues.
0d8ff3
0d8ff3
0d8ff3
diff -Naur tiff-3.9.4.orig/libtiff/tif_strip.c tiff-3.9.4/libtiff/tif_strip.c
0d8ff3
--- tiff-3.9.4.orig/libtiff/tif_strip.c	2010-06-08 14:50:43.000000000 -0400
0d8ff3
+++ tiff-3.9.4/libtiff/tif_strip.c	2012-06-27 12:37:55.054788399 -0400
0d8ff3
@@ -107,6 +107,7 @@
0d8ff3
 TIFFVStripSize(TIFF* tif, uint32 nrows)
0d8ff3
 {
0d8ff3
 	TIFFDirectory *td = &tif->tif_dir;
0d8ff3
+	uint32 stripsize;
0d8ff3
 
0d8ff3
 	if (nrows == (uint32) -1)
0d8ff3
 		nrows = td->td_imagelength;
0d8ff3
@@ -122,7 +123,7 @@
0d8ff3
 		 * YCbCr data for the extended image.
0d8ff3
 		 */
0d8ff3
 		uint16 ycbcrsubsampling[2];
0d8ff3
-		tsize_t w, scanline, samplingarea;
0d8ff3
+		uint32 w, scanline, samplingarea;
0d8ff3
 
0d8ff3
 		TIFFGetFieldDefaulted(tif, TIFFTAG_YCBCRSUBSAMPLING,
0d8ff3
 				      ycbcrsubsampling + 0,
0d8ff3
@@ -141,13 +142,27 @@
0d8ff3
 		nrows = TIFFroundup(nrows, ycbcrsubsampling[1]);
0d8ff3
 		/* NB: don't need TIFFhowmany here 'cuz everything is rounded */
0d8ff3
 		scanline = multiply(tif, nrows, scanline, "TIFFVStripSize");
0d8ff3
-		return ((tsize_t)
0d8ff3
-		    summarize(tif, scanline,
0d8ff3
-			      multiply(tif, 2, scanline / samplingarea,
0d8ff3
-				       "TIFFVStripSize"), "TIFFVStripSize"));
0d8ff3
+		/* a zero anywhere in here means overflow, must return zero */
0d8ff3
+		if (scanline > 0) {
0d8ff3
+			uint32 extra =
0d8ff3
+			    multiply(tif, 2, scanline / samplingarea,
0d8ff3
+				     "TIFFVStripSize");
0d8ff3
+			if (extra > 0)
0d8ff3
+				stripsize = summarize(tif, scanline, extra,
0d8ff3
+						      "TIFFVStripSize");
0d8ff3
+			else
0d8ff3
+				stripsize = 0;
0d8ff3
+		} else
0d8ff3
+			stripsize = 0;
0d8ff3
 	} else
0d8ff3
-		return ((tsize_t) multiply(tif, nrows, TIFFScanlineSize(tif),
0d8ff3
-					   "TIFFVStripSize"));
0d8ff3
+		stripsize = multiply(tif, nrows, TIFFScanlineSize(tif),
0d8ff3
+				     "TIFFVStripSize");
0d8ff3
+	/* Because tsize_t is signed, we might have conversion overflow */
0d8ff3
+	if (((tsize_t) stripsize) < 0) {
0d8ff3
+		TIFFErrorExt(tif->tif_clientdata, tif->tif_name, "Integer overflow in %s", "TIFFVStripSize");
0d8ff3
+		stripsize = 0;
0d8ff3
+	}
0d8ff3
+	return (tsize_t) stripsize;
0d8ff3
 }
0d8ff3
 
0d8ff3
 
0d8ff3
diff -Naur tiff-3.9.4.orig/libtiff/tif_tile.c tiff-3.9.4/libtiff/tif_tile.c
0d8ff3
--- tiff-3.9.4.orig/libtiff/tif_tile.c	2010-06-08 14:50:43.000000000 -0400
0d8ff3
+++ tiff-3.9.4/libtiff/tif_tile.c	2012-06-27 12:37:55.055788446 -0400
0d8ff3
@@ -174,7 +174,7 @@
0d8ff3
 TIFFTileRowSize(TIFF* tif)
0d8ff3
 {
0d8ff3
 	TIFFDirectory *td = &tif->tif_dir;
0d8ff3
-	tsize_t rowsize;
0d8ff3
+	uint32 rowsize;
0d8ff3
 	
0d8ff3
 	if (td->td_tilelength == 0 || td->td_tilewidth == 0)
0d8ff3
 		return ((tsize_t) 0);
0d8ff3
@@ -193,7 +193,7 @@
0d8ff3
 TIFFVTileSize(TIFF* tif, uint32 nrows)
0d8ff3
 {
0d8ff3
 	TIFFDirectory *td = &tif->tif_dir;
0d8ff3
-	tsize_t tilesize;
0d8ff3
+	uint32 tilesize;
0d8ff3
 
0d8ff3
 	if (td->td_tilelength == 0 || td->td_tilewidth == 0 ||
0d8ff3
 	    td->td_tiledepth == 0)
0d8ff3
@@ -209,12 +209,12 @@
0d8ff3
 		 * horizontal/vertical subsampling area include
0d8ff3
 		 * YCbCr data for the extended image.
0d8ff3
 		 */
0d8ff3
-		tsize_t w =
0d8ff3
+		uint32 w =
0d8ff3
 		    TIFFroundup(td->td_tilewidth, td->td_ycbcrsubsampling[0]);
0d8ff3
-		tsize_t rowsize =
0d8ff3
+		uint32 rowsize =
0d8ff3
 		    TIFFhowmany8(multiply(tif, w, td->td_bitspersample,
0d8ff3
 					  "TIFFVTileSize"));
0d8ff3
-		tsize_t samplingarea =
0d8ff3
+		uint32 samplingarea =
0d8ff3
 		    td->td_ycbcrsubsampling[0]*td->td_ycbcrsubsampling[1];
0d8ff3
 		if (samplingarea == 0) {
0d8ff3
 			TIFFErrorExt(tif->tif_clientdata, tif->tif_name, "Invalid YCbCr subsampling");
0d8ff3
@@ -223,15 +223,27 @@
0d8ff3
 		nrows = TIFFroundup(nrows, td->td_ycbcrsubsampling[1]);
0d8ff3
 		/* NB: don't need TIFFhowmany here 'cuz everything is rounded */
0d8ff3
 		tilesize = multiply(tif, nrows, rowsize, "TIFFVTileSize");
0d8ff3
-		tilesize = summarize(tif, tilesize,
0d8ff3
-				     multiply(tif, 2, tilesize / samplingarea,
0d8ff3
-					      "TIFFVTileSize"),
0d8ff3
+		/* a zero anywhere in here means overflow, must return zero */
0d8ff3
+		if (tilesize > 0) {
0d8ff3
+			uint32 extra =
0d8ff3
+			    multiply(tif, 2, tilesize / samplingarea,
0d8ff3
 				     "TIFFVTileSize");
0d8ff3
+			if (extra > 0)
0d8ff3
+				tilesize = summarize(tif, tilesize, extra,
0d8ff3
+						     "TIFFVTileSize");
0d8ff3
+			else
0d8ff3
+				tilesize = 0;
0d8ff3
+		}
0d8ff3
 	} else
0d8ff3
 		tilesize = multiply(tif, nrows, TIFFTileRowSize(tif),
0d8ff3
 				    "TIFFVTileSize");
0d8ff3
-	return ((tsize_t)
0d8ff3
-	    multiply(tif, tilesize, td->td_tiledepth, "TIFFVTileSize"));
0d8ff3
+	tilesize = multiply(tif, tilesize, td->td_tiledepth, "TIFFVTileSize");
0d8ff3
+	/* Because tsize_t is signed, we might have conversion overflow */
0d8ff3
+	if (((tsize_t) tilesize) < 0) {
0d8ff3
+		TIFFErrorExt(tif->tif_clientdata, tif->tif_name, "Integer overflow in %s", "TIFFVTileSize");
0d8ff3
+		tilesize = 0;
0d8ff3
+	}
0d8ff3
+	return (tsize_t) tilesize;
0d8ff3
 }
0d8ff3
 
0d8ff3
 /*