address some more software overflows

replaces the old overflow clamping since we now have a function for this purpose

src/m_fixed.h

@@ -20,6 +20,7 @@ extern "C" {
 #endif
 
 #include "doomtype.h"
+
 #ifdef __GNUC__
 #include <stdlib.h>
 #endif
@@ -71,6 +72,14 @@ FUNCMATH FUNCINLINE static ATTRINLINE fixed_t DoubleToFixed(double f)
 #define FIXED_TO_FLOAT(x) FixedToFloat(x) // (((float)(x)) / ((float)FRACUNIT))
 #define FLOAT_TO_FIXED(f) FloatToFixed(f) // (fixed_t)((f) * ((float)FRACUNIT))
 
+FUNCMATH FUNCINLINE static ATTRINLINE fixed_t FixedClamp(INT64 val)
+{
+// stoopid includes
+#define FCLAMP(x, y, z) ((x) < (y) ? (y) : ((x) > (z) ? (z) : (x)))
+	return (fixed_t)(FCLAMP(val, (INT64)INT32_MIN, (INT64)INT32_MAX));
+#undef FCLAMP
+}
+
 /**	\brief	The FixedMul function
 
 	\param	a	fixed_t number

src/r_main.cpp

@@ -519,10 +519,10 @@ void R_GetRenderBlockMapDimensions(fixed_t drawdist, INT32 *xl, INT32 *xh, INT32
 	const fixed_t vyright = viewy + FixedMul(drawdist, FSIN(right));
 
 	// Try to narrow the search to within only the field of view
-	*xl = (unsigned)(std::min(viewx, std::min(vxleft, vxright)) - bmaporgx)>>MAPBLOCKSHIFT;
+	*xl = (unsigned)(FixedClamp((INT64)std::min(viewx, std::min(vxleft, vxright)) - bmaporgx))>>MAPBLOCKSHIFT;
-	*xh = (unsigned)(std::max(viewx, std::max(vxleft, vxright)) - bmaporgx)>>MAPBLOCKSHIFT;
+	*xh = (unsigned)(FixedClamp((INT64)std::max(viewx, std::max(vxleft, vxright)) - bmaporgx))>>MAPBLOCKSHIFT;
-	*yl = (unsigned)(std::min(viewy, std::min(vyleft, vyright)) - bmaporgy)>>MAPBLOCKSHIFT;
+	*yl = (unsigned)(FixedClamp((INT64)std::min(viewy, std::min(vyleft, vyright)) - bmaporgy))>>MAPBLOCKSHIFT;
-	*yh = (unsigned)(std::max(viewy, std::max(vyleft, vyright)) - bmaporgy)>>MAPBLOCKSHIFT;
+	*yh = (unsigned)(FixedClamp((INT64)std::max(viewy, std::max(vyleft, vyright)) - bmaporgy))>>MAPBLOCKSHIFT;
 
 	if (*xh >= bmapwidth)
 		*xh = bmapwidth - 1;

src/r_main.h

@@ -152,8 +152,8 @@ subsector_t *R_IsPointInSubsector(fixed_t x, fixed_t y);
 
 FUNCMATH FUNCINLINE static ATTRINLINE fixed_t R_QuickDist(fixed_t x1, fixed_t y1, fixed_t x, fixed_t y)
 {
-	const fixed_t absx = abs(x - x1);
+	const fixed_t absx = FixedClamp(labs((INT64)x - (INT64)x1));
-	const fixed_t absy = abs(y - y1);
+	const fixed_t absy = FixedClamp(labs((INT64)y - (INT64)y1));
 #ifdef __cplusplus
 	return std::max<fixed_t>(absx, absy);
 #else

src/r_plane.cpp

@@ -228,8 +228,8 @@ static void R_MapPlane(drawspandata_t *ds, spandrawfunc_t *localspanfunc, INT32
 	// to step from those to the proper texture coordinate to start drawing at.
 	// That way, the texture coordinate is always calculated by its position
 	// on the screen and not by its position relative to the edge of the visplane.
-	ds->xfrac = ds->xoffs + FixedMul(planecos, distance) + (x1 - centerx) * ds->xstep;
+	ds->xfrac = FixedClamp((INT64)ds->xoffs + (INT64)FixedMul(planecos, distance) + ((INT64)x1 - (INT64)centerx) * (INT64)ds->xstep);
-	ds->yfrac = ds->yoffs - FixedMul(planesin, distance) + (x1 - centerx) * ds->ystep;
+	ds->yfrac = FixedClamp((INT64)ds->yoffs - (INT64)FixedMul(planesin, distance) + ((INT64)x1 - (INT64)centerx) * (INT64)ds->ystep);
 
 	if (ds->planeripple.active)
 	{
@@ -1012,7 +1012,7 @@ void R_DrawSinglePlane(drawspandata_t* ds, visplane_t *pl, boolean allow_paralle
 	levelflat_t *levelflat;
 	void (*mapfunc)(drawspandata_t*, void(*)(drawspandata_t*), INT32, INT32, INT32, boolean) = R_MapPlane;
 
-	if (pl->minx > pl->maxx)
+	if (!(pl->minx <= pl->maxx))
 		return;
 
 	// sky flat

src/r_portal.c

@@ -312,8 +312,8 @@ void Portal_AddSkybox(const visplane_t* plane)
 			y = (viewy - skyboxmo[1]->y) * -mh->skybox_scaley;
 
 		// Apply transform to account for the skybox viewport angle.
-		portal->viewx += FixedMul(x,FINECOSINE(ang)) - FixedMul(y,  FINESINE(ang));
+		portal->viewx += FixedMul(x, FINECOSINE(ang)) - FixedMul(y,  FINESINE(ang));
-		portal->viewy += FixedMul(x,  FINESINE(ang)) + FixedMul(y,FINECOSINE(ang));
+		portal->viewy += FixedMul(x,  FINESINE(ang)) + FixedMul(y, FINECOSINE(ang));
 	}
 
 	if (mh->skybox_scalez > 0)

src/r_segs.cpp

@@ -226,10 +226,11 @@ static void R_RenderMaskedSegLoop(drawcolumndata_t* dc, drawseg_t *drawseg, INT3
 			leftheight  -= viewz;
 			rightheight -= viewz;
 
-			rlight->height         = (centeryfrac) - FixedMul(leftheight, drawseg->scale1);
+			rlight->height      = FixedClamp((INT64)centeryfrac - FixedMul(leftheight, drawseg->scale1));
-			rlight->heightstep     = (centeryfrac) - FixedMul(rightheight, drawseg->scale2);
+			rlight->heightstep  = FixedClamp((INT64)centeryfrac - FixedMul(rightheight, drawseg->scale2));
-			rlight->heightstep     = (rlight->heightstep-rlight->height)/(range);
+			rlight->heightstep  = FixedClamp((INT64)rlight->heightstep - rlight->height)/(range);
-			rlight->startheight    = rlight->height; // keep starting value here to reset for each repeat
+			rlight->startheight = rlight->height; // keep starting value here to reset for each repeat
+
 			rlight->lightlevel     = *light->lightlevel;
 			rlight->extra_colormap = light->extra_colormap;
 			rlight->flags = static_cast<ffloortype_e>(light->flags);
@@ -341,7 +342,7 @@ static void R_RenderMaskedSegLoop(drawcolumndata_t* dc, drawseg_t *drawseg, INT3
 					for (i = 0; i < dc->numlights; i++)
 					{
 						rlight = &dc->lightlist[i];
-						rlight->height += rlight->heightstep;
+						rlight->height = FixedClamp((INT64)rlight->height + rlight->heightstep);
 					}
 				}
 
@@ -355,9 +356,9 @@ static void R_RenderMaskedSegLoop(drawcolumndata_t* dc, drawseg_t *drawseg, INT3
 				lighttable_t **xwalllights;
 
 				sprbotscreen = INT32_MAX;
-				sprtopscreen = windowtop = (centeryfrac - FixedMul(dc->texturemid, spryscale));
+				sprtopscreen = windowtop = FixedClamp((INT64)centeryfrac - FixedMul(dc->texturemid, spryscale));
 
-				realbot = windowbottom = FixedMul(textureheight[texnum], spryscale) + sprtopscreen;
+				realbot = windowbottom = FixedClamp((INT64)FixedMul(textureheight[texnum], spryscale) + sprtopscreen);
 				dc->iscale = 0xffffffffu / (unsigned)spryscale;
 
 				// draw the texture
@@ -407,7 +408,8 @@ static void R_RenderMaskedSegLoop(drawcolumndata_t* dc, drawseg_t *drawseg, INT3
 					set_light_vars(i);
 
 					height = rlight->height;
-					rlight->height += rlight->heightstep;
+
+					rlight->height = FixedClamp((INT64)rlight->height + rlight->heightstep);
 
 					if (height <= windowtop)
 					{
@@ -423,7 +425,7 @@ static void R_RenderMaskedSegLoop(drawcolumndata_t* dc, drawseg_t *drawseg, INT3
 						for (i++; i < dc->numlights; i++)
 						{
 							rlight = &dc->lightlist[i];
-							rlight->height += rlight->heightstep;
+							rlight->height = FixedClamp((INT64)rlight->height + rlight->heightstep);
 						}
 
 						continue;
@@ -455,7 +457,7 @@ static void R_RenderMaskedSegLoop(drawcolumndata_t* dc, drawseg_t *drawseg, INT3
 			if (frontsector->extra_colormap)
 				dc->colormap = frontsector->extra_colormap->colormap + (dc->colormap - colormaps);
 
-			sprtopscreen = centeryfrac - FixedMul(dc->texturemid, spryscale);
+			sprtopscreen = FixedClamp((INT64)centeryfrac - FixedMul(dc->texturemid, spryscale));
 			dc->iscale = 0xffffffffu / (unsigned)spryscale;
 
 			// draw the texture
@@ -710,17 +712,6 @@ void R_RenderThickSideRange(drawseg_t *drawseg, INT32 x1, INT32 x2, ffloor_t *pf
 	rw_scalestep = drawseg->scalestep;
 	spryscale = drawseg->scale1 + (x1 - drawseg->x1)*rw_scalestep;
 
-#define CLAMPMAX INT32_MAX
-#define CLAMPMIN (-INT32_MAX) // This is not INT32_MIN on purpose! INT32_MIN makes the drawers freak out.
-	auto overflow_clamp = [&](INT64 overflow_test)
-	{
-		return (overflow_test > (INT64)CLAMPMAX) ? CLAMPMAX :
-		(overflow_test > (INT64)CLAMPMIN) ? (fixed_t)overflow_test :
-		CLAMPMIN;
-	};
-#undef CLAMPMAX
-#undef CLAMPMIN
-
 	dc->numlights = 0;
 	if (frontsector->numlights)
 	{
@@ -736,7 +727,6 @@ void R_RenderThickSideRange(drawseg_t *drawseg, INT32 x1, INT32 x2, ffloor_t *pf
 		{
 			fixed_t leftheight, rightheight;
 			fixed_t pfloorleft, pfloorright;
-			INT64 overflow_test;
 			light = &frontsector->lightlist[i];
 			rlight = &dc->lightlist[p];
 
@@ -760,17 +750,14 @@ void R_RenderThickSideRange(drawseg_t *drawseg, INT32 x1, INT32 x2, ffloor_t *pf
 					continue;
 			}
 
-			leftheight -= viewz;
+			leftheight  -= viewz;
 			rightheight -= viewz;
 
 			// Monster Iestyn (25/03/18): do not skip these lights if they fail overflow test, just clamp them instead so they behave.
-			overflow_test = (INT64)centeryfrac - (((INT64)leftheight*drawseg->scale1)>>FRACBITS);
+			rlight->height     = FixedClamp((INT64)centeryfrac - (((INT64)leftheight * drawseg->scale1)>>FRACBITS));
-			rlight->height = overflow_clamp(overflow_test);
+			rlight->heightstep = FixedClamp((INT64)centeryfrac - (((INT64)rightheight * drawseg->scale2)>>FRACBITS));
+			rlight->heightstep = FixedClamp((INT64)rlight->heightstep - rlight->height)/(range);
 
-			overflow_test = (INT64)centeryfrac - (((INT64)rightheight*drawseg->scale2)>>FRACBITS);
-			rlight->heightstep = overflow_clamp(overflow_test);
-
-			rlight->heightstep = (rlight->heightstep-rlight->height)/(range);
 			rlight->flags = static_cast<ffloortype_e>(light->flags);
 
 			if (light->flags & FF_CUTLEVEL)
@@ -781,13 +768,9 @@ void R_RenderThickSideRange(drawseg_t *drawseg, INT32 x1, INT32 x2, ffloor_t *pf
 				rightheight -= viewz;
 
 				// Monster Iestyn (25/03/18): do not skip these lights if they fail overflow test, just clamp them instead so they behave.
-				overflow_test = (INT64)centeryfrac - (((INT64)leftheight*drawseg->scale1)>>FRACBITS);
+				rlight->botheight     = FixedClamp((INT64)centeryfrac - (((INT64)leftheight*drawseg->scale1)>>FRACBITS));
-				rlight->botheight = overflow_clamp(overflow_test);
+				rlight->botheightstep = FixedClamp(((INT64)centeryfrac - (((INT64)rightheight*drawseg->scale2)>>FRACBITS)));
-
+				rlight->botheightstep = FixedClamp((INT64)rlight->botheightstep - rlight->botheight)/(range);
-				overflow_test = (INT64)centeryfrac - (((INT64)rightheight*drawseg->scale2)>>FRACBITS);
-				rlight->botheightstep = overflow_clamp(overflow_test);
-
-				rlight->botheightstep = (rlight->botheightstep-rlight->botheight)/(range);
 			}
 
 			rlight->lightlevel = *light->lightlevel;
@@ -896,10 +879,10 @@ void R_RenderThickSideRange(drawseg_t *drawseg, INT32 x1, INT32 x2, ffloor_t *pf
 		right_bottom = P_GetFFloorBottomZAt(pfloor, drawseg->rightpos.x, drawseg->rightpos.y) - viewz;
 
 		// using INT64 to avoid 32bit overflow
-		top_frac =    (INT64)centeryfrac - (((INT64)left_top     * drawseg->scale1) >> FRACBITS);
+		top_frac    = FixedClamp((INT64)centeryfrac - (((INT64)left_top     * drawseg->scale1) >> FRACBITS));
-		bottom_frac = (INT64)centeryfrac - (((INT64)left_bottom  * drawseg->scale1) >> FRACBITS);
+		top_step    = FixedClamp((INT64)centeryfrac - (((INT64)right_top    * drawseg->scale2) >> FRACBITS));
-		top_step =    (INT64)centeryfrac - (((INT64)right_top    * drawseg->scale2) >> FRACBITS);
+		bottom_frac = FixedClamp((INT64)centeryfrac - (((INT64)left_bottom  * drawseg->scale1) >> FRACBITS));
-		bottom_step = (INT64)centeryfrac - (((INT64)right_bottom * drawseg->scale2) >> FRACBITS);
+		bottom_step = FixedClamp((INT64)centeryfrac - (((INT64)right_bottom * drawseg->scale2) >> FRACBITS));
 
 		top_step = (top_step-top_frac)/(range);
 		bottom_step = (bottom_step-bottom_frac)/(range);
@@ -921,8 +904,8 @@ void R_RenderThickSideRange(drawseg_t *drawseg, INT32 x1, INT32 x2, ffloor_t *pf
 
 		// Calculate bounds
 		// clamp the values if necessary to avoid overflows and rendering glitches caused by them
-		sprtopscreen = windowtop = overflow_clamp(top_frac);
+		sprtopscreen = windowtop = top_frac;
-		sprbotscreen = windowbottom = overflow_clamp(bottom_frac);
+		sprbotscreen = windowbottom = bottom_frac;
 
 		top_frac += top_step;
 		bottom_frac += bottom_step;
@@ -935,9 +918,11 @@ void R_RenderThickSideRange(drawseg_t *drawseg, INT32 x1, INT32 x2, ffloor_t *pf
 				for (i = 0; i < dc->numlights; i++)
 				{
 					rlight = &dc->lightlist[i];
-					rlight->height += rlight->heightstep;
+
+					rlight->height = FixedClamp((INT64)rlight->height + rlight->heightstep);
+
 					if (rlight->flags & FF_CUTLEVEL)
-						rlight->botheight += rlight->botheightstep;
+						rlight->botheight = FixedClamp((INT64)rlight->botheight + rlight->botheightstep);
 				}
 			}
 
@@ -1045,12 +1030,12 @@ void R_RenderThickSideRange(drawseg_t *drawseg, INT32 x1, INT32 x2, ffloor_t *pf
 					solid = 0;
 
 				height = rlight->height;
-				rlight->height += rlight->heightstep;
+				rlight->height = FixedClamp((INT64)rlight->height + rlight->heightstep);
 
 				if (solid)
 				{
-					bheight = rlight->botheight - (FRACUNIT >> 1);
+					bheight = FixedClamp((INT64)rlight->botheight - (FRACUNIT >> 1));
-					rlight->botheight += rlight->botheightstep;
+					rlight->botheight = FixedClamp((INT64)rlight->botheight + rlight->botheightstep);
 				}
 
 				if (height <= windowtop)
@@ -1078,7 +1063,7 @@ void R_RenderThickSideRange(drawseg_t *drawseg, INT32 x1, INT32 x2, ffloor_t *pf
 					for (i++; i < dc->numlights; i++)
 					{
 						rlight = &dc->lightlist[i];
-						rlight->height += rlight->heightstep;
+						rlight->height = FixedClamp((INT64)rlight->height + rlight->heightstep);
 
 						if (rlight->flags & FF_CUTLEVEL)
 							rlight->botheight += rlight->botheightstep;
@@ -1212,7 +1197,7 @@ static void R_RenderSegLoop(drawcolumndata_t* dc)
 	for (; rw_x < rw_stopx; rw_x++)
 	{
 		// mark floor / ceiling areas
-		yl = (topfrac+HEIGHTUNIT-1)>>HEIGHTBITS;
+		yl = FixedClamp((INT64)topfrac + HEIGHTUNIT-1)>>HEIGHTBITS;
 		yh = bottomfrac>>HEIGHTBITS;
 
 		// Mark ceiling
@@ -1363,15 +1348,17 @@ static void R_RenderSegLoop(drawcolumndata_t* dc)
 		//SoM: Calculate offsets for Thick fake floors.
 		// calculate texture offset
 		angle = (rw_centerangle + xtoviewangle[rw_x])>>ANGLETOFINESHIFT;
+
 		// Mask 4095 to guarantee this index is within bounds
-		texturecolumn = rw_offset-FixedMul(FINETANGENT(angle & 4095), rw_distance);
+		texturecolumn = FixedClamp((INT64)rw_offset - FixedMul(FINETANGENT(angle & 4095), rw_distance));
 
 		if (oldtexturecolumn != -1)
 		{
-			rw_bottomtexturemid += FixedMul(rw_bottomtextureslide,  oldtexturecolumn-texturecolumn);
+			const fixed_t coloff = FixedClamp((INT64)oldtexturecolumn - texturecolumn);
-			rw_midtexturemid    += FixedMul(rw_midtextureslide,     oldtexturecolumn-texturecolumn);
+			rw_bottomtexturemid += FixedMul(rw_bottomtextureslide,  coloff);
-			rw_toptexturemid    += FixedMul(rw_toptextureslide,     oldtexturecolumn-texturecolumn);
+			rw_midtexturemid    += FixedMul(rw_midtextureslide,     coloff);
-			rw_midtextureback   += FixedMul(rw_midtexturebackslide, oldtexturecolumn-texturecolumn);
+			rw_toptexturemid    += FixedMul(rw_toptextureslide,     coloff);
+			rw_midtextureback   += FixedMul(rw_midtexturebackslide, coloff);
 		}
 		oldtexturecolumn = texturecolumn;
 
@@ -1381,7 +1368,7 @@ static void R_RenderSegLoop(drawcolumndata_t* dc)
 			// calculate lighting
 			pindex = FixedMul(rw_scale, LIGHTRESOLUTIONFIX)>>LIGHTSCALESHIFT;
 
-			if (pindex >=  MAXLIGHTSCALE)
+			if (pindex >= MAXLIGHTSCALE)
 				pindex = MAXLIGHTSCALE-1;
 
 			dc->colormap = walllights[pindex];
@@ -1463,8 +1450,8 @@ static void R_RenderSegLoop(drawcolumndata_t* dc)
 			if (toptexture)
 			{
 				// top wall
-				mid = pixhigh>>HEIGHTBITS;
+				mid = pixhigh >> HEIGHTBITS;
-				pixhigh += pixhighstep;
+				pixhigh = FixedClamp((INT64)pixhigh + pixhighstep);
 
 				if (mid >= floorclip[rw_x])
 					mid = floorclip[rw_x]-1;
@@ -1493,8 +1480,8 @@ static void R_RenderSegLoop(drawcolumndata_t* dc)
 			if (bottomtexture)
 			{
 				// bottom wall
-				mid = (pixlow+HEIGHTUNIT-1)>>HEIGHTBITS;
+				mid = FixedClamp((INT64)pixlow+HEIGHTUNIT-1)>>HEIGHTBITS;
-				pixlow += pixlowstep;
+				pixlow = FixedClamp((INT64)pixlow + pixlowstep);
 
 				// no space above wall?
 				if (mid <= ceilingclip[rw_x])
@@ -1545,10 +1532,10 @@ static void R_RenderSegLoop(drawcolumndata_t* dc)
 		{
 			for (i = 0; i < dc->numlights; i++)
 			{
-				dc->lightlist[i].height += dc->lightlist[i].heightstep;
+				dc->lightlist[i].height = FixedClamp((INT64)dc->lightlist[i].height + dc->lightlist[i].heightstep);
 
 				if (dc->lightlist[i].flags & FF_CUTSOLIDS)
-					dc->lightlist[i].botheight += dc->lightlist[i].botheightstep;
+					dc->lightlist[i].botheight = FixedClamp((INT64)dc->lightlist[i].botheight + dc->lightlist[i].botheightstep);;
 			}
 		}
 
@@ -1569,9 +1556,10 @@ static void R_RenderSegLoop(drawcolumndata_t* dc)
 			visffloor[i].b_frac += visffloor[i].b_step;
 		}
 
-		rw_scale += rw_scalestep;
+		rw_scale = FixedClamp((INT64)rw_scale + rw_scalestep);
-		topfrac += topstep;
+
-		bottomfrac += bottomstep;
+		topfrac    = FixedClamp((INT64)topfrac + topstep);
+		bottomfrac = FixedClamp((INT64)bottomfrac + bottomstep);
 	}
 }
 
@@ -1585,7 +1573,7 @@ static void R_MarkSegBounds(void)
 	for (; rw_x < rw_stopx; rw_x++)
 	{
 		// mark floor / ceiling areas
-		yl = (topfrac+HEIGHTUNIT-1)>>HEIGHTBITS;
+		yl = FixedClamp((INT64)topfrac + HEIGHTUNIT-1)>>HEIGHTBITS;
 		yh = bottomfrac>>HEIGHTBITS;
 
 		// Mark ceiling
@@ -1822,7 +1810,7 @@ void R_StoreWallRange(INT32 start, INT32 stop)
 
 	// big room fix
 	if (longboi)
-		rw_distance = (fixed_t)R_CalcSegDist(curline,viewx,viewy);
+		rw_distance = FixedClamp(R_CalcSegDist(curline, viewx, viewy));
 
 	ds_p->x1 = rw_x = start;
 	ds_p->x2 = stop;
@@ -2205,8 +2193,8 @@ void R_StoreWallRange(INT32 start, INT32 stop)
 			}
 		}
 
-		rw_toptexturemid += sidedef->rowoffset;
+		rw_toptexturemid = FixedClamp((INT64)rw_toptexturemid + sidedef->rowoffset);
-		rw_bottomtexturemid += sidedef->rowoffset;
+		rw_bottomtexturemid = FixedClamp((INT64)rw_bottomtexturemid + sidedef->rowoffset);
 
 		R_AllocTextureColumnTables(rw_stopx - start);
 
@@ -2491,8 +2479,9 @@ void R_StoreWallRange(INT32 start, INT32 stop)
 			rw_offset = -rw_offset;
 
 		/// don't use texture offset for splats
-		rw_offset2 = rw_offset + curline->offset;
+		rw_offset2 = FixedClamp((INT64)rw_offset + curline->offset);
-		rw_offset += sidedef->textureoffset + curline->offset;
+		rw_offset = FixedClamp((INT64)rw_offset + (INT64)sidedef->textureoffset + curline->offset);
+
 		rw_centerangle = ANGLE_90 + viewangle - rw_normalangle;
 
 		// calculate light table
@@ -2552,8 +2541,8 @@ void R_StoreWallRange(INT32 start, INT32 stop)
 
 		auto set_topstep_normal = [&]
 		{
-			topstep = -FixedMul (rw_scalestep, worldtop);
+			topstep = -FixedMul(rw_scalestep, worldtop);
-			topfrac = (centeryfrac>>4) - FixedMul (worldtop, rw_scale);
+			topfrac = FixedClamp((INT64)(centeryfrac>>4) - FixedMul(worldtop, rw_scale));
 		};
 
 		// untextured seg
@@ -2574,14 +2563,14 @@ void R_StoreWallRange(INT32 start, INT32 stop)
 			// this is an attempt to fix issues with textureless single sided lines drawing nothing where they should just draw sky instead
 			if (tophigh && !backsector && frontsector->ceilingpic == skyflatnum)
 			{
-				topstep = -FixedMul (rw_scalestep, worldbottom);
+				topstep = -FixedMul(rw_scalestep, worldbottom);
-				topfrac = (centeryfrac>>4) - FixedMul (worldbottom, rw_scale);
+				topfrac = FixedClamp((INT64)(centeryfrac>>4) - FixedMul(worldbottom, rw_scale));
 
 				// account for slopes to try and get rid of sharp edges from the black void
 				if (frontsector->f_slope || (backsector && backsector->f_slope))
 				{
-					topstep = -FixedMul (rw_scalestep, worldbottomslope);
+					topstep = -FixedMul(rw_scalestep, worldbottomslope);
-					topfrac = (centeryfrac>>4) - FixedMul (worldbottomslope, rw_scale);
+					topfrac = FixedClamp((INT64)(centeryfrac>>4) - FixedMul(worldbottomslope, rw_scale));
 				}
 			}
 			else
@@ -2590,18 +2579,18 @@ void R_StoreWallRange(INT32 start, INT32 stop)
 		else
 			set_topstep_normal();
 
-		bottomstep = -FixedMul (rw_scalestep, worldbottom);
+		bottomstep = -FixedMul(rw_scalestep, worldbottom);
-		bottomfrac = (centeryfrac>>4) - FixedMul (worldbottom, rw_scale);
+		bottomfrac = FixedClamp((INT64)(centeryfrac>>4) - FixedMul(worldbottom, rw_scale));
 
 		if (frontsector->c_slope)
 		{
-			fixed_t topfracend = (centeryfrac>>4) - FixedMul (worldtopslope, ds_p->scale2);
+			fixed_t topfracend = FixedClamp((INT64)(centeryfrac>>4) - FixedMul(worldtopslope, ds_p->scale2));
 			topstep = (topfracend-topfrac)/(range);
 		}
 
 		if (frontsector->f_slope)
 		{
-			fixed_t bottomfracend = (centeryfrac>>4) - FixedMul (worldbottomslope, ds_p->scale2);
+			fixed_t bottomfracend = FixedClamp((INT64)(centeryfrac>>4) - FixedMul(worldbottomslope, ds_p->scale2));
 			bottomstep = (bottomfracend-bottomfrac)/(range);
 		}
 	}
@@ -2643,9 +2632,9 @@ void R_StoreWallRange(INT32 start, INT32 stop)
 					continue;
 			}
 
-			rlight->height = (centeryfrac>>4) - FixedMul(leftheight, rw_scale);
+			rlight->height = FixedClamp((INT64)(centeryfrac>>4) - FixedMul(leftheight, rw_scale));
-			rlight->heightstep = (centeryfrac>>4) - FixedMul(rightheight, ds_p->scale2);
+			rlight->heightstep = FixedClamp((INT64)(centeryfrac>>4) - FixedMul(rightheight, ds_p->scale2));
-			rlight->heightstep = (rlight->heightstep-rlight->height)/(range);
+			rlight->heightstep = FixedClamp((INT64)rlight->heightstep - rlight->height)/(range);
 			rlight->flags = static_cast<ffloortype_e>(light->flags);
 
 			if (light->caster && light->caster->flags & FF_CUTSOLIDS)
@@ -2707,9 +2696,9 @@ void R_StoreWallRange(INT32 start, INT32 stop)
 
 		if (toptexture)
 		{
-			fixed_t topfracend = (centeryfrac>>4) - FixedMul(worldhighslope, ds_p->scale2);
+			fixed_t topfracend = FixedClamp((INT64)(centeryfrac>>4) - FixedMul(worldhighslope, ds_p->scale2));
 
-			pixhigh = (centeryfrac>>4) - FixedMul (worldhigh, rw_scale);
+			pixhigh = FixedClamp((INT64)(centeryfrac>>4) - FixedMul(worldhigh, rw_scale));
 			pixhighstep = (topfracend-pixhigh)/(range);
 
 			// If the lowest part of a ceiling stretching down covers the entire screen
@@ -2719,9 +2708,9 @@ void R_StoreWallRange(INT32 start, INT32 stop)
 
 		if (bottomtexture)
 		{
-			fixed_t bottomfracend = (centeryfrac>>4) - FixedMul (worldlowslope, ds_p->scale2);
+			fixed_t bottomfracend = FixedClamp((INT64)(centeryfrac>>4) - FixedMul(worldlowslope, ds_p->scale2));
 
-			pixlow = (centeryfrac>>4) - FixedMul (worldlow, rw_scale);
+			pixlow = FixedClamp((INT64)(centeryfrac>>4) - FixedMul(worldlow, rw_scale));
 			pixlowstep = (bottomfracend-pixlow)/(range);
 
 			// If the highest part of a floor stretching up covers the entire screen
@@ -2742,8 +2731,8 @@ void R_StoreWallRange(INT32 start, INT32 stop)
 				visffloor[i].b_pos_slope = roverright;
 				visffloor[i].b_pos >>= 4;
 				visffloor[i].b_pos_slope >>= 4;
-				visffloor[i].b_frac = (centeryfrac >> 4) - FixedMul(visffloor[i].b_pos, rw_scale);
+				visffloor[i].b_frac = FixedClamp((INT64)(centeryfrac>>4) - FixedMul(visffloor[i].b_pos, rw_scale));
-				visffloor[i].b_step = (centeryfrac >> 4) - FixedMul(visffloor[i].b_pos_slope, ds_p->scale2);
+				visffloor[i].b_step = FixedClamp((INT64)(centeryfrac>>4) - FixedMul(visffloor[i].b_pos_slope, ds_p->scale2));
 				visffloor[i].b_step = (visffloor[i].b_step-visffloor[i].b_frac)/(range);
 			};
 
@@ -2849,7 +2838,7 @@ void R_StoreWallRange(INT32 start, INT32 stop)
 					visffloor[i].b_pos = height;
 					visffloor[i].b_pos = (visffloor[i].b_pos - viewz) >> 4;
 					visffloor[i].b_step = FixedMul(-rw_scalestep, visffloor[i].b_pos);
-					visffloor[i].b_frac = (centeryfrac >> 4) - FixedMul(visffloor[i].b_pos, rw_scale);
+					visffloor[i].b_frac = FixedClamp((INT64)(centeryfrac>>4) - FixedMul(visffloor[i].b_pos, rw_scale));
 				};
 
 				while (i < numffloors && visffloor[i].polyobj != curline->polyseg)

src/r_things.cpp

@@ -11,6 +11,7 @@
 /// \file  r_things.c
 /// \brief Refresh of things, i.e. objects represented by sprites
 
+#include "m_fixed.h"
 #include "r_main.h" // stplyr
 #include "r_fps.h"
 #include "r_things.h"
@@ -616,10 +617,11 @@ void R_DrawMaskedColumn(drawcolumndata_t* dc, column_t *column)
 		if (topdelta <= prevdelta)
 			topdelta += prevdelta;
 		prevdelta = topdelta;
-		topscreen = sprtopscreen + spryscale*topdelta;
-		bottomscreen = topscreen + spryscale*column->length;
 
-		dc->yl = (topscreen+FRACUNIT-1)>>FRACBITS;
+		topscreen    = FixedClamp((INT64)sprtopscreen + (INT64)spryscale * (INT64)topdelta);
+		bottomscreen = FixedClamp((INT64)topscreen + (INT64)spryscale * (INT64)column->length);
+
+		dc->yl = FixedClamp((INT64)topscreen + (FRACUNIT-1))>>FRACBITS;
 		dc->yh = (bottomscreen-1)>>FRACBITS;
 
 		if (windowtop != INT32_MAX && windowbottom != INT32_MAX)
@@ -885,7 +887,7 @@ static void R_DrawVisSprite(vissprite_t *vis)
 		for (dc.x = vis->x1; dc.x <= vis->x2; dc.x++, spryscale += scalestep)
 		{
 			angle_t angle = ((vis->centerangle + xtoviewangle[dc.x]) >> ANGLETOFINESHIFT) & 0xFFF;
-			texturecolumn = (vis->paperoffset - FixedMul(FINETANGENT(angle), vis->paperdistance)) / horzscale;
+			texturecolumn = FixedClamp((INT64)vis->paperoffset - FixedMul(FINETANGENT(angle), vis->paperdistance)) / horzscale;
 
 			if (texturecolumn < 0 || texturecolumn >= pwidth)
 				continue;