[PATCH 2/5] drm/rect: Handle rounding errors in drm_rect_clip_scaled, v2.

[Maarten Lankhorst]

No matter how you perform the clip adjustments, a small
error may push the scaling factor to the other side of
0x10000. Solve this with a macro that will fixup the
scale to 0x10000 if we accidentally wrap to the other side.

Changes since v1:
- Adjust dst immediately, else drm_rect_width/height on dst gives bogus
  results.

Signed-off-by: Maarten Lankhorst <maarten.lankhorst at linux.intel.com>
---
 drivers/gpu/drm/drm_rect.c | 71 ++++++++++++++++++++++++++++----------
 1 file changed, 52 insertions(+), 19 deletions(-)

diff --git a/drivers/gpu/drm/drm_rect.c b/drivers/gpu/drm/drm_rect.c
index b179c7c73cc5..9d6194780a74 100644
--- a/drivers/gpu/drm/drm_rect.c
+++ b/drivers/gpu/drm/drm_rect.c
@@ -50,6 +50,24 @@ bool drm_rect_intersect(struct drm_rect *r1, const struct drm_rect *r2)
 }
 EXPORT_SYMBOL(drm_rect_intersect);
 
+static int drm_calc_scale(int src, int dst)
+{
+	int scale = 0;
+
+	if (WARN_ON(src < 0 || dst < 0))
+		return -EINVAL;
+
+	if (dst == 0)
+		return 0;
+
+	if (src > (dst << 16))
+		return DIV_ROUND_UP(src, dst);
+	else
+		scale = src / dst;
+
+	return scale;
+}
+
 /**
  * drm_rect_clip_scaled - perform a scaled clip operation
  * @src: source window rectangle
@@ -71,49 +89,64 @@ bool drm_rect_clip_scaled(struct drm_rect *src, struct drm_rect *dst,
 {
 	int diff;
 
+	/*
+	 * When scale is near 1.0 rounding errors may cause the scaling
+	 * factor to the other side. Some hardware may support
+	 * upsampling, but not downsampling, and that would break when
+	 * rounding.
+	 */
+#define FIXUP(oldscale, fn, m, second) do { \
+	if (oldscale != 1 << 16) { \
+		int newscale = drm_calc_scale(fn(src), fn(dst)); \
+ \
+		if (newscale < 0) \
+			return false; \
+ \
+		if ((oldscale < 0x10000) != (newscale < 0x10000)) { \
+			if (!second) \
+				src->m##1 = src->m##2 - (fn(dst) << 16); \
+			else \
+				src->m##2 = src->m##1 + (fn(dst) << 16); \
+		} \
+	} \
+ } while (0)
+
 	diff = clip->x1 - dst->x1;
 	if (diff > 0) {
 		int64_t tmp = src->x1 + (int64_t) diff * hscale;
 		src->x1 = clamp_t(int64_t, tmp, INT_MIN, INT_MAX);
+		dst->x1 = clip->x1;
+		FIXUP(hscale, drm_rect_width, x, 0);
 	}
+
 	diff = clip->y1 - dst->y1;
 	if (diff > 0) {
 		int64_t tmp = src->y1 + (int64_t) diff * vscale;
 		src->y1 = clamp_t(int64_t, tmp, INT_MIN, INT_MAX);
+		dst->y1 = clip->y1;
+		FIXUP(vscale, drm_rect_height, y, 0);
 	}
+
 	diff = dst->x2 - clip->x2;
 	if (diff > 0) {
 		int64_t tmp = src->x2 - (int64_t) diff * hscale;
 		src->x2 = clamp_t(int64_t, tmp, INT_MIN, INT_MAX);
+		dst->x2 = clip->x2;
+		FIXUP(hscale, drm_rect_width, x, 1);
 	}
 	diff = dst->y2 - clip->y2;
 	if (diff > 0) {
 		int64_t tmp = src->y2 - (int64_t) diff * vscale;
 		src->y2 = clamp_t(int64_t, tmp, INT_MIN, INT_MAX);
+		dst->y2 = clip->y2;
+		FIXUP(vscale, drm_rect_height, y, 1);
 	}
+#undef FIXUP
 
-	return drm_rect_intersect(dst, clip);
+	return drm_rect_visible(dst);
 }
 EXPORT_SYMBOL(drm_rect_clip_scaled);
 
-static int drm_calc_scale(int src, int dst)
-{
-	int scale = 0;
-
-	if (WARN_ON(src < 0 || dst < 0))
-		return -EINVAL;
-
-	if (dst == 0)
-		return 0;
-
-	if (src > (dst << 16))
-		return DIV_ROUND_UP(src, dst);
-	else
-		scale = src / dst;
-
-	return scale;
-}
-
 /**
  * drm_rect_calc_hscale - calculate the horizontal scaling factor
  * @src: source window rectangle
-- 
2.17.0