[PATCH v2] xfree86/modes: Take rotation into account when checking mode size

[ville.syrjala at nokia.com]

From: Ville Syrjälä <ville.syrjala at nokia.com>

Assume that a mode can be used in either landscape or portrait
orientation. I suppose the correct thing to do would be to
collect all the supported rotations from the CRTCs that can be used
with a specific output, but that information doesn't seem to be
readily available when these checks are done. So just assume that
either orientation is fine.

Signed-off-by: Ville Syrjälä <ville.syrjala at nokia.com>
---
 hw/xfree86/modes/xf86Modes.c |   33 +++++++++++++++++++++++++--------
 1 files changed, 25 insertions(+), 8 deletions(-)

diff --git a/hw/xfree86/modes/xf86Modes.c b/hw/xfree86/modes/xf86Modes.c
index 05f4319..ef89050 100644
--- a/hw/xfree86/modes/xf86Modes.c
+++ b/hw/xfree86/modes/xf86Modes.c
@@ -364,15 +364,32 @@ xf86ValidateModesSize(ScrnInfoPtr pScrn, DisplayModePtr modeList,
 {
     DisplayModePtr mode;
 
-    for (mode = modeList; mode != NULL; mode = mode->next) {
-	if (maxPitch > 0 && mode->HDisplay > maxPitch)
-	    mode->status = MODE_BAD_WIDTH;
-
-	if (maxX > 0 && mode->HDisplay > maxX)
-	    mode->status = MODE_VIRTUAL_X;
+    if (maxPitch <= 0)
+	    maxPitch = MAXINT;
+    if (maxX <= 0)
+	    maxX = MAXINT;
+    if (maxY <= 0)
+	    maxY = MAXINT;
 
-	if (maxY > 0 && mode->VDisplay > maxY)
-	    mode->status = MODE_VIRTUAL_Y;
+    for (mode = modeList; mode != NULL; mode = mode->next) {
+	if ((xf86ModeWidth(mode, RR_Rotate_0) > maxPitch ||
+	     xf86ModeWidth(mode, RR_Rotate_0) > maxX ||
+	     xf86ModeHeight(mode, RR_Rotate_0) > maxY) &&
+	    (xf86ModeWidth(mode, RR_Rotate_90) > maxPitch ||
+	     xf86ModeWidth(mode, RR_Rotate_90) > maxX ||
+	     xf86ModeHeight(mode, RR_Rotate_90) > maxY)) {
+	    if (xf86ModeWidth(mode, RR_Rotate_0) > maxPitch ||
+		xf86ModeWidth(mode, RR_Rotate_90) > maxPitch)
+		mode->status = MODE_BAD_WIDTH;
+
+	    if (xf86ModeWidth(mode, RR_Rotate_0) > maxX ||
+		xf86ModeWidth(mode, RR_Rotate_90) > maxX)
+		mode->status = MODE_VIRTUAL_X;
+
+	    if (xf86ModeHeight(mode, RR_Rotate_0) > maxY ||
+		xf86ModeHeight(mode, RR_Rotate_90) > maxY)
+		mode->status = MODE_VIRTUAL_Y;
+	}
 
 	if (mode->next == modeList)
 	    break;
-- 
1.7.2.2