[PATCH 9/9] drm/vmwgfx: Properly check display/scanout surface size

[Sinclair Yeh]

The scanout surface size is the smaller of max texture size and
max STDU size.

Signed-off-by: Sinclair Yeh <syeh at vmware.com>
Reviewed-by: Thomas Hellstrom <thellstrom at vmware.com>
---
 drivers/gpu/drm/vmwgfx/vmwgfx_kms.c     |  3 +++
 drivers/gpu/drm/vmwgfx/vmwgfx_stdu.c    | 16 +++++++++++++++-
 drivers/gpu/drm/vmwgfx/vmwgfx_surface.c | 14 ++++++++++++++
 3 files changed, 32 insertions(+), 1 deletion(-)

diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
index 881ffd4..9b0bf2c 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
@@ -2218,7 +2218,10 @@ int vmw_du_connector_fill_modes(struct drm_connector *connector,
 
 	if (dev_priv->active_display_unit == vmw_du_screen_target) {
 		max_width  = min(max_width,  dev_priv->stdu_max_width);
+		max_width  = min(max_width,  dev_priv->texture_max_width);
+
 		max_height = min(max_height, dev_priv->stdu_max_height);
+		max_height = min(max_height, dev_priv->texture_max_height);
 	}
 
 	/* Add preferred mode */
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_stdu.c b/drivers/gpu/drm/vmwgfx/vmwgfx_stdu.c
index ce1191a..6ae2042 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_stdu.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_stdu.c
@@ -1602,7 +1602,21 @@ int vmw_kms_stdu_init_display(struct vmw_private *dev_priv)
 
 	dev_priv->active_display_unit = vmw_du_screen_target;
 
-	if (!(dev_priv->capabilities & SVGA_CAP_3D)) {
+	if (dev_priv->capabilities & SVGA_CAP_3D) {
+		/*
+		 * For 3D VMs, display (scanout) buffer size is the smaller of
+		 * max texture and max STDU
+		 */
+		uint32_t max_width, max_height;
+
+		max_width = min(dev_priv->texture_max_width,
+				dev_priv->stdu_max_width);
+		max_height = min(dev_priv->texture_max_height,
+				 dev_priv->stdu_max_height);
+
+		dev->mode_config.max_width = max_width;
+		dev->mode_config.max_height = max_height;
+	} else {
 		/*
 		 * Given various display aspect ratios, there's no way to
 		 * estimate these using prim_bb_mem.  So just set these to
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_surface.c b/drivers/gpu/drm/vmwgfx/vmwgfx_surface.c
index 6abcf82..41b9d20 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_surface.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_surface.c
@@ -1479,10 +1479,24 @@ int vmw_surface_gb_priv_define(struct drm_device *dev,
 	*srf_out = NULL;
 
 	if (for_scanout) {
+		uint32_t max_width, max_height;
+
 		if (!svga3dsurface_is_screen_target_format(format)) {
 			DRM_ERROR("Invalid Screen Target surface format.");
 			return -EINVAL;
 		}
+
+		max_width = min(dev_priv->texture_max_width,
+				dev_priv->stdu_max_width);
+		max_height = min(dev_priv->texture_max_height,
+				 dev_priv->stdu_max_height);
+
+		if (size.width > max_width || size.height > max_height) {
+			DRM_ERROR("%ux%u\n, exeeds max surface size %ux%u",
+				  size.width, size.height,
+				  max_width, max_height);
+			return -EINVAL;
+		}
 	} else {
 		const struct svga3d_surface_desc *desc;
 
-- 
2.7.4