[PATCH] drm/vmwgfx: fix incorrect VRAM size check in vmw_kms_fb_create()

[Xi Wang]

The commit e133e737 didn't correctly fix the overflow issue.

-	unsigned int required_size;
+	u64 required_size;
	...
	required_size = mode_cmd->pitch * mode_cmd->height;
-	if (unlikely(required_size > dev_priv->vram_size)) {
+	if (unlikely(required_size > (u64) dev_priv->vram_size)) {

Since pitch and height are u32, their product is still 32-bit and
would overflow.  Converting the result to u64 cannot help.  A correct
way is to convert pitch and height to u64 before the multiplication.

	required_size = (u64)mode_cmd->pitch * (u64)mode_cmd->height;

This fix calls vmw_kms_validate_mode_vram() for validation.

Signed-off-by: Xi Wang <xi.wang at gmail.com>
Cc: Thomas Hellstrom <thellstrom at vmware.com>
Cc: Dave Airlie <airlied at redhat.com>
---
 drivers/gpu/drm/vmwgfx/vmwgfx_kms.c |    6 +++---
 1 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
index 37d4054..582a4d7 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
@@ -1003,7 +1003,6 @@ static struct drm_framebuffer *vmw_kms_fb_create(struct drm_device *dev,
 	struct vmw_surface *surface = NULL;
 	struct vmw_dma_buffer *bo = NULL;
 	struct ttm_base_object *user_obj;
-	u64 required_size;
 	int ret;
 
 	/**
@@ -1012,8 +1011,9 @@ static struct drm_framebuffer *vmw_kms_fb_create(struct drm_device *dev,
 	 * requested framebuffer.
 	 */
 
-	required_size = mode_cmd->pitch * mode_cmd->height;
-	if (unlikely(required_size > (u64) dev_priv->vram_size)) {
+	if (!vmw_kms_validate_mode_vram(dev_priv,
+					mode_cmd->pitch,
+					mode_cmd->height)) {
 		DRM_ERROR("VRAM size is too small for requested mode.\n");
 		return ERR_PTR(-ENOMEM);
 	}
-- 
1.7.5.4