[RFC][PATCH 4/7] drm: Add drm_framebuffer_check() utility function

[ville.syrjala at linux.intel.com]

From: Ville Syrjälä <ville.syrjala at linux.intel.com>

This function performs a battery of sanity checks on the requested
framebuffer layout. Drivers can call it from their fb_create hook.

Signed-off-by: Ville Syrjälä <ville.syrjala at linux.intel.com>
---
 drivers/gpu/drm/drm_crtc_helper.c |   70 +++++++++++++++++++++++++++++++++++++
 include/drm/drm_crtc_helper.h     |    2 +
 2 files changed, 72 insertions(+), 0 deletions(-)

diff --git a/drivers/gpu/drm/drm_crtc_helper.c b/drivers/gpu/drm/drm_crtc_helper.c
index 1528068..af5a8f0 100644
--- a/drivers/gpu/drm/drm_crtc_helper.c
+++ b/drivers/gpu/drm/drm_crtc_helper.c
@@ -1154,3 +1154,73 @@ int drm_format_vert_chroma_subsampling(uint32_t format)
 	}
 }
 EXPORT_SYMBOL(drm_format_vert_chroma_subsampling);
+
+/**
+ * drm_framebuffer_check - check the framebuffer layout
+ * @r: cmd from ioctl
+ * @size: BO size
+ *
+ * RETURNS:
+ * Zero on success, error code on failure.
+ */
+int drm_framebuffer_check(const struct drm_mode_fb_cmd2 *r, unsigned int size)
+{
+	struct {
+		unsigned int start, end;
+	} ranges[4];
+	unsigned int required_size = 0;
+	int hsub = drm_format_horz_chroma_subsampling(r->pixel_format);
+	int vsub = drm_format_vert_chroma_subsampling(r->pixel_format);
+	int num_planes = drm_format_num_planes(r->pixel_format);
+	int i, j;
+
+	if (r->width == 0 || r->height == 0)
+		return -EINVAL;
+
+	/* Keep things safe for s15.16 fixed point math. */
+	if (r->width > 0x7fff || r->height > 0x7fff)
+		return -EINVAL;
+
+	if (r->width % hsub || r->height % vsub)
+		return -EINVAL;
+
+	for (i = 0; i < num_planes; i++) {
+		unsigned int height = r->height / (i != 0 ? vsub : 1);
+		unsigned int width = r->width / (i != 0 ? hsub : 1);
+		unsigned int size = r->pitches[i] * height;
+		unsigned int cpp = drm_format_plane_cpp(r->pixel_format, i);
+		unsigned int min_pitch = cpp * width;
+
+		if (size < r->pitches[i] || size < height)
+			return -EINVAL;
+
+		if (min_pitch < width || min_pitch < cpp)
+			return -EINVAL;
+
+		if (r->pitches[i] < min_pitch)
+			return -EINVAL;
+
+		ranges[i].start = r->offsets[i];
+		ranges[i].end = ranges[i].start + size;
+
+		if (ranges[i].end < ranges[i].start)
+			return -EINVAL;
+
+		required_size = max(required_size, ranges[i].end);
+	}
+
+	if (size < required_size)
+		return -EINVAL;
+
+	/* Check for overlapping ranges */
+	for (i = 0; i < num_planes; i++) {
+		for (j = i + 1; j < num_planes; j++) {
+			if (ranges[i].start < ranges[j].end &&
+			    ranges[j].start < ranges[i].end)
+				return -EINVAL;
+		}
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL(drm_framebuffer_check);
diff --git a/include/drm/drm_crtc_helper.h b/include/drm/drm_crtc_helper.h
index 25d46b8..4d2a53a 100644
--- a/include/drm/drm_crtc_helper.h
+++ b/include/drm/drm_crtc_helper.h
@@ -151,5 +151,7 @@ extern int drm_format_num_planes(uint32_t format);
 extern int drm_format_plane_cpp(uint32_t format, int plane);
 extern int drm_format_horz_chroma_subsampling(uint32_t format);
 extern int drm_format_vert_chroma_subsampling(uint32_t format);
+extern int drm_framebuffer_check(const struct drm_mode_fb_cmd2 *r,
+				 unsigned int size);
 
 #endif
-- 
1.7.3.4