[PATCH weston v2 20/20] compositor-drm: use weston_view_to_output_matrix() to test plane viability

[Derek Foreman]

Instead of comparing buffer transforms to output transforms we now
use weston_view_to_output_matrix() and weston_matrix_to_transform() to
test if we can use a drm plane.

We no longer test scaling, since the drm plane api supports scaling.
Unfortunately the drmSetPlane() call is far from the viability test and
has no reasonable fallback, so scaling will need to be revisited in the
future when atomic mode setting is viable and sprites_are_broken stops
being universally true...
---
 src/compositor-drm.c | 41 ++++++++++++++++++-----------------------
 1 file changed, 18 insertions(+), 23 deletions(-)

diff --git a/src/compositor-drm.c b/src/compositor-drm.c
index c0f451e..7d81983 100644
--- a/src/compositor-drm.c
+++ b/src/compositor-drm.c
@@ -824,37 +824,26 @@ drm_output_check_sprite_format(struct drm_sprite *s,
 	return 0;
 }
 
-static int
-drm_view_transform_supported(struct weston_view *ev)
-{
-	return !ev->transform.enabled ||
-		(ev->transform.matrix.type < WESTON_MATRIX_TRANSFORM_ROTATE);
-}
-
 static struct weston_plane *
 drm_output_prepare_overlay_view(struct weston_output *output_base,
 				struct weston_view *ev)
 {
 	struct weston_compositor *ec = output_base->compositor;
 	struct drm_compositor *c =(struct drm_compositor *) ec;
-	struct weston_buffer_viewport *viewport = &ev->surface->buffer_viewport;
+	struct weston_matrix matrix;
 	struct drm_sprite *s;
 	int found = 0;
 	struct gbm_bo *bo;
 	pixman_region32_t dest_rect, src_rect;
 	pixman_box32_t *box, tbox;
+	enum wl_output_transform transform;
 	uint32_t format;
+	float scalex, scaley, transx, transy;
 	int32_t sx1, sy1, sx2, sy2;
 
 	if (c->gbm == NULL)
 		return NULL;
 
-	if (viewport->buffer.transform != output_base->transform)
-		return NULL;
-
-	if (viewport->buffer.scale != output_base->current_scale)
-		return NULL;
-
 	if (c->sprites_are_broken)
 		return NULL;
 
@@ -870,7 +859,14 @@ drm_output_prepare_overlay_view(struct weston_output *output_base,
 	if (wl_shm_buffer_get(ev->surface->buffer_ref.buffer->resource))
 		return NULL;
 
-	if (!drm_view_transform_supported(ev))
+	weston_view_to_output_matrix(ev, output_base, false, &matrix);
+
+	if (!weston_matrix_to_transform(&matrix, &transform,
+					&scalex, &scaley,
+					&transx, &transy))
+		return NULL;
+
+	if (transform != WL_OUTPUT_TRANSFORM_NORMAL)
 		return NULL;
 
 	wl_list_for_each(s, &c->sprite_list, link) {
@@ -936,14 +932,13 @@ drm_output_prepare_overlay_view(struct weston_output *output_base,
 	weston_view_from_global(ev, box->x1, box->y1, &sx1, &sy1);
 	weston_view_from_global(ev, box->x2, box->y2, &sx2, &sy2);
 
-	if (sx1 < 0)
-		sx1 = 0;
-	if (sy1 < 0)
-		sy1 = 0;
-	if (sx2 > ev->surface->width)
-		sx2 = ev->surface->width;
-	if (sy2 > ev->surface->height)
-		sy2 = ev->surface->height;
+
+	/* Previously we clamped to the surface edge here, but that will
+	 * result in incorrect scaling, so we just bail.
+	 */
+	if (sx1 < 0 || sy1 < 0 ||
+	    sx2 > ev->surface->width || sy2 > ev->surface->height)
+		return NULL;
 
 	tbox.x1 = sx1;
 	tbox.y1 = sy1;
-- 
2.1.1