[PATCH v16 18/23] compositor-drm: Ignore occluded views

[Daniel Stone]

When trying to assign planes, keep track of the areas which are
already occluded, and ignore views which are completely occluded. This
allows us to build a state using planes only, when there are occluded
views which cannot go into a plane behind views which can.

Signed-off-by: Daniel Stone <daniels at collabora.com>
Tested-by: Emre Ucan <eucan at de.adit-jv.com>
---
 libweston/compositor-drm.c | 37 ++++++++++++++++++++++++++++++++-----
 1 file changed, 32 insertions(+), 5 deletions(-)

diff --git a/libweston/compositor-drm.c b/libweston/compositor-drm.c
index 0a49bac5a..c546aec83 100644
--- a/libweston/compositor-drm.c
+++ b/libweston/compositor-drm.c
@@ -3166,7 +3166,7 @@ drm_output_propose_state(struct weston_output *output_base,
 	struct drm_output *output = to_drm_output(output_base);
 	struct drm_output_state *state;
 	struct weston_view *ev;
-	pixman_region32_t surface_overlap, renderer_region;
+	pixman_region32_t surface_overlap, renderer_region, occluded_region;
 	struct weston_plane *primary = &output_base->compositor->primary_plane;
 
 	assert(!output->state_last);
@@ -3188,9 +3188,12 @@ drm_output_propose_state(struct weston_output *output_base,
 	 * as we do for flipping full screen surfaces.
 	 */
 	pixman_region32_init(&renderer_region);
+	pixman_region32_init(&occluded_region);
 
 	wl_list_for_each(ev, &output_base->compositor->view_list, link) {
 		struct weston_plane *next_plane = NULL;
+		pixman_region32_t clipped_view;
+		bool occluded = false;
 
 		/* If this view doesn't touch our output at all, there's no
 		 * reason to do anything with it. */
@@ -3202,13 +3205,27 @@ drm_output_propose_state(struct weston_output *output_base,
 		if (ev->output_mask != (1u << output->base.id))
 			next_plane = primary;
 
+		/* Ignore views we know to be totally occluded. */
+		pixman_region32_init(&clipped_view);
+		pixman_region32_intersect(&clipped_view,
+					  &ev->transform.boundingbox,
+					  &output->base.region);
+
+		pixman_region32_init(&surface_overlap);
+		pixman_region32_subtract(&surface_overlap, &clipped_view,
+					 &occluded_region);
+		occluded = !pixman_region32_not_empty(&surface_overlap);
+		if (occluded) {
+			pixman_region32_fini(&surface_overlap);
+			pixman_region32_fini(&clipped_view);
+			continue;
+		}
+
 		/* Since we process views from top to bottom, we know that if
 		 * the view intersects the calculated renderer region, it must
 		 * be part of, or occluded by, it, and cannot go on a plane. */
-		pixman_region32_init(&surface_overlap);
 		pixman_region32_intersect(&surface_overlap, &renderer_region,
-					  &ev->transform.boundingbox);
-
+					  &clipped_view);
 		if (pixman_region32_not_empty(&surface_overlap))
 			next_plane = primary;
 		pixman_region32_fini(&surface_overlap);
@@ -3216,6 +3233,9 @@ drm_output_propose_state(struct weston_output *output_base,
 		if (next_plane == NULL)
 			next_plane = drm_output_prepare_cursor_view(state, ev);
 
+		if (next_plane == NULL && !drm_view_is_opaque(ev))
+			next_plane = primary;
+
 		if (next_plane == NULL)
 			next_plane = drm_output_prepare_scanout_view(state, ev);
 
@@ -3228,9 +3248,16 @@ drm_output_propose_state(struct weston_output *output_base,
 		if (next_plane == primary)
 			pixman_region32_union(&renderer_region,
 					      &renderer_region,
-					      &ev->transform.boundingbox);
+					      &clipped_view);
+		else if (output->cursor_plane &&
+			 next_plane != &output->cursor_plane->base)
+			pixman_region32_union(&occluded_region,
+					      &occluded_region,
+					      &clipped_view);
+		pixman_region32_fini(&clipped_view);
 	}
 	pixman_region32_fini(&renderer_region);
+	pixman_region32_fini(&occluded_region);
 
 	return state;
 }
-- 
2.17.1