[PATCH 2/2] drm/vkms: Properly order plane for blending

[Louis Chauvet]

The current blending algorithm requires that vkms_state->active_planes be
ordered by z-order. This has not been an issue so far because the planes
are registered in the correct order in DRM (primary-overlay-cursor).
This new algorithm now uses the configured z-order to populate
active_planes.

Signed-off-by: Louis Chauvet <louis.chauvet at bootlin.com>
---
 drivers/gpu/drm/vkms/vkms_crtc.c | 26 ++++++++++++++++++++------
 drivers/gpu/drm/vkms/vkms_drv.c  |  1 +
 2 files changed, 21 insertions(+), 6 deletions(-)

diff --git a/drivers/gpu/drm/vkms/vkms_crtc.c b/drivers/gpu/drm/vkms/vkms_crtc.c
index e60573e0f3e9510252e1f198b00e28bcc7987620..f3f3ad8ede29987b385d53834970bcd0c6adefa7 100644
--- a/drivers/gpu/drm/vkms/vkms_crtc.c
+++ b/drivers/gpu/drm/vkms/vkms_crtc.c
@@ -200,14 +200,28 @@ static int vkms_crtc_atomic_check(struct drm_crtc *crtc,
 	vkms_state->num_active_planes = i;
 
 	i = 0;
-	drm_for_each_plane_mask(plane, crtc->dev, crtc_state->plane_mask) {
-		plane_state = drm_atomic_get_existing_plane_state(crtc_state->state, plane);
 
-		if (!plane_state->visible)
-			continue;
+	unsigned int current_zpos = 0;
+
+	while (i < vkms_state->num_active_planes) {
+		unsigned int next_zpos = UINT_MAX;
+
+		drm_for_each_plane_mask(plane, crtc->dev, crtc_state->plane_mask) {
+			plane_state = drm_atomic_get_existing_plane_state(crtc_state->state, plane);
+
+			if (!plane_state->visible)
+				continue;
+
+			if (plane_state->normalized_zpos == current_zpos)
+				vkms_state->active_planes[i++] = to_vkms_plane_state(plane_state);
+
+			if (plane_state->normalized_zpos > current_zpos &&
+			    plane_state->normalized_zpos < next_zpos)
+				next_zpos = plane_state->normalized_zpos;
+		}
+
+		current_zpos = next_zpos;
 
-		vkms_state->active_planes[i++] =
-			to_vkms_plane_state(plane_state);
 	}
 
 	return 0;
diff --git a/drivers/gpu/drm/vkms/vkms_drv.c b/drivers/gpu/drm/vkms/vkms_drv.c
index e8472d9b6e3b2b5d6d497763288bf3dc6fde5987..e492791a7a970b768184292bf82318e3fca6b36a 100644
--- a/drivers/gpu/drm/vkms/vkms_drv.c
+++ b/drivers/gpu/drm/vkms/vkms_drv.c
@@ -135,6 +135,7 @@ static int vkms_modeset_init(struct vkms_device *vkmsdev)
 	dev->mode_config.max_height = YRES_MAX;
 	dev->mode_config.cursor_width = 512;
 	dev->mode_config.cursor_height = 512;
+	dev->mode_config.normalize_zpos = true;
 	/*
 	 * FIXME: There's a confusion between bpp and depth between this and
 	 * fbdev helpers. We have to go with 0, meaning "pick the default",

-- 
2.50.1