[Mesa-dev] [PATCH] st/vdpau: avoid sending buffers with 10bit channels to X

[Nayan Deshmukh]

PresentPixmap only works if the pixmap depth matches the window
depth, otherwise it returns a BadMatch protocol error.

Signed-off-by: Nayan Deshmukh <nayan26deshmukh at gmail.com>
---
 src/gallium/state_trackers/vdpau/output.c        | 6 ++++++
 src/gallium/state_trackers/vdpau/presentation.c  | 6 +++---
 src/gallium/state_trackers/vdpau/vdpau_private.h | 1 +
 3 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/src/gallium/state_trackers/vdpau/output.c b/src/gallium/state_trackers/vdpau/output.c
index 98a8011..4466483 100644
--- a/src/gallium/state_trackers/vdpau/output.c
+++ b/src/gallium/state_trackers/vdpau/output.c
@@ -75,6 +75,12 @@ vlVdpOutputSurfaceCreate(VdpDevice device,
 
    memset(&res_tmpl, 0, sizeof(res_tmpl));
 
+   if (rgba_format == VDP_RGBA_FORMAT_B10G10R10A2 ||
+       rgba_format ==  VDP_RGBA_FORMAT_R10G10B10A2)
+      vlsurface->send_to_X = false;
+   else
+      vlsurface->send_to_X = true;
+
    res_tmpl.target = PIPE_TEXTURE_2D;
    res_tmpl.format = VdpFormatRGBAToPipe(rgba_format);
    res_tmpl.width0 = width;
diff --git a/src/gallium/state_trackers/vdpau/presentation.c b/src/gallium/state_trackers/vdpau/presentation.c
index d479369..78cafc8 100644
--- a/src/gallium/state_trackers/vdpau/presentation.c
+++ b/src/gallium/state_trackers/vdpau/presentation.c
@@ -231,7 +231,7 @@ vlVdpPresentationQueueDisplay(VdpPresentationQueue presentation_queue,
    vscreen = pq->device->vscreen;
 
    pipe_mutex_lock(pq->device->mutex);
-   if (vscreen->set_back_texture_from_output)
+   if (vscreen->set_back_texture_from_output && surf->send_to_X)
       vscreen->set_back_texture_from_output(vscreen, surf->surface->texture, clip_width, clip_height);
    tex = vscreen->texture_from_drawable(vscreen, (void *)pq->drawable);
    if (!tex) {
@@ -239,7 +239,7 @@ vlVdpPresentationQueueDisplay(VdpPresentationQueue presentation_queue,
       return VDP_STATUS_INVALID_HANDLE;
    }
 
-   if (!vscreen->set_back_texture_from_output) {
+   if (!vscreen->set_back_texture_from_output || !surf->send_to_X) {
       dirty_area = vscreen->get_dirty_area(vscreen);
 
       memset(&surf_templ, 0, sizeof(surf_templ));
@@ -289,7 +289,7 @@ vlVdpPresentationQueueDisplay(VdpPresentationQueue presentation_queue,
       framenum++;
    }
 
-   if (!vscreen->set_back_texture_from_output) {
+   if (!vscreen->set_back_texture_from_output || !surf->send_to_X) {
       pipe_resource_reference(&tex, NULL);
       pipe_surface_reference(&surf_draw, NULL);
    }
diff --git a/src/gallium/state_trackers/vdpau/vdpau_private.h b/src/gallium/state_trackers/vdpau/vdpau_private.h
index 490a0bd..8356608 100644
--- a/src/gallium/state_trackers/vdpau/vdpau_private.h
+++ b/src/gallium/state_trackers/vdpau/vdpau_private.h
@@ -415,6 +415,7 @@ typedef struct
    struct pipe_fence_handle *fence;
    struct vl_compositor_state cstate;
    struct u_rect dirty_area;
+   bool send_to_X;
 } vlVdpOutputSurface;
 
 typedef struct
-- 
2.9.3