[Mesa-dev] [PATCH v3] st/vdpau: only send buffers with B8G8R8A8 format to X

[Christian König]

Am 19.01.2017 um 10:07 schrieb Nayan Deshmukh:
> PresentPixmap only works if the pixmap depth matches with the
> window depth, otherwise it returns a BadMatch protocol error.
> Even if the depths match, the result won't look correctly
> if the VDPAU RGB component order doesn't match the X11 one so
> we only allow the X11 format.
> For other buffers we copy them to a buffer which is send to X.
>
> v2: only send buffers with format VDP_RGBA_FORMAT_B8G8R8A8
> v3: reword commit message
>
> Signed-off-by: Nayan Deshmukh <nayan26deshmukh at gmail.com>

I was a bit to late with my response to the last version.

> ---
>   src/gallium/state_trackers/vdpau/output.c        | 2 ++
>   src/gallium/state_trackers/vdpau/presentation.c  | 6 +++---
>   src/gallium/state_trackers/vdpau/vdpau_private.h | 1 +
>   3 files changed, 6 insertions(+), 3 deletions(-)
>
> diff --git a/src/gallium/state_trackers/vdpau/output.c b/src/gallium/state_trackers/vdpau/output.c
> index 98a8011..e800d2c 100644
> --- a/src/gallium/state_trackers/vdpau/output.c
> +++ b/src/gallium/state_trackers/vdpau/output.c
> @@ -75,6 +75,8 @@ vlVdpOutputSurfaceCreate(VdpDevice device,
>   
>      memset(&res_tmpl, 0, sizeof(res_tmpl));
>   
> +   vlsurface->send_to_X = rgba_format == VDP_RGBA_FORMAT_B8G8R8A8;
> +

I would add the commit message here as code comment as well.

With that fixed the patch is Reviewed-by: Christian König 
<christian.koenig at amd.com>.

If nobody else responses I'm going to make this small change myself and 
commit the result.

Regards,
Christian.

>      res_tmpl.target = PIPE_TEXTURE_2D;
>      res_tmpl.format = VdpFormatRGBAToPipe(rgba_format);
>      res_tmpl.width0 = width;
> diff --git a/src/gallium/state_trackers/vdpau/presentation.c b/src/gallium/state_trackers/vdpau/presentation.c
> index d479369..78cafc8 100644
> --- a/src/gallium/state_trackers/vdpau/presentation.c
> +++ b/src/gallium/state_trackers/vdpau/presentation.c
> @@ -231,7 +231,7 @@ vlVdpPresentationQueueDisplay(VdpPresentationQueue presentation_queue,
>      vscreen = pq->device->vscreen;
>   
>      pipe_mutex_lock(pq->device->mutex);
> -   if (vscreen->set_back_texture_from_output)
> +   if (vscreen->set_back_texture_from_output && surf->send_to_X)
>         vscreen->set_back_texture_from_output(vscreen, surf->surface->texture, clip_width, clip_height);
>      tex = vscreen->texture_from_drawable(vscreen, (void *)pq->drawable);
>      if (!tex) {
> @@ -239,7 +239,7 @@ vlVdpPresentationQueueDisplay(VdpPresentationQueue presentation_queue,
>         return VDP_STATUS_INVALID_HANDLE;
>      }
>   
> -   if (!vscreen->set_back_texture_from_output) {
> +   if (!vscreen->set_back_texture_from_output || !surf->send_to_X) {
>         dirty_area = vscreen->get_dirty_area(vscreen);
>   
>         memset(&surf_templ, 0, sizeof(surf_templ));
> @@ -289,7 +289,7 @@ vlVdpPresentationQueueDisplay(VdpPresentationQueue presentation_queue,
>         framenum++;
>      }
>   
> -   if (!vscreen->set_back_texture_from_output) {
> +   if (!vscreen->set_back_texture_from_output || !surf->send_to_X) {
>         pipe_resource_reference(&tex, NULL);
>         pipe_surface_reference(&surf_draw, NULL);
>      }
> diff --git a/src/gallium/state_trackers/vdpau/vdpau_private.h b/src/gallium/state_trackers/vdpau/vdpau_private.h
> index 490a0bd..8356608 100644
> --- a/src/gallium/state_trackers/vdpau/vdpau_private.h
> +++ b/src/gallium/state_trackers/vdpau/vdpau_private.h
> @@ -415,6 +415,7 @@ typedef struct
>      struct pipe_fence_handle *fence;
>      struct vl_compositor_state cstate;
>      struct u_rect dirty_area;
> +   bool send_to_X;
>   } vlVdpOutputSurface;
>   
>   typedef struct