[Libva] [PATCH v2] vaDeriveImage: properly set VAImage format

[Xiang, Haihao]

On Sun, 2015-10-11 at 16:23 +0100, Julien Isorce wrote:
> This is required to allow gstvaapipostproc
> outputing RGBx va surfaces.
> 
> https://bugs.freedesktop.org/show_bug.cgi?id=92088
> 
> Signed-off-by: Julien Isorce <j.isorce at samsung.com>
> ---
>  src/i965_drv_video.c | 40 +++++++++++++++++++++++++++++++++++++++-
>  1 file changed, 39 insertions(+), 1 deletion(-)
> 
> diff --git a/src/i965_drv_video.c b/src/i965_drv_video.c
> index cb950e1..8df0c61 100644
> --- a/src/i965_drv_video.c
> +++ b/src/i965_drv_video.c
> @@ -4060,10 +4060,10 @@ VAStatus i965_DeriveImage(VADriverContextP ctx,
>  
>      image->format.fourcc = obj_surface->fourcc;
>      image->format.byte_order = VA_LSB_FIRST;
> -    image->format.bits_per_pixel = 12;
>  
>      switch (image->format.fourcc) {
>      case VA_FOURCC_YV12:
> +        image->format.bits_per_pixel = 12;
>          image->num_planes = 3;
>          image->pitches[0] = w_pitch; /* Y */
>          image->offsets[0] = 0;
> @@ -4074,6 +4074,7 @@ VAStatus i965_DeriveImage(VADriverContextP ctx,
>          break;
>  
>      case VA_FOURCC_YV16:
> +        image->format.bits_per_pixel = 16;
>          image->num_planes = 3;
>          image->pitches[0] = w_pitch; /* Y */
>          image->offsets[0] = 0;
> @@ -4084,6 +4085,7 @@ VAStatus i965_DeriveImage(VADriverContextP ctx,
>          break;
>  
>      case VA_FOURCC_NV12:
> +        image->format.bits_per_pixel = 12;
>          image->num_planes = 2;
>          image->pitches[0] = w_pitch; /* Y */
>          image->offsets[0] = 0;
> @@ -4097,6 +4099,7 @@ VAStatus i965_DeriveImage(VADriverContextP ctx,
>      case VA_FOURCC_444P:
>      case VA_FOURCC_422V:
>      case VA_FOURCC_411P:
> +        image->format.bits_per_pixel = 12;

It is 24 for 444P, 16 for 422V.

Actually you could get bits_per_pixel via i965_fourcc_infos[].

>          image->num_planes = 3;
>          image->pitches[0] = w_pitch; /* Y */
>          image->offsets[0] = 0;
> @@ -4109,6 +4112,7 @@ VAStatus i965_DeriveImage(VADriverContextP ctx,
>      case VA_FOURCC_YUY2:
>      case VA_FOURCC_UYVY:
>      case VA_FOURCC_Y800:
> +        image->format.bits_per_pixel = 16;

It is 8 for Y800.


>          image->num_planes = 1;
>          image->pitches[0] = obj_surface->width; /* Y, width is aligned already */
>          image->offsets[0] = 0;
> @@ -4117,8 +4121,42 @@ VAStatus i965_DeriveImage(VADriverContextP ctx,
>      case VA_FOURCC_RGBX:
>      case VA_FOURCC_BGRA:
>      case VA_FOURCC_BGRX:
> +        image->format.bits_per_pixel = 32;
>          image->num_planes = 1;
>          image->pitches[0] = obj_surface->width;
> +
> +        switch (image->format.fourcc) {
> +        case VA_FOURCC_RGBA:
> +        case VA_FOURCC_RGBX:
> +            image->format.red_mask = 0x000000ff;
> +            image->format.green_mask = 0x0000ff00;
> +            image->format.blue_mask = 0x00ff0000;
> +            image->format.alpha_mask = 0x00000000;
> +            break;
> +        case VA_FOURCC_BGRA:
> +        case VA_FOURCC_BGRX:
> +            image->format.red_mask = 0x00ff0000;
> +            image->format.green_mask = 0x0000ff00;
> +            image->format.blue_mask = 0x000000ff;
> +            image->format.alpha_mask = 0x00000000;
> +            break;
> +        default:
> +            goto error;
> +        }
> +
> +        switch (image->format.fourcc) {
> +        case VA_FOURCC_RGBA:
> +        case VA_FOURCC_BGRA:
> +            image->format.depth = 32;
> +            break;
> +        case VA_FOURCC_RGBX:
> +        case VA_FOURCC_BGRX:
> +            image->format.depth = 24;
> +            break;
> +        default:
> +            goto error;
> +        }
> +
>          break;
>      default:
>          goto error;