[Libva] [PATCH v2] vaDeriveImage: properly set VAImage format

[Julien Isorce]

This is required to allow gstvaapipostproc
outputing RGBx va surfaces.

https://bugs.freedesktop.org/show_bug.cgi?id=92088

Signed-off-by: Julien Isorce <j.isorce at samsung.com>
---
 src/i965_drv_video.c | 40 +++++++++++++++++++++++++++++++++++++++-
 1 file changed, 39 insertions(+), 1 deletion(-)

diff --git a/src/i965_drv_video.c b/src/i965_drv_video.c
index cb950e1..8df0c61 100644
--- a/src/i965_drv_video.c
+++ b/src/i965_drv_video.c
@@ -4060,10 +4060,10 @@ VAStatus i965_DeriveImage(VADriverContextP ctx,
 
     image->format.fourcc = obj_surface->fourcc;
     image->format.byte_order = VA_LSB_FIRST;
-    image->format.bits_per_pixel = 12;
 
     switch (image->format.fourcc) {
     case VA_FOURCC_YV12:
+        image->format.bits_per_pixel = 12;
         image->num_planes = 3;
         image->pitches[0] = w_pitch; /* Y */
         image->offsets[0] = 0;
@@ -4074,6 +4074,7 @@ VAStatus i965_DeriveImage(VADriverContextP ctx,
         break;
 
     case VA_FOURCC_YV16:
+        image->format.bits_per_pixel = 16;
         image->num_planes = 3;
         image->pitches[0] = w_pitch; /* Y */
         image->offsets[0] = 0;
@@ -4084,6 +4085,7 @@ VAStatus i965_DeriveImage(VADriverContextP ctx,
         break;
 
     case VA_FOURCC_NV12:
+        image->format.bits_per_pixel = 12;
         image->num_planes = 2;
         image->pitches[0] = w_pitch; /* Y */
         image->offsets[0] = 0;
@@ -4097,6 +4099,7 @@ VAStatus i965_DeriveImage(VADriverContextP ctx,
     case VA_FOURCC_444P:
     case VA_FOURCC_422V:
     case VA_FOURCC_411P:
+        image->format.bits_per_pixel = 12;
         image->num_planes = 3;
         image->pitches[0] = w_pitch; /* Y */
         image->offsets[0] = 0;
@@ -4109,6 +4112,7 @@ VAStatus i965_DeriveImage(VADriverContextP ctx,
     case VA_FOURCC_YUY2:
     case VA_FOURCC_UYVY:
     case VA_FOURCC_Y800:
+        image->format.bits_per_pixel = 16;
         image->num_planes = 1;
         image->pitches[0] = obj_surface->width; /* Y, width is aligned already */
         image->offsets[0] = 0;
@@ -4117,8 +4121,42 @@ VAStatus i965_DeriveImage(VADriverContextP ctx,
     case VA_FOURCC_RGBX:
     case VA_FOURCC_BGRA:
     case VA_FOURCC_BGRX:
+        image->format.bits_per_pixel = 32;
         image->num_planes = 1;
         image->pitches[0] = obj_surface->width;
+
+        switch (image->format.fourcc) {
+        case VA_FOURCC_RGBA:
+        case VA_FOURCC_RGBX:
+            image->format.red_mask = 0x000000ff;
+            image->format.green_mask = 0x0000ff00;
+            image->format.blue_mask = 0x00ff0000;
+            image->format.alpha_mask = 0x00000000;
+            break;
+        case VA_FOURCC_BGRA:
+        case VA_FOURCC_BGRX:
+            image->format.red_mask = 0x00ff0000;
+            image->format.green_mask = 0x0000ff00;
+            image->format.blue_mask = 0x000000ff;
+            image->format.alpha_mask = 0x00000000;
+            break;
+        default:
+            goto error;
+        }
+
+        switch (image->format.fourcc) {
+        case VA_FOURCC_RGBA:
+        case VA_FOURCC_BGRA:
+            image->format.depth = 32;
+            break;
+        case VA_FOURCC_RGBX:
+        case VA_FOURCC_BGRX:
+            image->format.depth = 24;
+            break;
+        default:
+            goto error;
+        }
+
         break;
     default:
         goto error;
-- 
1.9.1