[Libva] [PATCH intel-driver 2/4] decoder: h264: allocate reference frames earlier, if needed.

[Gwenole Beauchesne]

Sometimes, a dummy frame comes from the codec layer and it is used
as a reference, per the comment in the existing code. Even though
this looks suspicious, keep this criterion but make sure to try
allocating the VA surface, if needed, earlier in the function that
sanity checks the parameters for decoding the current frame.

This makes it possible to fail at a much earlier time, and actually
make it possible to return a sensible error code to the upper layer.

Signed-off-by: Gwenole Beauchesne <gwenole.beauchesne at intel.com>
---
 src/i965_decoder_utils.c | 44 +++++++++++++++++++++-----------------------
 1 file changed, 21 insertions(+), 23 deletions(-)

diff --git a/src/i965_decoder_utils.c b/src/i965_decoder_utils.c
index 9d537ff..780d239 100644
--- a/src/i965_decoder_utils.c
+++ b/src/i965_decoder_utils.c
@@ -410,12 +410,6 @@ intel_update_avc_frame_store_index(VADriverContextP ctx,
             int slot_found;
             struct object_surface *obj_surface = decode_state->reference_objects[i];
 
-            /* 
-             * Sometimes a dummy frame comes from the upper layer library, call i965_check_alloc_surface_bo()
-             * to ake sure the store buffer is allocated for this reference frame
-             */
-            i965_check_alloc_surface_bo(ctx, obj_surface, 1, VA_FOURCC_NV12, SUBSAMPLE_YUV420);
-
             slot_found = 0;
             frame_idx = -1;
             /* Find a free frame store index */
@@ -539,6 +533,7 @@ intel_decoder_check_avc_parameter(VADriverContextP ctx,
 {
     struct i965_driver_data *i965 = i965_driver_data(ctx);
     VAPictureParameterBufferH264 *pic_param = (VAPictureParameterBufferH264 *)decode_state->pic_param->buffer;
+    VAStatus va_status;
     struct object_surface *obj_surface;	
     int i;
 
@@ -562,28 +557,31 @@ intel_decoder_check_avc_parameter(VADriverContextP ctx,
        }
     }
 
-    for (i = 0; i < 16; i++) {
-        if (pic_param->ReferenceFrames[i].flags & VA_PICTURE_H264_INVALID ||
-            pic_param->ReferenceFrames[i].picture_id == VA_INVALID_SURFACE)
-            break;
-        else {
-            obj_surface = SURFACE(pic_param->ReferenceFrames[i].picture_id);
-            assert(obj_surface);
+    /* Fill in the reference objects array with the actual VA surface objects
+       with 1:1 correspondance with any entry in ReferenceFrames[] */
+    for (i = 0; i < ARRAY_ELEMS(pic_param->ReferenceFrames); i++) {
+        const VAPictureH264 * const va_pic = &pic_param->ReferenceFrames[i];
 
+        obj_surface = NULL;
+        if (!(va_pic->flags & VA_PICTURE_H264_INVALID) &&
+            va_pic->picture_id != VA_INVALID_ID) {
+            obj_surface = SURFACE(pic_param->ReferenceFrames[i].picture_id);
             if (!obj_surface)
-                goto error;
-
-            if (!obj_surface->bo) { /* a reference frame  without store buffer */
-                WARN_ONCE("Invalid reference frame!!!\n");
-            }
+                return VA_STATUS_ERROR_INVALID_SURFACE;
 
-            decode_state->reference_objects[i] = obj_surface;
+            /* 
+             * Sometimes a dummy frame comes from the upper layer
+             * library, call i965_check_alloc_surface_bo() to make
+             * sure the store buffer is allocated for this reference
+             * frame
+             */
+            va_status = i965_check_alloc_surface_bo(ctx, obj_surface, 1,
+                VA_FOURCC_NV12, SUBSAMPLE_YUV420);
+            if (va_status != VA_STATUS_SUCCESS)
+                return va_status;
         }
+        decode_state->reference_objects[i] = obj_surface;
     }
-
-    for ( ; i < 16; i++)
-        decode_state->reference_objects[i] = NULL;
-
     return VA_STATUS_SUCCESS;
 
 error:
-- 
1.9.1