[Libva] [PATCH] h264: fix 4K decoding (IVB).

[Xiang, Haihao]

On Wed, 2012-06-27 at 15:34 +0200, Gwenole Beauchesne wrote: 
> Hi,
> 
> I pushed the following as obvious. Tested with GStreamer only.
> MPlayer has internal limitations to 2K pixels in either dimension.

I can play a 3840x2160 video by MPlayer-vaapi on my machine.

...
VO: [vaapi] 3840x2160 => 3840x2160 H.264 VA-API Acceleration 
[VD_FFMPEG] XVMC-accelerated MPEG-2.
V:  15.6   0/  0  4%  2%  0.0% 0 0

> 
> Fix decoding of 4K videos on Ivy Bridge. 
> Note that either dimension
> shall not exceed 4096 pixels.

> Signed-off-by: Gwenole Beauchesne <gwenole.beauchesne at intel.com>
> ---
>  src/gen7_mfd.c |   17 +++++++++++------
>  1 file changed, 11 insertions(+), 6 deletions(-)
> 
> diff --git a/src/gen7_mfd.c b/src/gen7_mfd.c
> index 9c72a08..e01c343 100644
> --- a/src/gen7_mfd.c
> +++ b/src/gen7_mfd.c
> @@ -194,8 +194,8 @@ gen7_mfd_init_avc_surface(VADriverContextP ctx,
>      int width_in_mbs, height_in_mbs;
>  
>      obj_surface->free_private_data = gen7_mfd_free_avc_surface;
> -    width_in_mbs = ((pic_param->picture_width_in_mbs_minus1 + 1) & 0xff);
> -    height_in_mbs = ((pic_param->picture_height_in_mbs_minus1 + 1) & 0xff); /* frame height */
> +    width_in_mbs = pic_param->picture_width_in_mbs_minus1 + 1;
> +    height_in_mbs = pic_param->picture_height_in_mbs_minus1 + 1; /* frame height */
>  
>      if (!gen7_avc_surface) {
>          gen7_avc_surface = calloc(sizeof(struct gen7_avc_surface), 1);
> @@ -211,6 +211,7 @@ gen7_mfd_init_avc_surface(VADriverContextP ctx,
>                                                   "direct mv w/r buffer",
>                                                   width_in_mbs * height_in_mbs * 64,
>                                                   0x1000);
> +        assert(gen7_avc_surface->dmv_top);
>      }
>  
>      if (gen7_avc_surface->dmv_bottom_flag &&
> @@ -219,6 +220,7 @@ gen7_mfd_init_avc_surface(VADriverContextP ctx,
>                                                      "direct mv w/r buffer",
>                                                      width_in_mbs * height_in_mbs * 64,                                                    
>                                                      0x1000);
> +        assert(gen7_avc_surface->dmv_bottom);
>      }
>  }
>  
> @@ -496,8 +498,8 @@ gen7_mfd_avc_img_state(VADriverContextP ctx,
>      mbaff_frame_flag = (pic_param->seq_fields.bits.mb_adaptive_frame_field_flag &&
>                          !pic_param->pic_fields.bits.field_pic_flag);
>  
> -    width_in_mbs = ((pic_param->picture_width_in_mbs_minus1 + 1) & 0xff);
> -    height_in_mbs = ((pic_param->picture_height_in_mbs_minus1 + 1) & 0xff); /* frame height */
> +    width_in_mbs = pic_param->picture_width_in_mbs_minus1 + 1;
> +    height_in_mbs = pic_param->picture_height_in_mbs_minus1 + 1; /* frame height */
>  
>      /* MFX unit doesn't support 4:2:2 and 4:4:4 picture */
>      assert(pic_param->seq_fields.bits.chroma_format_idc == 0 || /* monochrome picture */
> @@ -879,7 +881,7 @@ gen7_mfd_avc_decode_init(VADriverContextP ctx,
>      struct object_surface *obj_surface;
>      dri_bo *bo;
>      int i, j, enable_avc_ildb = 0;
> -    int width_in_mbs;
> +    unsigned int width_in_mbs, height_in_mbs;
>  
>      for (j = 0; j < decode_state->num_slice_params && enable_avc_ildb == 0; j++) {
>          assert(decode_state->slice_params && decode_state->slice_params[j]->buffer);
> @@ -905,7 +907,10 @@ gen7_mfd_avc_decode_init(VADriverContextP ctx,
>      assert(decode_state->pic_param && decode_state->pic_param->buffer);
>      pic_param = (VAPictureParameterBufferH264 *)decode_state->pic_param->buffer;
>      gen7_mfd_avc_frame_store_index(ctx, pic_param, gen7_mfd_context);
> -    width_in_mbs = ((pic_param->picture_width_in_mbs_minus1 + 1) & 0xff);
> +    width_in_mbs = pic_param->picture_width_in_mbs_minus1 + 1;
> +    height_in_mbs = pic_param->picture_height_in_mbs_minus1 + 1;
> +    assert(width_in_mbs > 0 && width_in_mbs <= 256); /* 4K */
> +    assert(height_in_mbs > 0 && height_in_mbs <= 256);

vaCreateContext() will fail if either dimension exceeds 4096 pixels, so
assert() won't be triggered.

> 
>      /* Current decoded picture */
>      va_pic = &pic_param->CurrPic;