[Bug 732266] vaapidecode: h264: add support for decoding SVC base layers only

[GStreamer (GNOME Bugzilla)]

https://bugzilla.gnome.org/show_bug.cgi?id=732266

--- Comment #67 from Orestis Floros <orestisf1993 at gmail.com> ---
By the way, isn't the code in gstvaapidecoder_h264.c:4589

> if (priv->is_avcC)
>   result = gst_h264_parser_identify_nalu_avc (priv->parser,
>       buf, 0, buf_size, priv->nal_length_size, &pi->nalu);
> else
>   result = gst_h264_parser_identify_nalu_unchecked (priv->parser,
>       buf, 0, buf_size, &pi->nalu);
> status = get_status (result);

dangerous in the same way? If h264parser fails to parse them and they have
garbage data (eg as with the PPS case you mention) it can lead to a segfault in
_finalize() which is called after goto exit IIUC.

This is a bit of uncharted territory for me. A workaround for this without
parsing the ssps could be to mark the nalu as not valid and check in
_finalize():

> diff --git a/gst-libs/gst/vaapi/gstvaapidecoder_h264.c b/gst-libs/gst/vaapi/gstvaapidecoder_h264.c
> index 8b8a08ee..f779ac71 100644
> --- a/gst-libs/gst/vaapi/gstvaapidecoder_h264.c
> +++ b/gst-libs/gst/vaapi/gstvaapidecoder_h264.c
> @@ -100,6 +100,9 @@ struct _GstVaapiParserInfoH264
>  static void
>  gst_vaapi_parser_info_h264_finalize (GstVaapiParserInfoH264 * pi)
>  {
> +  if (!pi->nalu.valid)
> +    return;
> +
>    switch (pi->nalu.type) {
>      case GST_H264_NAL_SPS:
>      case GST_H264_NAL_SUBSET_SPS:
> @@ -1811,8 +1814,10 @@ parse_pps (GstVaapiDecoderH264 * decoder, GstVaapiDecoderUnit * unit)
>    result = gst_h264_parser_parse_pps (priv->parser, &pi->nalu, pps);
>
>    /* PPS's sps id might be an ignored subset sps in SVC streams */
> -  if (priv->base_only && result == GST_H264_PARSER_BROKEN_LINK)
> +  if (priv->base_only && result == GST_H264_PARSER_BROKEN_LINK){
> +    pi->nalu.valid = FALSE;
>      return GST_VAAPI_DECODER_STATUS_SUCCESS;
> +  }
>
>    priv->parser_state &= GST_H264_VIDEO_STATE_GOT_SPS;
>
> @@ -4593,6 +4598,7 @@ gst_vaapi_decoder_h264_parse (GstVaapiDecoder * base_decoder,
>            || pi->nalu.type == GST_H264_NAL_SUBSET_SPS
>            || pi->nalu.type == GST_H264_NAL_SLICE_EXT)) {
>      GST_VAAPI_DECODER_UNIT_FLAG_SET (unit, GST_VAAPI_DECODER_UNIT_FLAG_SKIP);
> +    pi->nalu.valid = FALSE;
>      return GST_VAAPI_DECODER_STATUS_SUCCESS;
>    }
>    switch (pi->nalu.type) {

-- 
You are receiving this mail because:
You are the QA Contact for the bug.
You are the assignee for the bug.