Mesa (main): frontends/va: Return error in vaRenderPicture if decoder is NULL

[GitLab Mirror]

Module: Mesa
Branch: main
Commit: 65afcddbf1892a74397cb8ba0d29618544fb507a
URL:    http://cgit.freedesktop.org/mesa/mesa/commit/?id=65afcddbf1892a74397cb8ba0d29618544fb507a

Author: Lorenz Brun <lorenz at brun.one>
Date:   Wed Jun  9 17:41:12 2021 +0200

frontends/va: Return error in vaRenderPicture if decoder is NULL

This fixes a crash if a data slice is submitted before the decoder
is initialized. A well-behaved application shouldn't encounter this
but returning an error is still better than crashing the entire
process and the rest of the code is similarly defensive.

Signed-off-by: Lorenz Brun <lorenz at brun.one>
Reviewed-by: Leo Liu <leo.liu at amd.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/11792>

---

 src/gallium/frontends/va/picture.c | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/gallium/frontends/va/picture.c b/src/gallium/frontends/va/picture.c
index 2843983a753..9e630eefc97 100644
--- a/src/gallium/frontends/va/picture.c
+++ b/src/gallium/frontends/va/picture.c
@@ -278,7 +278,7 @@ handleVAProtectedSliceDataBufferType(vlVaContext *context, vlVaBuffer *buf)
 	context->desc.base.protected_playback = true;
 }
 
-static void
+static VAStatus
 handleVASliceDataBufferType(vlVaContext *context, vlVaBuffer *buf)
 {
    enum pipe_video_format format = u_reduce_video_profile(context->templat.profile);
@@ -290,6 +290,9 @@ handleVASliceDataBufferType(vlVaContext *context, vlVaBuffer *buf)
    static const uint8_t start_code_vc1[] = { 0x00, 0x00, 0x01, 0x0d };
    static const uint8_t eoi_jpeg[] = { 0xff, 0xd9 };
 
+   if (!context->decoder)
+      return VA_STATUS_ERROR_INVALID_CONTEXT;
+
    format = u_reduce_video_profile(context->templat.profile);
    if (!context->desc.base.protected_playback) {
       switch (format) {
@@ -358,6 +361,7 @@ handleVASliceDataBufferType(vlVaContext *context, vlVaBuffer *buf)
    }
    context->decoder->decode_bitstream(context->decoder, context->target, &context->desc.base,
       num_buffers, (const void * const*)buffers, sizes);
+   return VA_STATUS_SUCCESS;
 }
 
 static VAStatus
@@ -607,7 +611,7 @@ vlVaRenderPicture(VADriverContextP ctx, VAContextID context_id, VABufferID *buff
          break;
 
       case VASliceDataBufferType:
-         handleVASliceDataBufferType(context, buf);
+         vaStatus = handleVASliceDataBufferType(context, buf);
          break;
 
       case VAProcPipelineParameterBufferType: