[Libva] support raw yuv stream in vaapisink (especially for wayland platform)
Zhao, Halley
halley.zhao at intel.com
Fri Sep 7 02:25:29 PDT 2012
Hi Gwenole:
In order to support raw video stream (decoded by sw decoder) on wayland platform, I heard that you have
ideas to add upload functionality in vaapisink, right?
Here is a patch to make that work, could you give some comments.
Do you prefer to separate the functionality to a new GObject? If yes, I could give more try.
>From d599f18a654025328c34d49a09c60edfb3de147f Mon Sep 17 00:00:00 2001
From: Zhao Halley <halley.zhao at intel.com>
Date: Fri, 7 Sep 2012 13:14:04 +0800
Subject: [PATCH] add yuv upload functionality in vaapisink to support raw
video stream
---
gst/vaapi/gstvaapisink.c | 330 ++++++++++++++++++++++++++++++++++++++++++++--
gst/vaapi/gstvaapisink.h | 9 ++
2 files changed, 328 insertions(+), 11 deletions(-)
mode change 100644 => 100755 gst/vaapi/gstvaapisink.c
mode change 100644 => 100755 gst/vaapi/gstvaapisink.h
diff --git a/gst/vaapi/gstvaapisink.c b/gst/vaapi/gstvaapisink.c
old mode 100644
new mode 100755
index 4611974..9aa453e
--- a/gst/vaapi/gstvaapisink.c
+++ b/gst/vaapi/gstvaapisink.c
@@ -35,6 +35,9 @@
#include <gst/video/videocontext.h>
#include <gst/vaapi/gstvaapivideobuffer.h>
#include <gst/vaapi/gstvaapivideosink.h>
+#include <gst/vaapi/gstvaapiimagepool.h>
+#include <gst/vaapi/gstvaapisurfacepool.h>
+#include <gst/vaapi/gstvaapipluginbuffer.h>
#if USE_DRM
# include <gst/vaapi/gstvaapidisplay_drm.h>
#endif
@@ -72,12 +75,25 @@ static const GstElementDetails gst_vaapisink_details =
"Gwenole Beauchesne <gwenole.beauchesne at intel.com>");
/* Default template */
+static const char gst_vaapisink_raw_caps_str[] =
+ "video/x-raw-yuv, "
+ "width = (int) [ 1, MAX ], "
+ "height = (int) [ 1, MAX ]; ";
+
+static const char gst_vaapisink_sink_caps_str[] =
+ "video/x-raw-yuv, "
+ "width = (int) [ 1, MAX ], "
+ "height = (int) [ 1, MAX ]; "
+ GST_VAAPI_SURFACE_CAPS;
+
static GstStaticPadTemplate gst_vaapisink_sink_factory =
GST_STATIC_PAD_TEMPLATE(
"sink",
GST_PAD_SINK,
GST_PAD_ALWAYS,
- GST_STATIC_CAPS(GST_VAAPI_SURFACE_CAPS));
+ GST_STATIC_CAPS(gst_vaapisink_sink_caps_str));
+
+static GstStaticCaps in_raw_caps = GST_STATIC_CAPS (gst_vaapisink_raw_caps_str);
static void
gst_vaapisink_implements_iface_init(GstImplementsInterfaceClass *iface);
@@ -223,6 +239,8 @@ gst_vaapisink_xoverlay_iface_init(GstXOverlayClass *iface)
static void
gst_vaapisink_destroy(GstVaapiSink *sink)
{
+ g_clear_object(&sink->images);
+ g_clear_object(&sink->surfaces);
g_clear_object(&sink->texture);
g_clear_object(&sink->display);
@@ -475,13 +493,145 @@ gst_vaapisink_stop(GstBaseSink *base_sink)
}
static gboolean
+gst_vaapisink_ensure_image_pool(GstVaapiSink *sink, GstCaps *caps)
+{
+ GstStructure * const structure = gst_caps_get_structure(caps, 0);
+ gint width, height;
+
+ gst_structure_get_int(structure, "width", &width);
+ gst_structure_get_int(structure, "height", &height);
+
+ if (width != sink->video_width || height != sink->video_height) {
+ sink->video_width = width;
+ sink->video_height = height;
+ g_clear_object(&sink->images);
+ sink->images = gst_vaapi_image_pool_new(sink->display, caps);
+ if (!sink->images)
+ return FALSE;
+ sink->images_reset = TRUE;
+ }
+ return TRUE;
+}
+
+static gboolean
+gst_vaapisink_ensure_surface_pool(GstVaapiSink *sink, GstCaps *caps)
+{
+ GstStructure * const structure = gst_caps_get_structure(caps, 0);
+ gint width, height;
+
+ gst_structure_get_int(structure, "width", &width);
+ gst_structure_get_int(structure, "height", &height);
+
+ if (width != sink->surface_width || height != sink->surface_height) {
+ sink->surface_width = width;
+ sink->surface_height = height;
+ g_clear_object(&sink->surfaces);
+ sink->surfaces = gst_vaapi_surface_pool_new(sink->display, caps);
+ if (!sink->surfaces)
+ return FALSE;
+ sink->surfaces_reset = TRUE;
+ }
+ return TRUE;
+}
+
+static void
+gst_vaapisink_ensure_direct_rendering_caps(
+ GstVaapiSink *sink,
+ GstCaps *caps
+)
+{
+ GstVaapiSurface *surface;
+ GstVaapiImage *image;
+ GstVaapiImageFormat vaformat;
+ GstVideoFormat vformat;
+ GstStructure *structure;
+ gint width, height;
+
+ if (!sink->images_reset && !sink->surfaces_reset)
+ return;
+
+ sink->images_reset = FALSE;
+ sink->surfaces_reset = FALSE;
+ sink->direct_rendering_caps = 0;
+
+ structure = gst_caps_get_structure(caps, 0);
+ if (!structure)
+ return;
+ gst_structure_get_int(structure, "width", &width);
+ gst_structure_get_int(structure, "height", &height);
+
+ /* Translate from Gst video format to VA image format */
+ if (!gst_video_format_parse_caps(caps, &vformat, NULL, NULL))
+ return;
+ if (!gst_video_format_is_yuv(vformat))
+ return;
+ vaformat = gst_vaapi_image_format_from_video(vformat);
+ if (!vaformat)
+ return;
+
+ /* Check if we can alias sink & output buffers (same data_size) */
+ image = gst_vaapi_video_pool_get_object(sink->images);
+ if (image) {
+ if (sink->direct_rendering_caps == 0 &&
+ (gst_vaapi_image_get_format(image) == vaformat &&
+ gst_vaapi_image_is_linear(image) &&
+ (gst_vaapi_image_get_data_size(image) ==
+ gst_video_format_get_size(vformat, width, height))))
+ sink->direct_rendering_caps = 1;
+ gst_vaapi_video_pool_put_object(sink->images, image);
+ }
+
+ /* Check if we can access to the surface pixels directly */
+ surface = gst_vaapi_video_pool_get_object(sink->surfaces);
+ if (surface) {
+ image = gst_vaapi_surface_derive_image(surface);
+ if (image) {
+ if (gst_vaapi_image_map(image)) {
+ if (sink->direct_rendering_caps == 1 &&
+ (gst_vaapi_image_get_format(image) == vaformat &&
+ gst_vaapi_image_is_linear(image) &&
+ (gst_vaapi_image_get_data_size(image) ==
+ gst_video_format_get_size(vformat, width, height))))
+ sink->direct_rendering_caps = 2;
+ gst_vaapi_image_unmap(image);
+ }
+ g_object_unref(image);
+ }
+ gst_vaapi_video_pool_put_object(sink->surfaces, surface);
+ }
+}
+
+static gboolean
+gst_vaapisink_negotiate_buffers(
+ GstVaapiSink *sink,
+ GstCaps *caps
+)
+{
+ guint dr;
+
+ if (!gst_vaapisink_ensure_image_pool(sink, caps))
+ return FALSE;
+
+ if (!gst_vaapisink_ensure_surface_pool(sink, caps))
+ return FALSE;
+
+ gst_vaapisink_ensure_direct_rendering_caps(sink, caps);
+ dr = MIN(sink->direct_rendering, sink->direct_rendering_caps);
+ if (sink->direct_rendering != dr) {
+ sink->direct_rendering = dr;
+ GST_DEBUG("direct-rendering level: %d", dr);
+ }
+ return TRUE;
+}
+
+static gboolean
gst_vaapisink_set_caps(GstBaseSink *base_sink, GstCaps *caps)
{
GstVaapiSink * const sink = GST_VAAPISINK(base_sink);
GstStructure * const structure = gst_caps_get_structure(caps, 0);
guint win_width, win_height, display_width, display_height;
gint video_width, video_height, video_par_n = 1, video_par_d = 1;
-
+
#if USE_DRM
if (sink->display_type == GST_VAAPI_DISPLAY_TYPE_DRM)
return TRUE;
@@ -489,6 +639,11 @@ gst_vaapisink_set_caps(GstBaseSink *base_sink, GstCaps *caps)
if (!structure)
return FALSE;
+ if (gst_structure_has_name(structure, "video/x-raw-yuv")) {
+ sink->raw_yuv_stream = TRUE;
+ if (!gst_vaapisink_negotiate_buffers(sink, caps))
+ return FALSE;
+ }
if (!gst_structure_get_int(structure, "width", &video_width))
return FALSE;
if (!gst_structure_get_int(structure, "height", &video_height))
@@ -714,34 +869,94 @@ gst_vaapisink_put_surface(
return TRUE;
}
+
+static GstVaapiSurface *
+gst_vaapisink_upload_raw_yuv(
+ GstVaapiSink *sink,
+ GstBuffer *inbuf)
+{
+ GstVaapiVideoBuffer *vbuffer = NULL;
+ GstVaapiImage *image = NULL;
+ GstVaapiSurface *surface = NULL;
+ gboolean success = FALSE;
+
+ if (sink->direct_rendering) {
+ vbuffer = GST_VAAPI_VIDEO_BUFFER(inbuf);
+ }
+
+ if (sink->direct_rendering == 2) {
+ surface = gst_vaapi_video_buffer_get_surface(vbuffer);
+ g_assert(surface);
+ return surface;
+ }
+
+ surface = gst_vaapi_video_pool_get_object(sink->surfaces);
+ g_assert(surface);
+ if (!surface)
+ return NULL;
+
+ if (sink->direct_rendering == 1) {
+ image = gst_vaapi_video_buffer_get_image(vbuffer);
+ g_assert(image);
+ }
+ else if (sink->direct_rendering == 0) {
+ image = gst_vaapi_video_pool_get_object(sink->images);
+ gst_vaapi_image_update_from_buffer(image, inbuf, NULL);
+ }
+
+ success = gst_vaapi_surface_put_image(surface, image);
+
+ if (sink->direct_rendering = 0) {
+ gst_vaapi_video_pool_put_object(sink->images, image);
+ }
+
+ if (!success)
+ goto error_put_image;
+ return surface;
+
+error_put_image:
+ {
+ GST_WARNING("failed to upload %" GST_FOURCC_FORMAT " image "
+ "to surface 0x%08x",
+ GST_FOURCC_ARGS(gst_vaapi_image_get_format(image)),
+ gst_vaapi_surface_get_id(surface));
+ return GST_FLOW_OK;
+ }
+}
+
static GstFlowReturn
gst_vaapisink_show_frame(GstBaseSink *base_sink, GstBuffer *buffer)
{
GstVaapiSink * const sink = GST_VAAPISINK(base_sink);
- GstVaapiVideoBuffer * const vbuffer = GST_VAAPI_VIDEO_BUFFER(buffer);
+ GstVaapiVideoBuffer * vbuffer = NULL;
GstVaapiSurface *surface;
guint flags;
gboolean success;
GstVideoOverlayComposition * const composition =
gst_video_buffer_get_overlay_composition(buffer);
- if (sink->display != gst_vaapi_video_buffer_get_display (vbuffer)) {
- g_clear_object(&sink->display);
- sink->display = g_object_ref (gst_vaapi_video_buffer_get_display (vbuffer));
- }
-
if (!sink->window)
return GST_FLOW_UNEXPECTED;
- surface = gst_vaapi_video_buffer_get_surface(vbuffer);
+ if (sink->raw_yuv_stream) {
+ surface = gst_vaapisink_upload_raw_yuv(sink, buffer);
+ flags = 0; // todo
+ }
+ else {
+ vbuffer = GST_VAAPI_VIDEO_BUFFER(buffer);
+ surface = gst_vaapi_video_buffer_get_surface(vbuffer);
+ flags = gst_vaapi_video_buffer_get_render_flags(vbuffer);
+ if (sink->display != gst_vaapi_video_buffer_get_display (vbuffer)) {
+ g_clear_object(&sink->display);
+ sink->display = g_object_ref (gst_vaapi_video_buffer_get_display (vbuffer));
+ }
+ }
if (!surface)
return GST_FLOW_UNEXPECTED;
GST_DEBUG("render surface %" GST_VAAPI_ID_FORMAT,
GST_VAAPI_ID_ARGS(gst_vaapi_surface_get_id(surface)));
- flags = gst_vaapi_video_buffer_get_render_flags(vbuffer);
-
if (!gst_vaapi_surface_set_subpictures_from_composition(surface,
composition, TRUE))
GST_WARNING("could not update subtitles");
@@ -772,9 +987,92 @@ gst_vaapisink_show_frame(GstBaseSink *base_sink, GstBuffer *buffer)
success = FALSE;
break;
}
+
+ if (sink->direct_rendering <2) {
+ gst_vaapi_video_pool_put_object(sink->surfaces, surface);
+ }
return success ? GST_FLOW_OK : GST_FLOW_UNEXPECTED;
}
+static GstFlowReturn
+gst_vaapisink_buffer_alloc(
+ GstBaseSink *base_sink,
+ guint64 offset,
+ guint size,
+ GstCaps *caps,
+ GstBuffer **pbuf
+)
+{
+ GstVaapiSink *sink = GST_VAAPISINK(base_sink);
+ GstBuffer *buffer = NULL;
+ GstVaapiImage *image = NULL;
+ GstVaapiSurface *surface = NULL;
+ GstVaapiVideoBuffer *vbuffer;
+ GstStructure *structure = NULL;
+
+ *pbuf = NULL;
+
+ structure = gst_caps_get_structure(caps, 0);
+ if (!gst_structure_has_name(structure, "video/x-raw-yuv")) {
+ return GST_FLOW_OK;
+ }
+
+ /* Check if we can use direct-rendering */
+ if (!gst_vaapisink_negotiate_buffers(sink, caps))
+ goto error;
+ if (!sink->direct_rendering)
+ return GST_FLOW_OK;
+
+ switch (sink->direct_rendering) {
+ case 2:
+ buffer = gst_vaapi_video_buffer_new_from_pool(sink->surfaces);
+ if (!buffer)
+ goto error;
+ vbuffer = GST_VAAPI_VIDEO_BUFFER(buffer);
+
+ surface = gst_vaapi_video_buffer_get_surface(vbuffer);
+ image = gst_vaapi_surface_derive_image(surface);
+ if (image && gst_vaapi_image_get_data_size(image) == size) {
+ gst_vaapi_video_buffer_set_image(vbuffer, image);
+ g_object_unref(image); /* video buffer owns an extra reference */
+ break;
+ }
+
+ /* We can't use the derive-image optimization. Disable it. */
+ sink->direct_rendering = 1;
+ gst_buffer_unref(buffer);
+ buffer = NULL;
+
+ case 1:
+ buffer = gst_vaapi_video_buffer_new_from_pool(sink->images);
+ if (!buffer)
+ goto error;
+ vbuffer = GST_VAAPI_VIDEO_BUFFER(buffer);
+
+ image = gst_vaapi_video_buffer_get_image(vbuffer);
+ break;
+ }
+ g_assert(image);
+
+ if (!gst_vaapi_image_map(image))
+ goto error;
+
+ GST_BUFFER_DATA(buffer) = gst_vaapi_image_get_plane(image, 0);
+ GST_BUFFER_SIZE(buffer) = gst_vaapi_image_get_data_size(image);
+
+ gst_buffer_set_caps(buffer, caps);
+ *pbuf = buffer;
+ return GST_FLOW_OK;
+
+error:
+ /* We can't use the inout-buffers optimization. Disable it. */
+ GST_DEBUG("disable in/out buffer optimization");
+ if (buffer)
+ gst_buffer_unref(buffer);
+ sink->direct_rendering = 0;
+ return GST_FLOW_OK;
+}
+
static gboolean
gst_vaapisink_query(GstBaseSink *base_sink, GstQuery *query)
{
@@ -870,6 +1168,7 @@ gst_vaapisink_class_init(GstVaapiSinkClass *klass)
basesink_class->preroll = gst_vaapisink_show_frame;
basesink_class->render = gst_vaapisink_show_frame;
basesink_class->query = gst_vaapisink_query;
+ basesink_class->buffer_alloc = gst_vaapisink_buffer_alloc;
gst_element_class_set_details_simple(
element_class,
@@ -942,9 +1241,18 @@ gst_vaapisink_init(GstVaapiSink *sink)
sink->video_height = 0;
sink->video_par_n = 1;
sink->video_par_d = 1;
+ sink->surface_width = 0;
+ sink->surface_height = 0;
sink->foreign_window = FALSE;
sink->fullscreen = FALSE;
sink->synchronous = FALSE;
sink->display_type = DEFAULT_DISPLAY_TYPE;
sink->use_reflection = FALSE;
+ sink->images = NULL;
+ sink->surfaces = NULL;
+ sink->raw_yuv_stream = FALSE;
+ sink->images_reset = FALSE;
+ sink->surfaces_reset = FALSE;
+ sink->direct_rendering = G_MAXUINT32;
+ sink->direct_rendering_caps = 0;
}
diff --git a/gst/vaapi/gstvaapisink.h b/gst/vaapi/gstvaapisink.h
old mode 100644
new mode 100755
index c5883b3..9445cd4
--- a/gst/vaapi/gstvaapisink.h
+++ b/gst/vaapi/gstvaapisink.h
@@ -76,11 +76,20 @@ struct _GstVaapiSink {
guint video_height;
gint video_par_n;
gint video_par_d;
+ gint surface_width;
+ gint surface_height; // for yuv GstBuffer's surface pool
+ GstVaapiVideoPool *images;
+ GstVaapiVideoPool *surfaces;
GstVaapiRectangle display_rect;
+ guint direct_rendering_caps;
+ guint direct_rendering;
guint foreign_window : 1;
guint fullscreen : 1;
guint synchronous : 1;
guint use_reflection : 1;
+ guint raw_yuv_stream : 1;
+ guint images_reset : 1;
+ guint surfaces_reset : 1;
};
struct _GstVaapiSinkClass {
--
1.7.9.5
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.freedesktop.org/archives/libva/attachments/20120907/21eb74a6/attachment-0001.html>
More information about the Libva
mailing list