appsrc + x11 + GLMemory
Nikolay Frey
nikolay.frey at gmail.com
Wed Jun 3 10:59:33 UTC 2020
Re: appsrc + x11 + GLMemory
I did not see my attachments in the mailing list.
source code:
"
#define GL_GLEXT_PROTOTYPES
#include <gst/gst.h>
#include <gst/gl/gl.h>
#include <gst/app/gstappsink.h>
#include <gst/gl/x11/gstgldisplay_x11.h>
#include <gst/gl/gstglmemory.h>
#include <math.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <sys/time.h>
#include <unistd.h>
#include <X11/Xlib.h>
#include <X11/keysym.h>
#include <GL/gl.h>
#include <GL/glut.h>
#include <GL/glx.h>
#include <GL/glxext.h>
*typedef* *struct* app_s
{
Display *display;
Window win;
GLXContext ctx;
GstGLDisplay *gl_display;
GstPipeline *pipeline;
GMainLoop *loop;
GstElement *source;
*unsigned* *int* framebuffer;
*unsigned* *int* texture;
GstContext *x11context;
GstContext *ctxcontext;
GstGLContext *gl_context;
GstGLVideoAllocationParams *allocation_params;
*int* width;
*int* height;
} app_t;
app_t g_app = {0};
*static* *void* _gl_draw(*void*)
{
//opengl draw objects
}
*static* *void* _gl_draw_frame(app_t *app)
{
*static* *int* frames = 0;
GstBuffer *buffer;
gpointer wrapped[1];
GstVideoFrame out_frame;
gboolean ret;
GstGLMemoryAllocator *allocator =
gst_gl_memory_allocator_get_default(app->gl_context);
GstVideoMeta *vmeta;
wrapped[0] = (gpointer) app->texture;
buffer = gst_buffer_new();
*if* (!buffer)
{
printf("no buffer\n");
exit(1);
}
printf("add meta\n");
vmeta = gst_buffer_add_video_meta(buffer, GST_VIDEO_FRAME_FLAG_NONE,
GST_VIDEO_FORMAT_RGBA, 512, 512);
*if* (!vmeta)
{
printf("add meta failed\n");
exit(1);
}
printf("setup buffer\n");
ret = gst_gl_memory_setup_buffer(allocator, buffer, app->allocation_params,
*NULL*, wrapped, 1);
printf("setup buffer ret %d\n", ret);
*if* (!gst_video_frame_map(&out_frame, app->allocation_params->v_info,
buffer, (GstMapFlags) (GST_MAP_WRITE | GST_MAP_GL)))
{
printf("failed on gst_video_frame_map\n");
exit(1);
}
guint next_tex = *(guint *) out_frame.data[0];
printf("next_tex %d\n", next_tex);
_gl_draw();
gst_video_frame_unmap(&out_frame);
g_signal_emit_by_name(app->source, "push-buffer", buffer, &ret);
gst_buffer_unref(buffer);
printf("push ret %d\n", ret);
glXSwapBuffers(app->display, app->win);
frames++;
}
*static* *void* _gl_reshape(app_t *app, *int* width, *int* height)
{
glGenFramebuffers(1, &app->framebuffer);
glBindFramebuffer(GL_FRAMEBUFFER, app->framebuffer);
printf("FramebufferName %d\n", app->framebuffer);
glGenTextures(1, &app->texture);
glBindTexture(GL_TEXTURE_2D, app->texture);
printf("texture %d\n", app->texture);
glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, width, height, 0, GL_RGB,
GL_UNSIGNED_BYTE, *NULL*);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
glBindTexture(GL_TEXTURE_2D, 0);
glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D,
app->texture, 0);
*unsigned* *int* rbo;
glGenRenderbuffers(1, &rbo);
glBindRenderbuffer(GL_RENDERBUFFER, rbo);
glRenderbufferStorage(GL_RENDERBUFFER, GL_DEPTH24_STENCIL8, width, height);
glBindRenderbuffer(GL_RENDERBUFFER, 0);
glFramebufferRenderbuffer(GL_FRAMEBUFFER, GL_DEPTH_STENCIL_ATTACHMENT,
GL_RENDERBUFFER, rbo);
*if*(glCheckFramebufferStatus(GL_FRAMEBUFFER) != GL_FRAMEBUFFER_COMPLETE)
printf("failed on glCheckFramebufferStatus\n");
glBindFramebuffer(GL_FRAMEBUFFER, app->framebuffer);
glViewport(0, 0, (GLint) width, (GLint) height);
glMatrixMode(GL_PROJECTION);
glLoadIdentity();
*float* nearp = 1, farp = 500.0f, hht, hwd;
hht = nearp * tan(45.0 / 2.0 / 180.0 * M_PI);
hwd = hht * width / height;
glFrustum(-hwd, hwd, -hht, hht, nearp, farp);
glMatrixMode(GL_MODELVIEW);
glLoadIdentity();
glTranslatef(0.0, 0.0, -40.0);
}
*static* *void* _gl_init(*void*)
{
glEnable(GL_DEPTH_TEST);
*static* GLfloat pos[4] = { 5.0, 5.0, 10.0, 0.0 };
glShadeModel(GL_FLAT);
glLightfv(GL_LIGHT0, GL_POSITION, pos);
glEnable(GL_CULL_FACE);
glEnable(GL_LIGHTING);
glEnable(GL_LIGHT0);
glEnable(GL_DEPTH_TEST);
glEnable(GL_NORMALIZE);
}
*static* *void* _gst_gl_init(app_t *app, *int* width, *int* height)
{
GstVideoInfo *vinfo = gst_video_info_new();
printf("new video info %p\n", vinfo);
gboolean ret = gst_video_info_set_format(vinfo, GST_VIDEO_FORMAT_RGBA,
width, height);
printf("set video info ret %d\n", ret);
printf("allocation params\n");
app->allocation_params =
gst_gl_video_allocation_params_new_wrapped_texture(app->gl_context,
*NULL*, vinfo, 0, *NULL*, GST_GL_TEXTURE_TARGET_2D, GST_GL_RGB,
app->texture, *NULL*, 0);
printf("allocation params %p gl_context %p\n", app->allocation_params,
app->gl_context);
}
*static* gboolean _gst_bus_call(GstBus *bus, GstMessage *msg, gpointer data)
{
GMainLoop *loop = (GMainLoop*) data;
*switch* (GST_MESSAGE_TYPE (msg))
{
*case* GST_MESSAGE_EOS:
printf("eos\n");
g_main_loop_quit (loop);
*break*;
*case* GST_MESSAGE_ERROR:
{
gchar *debug = *NULL*;
GError *err = *NULL*;
gst_message_parse_error(msg, &err, &debug);
printf("error '%s'\n", err->message);
g_error_free(err);
*if* (debug)
{
printf("deails '%s'\n", debug);
g_free(debug);
}
g_main_loop_quit (loop);
*break*;
}
*case* GST_MESSAGE_NEED_CONTEXT:
{
*const* gchar *context_type;
gst_message_parse_context_type(msg, &context_type);
*if* (g_strcmp0(context_type, "gst.gl.app_context") == 0)
{
printf("context type '%s'\n", context_type);
gst_element_set_context(GST_ELEMENT (msg->src), g_app.ctxcontext);
}
*else* *if* (g_strcmp0(context_type, GST_GL_DISPLAY_CONTEXT_TYPE) == 0)
{
printf("context type '%s'\n", context_type);
gst_element_set_context(GST_ELEMENT (msg->src), g_app.x11context);
}
*break*;
}
*case* GST_MESSAGE_HAVE_CONTEXT:
printf("GST_MESSAGE_HAVE_CONTEXT\n");
*break*;
*default*:
*break*;
}
*return* *TRUE*;
}
*static* *void* _x11_create_window(*const* *char* *name,*int* x, *int* y,
*int* width, *int* height, app_t *app, VisualID *visRet)
{
*int* attribs[64];
*int* i = 0;
*int* scrnum;
XSetWindowAttributes attr;
*unsigned* *long* mask;
Window root;
XSizeHints sizehints;
XVisualInfo *visinfo;
attribs[i++] = GLX_RGBA;
attribs[i++] = GLX_DOUBLEBUFFER;
attribs[i++] = GLX_RED_SIZE;
attribs[i++] = 1;
attribs[i++] = GLX_GREEN_SIZE;
attribs[i++] = 1;
attribs[i++] = GLX_BLUE_SIZE;
attribs[i++] = 1;
attribs[i++] = GLX_DEPTH_SIZE;
attribs[i++] = 1;
attribs[i++] = None;
scrnum = DefaultScreen(app->display);
root = RootWindow(app->display, scrnum);
visinfo = glXChooseVisual(app->display, scrnum, attribs);
*if* (!visinfo)
{
printf("Error: couldn't get an RGB, Double-buffered");
exit(1);
}
attr.background_pixel = 0;
attr.border_pixel = 0;
attr.colormap = XCreateColormap(app->display, root, visinfo->visual,
AllocNone);
attr.event_mask = StructureNotifyMask | ExposureMask | KeyPressMask;
mask = CWBackPixel | CWBorderPixel | CWColormap | CWEventMask;
app->win = XCreateWindow(app->display, root, x, y, width, height, 0,
visinfo->depth, InputOutput, visinfo->visual, mask, &attr );
sizehints.x = x;
sizehints.y = y;
sizehints.width = width;
sizehints.height = height;
sizehints.flags = USSize | USPosition;
XSetNormalHints(app->display, app->win, &sizehints);
XSetStandardProperties(app->display, app->win, name, name, None, (*char*
**) *NULL*, 0, &sizehints);
app->ctx = glXCreateContext(app->display, visinfo, *NULL*, True);
*if* (!app->ctx)
{
printf("Error: glXCreateContext failed\n");
exit(1);
}
*visRet = visinfo->visualid;
XFree(visinfo);
}
*static* *int* _gl_is_glx_extension_supported(Display *display, *const*
*char* *query)
{
*const* *int* scrnum = DefaultScreen(display);
*const* *char* *glx_extensions = *NULL*;
*const* size_t len = strlen(query);
*const* *char* *ptr;
*if* (glx_extensions == *NULL*)
glx_extensions = glXQueryExtensionsString(display, scrnum);
ptr = strstr(glx_extensions, query);
*return* ((ptr != *NULL*) && ((ptr[len] == ' ') || (ptr[len] == '\0')));
}
*static* *void* _gl_query_vsync(Display *display, GLXDrawable drawable)
{
*int* interval = 0;
*if* (_gl_is_glx_extension_supported(display, "GLX_MESA_swap_control"))
{
PFNGLXGETSWAPINTERVALMESAPROC pglXGetSwapIntervalMESA =
(PFNGLXGETSWAPINTERVALMESAPROC)
glXGetProcAddressARB((*const* GLubyte *) "glXGetSwapIntervalMESA");
interval = (*pglXGetSwapIntervalMESA)();
}
*else* *if* (_gl_is_glx_extension_supported(display, "GLX_SGI_swap_control"
))
interval = 1;
*if* (interval > 0)
{
printf("Running synchronized to the vertical refresh. The framerate should
be\n");
*if* (interval == 1)
printf("approximately the same as the monitor refresh rate.\n");
*else* *if* (interval > 1)
printf("approximately 1/%d the monitor refresh rate.\n", interval);
}
}
*static* *void* _gst_start_pipeline(app_t *app)
{
gst_element_set_context(GST_ELEMENT(app->pipeline), app->ctxcontext);
gst_element_set_context(GST_ELEMENT(app->pipeline), app->x11context);
gst_element_set_state(GST_ELEMENT(app->pipeline), GST_STATE_PLAYING);
printf("Pipeline play\n");
}
*static* gboolean _gst_idle_loop(gpointer data)
{
*static* *int* inited = 0;
*if* (!inited)
{
printf("idle pid %d\n", getpid());
inited = 1;
}
_gl_draw_frame((app_t *) data);
*return* *TRUE*;
}
*static* *void* _gst_load_pipeline(app_t *app, *const* *char* *text)
{
app->pipeline = GST_PIPELINE(gst_parse_launch(text, *NULL*));
GstBus *bus = gst_pipeline_get_bus(GST_PIPELINE(app->pipeline));
gst_bus_add_watch(bus, _gst_bus_call, app->loop);
gst_object_unref(bus);
app->source = gst_bin_get_by_name(GST_BIN(app->pipeline), "source");
}
*int* main(*int* argc, *char* **argv)
{
*int* x = 0, y = 0;
*char* pl[1024];
g_app.width = 512;
g_app.height = 512;
*char* *displayName = *NULL*;
VisualID visId;
g_app.display = XOpenDisplay(displayName);
*if* (!g_app.display)
{
printf("Error: couldn't open display %s\n", displayName ? displayName :
getenv("DISPLAY"));
exit(-1);
}
_x11_create_window("window", x, y, g_app.width, g_app.height, &g_app,
&visId);
XMapWindow(g_app.display, g_app.win);
glXMakeCurrent(g_app.display, g_app.win, g_app.ctx);
_gl_query_vsync(g_app.display, g_app.win);
printf("GL_RENDERER = %s\n", (*char* *) glGetString(GL_RENDERER));
printf("GL_VERSION = %s\n", (*char* *) glGetString(GL_VERSION));
printf("GL_VENDOR = %s\n", (*char* *) glGetString(GL_VENDOR));
printf("GL_EXTENSIONS = %s\n", (*char* *) glGetString(GL_EXTENSIONS));
printf("VisualID %d, 0x%x\n", (*int*) visId, (*int*) visId);
_gl_init();
_gl_reshape(&g_app, g_app.width, g_app.height);
gst_init(*NULL*, *NULL*);
g_app.gl_display =
GST_GL_DISPLAY(gst_gl_display_x11_new_with_display(g_app.display));
g_app.x11context = gst_context_new(GST_GL_DISPLAY_CONTEXT_TYPE, *TRUE*);
gst_context_set_gl_display(g_app.x11context, g_app.gl_display);
g_app.gl_context = gst_gl_context_new_wrapped(g_app.gl_display, (guintptr)
g_app.ctx, GST_GL_PLATFORM_GLX,GST_GL_API_OPENGL);
g_app.ctxcontext = gst_context_new("gst.gl.app_context", *TRUE*);
gst_structure_set(gst_context_writable_structure(g_app.ctxcontext),
"context", GST_TYPE_GL_CONTEXT, g_app.gl_context, *NULL*);
_gst_gl_init(&g_app, g_app.width, g_app.height);
snprintf(pl, *sizeof*(pl), "appsrc stream-type=0 do-timestamp=1
emit-signals=0 format=3 is-live=1 name=source
caps=\"video/x-raw(memory:GLMemory), "
"width=%d, height=%d, framerate=(fraction)10/1, format=(string)RGBA,
texture-target=(string)external-oes\" ! glcolorconvert ! gldownload !
x264enc tune=0x4 b-adapt=0 ! h264parse "
"! matroskamux ! filesink location=\"/tmp/test.mkv\" sync=0", g_app.width,
g_app.height);
printf("pipeline '%s'\n", pl);
_gst_load_pipeline(&g_app, pl);
_gst_start_pipeline(&g_app);
sleep(1);
g_app.loop = g_main_loop_new(*NULL*, *FALSE*);
printf("main pid %d\n", getpid());
g_idle_add(_gst_idle_loop, &g_app);
g_main_loop_run(g_app.loop);
glXMakeCurrent(g_app.display, None, *NULL*);
glXDestroyContext(g_app.display, g_app.ctx);
XDestroyWindow(g_app.display, g_app.win);
XCloseDisplay(g_app.display);
*return* 0;
}
"
out log:
"
GL_RENDERER = SVGA3D; build: RELEASE; LLVM;
GL_VERSION = 2.1 Mesa 19.2.8
GL_VENDOR = VMware, Inc.
GL_EXTENSIONS = GL_ARB_multisample GL_EXT_abgr GL_EXT_bgra
GL_EXT_blend_color GL_EXT_blend_minmax GL_EXT_blend_subtract
GL_EXT_copy_texture GL_EXT_subtexture GL_EXT_texture_object
GL_EXT_vertex_array GL_EXT_compiled_vertex_array GL_EXT_texture
GL_EXT_texture3D GL_IBM_rasterpos_clip GL_ARB_point_parameters
GL_EXT_draw_range_elements GL_EXT_packed_pixels GL_EXT_point_parameters
GL_EXT_rescale_normal GL_EXT_separate_specular_color
GL_EXT_texture_edge_clamp GL_SGIS_generate_mipmap
GL_SGIS_texture_border_clamp GL_SGIS_texture_edge_clamp GL_SGIS_texture_lod
GL_ARB_framebuffer_sRGB GL_ARB_multitexture GL_EXT_framebuffer_sRGB
GL_IBM_multimode_draw_arrays GL_IBM_texture_mirrored_repeat
GL_ARB_texture_cube_map GL_ARB_texture_env_add GL_ARB_transpose_matrix
GL_EXT_blend_func_separate GL_EXT_fog_coord GL_EXT_multi_draw_arrays
GL_EXT_secondary_color GL_EXT_texture_env_add
GL_EXT_texture_filter_anisotropic GL_EXT_texture_lod_bias
GL_INGR_blend_func_separate GL_NV_blend_square GL_NV_light_max_exponent
GL_NV_texgen_reflection GL_NV_texture_env_combine4 GL_S3_s3tc
GL_SUN_multi_draw_arrays GL_ARB_texture_border_clamp
GL_ARB_texture_compression GL_EXT_framebuffer_object
GL_EXT_texture_compression_s3tc GL_EXT_texture_env_combine
GL_EXT_texture_env_dot3 GL_MESA_window_pos GL_NV_packed_depth_stencil
GL_NV_texture_rectangle GL_ARB_depth_texture GL_ARB_occlusion_query
GL_ARB_shadow GL_ARB_texture_env_combine GL_ARB_texture_env_crossbar
GL_ARB_texture_env_dot3 GL_ARB_texture_mirrored_repeat GL_ARB_window_pos
GL_ATI_fragment_shader GL_EXT_stencil_two_side GL_EXT_texture_cube_map
GL_NV_fog_distance GL_APPLE_packed_pixels GL_ARB_draw_buffers
GL_ARB_fragment_program GL_ARB_fragment_shader GL_ARB_shader_objects
GL_ARB_vertex_program GL_ARB_vertex_shader GL_ATI_draw_buffers
GL_ATI_texture_env_combine3 GL_ATI_texture_float GL_EXT_shadow_funcs
GL_EXT_stencil_wrap GL_MESA_pack_invert GL_NV_primitive_restart
GL_ARB_fragment_program_shadow GL_ARB_half_float_pixel
GL_ARB_occlusion_query2 GL_ARB_point_sprite GL_ARB_shading_language_100
GL_ARB_sync GL_ARB_texture_non_power_of_two GL_ARB_vertex_buffer_object
GL_ATI_blend_equation_separate GL_EXT_blend_equation_separate
GL_OES_read_format GL_ARB_color_buffer_float GL_ARB_pixel_buffer_object
GL_ARB_texture_float GL_ARB_texture_rectangle GL_EXT_pixel_buffer_object
GL_EXT_texture_compression_dxt1 GL_EXT_texture_rectangle
GL_EXT_texture_sRGB GL_ARB_framebuffer_object GL_EXT_framebuffer_blit
GL_EXT_packed_depth_stencil GL_ARB_vertex_array_object
GL_ATI_separate_stencil GL_EXT_gpu_program_parameters
GL_EXT_texture_sRGB_decode GL_OES_EGL_image GL_ARB_copy_buffer
GL_ARB_half_float_vertex GL_ARB_map_buffer_range GL_ARB_texture_swizzle
GL_ARB_vertex_array_bgra GL_EXT_texture_swizzle GL_EXT_vertex_array_bgra
GL_ARB_ES2_compatibility GL_ARB_debug_output
GL_ARB_draw_elements_base_vertex GL_ARB_explicit_attrib_location
GL_ARB_fragment_coord_conventions GL_ARB_provoking_vertex
GL_ARB_sampler_objects GL_ARB_shader_texture_lod GL_EXT_provoking_vertex
GL_ARB_get_program_binary GL_ARB_robustness GL_ARB_separate_shader_objects
GL_ANGLE_texture_compression_dxt3 GL_ANGLE_texture_compression_dxt5
GL_ARB_compressed_texture_pixel_storage GL_ARB_internalformat_query
GL_ARB_map_buffer_alignment GL_ARB_texture_storage
GL_AMD_shader_trinary_minmax GL_ARB_clear_buffer_object
GL_ARB_explicit_uniform_location GL_ARB_invalidate_subdata
GL_ARB_program_interface_query GL_ARB_vertex_attrib_binding GL_KHR_debug
GL_ARB_internalformat_query2 GL_ARB_multi_bind GL_ARB_get_texture_sub_image
GL_KHR_context_flush_control GL_ARB_parallel_shader_compile GL_KHR_no_error
GL_ARB_texture_filter_anisotropic GL_KHR_parallel_shader_compile
VisualID 311, 0x137
FramebufferName 1
texture 1
new video info 0x5578cf6a58e0
set video info ret 1
allocation params
allocation params 0x5578cf60a460 gl_context 0x5578cf673070
pipeline 'appsrc stream-type=0 do-timestamp=1 emit-signals=0 format=3
is-live=1 name=source caps="video/x-raw(memory:GLMemory), width=512,
height=512, framerate=(fraction)10/1, format=(string)RGBA,
texture-target=(string)external-oes" ! glcolorconvert ! gldownload !
x264enc tune=0x4 b-adapt=0 ! h264parse ! matroskamux ! filesink
location="/tmp/test.mkv" sync=0'
Pipeline play
main pid 4625
idle pid 4625
add meta
setup buffer
** (app:4625): CRITICAL **: 13:20:33.001: gst_gl_context_thread_add:
assertion 'context->priv->active_thread == g_thread_self ()' failed
0:00:01.014606602 4625 0x5578cf7b1400 ERROR glbasememory
gstglbasememory.c:178:gst_gl_base_memory_init: Could not create GL buffer
with context:0x5578cf673070
setup buffer ret 1
** (app:4625): CRITICAL **: 13:20:33.001: gst_gl_context_thread_add:
assertion 'context->priv->active_thread == g_thread_self ()' failed
** (app:4625): CRITICAL **: 13:20:33.001: gst_gl_context_thread_add:
assertion 'context->priv->active_thread == g_thread_self ()' failed
0:00:01.014683888 4625 0x5578cf7b1400 ERROR videometa
gstvideometa.c:245:default_map: cannot map memory range 0-1
0:00:01.014692615 4625 0x5578cf7b1400 ERROR default
video-frame.c:168:gst_video_frame_map_id: failed to map video frame plane 0
failed on gst_video_frame_map
"
Nikolay Frey <nikolay.frey at gmail.com>:
> Hi all!
>
> I wrote a simple program to load the opengl textures into the gstreamer
> pipeline, but it does not work. I can’t understand what I am doing wrong. I
> want render graphics to opengl texture and then use the texture as a source
> for the pipeline.
> I would be grateful for any help.
> I provide makefile, app source code and app output log.
> I use gstreamer 1.16.2 and linux ubuntu 19.10.
>
> Thank you!
>
>
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://lists.freedesktop.org/archives/gstreamer-devel/attachments/20200603/fde8d226/attachment-0001.htm>
More information about the gstreamer-devel
mailing list