<div dir="ltr">Thanks for your answer!<div> </div><div>There are some 50 threads with 2 pipelines.</div><div> </div><div>$ top -p 317562 317562 viljar 20 0 3871,0m 121,2m 80,3m S 0,0 0,8 0:00.09 PanoramaTest 317563 viljar 20 0 3871,0m 121,2m 80,3m S 0,0 0,8 0:00.04 QXcbEventQueue 317594 viljar 20 0 3871,0m 121,2m 80,3m S 0,2 0,8 0:03.01 PanoramaTe:cs0 317595 viljar 39 19 3871,0m 121,2m 80,3m S 0,0 0,8 0:00.00 Panoram:disk$0 317596 viljar 20 0 3871,0m 121,2m 80,3m S 0,0 0,8 0:00.00 PanoramaTe:sh0 317597 viljar 20 0 3871,0m 121,2m 80,3m S 0,0 0,8 0:00.00 PanoramaTe:sh1 317598 viljar 20 0 3871,0m 121,2m 80,3m S 0,0 0,8 0:00.00 PanoramaTe:sh2 317599 viljar 20 0 3871,0m 121,2m 80,3m S 0,0 0,8 0:00.00 PanoramaTe:sh3 317600 viljar 20 0 3871,0m 121,2m 80,3m S 0,0 0,8 0:00.00 PanoramaTe:sh4 317601 viljar 20 0 3871,0m 121,2m 80,3m S 0,0 0,8 0:00.00 PanoramaTe:sh5 317602 viljar 20 0 3871,0m 121,2m 80,3m S 0,0 0,8 0:00.00 PanoramaTe:sh6 317603 viljar 20 0 3871,0m 121,2m 80,3m S 0,0 0,8 0:00.00 PanoramaTe:sh7 317604 viljar 20 0 3871,0m 121,2m 80,3m S 0,0 0,8 0:00.00 PanoramaTe:sh8 317605 viljar 39 19 3871,0m 121,2m 80,3m S 0,0 0,8 0:00.00 Panorama:shlo0 317606 viljar 39 19 3871,0m 121,2m 80,3m S 0,0 0,8 0:00.00 Panorama:shlo1 317607 viljar 39 19 3871,0m 121,2m 80,3m S 0,0 0,8 0:00.00 Panorama:shlo2 317608 viljar 39 19 3871,0m 121,2m 80,3m S 0,0 0,8 0:00.00 Panorama:shlo3 317609 viljar 39 19 3871,0m 121,2m 80,3m S 0,0 0,8 0:00.00 Panoram:disk$0 317610 viljar 20 0 3871,0m 121,2m 80,3m S 0,0 0,8 0:00.00 PanoramaTe:sh0 317611 viljar 20 0 3871,0m 121,2m 80,3m S 0,0 0,8 0:00.00 PanoramaTe:sh1 317612 viljar 20 0 3871,0m 121,2m 80,3m S 0,0 0,8 0:00.00 PanoramaTe:sh2 317613 viljar 20 0 3871,0m 121,2m 80,3m S 0,0 0,8 0:00.00 PanoramaTe:sh3 317614 viljar 20 0 3871,0m 121,2m 80,3m S 0,0 0,8 0:00.00 PanoramaTe:sh4 317615 viljar 20 0 3871,0m 121,2m 80,3m S 0,0 0,8 0:00.00 PanoramaTe:sh5 317616 viljar 20 0 3871,0m 121,2m 80,3m S 0,0 0,8 0:00.00 PanoramaTe:sh6 317617 viljar 20 0 3871,0m 121,2m 80,3m S 0,0 0,8 0:00.00 PanoramaTe:sh7 317618 viljar 20 0 3871,0m 121,2m 80,3m S 0,0 0,8 0:00.00 PanoramaTe:sh8 317619 viljar 39 19 3871,0m 121,2m 80,3m S 0,0 0,8 0:00.00 Panorama:shlo0 317620 viljar 39 19 3871,0m 121,2m 80,3m S 0,0 0,8 0:00.00 Panorama:shlo1 317621 viljar 39 19 3871,0m 121,2m 80,3m S 0,0 0,8 0:00.00 Panorama:shlo2 317622 viljar 39 19 3871,0m 121,2m 80,3m S 0,0 0,8 0:00.00 Panorama:shlo3 317623 viljar 20 0 3871,0m 121,2m 80,3m S 0,0 0,8 0:00.00 PanoramaTest 317624 viljar 20 0 3871,0m 121,2m 80,3m S 0,0 0,8 0:00.02 videotestsrc0:s 317626 viljar 39 19 3871,0m 121,2m 80,3m S 0,0 0,8 0:00.00 Panoram:disk$0 317627 viljar 20 0 3871,0m 121,2m 80,3m S 0,0 0,8 0:00.00 PanoramaTe:sh0 317628 viljar 20 0 3871,0m 121,2m 80,3m S 0,0 0,8 0:00.00 PanoramaTe:sh1 317629 viljar 20 0 3871,0m 121,2m 80,3m S 0,0 0,8 0:00.00 PanoramaTe:sh2 317630 viljar 20 0 3871,0m 121,2m 80,3m S 0,0 0,8 0:00.00 PanoramaTe:sh3 317631 viljar 20 0 3871,0m 121,2m 80,3m S 0,0 0,8 0:00.00 PanoramaTe:sh4 317632 viljar 20 0 3871,0m 121,2m 80,3m S 0,0 0,8 0:00.00 PanoramaTe:sh5 317633 viljar 20 0 3871,0m 121,2m 80,3m S 0,0 0,8 0:00.00 PanoramaTe:sh6 317634 viljar 20 0 3871,0m 121,2m 80,3m S 0,0 0,8 0:00.00 PanoramaTe:sh7 317635 viljar 20 0 3871,0m 121,2m 80,3m S 0,0 0,8 0:00.00 PanoramaTe:sh8 317636 viljar 39 19 3871,0m 121,2m 80,3m S 0,0 0,8 0:00.00 Panorama:shlo0 317637 viljar 39 19 3871,0m 121,2m 80,3m S 0,0 0,8 0:00.00 Panorama:shlo1 317638 viljar 39 19 3871,0m 121,2m 80,3m S 0,0 0,8 0:00.00 Panorama:shlo2 317639 viljar 39 19 3871,0m 121,2m 80,3m S 0,0 0,8 0:00.00 Panorama:shlo3 317640 viljar 20 0 3871,0m 121,2m 80,3m S 0,0 0,8 0:00.46 PanoramaTest 317641 viljar 20 0 3871,0m 121,2m 80,3m S 0,0 0,8 0:14.36 videotestsrc1:s 317642 viljar 20 0 3871,0m 121,2m 80,3m S 0,0 0,8 0:00.00 QDBusConnection 2 threads have some activities: # strace -p 317594 ... futex(0x55873a01f390, FUTEX_WAKE_PRIVATE, 1) = 0 ioctl(12, DRM_IOCTL_AMDGPU_CS, 0x7fc2adc76950) = 0 futex(0x55873a24da34, FUTEX_WAKE, 2147483647) = 1 futex(0x55873a01f3e0, FUTEX_WAIT_BITSET_PRIVATE|FUTEX_CLOCK_REALTIME, 0, NULL, FUTEX_BITSET_MATCH_ANY) = -1 EAGAIN (Resource temporarily unavailable) ... gdb) thread 3 [Switching to thread 3 (Thread 0x7fc2adc77640 (LWP 317594))] #0 0x00007fc2b37a48ca in __futex_abstimed_wait_common64 () from /usr/lib/libpthread.so.0 (gdb) bt full #0 0x00007fc2b37a48ca in __futex_abstimed_wait_common64 () at /usr/lib/libpthread.so.0 #1 0x00007fc2b379e270 in pthread_cond_wait@@GLIBC_2.3.2 () at /usr/lib/libpthread.so.0 #2 0x00007fc2ae4087bc in () at /usr/lib/dri/radeonsi_drv_video.so #3 0x00007fc2ae407a98 in () at /usr/lib/dri/radeonsi_drv_video.so #4 0x00007fc2b3798259 in start_thread () at /usr/lib/libpthread.so.0 #5 0x00007fc2b36bf5e3 in clone () at /usr/lib/libc.so.6 # strace -p 317641 ... futex(0x55873a028a20, FUTEX_WAIT_BITSET_PRIVATE, 0, {tv_sec=318695, tv_nsec=960885939}, FUTEX_BITSET_MATCH_ANY) = -1 ETIMEDOUT (Connection timed out) futex(0x55873a01f3e0, FUTEX_WAKE_PRIVATE, 1) = 1 futex(0x55873a01f390, FUTEX_WAKE_PRIVATE, 1) = 1 futex(0x55873a24da34, FUTEX_WAIT_BITSET, 2, NULL, FUTEX_BITSET_MATCH_ANY) = 0 futex(0x55873a01f3e4, FUTEX_WAKE_PRIVATE, 1) = 1 futex(0x55873a01f390, FUTEX_WAKE_PRIVATE, 1) = 1 futex(0x55873a24da34, FUTEX_WAIT_BITSET, 2, NULL, FUTEX_BITSET_MATCH_ANY) = -1 EAGAIN (Resource temporarily unavailable) ioctl(12, DRM_IOCTL_AMDGPU_WAIT_CS, 0x7fc1ceffc180) = 0 futex(0x55873a01f3e0, FUTEX_WAKE_PRIVATE, 1) = 1 futex(0x55873a01f390, FUTEX_WAKE_PRIVATE, 1) = 1 futex(0x55873a24da34, FUTEX_WAIT_BITSET, 2, NULL, FUTEX_BITSET_MATCH_ANY) = -1 EAGAIN (Resource temporarily unavailable) poll([{fd=23, events=POLLIN|POLLOUT}], 1, -1) = 1 ([{fd=23, revents=POLLOUT}]) writev(23, [{iov_base="\212\v\4\0\21\0\0\t\0\0\0\0\225\3\345\3\224\1\22\0\v\0\240\7\17\0\0\ti\274\0\0\0\0\0\0\21\0\0\t\0\0\0\0\0\0\0\0\0\0\0\0\20\0\0\t\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", iov_len=88}], 1) = 88 ... gdb) thread 49 [Switching to thread 49 (Thread 0x7fc1ceffd640 (LWP 317641))] #0 0x00007fc2b36ba18d in syscall () from /usr/lib/libc.so.6 (gdb) bt full #0 0x00007fc2b36ba18d in syscall () at /usr/lib/libc.so.6 #1 0x00007fc2b4c0178e in () at /usr/lib/libgstreamer-1.0.so.0 #2 0x00007fc2b4c01c13 in () at /usr/lib/libgstreamer-1.0.so.0 #3 0x00007fc2b4b9ac9e in gst_clock_id_wait () at /usr/lib/libgstreamer-1.0.so.0 #4 0x00007fc2b4cc8608 in gst_base_sink_wait_clock () at /usr/lib/libgstbase-1.0.so.0 #5 0x00007fc2b4ced895 in () at /usr/lib/libgstbase-1.0.so.0 #6 0x00007fc2b4ceeaff in () at /usr/lib/libgstbase-1.0.so.0 #7 0x00007fc2b4cbe3a1 in () at /usr/lib/libgstbase-1.0.so.0 #8 0x00007fc2b4bd1f85 in () at /usr/lib/libgstreamer-1.0.so.0 #9 0x00007fc2b4bd567b in () at /usr/lib/libgstreamer-1.0.so.0 #10 0x00007fc2b4bd5aae in gst_pad_push () at /usr/lib/libgstreamer-1.0.so.0 #11 0x00007fc2b4ccff9e in () at /usr/lib/libgstbase-1.0.so.0 #12 0x00007fc2b4bfcbe1 in () at /usr/lib/libgstreamer-1.0.so.0 #13 0x00007fc2b4a3d8f7 in () at /usr/lib/libglib-2.0.so.0 #14 0x00007fc2b4a3ad11 in () at /usr/lib/libglib-2.0.so.0 #15 0x00007fc2b3798259 in start_thread () at /usr/lib/libpthread.so.0 #16 0x00007fc2b36bf5e3 in clone () at /usr/lib/libc.so.6 </div><div> </div><div>However most threads seem to wait on something, and bactrace will print: </div><div> </div><div>(gdb) thread 6 [Switching to thread 6 (Thread 0x7fc2897fa640 (LWP 317597))] #0 0x00007fc2b37a48ca in __futex_abstimed_wait_common64 () from /usr/lib/libpthread.so.0 (gdb) bt full #0 0x00007fc2b37a48ca in __futex_abstimed_wait_common64 () at /usr/lib/libpthread.so.0 #1 0x00007fc2b379e270 in pthread_cond_wait@@GLIBC_2.3.2 () at /usr/lib/libpthread.so.0 #2 0x00007fc2a04168cc in () at /usr/lib/dri/radeonsi_dri.so #3 0x00007fc2a040ffa8 in () at /usr/lib/dri/radeonsi_dri.so #4 0x00007fc2b3798259 in start_thread () at /usr/lib/libpthread.so.0 #5 0x00007fc2b36bf5e3 in clone () at /usr/lib/libc.so.6 </div><div>Well this does not tell me personally what is going on, but it seems something to do with the vaapi driver</div></div> <div class="gmail_quote"><div dir="ltr" class="gmail_attr">Kontakt Nirbheek Chauhan (<<a href="mailto:nirbheek.chauhan@gmail.com">nirbheek.chauhan@gmail.com</a>>) kirjutas kuupäeval T, 9. november 2021 kell 23:58: </div><blockquote class="gmail_quote" style="margin:0px 0px 0px 0.8ex;border-left:1px solid rgb(204,204,204);padding-left:1ex">Hi Viljar, You should probably dig into the `gdb` backtrace output and see if there's a deadlock, and if so where. Install debug symbols, run your test program. When it deadlocks, attach gdb to it using `gdb -p <pid>` and check the output of "bt full". Look for any mutexes that are locked or are waiting for a lock. If a mutex is owned by a thread and another thread is waiting forever trying to lock it, that is a deadlock. Analyzing the owners of various locked mutexes and threads that are trying to lock the same mutexes will tell you what's happening. Cheers, Nirbheek On Tue, Nov 9, 2021 at 5:00 PM Viljar Hera via gstreamer-devel <<a href="mailto:gstreamer-devel@lists.freedesktop.org" target="_blank">gstreamer-devel@lists.freedesktop.org</a>> wrote: > > Oops, > > switch (gst_element_set_state(pipeline, GST_STATE_PLAYING)) // <- blocks indefinitely > > that comment landed on wrong place, should be: > > > switch (gst_element_set_state(pipelines[i], GST_STATE_NULL)) // <- blocks indefinitely > > > Kontakt meistrimees--- via gstreamer-devel (<<a href="mailto:gstreamer-devel@lists.freedesktop.org" target="_blank">gstreamer-devel@lists.freedesktop.org</a>>) kirjutas kuupäeval T, 9. november 2021 kell 11:55: >> >> >> I prepared a simple test program that I could reproduce the issue. I have tested with latest stable version of GStreamer 1.18.5, using radeonsi driver, but it will probably happen with Intel drivers as well. >> Increasing the number of pipeline-s will increase the chance of getting one of them stuck on teardown. >> >> I'm considering opening a bugreport, but I want a second opinion before I do. Maybe I'm missing something here. >> >> #include <QApplication> >> >> #include <QDebug> >> >> #include <QWidget> >> >> #include <QGridLayout> >> >> >> #include <gst/gst.h> >> >> #include <gst/video/videooverlay.h> >> >> >> int main(int argc, char *argv[]) >> >> { >> >> gst_init(0, 0); >> >> >> QApplication app(argc, argv); >> >> app.connect(&app, &QApplication::lastWindowClosed, &app, &QApplication::quit); >> >> >> // We will arrenge videoWidgets into the colums and rows >> >> QWidget mainWidget; >> >> QGridLayout *layout = new QGridLayout; >> >> layout->setHorizontalSpacing(0); >> >> mainWidget.setLayout(layout); >> >> >> int streamCount = 9; >> >> int rowCount = 3; >> >> int colCount = streamCount % rowCount == 0 ? streamCount / rowCount : streamCount / rowCount + 1; >> >> >> const char *pipelineStr = "videotestsrc ! vaapisink name=\"thesink\""; >> >> QVector<GstElement *> pipelines; >> >> >> for (int i = 0; i < streamCount; ++i) >> >> { >> >> GError *error = NULL; >> >> GstElement *pipeline = gst_parse_launch(pipelineStr, &error); >> >> >> if (error) { >> >> qInfo() << "Error: " << error->message; >> >> g_clear_error(&error); >> >> } >> >> >> if (!pipeline) { >> >> qInfo() << "Unable to create pipeline"; >> >> continue; >> >> } >> >> >> GstElement *sink = gst_bin_get_by_name(GST_BIN(pipeline), "thesink"); >> >> >> if (!sink) >> >> { >> >> qInfo() << "Failed to get vaapisink element"; >> >> continue; >> >> } >> >> >> QWidget *videoWidget = new QWidget(); >> >> layout->addWidget(videoWidget, i / colCount, i % colCount); >> >> >> if (sink) >> >> { >> >> gst_video_overlay_set_window_handle(GST_VIDEO_OVERLAY(sink), videoWidget->winId()); >> >> g_object_unref(sink); >> >> } >> >> >> switch (gst_element_set_state(pipeline, GST_STATE_PLAYING)) // <- blocks indefinitely >> >> { >> >> case GST_STATE_CHANGE_FAILURE: >> >> qInfo() << "Failed to start pipeline" << i; >> >> break; >> >> case GST_STATE_CHANGE_SUCCESS: >> >> qInfo() << "Pipeline" < >> >> break; >> >> case GST_STATE_CHANGE_ASYNC: >> >> qInfo() << "Pipeline" < >> >> break; >> >> case GST_STATE_CHANGE_NO_PREROLL: >> >> qInfo() << "Pipeline" < >> >> break; >> >> default: >> >> qInfo() << "Unknown value"; >> >> break; >> >> } >> >> >> pipelines.append(pipeline); >> >> } >> >> >> mainWidget.resize(1280, 720); >> >> mainWidget.showMaximized(); >> >> >> int ret = app.exec(); >> >> >> qInfo() << "Terminating"; >> >> mainWidget.hide(); >> >> >> for (int i = 0; i < pipelines.size(); ++i) { >> >> qInfo() << "Stopping pipeline" << i; >> >> switch (gst_element_set_state(pipelines[i], GST_STATE_NULL)) >> >> { >> >> case GST_STATE_CHANGE_FAILURE: >> >> qInfo() << "Failed to stop pipeline" << i; >> >> break; >> >> case GST_STATE_CHANGE_SUCCESS: >> >> qInfo() << "Pipeline" < >> >> break; >> >> case GST_STATE_CHANGE_ASYNC: >> >> qInfo() << "Pipeline" < >> >> break; >> >> case GST_STATE_CHANGE_NO_PREROLL: >> >> qInfo() << "Preroll" << i; >> >> break; >> >> default: >> >> qInfo() << "Unknown value"; >> >> break; >> >> } >> >> >> g_object_unref(pipelines[i]); >> >> } >> >> >> return ret; >> >> } >> >> >> If someone could confirm the issue I would open a bug report. >> >> </blockquote></div>