[Spice-commits] configure.ac server/Makefile.am server/gstreamer-encoder.c

[Frediano Ziglio]

 configure.ac               |    6 ++++++
 server/Makefile.am         |    2 ++
 server/gstreamer-encoder.c |   30 ++++++++++++++++++++++++++++++
 3 files changed, 38 insertions(+)

New commits:
commit 14aee7cd745cbb6d2f3d5203dbb878dbe966dbc5
Author: Frediano Ziglio <fziglio at redhat.com>
Date:   Fri May 19 12:21:48 2017 +0100

    gstreamer: Check if ORC library can work
    
    ORC library is used internally by GStreamer to generate code
    dynamically.
    If ORC cannot allocate executable memory, the failure causes
    an abort(3) to be called.
    This happens on some SELinux configurations that disable executable
    memory allocation (execmem boolean).
    Check that ORC could work before attempting to use GStreamer to
    avoid crashes.
    While this check is done, the ORC library outputs an error which will
    be well visible in Qemu output.
    
    Signed-off-by: Frediano Ziglio <fziglio at redhat.com>
    Acked-by: Christophe Fergeau <cfergeau at redhat.com>

diff --git a/configure.ac b/configure.ac
index a840eebd..f80193b3 100644
--- a/configure.ac
+++ b/configure.ac
@@ -116,6 +116,12 @@ AS_IF([test x"$missing_gstreamer_elements" = xyes],
     [SPICE_WARNING([The GStreamer video encoder can be built but may not work.])
 ])
 
+if test "x$have_gstreamer_0_10" = "xyes" -o "x$have_gstreamer_1_0" = "xyes"; then
+    PKG_CHECK_MODULES(ORC, orc-0.4)
+    AC_SUBST(ORC_CFLAGS)
+    AC_SUBST(ORC_LIBS)
+fi
+
 AC_ARG_ENABLE([automated_tests],
               AS_HELP_STRING([--enable-automated-tests], [Enable automated tests using spicy-screenshot (part of spice-gtk)]),,
               [enable_automated_tests="no"])
diff --git a/server/Makefile.am b/server/Makefile.am
index ef8d31fc..5d5590af 100644
--- a/server/Makefile.am
+++ b/server/Makefile.am
@@ -20,6 +20,7 @@ AM_CPPFLAGS =					\
 	$(SSL_CFLAGS)				\
 	$(VISIBILITY_HIDDEN_CFLAGS)		\
 	$(WARN_CFLAGS)				\
+	$(ORC_CFLAGS)				\
 	$(NULL)
 
 noinst_LTLIBRARIES = libserver.la
@@ -54,6 +55,7 @@ libserver_la_LIBADD =							\
 	$(SSL_LIBS)							\
 	$(Z_LIBS)							\
 	$(SPICE_NONPKGCONFIG_LIBS)					\
+	$(ORC_LIBS)							\
 	$(NULL)
 
 libspice_serverincludedir = $(includedir)/spice-server
diff --git a/server/gstreamer-encoder.c b/server/gstreamer-encoder.c
index bb4f27ba..b2ddfea9 100644
--- a/server/gstreamer-encoder.c
+++ b/server/gstreamer-encoder.c
@@ -27,6 +27,7 @@
 #include <gst/app/gstappsrc.h>
 #include <gst/app/gstappsink.h>
 #include <gst/video/video.h>
+#include <orc/orcprogram.h>
 
 #include "red-common.h"
 #include "video-encoder.h"
@@ -1702,6 +1703,30 @@ static void spice_gst_encoder_get_stats(VideoEncoder *video_encoder,
     }
 }
 
+/* Check if ORC library can work.
+ * ORC library is used quite extensively by GStreamer
+ * to generate code dynamically. If ORC cannot work, GStreamer
+ * will abort(3) the entire process.
+ */
+static bool orc_check(void)
+{
+    static bool orc_checked = false;
+    static bool orc_dynamic_code_ok = false;
+
+    if (SPICE_UNLIKELY(!orc_checked)) {
+        OrcCode *code = orc_code_new();
+        if (code) {
+            /* allocating 0 byte for code makes the function not crash
+             * but it does all the initializations and checks */
+            orc_code_allocate_codemem(code, 0);
+            orc_dynamic_code_ok = code->code != NULL;
+            orc_code_free(code);
+        }
+        orc_checked = true;
+    }
+    return orc_dynamic_code_ok;
+}
+
 VideoEncoder *gstreamer_encoder_new(SpiceVideoCodecType codec_type,
                                     uint64_t starting_bit_rate,
                                     VideoEncoderRateControlCbs *cbs,
@@ -1721,6 +1746,11 @@ VideoEncoder *gstreamer_encoder_new(SpiceVideoCodecType codec_type,
         return NULL;
     }
 
+    // avoid aborting the process
+    if (!orc_check()) {
+        return NULL;
+    }
+
     SpiceGstEncoder *encoder = spice_new0(SpiceGstEncoder, 1);
     encoder->base.destroy = spice_gst_encoder_destroy;
     encoder->base.encode_frame = spice_gst_encoder_encode_frame;