[VDPAU] [PATCH v2] Fix leaked extension info on library unload

[Robert Morell]

In this sequence:
dlopen(libvdpau.so)
vdp_device_create_x11(dpy, ...)
dlclose(libvdpau.so)
XCloseDisplay(dpy)

the process will attempt to call the address at which DRI2CloseDisplay
was previously mapped, possibly resulting in a SEGV.

Instead of tracking displays to which we've added hooks and cleaning up
the extension on library unload or display close, simply clean up after
ourselves once we have the data we need.

Signed-off-by: Robert Morell <rmorell at nvidia.com>
---

Apologies for the duplicated post earlier.  Here is an updated version of the
patch that actually destroys the extension hooks entirely using
XextDestroyExtension(), rather than just removing this particular display.

 src/mesa_dri2.c     | 19 +++++++++++++++++--
 src/mesa_dri2.h     |  3 +++
 src/vdpau_wrapper.c |  3 +++
 3 files changed, 23 insertions(+), 2 deletions(-)

diff --git a/src/mesa_dri2.c b/src/mesa_dri2.c
index dbf9aa8..3bc75ef 100644
--- a/src/mesa_dri2.c
+++ b/src/mesa_dri2.c
@@ -42,7 +42,6 @@
 
 static char dri2ExtensionName[] = DRI2_NAME;
 static XExtensionInfo *dri2Info;
-static XEXT_GENERATE_CLOSE_DISPLAY (DRI2CloseDisplay, dri2Info)
 
 static /* const */ XExtensionHooks dri2ExtensionHooks = {
   NULL,                   /* create_gc */
@@ -51,7 +50,7 @@ static /* const */ XExtensionHooks dri2ExtensionHooks = {
   NULL,                   /* free_gc */
   NULL,                   /* create_font */
   NULL,                   /* free_font */
-  DRI2CloseDisplay,       /* close_display */
+  NULL,                   /* close_display */
   NULL,                   /* wire_to_event */
   NULL,                   /* event_to_wire */
   NULL,                   /* error */
@@ -75,6 +74,14 @@ _vdp_DRI2QueryExtension(Display * dpy, int *eventBase, int *errorBase)
       return True;
    }
 
+   if (dri2Info) {
+      if (info) {
+         XextRemoveDisplay(dri2Info, dpy);
+      }
+      XextDestroyExtension(dri2Info);
+      dri2Info = NULL;
+   }
+
    return False;
 }
 
@@ -161,3 +168,11 @@ _vdp_DRI2Connect(Display * dpy, XID window, char **driverName, char **deviceName
 
    return True;
 }
+
+void
+_vdp_DRI2RemoveExtension(Display * dpy)
+{
+   XextRemoveDisplay(dri2Info, dpy);
+   XextDestroyExtension(dri2Info);
+   dri2Info = NULL;
+}
diff --git a/src/mesa_dri2.h b/src/mesa_dri2.h
index 5c5fb12..09bde8c 100644
--- a/src/mesa_dri2.h
+++ b/src/mesa_dri2.h
@@ -47,4 +47,7 @@ extern Bool
 _vdp_DRI2Connect(Display * display, XID window, char **driverName,
                  char **deviceName);
 
+extern void
+_vdp_DRI2RemoveExtension(Display * display);
+
 #endif
diff --git a/src/vdpau_wrapper.c b/src/vdpau_wrapper.c
index a1d5d40..d847a87 100644
--- a/src/vdpau_wrapper.c
+++ b/src/vdpau_wrapper.c
@@ -86,14 +86,17 @@ static char * _vdp_get_driver_name_from_dri2(
 
     if (!_vdp_DRI2QueryVersion(display, &major, &minor) ||
             (major < 1 || (major == 1 && minor < 2))) {
+        _vdp_DRI2RemoveExtension(display);
         return NULL;
     }
 
     if (!_vdp_DRI2Connect(display, root, &driver_name, &device_name)) {
+        _vdp_DRI2RemoveExtension(display);
         return NULL;
     }
 
     XFree(device_name);
+    _vdp_DRI2RemoveExtension(display);
 #endif /* DRI2 */
     return driver_name;
 }
-- 
1.7.12.4