[PATCH xserver] Xdmx: fix crashes caused by OpenGL applications

[Raimonds Cicans]

Signed-off-by: Raimonds Cicans <ray at apollo.lv>
---
 hw/dmx/glxProxy/glxcmds.c | 38 ++++++++++++++++++--------------------
 1 file changed, 18 insertions(+), 20 deletions(-)

diff --git a/hw/dmx/glxProxy/glxcmds.c b/hw/dmx/glxProxy/glxcmds.c
index 330d5c4ad..871b416d3 100644
--- a/hw/dmx/glxProxy/glxcmds.c
+++ b/hw/dmx/glxProxy/glxcmds.c
@@ -159,7 +159,7 @@ CreateContext(__GLXclientState * cl,
     ScreenPtr pScreen;
     __GLXcontext *glxc, *shareglxc;
     __GLXvisualConfig *pGlxVisual;
-    __GLXscreenInfo *pGlxScreen;
+    DMXScreenInfo *dmxScreenGlx;
     VisualID visual = vid;
     GLint i;
     int from_screen = screen;
@@ -210,7 +210,7 @@ CreateContext(__GLXclientState * cl,
     }
 
     pScreen = screenInfo.screens[screen];
-    pGlxScreen = &__glXActiveScreens[screen];
+    dmxScreenGlx = &dmxScreens[screen];
 
     if (fbconfigId != None) {
         glxc->pFBConfig = glxLookupFBConfig(fbconfigId);
@@ -241,13 +241,13 @@ CreateContext(__GLXclientState * cl,
             return BadValue;
         }
 
-        pGlxVisual = pGlxScreen->pGlxVisual;
-        for (i = 0; i < pGlxScreen->numVisuals; i++, pGlxVisual++) {
+        pGlxVisual = &dmxScreenGlx->glxVisuals[i];
+        for (i = 0; i < dmxScreenGlx->numGlxVisuals; i++, pGlxVisual++) {
             if (pGlxVisual->vid == visual) {
                 break;
             }
         }
-        if (i == pGlxScreen->numVisuals) {
+        if (i == dmxScreenGlx->numGlxVisuals) {
             /*
              ** Visual not support on this screen by this OpenGL implementation.
              */
@@ -275,7 +275,6 @@ CreateContext(__GLXclientState * cl,
     }
 
     glxc->pScreen = pScreen;
-    glxc->pGlxScreen = pGlxScreen;
     glxc->pVisual = pVisual;
     glxc->pGlxVisual = pGlxVisual;
 
@@ -296,7 +295,6 @@ CreateContext(__GLXclientState * cl,
         int sent = 0;
 
         pScreen = screenInfo.screens[screen];
-        pGlxScreen = &__glXActiveScreens[screen];
         dmxScreen = &dmxScreens[screen];
 
         if (glxc->pFBConfig) {
@@ -1627,7 +1625,7 @@ __glXCopyContext(__GLXclientState * cl, GLbyte * pc)
     /*
      ** They must be in the same address space, and same screen.
      */
-    if (src->pGlxScreen != dst->pGlxScreen) {
+    if (src->pScreen != dst->pScreen) {
         client->errorValue = source;
         return BadMatch;
     }
@@ -1690,7 +1688,7 @@ __glXGetVisualConfigs(__GLXclientState * cl, GLbyte * pc)
     ClientPtr client = cl->client;
     xGLXGetVisualConfigsReq *req = (xGLXGetVisualConfigsReq *) pc;
     xGLXGetVisualConfigsReply reply;
-    __GLXscreenInfo *pGlxScreen;
+    DMXScreenInfo *dmxScreenGlx;
     __GLXvisualConfig *pGlxVisual;
     CARD32 buf[__GLX_TOTAL_CONFIG];
     unsigned int screen;
@@ -1702,22 +1700,22 @@ __glXGetVisualConfigs(__GLXclientState * cl, GLbyte * pc)
         client->errorValue = screen;
         return BadValue;
     }
-    pGlxScreen = &__glXActiveScreens[screen];
+    dmxScreenGlx = &dmxScreens[screen];
 
     reply = (xGLXGetVisualConfigsReply) {
         .type = X_Reply,
         .sequenceNumber = client->sequence,
-        .numVisuals = pGlxScreen->numGLXVisuals,
+        .numVisuals = dmxScreenGlx->numGlxVisuals,
         .numProps = __GLX_TOTAL_CONFIG,
-        .length = (pGlxScreen->numGLXVisuals * __GLX_SIZE_CARD32 *
+        .length = (dmxScreenGlx->numGlxVisuals * __GLX_SIZE_CARD32 *
                     __GLX_TOTAL_CONFIG) >> 2
     };
 
     WriteToClient(client, sz_xGLXGetVisualConfigsReply, &reply);
 
-    for (i = 0; i < pGlxScreen->numVisuals; i++) {
-        pGlxVisual = &pGlxScreen->pGlxVisual[i];
-        if (!pGlxScreen->isGLXvis[i] || pGlxVisual->vid == 0) {
+    for (i = 0; i < dmxScreenGlx->numGlxVisuals; i++) {
+        pGlxVisual = &dmxScreenGlx->glxVisuals[i];
+        if (pGlxVisual->vid == 0) {
             /* not a usable visual */
             continue;
         }
@@ -2871,7 +2869,7 @@ __glXGetFBConfigs(__GLXclientState * cl, GLbyte * pc)
     int numAttribs = __GLX_TOTAL_FBCONFIG_PROPS;
     unsigned int screen = req->screen;
     int numFBConfigs, i, p;
-    __GLXscreenInfo *pGlxScreen;
+    DMXScreenInfo *dmxScreenGlx;
 
     if (screen >= screenInfo.numScreens) {
         /* The client library must send a valid screen number. */
@@ -2879,7 +2877,7 @@ __glXGetFBConfigs(__GLXclientState * cl, GLbyte * pc)
         return BadValue;
     }
 
-    pGlxScreen = &__glXActiveScreens[screen];
+    dmxScreenGlx = &dmxScreens[screen];
     numFBConfigs = __glXNumFBConfigs;
 
     reply = (xGLXGetFBConfigsReply) {
@@ -2989,9 +2987,9 @@ __glXGetFBConfigs(__GLXclientState * cl, GLbyte * pc)
             }
 
             if (pGlxVisual) {
-                for (v = 0; v < pGlxScreen->numVisuals; v++) {
-                    if (glxVisualsMatch(&pGlxScreen->pGlxVisual[v], pGlxVisual)) {
-                        associatedVisualId = pGlxScreen->pGlxVisual[v].vid;
+                for (v = 0; v < dmxScreenGlx->numGlxVisuals; v++) {
+                    if (glxVisualsMatch(&dmxScreenGlx->glxVisuals[v], pGlxVisual)) {
+                        associatedVisualId = dmxScreenGlx->glxVisuals[v].vid;
                         found = 1;
                         break;
                     }
-- 
2.13.6

Problem: _ANY_ OpenGL application causes Xdmx crash

Cause: variable __glXActiveScreens is defined and used but
_NOT_INITIALIZED_ANYWHERE_. So on first use of this variable
Xdmx segfaults. It looks it is remains of some half way refactory.
Problem introduced somewhere 1.10.0.

Fix: replace variable __glXActiveScreens with variable dmxScreens
where possible. Patch contains minimum replacements needed to get
things to work. Perfect solution - remove variable __glXActiveScreens
alltogether.

Tested applications:
glxinfo - semi-works (some application's errors, no Xdmx errors)
glxgears - works
gl-117 - works
glmark2 - failed (application failed to open screen, no Xdmx errors)
nexuiz - works