Xfbdev color problems due to endianess

[Leon Ljunggren]

I got back to this after working on some other things for a while and
I've made some progress. I looked at an old version of xfree that we
had since old with the endian problems fixed and tried to apply the
changes to xorg 7.3 (see diff form normal xorg 7.3 and my patched
version below) and it did help somewhat. Before I had problems with
"dirt" in for example xeyes and fonts being displaced (they looked ok,
they were just moved to the left) and this were fixed, but I still
have problems with the colours looking off.

Since the changes did fix it for the old xfree 4.5.0 I'm guessing
there's some new parts in xorg7.3 that I would have to patch in
similar manner? Anyone got any pointers of where that could be?

Thanks
/Leon

--- xs_x11/xorg_true/xserver/include/servermd.h	2008-09-16
01:44:34.000000000 +0200
+++ x11_color/xorg_true/xserver/include/servermd.h	2008-09-29
12:37:54.000000000 +0200
@@ -257,8 +257,12 @@

 /* linux on ARM */
 #if defined(linux) && defined(__arm__)
+/*
 #define IMAGE_BYTE_ORDER	LSBFirst
 #define BITMAP_BIT_ORDER	LSBFirst
+*/
+#define IMAGE_BYTE_ORDER	MSBFirst
+#define BITMAP_BIT_ORDER	MSBFirst
 #define GLYPHPADBYTES		4
 #endif

--- xs_x11/xorg_true/xserver/miext/shadow/shadow.h	2008-09-16
01:44:33.000000000 +0200
+++ x11_color/xorg_true/xserver/miext/shadow/shadow.h	2008-09-29
12:22:34.000000000 +0200
@@ -24,6 +24,8 @@
 #ifndef _SHADOW_H_
 #define _SHADOW_H_

+#define ccpxsswap(x)  (((x >> 8) & 0x00FF00FF) | ((x << 8) & 0xFF00FF00))
+
 #include "scrnintstr.h"

 #ifdef RENDER
--- xs_x11/xorg_true/xserver/miext/shadow/shpacked.c	2008-09-16
01:44:33.000000000 +0200
+++ x11_color/xorg_true/xserver/miext/shadow/shpacked.c	2008-09-29
12:24:24.000000000 +0200
@@ -103,7 +103,11 @@
 		scr += i;
 #define PickBit(a,i)	(((a) >> (i)) & 1)
 		while (i--)
-		    *win++ = *sha++;
+		{
+		  /* *win++ = *sha++;*/
+		  *win++ = ccpxsswap(*sha);
+		  sha++;
+		}
 	    }
 	    shaLine += shaStride;
 	    y++;
--- xs_x11/xorg_true/xserver/miext/shadow/shplanar8.c	2008-09-16
01:44:33.000000000 +0200
+++ x11_color/xorg_true/xserver/miext/shadow/shplanar8.c	2008-09-29
12:26:26.000000000 +0200
@@ -161,7 +161,8 @@
 			GetBits(plane,2,s2);
 			GetBits(plane,4,s3);
 			GetBits(plane,6,s4);
-			*win++ = s1 | (s2 << 8) | (s3 << 16) | (s4 << 24);
+			/**win++ = s1 | (s2 << 8) | (s3 << 16) | (s4 << 24);*/
+			*win++ = ccpxsswap(s1 | (s2 << 8) | (s3 << 16) | (s4 << 24));
 			sha += 8;
 		    }
 		}
--- xs_x11/xorg_true/xserver/miext/shadow/shplanar.c	2008-09-16
01:44:33.000000000 +0200
+++ x11_color/xorg_true/xserver/miext/shadow/shplanar.c	2008-09-29
12:27:30.000000000 +0200
@@ -159,7 +159,8 @@
 			GetBits(plane,1,s2);
 			GetBits(plane,2,s3);
 			GetBits(plane,3,s4);
-			*win++ = s1 | (s2 << 8) | (s3 << 16) | (s4 << 24);
+			/**win++ = s1 | (s2 << 8) | (s3 << 16) | (s4 << 24);*/
+			*win++ = ccpxsswap(s1 | (s2 << 8) | (s3 << 16) | (s4 << 24));
 			sha += 4;
 		    }
 		}
--- xs_x11/xorg_true/xserver/miext/shadow/shrotate.c	2008-09-16
01:44:33.000000000 +0200
+++ x11_color/xorg_true/xserver/miext/shadow/shrotate.c	2008-09-29
12:28:40.000000000 +0200
@@ -288,7 +288,8 @@
 			}
 			sha += shaStepOverY;
 		    }
-		    *win++ = bits;
+		    /**win++ = bits;*/
+		    *win++ = ccpxsswap(bits);
 		}
 	    }
 	    scr_y++;
--- xs_x11/xorg_true/xserver/hw/kdrive/vesa/vesa.c	2008-09-16
01:44:41.000000000 +0200
+++ x11_color/xorg_true/xserver/hw/kdrive/vesa/vesa.c	2008-10-02
11:10:03.000000000 +0200
@@ -758,7 +758,7 @@
 		{
 		    bits = *sha++;
 		    vesaInvertBits32(bits);
-		    *win++ = bits;
+		    *win++ = ccpxsswap(bits);
 		}
 	    }
 	    shaLine += shaStride;


On Wed, Sep 10, 2008 at 3:05 PM, Leon Ljunggren
<leon.ljunggren at gmail.com> wrote:
> Hi,
> I have compiled and got kdrive (Xfbdev), using the latest stable
> source from git, running on my ARM based system but I'm experiencing
> problems with colours., they are "inverted". This is casued by the
> fact that my machine is big endian while the gfx module is little
> endian. This means that:
>
> BBBBBGGGGGGRRRRR
>
> becomes
>
> GGGRRRRRBBBBBGGG
>
> In fbdev.c there's a color masking feature (in
> fbdevScreenInitialize(...), line 264-266), that I thought I could
> modify to fix the problem:
>
> screen->fb[0].redMask = Mask (3, 5);
> screen->fb[0].greenMask = Mask (0, 3) | Mask (13, 3);
> screen->fb[0].blueMask = Mask (8, 5);
>
> However, when I run with these changes the server quits with a
> segmentation fault error. It would seem that it doesn't like having
> the green section split up in two part. I can swap the masks around as
> I wish as long as I don't try to split the bits for any one mask.
>
> Anyone that has any idea on what I can do to fix this? I guess I could
> swap the byte order of each pixel as they are written to the
> framebuffer, but I haven't been able to find where this writing is
> done in the server.
>
> Thanks
> Leon Ljunggren
>