[PATCH xserver] xfree86: fix gamma compute when palette_size > 256

[walter harms]

Am 30.10.2017 07:33, schrieb Qiang Yu:
> palette_(red|green|blue)_size > crtc->gamma_size (=256)
> this may happen when screen has per RGB chanel > 8bit,
> i.e. 30bit depth screen 10bit per RGB.
> 
> Signed-off-by: Qiang Yu <Qiang.Yu at amd.com>
> ---
>  hw/xfree86/modes/xf86RandR12.c | 96 ++++++++++++++++++++++++++++++------------
>  1 file changed, 69 insertions(+), 27 deletions(-)
> 
> diff --git a/hw/xfree86/modes/xf86RandR12.c b/hw/xfree86/modes/xf86RandR12.c
> index fe8770d..881c8a6 100644
> --- a/hw/xfree86/modes/xf86RandR12.c
> +++ b/hw/xfree86/modes/xf86RandR12.c
> @@ -1258,40 +1258,82 @@ xf86RandR12CrtcComputeGamma(xf86CrtcPtr crtc, LOCO *palette,
>  
>      for (shift = 0; (gamma_size << shift) < (1 << 16); shift++);
>  
> -    gamma_slots = crtc->gamma_size / palette_red_size;
> -    for (i = 0; i < palette_red_size; i++) {
> -        value = palette[i].red;
> -        if (gamma_red)
> -            value = gamma_red[value];
> -        else
> -            value <<= shift;
> +    if (crtc->gamma_size >= palette_red_size) {

minor nitpick:

IMHO: if you use if (crtc->gamma_size < palette_red_size) and switch the
blocks it is more easy to read.

And since the if () are a bit large, did you consider to make it into
3 separate functions to improve readability ?

re,
 wh

> +        gamma_slots = crtc->gamma_size / palette_red_size;
> +        for (i = 0; i < palette_red_size; i++) {
> +            value = palette[i].red;
> +            if (gamma_red)
> +                value = gamma_red[value];
> +            else
> +                value <<= shift;
>  
> -        for (j = 0; j < gamma_slots; j++)
> -            crtc->gamma_red[i * gamma_slots + j] = value;
> +            for (j = 0; j < gamma_slots; j++)
> +                crtc->gamma_red[i * gamma_slots + j] = value;
> +        }
>      }
> +    else {
> +        gamma_slots = palette_red_size / crtc->gamma_size;
> +        for (i = 0; i < crtc->gamma_size; i++) {
> +            value = palette[i * gamma_slots].red;
> +            if (gamma_red)
> +                value = gamma_red[value];
> +            else
> +                value <<= shift;
>  
> -    gamma_slots = crtc->gamma_size / palette_green_size;
> -    for (i = 0; i < palette_green_size; i++) {
> -        value = palette[i].green;
> -        if (gamma_green)
> -            value = gamma_green[value];
> -        else
> -            value <<= shift;
> +            crtc->gamma_red[i] = value;
> +        }
> +    }
> +
> +    if (crtc->gamma_size >= palette_green_size) {
> +        gamma_slots = crtc->gamma_size / palette_green_size;
> +        for (i = 0; i < palette_green_size; i++) {
> +            value = palette[i].green;
> +            if (gamma_green)
> +                value = gamma_green[value];
> +            else
> +                value <<= shift;
> +
> +            for (j = 0; j < gamma_slots; j++)
> +                crtc->gamma_green[i * gamma_slots + j] = value;
> +        }
> +    }
> +    else {
> +        gamma_slots = palette_green_size / crtc->gamma_size;
> +        for (i = 0; i < crtc->gamma_size; i++) {
> +            value = palette[i * gamma_slots].green;
> +            if (gamma_green)
> +                value = gamma_green[value];
> +            else
> +                value <<= shift;
>  
> -        for (j = 0; j < gamma_slots; j++)
> -            crtc->gamma_green[i * gamma_slots + j] = value;
> +            crtc->gamma_green[i] = value;
> +        }
>      }
>  
> -    gamma_slots = crtc->gamma_size / palette_blue_size;
> -    for (i = 0; i < palette_blue_size; i++) {
> -        value = palette[i].blue;
> -        if (gamma_blue)
> -            value = gamma_blue[value];
> -        else
> -            value <<= shift;
> +    if (crtc->gamma_size >= palette_blue_size) {
> +        gamma_slots = crtc->gamma_size / palette_blue_size;
> +        for (i = 0; i < palette_blue_size; i++) {
> +            value = palette[i].blue;
> +            if (gamma_blue)
> +                value = gamma_blue[value];
> +            else
> +                value <<= shift;
>  
> -        for (j = 0; j < gamma_slots; j++)
> -            crtc->gamma_blue[i * gamma_slots + j] = value;
> +            for (j = 0; j < gamma_slots; j++)
> +                crtc->gamma_blue[i * gamma_slots + j] = value;
> +        }
> +    }
> +    else {
> +        gamma_slots = palette_blue_size / crtc->gamma_size;
> +        for (i = 0; i < crtc->gamma_size; i++) {
> +            value = palette[i * gamma_slots].blue;
> +            if (gamma_blue)
> +                value = gamma_blue[value];
> +            else
> +                value <<= shift;
> +
> +            crtc->gamma_blue[i] = value;
> +        }
>      }
>  }
>