[Spice-devel] [PATCH] Add support for LZ compression of A8 images

[Søren Sandmann]

From: Søren Sandmann Pedersen <ssp at redhat.com>

Graduality is irrelevant for A8 images, so instead of using RGB-ness
as a short-cut, add a new macro BITMAP_FMT_HAS_GRADUALITY() that
returns true for the existing RGB images, but false for A8.
---
 server/red_worker.c |   25 ++++++++++++++++---------
 spice-common        |    2 +-
 2 files changed, 17 insertions(+), 10 deletions(-)

diff --git a/server/red_worker.c b/server/red_worker.c
index c712180..154cacc 100644
--- a/server/red_worker.c
+++ b/server/red_worker.c
@@ -430,9 +430,13 @@ struct RedCompressBuf {
     RedCompressBuf *send_next;
 };
 
-static const int BITMAP_FMT_IS_PLT[] = {0, 1, 1, 1, 1, 1, 0, 0, 0, 0};
-static const int BITMAP_FMT_IS_RGB[] = {0, 0, 0, 0, 0, 0, 1, 1, 1, 1};
-static const int BITMAP_FMP_BYTES_PER_PIXEL[] = {0, 0, 0, 0, 0, 1, 2, 3, 4, 4};
+static const int BITMAP_FMT_IS_PLT[] = {0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0};
+static const int BITMAP_FMT_IS_RGB[] = {0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1};
+static const int BITMAP_FMP_BYTES_PER_PIXEL[] = {0, 0, 0, 0, 0, 1, 2, 3, 4, 4, 1};
+
+#define BITMAP_FMT_HAS_GRADUALITY(f)                                    \
+    (BITMAP_FMT_IS_RGB[(f)]        &&                                   \
+     ((f) != SPICE_BITMAP_FMT_8BIT_A))
 
 pthread_mutex_t cache_lock = PTHREAD_MUTEX_INITIALIZER;
 Ring pixmap_cache_list = {&pixmap_cache_list, &pixmap_cache_list};
@@ -3073,7 +3077,7 @@ static inline void red_update_copy_graduality(RedWorker* worker, Drawable *drawa
 
     bitmap = &drawable->red_drawable->u.copy.src_bitmap->u.bitmap;
 
-    if (!BITMAP_FMT_IS_RGB[bitmap->format] || _stride_is_extra(bitmap) ||
+    if (!BITMAP_FMT_HAS_GRADUALITY(bitmap->format) || _stride_is_extra(bitmap) ||
         (bitmap->data->flags & SPICE_CHUNKS_FLAGS_UNSTABLE)) {
         drawable->copy_bitmap_graduality = BITMAP_GRADUAL_NOT_AVAIL;
     } else  {
@@ -3633,6 +3637,8 @@ static int surface_format_to_image_type(uint32_t surface_format)
         return SPICE_BITMAP_FMT_32BIT;
     case SPICE_SURFACE_FMT_32_ARGB:
         return SPICE_BITMAP_FMT_RGBA;
+    case SPICE_SURFACE_FMT_8_A:
+        return SPICE_BITMAP_FMT_8BIT_A;
     default:
         spice_critical("Unsupported surface format");
     }
@@ -5845,7 +5851,8 @@ static const LzImageType MAP_BITMAP_FMT_TO_LZ_IMAGE_TYPE[] = {
     LZ_IMAGE_TYPE_RGB16,
     LZ_IMAGE_TYPE_RGB24,
     LZ_IMAGE_TYPE_RGB32,
-    LZ_IMAGE_TYPE_RGBA
+    LZ_IMAGE_TYPE_RGBA,
+    LZ_IMAGE_TYPE_A8
 };
 
 typedef struct compress_send_data_t {
@@ -6303,8 +6310,8 @@ static inline int red_compress_image(DisplayChannelClient *dcc,
                     quic_compress = FALSE;
                 } else {
                     if (drawable->copy_bitmap_graduality == BITMAP_GRADUAL_INVALID) {
-                        quic_compress = BITMAP_FMT_IS_RGB[src->format] &&
-                            (_get_bitmap_graduality_level(display_channel->common.worker, src,
+                        quic_compress = BITMAP_FMT_HAS_GRADUALITY(src->format) &&
+			    (_get_bitmap_graduality_level(display_channel->common.worker, src,
                                                           drawable->group_id) ==
                              BITMAP_GRADUAL_HIGH);
                     } else {
@@ -6338,7 +6345,7 @@ static inline int red_compress_image(DisplayChannelClient *dcc,
         int ret;
         if ((image_compression == SPICE_IMAGE_COMPRESS_AUTO_GLZ) ||
             (image_compression == SPICE_IMAGE_COMPRESS_GLZ)) {
-            glz = BITMAP_FMT_IS_RGB[src->format] && (
+            glz = BITMAP_FMT_HAS_GRADUALITY(src->format) && (
                     (src->x * src->y) < glz_enc_dictionary_get_size(
                         dcc->glz_dict->dict));
         } else if ((image_compression == SPICE_IMAGE_COMPRESS_AUTO_LZ) ||
@@ -8411,7 +8418,7 @@ static void red_marshall_image(RedChannelClient *rcc, SpiceMarshaller *m, ImageI
 
     if ((comp_mode == SPICE_IMAGE_COMPRESS_AUTO_LZ) ||
         (comp_mode == SPICE_IMAGE_COMPRESS_AUTO_GLZ)) {
-        if (BITMAP_FMT_IS_RGB[item->image_format]) {
+        if (BITMAP_FMT_HAS_GRADUALITY(item->image_format)) {
             if (!_stride_is_extra(&bitmap)) {
                 BitmapGradualType grad_level;
                 grad_level = _get_bitmap_graduality_level(display_channel->common.worker,
diff --git a/spice-common b/spice-common
index 22fc0b0..288a7c2 160000
--- a/spice-common
+++ b/spice-common
@@ -1 +1 @@
-Subproject commit 22fc0b0145876b90385c1c88923bcd72a6380812
+Subproject commit 288a7c2f482439499c977bce418262df4661f2f7
-- 
1.7.4