[Mesa-dev] [PATCH 5/8] gallium/vl: Simplify usage of full range matrices

[Thomas Hellstrom]

When looking at the full range matrices, it becomes obvious that the difference
between the standard matrices and the full range matrices is that the full
range matrices are multiplied by 1.164. Together with offsetting the y value
with -16/255, this will scale and offset RGB with the desired quantities.

However, the standard SMPTE 240M matrix seems to differ a bit since the
U and V coefficients are only multiplied with 1.138 to get the full range
matrix. This would actually alter the color somewhat so I figure that's an
error. The full range matrix is consistent with Nvidia's VDPAU implementation.

We can also incorporate the ybias in the brightness simplifying the
calculation somewhat.

Signed-off-by: Thomas Hellstrom <thellstrom at vmware.com>
Reviewed-by: Sinclair Yeh <syeh at vmware.com>
---
 src/gallium/auxiliary/vl/vl_csc.c | 55 ++++++++++++---------------------------
 1 file changed, 17 insertions(+), 38 deletions(-)

diff --git a/src/gallium/auxiliary/vl/vl_csc.c b/src/gallium/auxiliary/vl/vl_csc.c
index 1587e6c..d70ab14 100644
--- a/src/gallium/auxiliary/vl/vl_csc.c
+++ b/src/gallium/auxiliary/vl/vl_csc.c
@@ -108,18 +108,6 @@ static const vl_csc_matrix bt_601 =
 };
 
 /*
- * Converts ITU-R BT.601 YCbCr pixels to RGB pixels where:
- * Y is in [16,235], Cb and Cr are in [16,240]
- * R, G, and B are in [0,255]
- */
-static const vl_csc_matrix bt_601_full =
-{
-   { 1.164f,  0.0f,    1.596f, 0.0f, },
-   { 1.164f, -0.391f, -0.813f, 0.0f, },
-   { 1.164f,  2.018f,  0.0f,   0.0f, }
-};
-
-/*
  * Converts ITU-R BT.709 YCbCr pixels to RGB pixels where:
  * Y is in [16,235], Cb and Cr are in [16,240]
  * R, G, and B are in [16,235]
@@ -132,29 +120,15 @@ static const vl_csc_matrix bt_709 =
 };
 
 /*
- * Converts ITU-R BT.709 YCbCr pixels to RGB pixels where:
+ * Converts SMPTE 240M YCbCr pixels to RGB pixels where:
  * Y is in [16,235], Cb and Cr are in [16,240]
- * R, G, and B are in [0,255]
+ * R, G, and B are in [16,235]
  */
-static const vl_csc_matrix bt_709_full =
-{
-   { 1.164f,  0.0f,    1.793f, 0.0f, },
-   { 1.164f, -0.213f, -0.534f, 0.0f, },
-   { 1.164f,  2.115f,  0.0f,   0.0f, }
-};
-
 static const vl_csc_matrix smpte240m =
 {
-   { 1.0f,  0.0f,    1.582f, 0.0f, },
-   { 1.0f, -0.228f, -0.478f, 0.0f, },
-   { 1.0f,  1.833f,  0.0f,   0.0f, }
-};
-
-static const vl_csc_matrix smpte240m_full =
-{
-   { 1.164f,  0.0f,    1.794f, 0.0f, },
-   { 1.164f, -0.258f, -0.543f, 0.0f, },
-   { 1.164f,  2.079f,  0.0f,   0.0f, }
+   { 1.0f,  0.0f,    1.541f, 0.0f, },
+   { 1.0f, -0.221f, -0.466f, 0.0f, },
+   { 1.0f,  1.785f,  0.0f,   0.0f, }
 };
 
 static const vl_csc_matrix identity =
@@ -176,7 +150,6 @@ void vl_csc_get_matrix(enum VL_CSC_COLOR_STANDARD cs,
                        bool full_range,
                        vl_csc_matrix *matrix)
 {
-   float ybias = full_range ? -16.0f/255.0f : 0.0f;
    float cbbias = -128.0f/255.0f;
    float crbias = -128.0f/255.0f;
 
@@ -188,17 +161,23 @@ void vl_csc_get_matrix(enum VL_CSC_COLOR_STANDARD cs,
 
    const vl_csc_matrix *cstd;
 
+   if (full_range) {
+      c *= 1.164f;              /* Adjust for the y range */
+      b *= 1.164f;              /* Adjust for the y range */
+      b -= c * 16.0f  / 255.0f; /* Adjust for the y bias */
+   }
+
    assert(matrix);
 
    switch (cs) {
       case VL_CSC_COLOR_STANDARD_BT_601:
-         cstd = full_range ? &bt_601_full : &bt_601;
+         cstd = &bt_601;
          break;
       case VL_CSC_COLOR_STANDARD_BT_709:
-         cstd = full_range ? &bt_709_full : &bt_709;
+         cstd = &bt_709;
          break;
       case VL_CSC_COLOR_STANDARD_SMPTE_240M:
-         cstd = full_range ? &smpte240m_full : &smpte240m;
+         cstd = &smpte240m;
          break;
       case VL_CSC_COLOR_STANDARD_IDENTITY:
       default:
@@ -210,21 +189,21 @@ void vl_csc_get_matrix(enum VL_CSC_COLOR_STANDARD cs,
    (*matrix)[0][0] = c * (*cstd)[0][0];
    (*matrix)[0][1] = c * (*cstd)[0][1] * s * cosf(h) - c * (*cstd)[0][2] * s * sinf(h);
    (*matrix)[0][2] = c * (*cstd)[0][2] * s * cosf(h) + c * (*cstd)[0][1] * s * sinf(h);
-   (*matrix)[0][3] = (*cstd)[0][3] + (*cstd)[0][0] * (b + c * ybias) +
+   (*matrix)[0][3] = (*cstd)[0][3] + (*cstd)[0][0] * b +
                      (*cstd)[0][1] * (c * cbbias * s * cosf(h) + c * crbias * s * sinf(h)) +
                      (*cstd)[0][2] * (c * crbias * s * cosf(h) - c * cbbias * s * sinf(h));
 
    (*matrix)[1][0] = c * (*cstd)[1][0];
    (*matrix)[1][1] = c * (*cstd)[1][1] * s * cosf(h) - c * (*cstd)[1][2] * s * sinf(h);
    (*matrix)[1][2] = c * (*cstd)[1][2] * s * cosf(h) + c * (*cstd)[1][1] * s * sinf(h);
-   (*matrix)[1][3] = (*cstd)[1][3] + (*cstd)[1][0] * (b + c * ybias) +
+   (*matrix)[1][3] = (*cstd)[1][3] + (*cstd)[1][0] * b +
                      (*cstd)[1][1] * (c * cbbias * s * cosf(h) + c * crbias * s * sinf(h)) +
                      (*cstd)[1][2] * (c * crbias * s * cosf(h) - c * cbbias * s * sinf(h));
 
    (*matrix)[2][0] = c * (*cstd)[2][0];
    (*matrix)[2][1] = c * (*cstd)[2][1] * s * cosf(h) - c * (*cstd)[2][2] * s * sinf(h);
    (*matrix)[2][2] = c * (*cstd)[2][2] * s * cosf(h) + c * (*cstd)[2][1] * s * sinf(h);
-   (*matrix)[2][3] = (*cstd)[2][3] + (*cstd)[2][0] * (b + c * ybias) +
+   (*matrix)[2][3] = (*cstd)[2][3] + (*cstd)[2][0] * b +
                      (*cstd)[2][1] * (c * cbbias * s * cosf(h) + c * crbias * s * sinf(h)) +
                      (*cstd)[2][2] * (c * crbias * s * cosf(h) - c * cbbias * s * sinf(h));
 }
-- 
2.5.0