[Cogl] [PATCH 4/4] cogl-gst: video-sink: add video balance support
Lionel Landwerlin
llandwerlin at gmail.com
Fri Jan 10 10:14:23 PST 2014
---
cogl-gst/cogl-gst-video-sink.c | 426 ++++++++++++++++++++++++++++++++++++++---
1 file changed, 401 insertions(+), 25 deletions(-)
diff --git a/cogl-gst/cogl-gst-video-sink.c b/cogl-gst/cogl-gst-video-sink.c
index 3742c6e..7168cf6 100644
--- a/cogl-gst/cogl-gst-video-sink.c
+++ b/cogl-gst/cogl-gst-video-sink.c
@@ -38,6 +38,7 @@
#include <gst/video/video.h>
#include <gst/riff/riff-ids.h>
#include <string.h>
+#include <math.h>
/* We just need the public Cogl api for cogl-gst but we first need to
* undef COGL_COMPILATION to avoid getting an error that normally
@@ -77,7 +78,12 @@ static GstStaticPadTemplate sinktemplate_all =
GST_PAD_ALWAYS,
GST_STATIC_CAPS (SINK_CAPS));
-G_DEFINE_TYPE (CoglGstVideoSink, cogl_gst_video_sink, GST_TYPE_BASE_SINK);
+static void color_balance_iface_init (GstColorBalanceInterface *iface);
+
+G_DEFINE_TYPE_WITH_CODE (CoglGstVideoSink,
+ cogl_gst_video_sink,
+ GST_TYPE_BASE_SINK,
+ G_IMPLEMENT_INTERFACE (GST_TYPE_COLOR_BALANCE, color_balance_iface_init))
enum
{
@@ -159,17 +165,31 @@ struct _CoglGstVideoSinkPrivate
CoglPipeline *pipeline;
CoglTexture *frame[3];
CoglBool frame_dirty;
+ CoglBool had_upload_once;
+
CoglGstVideoFormat format;
CoglBool bgr;
+
CoglGstSource *source;
GSList *renderers;
GstCaps *caps;
CoglGstRenderer *renderer;
GstFlowReturn flow_return;
int custom_start;
+ int video_start;
int free_layer;
CoglBool default_sample;
GstVideoInfo info;
+
+ gdouble brightness;
+ gdouble contrast;
+ gdouble hue;
+ gdouble saturation;
+ CoglBool correction_dirty;
+
+ guint8 *tabley;
+ guint8 *tableu;
+ guint8 *tablev;
};
/* Snippet cache */
@@ -185,7 +205,7 @@ get_layer_cache_entry (CoglGstVideoSink *sink,
{
SnippetCacheEntry *entry = l->data;
- if (entry->start_position == priv->custom_start)
+ if (entry->start_position == priv->video_start)
return entry;
}
@@ -201,7 +221,7 @@ add_layer_cache_entry (CoglGstVideoSink *sink,
SnippetCacheEntry *entry = g_slice_new (SnippetCacheEntry);
char *default_source;
- entry->start_position = priv->custom_start;
+ entry->start_position = priv->video_start;
entry->vertex_snippet =
cogl_snippet_new (COGL_SNIPPET_HOOK_VERTEX_GLOBALS,
@@ -215,8 +235,8 @@ add_layer_cache_entry (CoglGstVideoSink *sink,
default_source =
g_strdup_printf (" cogl_layer *= cogl_gst_sample_video%i "
"(cogl_tex_coord%i_in.st);\n",
- priv->custom_start,
- priv->custom_start);
+ priv->video_start,
+ priv->video_start);
entry->default_sample_snippet =
cogl_snippet_new (COGL_SNIPPET_HOOK_LAYER_FRAGMENT,
NULL, /* declarations */
@@ -290,14 +310,14 @@ setup_pipeline_from_cache_entry (CoglGstVideoSink *sink,
* the intermediate textures */
for (i = 0; i < n_layers; i++) {
cogl_pipeline_set_layer_combine (pipeline,
- priv->custom_start + i,
+ priv->video_start + i,
"RGBA=REPLACE(PREVIOUS)",
NULL);
}
if (priv->default_sample) {
cogl_pipeline_add_layer_snippet (pipeline,
- priv->custom_start + n_layers - 1,
+ priv->video_start + n_layers - 1,
cache_entry->default_sample_snippet);
}
}
@@ -305,6 +325,191 @@ setup_pipeline_from_cache_entry (CoglGstVideoSink *sink,
priv->frame_dirty = TRUE;
}
+/**/
+
+#define DEFAULT_BRIGHTNESS (0.0f)
+#define DEFAULT_CONTRAST (1.0f)
+#define DEFAULT_HUE (0.0f)
+#define DEFAULT_SATURATION (1.0f)
+
+static void
+cogl_gst_video_sink_color_balance_update_tables (CoglGstVideoSink *sink)
+{
+ CoglGstVideoSinkPrivate *priv = sink->priv;
+ gint i, j;
+ gdouble y, u, v, hue_cos, hue_sin;
+
+ /* Y */
+ for (i = 0; i < 256; i++) {
+ y = 16 + ((i - 16) * priv->contrast + priv->brightness * 255);
+ if (y < 0)
+ y = 0;
+ else if (y > 255)
+ y = 255;
+ priv->tabley[i] = rint (y);
+ }
+
+ hue_cos = cos (G_PI * priv->hue);
+ hue_sin = sin (G_PI * priv->hue);
+
+ /* U/V lookup tables are 2D, since we need both U/V for each table
+ * separately. */
+ for (i = -128; i < 128; i++) {
+ for (j = -128; j < 128; j++) {
+ u = 128 + ((i * hue_cos + j * hue_sin) * priv->saturation);
+ v = 128 + ((-i * hue_sin + j * hue_cos) * priv->saturation);
+ if (u < 0)
+ u = 0;
+ else if (u > 255)
+ u = 255;
+ if (v < 0)
+ v = 0;
+ else if (v > 255)
+ v = 255;
+ priv->tableu[(i + 128) * 256 + j + 128] = rint (u);
+ priv->tablev[(i + 128) * 256 + j + 128] = rint (v);
+ }
+ }
+}
+
+static const GList *
+cogl_gst_video_sink_color_balance_list_channels (GstColorBalance *balance)
+{
+ static GList *channels = NULL;
+
+ if (channels == NULL) {
+ const gchar *str_channels[4] = { "HUE", "SATURATION",
+ "BRIGHTNESS", "CONTRAST"
+ };
+ gint i;
+
+ for (i = 0; i < G_N_ELEMENTS (str_channels); i++) {
+ GstColorBalanceChannel *channel;
+
+ channel = g_object_new (GST_TYPE_COLOR_BALANCE_CHANNEL, NULL);
+ channel->label = g_strdup (str_channels[i]);
+ channel->min_value = -1000;
+ channel->max_value = 1000;
+ channels = g_list_append (channels, channel);
+ }
+ }
+
+ return channels;
+}
+
+static gboolean
+cogl_gst_video_sink_get_variable (CoglGstVideoSink *sink,
+ const gchar *variable,
+ gdouble *minp,
+ gdouble *maxp,
+ gdouble **valuep)
+{
+ CoglGstVideoSinkPrivate *priv = sink->priv;
+ gdouble min, max, *value;
+
+ if (!g_strcmp0 (variable, "BRIGHTNESS"))
+ {
+ min = -1.0;
+ max = 1.0;
+ value = &priv->brightness;
+ }
+ else if (!g_strcmp0 (variable, "CONTRAST"))
+ {
+ min = 0.0;
+ max = 2.0;
+ value = &priv->contrast;
+ }
+ else if (!g_strcmp0 (variable, "HUE"))
+ {
+ min = -1.0;
+ max = 1.0;
+ value = &priv->hue;
+ }
+ else if (!g_strcmp0 (variable, "SATURATION"))
+ {
+ min = 0.0;
+ max = 2.0;
+ value = &priv->saturation;
+ }
+ else
+ {
+ GST_WARNING_OBJECT (sink, "color balance parameter not supported %s",
+ variable);
+ return FALSE;
+ }
+
+ if (maxp)
+ *maxp = max;
+ if (minp)
+ *minp = min;
+ if (valuep)
+ *valuep = value;
+
+ return TRUE;
+}
+
+static void
+cogl_gst_video_sink_color_balance_set_value (GstColorBalance *balance,
+ GstColorBalanceChannel *channel,
+ gint value)
+{
+ CoglGstVideoSink *sink = COGL_GST_VIDEO_SINK (balance);
+ gdouble *old_value, new_value, min, max;
+
+ if (!cogl_gst_video_sink_get_variable (sink, channel->label,
+ &min, &max, &old_value))
+ return;
+
+ new_value = (max - min) * ((gdouble) (value - channel->min_value) /
+ (gdouble) (channel->max_value - channel->min_value))
+ + min;
+
+ if (new_value != *old_value)
+ {
+ *old_value = new_value;
+ sink->priv->correction_dirty = TRUE;
+
+ gst_color_balance_value_changed (GST_COLOR_BALANCE (balance), channel,
+ gst_color_balance_get_value (GST_COLOR_BALANCE (balance), channel));
+ }
+}
+
+static gint
+cogl_gst_video_sink_color_balance_get_value (GstColorBalance *balance,
+ GstColorBalanceChannel *channel)
+{
+ CoglGstVideoSink *sink = COGL_GST_VIDEO_SINK (balance);
+ gdouble *old_value, min, max;
+ gint value;
+
+ if (!cogl_gst_video_sink_get_variable (sink, channel->label,
+ &min, &max, &old_value))
+ return 0;
+
+ value = (gint) (((*old_value + min) / (max - min)) *
+ (channel->max_value - channel->min_value))
+ + channel->min_value;
+
+ return value;
+}
+
+static GstColorBalanceType
+cogl_gst_video_sink_color_balance_get_balance_type (GstColorBalance *balance)
+{
+ return GST_COLOR_BALANCE_HARDWARE;
+}
+
+static void
+color_balance_iface_init (GstColorBalanceInterface *iface)
+{
+ iface->list_channels = cogl_gst_video_sink_color_balance_list_channels;
+ iface->set_value = cogl_gst_video_sink_color_balance_set_value;
+ iface->get_value = cogl_gst_video_sink_color_balance_get_value;
+
+ iface->get_balance_type = cogl_gst_video_sink_color_balance_get_balance_type;
+}
+
+
static void
cogl_gst_source_finalize (GSource *source)
{
@@ -333,10 +538,134 @@ cogl_gst_video_sink_attach_frame (CoglGstVideoSink *sink,
for (i = 0; i < G_N_ELEMENTS (priv->frame); i++)
if (priv->frame[i] != NULL)
- cogl_pipeline_set_layer_texture (pln, i + priv->custom_start,
+ cogl_pipeline_set_layer_texture (pln, i + priv->video_start,
priv->frame[i]);
}
+/* Color correction */
+
+static const gchar *no_color_correction_shader =
+ "#define cogl_gst_get_corrected_color_from_yuv(arg) (arg)\n"
+ "#define cogl_gst_get_corrected_color_from_rgb(arg) (arg)\n";
+
+static const gchar *color_correction_shader =
+ "vec3\n"
+ "cogl_gst_get_corrected_color_from_yuv (vec3 yuv)\n"
+ "{\n"
+ " vec2 ruv = vec2 (yuv[2] + 0.5, yuv[1] + 0.5);\n"
+ " return vec3 (texture2D (cogl_sampler%i, vec2 (yuv[0], 0)).a,\n"
+ " texture2D (cogl_sampler%i, ruv).a - 0.5,\n"
+ " texture2D (cogl_sampler%i, ruv).a - 0.5);\n"
+ "}\n"
+ "\n"
+ "vec3\n"
+ "cogl_gst_get_corrected_color_from_rgb (vec3 rgb)\n"
+ "{\n"
+ " vec3 yuv = cogl_gst_yuv_srgb_to_bt601 (rgb);\n"
+ " vec3 corrected_yuv = vec3 (texture2D (cogl_sampler%i, vec2 (yuv[0], 0)).a,\n"
+ " texture2D (cogl_sampler%i, vec2 (yuv[2], yuv[1])).a,\n"
+ " texture2D (cogl_sampler%i, vec2 (yuv[2], yuv[1])).a);\n"
+ " return cogl_gst_yuv_bt601_to_srgb (corrected_yuv);\n"
+ "}\n";
+
+static void
+cogl_gst_video_sink_setup_correction (CoglGstVideoSink *sink,
+ CoglPipeline *pipeline)
+{
+ CoglGstVideoSinkPrivate *priv = sink->priv;
+ static CoglSnippet *color_correction_snippet_vert = NULL,
+ *color_correction_snippet_frag = NULL,
+ *no_color_correction_snippet_vert = NULL,
+ *no_color_correction_snippet_frag = NULL;
+
+ /* GST_INFO_OBJECT */ g_message (/* sink, */ "attaching correction b=%.3f/c=%.3f/h=%.3f/s=%.3f",
+ priv->brightness, priv->contrast,
+ priv->hue, priv->saturation);
+
+
+ if (priv->brightness != DEFAULT_BRIGHTNESS ||
+ priv->contrast != DEFAULT_CONTRAST ||
+ priv->hue != DEFAULT_HUE ||
+ priv->saturation != DEFAULT_SATURATION)
+ {
+ int i;
+ guint8 *tables[3] = { priv->tabley, priv->tableu, priv->tablev };
+ gint tables_sizes[3][2] = { { 256, 1 },
+ { 256, 256 },
+ { 256, 256 } };
+
+ if (G_UNLIKELY (color_correction_snippet_vert == NULL))
+ {
+ gchar *shader = g_strdup_printf (color_correction_shader,
+ priv->custom_start,
+ priv->custom_start + 1,
+ priv->custom_start + 2,
+ priv->custom_start,
+ priv->custom_start + 1,
+ priv->custom_start + 2);
+
+ color_correction_snippet_vert =
+ cogl_snippet_new (COGL_SNIPPET_HOOK_VERTEX_GLOBALS,
+ shader, NULL);
+ color_correction_snippet_frag =
+ cogl_snippet_new (COGL_SNIPPET_HOOK_FRAGMENT_GLOBALS,
+ shader, NULL);
+ }
+
+ cogl_pipeline_add_snippet (pipeline, color_correction_snippet_vert);
+ cogl_pipeline_add_snippet (pipeline, color_correction_snippet_frag);
+
+ cogl_gst_video_sink_color_balance_update_tables (sink);
+
+ for (i = 0; i < 3; i++)
+ {
+ CoglTexture *lut_texture =
+ cogl_texture_2d_new_from_data (priv->ctx,
+ tables_sizes[i][0],
+ tables_sizes[i][1],
+ COGL_PIXEL_FORMAT_A_8,
+ tables_sizes[i][0],
+ tables[i],
+ NULL);
+
+ cogl_pipeline_set_layer_filters (pipeline,
+ priv->custom_start + i,
+ COGL_PIPELINE_FILTER_NEAREST,
+ COGL_PIPELINE_FILTER_LINEAR);
+ cogl_pipeline_set_layer_combine (pipeline,
+ priv->custom_start + i,
+ "RGBA=REPLACE(PREVIOUS)",
+ NULL);
+ cogl_pipeline_set_layer_texture (pipeline,
+ priv->custom_start + i,
+ lut_texture);
+
+ cogl_object_unref (lut_texture);
+ }
+
+ priv->video_start = priv->custom_start + 3;
+ }
+ else
+ {
+ if (G_UNLIKELY (no_color_correction_snippet_vert == NULL))
+ {
+ no_color_correction_snippet_vert =
+ cogl_snippet_new (COGL_SNIPPET_HOOK_VERTEX_GLOBALS,
+ no_color_correction_shader,
+ NULL);
+ no_color_correction_snippet_frag =
+ cogl_snippet_new (COGL_SNIPPET_HOOK_FRAGMENT_GLOBALS,
+ no_color_correction_shader,
+ NULL);
+ }
+
+ cogl_pipeline_add_snippet (pipeline, no_color_correction_snippet_vert);
+ cogl_pipeline_add_snippet (pipeline, no_color_correction_snippet_frag);
+
+ priv->video_start = priv->custom_start;
+ }
+}
+
/* YUV <-> RGB conversions */
static const gchar *color_conversions_shaders =
@@ -459,7 +788,8 @@ cogl_gst_source_check (GSource *source)
{
CoglGstSource *gst_source = (CoglGstSource *) source;
- return gst_source->buffer != NULL;
+ return (gst_source->buffer != NULL ||
+ gst_source->sink->priv->correction_dirty);
}
static void
@@ -479,6 +809,7 @@ dirty_default_pipeline (CoglGstVideoSink *sink)
{
cogl_object_unref (priv->pipeline);
priv->pipeline = NULL;
+ priv->had_upload_once = FALSE;
}
}
@@ -521,6 +852,7 @@ cogl_gst_video_sink_setup_pipeline (CoglGstVideoSink *sink,
if (sink->priv->renderer)
{
cogl_gst_video_sink_setup_conversions (sink, pipeline);
+ cogl_gst_video_sink_setup_correction (sink, pipeline);
sink->priv->renderer->setup_pipeline (sink, pipeline);
}
}
@@ -539,7 +871,16 @@ cogl_gst_video_sink_get_pipeline (CoglGstVideoSink *vt)
priv->pipeline = cogl_pipeline_new (priv->ctx);
cogl_gst_video_sink_setup_pipeline (vt, priv->pipeline);
cogl_gst_video_sink_attach_frame (vt, priv->pipeline);
- priv->frame_dirty = FALSE;
+ priv->correction_dirty = FALSE;
+ }
+ else if (priv->correction_dirty)
+ {
+ cogl_object_unref (priv->pipeline);
+ priv->pipeline = cogl_pipeline_new (priv->ctx);
+
+ cogl_gst_video_sink_setup_pipeline (vt, priv->pipeline);
+ cogl_gst_video_sink_attach_frame (vt, priv->pipeline);
+ priv->correction_dirty = FALSE;
}
else if (priv->frame_dirty)
{
@@ -548,9 +889,10 @@ cogl_gst_video_sink_get_pipeline (CoglGstVideoSink *vt)
priv->pipeline = pipeline;
cogl_gst_video_sink_attach_frame (vt, pipeline);
- priv->frame_dirty = FALSE;
}
+ priv->frame_dirty = FALSE;
+
return priv->pipeline;
}
@@ -592,10 +934,12 @@ cogl_gst_rgb_setup_pipeline (CoglGstVideoSink *sink,
g_strdup_printf ("vec4\n"
"cogl_gst_sample_video%i (vec2 UV)\n"
"{\n"
- " return texture2D (cogl_sampler%i, UV);\n"
+ " vec4 color = texture2D (cogl_sampler%i, UV);\n"
+ " vec3 corrected = cogl_gst_get_corrected_color_from_rgb (color.rgb);\n"
+ " return vec4(corrected.rgb, color.a);\n"
"}\n",
- priv->custom_start,
- priv->custom_start);
+ priv->video_start,
+ priv->video_start);
entry = add_layer_cache_entry (sink, &snippet_cache, source);
g_free (source);
@@ -871,19 +1215,19 @@ cogl_gst_yv12_glsl_setup_pipeline (CoglGstVideoSink *sink,
g_strdup_printf ("vec4\n"
"cogl_gst_sample_video%i (vec2 UV)\n"
"{\n"
- " float y = 1.1640625 * "
- "(texture2D (cogl_sampler%i, UV).a - 0.0625);\n"
+ " float y = 1.1640625 * (texture2D (cogl_sampler%i, UV).a - 0.0625);\n"
" float u = texture2D (cogl_sampler%i, UV).a - 0.5;\n"
" float v = texture2D (cogl_sampler%i, UV).a - 0.5;\n"
+ " vec3 corrected = cogl_gst_get_corrected_color_from_yuv (vec3 (y, u, v));\n"
" vec4 color;\n"
" color.rgb = cogl_gst_default_yuv_to_srgb (corrected);\n"
" color.a = 1.0;\n"
" return color;\n"
"}\n",
- priv->custom_start,
- priv->custom_start,
- priv->custom_start + 1,
- priv->custom_start + 2);
+ priv->video_start,
+ priv->video_start,
+ priv->video_start + 1,
+ priv->video_start + 2);
entry = add_layer_cache_entry (sink, &snippet_cache, source);
g_free (source);
@@ -936,11 +1280,13 @@ cogl_gst_ayuv_glsl_setup_pipeline (CoglGstVideoSink *sink,
" float y = 1.1640625 * (color.g - 0.0625);\n"
" float u = color.b - 0.5;\n"
" float v = color.a - 0.5;\n"
+ " vec3 corrected = cogl_gst_get_corrected_color_from_yuv (vec3 (y, u, v));\n"
" color.a = color.r;\n"
" color.rgb = cogl_gst_default_yuv_to_srgb (corrected);\n"
" return color;\n"
- "}\n", priv->custom_start,
- priv->custom_start);
+ "}\n",
+ priv->video_start,
+ priv->video_start);
entry = add_layer_cache_entry (sink, &snippet_cache, source);
g_free (source);
@@ -1261,6 +1607,8 @@ cogl_gst_source_dispatch (GSource *source,
if (!priv->renderer->upload (gst_source->sink, buffer))
goto fail_upload;
+ priv->had_upload_once = TRUE;
+
gst_buffer_unref (buffer);
}
else
@@ -1270,9 +1618,10 @@ cogl_gst_source_dispatch (GSource *source,
g_signal_emit (gst_source->sink,
video_sink_signals[PIPELINE_READY_SIGNAL],
0 /* detail */);
- g_signal_emit (gst_source->sink,
- video_sink_signals[NEW_FRAME_SIGNAL], 0,
- NULL);
+ if (priv->had_upload_once)
+ g_signal_emit (gst_source->sink,
+ video_sink_signals[NEW_FRAME_SIGNAL], 0,
+ NULL);
return TRUE;
@@ -1333,6 +1682,15 @@ cogl_gst_video_sink_init (CoglGstVideoSink *sink)
CoglGstVideoSinkPrivate);
priv->custom_start = 0;
priv->default_sample = TRUE;
+
+ priv->brightness = DEFAULT_BRIGHTNESS;
+ priv->contrast = DEFAULT_CONTRAST;
+ priv->hue = DEFAULT_HUE;
+ priv->saturation = DEFAULT_SATURATION;
+
+ priv->tabley = g_new0 (guint8, 256);
+ priv->tableu = g_new0 (guint8, 256 * 256);
+ priv->tablev = g_new0 (guint8, 256 * 256);
}
static GstFlowReturn
@@ -1388,6 +1746,24 @@ cogl_gst_video_sink_dispose (GObject *object)
priv->caps = NULL;
}
+ if (priv->tabley)
+ {
+ g_free (priv->tabley);
+ priv->tabley = NULL;
+ }
+
+ if (priv->tableu)
+ {
+ g_free (priv->tableu);
+ priv->tableu = NULL;
+ }
+
+ if (priv->tablev)
+ {
+ g_free (priv->tablev);
+ priv->tablev = NULL;
+ }
+
G_OBJECT_CLASS (cogl_gst_video_sink_parent_class)->dispose (object);
}
--
1.8.5.2
More information about the Cogl
mailing list