[pulseaudio-discuss] [PATCH v4 20/23] echo-cancel: Use webrtc's deinterleaved API

arun at accosted.net arun at accosted.net
Wed Feb 17 14:17:12 UTC 2016


From: Arun Raghavan <git at arunraghavan.net>

This is required to have unequal channel counts on capture in and out
streams, which is needed for beamforming to work. The deinterleaved API
only works with floating point samples.
---
 src/modules/echo-cancel/echo-cancel.h        |  1 +
 src/modules/echo-cancel/module-echo-cancel.c |  4 +-
 src/modules/echo-cancel/webrtc.cc            | 56 ++++++++++++++++------------
 3 files changed, 36 insertions(+), 25 deletions(-)

diff --git a/src/modules/echo-cancel/echo-cancel.h b/src/modules/echo-cancel/echo-cancel.h
index ab0c5e9..83f8b85 100644
--- a/src/modules/echo-cancel/echo-cancel.h
+++ b/src/modules/echo-cancel/echo-cancel.h
@@ -66,6 +66,7 @@ struct pa_echo_canceller_params {
             void *apm;
             unsigned int blocksize; /* in frames */
             pa_sample_spec rec_ss, play_ss, out_ss;
+            float *rec_buffer[PA_CHANNELS_MAX], *play_buffer[PA_CHANNELS_MAX]; /* for deinterleaved buffers */
             void *trace_callback;
             bool agc;
             bool first;
diff --git a/src/modules/echo-cancel/module-echo-cancel.c b/src/modules/echo-cancel/module-echo-cancel.c
index 484281f..4e898d0 100644
--- a/src/modules/echo-cancel/module-echo-cancel.c
+++ b/src/modules/echo-cancel/module-echo-cancel.c
@@ -2158,12 +2158,12 @@ int main(int argc, char* argv[]) {
         goto fail;
     }
 
-    source_ss.format = PA_SAMPLE_S16LE;
+    source_ss.format = PA_SAMPLE_FLOAT32LE;
     source_ss.rate = DEFAULT_RATE;
     source_ss.channels = DEFAULT_CHANNELS;
     pa_channel_map_init_auto(&source_map, source_ss.channels, PA_CHANNEL_MAP_DEFAULT);
 
-    sink_ss.format = PA_SAMPLE_S16LE;
+    sink_ss.format = PA_SAMPLE_FLOAT32LE;
     sink_ss.rate = DEFAULT_RATE;
     sink_ss.channels = DEFAULT_CHANNELS;
     pa_channel_map_init_auto(&sink_map, sink_ss.channels, PA_CHANNEL_MAP_DEFAULT);
diff --git a/src/modules/echo-cancel/webrtc.cc b/src/modules/echo-cancel/webrtc.cc
index 35ba93f..385c211 100644
--- a/src/modules/echo-cancel/webrtc.cc
+++ b/src/modules/echo-cancel/webrtc.cc
@@ -117,8 +117,8 @@ static void pa_webrtc_ec_fixate_spec(pa_sample_spec *rec_ss, pa_channel_map *rec
                                      pa_sample_spec *play_ss, pa_channel_map *play_map,
                                      pa_sample_spec *out_ss, pa_channel_map *out_map)
 {
-    rec_ss->format = PA_SAMPLE_S16NE;
-    play_ss->format = PA_SAMPLE_S16NE;
+    rec_ss->format = PA_SAMPLE_FLOAT32NE;
+    play_ss->format = PA_SAMPLE_FLOAT32NE;
 
     /* AudioProcessing expects one of the following rates */
     if (rec_ss->rate >= 48000)
@@ -147,7 +147,7 @@ bool pa_webrtc_ec_init(pa_core *c, pa_echo_canceller *ec,
     webrtc::ProcessingConfig pconfig;
     webrtc::Config config;
     bool hpf, ns, agc, dgc, mobile, cn, vad, ext_filter, intelligibility, experimental_agc;
-    int rm = -1;
+    int rm = -1, i;
     uint32_t agc_start_volume;
     pa_modargs *ma;
     bool trace = false;
@@ -340,6 +340,11 @@ bool pa_webrtc_ec_init(pa_core *c, pa_echo_canceller *ec,
     *nframes = ec->params.webrtc.blocksize;
     ec->params.webrtc.first = true;
 
+    for (i = 0; i < rec_ss->channels; i++)
+        ec->params.webrtc.rec_buffer[i] = pa_xnew(float, *nframes);
+    for (i = 0; i < play_ss->channels; i++)
+        ec->params.webrtc.play_buffer[i] = pa_xnew(float, *nframes);
+
     pa_modargs_free(ma);
     return true;
 
@@ -357,36 +362,34 @@ fail:
 
 void pa_webrtc_ec_play(pa_echo_canceller *ec, const uint8_t *play) {
     webrtc::AudioProcessing *apm = (webrtc::AudioProcessing*)ec->params.webrtc.apm;
-    webrtc::AudioFrame play_frame;
     const pa_sample_spec *ss = &ec->params.webrtc.play_ss;
+    int n = ec->params.webrtc.blocksize;
+    float **buf = ec->params.webrtc.play_buffer;
+    webrtc::StreamConfig config(ss->rate, ss->channels, false);
 
-    play_frame.num_channels_ = ss->channels;
-    play_frame.sample_rate_hz_ = ss->rate;
-    play_frame.interleaved_ = true;
-    play_frame.samples_per_channel_ = ec->params.webrtc.blocksize;
-
-    pa_assert(play_frame.samples_per_channel_ <= webrtc::AudioFrame::kMaxDataSizeSamples);
-    memcpy(play_frame.data_, play, ec->params.webrtc.blocksize * pa_frame_size(ss));
+    pa_deinterleave(play, (void **) buf, ss->channels, pa_sample_size(ss), n);
 
-    if (apm->ProcessReverseStream(&play_frame) != webrtc::AudioProcessing::kNoError)
+    if (apm->ProcessReverseStream(buf, config, config, buf) != webrtc::AudioProcessing::kNoError)
         pa_log("Failed to process playback stream");
+
+    /* FIXME: we need to be able to modify playback samples, which we can't
+     * currently do. This is because module-echo-cancel processes playback
+     * frames in the source thread, and just stores playback chunks as they
+     * pass through the sink. */
 }
 
 void pa_webrtc_ec_record(pa_echo_canceller *ec, const uint8_t *rec, uint8_t *out) {
     webrtc::AudioProcessing *apm = (webrtc::AudioProcessing*)ec->params.webrtc.apm;
-    webrtc::AudioFrame out_frame;
     const pa_sample_spec *rec_ss = &ec->params.webrtc.rec_ss;
     const pa_sample_spec *out_ss = &ec->params.webrtc.out_ss;
+    float **buf = ec->params.webrtc.rec_buffer;
+    int n = ec->params.webrtc.blocksize;
     pa_cvolume v;
     int old_volume, new_volume;
+    webrtc::StreamConfig rec_config(rec_ss->rate, rec_ss->channels, false);
+    webrtc::StreamConfig out_config(out_ss->rate, out_ss->channels, false);
 
-    out_frame.num_channels_ = rec_ss->channels;
-    out_frame.sample_rate_hz_ = rec_ss->rate;
-    out_frame.interleaved_ = true;
-    out_frame.samples_per_channel_ = ec->params.webrtc.blocksize;
-
-    pa_assert(out_frame.samples_per_channel_ <= webrtc::AudioFrame::kMaxDataSizeSamples);
-    memcpy(out_frame.data_, rec, ec->params.webrtc.blocksize * pa_frame_size(rec_ss));
+    pa_deinterleave(rec, (void **) buf, rec_ss->channels, pa_sample_size(rec_ss), n);
 
     if (ec->params.webrtc.agc) {
         pa_cvolume_init(&v);
@@ -396,7 +399,7 @@ void pa_webrtc_ec_record(pa_echo_canceller *ec, const uint8_t *rec, uint8_t *out
     }
 
     apm->set_stream_delay_ms(0);
-    pa_assert_se(apm->ProcessStream(&out_frame) == webrtc::AudioProcessing::kNoError);
+    pa_assert_se(apm->ProcessStream(buf, rec_config, out_config, buf) == webrtc::AudioProcessing::kNoError);
 
     if (ec->params.webrtc.agc) {
         if (PA_UNLIKELY(ec->params.webrtc.first)) {
@@ -411,12 +414,12 @@ void pa_webrtc_ec_record(pa_echo_canceller *ec, const uint8_t *rec, uint8_t *out
         }
 
         if (old_volume != new_volume) {
-            pa_cvolume_set(&v, rec_ss->channels, webrtc_volume_to_pa(new_volume));
+            pa_cvolume_set(&v, out_ss->channels, webrtc_volume_to_pa(new_volume));
             pa_echo_canceller_set_capture_volume(ec, &v);
         }
     }
 
-    memcpy(out, out_frame.data_, ec->params.webrtc.blocksize * pa_frame_size(out_ss));
+    pa_interleave((const void **) buf, out_ss->channels, out, pa_sample_size(out_ss), n);
 }
 
 void pa_webrtc_ec_set_drift(pa_echo_canceller *ec, float drift) {
@@ -431,6 +434,8 @@ void pa_webrtc_ec_run(pa_echo_canceller *ec, const uint8_t *rec, const uint8_t *
 }
 
 void pa_webrtc_ec_done(pa_echo_canceller *ec) {
+    int i;
+
     if (ec->params.webrtc.trace_callback) {
         webrtc::Trace::ReturnTrace();
         delete ((PaWebrtcTraceCallback *) ec->params.webrtc.trace_callback);
@@ -440,4 +445,9 @@ void pa_webrtc_ec_done(pa_echo_canceller *ec) {
         delete (webrtc::AudioProcessing*)ec->params.webrtc.apm;
         ec->params.webrtc.apm = NULL;
     }
+
+    for (i = 0; i < ec->params.webrtc.rec_ss.channels; i++)
+        pa_xfree(ec->params.webrtc.rec_buffer[i]);
+    for (i = 0; i < ec->params.webrtc.play_ss.channels; i++)
+        pa_xfree(ec->params.webrtc.play_buffer[i]);
 }
-- 
2.5.0



More information about the pulseaudio-discuss mailing list