[pulseaudio-discuss] [PATCH v3 22/24] echo-cancel: Use webrtc's deinterleaved API

arun at accosted.net arun at accosted.net
Sun Jan 17 23:36:36 PST 2016


From: Arun Raghavan <git at arunraghavan.net>

This is required to have unequal channel counts on capture in and out
streams, which is needed for beamforming to work.
---
 src/modules/echo-cancel/echo-cancel.h        |  1 +
 src/modules/echo-cancel/module-echo-cancel.c |  4 +--
 src/modules/echo-cancel/webrtc.cc            | 51 ++++++++++++++++------------
 3 files changed, 32 insertions(+), 24 deletions(-)

diff --git a/src/modules/echo-cancel/echo-cancel.h b/src/modules/echo-cancel/echo-cancel.h
index 613f7e3..881b373 100644
--- a/src/modules/echo-cancel/echo-cancel.h
+++ b/src/modules/echo-cancel/echo-cancel.h
@@ -66,6 +66,7 @@ struct pa_echo_canceller_params {
             void *apm;
             int32_t blocksize; /* in frames */
             pa_sample_spec rec_ss, play_ss, out_ss;
+            float *rec_buffer[PA_CHANNELS_MAX], *play_buffer[PA_CHANNELS_MAX]; /* for deinterleaved buffers */
             bool agc;
             bool trace;
             bool first;
diff --git a/src/modules/echo-cancel/module-echo-cancel.c b/src/modules/echo-cancel/module-echo-cancel.c
index 484281f..4e898d0 100644
--- a/src/modules/echo-cancel/module-echo-cancel.c
+++ b/src/modules/echo-cancel/module-echo-cancel.c
@@ -2158,12 +2158,12 @@ int main(int argc, char* argv[]) {
         goto fail;
     }
 
-    source_ss.format = PA_SAMPLE_S16LE;
+    source_ss.format = PA_SAMPLE_FLOAT32LE;
     source_ss.rate = DEFAULT_RATE;
     source_ss.channels = DEFAULT_CHANNELS;
     pa_channel_map_init_auto(&source_map, source_ss.channels, PA_CHANNEL_MAP_DEFAULT);
 
-    sink_ss.format = PA_SAMPLE_S16LE;
+    sink_ss.format = PA_SAMPLE_FLOAT32LE;
     sink_ss.rate = DEFAULT_RATE;
     sink_ss.channels = DEFAULT_CHANNELS;
     pa_channel_map_init_auto(&sink_map, sink_ss.channels, PA_CHANNEL_MAP_DEFAULT);
diff --git a/src/modules/echo-cancel/webrtc.cc b/src/modules/echo-cancel/webrtc.cc
index 6e6143d..ea70969 100644
--- a/src/modules/echo-cancel/webrtc.cc
+++ b/src/modules/echo-cancel/webrtc.cc
@@ -121,8 +121,8 @@ static void pa_webrtc_ec_fixate_spec(pa_sample_spec *rec_ss, pa_channel_map *rec
                                      pa_sample_spec *play_ss, pa_channel_map *play_map,
                                      pa_sample_spec *out_ss, pa_channel_map *out_map)
 {
-    rec_ss->format = PA_SAMPLE_S16NE;
-    play_ss->format = PA_SAMPLE_S16NE;
+    rec_ss->format = PA_SAMPLE_FLOAT32NE;
+    play_ss->format = PA_SAMPLE_FLOAT32NE;
 
     /* AudioProcessing expects one of the following rates */
     if (rec_ss->rate >= 48000)
@@ -221,7 +221,7 @@ bool pa_webrtc_ec_init(pa_core *c, pa_echo_canceller *ec,
     webrtc::ProcessingConfig pconfig;
     webrtc::Config config;
     bool hpf, ns, agc, dgc, mobile, cn, vad, ext_filter, intelligibility, experimental_agc, beamforming;
-    int rm = -1;
+    int rm = -1, i;
     uint32_t agc_start_volume;
     pa_modargs *ma;
 
@@ -475,6 +475,11 @@ bool pa_webrtc_ec_init(pa_core *c, pa_echo_canceller *ec,
     *nframes = ec->params.webrtc.blocksize;
     ec->params.webrtc.first = true;
 
+    for (i = 0; i < rec_ss->channels; i++)
+        ec->params.webrtc.rec_buffer[i] = pa_xnew(float, *nframes);
+    for (i = 0; i < play_ss->channels; i++)
+        ec->params.webrtc.play_buffer[i] = pa_xnew(float, *nframes);
+
     pa_modargs_free(ma);
     return true;
 
@@ -491,36 +496,31 @@ fail:
 
 void pa_webrtc_ec_play(pa_echo_canceller *ec, const uint8_t *play) {
     webrtc::AudioProcessing *apm = (webrtc::AudioProcessing*)ec->params.webrtc.apm;
-    webrtc::AudioFrame play_frame;
     const pa_sample_spec *ss = &ec->params.webrtc.play_ss;
+    int n = ec->params.webrtc.blocksize;
+    float **buf = ec->params.webrtc.play_buffer;
+    webrtc::StreamConfig config(ss->rate, ss->channels, false);
 
-    play_frame.num_channels_ = ss->channels;
-    play_frame.sample_rate_hz_ = ss->rate;
-    play_frame.interleaved_ = true;
-    play_frame.samples_per_channel_ = ec->params.webrtc.blocksize;
-
-    pa_assert(play_frame.samples_per_channel_ <= webrtc::AudioFrame::kMaxDataSizeSamples);
-    memcpy(play_frame.data_, play, ec->params.webrtc.blocksize * pa_frame_size(ss));
+    pa_deinterleave(play, (void **) buf, ss->channels, pa_sample_size(ss), n);
 
-    if (apm->ProcessReverseStream(&play_frame) != webrtc::AudioProcessing::kNoError)
+    if (apm->ProcessReverseStream(buf, config, config, buf) != webrtc::AudioProcessing::kNoError)
         pa_log("Failed to process playback stream");
+
+    /* FIXME: we need to be able to modify playback samples */
 }
 
 void pa_webrtc_ec_record(pa_echo_canceller *ec, const uint8_t *rec, uint8_t *out) {
     webrtc::AudioProcessing *apm = (webrtc::AudioProcessing*)ec->params.webrtc.apm;
-    webrtc::AudioFrame out_frame;
     const pa_sample_spec *rec_ss = &ec->params.webrtc.rec_ss;
     const pa_sample_spec *out_ss = &ec->params.webrtc.out_ss;
+    float **buf = ec->params.webrtc.rec_buffer;
+    int n = ec->params.webrtc.blocksize;
     pa_cvolume v;
     int old_volume, new_volume;
+    webrtc::StreamConfig rec_config(rec_ss->rate, rec_ss->channels, false);
+    webrtc::StreamConfig out_config(out_ss->rate, out_ss->channels, false);
 
-    out_frame.num_channels_ = rec_ss->channels;
-    out_frame.sample_rate_hz_ = rec_ss->rate;
-    out_frame.interleaved_ = true;
-    out_frame.samples_per_channel_ = ec->params.webrtc.blocksize;
-
-    pa_assert(out_frame.samples_per_channel_ <= webrtc::AudioFrame::kMaxDataSizeSamples);
-    memcpy(out_frame.data_, rec, ec->params.webrtc.blocksize * pa_frame_size(rec_ss));
+    pa_deinterleave(rec, (void **) buf, rec_ss->channels, pa_sample_size(rec_ss), n);
 
     if (ec->params.webrtc.agc) {
         pa_cvolume_init(&v);
@@ -530,7 +530,7 @@ void pa_webrtc_ec_record(pa_echo_canceller *ec, const uint8_t *rec, uint8_t *out
     }
 
     apm->set_stream_delay_ms(0);
-    if (apm->ProcessStream(&out_frame) != webrtc::AudioProcessing::kNoError)
+    if (apm->ProcessStream(buf, rec_config, out_config, buf) != webrtc::AudioProcessing::kNoError)
         pa_log("Failed to process capture stream");
 
     if (ec->params.webrtc.agc) {
@@ -551,7 +551,7 @@ void pa_webrtc_ec_record(pa_echo_canceller *ec, const uint8_t *rec, uint8_t *out
         }
     }
 
-    memcpy(out, out_frame.data_, ec->params.webrtc.blocksize * pa_frame_size(out_ss));
+    pa_interleave((const void **) buf, out_ss->channels, out, pa_sample_size(out_ss), n);
 }
 
 void pa_webrtc_ec_set_drift(pa_echo_canceller *ec, float drift) {
@@ -566,6 +566,8 @@ void pa_webrtc_ec_run(pa_echo_canceller *ec, const uint8_t *rec, const uint8_t *
 }
 
 void pa_webrtc_ec_done(pa_echo_canceller *ec) {
+    int i;
+
     if (ec->params.webrtc.trace)
         webrtc::Trace::ReturnTrace();
 
@@ -573,4 +575,9 @@ void pa_webrtc_ec_done(pa_echo_canceller *ec) {
         delete (webrtc::AudioProcessing*)ec->params.webrtc.apm;
         ec->params.webrtc.apm = NULL;
     }
+
+    for (i = 0; i < ec->params.webrtc.rec_ss.channels; i++)
+        pa_xfree(ec->params.webrtc.rec_buffer[i]);
+    for (i = 0; i < ec->params.webrtc.play_ss.channels; i++)
+        pa_xfree(ec->params.webrtc.play_buffer[i]);
 }
-- 
2.5.0



More information about the pulseaudio-discuss mailing list