aboutsummaryrefslogtreecommitdiff
path: root/webrtc/modules
diff options
context:
space:
mode:
authorandrew@webrtc.org <andrew@webrtc.org@4adac7df-926f-26a2-2b94-8c16560cd09d>2013-09-25 23:17:38 +0000
committerandrew@webrtc.org <andrew@webrtc.org@4adac7df-926f-26a2-2b94-8c16560cd09d>2013-09-25 23:17:38 +0000
commit1760a17b8e176f38bcc203bd0afbc9c59df9b502 (patch)
treeb2c23d26e2c699861aedeb1e7f66539535f98dbf /webrtc/modules
parentbecbefaee6e97f9ca6d478d146b55985fc7b9625 (diff)
downloadwebrtc-1760a17b8e176f38bcc203bd0afbc9c59df9b502.tar.gz
Add an extended filter mode to AEC.
Re-land: http://review.webrtc.org/2151007/ TBR=bjornv@webrtc.org Original change description: This mode extends the filter length from the current 48 ms to 128 ms. It is runtime selectable which allows it to be enabled through experiment. We reuse the DelayCorrection infrastructure to avoid having to replumb everything up to libjingle. Increases AEC complexity by ~50% on modern x86 CPUs. Measurements (in percent of usage on one core): Machine/CPU Normal Extended MacBook Retina (Early 2013), Core i7 Ivy Bridge (2.7 GHz, hyperthreaded) 0.6% 0.9% MacBook Air (Late 2010), Core 2 Duo (2.13 GHz) 1.4% 2.7% Chromebook Pixel, Core i5 Ivy Bridge (1.8 GHz) 0.6% 1.0% Samsung ARM Chromebook, Samsung Exynos 5 Dual (1.7 GHz) 3.2% 5.6% The relative value is large of course but the absolute should be acceptable in order to have a working AEC on some platforms. Detailed changes to the algorithm: - The filter length is changed from 48 to 128 ms. This comes with tuning of several parameters: i) filter adaptation stepsize and error threshold; ii) non-linear processing smoothing and overdrive. - Option to ignore the reported delays on platforms which we deem sufficiently unreliable. Currently this will be enabled in Chromium for Mac. - Faster startup times by removing the excessive "startup phase" processing of reported delays. - Much more conservative adjustments to the far-end read pointer. We smooth the delay difference more heavily, and back off from the difference more. Adjustments force a readaptation of the filter, so they should be avoided except when really necessary. Corresponds to these changes: https://chromereviews.googleplex.com/9412014 https://chromereviews.googleplex.com/9514013 https://chromereviews.googleplex.com/9960013 BUG=454,827,1261 Review URL: https://webrtc-codereview.appspot.com/2295006 git-svn-id: http://webrtc.googlecode.com/svn/trunk@4848 4adac7df-926f-26a2-2b94-8c16560cd09d
Diffstat (limited to 'webrtc/modules')
-rw-r--r--webrtc/modules/audio_processing/aec/aec_core.c124
-rw-r--r--webrtc/modules/audio_processing/aec/aec_core.h15
-rw-r--r--webrtc/modules/audio_processing/aec/aec_core_internal.h27
-rw-r--r--webrtc/modules/audio_processing/aec/aec_core_sse2.c56
-rw-r--r--webrtc/modules/audio_processing/aec/echo_cancellation.c537
-rw-r--r--webrtc/modules/audio_processing/aec/echo_cancellation_internal.h6
-rw-r--r--webrtc/modules/audio_processing/aec/system_delay_unittest.cc6
-rw-r--r--webrtc/modules/audio_processing/audio_processing.gypi8
-rw-r--r--webrtc/modules/audio_processing/echo_cancellation_impl.cc16
-rw-r--r--webrtc/modules/audio_processing/echo_cancellation_impl.h26
-rw-r--r--webrtc/modules/audio_processing/echo_cancellation_impl_unittest.cc51
-rw-r--r--webrtc/modules/modules.gyp1
12 files changed, 603 insertions, 270 deletions
diff --git a/webrtc/modules/audio_processing/aec/aec_core.c b/webrtc/modules/audio_processing/aec/aec_core.c
index deaf74aa2d..7fcdec6a36 100644
--- a/webrtc/modules/audio_processing/aec/aec_core.c
+++ b/webrtc/modules/audio_processing/aec/aec_core.c
@@ -106,7 +106,17 @@ const float WebRtcAec_overDriveCurve[65] = {
// Target suppression levels for nlp modes.
// log{0.001, 0.00001, 0.00000001}
static const float kTargetSupp[3] = { -6.9f, -11.5f, -18.4f };
-static const float kMinOverDrive[3] = { 1.0f, 2.0f, 5.0f };
+
+// Two sets of parameters, one for the extended filter mode.
+static const float kExtendedMinOverDrive[3] = { 3.0f, 6.0f, 15.0f };
+static const float kNormalMinOverDrive[3] = { 1.0f, 2.0f, 5.0f };
+static const float kExtendedSmoothingCoefficients[2][2] =
+ { { 0.9f, 0.1f }, { 0.92f, 0.08f } };
+static const float kNormalSmoothingCoefficients[2][2] =
+ { { 0.9f, 0.1f }, { 0.93f, 0.07f } };
+
+// Number of partitions forming the NLP's "preferred" bands.
+enum { kPrefBandSize = 24 };
#ifdef WEBRTC_AEC_DEBUG_DUMP
extern int webrtc_aec_instance_count;
@@ -278,13 +288,13 @@ int WebRtcAec_FreeAec(AecCore* aec)
static void FilterFar(AecCore* aec, float yf[2][PART_LEN1])
{
int i;
- for (i = 0; i < NR_PART; i++) {
+ for (i = 0; i < aec->num_partitions; i++) {
int j;
int xPos = (i + aec->xfBufBlockPos) * PART_LEN1;
int pos = i * PART_LEN1;
// Check for wrap
- if (i + aec->xfBufBlockPos >= NR_PART) {
- xPos -= NR_PART*(PART_LEN1);
+ if (i + aec->xfBufBlockPos >= aec->num_partitions) {
+ xPos -= aec->num_partitions*(PART_LEN1);
}
for (j = 0; j < PART_LEN1; j++) {
@@ -298,22 +308,25 @@ static void FilterFar(AecCore* aec, float yf[2][PART_LEN1])
static void ScaleErrorSignal(AecCore* aec, float ef[2][PART_LEN1])
{
+ const float mu = aec->extended_filter_enabled ? kExtendedMu : aec->normal_mu;
+ const float error_threshold = aec->extended_filter_enabled ?
+ kExtendedErrorThreshold : aec->normal_error_threshold;
int i;
- float absEf;
+ float abs_ef;
for (i = 0; i < (PART_LEN1); i++) {
ef[0][i] /= (aec->xPow[i] + 1e-10f);
ef[1][i] /= (aec->xPow[i] + 1e-10f);
- absEf = sqrtf(ef[0][i] * ef[0][i] + ef[1][i] * ef[1][i]);
+ abs_ef = sqrtf(ef[0][i] * ef[0][i] + ef[1][i] * ef[1][i]);
- if (absEf > aec->errThresh) {
- absEf = aec->errThresh / (absEf + 1e-10f);
- ef[0][i] *= absEf;
- ef[1][i] *= absEf;
+ if (abs_ef > error_threshold) {
+ abs_ef = error_threshold / (abs_ef + 1e-10f);
+ ef[0][i] *= abs_ef;
+ ef[1][i] *= abs_ef;
}
// Stepsize factor
- ef[0][i] *= aec->mu;
- ef[1][i] *= aec->mu;
+ ef[0][i] *= mu;
+ ef[1][i] *= mu;
}
}
@@ -322,35 +335,35 @@ static void ScaleErrorSignal(AecCore* aec, float ef[2][PART_LEN1])
//static void FilterAdaptationUnconstrained(AecCore* aec, float *fft,
// float ef[2][PART_LEN1]) {
// int i, j;
-// for (i = 0; i < NR_PART; i++) {
+// for (i = 0; i < aec->num_partitions; i++) {
// int xPos = (i + aec->xfBufBlockPos)*(PART_LEN1);
// int pos;
// // Check for wrap
-// if (i + aec->xfBufBlockPos >= NR_PART) {
-// xPos -= NR_PART * PART_LEN1;
+// if (i + aec->xfBufBlockPos >= aec->num_partitions) {
+// xPos -= aec->num_partitions * PART_LEN1;
// }
//
// pos = i * PART_LEN1;
//
// for (j = 0; j < PART_LEN1; j++) {
-// aec->wfBuf[pos + j][0] += MulRe(aec->xfBuf[xPos + j][0],
-// -aec->xfBuf[xPos + j][1],
-// ef[j][0], ef[j][1]);
-// aec->wfBuf[pos + j][1] += MulIm(aec->xfBuf[xPos + j][0],
-// -aec->xfBuf[xPos + j][1],
-// ef[j][0], ef[j][1]);
+// aec->wfBuf[0][pos + j] += MulRe(aec->xfBuf[0][xPos + j],
+// -aec->xfBuf[1][xPos + j],
+// ef[0][j], ef[1][j]);
+// aec->wfBuf[1][pos + j] += MulIm(aec->xfBuf[0][xPos + j],
+// -aec->xfBuf[1][xPos + j],
+// ef[0][j], ef[1][j]);
// }
// }
//}
static void FilterAdaptation(AecCore* aec, float *fft, float ef[2][PART_LEN1]) {
int i, j;
- for (i = 0; i < NR_PART; i++) {
+ for (i = 0; i < aec->num_partitions; i++) {
int xPos = (i + aec->xfBufBlockPos)*(PART_LEN1);
int pos;
// Check for wrap
- if (i + aec->xfBufBlockPos >= NR_PART) {
- xPos -= NR_PART * PART_LEN1;
+ if (i + aec->xfBufBlockPos >= aec->num_partitions) {
+ xPos -= aec->num_partitions * PART_LEN1;
}
pos = i * PART_LEN1;
@@ -424,12 +437,12 @@ int WebRtcAec_InitAec(AecCore* aec, int sampFreq)
aec->sampFreq = sampFreq;
if (sampFreq == 8000) {
- aec->mu = 0.6f;
- aec->errThresh = 2e-6f;
+ aec->normal_mu = 0.6f;
+ aec->normal_error_threshold = 2e-6f;
}
else {
- aec->mu = 0.5f;
- aec->errThresh = 1.5e-6f;
+ aec->normal_mu = 0.5f;
+ aec->normal_error_threshold = 1.5e-6f;
}
if (WebRtc_InitBuffer(aec->nearFrBuf) == -1) {
@@ -471,6 +484,9 @@ int WebRtcAec_InitAec(AecCore* aec, int sampFreq)
aec->delay_logging_enabled = 0;
memset(aec->delay_histogram, 0, sizeof(aec->delay_histogram));
+ aec->extended_filter_enabled = 0;
+ aec->num_partitions = kNormalNumPartitions;
+
// Default target suppression mode.
aec->nlp_mode = 1;
@@ -480,7 +496,7 @@ int WebRtcAec_InitAec(AecCore* aec, int sampFreq)
aec->mult = (short)aec->sampFreq / 16000;
}
else {
- aec->mult = (short)aec->sampFreq / 8000;
+ aec->mult = (short)aec->sampFreq / 8000;
}
aec->farBufWritePos = 0;
@@ -511,11 +527,14 @@ int WebRtcAec_InitAec(AecCore* aec, int sampFreq)
aec->xfBufBlockPos = 0;
// TODO: Investigate need for these initializations. Deleting them doesn't
// change the output at all and yields 0.4% overall speedup.
- memset(aec->xfBuf, 0, sizeof(complex_t) * NR_PART * PART_LEN1);
- memset(aec->wfBuf, 0, sizeof(complex_t) * NR_PART * PART_LEN1);
+ memset(aec->xfBuf, 0, sizeof(complex_t) * kExtendedNumPartitions *
+ PART_LEN1);
+ memset(aec->wfBuf, 0, sizeof(complex_t) * kExtendedNumPartitions *
+ PART_LEN1);
memset(aec->sde, 0, sizeof(complex_t) * PART_LEN1);
memset(aec->sxd, 0, sizeof(complex_t) * PART_LEN1);
- memset(aec->xfwBuf, 0, sizeof(complex_t) * NR_PART * PART_LEN1);
+ memset(aec->xfwBuf, 0, sizeof(complex_t) * kExtendedNumPartitions *
+ PART_LEN1);
memset(aec->se, 0, sizeof(float) * PART_LEN1);
// To prevent numerical instability in the first block.
@@ -731,13 +750,11 @@ int WebRtcAec_GetDelayMetricsCore(AecCore* self, int* median, int* std) {
}
int WebRtcAec_echo_state(AecCore* self) {
- assert(self != NULL);
return self->echoState;
}
void WebRtcAec_GetEchoStats(AecCore* self, Stats* erl, Stats* erle,
Stats* a_nlp) {
- assert(self != NULL);
assert(erl != NULL);
assert(erle != NULL);
assert(a_nlp != NULL);
@@ -748,14 +765,12 @@ void WebRtcAec_GetEchoStats(AecCore* self, Stats* erl, Stats* erle,
#ifdef WEBRTC_AEC_DEBUG_DUMP
void* WebRtcAec_far_time_buf(AecCore* self) {
- assert(self != NULL);
return self->far_time_buf;
}
#endif
void WebRtcAec_SetConfigCore(AecCore* self, int nlp_mode, int metrics_mode,
int delay_logging) {
- assert(self != NULL);
assert(nlp_mode >= 0 && nlp_mode < 3);
self->nlp_mode = nlp_mode;
self->metricsMode = metrics_mode;
@@ -768,13 +783,20 @@ void WebRtcAec_SetConfigCore(AecCore* self, int nlp_mode, int metrics_mode,
}
}
+void WebRtcAec_enable_delay_correction(AecCore* self, int enable) {
+ self->extended_filter_enabled = enable;
+ self->num_partitions = enable ? kExtendedNumPartitions : kNormalNumPartitions;
+}
+
+int WebRtcAec_delay_correction_enabled(AecCore* self) {
+ return self->extended_filter_enabled;
+}
+
int WebRtcAec_system_delay(AecCore* self) {
- assert(self != NULL);
return self->system_delay;
}
void WebRtcAec_SetSystemDelay(AecCore* self, int delay) {
- assert(self != NULL);
assert(delay >= 0);
self->system_delay = delay;
}
@@ -850,7 +872,8 @@ static void ProcessBlock(AecCore* aec) {
for (i = 0; i < PART_LEN1; i++) {
far_spectrum = (xf_ptr[i] * xf_ptr[i]) +
(xf_ptr[PART_LEN1 + i] * xf_ptr[PART_LEN1 + i]);
- aec->xPow[i] = gPow[0] * aec->xPow[i] + gPow[1] * NR_PART * far_spectrum;
+ aec->xPow[i] = gPow[0] * aec->xPow[i] + gPow[1] * aec->num_partitions *
+ far_spectrum;
// Calculate absolute spectra
abs_far_spectrum[i] = sqrtf(far_spectrum);
@@ -910,7 +933,7 @@ static void ProcessBlock(AecCore* aec) {
// Update the xfBuf block position.
aec->xfBufBlockPos--;
if (aec->xfBufBlockPos == -1) {
- aec->xfBufBlockPos = NR_PART - 1;
+ aec->xfBufBlockPos = aec->num_partitions - 1;
}
// Buffer xf
@@ -1011,18 +1034,21 @@ static void NonLinearProcessing(AecCore* aec, short *output, short *outputH)
float cohde[PART_LEN1], cohxd[PART_LEN1];
float hNlDeAvg, hNlXdAvg;
float hNl[PART_LEN1];
- float hNlPref[PREF_BAND_SIZE];
+ float hNlPref[kPrefBandSize];
float hNlFb = 0, hNlFbLow = 0;
const float prefBandQuant = 0.75f, prefBandQuantLow = 0.5f;
- const int prefBandSize = PREF_BAND_SIZE / aec->mult;
+ const int prefBandSize = kPrefBandSize / aec->mult;
const int minPrefBand = 4 / aec->mult;
// Near and error power sums
float sdSum = 0, seSum = 0;
- // Power estimate smoothing coefficients
- const float gCoh[2][2] = {{0.9f, 0.1f}, {0.93f, 0.07f}};
- const float *ptrGCoh = gCoh[aec->mult - 1];
+ // Power estimate smoothing coefficients.
+ const float *ptrGCoh = aec->extended_filter_enabled ?
+ kExtendedSmoothingCoefficients[aec->mult - 1] :
+ kNormalSmoothingCoefficients[aec->mult - 1];
+ const float* min_overdrive = aec->extended_filter_enabled ?
+ kExtendedMinOverDrive : kNormalMinOverDrive;
// Filter energy
float wfEnMax = 0, wfEn = 0;
@@ -1045,7 +1071,7 @@ static void NonLinearProcessing(AecCore* aec, short *output, short *outputH)
if (aec->delayEstCtr == 0) {
wfEnMax = 0;
aec->delayIdx = 0;
- for (i = 0; i < NR_PART; i++) {
+ for (i = 0; i < aec->num_partitions; i++) {
pos = i * PART_LEN1;
wfEn = 0;
for (j = 0; j < PART_LEN1; j++) {
@@ -1186,7 +1212,7 @@ static void NonLinearProcessing(AecCore* aec, short *output, short *outputH)
if (aec->hNlXdAvgMin == 1) {
aec->echoState = 0;
- aec->overDrive = kMinOverDrive[aec->nlp_mode];
+ aec->overDrive = min_overdrive[aec->nlp_mode];
if (aec->stNearState == 1) {
memcpy(hNl, cohde, sizeof(hNl));
@@ -1242,7 +1268,7 @@ static void NonLinearProcessing(AecCore* aec, short *output, short *outputH)
aec->hNlMinCtr = 0;
aec->overDrive = WEBRTC_SPL_MAX(kTargetSupp[aec->nlp_mode] /
((float)log(aec->hNlFbMin + 1e-10f) + 1e-10f),
- kMinOverDrive[aec->nlp_mode]);
+ min_overdrive[aec->nlp_mode]);
}
// Smooth the overdrive.
@@ -1462,7 +1488,6 @@ static void InitStats(Stats* stats) {
}
static void InitMetrics(AecCore* self) {
- assert(self != NULL);
self->stateCounter = 0;
InitLevel(&self->farlevel);
InitLevel(&self->nearlevel);
@@ -1684,3 +1709,4 @@ static void TimeToFrequency(float time_data[PART_LEN2],
freq_data[1][i] = time_data[2 * i + 1];
}
}
+
diff --git a/webrtc/modules/audio_processing/aec/aec_core.h b/webrtc/modules/audio_processing/aec/aec_core.h
index 638071735d..f83c37c8c4 100644
--- a/webrtc/modules/audio_processing/aec/aec_core.h
+++ b/webrtc/modules/audio_processing/aec/aec_core.h
@@ -70,23 +70,38 @@ void WebRtcAec_ProcessFrame(AecCore* aec,
// Returns the number of elements moved, and adjusts |system_delay| by the
// corresponding amount in ms.
int WebRtcAec_MoveFarReadPtr(AecCore* aec, int elements);
+
// Calculates the median and standard deviation among the delay estimates
// collected since the last call to this function.
int WebRtcAec_GetDelayMetricsCore(AecCore* self, int* median, int* std);
+
// Returns the echo state (1: echo, 0: no echo).
int WebRtcAec_echo_state(AecCore* self);
+
// Gets statistics of the echo metrics ERL, ERLE, A_NLP.
void WebRtcAec_GetEchoStats(AecCore* self, Stats* erl, Stats* erle,
Stats* a_nlp);
#ifdef WEBRTC_AEC_DEBUG_DUMP
void* WebRtcAec_far_time_buf(AecCore* self);
#endif
+
// Sets local configuration modes.
void WebRtcAec_SetConfigCore(AecCore* self, int nlp_mode, int metrics_mode,
int delay_logging);
+
+// We now interpret delay correction to mean an extended filter length feature.
+// We reuse the delay correction infrastructure to avoid changes through to
+// libjingle. See details along with |DelayCorrection| in
+// echo_cancellation_impl.h. Non-zero enables, zero disables.
+void WebRtcAec_enable_delay_correction(AecCore* self, int enable);
+
+// Returns non-zero if delay correction is enabled and zero if disabled.
+int WebRtcAec_delay_correction_enabled(AecCore* self);
+
// Returns the current |system_delay|, i.e., the buffered difference between
// far-end and near-end.
int WebRtcAec_system_delay(AecCore* self);
+
// Sets the |system_delay| to |value|. Note that if the value is changed
// improperly, there can be a performance regression. So it should be used with
// care.
diff --git a/webrtc/modules/audio_processing/aec/aec_core_internal.h b/webrtc/modules/audio_processing/aec/aec_core_internal.h
index 3b92bd609e..4480101f44 100644
--- a/webrtc/modules/audio_processing/aec/aec_core_internal.h
+++ b/webrtc/modules/audio_processing/aec/aec_core_internal.h
@@ -19,8 +19,15 @@
#include "webrtc/modules/audio_processing/utility/ring_buffer.h"
#include "webrtc/typedefs.h"
-#define NR_PART 12 // Number of partitions in filter.
-#define PREF_BAND_SIZE 24
+// Number of partitions for the extended filter mode. The first one is an enum
+// to be used in array declarations, as it represents the maximum filter length.
+enum { kExtendedNumPartitions = 32 };
+static const int kNormalNumPartitions = 12;
+
+// Extended filter adaptation parameters.
+// TODO(ajm): No narrowband tuning yet.
+static const float kExtendedMu = 0.4f;
+static const float kExtendedErrorThreshold = 1.0e-6f;
typedef struct PowerLevel {
float sfrsum;
@@ -56,11 +63,12 @@ struct AecCore {
float dInitMinPow[PART_LEN1];
float *noisePow;
- float xfBuf[2][NR_PART * PART_LEN1]; // farend fft buffer
- float wfBuf[2][NR_PART * PART_LEN1]; // filter fft
+ float xfBuf[2][kExtendedNumPartitions * PART_LEN1]; // farend fft buffer
+ float wfBuf[2][kExtendedNumPartitions * PART_LEN1]; // filter fft
complex_t sde[PART_LEN1]; // cross-psd of nearend and error
complex_t sxd[PART_LEN1]; // cross-psd of farend and nearend
- complex_t xfwBuf[NR_PART * PART_LEN1]; // farend windowed fft buffer
+ // Farend windowed fft buffer.
+ complex_t xfwBuf[kExtendedNumPartitions * PART_LEN1];
float sx[PART_LEN1], sd[PART_LEN1], se[PART_LEN1]; // far, near, error psd
float hNs[PART_LEN1];
@@ -85,8 +93,8 @@ struct AecCore {
int sampFreq;
uint32_t seed;
- float mu; // stepsize
- float errThresh; // error threshold
+ float normal_mu; // stepsize
+ float normal_error_threshold; // error threshold
int noiseEstCtr;
@@ -112,6 +120,11 @@ struct AecCore {
void* delay_estimator_farend;
void* delay_estimator;
+ // 1 = extended filter mode enabled, 0 = disabled.
+ int extended_filter_enabled;
+ // Runtime selection of number of filter partitions.
+ int num_partitions;
+
#ifdef WEBRTC_AEC_DEBUG_DUMP
RingBuffer* far_time_buf;
FILE *farFile;
diff --git a/webrtc/modules/audio_processing/aec/aec_core_sse2.c b/webrtc/modules/audio_processing/aec/aec_core_sse2.c
index fdc68723ef..61602a8234 100644
--- a/webrtc/modules/audio_processing/aec/aec_core_sse2.c
+++ b/webrtc/modules/audio_processing/aec/aec_core_sse2.c
@@ -34,13 +34,14 @@ __inline static float MulIm(float aRe, float aIm, float bRe, float bIm)
static void FilterFarSSE2(AecCore* aec, float yf[2][PART_LEN1])
{
int i;
- for (i = 0; i < NR_PART; i++) {
+ const int num_partitions = aec->num_partitions;
+ for (i = 0; i < num_partitions; i++) {
int j;
int xPos = (i + aec->xfBufBlockPos) * PART_LEN1;
int pos = i * PART_LEN1;
// Check for wrap
- if (i + aec->xfBufBlockPos >= NR_PART) {
- xPos -= NR_PART*(PART_LEN1);
+ if (i + aec->xfBufBlockPos >= num_partitions) {
+ xPos -= num_partitions*(PART_LEN1);
}
// vectorized code (four at once)
@@ -75,8 +76,11 @@ static void FilterFarSSE2(AecCore* aec, float yf[2][PART_LEN1])
static void ScaleErrorSignalSSE2(AecCore* aec, float ef[2][PART_LEN1])
{
const __m128 k1e_10f = _mm_set1_ps(1e-10f);
- const __m128 kThresh = _mm_set1_ps(aec->errThresh);
- const __m128 kMu = _mm_set1_ps(aec->mu);
+ const __m128 kMu = aec->extended_filter_enabled ?
+ _mm_set1_ps(kExtendedMu) : _mm_set1_ps(aec->normal_mu);
+ const __m128 kThresh = aec->extended_filter_enabled ?
+ _mm_set1_ps(kExtendedErrorThreshold) :
+ _mm_set1_ps(aec->normal_error_threshold);
int i;
// vectorized code (four at once)
@@ -110,32 +114,39 @@ static void ScaleErrorSignalSSE2(AecCore* aec, float ef[2][PART_LEN1])
_mm_storeu_ps(&ef[1][i], ef_im);
}
// scalar code for the remaining items.
- for (; i < (PART_LEN1); i++) {
- float absEf;
- ef[0][i] /= (aec->xPow[i] + 1e-10f);
- ef[1][i] /= (aec->xPow[i] + 1e-10f);
- absEf = sqrtf(ef[0][i] * ef[0][i] + ef[1][i] * ef[1][i]);
-
- if (absEf > aec->errThresh) {
- absEf = aec->errThresh / (absEf + 1e-10f);
- ef[0][i] *= absEf;
- ef[1][i] *= absEf;
- }
+ {
+ const float mu = aec->extended_filter_enabled ?
+ kExtendedMu : aec->normal_mu;
+ const float error_threshold = aec->extended_filter_enabled ?
+ kExtendedErrorThreshold : aec->normal_error_threshold;
+ for (; i < (PART_LEN1); i++) {
+ float abs_ef;
+ ef[0][i] /= (aec->xPow[i] + 1e-10f);
+ ef[1][i] /= (aec->xPow[i] + 1e-10f);
+ abs_ef = sqrtf(ef[0][i] * ef[0][i] + ef[1][i] * ef[1][i]);
+
+ if (abs_ef > error_threshold) {
+ abs_ef = error_threshold / (abs_ef + 1e-10f);
+ ef[0][i] *= abs_ef;
+ ef[1][i] *= abs_ef;
+ }
- // Stepsize factor
- ef[0][i] *= aec->mu;
- ef[1][i] *= aec->mu;
+ // Stepsize factor
+ ef[0][i] *= mu;
+ ef[1][i] *= mu;
+ }
}
}
static void FilterAdaptationSSE2(AecCore* aec, float *fft, float ef[2][PART_LEN1]) {
int i, j;
- for (i = 0; i < NR_PART; i++) {
+ const int num_partitions = aec->num_partitions;
+ for (i = 0; i < num_partitions; i++) {
int xPos = (i + aec->xfBufBlockPos)*(PART_LEN1);
int pos = i * PART_LEN1;
// Check for wrap
- if (i + aec->xfBufBlockPos >= NR_PART) {
- xPos -= NR_PART * PART_LEN1;
+ if (i + aec->xfBufBlockPos >= num_partitions) {
+ xPos -= num_partitions * PART_LEN1;
}
// Process the whole array...
@@ -413,3 +424,4 @@ void WebRtcAec_InitAec_SSE2(void) {
WebRtcAec_FilterAdaptation = FilterAdaptationSSE2;
WebRtcAec_OverdriveAndSuppress = OverdriveAndSuppressSSE2;
}
+
diff --git a/webrtc/modules/audio_processing/aec/echo_cancellation.c b/webrtc/modules/audio_processing/aec/echo_cancellation.c
index 2d4135982c..07bca559af 100644
--- a/webrtc/modules/audio_processing/aec/echo_cancellation.c
+++ b/webrtc/modules/audio_processing/aec/echo_cancellation.c
@@ -27,6 +27,61 @@
#include "webrtc/modules/audio_processing/utility/ring_buffer.h"
#include "webrtc/typedefs.h"
+// Measured delays [ms]
+// Device Chrome GTP
+// MacBook Air 10
+// MacBook Retina 10 100
+// MacPro 30?
+//
+// Win7 Desktop 70 80?
+// Win7 T430s 110
+// Win8 T420s 70
+//
+// Daisy 50
+// Pixel (w/ preproc?) 240
+// Pixel (w/o preproc?) 110 110
+
+// The extended filter mode gives us the flexibility to ignore the system's
+// reported delays. We do this for platforms which we believe provide results
+// which are incompatible with the AEC's expectations. Based on measurements
+// (some provided above) we set a conservative (i.e. lower than measured)
+// fixed delay.
+//
+// WEBRTC_UNTRUSTED_DELAY will only have an impact when |extended_filter_mode|
+// is enabled. See the note along with |DelayCorrection| in
+// echo_cancellation_impl.h for more details on the mode.
+//
+// Justification:
+// Chromium/Mac: Here, the true latency is so low (~10-20 ms), that it plays
+// havoc with the AEC's buffering. To avoid this, we set a fixed delay of 20 ms
+// and then compensate by rewinding by 10 ms (in wideband) through
+// kDelayDiffOffsetSamples. This trick does not seem to work for larger rewind
+// values, but fortunately this is sufficient.
+//
+// Chromium/Linux(ChromeOS): The values we get on this platform don't correspond
+// well to reality. The variance doesn't match the AEC's buffer changes, and the
+// bulk values tend to be too low. However, the range across different hardware
+// appears to be too large to choose a single value.
+//
+// GTP/Linux(ChromeOS): TBD, but for the moment we will trust the values.
+#if defined(WEBRTC_CHROMIUM_BUILD) && defined(WEBRTC_MAC)
+#define WEBRTC_UNTRUSTED_DELAY
+#endif
+
+#if defined(WEBRTC_MAC)
+static const int kFixedDelayMs = 20;
+static const int kDelayDiffOffsetSamples = -160;
+#elif defined(WEBRTC_WIN)
+static const int kFixedDelayMs = 50;
+static const int kDelayDiffOffsetSamples = 0;
+#else
+// Essentially ChromeOS.
+static const int kFixedDelayMs = 50;
+static const int kDelayDiffOffsetSamples = 0;
+#endif
+static const int kMinTrustedDelayMs = 20;
+static const int kMaxTrustedDelayMs = 500;
+
// Maximum length of resampled signal. Must be an integer multiple of frames
// (ceil(1/(1 + MIN_SKEW)*2) + 1)*FRAME_LEN
// The factor of 2 handles wb, and the + 1 is as a safety margin
@@ -43,7 +98,14 @@ int webrtc_aec_instance_count = 0;
// Estimates delay to set the position of the far-end buffer read pointer
// (controlled by knownDelay)
-static int EstBufDelay(aecpc_t *aecInst);
+static void EstBufDelayNormal(aecpc_t *aecInst);
+static void EstBufDelayExtended(aecpc_t *aecInst);
+static int ProcessNormal(aecpc_t* self, const int16_t* near,
+ const int16_t* near_high, int16_t* out, int16_t* out_high,
+ int16_t num_samples, int16_t reported_delay_ms, int32_t skew);
+static void ProcessExtended(aecpc_t* self, const int16_t* near,
+ const int16_t* near_high, int16_t* out, int16_t* out_high,
+ int16_t num_samples, int16_t reported_delay_ms, int32_t skew);
int32_t WebRtcAec_Create(void **aecInst)
{
@@ -135,10 +197,6 @@ int32_t WebRtcAec_Init(void *aecInst, int32_t sampFreq, int32_t scSampFreq)
aecpc_t *aecpc = aecInst;
AecConfig aecConfig;
- if (aecpc == NULL) {
- return -1;
- }
-
if (sampFreq != 8000 && sampFreq != 16000 && sampFreq != 32000) {
aecpc->lastError = AEC_BAD_PARAMETER_ERROR;
return -1;
@@ -177,31 +235,31 @@ int32_t WebRtcAec_Init(void *aecInst, int32_t sampFreq, int32_t scSampFreq)
aecpc->splitSampFreq = sampFreq;
}
- aecpc->skewFrCtr = 0;
- aecpc->activity = 0;
-
aecpc->delayCtr = 0;
+ aecpc->sampFactor = (aecpc->scSampFreq * 1.0f) / aecpc->splitSampFreq;
+ // Sampling frequency multiplier (SWB is processed as 160 frame size).
+ aecpc->rate_factor = aecpc->splitSampFreq / 8000;
aecpc->sum = 0;
aecpc->counter = 0;
aecpc->checkBuffSize = 1;
aecpc->firstVal = 0;
- aecpc->ECstartup = 1;
+ aecpc->startup_phase = 1;
aecpc->bufSizeStart = 0;
aecpc->checkBufSizeCtr = 0;
- aecpc->filtDelay = 0;
+ aecpc->msInSndCardBuf = 0;
+ aecpc->filtDelay = -1; // -1 indicates an initialized state.
aecpc->timeForDelayChange = 0;
aecpc->knownDelay = 0;
aecpc->lastDelayDiff = 0;
- aecpc->skew = 0;
+ aecpc->skewFrCtr = 0;
aecpc->resample = kAecFalse;
aecpc->highSkewCtr = 0;
- aecpc->sampFactor = (aecpc->scSampFreq * 1.0f) / aecpc->splitSampFreq;
+ aecpc->skew = 0;
- // Sampling frequency multiplier (SWB is processed as 160 frame size).
- aecpc->rate_factor = aecpc->splitSampFreq / 8000;
+ aecpc->farend_started = 0;
// Default settings.
aecConfig.nlpMode = kAecNlpModerate;
@@ -239,10 +297,6 @@ int32_t WebRtcAec_BufferFarend(void *aecInst, const int16_t *farend,
float skew;
int i = 0;
- if (aecpc == NULL) {
- return -1;
- }
-
if (farend == NULL) {
aecpc->lastError = AEC_NULL_POINTER_ERROR;
return -1;
@@ -268,6 +322,7 @@ int32_t WebRtcAec_BufferFarend(void *aecInst, const int16_t *farend,
farend_ptr = (const int16_t*) newFarend;
}
+ aecpc->farend_started = 1;
WebRtcAec_SetSystemDelay(aecpc->aec, WebRtcAec_system_delay(aecpc->aec) +
newNrOfSamples);
@@ -311,17 +366,6 @@ int32_t WebRtcAec_Process(void *aecInst, const int16_t *nearend,
{
aecpc_t *aecpc = aecInst;
int32_t retVal = 0;
- short i;
- short nBlocks10ms;
- short nFrames;
- // Limit resampling to doubling/halving of signal
- const float minSkewEst = -0.5f;
- const float maxSkewEst = 1.0f;
-
- if (aecpc == NULL) {
- return -1;
- }
-
if (nearend == NULL) {
aecpc->lastError = AEC_NULL_POINTER_ERROR;
return -1;
@@ -354,144 +398,21 @@ int32_t WebRtcAec_Process(void *aecInst, const int16_t *nearend,
aecpc->lastError = AEC_BAD_PARAMETER_WARNING;
retVal = -1;
}
- else if (msInSndCardBuf > 500) {
- msInSndCardBuf = 500;
+ else if (msInSndCardBuf > kMaxTrustedDelayMs) {
+ // The clamping is now done in ProcessExtended/Normal().
aecpc->lastError = AEC_BAD_PARAMETER_WARNING;
retVal = -1;
}
- // TODO(andrew): we need to investigate if this +10 is really wanted.
- msInSndCardBuf += 10;
- aecpc->msInSndCardBuf = msInSndCardBuf;
-
- if (aecpc->skewMode == kAecTrue) {
- if (aecpc->skewFrCtr < 25) {
- aecpc->skewFrCtr++;
- }
- else {
- retVal = WebRtcAec_GetSkew(aecpc->resampler, skew, &aecpc->skew);
- if (retVal == -1) {
- aecpc->skew = 0;
- aecpc->lastError = AEC_BAD_PARAMETER_WARNING;
- }
-
- aecpc->skew /= aecpc->sampFactor*nrOfSamples;
-
- if (aecpc->skew < 1.0e-3 && aecpc->skew > -1.0e-3) {
- aecpc->resample = kAecFalse;
- }
- else {
- aecpc->resample = kAecTrue;
- }
-
- if (aecpc->skew < minSkewEst) {
- aecpc->skew = minSkewEst;
- }
- else if (aecpc->skew > maxSkewEst) {
- aecpc->skew = maxSkewEst;
- }
-#ifdef WEBRTC_AEC_DEBUG_DUMP
- (void)fwrite(&aecpc->skew, sizeof(aecpc->skew), 1, aecpc->skewFile);
-#endif
- }
- }
-
- nFrames = nrOfSamples / FRAME_LEN;
- nBlocks10ms = nFrames / aecpc->rate_factor;
-
- if (aecpc->ECstartup) {
- if (nearend != out) {
- // Only needed if they don't already point to the same place.
- memcpy(out, nearend, sizeof(short) * nrOfSamples);
- }
-
- // The AEC is in the start up mode
- // AEC is disabled until the system delay is OK
-
- // Mechanism to ensure that the system delay is reasonably stable.
- if (aecpc->checkBuffSize) {
- aecpc->checkBufSizeCtr++;
- // Before we fill up the far-end buffer we require the system delay
- // to be stable (+/-8 ms) compared to the first value. This
- // comparison is made during the following 6 consecutive 10 ms
- // blocks. If it seems to be stable then we start to fill up the
- // far-end buffer.
- if (aecpc->counter == 0) {
- aecpc->firstVal = aecpc->msInSndCardBuf;
- aecpc->sum = 0;
- }
-
- if (abs(aecpc->firstVal - aecpc->msInSndCardBuf) <
- WEBRTC_SPL_MAX(0.2 * aecpc->msInSndCardBuf, sampMsNb)) {
- aecpc->sum += aecpc->msInSndCardBuf;
- aecpc->counter++;
- }
- else {
- aecpc->counter = 0;
- }
-
- if (aecpc->counter * nBlocks10ms >= 6) {
- // The far-end buffer size is determined in partitions of
- // PART_LEN samples. Use 75% of the average value of the system
- // delay as buffer size to start with.
- aecpc->bufSizeStart = WEBRTC_SPL_MIN((3 * aecpc->sum *
- aecpc->rate_factor * 8) / (4 * aecpc->counter * PART_LEN),
- kMaxBufSizeStart);
- // Buffer size has now been determined.
- aecpc->checkBuffSize = 0;
- }
-
- if (aecpc->checkBufSizeCtr * nBlocks10ms > 50) {
- // For really bad systems, don't disable the echo canceller for
- // more than 0.5 sec.
- aecpc->bufSizeStart = WEBRTC_SPL_MIN((aecpc->msInSndCardBuf *
- aecpc->rate_factor * 3) / 40, kMaxBufSizeStart);
- aecpc->checkBuffSize = 0;
- }
- }
-
- // If |checkBuffSize| changed in the if-statement above.
- if (!aecpc->checkBuffSize) {
- // The system delay is now reasonably stable (or has been unstable
- // for too long). When the far-end buffer is filled with
- // approximately the same amount of data as reported by the system
- // we end the startup phase.
- int overhead_elements =
- WebRtcAec_system_delay(aecpc->aec) / PART_LEN -
- aecpc->bufSizeStart;
- if (overhead_elements == 0) {
- // Enable the AEC
- aecpc->ECstartup = 0;
- } else if (overhead_elements > 0) {
- // TODO(bjornv): Do we need a check on how much we actually
- // moved the read pointer? It should always be possible to move
- // the pointer |overhead_elements| since we have only added data
- // to the buffer and no delay compensation nor AEC processing
- // has been done.
- WebRtcAec_MoveFarReadPtr(aecpc->aec, overhead_elements);
-
- // Enable the AEC
- aecpc->ECstartup = 0;
- }
- }
+ // This returns the value of aec->extended_filter_enabled.
+ if (WebRtcAec_delay_correction_enabled(aecpc->aec)) {
+ ProcessExtended(aecpc, nearend, nearendH, out, outH, nrOfSamples,
+ msInSndCardBuf, skew);
} else {
- // AEC is enabled.
-
- EstBufDelay(aecpc);
-
- // Note that 1 frame is supported for NB and 2 frames for WB.
- for (i = 0; i < nFrames; i++) {
- // Call the AEC.
- WebRtcAec_ProcessFrame(aecpc->aec,
- &nearend[FRAME_LEN * i],
- &nearendH[FRAME_LEN * i],
- aecpc->knownDelay,
- &out[FRAME_LEN * i],
- &outH[FRAME_LEN * i]);
- // TODO(bjornv): Re-structure such that we don't have to pass
- // |aecpc->knownDelay| as input. Change name to something like
- // |system_buffer_diff|.
- }
+ if (ProcessNormal(aecpc, nearend, nearendH, out, outH, nrOfSamples,
+ msInSndCardBuf, skew) != 0) {
+ retVal = -1;
+ }
}
#ifdef WEBRTC_AEC_DEBUG_DUMP
@@ -509,11 +430,6 @@ int32_t WebRtcAec_Process(void *aecInst, const int16_t *nearend,
int WebRtcAec_set_config(void* handle, AecConfig config) {
aecpc_t* self = (aecpc_t*)handle;
-
- if (handle == NULL ) {
- return -1;
- }
-
if (self->initFlag != initCheck) {
self->lastError = AEC_UNINITIALIZED_ERROR;
return -1;
@@ -548,10 +464,6 @@ int WebRtcAec_set_config(void* handle, AecConfig config) {
int WebRtcAec_get_echo_status(void* handle, int* status) {
aecpc_t* self = (aecpc_t*)handle;
-
- if (handle == NULL ) {
- return -1;
- }
if (status == NULL ) {
self->lastError = AEC_NULL_POINTER_ERROR;
return -1;
@@ -665,10 +577,6 @@ int WebRtcAec_GetMetrics(void* handle, AecMetrics* metrics) {
int WebRtcAec_GetDelayMetrics(void* handle, int* median, int* std) {
aecpc_t* self = handle;
-
- if (handle == NULL) {
- return -1;
- }
if (median == NULL) {
self->lastError = AEC_NULL_POINTER_ERROR;
return -1;
@@ -693,11 +601,6 @@ int WebRtcAec_GetDelayMetrics(void* handle, int* median, int* std) {
int32_t WebRtcAec_get_error_code(void *aecInst)
{
aecpc_t *aecpc = aecInst;
-
- if (aecpc == NULL) {
- return -1;
- }
-
return aecpc->lastError;
}
@@ -708,7 +611,220 @@ AecCore* WebRtcAec_aec_core(void* handle) {
return ((aecpc_t*) handle)->aec;
}
-static int EstBufDelay(aecpc_t* aecpc) {
+static int ProcessNormal(aecpc_t *aecpc, const int16_t *nearend,
+ const int16_t *nearendH, int16_t *out, int16_t *outH,
+ int16_t nrOfSamples, int16_t msInSndCardBuf,
+ int32_t skew) {
+ int retVal = 0;
+ short i;
+ short nBlocks10ms;
+ short nFrames;
+ // Limit resampling to doubling/halving of signal
+ const float minSkewEst = -0.5f;
+ const float maxSkewEst = 1.0f;
+
+ msInSndCardBuf = msInSndCardBuf > kMaxTrustedDelayMs ?
+ kMaxTrustedDelayMs : msInSndCardBuf;
+ // TODO(andrew): we need to investigate if this +10 is really wanted.
+ msInSndCardBuf += 10;
+ aecpc->msInSndCardBuf = msInSndCardBuf;
+
+ if (aecpc->skewMode == kAecTrue) {
+ if (aecpc->skewFrCtr < 25) {
+ aecpc->skewFrCtr++;
+ }
+ else {
+ retVal = WebRtcAec_GetSkew(aecpc->resampler, skew, &aecpc->skew);
+ if (retVal == -1) {
+ aecpc->skew = 0;
+ aecpc->lastError = AEC_BAD_PARAMETER_WARNING;
+ }
+
+ aecpc->skew /= aecpc->sampFactor*nrOfSamples;
+
+ if (aecpc->skew < 1.0e-3 && aecpc->skew > -1.0e-3) {
+ aecpc->resample = kAecFalse;
+ }
+ else {
+ aecpc->resample = kAecTrue;
+ }
+
+ if (aecpc->skew < minSkewEst) {
+ aecpc->skew = minSkewEst;
+ }
+ else if (aecpc->skew > maxSkewEst) {
+ aecpc->skew = maxSkewEst;
+ }
+
+#ifdef WEBRTC_AEC_DEBUG_DUMP
+ (void)fwrite(&aecpc->skew, sizeof(aecpc->skew), 1, aecpc->skewFile);
+#endif
+ }
+ }
+
+ nFrames = nrOfSamples / FRAME_LEN;
+ nBlocks10ms = nFrames / aecpc->rate_factor;
+
+ if (aecpc->startup_phase) {
+ // Only needed if they don't already point to the same place.
+ if (nearend != out) {
+ memcpy(out, nearend, sizeof(short) * nrOfSamples);
+ }
+ if (nearendH != outH) {
+ memcpy(outH, nearendH, sizeof(short) * nrOfSamples);
+ }
+
+ // The AEC is in the start up mode
+ // AEC is disabled until the system delay is OK
+
+ // Mechanism to ensure that the system delay is reasonably stable.
+ if (aecpc->checkBuffSize) {
+ aecpc->checkBufSizeCtr++;
+ // Before we fill up the far-end buffer we require the system delay
+ // to be stable (+/-8 ms) compared to the first value. This
+ // comparison is made during the following 6 consecutive 10 ms
+ // blocks. If it seems to be stable then we start to fill up the
+ // far-end buffer.
+ if (aecpc->counter == 0) {
+ aecpc->firstVal = aecpc->msInSndCardBuf;
+ aecpc->sum = 0;
+ }
+
+ if (abs(aecpc->firstVal - aecpc->msInSndCardBuf) <
+ WEBRTC_SPL_MAX(0.2 * aecpc->msInSndCardBuf, sampMsNb)) {
+ aecpc->sum += aecpc->msInSndCardBuf;
+ aecpc->counter++;
+ }
+ else {
+ aecpc->counter = 0;
+ }
+
+ if (aecpc->counter * nBlocks10ms >= 6) {
+ // The far-end buffer size is determined in partitions of
+ // PART_LEN samples. Use 75% of the average value of the system
+ // delay as buffer size to start with.
+ aecpc->bufSizeStart = WEBRTC_SPL_MIN((3 * aecpc->sum *
+ aecpc->rate_factor * 8) / (4 * aecpc->counter * PART_LEN),
+ kMaxBufSizeStart);
+ // Buffer size has now been determined.
+ aecpc->checkBuffSize = 0;
+ }
+
+ if (aecpc->checkBufSizeCtr * nBlocks10ms > 50) {
+ // For really bad systems, don't disable the echo canceller for
+ // more than 0.5 sec.
+ aecpc->bufSizeStart = WEBRTC_SPL_MIN((aecpc->msInSndCardBuf *
+ aecpc->rate_factor * 3) / 40, kMaxBufSizeStart);
+ aecpc->checkBuffSize = 0;
+ }
+ }
+
+ // If |checkBuffSize| changed in the if-statement above.
+ if (!aecpc->checkBuffSize) {
+ // The system delay is now reasonably stable (or has been unstable
+ // for too long). When the far-end buffer is filled with
+ // approximately the same amount of data as reported by the system
+ // we end the startup phase.
+ int overhead_elements =
+ WebRtcAec_system_delay(aecpc->aec) / PART_LEN - aecpc->bufSizeStart;
+ if (overhead_elements == 0) {
+ // Enable the AEC
+ aecpc->startup_phase = 0;
+ } else if (overhead_elements > 0) {
+ // TODO(bjornv): Do we need a check on how much we actually
+ // moved the read pointer? It should always be possible to move
+ // the pointer |overhead_elements| since we have only added data
+ // to the buffer and no delay compensation nor AEC processing
+ // has been done.
+ WebRtcAec_MoveFarReadPtr(aecpc->aec, overhead_elements);
+
+ // Enable the AEC
+ aecpc->startup_phase = 0;
+ }
+ }
+ } else {
+ // AEC is enabled.
+ EstBufDelayNormal(aecpc);
+
+ // Note that 1 frame is supported for NB and 2 frames for WB.
+ for (i = 0; i < nFrames; i++) {
+ // Call the AEC.
+ WebRtcAec_ProcessFrame(aecpc->aec,
+ &nearend[FRAME_LEN * i],
+ &nearendH[FRAME_LEN * i],
+ aecpc->knownDelay,
+ &out[FRAME_LEN * i],
+ &outH[FRAME_LEN * i]);
+ // TODO(bjornv): Re-structure such that we don't have to pass
+ // |aecpc->knownDelay| as input. Change name to something like
+ // |system_buffer_diff|.
+ }
+ }
+
+ return retVal;
+}
+
+static void ProcessExtended(aecpc_t* self, const int16_t* near,
+ const int16_t* near_high, int16_t* out, int16_t* out_high,
+ int16_t num_samples, int16_t reported_delay_ms, int32_t skew) {
+ int i;
+ const int num_frames = num_samples / FRAME_LEN;
+#if defined(WEBRTC_UNTRUSTED_DELAY)
+ const int delay_diff_offset = kDelayDiffOffsetSamples;
+ reported_delay_ms = kFixedDelayMs;
+#else
+ // This is the usual mode where we trust the reported system delay values.
+ const int delay_diff_offset = 0;
+ // Due to the longer filter, we no longer add 10 ms to the reported delay
+ // to reduce chance of non-causality. Instead we apply a minimum here to avoid
+ // issues with the read pointer jumping around needlessly.
+ reported_delay_ms = reported_delay_ms < kMinTrustedDelayMs ?
+ kMinTrustedDelayMs : reported_delay_ms;
+ // If the reported delay appears to be bogus, we attempt to recover by using
+ // the measured fixed delay values. We use >= here because higher layers
+ // may already clamp to this maximum value, and we would otherwise not
+ // detect it here.
+ reported_delay_ms = reported_delay_ms >= kMaxTrustedDelayMs ?
+ kFixedDelayMs : reported_delay_ms;
+#endif
+ self->msInSndCardBuf = reported_delay_ms;
+
+ if (!self->farend_started) {
+ // Only needed if they don't already point to the same place.
+ if (near != out) {
+ memcpy(out, near, sizeof(short) * num_samples);
+ }
+ if (near_high != out_high) {
+ memcpy(out_high, near_high, sizeof(short) * num_samples);
+ }
+ return;
+ }
+ if (self->startup_phase) {
+ // In the extended mode, there isn't a startup "phase", just a special
+ // action on the first frame. In the trusted delay case, we'll take the
+ // current reported delay, unless it's less then our conservative
+ // measurement.
+ int startup_size_ms = reported_delay_ms < kFixedDelayMs ?
+ kFixedDelayMs : reported_delay_ms;
+ int overhead_elements = (WebRtcAec_system_delay(self->aec) -
+ startup_size_ms / 2 * self->rate_factor * 8) / PART_LEN;
+ WebRtcAec_MoveFarReadPtr(self->aec, overhead_elements);
+ self->startup_phase = 0;
+ }
+
+ EstBufDelayExtended(self);
+
+ for (i = 0; i < num_frames; ++i) {
+ // |delay_diff_offset| gives us the option to manually rewind the delay on
+ // very low delay platforms which can't be expressed purely through
+ // |reported_delay_ms|.
+ WebRtcAec_ProcessFrame(self->aec, &near[FRAME_LEN * i],
+ &near_high[FRAME_LEN * i], self->knownDelay + delay_diff_offset,
+ &out[FRAME_LEN * i], &out_high[FRAME_LEN * i]);
+ }
+}
+
+static void EstBufDelayNormal(aecpc_t* aecpc) {
int nSampSndCard = aecpc->msInSndCardBuf * sampMsNb * aecpc->rate_factor;
int current_delay = nSampSndCard - WebRtcAec_system_delay(aecpc->aec);
int delay_difference = 0;
@@ -732,8 +848,11 @@ static int EstBufDelay(aecpc_t* aecpc) {
current_delay += WebRtcAec_MoveFarReadPtr(aecpc->aec, 1) * PART_LEN;
}
+ // We use -1 to signal an initialized state in the "extended" implementation;
+ // compensate for that.
+ aecpc->filtDelay = aecpc->filtDelay < 0 ? 0 : aecpc->filtDelay;
aecpc->filtDelay = WEBRTC_SPL_MAX(0, (short) (0.8 * aecpc->filtDelay +
- 0.2 * current_delay));
+ 0.2 * current_delay));
delay_difference = aecpc->filtDelay - aecpc->knownDelay;
if (delay_difference > 224) {
@@ -756,6 +875,58 @@ static int EstBufDelay(aecpc_t* aecpc) {
if (aecpc->timeForDelayChange > 25) {
aecpc->knownDelay = WEBRTC_SPL_MAX((int) aecpc->filtDelay - 160, 0);
}
+}
- return 0;
+static void EstBufDelayExtended(aecpc_t* self) {
+ int reported_delay = self->msInSndCardBuf * sampMsNb * self->rate_factor;
+ int current_delay = reported_delay - WebRtcAec_system_delay(self->aec);
+ int delay_difference = 0;
+
+ // Before we proceed with the delay estimate filtering we:
+ // 1) Compensate for the frame that will be read.
+ // 2) Compensate for drift resampling.
+ // 3) Compensate for non-causality if needed, since the estimated delay can't
+ // be negative.
+
+ // 1) Compensating for the frame(s) that will be read/processed.
+ current_delay += FRAME_LEN * self->rate_factor;
+
+ // 2) Account for resampling frame delay.
+ if (self->skewMode == kAecTrue && self->resample == kAecTrue) {
+ current_delay -= kResamplingDelay;
+ }
+
+ // 3) Compensate for non-causality, if needed, by flushing two blocks.
+ if (current_delay < PART_LEN) {
+ current_delay += WebRtcAec_MoveFarReadPtr(self->aec, 2) * PART_LEN;
+ }
+
+ if (self->filtDelay == -1) {
+ self->filtDelay = WEBRTC_SPL_MAX(0, 0.5 * current_delay);
+ } else {
+ self->filtDelay = WEBRTC_SPL_MAX(0, (short) (0.95 * self->filtDelay +
+ 0.05 * current_delay));
+ }
+
+ delay_difference = self->filtDelay - self->knownDelay;
+ if (delay_difference > 384) {
+ if (self->lastDelayDiff < 128) {
+ self->timeForDelayChange = 0;
+ } else {
+ self->timeForDelayChange++;
+ }
+ } else if (delay_difference < 128 && self->knownDelay > 0) {
+ if (self->lastDelayDiff > 384) {
+ self->timeForDelayChange = 0;
+ } else {
+ self->timeForDelayChange++;
+ }
+ } else {
+ self->timeForDelayChange = 0;
+ }
+ self->lastDelayDiff = delay_difference;
+
+ if (self->timeForDelayChange > 25) {
+ self->knownDelay = WEBRTC_SPL_MAX((int) self->filtDelay - 256, 0);
+ }
}
diff --git a/webrtc/modules/audio_processing/aec/echo_cancellation_internal.h b/webrtc/modules/audio_processing/aec/echo_cancellation_internal.h
index 1298901aeb..e939c4297b 100644
--- a/webrtc/modules/audio_processing/aec/echo_cancellation_internal.h
+++ b/webrtc/modules/audio_processing/aec/echo_cancellation_internal.h
@@ -20,8 +20,6 @@ typedef struct {
int splitSampFreq;
int scSampFreq;
float sampFactor; // scSampRate / sampFreq
- short autoOnOff;
- short activity;
short skewMode;
int bufSizeStart;
int knownDelay;
@@ -39,7 +37,7 @@ typedef struct {
short msInSndCardBuf;
short filtDelay; // Filtered delay estimate.
int timeForDelayChange;
- int ECstartup;
+ int startup_phase;
int checkBuffSize;
short lastDelayDiff;
@@ -62,6 +60,8 @@ typedef struct {
int lastError;
+ int farend_started;
+
AecCore* aec;
} aecpc_t;
diff --git a/webrtc/modules/audio_processing/aec/system_delay_unittest.cc b/webrtc/modules/audio_processing/aec/system_delay_unittest.cc
index 97ebea3dfb..db37f0e83a 100644
--- a/webrtc/modules/audio_processing/aec/system_delay_unittest.cc
+++ b/webrtc/modules/audio_processing/aec/system_delay_unittest.cc
@@ -128,7 +128,7 @@ void SystemDelayTest::RunStableStartup() {
for (; process_time_ms < kStableConvergenceMs; process_time_ms += 10) {
RenderAndCapture(kDeviceBufMs);
buffer_size += samples_per_frame_;
- if (self_->ECstartup == 0) {
+ if (self_->startup_phase == 0) {
// We have left the startup phase.
break;
}
@@ -222,7 +222,7 @@ TEST_F(SystemDelayTest, CorrectDelayAfterUnstableStartup) {
RenderAndCapture(reported_delay_ms);
buffer_size += samples_per_frame_;
buffer_offset_ms = -buffer_offset_ms;
- if (self_->ECstartup == 0) {
+ if (self_->startup_phase == 0) {
// We have left the startup phase.
break;
}
@@ -268,7 +268,7 @@ TEST_F(SystemDelayTest, CorrectDelayAfterStableBufferBuildUp) {
for (; process_time_ms <= kMaxConvergenceMs; process_time_ms += 10) {
RenderAndCapture(kDeviceBufMs);
buffer_size += samples_per_frame_;
- if (self_->ECstartup == 0) {
+ if (self_->startup_phase == 0) {
// We have left the startup phase.
break;
}
diff --git a/webrtc/modules/audio_processing/audio_processing.gypi b/webrtc/modules/audio_processing/audio_processing.gypi
index 02ba74b303..d2655ff7f8 100644
--- a/webrtc/modules/audio_processing/audio_processing.gypi
+++ b/webrtc/modules/audio_processing/audio_processing.gypi
@@ -14,6 +14,11 @@
'variables': {
# Outputs some low-level debug files.
'aec_debug_dump%': 0,
+
+ # Disables the usual mode where we trust the reported system delay
+ # values the AEC receives. The corresponding define is set appropriately
+ # in the code, but it can be force-enabled here for testing.
+ 'aec_untrusted_delay_for_testing%': 0,
},
'dependencies': [
'<(webrtc_root)/common_audio/common_audio.gyp:common_audio',
@@ -92,6 +97,9 @@
['aec_debug_dump==1', {
'defines': ['WEBRTC_AEC_DEBUG_DUMP',],
}],
+ ['aec_untrusted_delay_for_testing==1', {
+ 'defines': ['WEBRTC_UNTRUSTED_DELAY',],
+ }],
['enable_protobuf==1', {
'dependencies': ['audioproc_debug_proto'],
'defines': ['WEBRTC_AUDIOPROC_DEBUG_DUMP'],
diff --git a/webrtc/modules/audio_processing/echo_cancellation_impl.cc b/webrtc/modules/audio_processing/echo_cancellation_impl.cc
index 47ee802483..cd12363ec5 100644
--- a/webrtc/modules/audio_processing/echo_cancellation_impl.cc
+++ b/webrtc/modules/audio_processing/echo_cancellation_impl.cc
@@ -13,12 +13,14 @@
#include <assert.h>
#include <string.h>
+extern "C" {
+#include "webrtc/modules/audio_processing/aec/aec_core.h"
+}
+#include "webrtc/modules/audio_processing/aec/include/echo_cancellation.h"
#include "webrtc/modules/audio_processing/audio_buffer.h"
#include "webrtc/modules/audio_processing/audio_processing_impl.h"
#include "webrtc/system_wrappers/interface/critical_section_wrapper.h"
-#include "webrtc/modules/audio_processing/aec/include/echo_cancellation.h"
-
namespace webrtc {
typedef void Handle;
@@ -69,7 +71,8 @@ EchoCancellationImpl::EchoCancellationImpl(const AudioProcessingImpl* apm)
stream_drift_samples_(0),
was_stream_drift_set_(false),
stream_has_echo_(false),
- delay_logging_enabled_(false) {}
+ delay_logging_enabled_(false),
+ delay_correction_enabled_(false) {}
EchoCancellationImpl::~EchoCancellationImpl() {}
@@ -338,6 +341,11 @@ int EchoCancellationImpl::Initialize() {
return apm_->kNoError;
}
+void EchoCancellationImpl::SetExtraOptions(const Config& config) {
+ delay_correction_enabled_ = config.Get<DelayCorrection>().enabled;
+ Configure();
+}
+
void* EchoCancellationImpl::CreateHandle() const {
Handle* handle = NULL;
if (WebRtcAec_Create(&handle) != apm_->kNoError) {
@@ -369,6 +377,8 @@ int EchoCancellationImpl::ConfigureHandle(void* handle) const {
config.skewMode = drift_compensation_enabled_;
config.delay_logging = delay_logging_enabled_;
+ WebRtcAec_enable_delay_correction(WebRtcAec_aec_core(
+ static_cast<Handle*>(handle)), delay_correction_enabled_ ? 1 : 0);
return WebRtcAec_set_config(static_cast<Handle*>(handle), config);
}
diff --git a/webrtc/modules/audio_processing/echo_cancellation_impl.h b/webrtc/modules/audio_processing/echo_cancellation_impl.h
index 07506d4e03..2b57ebb508 100644
--- a/webrtc/modules/audio_processing/echo_cancellation_impl.h
+++ b/webrtc/modules/audio_processing/echo_cancellation_impl.h
@@ -15,6 +15,30 @@
namespace webrtc {
+// Use to enable the delay correction feature. This now engages an extended
+// filter mode in the AEC, along with robustness measures around the reported
+// system delays. It comes with a significant increase in AEC complexity, but is
+// much more robust to unreliable reported delays.
+//
+// Detailed changes to the algorithm:
+// - The filter length is changed from 48 to 128 ms. This comes with tuning of
+// several parameters: i) filter adaptation stepsize and error threshold;
+// ii) non-linear processing smoothing and overdrive.
+// - Option to ignore the reported delays on platforms which we deem
+// sufficiently unreliable. See WEBRTC_UNTRUSTED_DELAY in echo_cancellation.c.
+// - Faster startup times by removing the excessive "startup phase" processing
+// of reported delays.
+// - Much more conservative adjustments to the far-end read pointer. We smooth
+// the delay difference more heavily, and back off from the difference more.
+// Adjustments force a readaptation of the filter, so they should be avoided
+// except when really necessary.
+struct DelayCorrection {
+ DelayCorrection() : enabled(false) {}
+ DelayCorrection(bool enabled) : enabled(enabled) {}
+
+ bool enabled;
+};
+
class AudioProcessingImpl;
class AudioBuffer;
@@ -34,6 +58,7 @@ class EchoCancellationImpl : public EchoCancellationImplWrapper {
// ProcessingComponent implementation.
virtual int Initialize() OVERRIDE;
+ virtual void SetExtraOptions(const Config& config) OVERRIDE;
private:
// EchoCancellation implementation.
@@ -70,6 +95,7 @@ class EchoCancellationImpl : public EchoCancellationImplWrapper {
bool was_stream_drift_set_;
bool stream_has_echo_;
bool delay_logging_enabled_;
+ bool delay_correction_enabled_;
};
} // namespace webrtc
diff --git a/webrtc/modules/audio_processing/echo_cancellation_impl_unittest.cc b/webrtc/modules/audio_processing/echo_cancellation_impl_unittest.cc
new file mode 100644
index 0000000000..16ecf02e4b
--- /dev/null
+++ b/webrtc/modules/audio_processing/echo_cancellation_impl_unittest.cc
@@ -0,0 +1,51 @@
+/*
+ * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "testing/gtest/include/gtest/gtest.h"
+extern "C" {
+#include "webrtc/modules/audio_processing/aec/aec_core.h"
+}
+#include "webrtc/modules/audio_processing/echo_cancellation_impl.h"
+#include "webrtc/modules/audio_processing/include/audio_processing.h"
+#include "webrtc/system_wrappers/interface/scoped_ptr.h"
+
+namespace webrtc {
+
+TEST(EchoCancellationInternalTest, DelayCorrection) {
+ scoped_ptr<AudioProcessing> ap(AudioProcessing::Create(0));
+ EXPECT_TRUE(ap->echo_cancellation()->aec_core() == NULL);
+
+ EXPECT_EQ(ap->kNoError, ap->echo_cancellation()->Enable(true));
+ EXPECT_TRUE(ap->echo_cancellation()->is_enabled());
+
+ AecCore* aec_core = ap->echo_cancellation()->aec_core();
+ ASSERT_TRUE(aec_core != NULL);
+ // Disabled by default.
+ EXPECT_EQ(0, WebRtcAec_delay_correction_enabled(aec_core));
+
+ Config config;
+ config.Set<DelayCorrection>(new DelayCorrection(true));
+ ap->SetExtraOptions(config);
+ EXPECT_EQ(1, WebRtcAec_delay_correction_enabled(aec_core));
+
+ // Retains setting after initialization.
+ EXPECT_EQ(ap->kNoError, ap->Initialize());
+ EXPECT_EQ(1, WebRtcAec_delay_correction_enabled(aec_core));
+
+ config.Set<DelayCorrection>(new DelayCorrection(false));
+ ap->SetExtraOptions(config);
+ EXPECT_EQ(0, WebRtcAec_delay_correction_enabled(aec_core));
+
+ // Retains setting after initialization.
+ EXPECT_EQ(ap->kNoError, ap->Initialize());
+ EXPECT_EQ(0, WebRtcAec_delay_correction_enabled(aec_core));
+}
+
+} // namespace webrtc
diff --git a/webrtc/modules/modules.gyp b/webrtc/modules/modules.gyp
index d5d3cbe828..5dab6f7cd2 100644
--- a/webrtc/modules/modules.gyp
+++ b/webrtc/modules/modules.gyp
@@ -145,6 +145,7 @@
'audio_coding/neteq4/mock/mock_payload_splitter.h',
'audio_processing/aec/system_delay_unittest.cc',
'audio_processing/aec/echo_cancellation_unittest.cc',
+ 'audio_processing/echo_cancellation_impl_unittest.cc',
'audio_processing/test/audio_processing_unittest.cc',
'audio_processing/utility/delay_estimator_unittest.cc',
'audio_processing/utility/ring_buffer_unittest.cc',