diff options
author | peah <peah@webrtc.org> | 2015-11-16 16:27:42 -0800 |
---|---|---|
committer | Commit bot <commit-bot@chromium.org> | 2015-11-17 00:27:50 +0000 |
commit | fa6228e221d818af55e3d8343c792f2c1ecc7252 (patch) | |
tree | d41b1c1dffb204886f84539fc3368822bbde28f6 /webrtc | |
parent | 4c27e4b62da2047063d88eedfeec3e939fea7843 (diff) | |
download | webrtc-fa6228e221d818af55e3d8343c792f2c1ecc7252.tar.gz |
Introduced the render sample queue for the aec and aecm.
BUG=webrtc:5099
Review URL: https://codereview.webrtc.org/1410833002
Cr-Commit-Position: refs/heads/master@{#10662}
Diffstat (limited to 'webrtc')
6 files changed, 250 insertions, 24 deletions
diff --git a/webrtc/modules/audio_processing/audio_processing_impl.cc b/webrtc/modules/audio_processing/audio_processing_impl.cc index 668ec11e91..0daaf1f449 100644 --- a/webrtc/modules/audio_processing/audio_processing_impl.cc +++ b/webrtc/modules/audio_processing/audio_processing_impl.cc @@ -530,6 +530,9 @@ int AudioProcessingImpl::ProcessStream(const float* const* src, return kNullPointerError; } + echo_cancellation_->ReadQueuedRenderData(); + echo_control_mobile_->ReadQueuedRenderData(); + ProcessingConfig processing_config = api_format_; processing_config.input_stream() = input_config; processing_config.output_stream() = output_config; @@ -571,6 +574,9 @@ int AudioProcessingImpl::ProcessStream(const float* const* src, int AudioProcessingImpl::ProcessStream(AudioFrame* frame) { CriticalSectionScoped crit_scoped(crit_); + echo_cancellation_->ReadQueuedRenderData(); + echo_control_mobile_->ReadQueuedRenderData(); + if (!frame) { return kNullPointerError; } diff --git a/webrtc/modules/audio_processing/echo_cancellation_impl.cc b/webrtc/modules/audio_processing/echo_cancellation_impl.cc index 0343a0eb44..c6f92005a5 100644 --- a/webrtc/modules/audio_processing/echo_cancellation_impl.cc +++ b/webrtc/modules/audio_processing/echo_cancellation_impl.cc @@ -55,6 +55,9 @@ AudioProcessing::Error MapError(int err) { } } // namespace +const size_t EchoCancellationImpl::kAllowedValuesOfSamplesPerFrame1; +const size_t EchoCancellationImpl::kAllowedValuesOfSamplesPerFrame2; + EchoCancellationImpl::EchoCancellationImpl(const AudioProcessing* apm, CriticalSectionWrapper* crit) : ProcessingComponent(), @@ -68,7 +71,9 @@ EchoCancellationImpl::EchoCancellationImpl(const AudioProcessing* apm, stream_has_echo_(false), delay_logging_enabled_(false), extended_filter_enabled_(false), - delay_agnostic_enabled_(false) { + delay_agnostic_enabled_(false), + render_queue_element_max_size_(0) { + AllocateRenderQueue(); } EchoCancellationImpl::~EchoCancellationImpl() {} @@ -85,25 +90,65 @@ int EchoCancellationImpl::ProcessRenderAudio(const AudioBuffer* audio) { // The ordering convention must be followed to pass to the correct AEC. size_t handle_index = 0; + render_queue_buffer_.clear(); for (int i = 0; i < apm_->num_output_channels(); i++) { for (int j = 0; j < audio->num_channels(); j++) { Handle* my_handle = static_cast<Handle*>(handle(handle_index)); - err = WebRtcAec_BufferFarend( - my_handle, - audio->split_bands_const_f(j)[kBand0To8kHz], + // Retrieve any error code produced by the buffering of the farend + // signal + err = WebRtcAec_GetBufferFarendError( + my_handle, audio->split_bands_const_f(j)[kBand0To8kHz], audio->num_frames_per_band()); if (err != apm_->kNoError) { return MapError(err); // TODO(ajm): warning possible? } - handle_index++; + // Buffer the samples in the render queue. + render_queue_buffer_.insert(render_queue_buffer_.end(), + audio->split_bands_const_f(j)[kBand0To8kHz], + (audio->split_bands_const_f(j)[kBand0To8kHz] + + audio->num_frames_per_band())); } } + // Insert the samples into the queue. + if (!render_signal_queue_->Insert(&render_queue_buffer_)) { + ReadQueuedRenderData(); + + // Retry the insert (should always work). + RTC_DCHECK_EQ(render_signal_queue_->Insert(&render_queue_buffer_), true); + } + return apm_->kNoError; } +// Read chunks of data that were received and queued on the render side from +// a queue. All the data chunks are buffered into the farend signal of the AEC. +void EchoCancellationImpl::ReadQueuedRenderData() { + if (!is_component_enabled()) { + return; + } + + while (render_signal_queue_->Remove(&capture_queue_buffer_)) { + size_t handle_index = 0; + int buffer_index = 0; + const int num_frames_per_band = + capture_queue_buffer_.size() / + (apm_->num_output_channels() * apm_->num_reverse_channels()); + for (int i = 0; i < apm_->num_output_channels(); i++) { + for (int j = 0; j < apm_->num_reverse_channels(); j++) { + Handle* my_handle = static_cast<Handle*>(handle(handle_index)); + WebRtcAec_BufferFarend(my_handle, &capture_queue_buffer_[buffer_index], + num_frames_per_band); + + buffer_index += num_frames_per_band; + handle_index++; + } + } + } +} + int EchoCancellationImpl::ProcessCaptureAudio(AudioBuffer* audio) { if (!is_component_enabled()) { return apm_->kNoError; @@ -333,9 +378,37 @@ int EchoCancellationImpl::Initialize() { return err; } + AllocateRenderQueue(); + return apm_->kNoError; } +void EchoCancellationImpl::AllocateRenderQueue() { + const size_t max_frame_size = std::max<size_t>( + kAllowedValuesOfSamplesPerFrame1, kAllowedValuesOfSamplesPerFrame2); + + const size_t new_render_queue_element_max_size = std::max<size_t>( + static_cast<size_t>(1), max_frame_size * num_handles_required()); + + // Reallocate the queue if the queue item size is too small to fit the + // data to put in the queue. + if (new_render_queue_element_max_size > render_queue_element_max_size_) { + render_queue_element_max_size_ = new_render_queue_element_max_size; + + std::vector<float> template_queue_element(render_queue_element_max_size_); + + render_signal_queue_.reset( + new SwapQueue<std::vector<float>, RenderQueueItemVerifier<float>>( + kMaxNumFramesToBuffer, template_queue_element, + RenderQueueItemVerifier<float>(render_queue_element_max_size_))); + } else { + render_signal_queue_->Clear(); + } + + render_queue_buffer_.resize(new_render_queue_element_max_size); + capture_queue_buffer_.resize(new_render_queue_element_max_size); +} + void EchoCancellationImpl::SetExtraOptions(const Config& config) { extended_filter_enabled_ = config.Get<ExtendedFilter>().enabled; delay_agnostic_enabled_ = config.Get<DelayAgnostic>().enabled; diff --git a/webrtc/modules/audio_processing/echo_cancellation_impl.h b/webrtc/modules/audio_processing/echo_cancellation_impl.h index 070dcabc5d..d4dfc6dd9f 100644 --- a/webrtc/modules/audio_processing/echo_cancellation_impl.h +++ b/webrtc/modules/audio_processing/echo_cancellation_impl.h @@ -11,6 +11,8 @@ #ifndef WEBRTC_MODULES_AUDIO_PROCESSING_ECHO_CANCELLATION_IMPL_H_ #define WEBRTC_MODULES_AUDIO_PROCESSING_ECHO_CANCELLATION_IMPL_H_ +#include "webrtc/base/scoped_ptr.h" +#include "webrtc/common_audio/swap_queue.h" #include "webrtc/modules/audio_processing/include/audio_processing.h" #include "webrtc/modules/audio_processing/processing_component.h" @@ -42,7 +44,16 @@ class EchoCancellationImpl : public EchoCancellation, bool is_delay_agnostic_enabled() const; bool is_extended_filter_enabled() const; + // Reads render side data that has been queued on the render call. + void ReadQueuedRenderData(); + private: + static const size_t kAllowedValuesOfSamplesPerFrame1 = 80; + static const size_t kAllowedValuesOfSamplesPerFrame2 = 160; + // TODO(peah): Decrease this once we properly handle hugely unbalanced + // reverse and forward call numbers. + static const size_t kMaxNumFramesToBuffer = 100; + // EchoCancellation implementation. int Enable(bool enable) override; int enable_drift_compensation(bool enable) override; @@ -68,6 +79,8 @@ class EchoCancellationImpl : public EchoCancellation, int num_handles_required() const override; int GetHandleError(void* handle) const override; + void AllocateRenderQueue(); + const AudioProcessing* apm_; CriticalSectionWrapper* crit_; bool drift_compensation_enabled_; @@ -79,6 +92,12 @@ class EchoCancellationImpl : public EchoCancellation, bool delay_logging_enabled_; bool extended_filter_enabled_; bool delay_agnostic_enabled_; + + size_t render_queue_element_max_size_; + std::vector<float> render_queue_buffer_; + std::vector<float> capture_queue_buffer_; + rtc::scoped_ptr<SwapQueue<std::vector<float>, RenderQueueItemVerifier<float>>> + render_signal_queue_; }; } // namespace webrtc diff --git a/webrtc/modules/audio_processing/echo_control_mobile_impl.cc b/webrtc/modules/audio_processing/echo_control_mobile_impl.cc index 8cc9dce27e..b9e1e517c9 100644 --- a/webrtc/modules/audio_processing/echo_control_mobile_impl.cc +++ b/webrtc/modules/audio_processing/echo_control_mobile_impl.cc @@ -40,20 +40,42 @@ int16_t MapSetting(EchoControlMobile::RoutingMode mode) { return -1; } +AudioProcessing::Error MapError(int err) { + switch (err) { + case AECM_UNSUPPORTED_FUNCTION_ERROR: + return AudioProcessing::kUnsupportedFunctionError; + case AECM_NULL_POINTER_ERROR: + return AudioProcessing::kNullPointerError; + case AECM_BAD_PARAMETER_ERROR: + return AudioProcessing::kBadParameterError; + case AECM_BAD_PARAMETER_WARNING: + return AudioProcessing::kBadStreamParameterWarning; + default: + // AECM_UNSPECIFIED_ERROR + // AECM_UNINITIALIZED_ERROR + return AudioProcessing::kUnspecifiedError; + } +} } // namespace +const size_t EchoControlMobileImpl::kAllowedValuesOfSamplesPerFrame1; +const size_t EchoControlMobileImpl::kAllowedValuesOfSamplesPerFrame2; + size_t EchoControlMobile::echo_path_size_bytes() { return WebRtcAecm_echo_path_size_bytes(); } EchoControlMobileImpl::EchoControlMobileImpl(const AudioProcessing* apm, CriticalSectionWrapper* crit) - : ProcessingComponent(), - apm_(apm), - crit_(crit), - routing_mode_(kSpeakerphone), - comfort_noise_enabled_(true), - external_echo_path_(NULL) {} + : ProcessingComponent(), + apm_(apm), + crit_(crit), + routing_mode_(kSpeakerphone), + comfort_noise_enabled_(true), + external_echo_path_(NULL), + render_queue_element_max_size_(0) { + AllocateRenderQueue(); +} EchoControlMobileImpl::~EchoControlMobileImpl() { if (external_echo_path_ != NULL) { @@ -74,25 +96,64 @@ int EchoControlMobileImpl::ProcessRenderAudio(const AudioBuffer* audio) { // The ordering convention must be followed to pass to the correct AECM. size_t handle_index = 0; + render_queue_buffer_.clear(); for (int i = 0; i < apm_->num_output_channels(); i++) { for (int j = 0; j < audio->num_channels(); j++) { Handle* my_handle = static_cast<Handle*>(handle(handle_index)); - err = WebRtcAecm_BufferFarend( - my_handle, - audio->split_bands_const(j)[kBand0To8kHz], + err = WebRtcAecm_GetBufferFarendError( + my_handle, audio->split_bands_const(j)[kBand0To8kHz], audio->num_frames_per_band()); - if (err != apm_->kNoError) { - return GetHandleError(my_handle); // TODO(ajm): warning possible? - } + if (err != apm_->kNoError) + return MapError(err); // TODO(ajm): warning possible?); + + // Buffer the samples in the render queue. + render_queue_buffer_.insert(render_queue_buffer_.end(), + audio->split_bands_const(j)[kBand0To8kHz], + (audio->split_bands_const(j)[kBand0To8kHz] + + audio->num_frames_per_band())); handle_index++; } } + // Insert the samples into the queue. + if (!render_signal_queue_->Insert(&render_queue_buffer_)) { + ReadQueuedRenderData(); + + // Retry the insert (should always work). + RTC_DCHECK_EQ(render_signal_queue_->Insert(&render_queue_buffer_), true); + } + return apm_->kNoError; } +// Read chunks of data that were received and queued on the render side from +// a queue. All the data chunks are buffered into the farend signal of the AEC. +void EchoControlMobileImpl::ReadQueuedRenderData() { + if (!is_component_enabled()) { + return; + } + + while (render_signal_queue_->Remove(&capture_queue_buffer_)) { + size_t handle_index = 0; + int buffer_index = 0; + const int num_frames_per_band = + capture_queue_buffer_.size() / + (apm_->num_output_channels() * apm_->num_reverse_channels()); + for (int i = 0; i < apm_->num_output_channels(); i++) { + for (int j = 0; j < apm_->num_reverse_channels(); j++) { + Handle* my_handle = static_cast<Handle*>(handle(handle_index)); + WebRtcAecm_BufferFarend(my_handle, &capture_queue_buffer_[buffer_index], + num_frames_per_band); + + buffer_index += num_frames_per_band; + handle_index++; + } + } + } +} + int EchoControlMobileImpl::ProcessCaptureAudio(AudioBuffer* audio) { if (!is_component_enabled()) { return apm_->kNoError; @@ -128,9 +189,8 @@ int EchoControlMobileImpl::ProcessCaptureAudio(AudioBuffer* audio) { audio->num_frames_per_band(), apm_->stream_delay_ms()); - if (err != apm_->kNoError) { - return GetHandleError(my_handle); // TODO(ajm): warning possible? - } + if (err != apm_->kNoError) + return MapError(err); handle_index++; } @@ -213,9 +273,9 @@ int EchoControlMobileImpl::GetEchoPath(void* echo_path, // Get the echo path from the first channel Handle* my_handle = static_cast<Handle*>(handle(0)); - if (WebRtcAecm_GetEchoPath(my_handle, echo_path, size_bytes) != 0) { - return GetHandleError(my_handle); - } + int32_t err = WebRtcAecm_GetEchoPath(my_handle, echo_path, size_bytes); + if (err != 0) + return MapError(err); return apm_->kNoError; } @@ -230,7 +290,39 @@ int EchoControlMobileImpl::Initialize() { return apm_->kBadSampleRateError; } - return ProcessingComponent::Initialize(); + int err = ProcessingComponent::Initialize(); + if (err != apm_->kNoError) { + return err; + } + + AllocateRenderQueue(); + + return apm_->kNoError; +} + +void EchoControlMobileImpl::AllocateRenderQueue() { + const size_t max_frame_size = std::max<size_t>( + kAllowedValuesOfSamplesPerFrame1, kAllowedValuesOfSamplesPerFrame2); + const size_t new_render_queue_element_max_size = std::max<size_t>( + static_cast<size_t>(1), max_frame_size * num_handles_required()); + + // Reallocate the queue if the queue item size is too small to fit the + // data to put in the queue. + if (new_render_queue_element_max_size > render_queue_element_max_size_) { + render_queue_element_max_size_ = new_render_queue_element_max_size; + + std::vector<int16_t> template_queue_element(render_queue_element_max_size_); + + render_signal_queue_.reset( + new SwapQueue<std::vector<int16_t>, RenderQueueItemVerifier<int16_t>>( + kMaxNumFramesToBuffer, template_queue_element, + RenderQueueItemVerifier<int16_t>(render_queue_element_max_size_))); + } else { + render_signal_queue_->Clear(); + } + + render_queue_buffer_.resize(new_render_queue_element_max_size); + capture_queue_buffer_.resize(new_render_queue_element_max_size); } void* EchoControlMobileImpl::CreateHandle() const { diff --git a/webrtc/modules/audio_processing/echo_control_mobile_impl.h b/webrtc/modules/audio_processing/echo_control_mobile_impl.h index da7022545f..87c5376d68 100644 --- a/webrtc/modules/audio_processing/echo_control_mobile_impl.h +++ b/webrtc/modules/audio_processing/echo_control_mobile_impl.h @@ -11,6 +11,8 @@ #ifndef WEBRTC_MODULES_AUDIO_PROCESSING_ECHO_CONTROL_MOBILE_IMPL_H_ #define WEBRTC_MODULES_AUDIO_PROCESSING_ECHO_CONTROL_MOBILE_IMPL_H_ +#include "webrtc/base/scoped_ptr.h" +#include "webrtc/common_audio/swap_queue.h" #include "webrtc/modules/audio_processing/include/audio_processing.h" #include "webrtc/modules/audio_processing/processing_component.h" @@ -37,7 +39,16 @@ class EchoControlMobileImpl : public EchoControlMobile, // ProcessingComponent implementation. int Initialize() override; + // Reads render side data that has been queued on the render call. + void ReadQueuedRenderData(); + private: + static const size_t kAllowedValuesOfSamplesPerFrame1 = 80; + static const size_t kAllowedValuesOfSamplesPerFrame2 = 160; + // TODO(peah): Decrease this once we properly handle hugely unbalanced + // reverse and forward call numbers. + static const size_t kMaxNumFramesToBuffer = 100; + // EchoControlMobile implementation. int Enable(bool enable) override; int set_routing_mode(RoutingMode mode) override; @@ -53,11 +64,20 @@ class EchoControlMobileImpl : public EchoControlMobile, int num_handles_required() const override; int GetHandleError(void* handle) const override; + void AllocateRenderQueue(); + const AudioProcessing* apm_; CriticalSectionWrapper* crit_; RoutingMode routing_mode_; bool comfort_noise_enabled_; unsigned char* external_echo_path_; + + size_t render_queue_element_max_size_; + std::vector<int16_t> render_queue_buffer_; + std::vector<int16_t> capture_queue_buffer_; + rtc::scoped_ptr< + SwapQueue<std::vector<int16_t>, RenderQueueItemVerifier<int16_t>>> + render_signal_queue_; }; } // namespace webrtc diff --git a/webrtc/modules/audio_processing/processing_component.h b/webrtc/modules/audio_processing/processing_component.h index 8ee3ac6c7d..291aea3922 100644 --- a/webrtc/modules/audio_processing/processing_component.h +++ b/webrtc/modules/audio_processing/processing_component.h @@ -17,6 +17,22 @@ namespace webrtc { +// Functor to use when supplying a verifier function for the queue item +// verifcation. +template <typename T> +class RenderQueueItemVerifier { + public: + explicit RenderQueueItemVerifier(size_t minimum_capacity) + : minimum_capacity_(minimum_capacity) {} + + bool operator()(const std::vector<T>& v) const { + return v.capacity() >= minimum_capacity_; + } + + private: + size_t minimum_capacity_; +}; + class ProcessingComponent { public: ProcessingComponent(); |