diff options
Diffstat (limited to 'webrtc/modules/audio_processing/audio_processing_impl.cc')
| -rw-r--r-- | webrtc/modules/audio_processing/audio_processing_impl.cc | 1099 |
1 files changed, 676 insertions, 423 deletions
diff --git a/webrtc/modules/audio_processing/audio_processing_impl.cc b/webrtc/modules/audio_processing/audio_processing_impl.cc index c6574151d0..744309c774 100644 --- a/webrtc/modules/audio_processing/audio_processing_impl.cc +++ b/webrtc/modules/audio_processing/audio_processing_impl.cc @@ -15,6 +15,7 @@ #include "webrtc/base/checks.h" #include "webrtc/base/platform_file.h" +#include "webrtc/base/trace_event.h" #include "webrtc/common_audio/audio_converter.h" #include "webrtc/common_audio/channel_buffer.h" #include "webrtc/common_audio/include/audio_util.h" @@ -36,8 +37,7 @@ extern "C" { #include "webrtc/modules/audio_processing/processing_component.h" #include "webrtc/modules/audio_processing/transient/transient_suppressor.h" #include "webrtc/modules/audio_processing/voice_detection_impl.h" -#include "webrtc/modules/interface/module_common_types.h" -#include "webrtc/system_wrappers/include/critical_section_wrapper.h" +#include "webrtc/modules/include/module_common_types.h" #include "webrtc/system_wrappers/include/file_wrapper.h" #include "webrtc/system_wrappers/include/logging.h" #include "webrtc/system_wrappers/include/metrics.h" @@ -75,7 +75,6 @@ static bool LayoutHasKeyboard(AudioProcessing::ChannelLayout layout) { assert(false); return false; } - } // namespace // Throughout webrtc, it's assumed that success is represented by zero. @@ -147,6 +146,35 @@ class GainControlForNewAgc : public GainControl, public VolumeCallbacks { int volume_; }; +struct AudioProcessingImpl::ApmPublicSubmodules { + ApmPublicSubmodules() + : echo_cancellation(nullptr), + echo_control_mobile(nullptr), + gain_control(nullptr) {} + // Accessed externally of APM without any lock acquired. + EchoCancellationImpl* echo_cancellation; + EchoControlMobileImpl* echo_control_mobile; + GainControlImpl* gain_control; + rtc::scoped_ptr<HighPassFilterImpl> high_pass_filter; + rtc::scoped_ptr<LevelEstimatorImpl> level_estimator; + rtc::scoped_ptr<NoiseSuppressionImpl> noise_suppression; + rtc::scoped_ptr<VoiceDetectionImpl> voice_detection; + rtc::scoped_ptr<GainControlForNewAgc> gain_control_for_new_agc; + + // Accessed internally from both render and capture. + rtc::scoped_ptr<TransientSuppressor> transient_suppressor; + rtc::scoped_ptr<IntelligibilityEnhancer> intelligibility_enhancer; +}; + +struct AudioProcessingImpl::ApmPrivateSubmodules { + explicit ApmPrivateSubmodules(Beamformer<float>* beamformer) + : beamformer(beamformer) {} + // Accessed internally from capture or during initialization + std::list<ProcessingComponent*> component_list; + rtc::scoped_ptr<Beamformer<float>> beamformer; + rtc::scoped_ptr<AgcManagerDirect> agc_manager; +}; + const int AudioProcessing::kNativeSampleRatesHz[] = { AudioProcessing::kSampleRate8kHz, AudioProcessing::kSampleRate16kHz, @@ -172,7 +200,7 @@ AudioProcessing* AudioProcessing::Create(const Config& config, AudioProcessingImpl* apm = new AudioProcessingImpl(config, beamformer); if (apm->Initialize() != kNoError) { delete apm; - apm = NULL; + apm = nullptr; } return apm; @@ -183,102 +211,82 @@ AudioProcessingImpl::AudioProcessingImpl(const Config& config) AudioProcessingImpl::AudioProcessingImpl(const Config& config, Beamformer<float>* beamformer) - : echo_cancellation_(NULL), - echo_control_mobile_(NULL), - gain_control_(NULL), - high_pass_filter_(NULL), - level_estimator_(NULL), - noise_suppression_(NULL), - voice_detection_(NULL), - crit_(CriticalSectionWrapper::CreateCriticalSection()), -#ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP - debug_file_(FileWrapper::Create()), - event_msg_(new audioproc::Event()), -#endif - api_format_({{{kSampleRate16kHz, 1, false}, - {kSampleRate16kHz, 1, false}, - {kSampleRate16kHz, 1, false}, - {kSampleRate16kHz, 1, false}}}), - fwd_proc_format_(kSampleRate16kHz), - rev_proc_format_(kSampleRate16kHz, 1), - split_rate_(kSampleRate16kHz), - stream_delay_ms_(0), - delay_offset_ms_(0), - was_stream_delay_set_(false), - last_stream_delay_ms_(0), - last_aec_system_delay_ms_(0), - stream_delay_jumps_(-1), - aec_system_delay_jumps_(-1), - output_will_be_muted_(false), - key_pressed_(false), + : public_submodules_(new ApmPublicSubmodules()), + private_submodules_(new ApmPrivateSubmodules(beamformer)), + constants_(config.Get<ExperimentalAgc>().startup_min_volume, #if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS) - use_new_agc_(false), + false, #else - use_new_agc_(config.Get<ExperimentalAgc>().enabled), + config.Get<ExperimentalAgc>().enabled, #endif - agc_startup_min_volume_(config.Get<ExperimentalAgc>().startup_min_volume), + config.Get<Intelligibility>().enabled), + #if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS) - transient_suppressor_enabled_(false), + capture_(false, #else - transient_suppressor_enabled_(config.Get<ExperimentalNs>().enabled), + capture_(config.Get<ExperimentalNs>().enabled, #endif - beamformer_enabled_(config.Get<Beamforming>().enabled), - beamformer_(beamformer), - array_geometry_(config.Get<Beamforming>().array_geometry), - target_direction_(config.Get<Beamforming>().target_direction), - intelligibility_enabled_(config.Get<Intelligibility>().enabled) { - echo_cancellation_ = new EchoCancellationImpl(this, crit_); - component_list_.push_back(echo_cancellation_); - - echo_control_mobile_ = new EchoControlMobileImpl(this, crit_); - component_list_.push_back(echo_control_mobile_); - - gain_control_ = new GainControlImpl(this, crit_); - component_list_.push_back(gain_control_); - - high_pass_filter_ = new HighPassFilterImpl(this, crit_); - component_list_.push_back(high_pass_filter_); - - level_estimator_ = new LevelEstimatorImpl(this, crit_); - component_list_.push_back(level_estimator_); - - noise_suppression_ = new NoiseSuppressionImpl(this, crit_); - component_list_.push_back(noise_suppression_); - - voice_detection_ = new VoiceDetectionImpl(this, crit_); - component_list_.push_back(voice_detection_); - - gain_control_for_new_agc_.reset(new GainControlForNewAgc(gain_control_)); + config.Get<Beamforming>().array_geometry, + config.Get<Beamforming>().target_direction), + capture_nonlocked_(config.Get<Beamforming>().enabled) +{ + { + rtc::CritScope cs_render(&crit_render_); + rtc::CritScope cs_capture(&crit_capture_); + + public_submodules_->echo_cancellation = + new EchoCancellationImpl(this, &crit_render_, &crit_capture_); + public_submodules_->echo_control_mobile = + new EchoControlMobileImpl(this, &crit_render_, &crit_capture_); + public_submodules_->gain_control = + new GainControlImpl(this, &crit_capture_, &crit_capture_); + public_submodules_->high_pass_filter.reset( + new HighPassFilterImpl(&crit_capture_)); + public_submodules_->level_estimator.reset( + new LevelEstimatorImpl(&crit_capture_)); + public_submodules_->noise_suppression.reset( + new NoiseSuppressionImpl(&crit_capture_)); + public_submodules_->voice_detection.reset( + new VoiceDetectionImpl(&crit_capture_)); + public_submodules_->gain_control_for_new_agc.reset( + new GainControlForNewAgc(public_submodules_->gain_control)); + + private_submodules_->component_list.push_back( + public_submodules_->echo_cancellation); + private_submodules_->component_list.push_back( + public_submodules_->echo_control_mobile); + private_submodules_->component_list.push_back( + public_submodules_->gain_control); + } SetExtraOptions(config); } AudioProcessingImpl::~AudioProcessingImpl() { - { - CriticalSectionScoped crit_scoped(crit_); - // Depends on gain_control_ and gain_control_for_new_agc_. - agc_manager_.reset(); - // Depends on gain_control_. - gain_control_for_new_agc_.reset(); - while (!component_list_.empty()) { - ProcessingComponent* component = component_list_.front(); - component->Destroy(); - delete component; - component_list_.pop_front(); - } + // Depends on gain_control_ and + // public_submodules_->gain_control_for_new_agc. + private_submodules_->agc_manager.reset(); + // Depends on gain_control_. + public_submodules_->gain_control_for_new_agc.reset(); + while (!private_submodules_->component_list.empty()) { + ProcessingComponent* component = + private_submodules_->component_list.front(); + component->Destroy(); + delete component; + private_submodules_->component_list.pop_front(); + } #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP - if (debug_file_->Open()) { - debug_file_->CloseFile(); - } -#endif + if (debug_dump_.debug_file->Open()) { + debug_dump_.debug_file->CloseFile(); } - delete crit_; - crit_ = NULL; +#endif } int AudioProcessingImpl::Initialize() { - CriticalSectionScoped crit_scoped(crit_); + // Run in a single-threaded manner during initialization. + rtc::CritScope cs_render(&crit_render_); + rtc::CritScope cs_capture(&crit_capture_); return InitializeLocked(); } @@ -306,44 +314,73 @@ int AudioProcessingImpl::Initialize(int input_sample_rate_hz, } int AudioProcessingImpl::Initialize(const ProcessingConfig& processing_config) { - CriticalSectionScoped crit_scoped(crit_); + // Run in a single-threaded manner during initialization. + rtc::CritScope cs_render(&crit_render_); + rtc::CritScope cs_capture(&crit_capture_); + return InitializeLocked(processing_config); +} + +int AudioProcessingImpl::MaybeInitializeRender( + const ProcessingConfig& processing_config) { + return MaybeInitialize(processing_config); +} + +int AudioProcessingImpl::MaybeInitializeCapture( + const ProcessingConfig& processing_config) { + return MaybeInitialize(processing_config); +} + +// Calls InitializeLocked() if any of the audio parameters have changed from +// their current values (needs to be called while holding the crit_render_lock). +int AudioProcessingImpl::MaybeInitialize( + const ProcessingConfig& processing_config) { + // Called from both threads. Thread check is therefore not possible. + if (processing_config == formats_.api_format) { + return kNoError; + } + + rtc::CritScope cs_capture(&crit_capture_); return InitializeLocked(processing_config); } int AudioProcessingImpl::InitializeLocked() { const int fwd_audio_buffer_channels = - beamformer_enabled_ ? api_format_.input_stream().num_channels() - : api_format_.output_stream().num_channels(); + capture_nonlocked_.beamformer_enabled + ? formats_.api_format.input_stream().num_channels() + : formats_.api_format.output_stream().num_channels(); const int rev_audio_buffer_out_num_frames = - api_format_.reverse_output_stream().num_frames() == 0 - ? rev_proc_format_.num_frames() - : api_format_.reverse_output_stream().num_frames(); - if (api_format_.reverse_input_stream().num_channels() > 0) { - render_audio_.reset(new AudioBuffer( - api_format_.reverse_input_stream().num_frames(), - api_format_.reverse_input_stream().num_channels(), - rev_proc_format_.num_frames(), rev_proc_format_.num_channels(), + formats_.api_format.reverse_output_stream().num_frames() == 0 + ? formats_.rev_proc_format.num_frames() + : formats_.api_format.reverse_output_stream().num_frames(); + if (formats_.api_format.reverse_input_stream().num_channels() > 0) { + render_.render_audio.reset(new AudioBuffer( + formats_.api_format.reverse_input_stream().num_frames(), + formats_.api_format.reverse_input_stream().num_channels(), + formats_.rev_proc_format.num_frames(), + formats_.rev_proc_format.num_channels(), rev_audio_buffer_out_num_frames)); if (rev_conversion_needed()) { - render_converter_ = AudioConverter::Create( - api_format_.reverse_input_stream().num_channels(), - api_format_.reverse_input_stream().num_frames(), - api_format_.reverse_output_stream().num_channels(), - api_format_.reverse_output_stream().num_frames()); + render_.render_converter = AudioConverter::Create( + formats_.api_format.reverse_input_stream().num_channels(), + formats_.api_format.reverse_input_stream().num_frames(), + formats_.api_format.reverse_output_stream().num_channels(), + formats_.api_format.reverse_output_stream().num_frames()); } else { - render_converter_.reset(nullptr); + render_.render_converter.reset(nullptr); } } else { - render_audio_.reset(nullptr); - render_converter_.reset(nullptr); + render_.render_audio.reset(nullptr); + render_.render_converter.reset(nullptr); } - capture_audio_.reset(new AudioBuffer( - api_format_.input_stream().num_frames(), - api_format_.input_stream().num_channels(), fwd_proc_format_.num_frames(), - fwd_audio_buffer_channels, api_format_.output_stream().num_frames())); + capture_.capture_audio.reset( + new AudioBuffer(formats_.api_format.input_stream().num_frames(), + formats_.api_format.input_stream().num_channels(), + capture_nonlocked_.fwd_proc_format.num_frames(), + fwd_audio_buffer_channels, + formats_.api_format.output_stream().num_frames())); // Initialize all components. - for (auto item : component_list_) { + for (auto item : private_submodules_->component_list) { int err = item->Initialize(); if (err != kNoError) { return err; @@ -351,15 +388,16 @@ int AudioProcessingImpl::InitializeLocked() { } InitializeExperimentalAgc(); - InitializeTransient(); - InitializeBeamformer(); - InitializeIntelligibility(); + InitializeHighPassFilter(); + InitializeNoiseSuppression(); + InitializeLevelEstimator(); + InitializeVoiceDetection(); #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP - if (debug_file_->Open()) { + if (debug_dump_.debug_file->Open()) { int err = WriteInitMessage(); if (err != kNoError) { return err; @@ -372,16 +410,13 @@ int AudioProcessingImpl::InitializeLocked() { int AudioProcessingImpl::InitializeLocked(const ProcessingConfig& config) { for (const auto& stream : config.streams) { - if (stream.num_channels() < 0) { - return kBadNumberChannelsError; - } if (stream.num_channels() > 0 && stream.sample_rate_hz() <= 0) { return kBadSampleRateError; } } - const int num_in_channels = config.input_stream().num_channels(); - const int num_out_channels = config.output_stream().num_channels(); + const size_t num_in_channels = config.input_stream().num_channels(); + const size_t num_out_channels = config.output_stream().num_channels(); // Need at least one input channel. // Need either one output channel or as many outputs as there are inputs. @@ -390,18 +425,17 @@ int AudioProcessingImpl::InitializeLocked(const ProcessingConfig& config) { return kBadNumberChannelsError; } - if (beamformer_enabled_ && - (static_cast<size_t>(num_in_channels) != array_geometry_.size() || - num_out_channels > 1)) { + if (capture_nonlocked_.beamformer_enabled && + num_in_channels != capture_.array_geometry.size()) { return kBadNumberChannelsError; } - api_format_ = config; + formats_.api_format = config; // We process at the closest native rate >= min(input rate, output rate)... const int min_proc_rate = - std::min(api_format_.input_stream().sample_rate_hz(), - api_format_.output_stream().sample_rate_hz()); + std::min(formats_.api_format.input_stream().sample_rate_hz(), + formats_.api_format.output_stream().sample_rate_hz()); int fwd_proc_rate; for (size_t i = 0; i < kNumNativeSampleRates; ++i) { fwd_proc_rate = kNativeSampleRatesHz[i]; @@ -410,20 +444,20 @@ int AudioProcessingImpl::InitializeLocked(const ProcessingConfig& config) { } } // ...with one exception. - if (echo_control_mobile_->is_enabled() && + if (public_submodules_->echo_control_mobile->is_enabled() && min_proc_rate > kMaxAECMSampleRateHz) { fwd_proc_rate = kMaxAECMSampleRateHz; } - fwd_proc_format_ = StreamConfig(fwd_proc_rate); + capture_nonlocked_.fwd_proc_format = StreamConfig(fwd_proc_rate); // We normally process the reverse stream at 16 kHz. Unless... int rev_proc_rate = kSampleRate16kHz; - if (fwd_proc_format_.sample_rate_hz() == kSampleRate8kHz) { + if (capture_nonlocked_.fwd_proc_format.sample_rate_hz() == kSampleRate8kHz) { // ...the forward stream is at 8 kHz. rev_proc_rate = kSampleRate8kHz; } else { - if (api_format_.reverse_input_stream().sample_rate_hz() == + if (formats_.api_format.reverse_input_stream().sample_rate_hz() == kSampleRate32kHz) { // ...or the input is at 32 kHz, in which case we use the splitting // filter rather than the resampler. @@ -433,66 +467,89 @@ int AudioProcessingImpl::InitializeLocked(const ProcessingConfig& config) { // Always downmix the reverse stream to mono for analysis. This has been // demonstrated to work well for AEC in most practical scenarios. - rev_proc_format_ = StreamConfig(rev_proc_rate, 1); + formats_.rev_proc_format = StreamConfig(rev_proc_rate, 1); - if (fwd_proc_format_.sample_rate_hz() == kSampleRate32kHz || - fwd_proc_format_.sample_rate_hz() == kSampleRate48kHz) { - split_rate_ = kSampleRate16kHz; + if (capture_nonlocked_.fwd_proc_format.sample_rate_hz() == kSampleRate32kHz || + capture_nonlocked_.fwd_proc_format.sample_rate_hz() == kSampleRate48kHz) { + capture_nonlocked_.split_rate = kSampleRate16kHz; } else { - split_rate_ = fwd_proc_format_.sample_rate_hz(); + capture_nonlocked_.split_rate = + capture_nonlocked_.fwd_proc_format.sample_rate_hz(); } return InitializeLocked(); } -// Calls InitializeLocked() if any of the audio parameters have changed from -// their current values. -int AudioProcessingImpl::MaybeInitializeLocked( - const ProcessingConfig& processing_config) { - if (processing_config == api_format_) { - return kNoError; - } - return InitializeLocked(processing_config); -} - void AudioProcessingImpl::SetExtraOptions(const Config& config) { - CriticalSectionScoped crit_scoped(crit_); - for (auto item : component_list_) { + // Run in a single-threaded manner when setting the extra options. + rtc::CritScope cs_render(&crit_render_); + rtc::CritScope cs_capture(&crit_capture_); + for (auto item : private_submodules_->component_list) { item->SetExtraOptions(config); } - if (transient_suppressor_enabled_ != config.Get<ExperimentalNs>().enabled) { - transient_suppressor_enabled_ = config.Get<ExperimentalNs>().enabled; + if (capture_.transient_suppressor_enabled != + config.Get<ExperimentalNs>().enabled) { + capture_.transient_suppressor_enabled = + config.Get<ExperimentalNs>().enabled; InitializeTransient(); } + +#ifdef WEBRTC_ANDROID_PLATFORM_BUILD + if (capture_nonlocked_.beamformer_enabled != + config.Get<Beamforming>().enabled) { + capture_nonlocked_.beamformer_enabled = config.Get<Beamforming>().enabled; + if (config.Get<Beamforming>().array_geometry.size() > 1) { + capture_.array_geometry = config.Get<Beamforming>().array_geometry; + } + capture_.target_direction = config.Get<Beamforming>().target_direction; + InitializeBeamformer(); + } +#endif // WEBRTC_ANDROID_PLATFORM_BUILD } +int AudioProcessingImpl::input_sample_rate_hz() const { + // Accessed from outside APM, hence a lock is needed. + rtc::CritScope cs(&crit_capture_); + return formats_.api_format.input_stream().sample_rate_hz(); +} int AudioProcessingImpl::proc_sample_rate_hz() const { - return fwd_proc_format_.sample_rate_hz(); + // Used as callback from submodules, hence locking is not allowed. + return capture_nonlocked_.fwd_proc_format.sample_rate_hz(); } int AudioProcessingImpl::proc_split_sample_rate_hz() const { - return split_rate_; + // Used as callback from submodules, hence locking is not allowed. + return capture_nonlocked_.split_rate; +} + +size_t AudioProcessingImpl::num_reverse_channels() const { + // Used as callback from submodules, hence locking is not allowed. + return formats_.rev_proc_format.num_channels(); } -int AudioProcessingImpl::num_reverse_channels() const { - return rev_proc_format_.num_channels(); +size_t AudioProcessingImpl::num_input_channels() const { + // Used as callback from submodules, hence locking is not allowed. + return formats_.api_format.input_stream().num_channels(); } -int AudioProcessingImpl::num_input_channels() const { - return api_format_.input_stream().num_channels(); +size_t AudioProcessingImpl::num_proc_channels() const { + // Used as callback from submodules, hence locking is not allowed. + return capture_nonlocked_.beamformer_enabled ? 1 : num_output_channels(); } -int AudioProcessingImpl::num_output_channels() const { - return api_format_.output_stream().num_channels(); +size_t AudioProcessingImpl::num_output_channels() const { + // Used as callback from submodules, hence locking is not allowed. + return formats_.api_format.output_stream().num_channels(); } void AudioProcessingImpl::set_output_will_be_muted(bool muted) { - CriticalSectionScoped lock(crit_); - output_will_be_muted_ = muted; - if (agc_manager_.get()) { - agc_manager_->SetCaptureMuted(output_will_be_muted_); + rtc::CritScope cs(&crit_capture_); + capture_.output_will_be_muted = muted; + if (private_submodules_->agc_manager.get()) { + private_submodules_->agc_manager->SetCaptureMuted( + capture_.output_will_be_muted); } } @@ -504,13 +561,21 @@ int AudioProcessingImpl::ProcessStream(const float* const* src, int output_sample_rate_hz, ChannelLayout output_layout, float* const* dest) { - CriticalSectionScoped crit_scoped(crit_); - StreamConfig input_stream = api_format_.input_stream(); + TRACE_EVENT0("webrtc", "AudioProcessing::ProcessStream_ChannelLayout"); + StreamConfig input_stream; + StreamConfig output_stream; + { + // Access the formats_.api_format.input_stream beneath the capture lock. + // The lock must be released as it is later required in the call + // to ProcessStream(,,,); + rtc::CritScope cs(&crit_capture_); + input_stream = formats_.api_format.input_stream(); + output_stream = formats_.api_format.output_stream(); + } + input_stream.set_sample_rate_hz(input_sample_rate_hz); input_stream.set_num_channels(ChannelsFromLayout(input_layout)); input_stream.set_has_keyboard(LayoutHasKeyboard(input_layout)); - - StreamConfig output_stream = api_format_.output_stream(); output_stream.set_sample_rate_hz(output_sample_rate_hz); output_stream.set_num_channels(ChannelsFromLayout(output_layout)); output_stream.set_has_keyboard(LayoutHasKeyboard(output_layout)); @@ -525,44 +590,64 @@ int AudioProcessingImpl::ProcessStream(const float* const* src, const StreamConfig& input_config, const StreamConfig& output_config, float* const* dest) { - CriticalSectionScoped crit_scoped(crit_); - if (!src || !dest) { - return kNullPointerError; + TRACE_EVENT0("webrtc", "AudioProcessing::ProcessStream_StreamConfig"); + ProcessingConfig processing_config; + { + // Acquire the capture lock in order to safely call the function + // that retrieves the render side data. This function accesses apm + // getters that need the capture lock held when being called. + rtc::CritScope cs_capture(&crit_capture_); + public_submodules_->echo_cancellation->ReadQueuedRenderData(); + public_submodules_->echo_control_mobile->ReadQueuedRenderData(); + public_submodules_->gain_control->ReadQueuedRenderData(); + + if (!src || !dest) { + return kNullPointerError; + } + + processing_config = formats_.api_format; } - ProcessingConfig processing_config = api_format_; processing_config.input_stream() = input_config; processing_config.output_stream() = output_config; - RETURN_ON_ERR(MaybeInitializeLocked(processing_config)); + { + // Do conditional reinitialization. + rtc::CritScope cs_render(&crit_render_); + RETURN_ON_ERR(MaybeInitializeCapture(processing_config)); + } + rtc::CritScope cs_capture(&crit_capture_); assert(processing_config.input_stream().num_frames() == - api_format_.input_stream().num_frames()); + formats_.api_format.input_stream().num_frames()); #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP - if (debug_file_->Open()) { + if (debug_dump_.debug_file->Open()) { RETURN_ON_ERR(WriteConfigMessage(false)); - event_msg_->set_type(audioproc::Event::STREAM); - audioproc::Stream* msg = event_msg_->mutable_stream(); + debug_dump_.capture.event_msg->set_type(audioproc::Event::STREAM); + audioproc::Stream* msg = debug_dump_.capture.event_msg->mutable_stream(); const size_t channel_size = - sizeof(float) * api_format_.input_stream().num_frames(); - for (int i = 0; i < api_format_.input_stream().num_channels(); ++i) + sizeof(float) * formats_.api_format.input_stream().num_frames(); + for (size_t i = 0; i < formats_.api_format.input_stream().num_channels(); + ++i) msg->add_input_channel(src[i], channel_size); } #endif - capture_audio_->CopyFrom(src, api_format_.input_stream()); + capture_.capture_audio->CopyFrom(src, formats_.api_format.input_stream()); RETURN_ON_ERR(ProcessStreamLocked()); - capture_audio_->CopyTo(api_format_.output_stream(), dest); + capture_.capture_audio->CopyTo(formats_.api_format.output_stream(), dest); #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP - if (debug_file_->Open()) { - audioproc::Stream* msg = event_msg_->mutable_stream(); + if (debug_dump_.debug_file->Open()) { + audioproc::Stream* msg = debug_dump_.capture.event_msg->mutable_stream(); const size_t channel_size = - sizeof(float) * api_format_.output_stream().num_frames(); - for (int i = 0; i < api_format_.output_stream().num_channels(); ++i) + sizeof(float) * formats_.api_format.output_stream().num_frames(); + for (size_t i = 0; i < formats_.api_format.output_stream().num_channels(); + ++i) msg->add_output_channel(dest[i], channel_size); - RETURN_ON_ERR(WriteMessageToDebugFile()); + RETURN_ON_ERR(WriteMessageToDebugFile(debug_dump_.debug_file.get(), + &crit_debug_, &debug_dump_.capture)); } #endif @@ -570,7 +655,20 @@ int AudioProcessingImpl::ProcessStream(const float* const* src, } int AudioProcessingImpl::ProcessStream(AudioFrame* frame) { - CriticalSectionScoped crit_scoped(crit_); + TRACE_EVENT0("webrtc", "AudioProcessing::ProcessStream_AudioFrame"); + { + // Acquire the capture lock in order to safely call the function + // that retrieves the render side data. This function accesses apm + // getters that need the capture lock held when being called. + // The lock needs to be released as + // public_submodules_->echo_control_mobile->is_enabled() aquires this lock + // as well. + rtc::CritScope cs_capture(&crit_capture_); + public_submodules_->echo_cancellation->ReadQueuedRenderData(); + public_submodules_->echo_control_mobile->ReadQueuedRenderData(); + public_submodules_->gain_control->ReadQueuedRenderData(); + } + if (!frame) { return kNullPointerError; } @@ -581,46 +679,62 @@ int AudioProcessingImpl::ProcessStream(AudioFrame* frame) { frame->sample_rate_hz_ != kSampleRate48kHz) { return kBadSampleRateError; } - if (echo_control_mobile_->is_enabled() && + + if (public_submodules_->echo_control_mobile->is_enabled() && frame->sample_rate_hz_ > kMaxAECMSampleRateHz) { LOG(LS_ERROR) << "AECM only supports 16 or 8 kHz sample rates"; return kUnsupportedComponentError; } - // TODO(ajm): The input and output rates and channels are currently - // constrained to be identical in the int16 interface. - ProcessingConfig processing_config = api_format_; + ProcessingConfig processing_config; + { + // Aquire lock for the access of api_format. + // The lock is released immediately due to the conditional + // reinitialization. + rtc::CritScope cs_capture(&crit_capture_); + // TODO(ajm): The input and output rates and channels are currently + // constrained to be identical in the int16 interface. + processing_config = formats_.api_format; + } processing_config.input_stream().set_sample_rate_hz(frame->sample_rate_hz_); processing_config.input_stream().set_num_channels(frame->num_channels_); processing_config.output_stream().set_sample_rate_hz(frame->sample_rate_hz_); processing_config.output_stream().set_num_channels(frame->num_channels_); - RETURN_ON_ERR(MaybeInitializeLocked(processing_config)); - if (frame->samples_per_channel_ != api_format_.input_stream().num_frames()) { + { + // Do conditional reinitialization. + rtc::CritScope cs_render(&crit_render_); + RETURN_ON_ERR(MaybeInitializeCapture(processing_config)); + } + rtc::CritScope cs_capture(&crit_capture_); + if (frame->samples_per_channel_ != + formats_.api_format.input_stream().num_frames()) { return kBadDataLengthError; } #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP - if (debug_file_->Open()) { - event_msg_->set_type(audioproc::Event::STREAM); - audioproc::Stream* msg = event_msg_->mutable_stream(); + if (debug_dump_.debug_file->Open()) { + debug_dump_.capture.event_msg->set_type(audioproc::Event::STREAM); + audioproc::Stream* msg = debug_dump_.capture.event_msg->mutable_stream(); const size_t data_size = sizeof(int16_t) * frame->samples_per_channel_ * frame->num_channels_; msg->set_input_data(frame->data_, data_size); } #endif - capture_audio_->DeinterleaveFrom(frame); + capture_.capture_audio->DeinterleaveFrom(frame); RETURN_ON_ERR(ProcessStreamLocked()); - capture_audio_->InterleaveTo(frame, output_copy_needed(is_data_processed())); + capture_.capture_audio->InterleaveTo(frame, + output_copy_needed(is_data_processed())); #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP - if (debug_file_->Open()) { - audioproc::Stream* msg = event_msg_->mutable_stream(); + if (debug_dump_.debug_file->Open()) { + audioproc::Stream* msg = debug_dump_.capture.event_msg->mutable_stream(); const size_t data_size = sizeof(int16_t) * frame->samples_per_channel_ * frame->num_channels_; msg->set_output_data(frame->data_, data_size); - RETURN_ON_ERR(WriteMessageToDebugFile()); + RETURN_ON_ERR(WriteMessageToDebugFile(debug_dump_.debug_file.get(), + &crit_debug_, &debug_dump_.capture)); } #endif @@ -629,22 +743,25 @@ int AudioProcessingImpl::ProcessStream(AudioFrame* frame) { int AudioProcessingImpl::ProcessStreamLocked() { #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP - if (debug_file_->Open()) { - audioproc::Stream* msg = event_msg_->mutable_stream(); - msg->set_delay(stream_delay_ms_); - msg->set_drift(echo_cancellation_->stream_drift_samples()); + if (debug_dump_.debug_file->Open()) { + audioproc::Stream* msg = debug_dump_.capture.event_msg->mutable_stream(); + msg->set_delay(capture_nonlocked_.stream_delay_ms); + msg->set_drift( + public_submodules_->echo_cancellation->stream_drift_samples()); msg->set_level(gain_control()->stream_analog_level()); - msg->set_keypress(key_pressed_); + msg->set_keypress(capture_.key_pressed); } #endif MaybeUpdateHistograms(); - AudioBuffer* ca = capture_audio_.get(); // For brevity. + AudioBuffer* ca = capture_.capture_audio.get(); // For brevity. - if (use_new_agc_ && gain_control_->is_enabled()) { - agc_manager_->AnalyzePreProcess(ca->channels()[0], ca->num_channels(), - fwd_proc_format_.num_frames()); + if (constants_.use_new_agc && + public_submodules_->gain_control->is_enabled()) { + private_submodules_->agc_manager->AnalyzePreProcess( + ca->channels()[0], ca->num_channels(), + capture_nonlocked_.fwd_proc_format.num_frames()); } bool data_processed = is_data_processed(); @@ -652,34 +769,41 @@ int AudioProcessingImpl::ProcessStreamLocked() { ca->SplitIntoFrequencyBands(); } - if (intelligibility_enabled_) { - intelligibility_enhancer_->AnalyzeCaptureAudio( - ca->split_channels_f(kBand0To8kHz), split_rate_, ca->num_channels()); + if (constants_.intelligibility_enabled) { + public_submodules_->intelligibility_enhancer->AnalyzeCaptureAudio( + ca->split_channels_f(kBand0To8kHz), capture_nonlocked_.split_rate, + ca->num_channels()); } - if (beamformer_enabled_) { - beamformer_->ProcessChunk(*ca->split_data_f(), ca->split_data_f()); + if (capture_nonlocked_.beamformer_enabled) { + private_submodules_->beamformer->ProcessChunk(*ca->split_data_f(), + ca->split_data_f()); ca->set_num_channels(1); } - RETURN_ON_ERR(high_pass_filter_->ProcessCaptureAudio(ca)); - RETURN_ON_ERR(gain_control_->AnalyzeCaptureAudio(ca)); - RETURN_ON_ERR(noise_suppression_->AnalyzeCaptureAudio(ca)); - RETURN_ON_ERR(echo_cancellation_->ProcessCaptureAudio(ca)); + public_submodules_->high_pass_filter->ProcessCaptureAudio(ca); + RETURN_ON_ERR(public_submodules_->gain_control->AnalyzeCaptureAudio(ca)); + public_submodules_->noise_suppression->AnalyzeCaptureAudio(ca); + RETURN_ON_ERR(public_submodules_->echo_cancellation->ProcessCaptureAudio(ca)); - if (echo_control_mobile_->is_enabled() && noise_suppression_->is_enabled()) { + if (public_submodules_->echo_control_mobile->is_enabled() && + public_submodules_->noise_suppression->is_enabled()) { ca->CopyLowPassToReference(); } - RETURN_ON_ERR(noise_suppression_->ProcessCaptureAudio(ca)); - RETURN_ON_ERR(echo_control_mobile_->ProcessCaptureAudio(ca)); - RETURN_ON_ERR(voice_detection_->ProcessCaptureAudio(ca)); + public_submodules_->noise_suppression->ProcessCaptureAudio(ca); + RETURN_ON_ERR( + public_submodules_->echo_control_mobile->ProcessCaptureAudio(ca)); + public_submodules_->voice_detection->ProcessCaptureAudio(ca); - if (use_new_agc_ && gain_control_->is_enabled() && - (!beamformer_enabled_ || beamformer_->is_target_present())) { - agc_manager_->Process(ca->split_bands_const(0)[kBand0To8kHz], - ca->num_frames_per_band(), split_rate_); + if (constants_.use_new_agc && + public_submodules_->gain_control->is_enabled() && + (!capture_nonlocked_.beamformer_enabled || + private_submodules_->beamformer->is_target_present())) { + private_submodules_->agc_manager->Process( + ca->split_bands_const(0)[kBand0To8kHz], ca->num_frames_per_band(), + capture_nonlocked_.split_rate); } - RETURN_ON_ERR(gain_control_->ProcessCaptureAudio(ca)); + RETURN_ON_ERR(public_submodules_->gain_control->ProcessCaptureAudio(ca)); if (synthesis_needed(data_processed)) { ca->MergeFrequencyBands(); @@ -687,21 +811,23 @@ int AudioProcessingImpl::ProcessStreamLocked() { // TODO(aluebs): Investigate if the transient suppression placement should be // before or after the AGC. - if (transient_suppressor_enabled_) { + if (capture_.transient_suppressor_enabled) { float voice_probability = - agc_manager_.get() ? agc_manager_->voice_probability() : 1.f; + private_submodules_->agc_manager.get() + ? private_submodules_->agc_manager->voice_probability() + : 1.f; - transient_suppressor_->Suppress( + public_submodules_->transient_suppressor->Suppress( ca->channels_f()[0], ca->num_frames(), ca->num_channels(), ca->split_bands_const_f(0)[kBand0To8kHz], ca->num_frames_per_band(), ca->keyboard_data(), ca->num_keyboard_frames(), voice_probability, - key_pressed_); + capture_.key_pressed); } // The level estimator operates on the recombined data. - RETURN_ON_ERR(level_estimator_->ProcessStream(ca)); + public_submodules_->level_estimator->ProcessStream(ca); - was_stream_delay_set_ = false; + capture_.was_stream_delay_set = false; return kNoError; } @@ -709,13 +835,15 @@ int AudioProcessingImpl::AnalyzeReverseStream(const float* const* data, size_t samples_per_channel, int rev_sample_rate_hz, ChannelLayout layout) { + TRACE_EVENT0("webrtc", "AudioProcessing::AnalyzeReverseStream_ChannelLayout"); + rtc::CritScope cs(&crit_render_); const StreamConfig reverse_config = { rev_sample_rate_hz, ChannelsFromLayout(layout), LayoutHasKeyboard(layout), }; if (samples_per_channel != reverse_config.num_frames()) { return kBadDataLengthError; } - return AnalyzeReverseStream(data, reverse_config, reverse_config); + return AnalyzeReverseStreamLocked(data, reverse_config, reverse_config); } int AudioProcessingImpl::ProcessReverseStream( @@ -723,13 +851,17 @@ int AudioProcessingImpl::ProcessReverseStream( const StreamConfig& reverse_input_config, const StreamConfig& reverse_output_config, float* const* dest) { - RETURN_ON_ERR( - AnalyzeReverseStream(src, reverse_input_config, reverse_output_config)); + TRACE_EVENT0("webrtc", "AudioProcessing::ProcessReverseStream_StreamConfig"); + rtc::CritScope cs(&crit_render_); + RETURN_ON_ERR(AnalyzeReverseStreamLocked(src, reverse_input_config, + reverse_output_config)); if (is_rev_processed()) { - render_audio_->CopyTo(api_format_.reverse_output_stream(), dest); - } else if (rev_conversion_needed()) { - render_converter_->Convert(src, reverse_input_config.num_samples(), dest, - reverse_output_config.num_samples()); + render_.render_audio->CopyTo(formats_.api_format.reverse_output_stream(), + dest); + } else if (render_check_rev_conversion_needed()) { + render_.render_converter->Convert(src, reverse_input_config.num_samples(), + dest, + reverse_output_config.num_samples()); } else { CopyAudioIfNeeded(src, reverse_input_config.num_frames(), reverse_input_config.num_channels(), dest); @@ -738,55 +870,61 @@ int AudioProcessingImpl::ProcessReverseStream( return kNoError; } -int AudioProcessingImpl::AnalyzeReverseStream( +int AudioProcessingImpl::AnalyzeReverseStreamLocked( const float* const* src, const StreamConfig& reverse_input_config, const StreamConfig& reverse_output_config) { - CriticalSectionScoped crit_scoped(crit_); - if (src == NULL) { + if (src == nullptr) { return kNullPointerError; } - if (reverse_input_config.num_channels() <= 0) { + if (reverse_input_config.num_channels() == 0) { return kBadNumberChannelsError; } - ProcessingConfig processing_config = api_format_; + ProcessingConfig processing_config = formats_.api_format; processing_config.reverse_input_stream() = reverse_input_config; processing_config.reverse_output_stream() = reverse_output_config; - RETURN_ON_ERR(MaybeInitializeLocked(processing_config)); + RETURN_ON_ERR(MaybeInitializeRender(processing_config)); assert(reverse_input_config.num_frames() == - api_format_.reverse_input_stream().num_frames()); + formats_.api_format.reverse_input_stream().num_frames()); #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP - if (debug_file_->Open()) { - event_msg_->set_type(audioproc::Event::REVERSE_STREAM); - audioproc::ReverseStream* msg = event_msg_->mutable_reverse_stream(); + if (debug_dump_.debug_file->Open()) { + debug_dump_.render.event_msg->set_type(audioproc::Event::REVERSE_STREAM); + audioproc::ReverseStream* msg = + debug_dump_.render.event_msg->mutable_reverse_stream(); const size_t channel_size = - sizeof(float) * api_format_.reverse_input_stream().num_frames(); - for (int i = 0; i < api_format_.reverse_input_stream().num_channels(); ++i) + sizeof(float) * formats_.api_format.reverse_input_stream().num_frames(); + for (size_t i = 0; + i < formats_.api_format.reverse_input_stream().num_channels(); ++i) msg->add_channel(src[i], channel_size); - RETURN_ON_ERR(WriteMessageToDebugFile()); + RETURN_ON_ERR(WriteMessageToDebugFile(debug_dump_.debug_file.get(), + &crit_debug_, &debug_dump_.render)); } #endif - render_audio_->CopyFrom(src, api_format_.reverse_input_stream()); + render_.render_audio->CopyFrom(src, + formats_.api_format.reverse_input_stream()); return ProcessReverseStreamLocked(); } int AudioProcessingImpl::ProcessReverseStream(AudioFrame* frame) { + TRACE_EVENT0("webrtc", "AudioProcessing::ProcessReverseStream_AudioFrame"); RETURN_ON_ERR(AnalyzeReverseStream(frame)); + rtc::CritScope cs(&crit_render_); if (is_rev_processed()) { - render_audio_->InterleaveTo(frame, true); + render_.render_audio->InterleaveTo(frame, true); } return kNoError; } int AudioProcessingImpl::AnalyzeReverseStream(AudioFrame* frame) { - CriticalSectionScoped crit_scoped(crit_); - if (frame == NULL) { + TRACE_EVENT0("webrtc", "AudioProcessing::AnalyzeReverseStream_AudioFrame"); + rtc::CritScope cs(&crit_render_); + if (frame == nullptr) { return kNullPointerError; } // Must be a native rate. @@ -797,7 +935,8 @@ int AudioProcessingImpl::AnalyzeReverseStream(AudioFrame* frame) { return kBadSampleRateError; } // This interface does not tolerate different forward and reverse rates. - if (frame->sample_rate_hz_ != api_format_.input_stream().sample_rate_hz()) { + if (frame->sample_rate_hz_ != + formats_.api_format.input_stream().sample_rate_hz()) { return kBadSampleRateError; } @@ -805,7 +944,7 @@ int AudioProcessingImpl::AnalyzeReverseStream(AudioFrame* frame) { return kBadNumberChannelsError; } - ProcessingConfig processing_config = api_format_; + ProcessingConfig processing_config = formats_.api_format; processing_config.reverse_input_stream().set_sample_rate_hz( frame->sample_rate_hz_); processing_config.reverse_input_stream().set_num_channels( @@ -815,44 +954,52 @@ int AudioProcessingImpl::AnalyzeReverseStream(AudioFrame* frame) { processing_config.reverse_output_stream().set_num_channels( frame->num_channels_); - RETURN_ON_ERR(MaybeInitializeLocked(processing_config)); + RETURN_ON_ERR(MaybeInitializeRender(processing_config)); if (frame->samples_per_channel_ != - api_format_.reverse_input_stream().num_frames()) { + formats_.api_format.reverse_input_stream().num_frames()) { return kBadDataLengthError; } #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP - if (debug_file_->Open()) { - event_msg_->set_type(audioproc::Event::REVERSE_STREAM); - audioproc::ReverseStream* msg = event_msg_->mutable_reverse_stream(); + if (debug_dump_.debug_file->Open()) { + debug_dump_.render.event_msg->set_type(audioproc::Event::REVERSE_STREAM); + audioproc::ReverseStream* msg = + debug_dump_.render.event_msg->mutable_reverse_stream(); const size_t data_size = sizeof(int16_t) * frame->samples_per_channel_ * frame->num_channels_; msg->set_data(frame->data_, data_size); - RETURN_ON_ERR(WriteMessageToDebugFile()); + RETURN_ON_ERR(WriteMessageToDebugFile(debug_dump_.debug_file.get(), + &crit_debug_, &debug_dump_.render)); } #endif - render_audio_->DeinterleaveFrom(frame); + render_.render_audio->DeinterleaveFrom(frame); return ProcessReverseStreamLocked(); } int AudioProcessingImpl::ProcessReverseStreamLocked() { - AudioBuffer* ra = render_audio_.get(); // For brevity. - if (rev_proc_format_.sample_rate_hz() == kSampleRate32kHz) { + AudioBuffer* ra = render_.render_audio.get(); // For brevity. + if (formats_.rev_proc_format.sample_rate_hz() == kSampleRate32kHz) { ra->SplitIntoFrequencyBands(); } - if (intelligibility_enabled_) { - intelligibility_enhancer_->ProcessRenderAudio( - ra->split_channels_f(kBand0To8kHz), split_rate_, ra->num_channels()); + if (constants_.intelligibility_enabled) { + // Currently run in single-threaded mode when the intelligibility + // enhancer is activated. + // TODO(peah): Fix to be properly multi-threaded. + rtc::CritScope cs(&crit_capture_); + public_submodules_->intelligibility_enhancer->ProcessRenderAudio( + ra->split_channels_f(kBand0To8kHz), capture_nonlocked_.split_rate, + ra->num_channels()); } - RETURN_ON_ERR(echo_cancellation_->ProcessRenderAudio(ra)); - RETURN_ON_ERR(echo_control_mobile_->ProcessRenderAudio(ra)); - if (!use_new_agc_) { - RETURN_ON_ERR(gain_control_->ProcessRenderAudio(ra)); + RETURN_ON_ERR(public_submodules_->echo_cancellation->ProcessRenderAudio(ra)); + RETURN_ON_ERR( + public_submodules_->echo_control_mobile->ProcessRenderAudio(ra)); + if (!constants_.use_new_agc) { + RETURN_ON_ERR(public_submodules_->gain_control->ProcessRenderAudio(ra)); } - if (rev_proc_format_.sample_rate_hz() == kSampleRate32kHz && + if (formats_.rev_proc_format.sample_rate_hz() == kSampleRate32kHz && is_rev_processed()) { ra->MergeFrequencyBands(); } @@ -861,9 +1008,10 @@ int AudioProcessingImpl::ProcessReverseStreamLocked() { } int AudioProcessingImpl::set_stream_delay_ms(int delay) { + rtc::CritScope cs(&crit_capture_); Error retval = kNoError; - was_stream_delay_set_ = true; - delay += delay_offset_ms_; + capture_.was_stream_delay_set = true; + delay += capture_.delay_offset_ms; if (delay < 0) { delay = 0; @@ -876,50 +1024,56 @@ int AudioProcessingImpl::set_stream_delay_ms(int delay) { retval = kBadStreamParameterWarning; } - stream_delay_ms_ = delay; + capture_nonlocked_.stream_delay_ms = delay; return retval; } int AudioProcessingImpl::stream_delay_ms() const { - return stream_delay_ms_; + // Used as callback from submodules, hence locking is not allowed. + return capture_nonlocked_.stream_delay_ms; } bool AudioProcessingImpl::was_stream_delay_set() const { - return was_stream_delay_set_; + // Used as callback from submodules, hence locking is not allowed. + return capture_.was_stream_delay_set; } void AudioProcessingImpl::set_stream_key_pressed(bool key_pressed) { - key_pressed_ = key_pressed; + rtc::CritScope cs(&crit_capture_); + capture_.key_pressed = key_pressed; } void AudioProcessingImpl::set_delay_offset_ms(int offset) { - CriticalSectionScoped crit_scoped(crit_); - delay_offset_ms_ = offset; + rtc::CritScope cs(&crit_capture_); + capture_.delay_offset_ms = offset; } int AudioProcessingImpl::delay_offset_ms() const { - return delay_offset_ms_; + rtc::CritScope cs(&crit_capture_); + return capture_.delay_offset_ms; } int AudioProcessingImpl::StartDebugRecording( const char filename[AudioProcessing::kMaxFilenameSize]) { - CriticalSectionScoped crit_scoped(crit_); + // Run in a single-threaded manner. + rtc::CritScope cs_render(&crit_render_); + rtc::CritScope cs_capture(&crit_capture_); static_assert(kMaxFilenameSize == FileWrapper::kMaxFileNameSize, ""); - if (filename == NULL) { + if (filename == nullptr) { return kNullPointerError; } #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP // Stop any ongoing recording. - if (debug_file_->Open()) { - if (debug_file_->CloseFile() == -1) { + if (debug_dump_.debug_file->Open()) { + if (debug_dump_.debug_file->CloseFile() == -1) { return kFileError; } } - if (debug_file_->OpenFile(filename, false) == -1) { - debug_file_->CloseFile(); + if (debug_dump_.debug_file->OpenFile(filename, false) == -1) { + debug_dump_.debug_file->CloseFile(); return kFileError; } @@ -932,21 +1086,23 @@ int AudioProcessingImpl::StartDebugRecording( } int AudioProcessingImpl::StartDebugRecording(FILE* handle) { - CriticalSectionScoped crit_scoped(crit_); + // Run in a single-threaded manner. + rtc::CritScope cs_render(&crit_render_); + rtc::CritScope cs_capture(&crit_capture_); - if (handle == NULL) { + if (handle == nullptr) { return kNullPointerError; } #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP // Stop any ongoing recording. - if (debug_file_->Open()) { - if (debug_file_->CloseFile() == -1) { + if (debug_dump_.debug_file->Open()) { + if (debug_dump_.debug_file->CloseFile() == -1) { return kFileError; } } - if (debug_file_->OpenFromFileHandle(handle, true, false) == -1) { + if (debug_dump_.debug_file->OpenFromFileHandle(handle, true, false) == -1) { return kFileError; } @@ -960,17 +1116,22 @@ int AudioProcessingImpl::StartDebugRecording(FILE* handle) { int AudioProcessingImpl::StartDebugRecordingForPlatformFile( rtc::PlatformFile handle) { + // Run in a single-threaded manner. + rtc::CritScope cs_render(&crit_render_); + rtc::CritScope cs_capture(&crit_capture_); FILE* stream = rtc::FdopenPlatformFileForWriting(handle); return StartDebugRecording(stream); } int AudioProcessingImpl::StopDebugRecording() { - CriticalSectionScoped crit_scoped(crit_); + // Run in a single-threaded manner. + rtc::CritScope cs_render(&crit_render_); + rtc::CritScope cs_capture(&crit_capture_); #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP // We just return if recording hasn't started. - if (debug_file_->Open()) { - if (debug_file_->CloseFile() == -1) { + if (debug_dump_.debug_file->Open()) { + if (debug_dump_.debug_file->CloseFile() == -1) { return kFileError; } } @@ -981,58 +1142,87 @@ int AudioProcessingImpl::StopDebugRecording() { } EchoCancellation* AudioProcessingImpl::echo_cancellation() const { - return echo_cancellation_; + // Adding a lock here has no effect as it allows any access to the submodule + // from the returned pointer. + return public_submodules_->echo_cancellation; } EchoControlMobile* AudioProcessingImpl::echo_control_mobile() const { - return echo_control_mobile_; + // Adding a lock here has no effect as it allows any access to the submodule + // from the returned pointer. + return public_submodules_->echo_control_mobile; } GainControl* AudioProcessingImpl::gain_control() const { - if (use_new_agc_) { - return gain_control_for_new_agc_.get(); + // Adding a lock here has no effect as it allows any access to the submodule + // from the returned pointer. + if (constants_.use_new_agc) { + return public_submodules_->gain_control_for_new_agc.get(); } - return gain_control_; + return public_submodules_->gain_control; } HighPassFilter* AudioProcessingImpl::high_pass_filter() const { - return high_pass_filter_; + // Adding a lock here has no effect as it allows any access to the submodule + // from the returned pointer. + return public_submodules_->high_pass_filter.get(); } LevelEstimator* AudioProcessingImpl::level_estimator() const { - return level_estimator_; + // Adding a lock here has no effect as it allows any access to the submodule + // from the returned pointer. + return public_submodules_->level_estimator.get(); } NoiseSuppression* AudioProcessingImpl::noise_suppression() const { - return noise_suppression_; + // Adding a lock here has no effect as it allows any access to the submodule + // from the returned pointer. + return public_submodules_->noise_suppression.get(); } VoiceDetection* AudioProcessingImpl::voice_detection() const { - return voice_detection_; + // Adding a lock here has no effect as it allows any access to the submodule + // from the returned pointer. + return public_submodules_->voice_detection.get(); } bool AudioProcessingImpl::is_data_processed() const { - if (beamformer_enabled_) { + if (capture_nonlocked_.beamformer_enabled) { return true; } int enabled_count = 0; - for (auto item : component_list_) { + for (auto item : private_submodules_->component_list) { if (item->is_component_enabled()) { enabled_count++; } } + if (public_submodules_->high_pass_filter->is_enabled()) { + enabled_count++; + } + if (public_submodules_->noise_suppression->is_enabled()) { + enabled_count++; + } + if (public_submodules_->level_estimator->is_enabled()) { + enabled_count++; + } + if (public_submodules_->voice_detection->is_enabled()) { + enabled_count++; + } - // Data is unchanged if no components are enabled, or if only level_estimator_ - // or voice_detection_ is enabled. + // Data is unchanged if no components are enabled, or if only + // public_submodules_->level_estimator + // or public_submodules_->voice_detection is enabled. if (enabled_count == 0) { return false; } else if (enabled_count == 1) { - if (level_estimator_->is_enabled() || voice_detection_->is_enabled()) { + if (public_submodules_->level_estimator->is_enabled() || + public_submodules_->voice_detection->is_enabled()) { return false; } } else if (enabled_count == 2) { - if (level_estimator_->is_enabled() && voice_detection_->is_enabled()) { + if (public_submodules_->level_estimator->is_enabled() && + public_submodules_->voice_detection->is_enabled()) { return false; } } @@ -1041,149 +1231,194 @@ bool AudioProcessingImpl::is_data_processed() const { bool AudioProcessingImpl::output_copy_needed(bool is_data_processed) const { // Check if we've upmixed or downmixed the audio. - return ((api_format_.output_stream().num_channels() != - api_format_.input_stream().num_channels()) || - is_data_processed || transient_suppressor_enabled_); + return ((formats_.api_format.output_stream().num_channels() != + formats_.api_format.input_stream().num_channels()) || + is_data_processed || capture_.transient_suppressor_enabled); } bool AudioProcessingImpl::synthesis_needed(bool is_data_processed) const { return (is_data_processed && - (fwd_proc_format_.sample_rate_hz() == kSampleRate32kHz || - fwd_proc_format_.sample_rate_hz() == kSampleRate48kHz)); + (capture_nonlocked_.fwd_proc_format.sample_rate_hz() == + kSampleRate32kHz || + capture_nonlocked_.fwd_proc_format.sample_rate_hz() == + kSampleRate48kHz)); } bool AudioProcessingImpl::analysis_needed(bool is_data_processed) const { - if (!is_data_processed && !voice_detection_->is_enabled() && - !transient_suppressor_enabled_) { - // Only level_estimator_ is enabled. + if (!is_data_processed && + !public_submodules_->voice_detection->is_enabled() && + !capture_.transient_suppressor_enabled) { + // Only public_submodules_->level_estimator is enabled. return false; - } else if (fwd_proc_format_.sample_rate_hz() == kSampleRate32kHz || - fwd_proc_format_.sample_rate_hz() == kSampleRate48kHz) { - // Something besides level_estimator_ is enabled, and we have super-wb. + } else if (capture_nonlocked_.fwd_proc_format.sample_rate_hz() == + kSampleRate32kHz || + capture_nonlocked_.fwd_proc_format.sample_rate_hz() == + kSampleRate48kHz) { + // Something besides public_submodules_->level_estimator is enabled, and we + // have super-wb. return true; } return false; } bool AudioProcessingImpl::is_rev_processed() const { - return intelligibility_enabled_ && intelligibility_enhancer_->active(); + return constants_.intelligibility_enabled && + public_submodules_->intelligibility_enhancer->active(); +} + +bool AudioProcessingImpl::render_check_rev_conversion_needed() const { + return rev_conversion_needed(); } bool AudioProcessingImpl::rev_conversion_needed() const { - return (api_format_.reverse_input_stream() != - api_format_.reverse_output_stream()); + return (formats_.api_format.reverse_input_stream() != + formats_.api_format.reverse_output_stream()); } void AudioProcessingImpl::InitializeExperimentalAgc() { - if (use_new_agc_) { - if (!agc_manager_.get()) { - agc_manager_.reset(new AgcManagerDirect(gain_control_, - gain_control_for_new_agc_.get(), - agc_startup_min_volume_)); + if (constants_.use_new_agc) { + if (!private_submodules_->agc_manager.get()) { + private_submodules_->agc_manager.reset(new AgcManagerDirect( + public_submodules_->gain_control, + public_submodules_->gain_control_for_new_agc.get(), + constants_.agc_startup_min_volume)); } - agc_manager_->Initialize(); - agc_manager_->SetCaptureMuted(output_will_be_muted_); + private_submodules_->agc_manager->Initialize(); + private_submodules_->agc_manager->SetCaptureMuted( + capture_.output_will_be_muted); } } void AudioProcessingImpl::InitializeTransient() { - if (transient_suppressor_enabled_) { - if (!transient_suppressor_.get()) { - transient_suppressor_.reset(new TransientSuppressor()); + if (capture_.transient_suppressor_enabled) { + if (!public_submodules_->transient_suppressor.get()) { + public_submodules_->transient_suppressor.reset(new TransientSuppressor()); } - transient_suppressor_->Initialize( - fwd_proc_format_.sample_rate_hz(), split_rate_, - api_format_.output_stream().num_channels()); + public_submodules_->transient_suppressor->Initialize( + capture_nonlocked_.fwd_proc_format.sample_rate_hz(), + capture_nonlocked_.split_rate, + num_proc_channels()); } } void AudioProcessingImpl::InitializeBeamformer() { - if (beamformer_enabled_) { - if (!beamformer_) { - beamformer_.reset( - new NonlinearBeamformer(array_geometry_, target_direction_)); + if (capture_nonlocked_.beamformer_enabled) { + if (!private_submodules_->beamformer) { + private_submodules_->beamformer.reset(new NonlinearBeamformer( + capture_.array_geometry, capture_.target_direction)); } - beamformer_->Initialize(kChunkSizeMs, split_rate_); + private_submodules_->beamformer->Initialize(kChunkSizeMs, + capture_nonlocked_.split_rate); } } void AudioProcessingImpl::InitializeIntelligibility() { - if (intelligibility_enabled_) { + if (constants_.intelligibility_enabled) { IntelligibilityEnhancer::Config config; - config.sample_rate_hz = split_rate_; - config.num_capture_channels = capture_audio_->num_channels(); - config.num_render_channels = render_audio_->num_channels(); - intelligibility_enhancer_.reset(new IntelligibilityEnhancer(config)); + config.sample_rate_hz = capture_nonlocked_.split_rate; + config.num_capture_channels = capture_.capture_audio->num_channels(); + config.num_render_channels = render_.render_audio->num_channels(); + public_submodules_->intelligibility_enhancer.reset( + new IntelligibilityEnhancer(config)); } } +void AudioProcessingImpl::InitializeHighPassFilter() { + public_submodules_->high_pass_filter->Initialize(num_proc_channels(), + proc_sample_rate_hz()); +} + +void AudioProcessingImpl::InitializeNoiseSuppression() { + public_submodules_->noise_suppression->Initialize(num_proc_channels(), + proc_sample_rate_hz()); +} + +void AudioProcessingImpl::InitializeLevelEstimator() { + public_submodules_->level_estimator->Initialize(); +} + +void AudioProcessingImpl::InitializeVoiceDetection() { + public_submodules_->voice_detection->Initialize(proc_split_sample_rate_hz()); +} + void AudioProcessingImpl::MaybeUpdateHistograms() { static const int kMinDiffDelayMs = 60; if (echo_cancellation()->is_enabled()) { // Activate delay_jumps_ counters if we know echo_cancellation is runnning. // If a stream has echo we know that the echo_cancellation is in process. - if (stream_delay_jumps_ == -1 && echo_cancellation()->stream_has_echo()) { - stream_delay_jumps_ = 0; + if (capture_.stream_delay_jumps == -1 && + echo_cancellation()->stream_has_echo()) { + capture_.stream_delay_jumps = 0; } - if (aec_system_delay_jumps_ == -1 && + if (capture_.aec_system_delay_jumps == -1 && echo_cancellation()->stream_has_echo()) { - aec_system_delay_jumps_ = 0; + capture_.aec_system_delay_jumps = 0; } // Detect a jump in platform reported system delay and log the difference. - const int diff_stream_delay_ms = stream_delay_ms_ - last_stream_delay_ms_; - if (diff_stream_delay_ms > kMinDiffDelayMs && last_stream_delay_ms_ != 0) { - RTC_HISTOGRAM_COUNTS("WebRTC.Audio.PlatformReportedStreamDelayJump", - diff_stream_delay_ms, kMinDiffDelayMs, 1000, 100); - if (stream_delay_jumps_ == -1) { - stream_delay_jumps_ = 0; // Activate counter if needed. + const int diff_stream_delay_ms = + capture_nonlocked_.stream_delay_ms - capture_.last_stream_delay_ms; + if (diff_stream_delay_ms > kMinDiffDelayMs && + capture_.last_stream_delay_ms != 0) { + RTC_HISTOGRAM_COUNTS_SPARSE( + "WebRTC.Audio.PlatformReportedStreamDelayJump", diff_stream_delay_ms, + kMinDiffDelayMs, 1000, 100); + if (capture_.stream_delay_jumps == -1) { + capture_.stream_delay_jumps = 0; // Activate counter if needed. } - stream_delay_jumps_++; + capture_.stream_delay_jumps++; } - last_stream_delay_ms_ = stream_delay_ms_; + capture_.last_stream_delay_ms = capture_nonlocked_.stream_delay_ms; // Detect a jump in AEC system delay and log the difference. - const int frames_per_ms = rtc::CheckedDivExact(split_rate_, 1000); + const int frames_per_ms = + rtc::CheckedDivExact(capture_nonlocked_.split_rate, 1000); const int aec_system_delay_ms = WebRtcAec_system_delay(echo_cancellation()->aec_core()) / frames_per_ms; const int diff_aec_system_delay_ms = - aec_system_delay_ms - last_aec_system_delay_ms_; + aec_system_delay_ms - capture_.last_aec_system_delay_ms; if (diff_aec_system_delay_ms > kMinDiffDelayMs && - last_aec_system_delay_ms_ != 0) { - RTC_HISTOGRAM_COUNTS("WebRTC.Audio.AecSystemDelayJump", - diff_aec_system_delay_ms, kMinDiffDelayMs, 1000, - 100); - if (aec_system_delay_jumps_ == -1) { - aec_system_delay_jumps_ = 0; // Activate counter if needed. + capture_.last_aec_system_delay_ms != 0) { + RTC_HISTOGRAM_COUNTS_SPARSE("WebRTC.Audio.AecSystemDelayJump", + diff_aec_system_delay_ms, kMinDiffDelayMs, + 1000, 100); + if (capture_.aec_system_delay_jumps == -1) { + capture_.aec_system_delay_jumps = 0; // Activate counter if needed. } - aec_system_delay_jumps_++; + capture_.aec_system_delay_jumps++; } - last_aec_system_delay_ms_ = aec_system_delay_ms; + capture_.last_aec_system_delay_ms = aec_system_delay_ms; } } void AudioProcessingImpl::UpdateHistogramsOnCallEnd() { - CriticalSectionScoped crit_scoped(crit_); - if (stream_delay_jumps_ > -1) { - RTC_HISTOGRAM_ENUMERATION( + // Run in a single-threaded manner. + rtc::CritScope cs_render(&crit_render_); + rtc::CritScope cs_capture(&crit_capture_); + + if (capture_.stream_delay_jumps > -1) { + RTC_HISTOGRAM_ENUMERATION_SPARSE( "WebRTC.Audio.NumOfPlatformReportedStreamDelayJumps", - stream_delay_jumps_, 51); + capture_.stream_delay_jumps, 51); } - stream_delay_jumps_ = -1; - last_stream_delay_ms_ = 0; + capture_.stream_delay_jumps = -1; + capture_.last_stream_delay_ms = 0; - if (aec_system_delay_jumps_ > -1) { - RTC_HISTOGRAM_ENUMERATION("WebRTC.Audio.NumOfAecSystemDelayJumps", - aec_system_delay_jumps_, 51); + if (capture_.aec_system_delay_jumps > -1) { + RTC_HISTOGRAM_ENUMERATION_SPARSE("WebRTC.Audio.NumOfAecSystemDelayJumps", + capture_.aec_system_delay_jumps, 51); } - aec_system_delay_jumps_ = -1; - last_aec_system_delay_ms_ = 0; + capture_.aec_system_delay_jumps = -1; + capture_.last_aec_system_delay_ms = 0; } #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP -int AudioProcessingImpl::WriteMessageToDebugFile() { - int32_t size = event_msg_->ByteSize(); +int AudioProcessingImpl::WriteMessageToDebugFile( + FileWrapper* debug_file, + rtc::CriticalSection* crit_debug, + ApmDebugDumpThreadState* debug_state) { + int32_t size = debug_state->event_msg->ByteSize(); if (size <= 0) { return kUnspecifiedError; } @@ -1192,82 +1427,100 @@ int AudioProcessingImpl::WriteMessageToDebugFile() { // pretty safe in assuming little-endian. #endif - if (!event_msg_->SerializeToString(&event_str_)) { + if (!debug_state->event_msg->SerializeToString(&debug_state->event_str)) { return kUnspecifiedError; } - // Write message preceded by its size. - if (!debug_file_->Write(&size, sizeof(int32_t))) { - return kFileError; - } - if (!debug_file_->Write(event_str_.data(), event_str_.length())) { - return kFileError; + { + // Ensure atomic writes of the message. + rtc::CritScope cs_capture(crit_debug); + // Write message preceded by its size. + if (!debug_file->Write(&size, sizeof(int32_t))) { + return kFileError; + } + if (!debug_file->Write(debug_state->event_str.data(), + debug_state->event_str.length())) { + return kFileError; + } } - event_msg_->Clear(); + debug_state->event_msg->Clear(); return kNoError; } int AudioProcessingImpl::WriteInitMessage() { - event_msg_->set_type(audioproc::Event::INIT); - audioproc::Init* msg = event_msg_->mutable_init(); - msg->set_sample_rate(api_format_.input_stream().sample_rate_hz()); - msg->set_num_input_channels(api_format_.input_stream().num_channels()); - msg->set_num_output_channels(api_format_.output_stream().num_channels()); - msg->set_num_reverse_channels( - api_format_.reverse_input_stream().num_channels()); + debug_dump_.capture.event_msg->set_type(audioproc::Event::INIT); + audioproc::Init* msg = debug_dump_.capture.event_msg->mutable_init(); + msg->set_sample_rate(formats_.api_format.input_stream().sample_rate_hz()); + + msg->set_num_input_channels(static_cast<google::protobuf::int32>( + formats_.api_format.input_stream().num_channels())); + msg->set_num_output_channels(static_cast<google::protobuf::int32>( + formats_.api_format.output_stream().num_channels())); + msg->set_num_reverse_channels(static_cast<google::protobuf::int32>( + formats_.api_format.reverse_input_stream().num_channels())); msg->set_reverse_sample_rate( - api_format_.reverse_input_stream().sample_rate_hz()); - msg->set_output_sample_rate(api_format_.output_stream().sample_rate_hz()); - // TODO(ekmeyerson): Add reverse output fields to event_msg_. - - RETURN_ON_ERR(WriteMessageToDebugFile()); + formats_.api_format.reverse_input_stream().sample_rate_hz()); + msg->set_output_sample_rate( + formats_.api_format.output_stream().sample_rate_hz()); + // TODO(ekmeyerson): Add reverse output fields to + // debug_dump_.capture.event_msg. + + RETURN_ON_ERR(WriteMessageToDebugFile(debug_dump_.debug_file.get(), + &crit_debug_, &debug_dump_.capture)); return kNoError; } int AudioProcessingImpl::WriteConfigMessage(bool forced) { audioproc::Config config; - config.set_aec_enabled(echo_cancellation_->is_enabled()); + config.set_aec_enabled(public_submodules_->echo_cancellation->is_enabled()); config.set_aec_delay_agnostic_enabled( - echo_cancellation_->is_delay_agnostic_enabled()); + public_submodules_->echo_cancellation->is_delay_agnostic_enabled()); config.set_aec_drift_compensation_enabled( - echo_cancellation_->is_drift_compensation_enabled()); + public_submodules_->echo_cancellation->is_drift_compensation_enabled()); config.set_aec_extended_filter_enabled( - echo_cancellation_->is_extended_filter_enabled()); - config.set_aec_suppression_level( - static_cast<int>(echo_cancellation_->suppression_level())); + public_submodules_->echo_cancellation->is_extended_filter_enabled()); + config.set_aec_suppression_level(static_cast<int>( + public_submodules_->echo_cancellation->suppression_level())); - config.set_aecm_enabled(echo_control_mobile_->is_enabled()); + config.set_aecm_enabled( + public_submodules_->echo_control_mobile->is_enabled()); config.set_aecm_comfort_noise_enabled( - echo_control_mobile_->is_comfort_noise_enabled()); - config.set_aecm_routing_mode( - static_cast<int>(echo_control_mobile_->routing_mode())); + public_submodules_->echo_control_mobile->is_comfort_noise_enabled()); + config.set_aecm_routing_mode(static_cast<int>( + public_submodules_->echo_control_mobile->routing_mode())); - config.set_agc_enabled(gain_control_->is_enabled()); - config.set_agc_mode(static_cast<int>(gain_control_->mode())); - config.set_agc_limiter_enabled(gain_control_->is_limiter_enabled()); - config.set_noise_robust_agc_enabled(use_new_agc_); + config.set_agc_enabled(public_submodules_->gain_control->is_enabled()); + config.set_agc_mode( + static_cast<int>(public_submodules_->gain_control->mode())); + config.set_agc_limiter_enabled( + public_submodules_->gain_control->is_limiter_enabled()); + config.set_noise_robust_agc_enabled(constants_.use_new_agc); - config.set_hpf_enabled(high_pass_filter_->is_enabled()); + config.set_hpf_enabled(public_submodules_->high_pass_filter->is_enabled()); - config.set_ns_enabled(noise_suppression_->is_enabled()); - config.set_ns_level(static_cast<int>(noise_suppression_->level())); + config.set_ns_enabled(public_submodules_->noise_suppression->is_enabled()); + config.set_ns_level( + static_cast<int>(public_submodules_->noise_suppression->level())); - config.set_transient_suppression_enabled(transient_suppressor_enabled_); + config.set_transient_suppression_enabled( + capture_.transient_suppressor_enabled); std::string serialized_config = config.SerializeAsString(); - if (!forced && last_serialized_config_ == serialized_config) { + if (!forced && + debug_dump_.capture.last_serialized_config == serialized_config) { return kNoError; } - last_serialized_config_ = serialized_config; + debug_dump_.capture.last_serialized_config = serialized_config; - event_msg_->set_type(audioproc::Event::CONFIG); - event_msg_->mutable_config()->CopyFrom(config); + debug_dump_.capture.event_msg->set_type(audioproc::Event::CONFIG); + debug_dump_.capture.event_msg->mutable_config()->CopyFrom(config); - RETURN_ON_ERR(WriteMessageToDebugFile()); + RETURN_ON_ERR(WriteMessageToDebugFile(debug_dump_.debug_file.get(), + &crit_debug_, &debug_dump_.capture)); return kNoError; } #endif // WEBRTC_AUDIOPROC_DEBUG_DUMP |