diff options
author | Jeroen de Borst <jeroendb@webrtc.org> | 2019-04-08 15:00:02 +0000 |
---|---|---|
committer | Commit Bot <commit-bot@chromium.org> | 2019-04-08 15:00:10 +0000 |
commit | e9d2b4efdd5dddaa3a476c0ac2a9cf9125b39929 (patch) | |
tree | 2e4b5f652005196c177be48618cad5aaccdf8dde /modules | |
parent | 57f2a5485a4c83c8045d0dbe9db6281d6bcda847 (diff) | |
download | webrtc-e9d2b4efdd5dddaa3a476c0ac2a9cf9125b39929.tar.gz |
Revert "Remove old audio device implementation."
This reverts commit 0cfa4cba5cae5e942f5d8e0e4e93b94982d0bfc3.
Reason for revert: audio_device_ios_objc target is removed, but still referenced by iPhone Meetins:Meeting_build_test, which now fails to build
Original change's description:
> Remove old audio device implementation.
>
> The iOS ADM implementation now lives in sdk/objc/native/api/audio_device_module.{h,mm}.
>
> Bug: webrtc:10514
> Change-Id: Ib0b162027b5680ebc40d621a57f1155f08e7a057
> Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/131326
> Commit-Queue: Kári Helgason <kthelgason@webrtc.org>
> Reviewed-by: Henrik Andreassson <henrika@webrtc.org>
> Reviewed-by: Niels Moller <nisse@webrtc.org>
> Cr-Commit-Position: refs/heads/master@{#27488}
TBR=henrika@webrtc.org,nisse@webrtc.org,kthelgason@webrtc.org
Change-Id: I5be10b3d17403a79ea30afc255cde01171bc9f5b
No-Presubmit: true
No-Tree-Checks: true
No-Try: true
Bug: webrtc:10514
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/131960
Reviewed-by: Jeroen de Borst <jeroendb@webrtc.org>
Commit-Queue: Jeroen de Borst <jeroendb@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#27492}
Diffstat (limited to 'modules')
-rw-r--r-- | modules/audio_device/BUILD.gn | 69 | ||||
-rw-r--r-- | modules/audio_device/DEPS | 23 | ||||
-rw-r--r-- | modules/audio_device/audio_device_impl.cc | 4 | ||||
-rw-r--r-- | modules/audio_device/ios/audio_device_ios.h | 296 | ||||
-rw-r--r-- | modules/audio_device/ios/audio_device_ios.mm | 908 | ||||
-rw-r--r-- | modules/audio_device/ios/audio_device_not_implemented_ios.mm | 205 | ||||
-rw-r--r-- | modules/audio_device/ios/audio_device_unittest_ios.mm | 877 | ||||
-rw-r--r-- | modules/audio_device/ios/audio_session_observer.h | 42 | ||||
-rw-r--r-- | modules/audio_device/ios/objc/RTCAudioSession.h | 11 | ||||
-rw-r--r-- | modules/audio_device/ios/objc/RTCAudioSessionConfiguration.h | 11 | ||||
-rw-r--r-- | modules/audio_device/ios/objc/RTCAudioSessionDelegateAdapter.h | 29 | ||||
-rw-r--r-- | modules/audio_device/ios/objc/RTCAudioSessionDelegateAdapter.mm | 89 | ||||
-rw-r--r-- | modules/audio_device/ios/voice_processing_audio_unit.h | 137 | ||||
-rw-r--r-- | modules/audio_device/ios/voice_processing_audio_unit.mm | 468 |
14 files changed, 3165 insertions, 4 deletions
diff --git a/modules/audio_device/BUILD.gn b/modules/audio_device/BUILD.gn index e9153201e6..02e54c01ca 100644 --- a/modules/audio_device/BUILD.gn +++ b/modules/audio_device/BUILD.gn @@ -45,6 +45,47 @@ rtc_source_set("audio_device") { ] } +if (rtc_include_internal_audio_device && is_ios) { + rtc_source_set("audio_device_ios_objc") { + visibility = [ + ":audio_device_impl", + ":audio_device_ios_objc_unittests", + ] + sources = [ + "ios/audio_device_ios.h", + "ios/audio_device_ios.mm", + "ios/audio_device_not_implemented_ios.mm", + "ios/audio_session_observer.h", + "ios/objc/RTCAudioSession.h", + "ios/objc/RTCAudioSessionConfiguration.h", + "ios/objc/RTCAudioSessionDelegateAdapter.h", + "ios/objc/RTCAudioSessionDelegateAdapter.mm", + "ios/voice_processing_audio_unit.h", + "ios/voice_processing_audio_unit.mm", + ] + libs = [ + "AudioToolbox.framework", + "AVFoundation.framework", + "Foundation.framework", + "UIKit.framework", + ] + deps = [ + ":audio_device_api", + ":audio_device_buffer", + ":audio_device_generic", + "../../api:array_view", + "../../rtc_base", + "../../rtc_base:checks", + "../../rtc_base:gtest_prod", + "../../rtc_base/system:fallthrough", + "../../sdk:audio_device", + "../../sdk:audio_objc", + "../../sdk:base_objc", + "../../system_wrappers:metrics", + ] + } +} + rtc_source_set("audio_device_api") { visibility = [ "*" ] sources = [ @@ -183,7 +224,7 @@ rtc_source_set("audio_device_impl") { "//third_party/abseil-cpp/absl/memory", ] if (rtc_include_internal_audio_device && is_ios) { - deps += [ "../../sdk:audio_device" ] + deps += [ ":audio_device_ios_objc" ] } sources = [ @@ -356,6 +397,32 @@ rtc_source_set("mock_audio_device") { } if (rtc_include_tests) { + # TODO(kthelgason): Reenable these tests on simulator. + # See bugs.webrtc.org/7812 + if (rtc_include_internal_audio_device && is_ios && !use_ios_simulator) { + rtc_source_set("audio_device_ios_objc_unittests") { + testonly = true + visibility = [ ":*" ] + sources = [ + "ios/audio_device_unittest_ios.mm", + ] + deps = [ + ":audio_device", + ":audio_device_buffer", + ":audio_device_impl", + ":audio_device_ios_objc", + ":mock_audio_device", + "../../api:scoped_refptr", + "../../rtc_base:rtc_base_approved", + "../../sdk:audio_objc", + "../../system_wrappers", + "../../test:fileutils", + "../../test:test_support", + "//third_party/ocmock", + ] + } + } + rtc_source_set("audio_device_unittests") { testonly = true diff --git a/modules/audio_device/DEPS b/modules/audio_device/DEPS index fc5eed7833..f74767a67e 100644 --- a/modules/audio_device/DEPS +++ b/modules/audio_device/DEPS @@ -7,7 +7,28 @@ specific_include_rules = { "ensure_initialized\.cc": [ "+base/android", ], - "audio_device_impl\.cc": [ + "audio_device_ios\.h": [ + "+sdk/objc", + ], + "audio_device_ios\.mm": [ + "+sdk/objc", + ], + "audio_device_unittest_ios\.mm": [ + "+sdk/objc", + ], + "RTCAudioSession\.h": [ + "+sdk/objc", + ], + "RTCAudioSessionConfiguration\.h": [ + "+sdk/objc", + ], + "RTCAudioSessionDelegateAdapter\.h": [ + "+sdk/objc", + ], + "RTCAudioSessionDelegateAdapter\.mm": [ + "+sdk/objc", + ], + "voice_processing_audio_unit\.mm": [ "+sdk/objc", ], } diff --git a/modules/audio_device/audio_device_impl.cc b/modules/audio_device/audio_device_impl.cc index f7efcede20..7b08a5af49 100644 --- a/modules/audio_device/audio_device_impl.cc +++ b/modules/audio_device/audio_device_impl.cc @@ -45,7 +45,7 @@ #include "modules/audio_device/linux/audio_device_pulse_linux.h" #endif #elif defined(WEBRTC_IOS) -#include "sdk/objc/native/src/audio/audio_device_ios.h" +#include "modules/audio_device/ios/audio_device_ios.h" #elif defined(WEBRTC_MAC) #include "modules/audio_device/mac/audio_device_mac.h" #endif @@ -287,7 +287,7 @@ int32_t AudioDeviceModuleImpl::CreatePlatformSpecificObjects() { // iOS ADM implementation. #if defined(WEBRTC_IOS) if (audio_layer == kPlatformDefaultAudio) { - audio_device_.reset(new ios_adm::AudioDeviceIOS()); + audio_device_.reset(new AudioDeviceIOS()); RTC_LOG(INFO) << "iPhone Audio APIs will be utilized."; } // END #if defined(WEBRTC_IOS) diff --git a/modules/audio_device/ios/audio_device_ios.h b/modules/audio_device/ios/audio_device_ios.h new file mode 100644 index 0000000000..e90bb440f4 --- /dev/null +++ b/modules/audio_device/ios/audio_device_ios.h @@ -0,0 +1,296 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_DEVICE_IOS_AUDIO_DEVICE_IOS_H_ +#define MODULES_AUDIO_DEVICE_IOS_AUDIO_DEVICE_IOS_H_ + +#include <memory> + +#include "modules/audio_device/audio_device_generic.h" +#include "modules/audio_device/ios/audio_session_observer.h" +#include "modules/audio_device/ios/voice_processing_audio_unit.h" +#include "rtc_base/buffer.h" +#include "rtc_base/gtest_prod_util.h" +#include "rtc_base/thread.h" +#include "rtc_base/thread_annotations.h" +#include "rtc_base/thread_checker.h" +#include "sdk/objc/base/RTCMacros.h" + +RTC_FWD_DECL_OBJC_CLASS(RTCAudioSessionDelegateAdapter); + +namespace webrtc { + +class FineAudioBuffer; + +// Implements full duplex 16-bit mono PCM audio support for iOS using a +// Voice-Processing (VP) I/O audio unit in Core Audio. The VP I/O audio unit +// supports audio echo cancellation. It also adds automatic gain control, +// adjustment of voice-processing quality and muting. +// +// An instance must be created and destroyed on one and the same thread. +// All supported public methods must also be called on the same thread. +// A thread checker will RTC_DCHECK if any supported method is called on an +// invalid thread. +// +// Recorded audio will be delivered on a real-time internal I/O thread in the +// audio unit. The audio unit will also ask for audio data to play out on this +// same thread. +class AudioDeviceIOS : public AudioDeviceGeneric, + public AudioSessionObserver, + public VoiceProcessingAudioUnitObserver, + public rtc::MessageHandler { + public: + AudioDeviceIOS(); + ~AudioDeviceIOS(); + + void AttachAudioBuffer(AudioDeviceBuffer* audioBuffer) override; + + InitStatus Init() override; + int32_t Terminate() override; + bool Initialized() const override; + + int32_t InitPlayout() override; + bool PlayoutIsInitialized() const override; + + int32_t InitRecording() override; + bool RecordingIsInitialized() const override; + + int32_t StartPlayout() override; + int32_t StopPlayout() override; + bool Playing() const override { return playing_; } + + int32_t StartRecording() override; + int32_t StopRecording() override; + bool Recording() const override { return recording_; } + + // These methods returns hard-coded delay values and not dynamic delay + // estimates. The reason is that iOS supports a built-in AEC and the WebRTC + // AEC will always be disabled in the Libjingle layer to avoid running two + // AEC implementations at the same time. And, it saves resources to avoid + // updating these delay values continuously. + // TODO(henrika): it would be possible to mark these two methods as not + // implemented since they are only called for A/V-sync purposes today and + // A/V-sync is not supported on iOS. However, we avoid adding error messages + // the log by using these dummy implementations instead. + int32_t PlayoutDelay(uint16_t& delayMS) const override; + + // Native audio parameters stored during construction. + // These methods are unique for the iOS implementation. + int GetPlayoutAudioParameters(AudioParameters* params) const override; + int GetRecordAudioParameters(AudioParameters* params) const override; + + // These methods are currently not fully implemented on iOS: + + // See audio_device_not_implemented.cc for trivial implementations. + int32_t ActiveAudioLayer( + AudioDeviceModule::AudioLayer& audioLayer) const override; + int32_t PlayoutIsAvailable(bool& available) override; + int32_t RecordingIsAvailable(bool& available) override; + int16_t PlayoutDevices() override; + int16_t RecordingDevices() override; + int32_t PlayoutDeviceName(uint16_t index, + char name[kAdmMaxDeviceNameSize], + char guid[kAdmMaxGuidSize]) override; + int32_t RecordingDeviceName(uint16_t index, + char name[kAdmMaxDeviceNameSize], + char guid[kAdmMaxGuidSize]) override; + int32_t SetPlayoutDevice(uint16_t index) override; + int32_t SetPlayoutDevice( + AudioDeviceModule::WindowsDeviceType device) override; + int32_t SetRecordingDevice(uint16_t index) override; + int32_t SetRecordingDevice( + AudioDeviceModule::WindowsDeviceType device) override; + int32_t InitSpeaker() override; + bool SpeakerIsInitialized() const override; + int32_t InitMicrophone() override; + bool MicrophoneIsInitialized() const override; + int32_t SpeakerVolumeIsAvailable(bool& available) override; + int32_t SetSpeakerVolume(uint32_t volume) override; + int32_t SpeakerVolume(uint32_t& volume) const override; + int32_t MaxSpeakerVolume(uint32_t& maxVolume) const override; + int32_t MinSpeakerVolume(uint32_t& minVolume) const override; + int32_t MicrophoneVolumeIsAvailable(bool& available) override; + int32_t SetMicrophoneVolume(uint32_t volume) override; + int32_t MicrophoneVolume(uint32_t& volume) const override; + int32_t MaxMicrophoneVolume(uint32_t& maxVolume) const override; + int32_t MinMicrophoneVolume(uint32_t& minVolume) const override; + int32_t MicrophoneMuteIsAvailable(bool& available) override; + int32_t SetMicrophoneMute(bool enable) override; + int32_t MicrophoneMute(bool& enabled) const override; + int32_t SpeakerMuteIsAvailable(bool& available) override; + int32_t SetSpeakerMute(bool enable) override; + int32_t SpeakerMute(bool& enabled) const override; + int32_t StereoPlayoutIsAvailable(bool& available) override; + int32_t SetStereoPlayout(bool enable) override; + int32_t StereoPlayout(bool& enabled) const override; + int32_t StereoRecordingIsAvailable(bool& available) override; + int32_t SetStereoRecording(bool enable) override; + int32_t StereoRecording(bool& enabled) const override; + + // AudioSessionObserver methods. May be called from any thread. + void OnInterruptionBegin() override; + void OnInterruptionEnd() override; + void OnValidRouteChange() override; + void OnCanPlayOrRecordChange(bool can_play_or_record) override; + void OnChangedOutputVolume() override; + + // VoiceProcessingAudioUnitObserver methods. + OSStatus OnDeliverRecordedData(AudioUnitRenderActionFlags* flags, + const AudioTimeStamp* time_stamp, + UInt32 bus_number, + UInt32 num_frames, + AudioBufferList* io_data) override; + OSStatus OnGetPlayoutData(AudioUnitRenderActionFlags* flags, + const AudioTimeStamp* time_stamp, + UInt32 bus_number, + UInt32 num_frames, + AudioBufferList* io_data) override; + + // Handles messages from posts. + void OnMessage(rtc::Message* msg) override; + + private: + // Called by the relevant AudioSessionObserver methods on |thread_|. + void HandleInterruptionBegin(); + void HandleInterruptionEnd(); + void HandleValidRouteChange(); + void HandleCanPlayOrRecordChange(bool can_play_or_record); + void HandleSampleRateChange(float sample_rate); + void HandlePlayoutGlitchDetected(); + void HandleOutputVolumeChange(); + + // Uses current |playout_parameters_| and |record_parameters_| to inform the + // audio device buffer (ADB) about our internal audio parameters. + void UpdateAudioDeviceBuffer(); + + // Since the preferred audio parameters are only hints to the OS, the actual + // values may be different once the AVAudioSession has been activated. + // This method asks for the current hardware parameters and takes actions + // if they should differ from what we have asked for initially. It also + // defines |playout_parameters_| and |record_parameters_|. + void SetupAudioBuffersForActiveAudioSession(); + + // Creates the audio unit. + bool CreateAudioUnit(); + + // Updates the audio unit state based on current state. + void UpdateAudioUnit(bool can_play_or_record); + + // Configures the audio session for WebRTC. + bool ConfigureAudioSession(); + // Unconfigures the audio session. + void UnconfigureAudioSession(); + + // Activates our audio session, creates and initializes the voice-processing + // audio unit and verifies that we got the preferred native audio parameters. + bool InitPlayOrRecord(); + + // Closes and deletes the voice-processing I/O unit. + void ShutdownPlayOrRecord(); + + // Resets thread-checkers before a call is restarted. + void PrepareForNewStart(); + + // Ensures that methods are called from the same thread as this object is + // created on. + rtc::ThreadChecker thread_checker_; + + // Native I/O audio thread checker. + rtc::ThreadChecker io_thread_checker_; + + // Thread that this object is created on. + rtc::Thread* thread_; + + // Raw pointer handle provided to us in AttachAudioBuffer(). Owned by the + // AudioDeviceModuleImpl class and called by AudioDeviceModule::Create(). + // The AudioDeviceBuffer is a member of the AudioDeviceModuleImpl instance + // and therefore outlives this object. + AudioDeviceBuffer* audio_device_buffer_; + + // Contains audio parameters (sample rate, #channels, buffer size etc.) for + // the playout and recording sides. These structure is set in two steps: + // first, native sample rate and #channels are defined in Init(). Next, the + // audio session is activated and we verify that the preferred parameters + // were granted by the OS. At this stage it is also possible to add a third + // component to the parameters; the native I/O buffer duration. + // A RTC_CHECK will be hit if we for some reason fail to open an audio session + // using the specified parameters. + AudioParameters playout_parameters_; + AudioParameters record_parameters_; + + // The AudioUnit used to play and record audio. + std::unique_ptr<VoiceProcessingAudioUnit> audio_unit_; + + // FineAudioBuffer takes an AudioDeviceBuffer which delivers audio data + // in chunks of 10ms. It then allows for this data to be pulled in + // a finer or coarser granularity. I.e. interacting with this class instead + // of directly with the AudioDeviceBuffer one can ask for any number of + // audio data samples. Is also supports a similar scheme for the recording + // side. + // Example: native buffer size can be 128 audio frames at 16kHz sample rate. + // WebRTC will provide 480 audio frames per 10ms but iOS asks for 128 + // in each callback (one every 8ms). This class can then ask for 128 and the + // FineAudioBuffer will ask WebRTC for new data only when needed and also + // cache non-utilized audio between callbacks. On the recording side, iOS + // can provide audio data frames of size 128 and these are accumulated until + // enough data to supply one 10ms call exists. This 10ms chunk is then sent + // to WebRTC and the remaining part is stored. + std::unique_ptr<FineAudioBuffer> fine_audio_buffer_; + + // Temporary storage for recorded data. AudioUnitRender() renders into this + // array as soon as a frame of the desired buffer size has been recorded. + // On real iOS devices, the size will be fixed and set once. For iOS + // simulators, the size can vary from callback to callback and the size + // will be changed dynamically to account for this behavior. + rtc::BufferT<int16_t> record_audio_buffer_; + + // Set to 1 when recording is active and 0 otherwise. + volatile int recording_; + + // Set to 1 when playout is active and 0 otherwise. + volatile int playing_; + + // Set to true after successful call to Init(), false otherwise. + bool initialized_ RTC_GUARDED_BY(thread_checker_); + + // Set to true after successful call to InitRecording() or InitPlayout(), + // false otherwise. + bool audio_is_initialized_; + + // Set to true if audio session is interrupted, false otherwise. + bool is_interrupted_; + + // Audio interruption observer instance. + RTCAudioSessionDelegateAdapter* audio_session_observer_ + RTC_GUARDED_BY(thread_checker_); + + // Set to true if we've activated the audio session. + bool has_configured_session_ RTC_GUARDED_BY(thread_checker_); + + // Counts number of detected audio glitches on the playout side. + int64_t num_detected_playout_glitches_ RTC_GUARDED_BY(thread_checker_); + int64_t last_playout_time_ RTC_GUARDED_BY(io_thread_checker_); + + // Counts number of playout callbacks per call. + // The value isupdated on the native I/O thread and later read on the + // creating thread (see thread_checker_) but at this stage no audio is + // active. Hence, it is a "thread safe" design and no lock is needed. + int64_t num_playout_callbacks_; + + // Contains the time for when the last output volume change was detected. + int64_t last_output_volume_change_time_ RTC_GUARDED_BY(thread_checker_); + + // Exposes private members for testing purposes only. + FRIEND_TEST_ALL_PREFIXES(AudioDeviceTest, testInterruptedAudioSession); +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_DEVICE_IOS_AUDIO_DEVICE_IOS_H_ diff --git a/modules/audio_device/ios/audio_device_ios.mm b/modules/audio_device/ios/audio_device_ios.mm new file mode 100644 index 0000000000..8fa34d10f8 --- /dev/null +++ b/modules/audio_device/ios/audio_device_ios.mm @@ -0,0 +1,908 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <AVFoundation/AVFoundation.h> +#import <Foundation/Foundation.h> + +#include "modules/audio_device/ios/audio_device_ios.h" + +#include <cmath> + +#include "api/array_view.h" +#include "modules/audio_device/fine_audio_buffer.h" +#include "rtc_base/atomic_ops.h" +#include "rtc_base/bind.h" +#include "rtc_base/checks.h" +#include "rtc_base/critical_section.h" +#include "rtc_base/logging.h" +#include "rtc_base/thread.h" +#include "rtc_base/thread_annotations.h" +#include "rtc_base/time_utils.h" +#include "sdk/objc/native/src/audio/helpers.h" +#include "system_wrappers/include/metrics.h" + +#import "modules/audio_device/ios/objc/RTCAudioSessionDelegateAdapter.h" +#import "sdk/objc/base/RTCLogging.h" +#import "sdk/objc/components/audio/RTCAudioSession+Private.h" +#import "sdk/objc/components/audio/RTCAudioSession.h" +#import "sdk/objc/components/audio/RTCAudioSessionConfiguration.h" + +namespace webrtc { + +#define LOGI() RTC_LOG(LS_INFO) << "AudioDeviceIOS::" + +#define LOG_AND_RETURN_IF_ERROR(error, message) \ + do { \ + OSStatus err = error; \ + if (err) { \ + RTC_LOG(LS_ERROR) << message << ": " << err; \ + return false; \ + } \ + } while (0) + +#define LOG_IF_ERROR(error, message) \ + do { \ + OSStatus err = error; \ + if (err) { \ + RTC_LOG(LS_ERROR) << message << ": " << err; \ + } \ + } while (0) + +// Hardcoded delay estimates based on real measurements. +// TODO(henrika): these value is not used in combination with built-in AEC. +// Can most likely be removed. +const UInt16 kFixedPlayoutDelayEstimate = 30; +const UInt16 kFixedRecordDelayEstimate = 30; + +enum AudioDeviceMessageType : uint32_t { + kMessageTypeInterruptionBegin, + kMessageTypeInterruptionEnd, + kMessageTypeValidRouteChange, + kMessageTypeCanPlayOrRecordChange, + kMessageTypePlayoutGlitchDetected, + kMessageOutputVolumeChange, +}; + +using ios::CheckAndLogError; + +#if !defined(NDEBUG) +// Returns true when the code runs on a device simulator. +static bool DeviceIsSimulator() { + return ios::GetDeviceName() == "x86_64"; +} + +// Helper method that logs essential device information strings. +static void LogDeviceInfo() { + RTC_LOG(LS_INFO) << "LogDeviceInfo"; + @autoreleasepool { + RTC_LOG(LS_INFO) << " system name: " << ios::GetSystemName(); + RTC_LOG(LS_INFO) << " system version: " << ios::GetSystemVersionAsString(); + RTC_LOG(LS_INFO) << " device type: " << ios::GetDeviceType(); + RTC_LOG(LS_INFO) << " device name: " << ios::GetDeviceName(); + RTC_LOG(LS_INFO) << " process name: " << ios::GetProcessName(); + RTC_LOG(LS_INFO) << " process ID: " << ios::GetProcessID(); + RTC_LOG(LS_INFO) << " OS version: " << ios::GetOSVersionString(); + RTC_LOG(LS_INFO) << " processing cores: " << ios::GetProcessorCount(); + RTC_LOG(LS_INFO) << " low power mode: " << ios::GetLowPowerModeEnabled(); +#if TARGET_IPHONE_SIMULATOR + RTC_LOG(LS_INFO) << " TARGET_IPHONE_SIMULATOR is defined"; +#endif + RTC_LOG(LS_INFO) << " DeviceIsSimulator: " << DeviceIsSimulator(); + } +} +#endif // !defined(NDEBUG) + +AudioDeviceIOS::AudioDeviceIOS() + : audio_device_buffer_(nullptr), + audio_unit_(nullptr), + recording_(0), + playing_(0), + initialized_(false), + audio_is_initialized_(false), + is_interrupted_(false), + has_configured_session_(false), + num_detected_playout_glitches_(0), + last_playout_time_(0), + num_playout_callbacks_(0), + last_output_volume_change_time_(0) { + LOGI() << "ctor" << ios::GetCurrentThreadDescription(); + io_thread_checker_.DetachFromThread(); + thread_ = rtc::Thread::Current(); + audio_session_observer_ = [[RTCAudioSessionDelegateAdapter alloc] initWithObserver:this]; +} + +AudioDeviceIOS::~AudioDeviceIOS() { + LOGI() << "~dtor" << ios::GetCurrentThreadDescription(); + audio_session_observer_ = nil; + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + Terminate(); +} + +void AudioDeviceIOS::AttachAudioBuffer(AudioDeviceBuffer* audioBuffer) { + LOGI() << "AttachAudioBuffer"; + RTC_DCHECK(audioBuffer); + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + audio_device_buffer_ = audioBuffer; +} + +AudioDeviceGeneric::InitStatus AudioDeviceIOS::Init() { + LOGI() << "Init"; + RTC_DCHECK_RUN_ON(&thread_checker_); + if (initialized_) { + return InitStatus::OK; + } +#if !defined(NDEBUG) + LogDeviceInfo(); +#endif + // Store the preferred sample rate and preferred number of channels already + // here. They have not been set and confirmed yet since configureForWebRTC + // is not called until audio is about to start. However, it makes sense to + // store the parameters now and then verify at a later stage. + RTCAudioSessionConfiguration* config = [RTCAudioSessionConfiguration webRTCConfiguration]; + playout_parameters_.reset(config.sampleRate, config.outputNumberOfChannels); + record_parameters_.reset(config.sampleRate, config.inputNumberOfChannels); + // Ensure that the audio device buffer (ADB) knows about the internal audio + // parameters. Note that, even if we are unable to get a mono audio session, + // we will always tell the I/O audio unit to do a channel format conversion + // to guarantee mono on the "input side" of the audio unit. + UpdateAudioDeviceBuffer(); + initialized_ = true; + return InitStatus::OK; +} + +int32_t AudioDeviceIOS::Terminate() { + LOGI() << "Terminate"; + RTC_DCHECK_RUN_ON(&thread_checker_); + if (!initialized_) { + return 0; + } + StopPlayout(); + StopRecording(); + initialized_ = false; + return 0; +} + +bool AudioDeviceIOS::Initialized() const { + RTC_DCHECK_RUN_ON(&thread_checker_); + return initialized_; +} + +int32_t AudioDeviceIOS::InitPlayout() { + LOGI() << "InitPlayout"; + RTC_DCHECK_RUN_ON(&thread_checker_); + RTC_DCHECK(initialized_); + RTC_DCHECK(!audio_is_initialized_); + RTC_DCHECK(!playing_); + if (!audio_is_initialized_) { + if (!InitPlayOrRecord()) { + RTC_LOG_F(LS_ERROR) << "InitPlayOrRecord failed for InitPlayout!"; + return -1; + } + } + audio_is_initialized_ = true; + return 0; +} + +bool AudioDeviceIOS::PlayoutIsInitialized() const { + RTC_DCHECK_RUN_ON(&thread_checker_); + return audio_is_initialized_; +} + +bool AudioDeviceIOS::RecordingIsInitialized() const { + RTC_DCHECK_RUN_ON(&thread_checker_); + return audio_is_initialized_; +} + +int32_t AudioDeviceIOS::InitRecording() { + LOGI() << "InitRecording"; + RTC_DCHECK_RUN_ON(&thread_checker_); + RTC_DCHECK(initialized_); + RTC_DCHECK(!audio_is_initialized_); + RTC_DCHECK(!recording_); + if (!audio_is_initialized_) { + if (!InitPlayOrRecord()) { + RTC_LOG_F(LS_ERROR) << "InitPlayOrRecord failed for InitRecording!"; + return -1; + } + } + audio_is_initialized_ = true; + return 0; +} + +int32_t AudioDeviceIOS::StartPlayout() { + LOGI() << "StartPlayout"; + RTC_DCHECK_RUN_ON(&thread_checker_); + RTC_DCHECK(audio_is_initialized_); + RTC_DCHECK(!playing_); + RTC_DCHECK(audio_unit_); + if (fine_audio_buffer_) { + fine_audio_buffer_->ResetPlayout(); + } + if (!recording_ && audio_unit_->GetState() == VoiceProcessingAudioUnit::kInitialized) { + if (!audio_unit_->Start()) { + RTCLogError(@"StartPlayout failed to start audio unit."); + return -1; + } + RTC_LOG(LS_INFO) << "Voice-Processing I/O audio unit is now started"; + } + rtc::AtomicOps::ReleaseStore(&playing_, 1); + num_playout_callbacks_ = 0; + num_detected_playout_glitches_ = 0; + return 0; +} + +int32_t AudioDeviceIOS::StopPlayout() { + LOGI() << "StopPlayout"; + RTC_DCHECK_RUN_ON(&thread_checker_); + if (!audio_is_initialized_ || !playing_) { + return 0; + } + if (!recording_) { + ShutdownPlayOrRecord(); + audio_is_initialized_ = false; + } + rtc::AtomicOps::ReleaseStore(&playing_, 0); + + // Derive average number of calls to OnGetPlayoutData() between detected + // audio glitches and add the result to a histogram. + int average_number_of_playout_callbacks_between_glitches = 100000; + RTC_DCHECK_GE(num_playout_callbacks_, num_detected_playout_glitches_); + if (num_detected_playout_glitches_ > 0) { + average_number_of_playout_callbacks_between_glitches = + num_playout_callbacks_ / num_detected_playout_glitches_; + } + RTC_HISTOGRAM_COUNTS_100000("WebRTC.Audio.AveragePlayoutCallbacksBetweenGlitches", + average_number_of_playout_callbacks_between_glitches); + RTCLog(@"Average number of playout callbacks between glitches: %d", + average_number_of_playout_callbacks_between_glitches); + return 0; +} + +int32_t AudioDeviceIOS::StartRecording() { + LOGI() << "StartRecording"; + RTC_DCHECK_RUN_ON(&thread_checker_); + RTC_DCHECK(audio_is_initialized_); + RTC_DCHECK(!recording_); + RTC_DCHECK(audio_unit_); + if (fine_audio_buffer_) { + fine_audio_buffer_->ResetRecord(); + } + if (!playing_ && audio_unit_->GetState() == VoiceProcessingAudioUnit::kInitialized) { + if (!audio_unit_->Start()) { + RTCLogError(@"StartRecording failed to start audio unit."); + return -1; + } + RTC_LOG(LS_INFO) << "Voice-Processing I/O audio unit is now started"; + } + rtc::AtomicOps::ReleaseStore(&recording_, 1); + return 0; +} + +int32_t AudioDeviceIOS::StopRecording() { + LOGI() << "StopRecording"; + RTC_DCHECK_RUN_ON(&thread_checker_); + if (!audio_is_initialized_ || !recording_) { + return 0; + } + if (!playing_) { + ShutdownPlayOrRecord(); + audio_is_initialized_ = false; + } + rtc::AtomicOps::ReleaseStore(&recording_, 0); + return 0; +} + +int32_t AudioDeviceIOS::PlayoutDelay(uint16_t& delayMS) const { + delayMS = kFixedPlayoutDelayEstimate; + return 0; +} + +int AudioDeviceIOS::GetPlayoutAudioParameters(AudioParameters* params) const { + LOGI() << "GetPlayoutAudioParameters"; + RTC_DCHECK(playout_parameters_.is_valid()); + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + *params = playout_parameters_; + return 0; +} + +int AudioDeviceIOS::GetRecordAudioParameters(AudioParameters* params) const { + LOGI() << "GetRecordAudioParameters"; + RTC_DCHECK(record_parameters_.is_valid()); + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + *params = record_parameters_; + return 0; +} + +void AudioDeviceIOS::OnInterruptionBegin() { + RTC_DCHECK(thread_); + LOGI() << "OnInterruptionBegin"; + thread_->Post(RTC_FROM_HERE, this, kMessageTypeInterruptionBegin); +} + +void AudioDeviceIOS::OnInterruptionEnd() { + RTC_DCHECK(thread_); + LOGI() << "OnInterruptionEnd"; + thread_->Post(RTC_FROM_HERE, this, kMessageTypeInterruptionEnd); +} + +void AudioDeviceIOS::OnValidRouteChange() { + RTC_DCHECK(thread_); + thread_->Post(RTC_FROM_HERE, this, kMessageTypeValidRouteChange); +} + +void AudioDeviceIOS::OnCanPlayOrRecordChange(bool can_play_or_record) { + RTC_DCHECK(thread_); + thread_->Post(RTC_FROM_HERE, + this, + kMessageTypeCanPlayOrRecordChange, + new rtc::TypedMessageData<bool>(can_play_or_record)); +} + +void AudioDeviceIOS::OnChangedOutputVolume() { + RTC_DCHECK(thread_); + thread_->Post(RTC_FROM_HERE, this, kMessageOutputVolumeChange); +} + +OSStatus AudioDeviceIOS::OnDeliverRecordedData(AudioUnitRenderActionFlags* flags, + const AudioTimeStamp* time_stamp, + UInt32 bus_number, + UInt32 num_frames, + AudioBufferList* /* io_data */) { + RTC_DCHECK_RUN_ON(&io_thread_checker_); + OSStatus result = noErr; + // Simply return if recording is not enabled. + if (!rtc::AtomicOps::AcquireLoad(&recording_)) return result; + + // Set the size of our own audio buffer and clear it first to avoid copying + // in combination with potential reallocations. + // On real iOS devices, the size will only be set once (at first callback). + record_audio_buffer_.Clear(); + record_audio_buffer_.SetSize(num_frames); + + // Allocate AudioBuffers to be used as storage for the received audio. + // The AudioBufferList structure works as a placeholder for the + // AudioBuffer structure, which holds a pointer to the actual data buffer + // in |record_audio_buffer_|. Recorded audio will be rendered into this memory + // at each input callback when calling AudioUnitRender(). + AudioBufferList audio_buffer_list; + audio_buffer_list.mNumberBuffers = 1; + AudioBuffer* audio_buffer = &audio_buffer_list.mBuffers[0]; + audio_buffer->mNumberChannels = record_parameters_.channels(); + audio_buffer->mDataByteSize = + record_audio_buffer_.size() * VoiceProcessingAudioUnit::kBytesPerSample; + audio_buffer->mData = reinterpret_cast<int8_t*>(record_audio_buffer_.data()); + + // Obtain the recorded audio samples by initiating a rendering cycle. + // Since it happens on the input bus, the |io_data| parameter is a reference + // to the preallocated audio buffer list that the audio unit renders into. + // We can make the audio unit provide a buffer instead in io_data, but we + // currently just use our own. + // TODO(henrika): should error handling be improved? + result = audio_unit_->Render(flags, time_stamp, bus_number, num_frames, &audio_buffer_list); + if (result != noErr) { + RTCLogError(@"Failed to render audio."); + return result; + } + + // Get a pointer to the recorded audio and send it to the WebRTC ADB. + // Use the FineAudioBuffer instance to convert between native buffer size + // and the 10ms buffer size used by WebRTC. + fine_audio_buffer_->DeliverRecordedData(record_audio_buffer_, kFixedRecordDelayEstimate); + return noErr; +} + +OSStatus AudioDeviceIOS::OnGetPlayoutData(AudioUnitRenderActionFlags* flags, + const AudioTimeStamp* time_stamp, + UInt32 bus_number, + UInt32 num_frames, + AudioBufferList* io_data) { + RTC_DCHECK_RUN_ON(&io_thread_checker_); + // Verify 16-bit, noninterleaved mono PCM signal format. + RTC_DCHECK_EQ(1, io_data->mNumberBuffers); + AudioBuffer* audio_buffer = &io_data->mBuffers[0]; + RTC_DCHECK_EQ(1, audio_buffer->mNumberChannels); + + // Produce silence and give audio unit a hint about it if playout is not + // activated. + if (!rtc::AtomicOps::AcquireLoad(&playing_)) { + const size_t size_in_bytes = audio_buffer->mDataByteSize; + RTC_CHECK_EQ(size_in_bytes / VoiceProcessingAudioUnit::kBytesPerSample, num_frames); + *flags |= kAudioUnitRenderAction_OutputIsSilence; + memset(static_cast<int8_t*>(audio_buffer->mData), 0, size_in_bytes); + return noErr; + } + + // Measure time since last call to OnGetPlayoutData() and see if it is larger + // than a well defined threshold which depends on the current IO buffer size. + // If so, we have an indication of a glitch in the output audio since the + // core audio layer will most likely run dry in this state. + ++num_playout_callbacks_; + const int64_t now_time = rtc::TimeMillis(); + if (time_stamp->mSampleTime != num_frames) { + const int64_t delta_time = now_time - last_playout_time_; + const int glitch_threshold = 1.6 * playout_parameters_.GetBufferSizeInMilliseconds(); + if (delta_time > glitch_threshold) { + RTCLogWarning(@"Possible playout audio glitch detected.\n" + " Time since last OnGetPlayoutData was %lld ms.\n", + delta_time); + // Exclude extreme delta values since they do most likely not correspond + // to a real glitch. Instead, the most probable cause is that a headset + // has been plugged in or out. There are more direct ways to detect + // audio device changes (see HandleValidRouteChange()) but experiments + // show that using it leads to more complex implementations. + // TODO(henrika): more tests might be needed to come up with an even + // better upper limit. + if (glitch_threshold < 120 && delta_time > 120) { + RTCLog(@"Glitch warning is ignored. Probably caused by device switch."); + } else { + thread_->Post(RTC_FROM_HERE, this, kMessageTypePlayoutGlitchDetected); + } + } + } + last_playout_time_ = now_time; + + // Read decoded 16-bit PCM samples from WebRTC (using a size that matches + // the native I/O audio unit) and copy the result to the audio buffer in the + // |io_data| destination. + fine_audio_buffer_->GetPlayoutData( + rtc::ArrayView<int16_t>(static_cast<int16_t*>(audio_buffer->mData), num_frames), + kFixedPlayoutDelayEstimate); + return noErr; +} + +void AudioDeviceIOS::OnMessage(rtc::Message* msg) { + switch (msg->message_id) { + case kMessageTypeInterruptionBegin: + HandleInterruptionBegin(); + break; + case kMessageTypeInterruptionEnd: + HandleInterruptionEnd(); + break; + case kMessageTypeValidRouteChange: + HandleValidRouteChange(); + break; + case kMessageTypeCanPlayOrRecordChange: { + rtc::TypedMessageData<bool>* data = static_cast<rtc::TypedMessageData<bool>*>(msg->pdata); + HandleCanPlayOrRecordChange(data->data()); + delete data; + break; + } + case kMessageTypePlayoutGlitchDetected: + HandlePlayoutGlitchDetected(); + break; + case kMessageOutputVolumeChange: + HandleOutputVolumeChange(); + break; + } +} + +void AudioDeviceIOS::HandleInterruptionBegin() { + RTC_DCHECK_RUN_ON(&thread_checker_); + RTCLog(@"Interruption begin. IsInterrupted changed from %d to 1.", is_interrupted_); + if (audio_unit_ && audio_unit_->GetState() == VoiceProcessingAudioUnit::kStarted) { + RTCLog(@"Stopping the audio unit due to interruption begin."); + if (!audio_unit_->Stop()) { + RTCLogError(@"Failed to stop the audio unit for interruption begin."); + } else { + PrepareForNewStart(); + } + } + is_interrupted_ = true; +} + +void AudioDeviceIOS::HandleInterruptionEnd() { + RTC_DCHECK_RUN_ON(&thread_checker_); + RTCLog(@"Interruption ended. IsInterrupted changed from %d to 0. " + "Updating audio unit state.", + is_interrupted_); + is_interrupted_ = false; + UpdateAudioUnit([RTCAudioSession sharedInstance].canPlayOrRecord); +} + +void AudioDeviceIOS::HandleValidRouteChange() { + RTC_DCHECK_RUN_ON(&thread_checker_); + RTCAudioSession* session = [RTCAudioSession sharedInstance]; + RTCLog(@"%@", session); + HandleSampleRateChange(session.sampleRate); +} + +void AudioDeviceIOS::HandleCanPlayOrRecordChange(bool can_play_or_record) { + RTCLog(@"Handling CanPlayOrRecord change to: %d", can_play_or_record); + UpdateAudioUnit(can_play_or_record); +} + +void AudioDeviceIOS::HandleSampleRateChange(float sample_rate) { + RTC_DCHECK_RUN_ON(&thread_checker_); + RTCLog(@"Handling sample rate change to %f.", sample_rate); + + // Don't do anything if we're interrupted. + if (is_interrupted_) { + RTCLog(@"Ignoring sample rate change to %f due to interruption.", sample_rate); + return; + } + + // If we don't have an audio unit yet, or the audio unit is uninitialized, + // there is no work to do. + if (!audio_unit_ || audio_unit_->GetState() < VoiceProcessingAudioUnit::kInitialized) { + return; + } + + // The audio unit is already initialized or started. + // Check to see if the sample rate or buffer size has changed. + RTCAudioSession* session = [RTCAudioSession sharedInstance]; + const double session_sample_rate = session.sampleRate; + const NSTimeInterval session_buffer_duration = session.IOBufferDuration; + const size_t session_frames_per_buffer = + static_cast<size_t>(session_sample_rate * session_buffer_duration + .5); + const double current_sample_rate = playout_parameters_.sample_rate(); + const size_t current_frames_per_buffer = playout_parameters_.frames_per_buffer(); + RTCLog(@"Handling playout sample rate change to: %f\n" + " Session sample rate: %f frames_per_buffer: %lu\n" + " ADM sample rate: %f frames_per_buffer: %lu", + sample_rate, + session_sample_rate, + (unsigned long)session_frames_per_buffer, + current_sample_rate, + (unsigned long)current_frames_per_buffer); + + // Sample rate and buffer size are the same, no work to do. + if (std::abs(current_sample_rate - session_sample_rate) <= DBL_EPSILON && + current_frames_per_buffer == session_frames_per_buffer) { + RTCLog(@"Ignoring sample rate change since audio parameters are intact."); + return; + } + + // Extra sanity check to ensure that the new sample rate is valid. + if (session_sample_rate <= 0.0) { + RTCLogError(@"Sample rate is invalid: %f", session_sample_rate); + return; + } + + // We need to adjust our format and buffer sizes. + // The stream format is about to be changed and it requires that we first + // stop and uninitialize the audio unit to deallocate its resources. + RTCLog(@"Stopping and uninitializing audio unit to adjust buffers."); + bool restart_audio_unit = false; + if (audio_unit_->GetState() == VoiceProcessingAudioUnit::kStarted) { + audio_unit_->Stop(); + restart_audio_unit = true; + PrepareForNewStart(); + } + if (audio_unit_->GetState() == VoiceProcessingAudioUnit::kInitialized) { + audio_unit_->Uninitialize(); + } + + // Allocate new buffers given the new stream format. + SetupAudioBuffersForActiveAudioSession(); + + // Initialize the audio unit again with the new sample rate. + RTC_DCHECK_EQ(playout_parameters_.sample_rate(), session_sample_rate); + if (!audio_unit_->Initialize(session_sample_rate)) { + RTCLogError(@"Failed to initialize the audio unit with sample rate: %f", session_sample_rate); + return; + } + + // Restart the audio unit if it was already running. + if (restart_audio_unit && !audio_unit_->Start()) { + RTCLogError(@"Failed to start audio unit with sample rate: %f", session_sample_rate); + return; + } + RTCLog(@"Successfully handled sample rate change."); +} + +void AudioDeviceIOS::HandlePlayoutGlitchDetected() { + RTC_DCHECK_RUN_ON(&thread_checker_); + // Don't update metrics if we're interrupted since a "glitch" is expected + // in this state. + if (is_interrupted_) { + RTCLog(@"Ignoring audio glitch due to interruption."); + return; + } + // Avoid doing glitch detection for two seconds after a volume change + // has been detected to reduce the risk of false alarm. + if (last_output_volume_change_time_ > 0 && + rtc::TimeSince(last_output_volume_change_time_) < 2000) { + RTCLog(@"Ignoring audio glitch due to recent output volume change."); + return; + } + num_detected_playout_glitches_++; + RTCLog(@"Number of detected playout glitches: %lld", num_detected_playout_glitches_); + + int64_t glitch_count = num_detected_playout_glitches_; + dispatch_async(dispatch_get_main_queue(), ^{ + RTCAudioSession* session = [RTCAudioSession sharedInstance]; + [session notifyDidDetectPlayoutGlitch:glitch_count]; + }); +} + +void AudioDeviceIOS::HandleOutputVolumeChange() { + RTC_DCHECK_RUN_ON(&thread_checker_); + RTCLog(@"Output volume change detected."); + // Store time of this detection so it can be used to defer detection of + // glitches too close in time to this event. + last_output_volume_change_time_ = rtc::TimeMillis(); +} + +void AudioDeviceIOS::UpdateAudioDeviceBuffer() { + LOGI() << "UpdateAudioDevicebuffer"; + // AttachAudioBuffer() is called at construction by the main class but check + // just in case. + RTC_DCHECK(audio_device_buffer_) << "AttachAudioBuffer must be called first"; + RTC_DCHECK_GT(playout_parameters_.sample_rate(), 0); + RTC_DCHECK_GT(record_parameters_.sample_rate(), 0); + RTC_DCHECK_EQ(playout_parameters_.channels(), 1); + RTC_DCHECK_EQ(record_parameters_.channels(), 1); + // Inform the audio device buffer (ADB) about the new audio format. + audio_device_buffer_->SetPlayoutSampleRate(playout_parameters_.sample_rate()); + audio_device_buffer_->SetPlayoutChannels(playout_parameters_.channels()); + audio_device_buffer_->SetRecordingSampleRate(record_parameters_.sample_rate()); + audio_device_buffer_->SetRecordingChannels(record_parameters_.channels()); +} + +void AudioDeviceIOS::SetupAudioBuffersForActiveAudioSession() { + LOGI() << "SetupAudioBuffersForActiveAudioSession"; + // Verify the current values once the audio session has been activated. + RTCAudioSession* session = [RTCAudioSession sharedInstance]; + double sample_rate = session.sampleRate; + NSTimeInterval io_buffer_duration = session.IOBufferDuration; + RTCLog(@"%@", session); + + // Log a warning message for the case when we are unable to set the preferred + // hardware sample rate but continue and use the non-ideal sample rate after + // reinitializing the audio parameters. Most BT headsets only support 8kHz or + // 16kHz. + RTCAudioSessionConfiguration* webRTCConfig = [RTCAudioSessionConfiguration webRTCConfiguration]; + if (sample_rate != webRTCConfig.sampleRate) { + RTC_LOG(LS_WARNING) << "Unable to set the preferred sample rate"; + } + + // Crash reports indicates that it can happen in rare cases that the reported + // sample rate is less than or equal to zero. If that happens and if a valid + // sample rate has already been set during initialization, the best guess we + // can do is to reuse the current sample rate. + if (sample_rate <= DBL_EPSILON && playout_parameters_.sample_rate() > 0) { + RTCLogError(@"Reported rate is invalid: %f. " + "Using %d as sample rate instead.", + sample_rate, playout_parameters_.sample_rate()); + sample_rate = playout_parameters_.sample_rate(); + } + + // At this stage, we also know the exact IO buffer duration and can add + // that info to the existing audio parameters where it is converted into + // number of audio frames. + // Example: IO buffer size = 0.008 seconds <=> 128 audio frames at 16kHz. + // Hence, 128 is the size we expect to see in upcoming render callbacks. + playout_parameters_.reset(sample_rate, playout_parameters_.channels(), io_buffer_duration); + RTC_DCHECK(playout_parameters_.is_complete()); + record_parameters_.reset(sample_rate, record_parameters_.channels(), io_buffer_duration); + RTC_DCHECK(record_parameters_.is_complete()); + RTC_LOG(LS_INFO) << " frames per I/O buffer: " << playout_parameters_.frames_per_buffer(); + RTC_LOG(LS_INFO) << " bytes per I/O buffer: " << playout_parameters_.GetBytesPerBuffer(); + RTC_DCHECK_EQ(playout_parameters_.GetBytesPerBuffer(), record_parameters_.GetBytesPerBuffer()); + + // Update the ADB parameters since the sample rate might have changed. + UpdateAudioDeviceBuffer(); + + // Create a modified audio buffer class which allows us to ask for, + // or deliver, any number of samples (and not only multiple of 10ms) to match + // the native audio unit buffer size. + RTC_DCHECK(audio_device_buffer_); + fine_audio_buffer_.reset(new FineAudioBuffer(audio_device_buffer_)); +} + +bool AudioDeviceIOS::CreateAudioUnit() { + RTC_DCHECK(!audio_unit_); + + audio_unit_.reset(new VoiceProcessingAudioUnit(this)); + if (!audio_unit_->Init()) { + audio_unit_.reset(); + return false; + } + + return true; +} + +void AudioDeviceIOS::UpdateAudioUnit(bool can_play_or_record) { + RTC_DCHECK_RUN_ON(&thread_checker_); + RTCLog(@"Updating audio unit state. CanPlayOrRecord=%d IsInterrupted=%d", + can_play_or_record, + is_interrupted_); + + if (is_interrupted_) { + RTCLog(@"Ignoring audio unit update due to interruption."); + return; + } + + // If we're not initialized we don't need to do anything. Audio unit will + // be initialized on initialization. + if (!audio_is_initialized_) return; + + // If we're initialized, we must have an audio unit. + RTC_DCHECK(audio_unit_); + + bool should_initialize_audio_unit = false; + bool should_uninitialize_audio_unit = false; + bool should_start_audio_unit = false; + bool should_stop_audio_unit = false; + + switch (audio_unit_->GetState()) { + case VoiceProcessingAudioUnit::kInitRequired: + RTCLog(@"VPAU state: InitRequired"); + RTC_NOTREACHED(); + break; + case VoiceProcessingAudioUnit::kUninitialized: + RTCLog(@"VPAU state: Uninitialized"); + should_initialize_audio_unit = can_play_or_record; + should_start_audio_unit = should_initialize_audio_unit && (playing_ || recording_); + break; + case VoiceProcessingAudioUnit::kInitialized: + RTCLog(@"VPAU state: Initialized"); + should_start_audio_unit = can_play_or_record && (playing_ || recording_); + should_uninitialize_audio_unit = !can_play_or_record; + break; + case VoiceProcessingAudioUnit::kStarted: + RTCLog(@"VPAU state: Started"); + RTC_DCHECK(playing_ || recording_); + should_stop_audio_unit = !can_play_or_record; + should_uninitialize_audio_unit = should_stop_audio_unit; + break; + } + + if (should_initialize_audio_unit) { + RTCLog(@"Initializing audio unit for UpdateAudioUnit"); + ConfigureAudioSession(); + SetupAudioBuffersForActiveAudioSession(); + if (!audio_unit_->Initialize(playout_parameters_.sample_rate())) { + RTCLogError(@"Failed to initialize audio unit."); + return; + } + } + + if (should_start_audio_unit) { + RTCLog(@"Starting audio unit for UpdateAudioUnit"); + // Log session settings before trying to start audio streaming. + RTCAudioSession* session = [RTCAudioSession sharedInstance]; + RTCLog(@"%@", session); + if (!audio_unit_->Start()) { + RTCLogError(@"Failed to start audio unit."); + return; + } + } + + if (should_stop_audio_unit) { + RTCLog(@"Stopping audio unit for UpdateAudioUnit"); + if (!audio_unit_->Stop()) { + RTCLogError(@"Failed to stop audio unit."); + return; + } + } + + if (should_uninitialize_audio_unit) { + RTCLog(@"Uninitializing audio unit for UpdateAudioUnit"); + audio_unit_->Uninitialize(); + UnconfigureAudioSession(); + } +} + +bool AudioDeviceIOS::ConfigureAudioSession() { + RTC_DCHECK_RUN_ON(&thread_checker_); + RTCLog(@"Configuring audio session."); + if (has_configured_session_) { + RTCLogWarning(@"Audio session already configured."); + return false; + } + RTCAudioSession* session = [RTCAudioSession sharedInstance]; + [session lockForConfiguration]; + bool success = [session configureWebRTCSession:nil]; + [session unlockForConfiguration]; + if (success) { + has_configured_session_ = true; + RTCLog(@"Configured audio session."); + } else { + RTCLog(@"Failed to configure audio session."); + } + return success; +} + +void AudioDeviceIOS::UnconfigureAudioSession() { + RTC_DCHECK_RUN_ON(&thread_checker_); + RTCLog(@"Unconfiguring audio session."); + if (!has_configured_session_) { + RTCLogWarning(@"Audio session already unconfigured."); + return; + } + RTCAudioSession* session = [RTCAudioSession sharedInstance]; + [session lockForConfiguration]; + [session unconfigureWebRTCSession:nil]; + [session unlockForConfiguration]; + has_configured_session_ = false; + RTCLog(@"Unconfigured audio session."); +} + +bool AudioDeviceIOS::InitPlayOrRecord() { + LOGI() << "InitPlayOrRecord"; + RTC_DCHECK_RUN_ON(&thread_checker_); + + // There should be no audio unit at this point. + if (!CreateAudioUnit()) { + return false; + } + + RTCAudioSession* session = [RTCAudioSession sharedInstance]; + // Subscribe to audio session events. + [session pushDelegate:audio_session_observer_]; + is_interrupted_ = session.isInterrupted ? true : false; + + // Lock the session to make configuration changes. + [session lockForConfiguration]; + NSError* error = nil; + if (![session beginWebRTCSession:&error]) { + [session unlockForConfiguration]; + RTCLogError(@"Failed to begin WebRTC session: %@", error.localizedDescription); + audio_unit_.reset(); + return false; + } + + // If we are ready to play or record, and if the audio session can be + // configured, then initialize the audio unit. + if (session.canPlayOrRecord) { + if (!ConfigureAudioSession()) { + // One possible reason for failure is if an attempt was made to use the + // audio session during or after a Media Services failure. + // See AVAudioSessionErrorCodeMediaServicesFailed for details. + [session unlockForConfiguration]; + audio_unit_.reset(); + return false; + } + SetupAudioBuffersForActiveAudioSession(); + audio_unit_->Initialize(playout_parameters_.sample_rate()); + } + + // Release the lock. + [session unlockForConfiguration]; + return true; +} + +void AudioDeviceIOS::ShutdownPlayOrRecord() { + LOGI() << "ShutdownPlayOrRecord"; + RTC_DCHECK_RUN_ON(&thread_checker_); + + // Stop the audio unit to prevent any additional audio callbacks. + audio_unit_->Stop(); + + // Close and delete the voice-processing I/O unit. + audio_unit_.reset(); + + // Detach thread checker for the AURemoteIO::IOThread to ensure that the + // next session uses a fresh thread id. + io_thread_checker_.DetachFromThread(); + + // Remove audio session notification observers. + RTCAudioSession* session = [RTCAudioSession sharedInstance]; + [session removeDelegate:audio_session_observer_]; + + // All I/O should be stopped or paused prior to deactivating the audio + // session, hence we deactivate as last action. + [session lockForConfiguration]; + UnconfigureAudioSession(); + [session endWebRTCSession:nil]; + [session unlockForConfiguration]; +} + +void AudioDeviceIOS::PrepareForNewStart() { + LOGI() << "PrepareForNewStart"; + // The audio unit has been stopped and preparations are needed for an upcoming + // restart. It will result in audio callbacks from a new native I/O thread + // which means that we must detach thread checkers here to be prepared for an + // upcoming new audio stream. + io_thread_checker_.DetachFromThread(); +} + +} // namespace webrtc diff --git a/modules/audio_device/ios/audio_device_not_implemented_ios.mm b/modules/audio_device/ios/audio_device_not_implemented_ios.mm new file mode 100644 index 0000000000..2e99aea97e --- /dev/null +++ b/modules/audio_device/ios/audio_device_not_implemented_ios.mm @@ -0,0 +1,205 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_device/ios/audio_device_ios.h" + +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" + +namespace webrtc { + +int32_t AudioDeviceIOS::ActiveAudioLayer(AudioDeviceModule::AudioLayer& audioLayer) const { + audioLayer = AudioDeviceModule::kPlatformDefaultAudio; + return 0; +} + +int16_t AudioDeviceIOS::PlayoutDevices() { + // TODO(henrika): improve. + RTC_LOG_F(LS_WARNING) << "Not implemented"; + return (int16_t)1; +} + +int16_t AudioDeviceIOS::RecordingDevices() { + // TODO(henrika): improve. + RTC_LOG_F(LS_WARNING) << "Not implemented"; + return (int16_t)1; +} + +int32_t AudioDeviceIOS::InitSpeaker() { + return 0; +} + +bool AudioDeviceIOS::SpeakerIsInitialized() const { + return true; +} + +int32_t AudioDeviceIOS::SpeakerVolumeIsAvailable(bool& available) { + available = false; + return 0; +} + +int32_t AudioDeviceIOS::SetSpeakerVolume(uint32_t volume) { + RTC_NOTREACHED() << "Not implemented"; + return -1; +} + +int32_t AudioDeviceIOS::SpeakerVolume(uint32_t& volume) const { + RTC_NOTREACHED() << "Not implemented"; + return -1; +} + +int32_t AudioDeviceIOS::MaxSpeakerVolume(uint32_t& maxVolume) const { + RTC_NOTREACHED() << "Not implemented"; + return -1; +} + +int32_t AudioDeviceIOS::MinSpeakerVolume(uint32_t& minVolume) const { + RTC_NOTREACHED() << "Not implemented"; + return -1; +} + +int32_t AudioDeviceIOS::SpeakerMuteIsAvailable(bool& available) { + available = false; + return 0; +} + +int32_t AudioDeviceIOS::SetSpeakerMute(bool enable) { + RTC_NOTREACHED() << "Not implemented"; + return -1; +} + +int32_t AudioDeviceIOS::SpeakerMute(bool& enabled) const { + RTC_NOTREACHED() << "Not implemented"; + return -1; +} + +int32_t AudioDeviceIOS::SetPlayoutDevice(uint16_t index) { + RTC_LOG_F(LS_WARNING) << "Not implemented"; + return 0; +} + +int32_t AudioDeviceIOS::SetPlayoutDevice(AudioDeviceModule::WindowsDeviceType) { + RTC_NOTREACHED() << "Not implemented"; + return -1; +} + +int32_t AudioDeviceIOS::InitMicrophone() { + return 0; +} + +bool AudioDeviceIOS::MicrophoneIsInitialized() const { + return true; +} + +int32_t AudioDeviceIOS::MicrophoneMuteIsAvailable(bool& available) { + available = false; + return 0; +} + +int32_t AudioDeviceIOS::SetMicrophoneMute(bool enable) { + RTC_NOTREACHED() << "Not implemented"; + return -1; +} + +int32_t AudioDeviceIOS::MicrophoneMute(bool& enabled) const { + RTC_NOTREACHED() << "Not implemented"; + return -1; +} + +int32_t AudioDeviceIOS::StereoRecordingIsAvailable(bool& available) { + available = false; + return 0; +} + +int32_t AudioDeviceIOS::SetStereoRecording(bool enable) { + RTC_LOG_F(LS_WARNING) << "Not implemented"; + return -1; +} + +int32_t AudioDeviceIOS::StereoRecording(bool& enabled) const { + enabled = false; + return 0; +} + +int32_t AudioDeviceIOS::StereoPlayoutIsAvailable(bool& available) { + available = false; + return 0; +} + +int32_t AudioDeviceIOS::SetStereoPlayout(bool enable) { + RTC_LOG_F(LS_WARNING) << "Not implemented"; + return -1; +} + +int32_t AudioDeviceIOS::StereoPlayout(bool& enabled) const { + enabled = false; + return 0; +} + +int32_t AudioDeviceIOS::MicrophoneVolumeIsAvailable(bool& available) { + available = false; + return 0; +} + +int32_t AudioDeviceIOS::SetMicrophoneVolume(uint32_t volume) { + RTC_NOTREACHED() << "Not implemented"; + return -1; +} + +int32_t AudioDeviceIOS::MicrophoneVolume(uint32_t& volume) const { + RTC_NOTREACHED() << "Not implemented"; + return -1; +} + +int32_t AudioDeviceIOS::MaxMicrophoneVolume(uint32_t& maxVolume) const { + RTC_NOTREACHED() << "Not implemented"; + return -1; +} + +int32_t AudioDeviceIOS::MinMicrophoneVolume(uint32_t& minVolume) const { + RTC_NOTREACHED() << "Not implemented"; + return -1; +} + +int32_t AudioDeviceIOS::PlayoutDeviceName(uint16_t index, + char name[kAdmMaxDeviceNameSize], + char guid[kAdmMaxGuidSize]) { + RTC_NOTREACHED() << "Not implemented"; + return -1; +} + +int32_t AudioDeviceIOS::RecordingDeviceName(uint16_t index, + char name[kAdmMaxDeviceNameSize], + char guid[kAdmMaxGuidSize]) { + RTC_NOTREACHED() << "Not implemented"; + return -1; +} + +int32_t AudioDeviceIOS::SetRecordingDevice(uint16_t index) { + RTC_LOG_F(LS_WARNING) << "Not implemented"; + return 0; +} + +int32_t AudioDeviceIOS::SetRecordingDevice(AudioDeviceModule::WindowsDeviceType) { + RTC_NOTREACHED() << "Not implemented"; + return -1; +} + +int32_t AudioDeviceIOS::PlayoutIsAvailable(bool& available) { + available = true; + return 0; +} + +int32_t AudioDeviceIOS::RecordingIsAvailable(bool& available) { + available = true; + return 0; +} + +} // namespace webrtc diff --git a/modules/audio_device/ios/audio_device_unittest_ios.mm b/modules/audio_device/ios/audio_device_unittest_ios.mm new file mode 100644 index 0000000000..74cfcc2c51 --- /dev/null +++ b/modules/audio_device/ios/audio_device_unittest_ios.mm @@ -0,0 +1,877 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include <algorithm> +#include <limits> +#include <list> +#include <memory> +#include <numeric> +#include <string> +#include <vector> + +#include "api/scoped_refptr.h" +#include "modules/audio_device/audio_device_impl.h" +#include "modules/audio_device/include/audio_device.h" +#include "modules/audio_device/include/mock_audio_transport.h" +#include "modules/audio_device/ios/audio_device_ios.h" +#include "rtc_base/arraysize.h" +#include "rtc_base/critical_section.h" +#include "rtc_base/event.h" +#include "rtc_base/format_macros.h" +#include "rtc_base/logging.h" +#include "rtc_base/time_utils.h" +#include "test/gmock.h" +#include "test/gtest.h" +#include "test/testsupport/file_utils.h" + +#import "sdk/objc/components/audio/RTCAudioSession+Private.h" +#import "sdk/objc/components/audio/RTCAudioSession.h" + +using std::cout; +using std::endl; +using ::testing::_; +using ::testing::AtLeast; +using ::testing::Gt; +using ::testing::Invoke; +using ::testing::NiceMock; +using ::testing::NotNull; +using ::testing::Return; + +// #define ENABLE_DEBUG_PRINTF +#ifdef ENABLE_DEBUG_PRINTF +#define PRINTD(...) fprintf(stderr, __VA_ARGS__); +#else +#define PRINTD(...) ((void)0) +#endif +#define PRINT(...) fprintf(stderr, __VA_ARGS__); + +namespace webrtc { + +// Number of callbacks (input or output) the tests waits for before we set +// an event indicating that the test was OK. +static const size_t kNumCallbacks = 10; +// Max amount of time we wait for an event to be set while counting callbacks. +static const int kTestTimeOutInMilliseconds = 10 * 1000; +// Number of bits per PCM audio sample. +static const size_t kBitsPerSample = 16; +// Number of bytes per PCM audio sample. +static const size_t kBytesPerSample = kBitsPerSample / 8; +// Average number of audio callbacks per second assuming 10ms packet size. +static const size_t kNumCallbacksPerSecond = 100; +// Play out a test file during this time (unit is in seconds). +static const int kFilePlayTimeInSec = 15; +// Run the full-duplex test during this time (unit is in seconds). +// Note that first |kNumIgnoreFirstCallbacks| are ignored. +static const int kFullDuplexTimeInSec = 10; +// Wait for the callback sequence to stabilize by ignoring this amount of the +// initial callbacks (avoids initial FIFO access). +// Only used in the RunPlayoutAndRecordingInFullDuplex test. +static const size_t kNumIgnoreFirstCallbacks = 50; +// Sets the number of impulses per second in the latency test. +// TODO(henrika): fine tune this setting for iOS. +static const int kImpulseFrequencyInHz = 1; +// Length of round-trip latency measurements. Number of transmitted impulses +// is kImpulseFrequencyInHz * kMeasureLatencyTimeInSec - 1. +// TODO(henrika): fine tune this setting for iOS. +static const int kMeasureLatencyTimeInSec = 5; +// Utilized in round-trip latency measurements to avoid capturing noise samples. +// TODO(henrika): fine tune this setting for iOS. +static const int kImpulseThreshold = 50; +static const char kTag[] = "[..........] "; + +enum TransportType { + kPlayout = 0x1, + kRecording = 0x2, +}; + +// Interface for processing the audio stream. Real implementations can e.g. +// run audio in loopback, read audio from a file or perform latency +// measurements. +class AudioStreamInterface { + public: + virtual void Write(const void* source, size_t num_frames) = 0; + virtual void Read(void* destination, size_t num_frames) = 0; + + protected: + virtual ~AudioStreamInterface() {} +}; + +// Reads audio samples from a PCM file where the file is stored in memory at +// construction. +class FileAudioStream : public AudioStreamInterface { + public: + FileAudioStream(size_t num_callbacks, + const std::string& file_name, + int sample_rate) + : file_size_in_bytes_(0), sample_rate_(sample_rate), file_pos_(0) { + file_size_in_bytes_ = test::GetFileSize(file_name); + sample_rate_ = sample_rate; + EXPECT_GE(file_size_in_callbacks(), num_callbacks) + << "Size of test file is not large enough to last during the test."; + const size_t num_16bit_samples = + test::GetFileSize(file_name) / kBytesPerSample; + file_.reset(new int16_t[num_16bit_samples]); + FILE* audio_file = fopen(file_name.c_str(), "rb"); + EXPECT_NE(audio_file, nullptr); + size_t num_samples_read = + fread(file_.get(), sizeof(int16_t), num_16bit_samples, audio_file); + EXPECT_EQ(num_samples_read, num_16bit_samples); + fclose(audio_file); + } + + // AudioStreamInterface::Write() is not implemented. + void Write(const void* source, size_t num_frames) override {} + + // Read samples from file stored in memory (at construction) and copy + // |num_frames| (<=> 10ms) to the |destination| byte buffer. + void Read(void* destination, size_t num_frames) override { + memcpy(destination, static_cast<int16_t*>(&file_[file_pos_]), + num_frames * sizeof(int16_t)); + file_pos_ += num_frames; + } + + int file_size_in_seconds() const { + return static_cast<int>( + file_size_in_bytes_ / (kBytesPerSample * sample_rate_)); + } + size_t file_size_in_callbacks() const { + return file_size_in_seconds() * kNumCallbacksPerSecond; + } + + private: + size_t file_size_in_bytes_; + int sample_rate_; + std::unique_ptr<int16_t[]> file_; + size_t file_pos_; +}; + +// Simple first in first out (FIFO) class that wraps a list of 16-bit audio +// buffers of fixed size and allows Write and Read operations. The idea is to +// store recorded audio buffers (using Write) and then read (using Read) these +// stored buffers with as short delay as possible when the audio layer needs +// data to play out. The number of buffers in the FIFO will stabilize under +// normal conditions since there will be a balance between Write and Read calls. +// The container is a std::list container and access is protected with a lock +// since both sides (playout and recording) are driven by its own thread. +class FifoAudioStream : public AudioStreamInterface { + public: + explicit FifoAudioStream(size_t frames_per_buffer) + : frames_per_buffer_(frames_per_buffer), + bytes_per_buffer_(frames_per_buffer_ * sizeof(int16_t)), + fifo_(new AudioBufferList), + largest_size_(0), + total_written_elements_(0), + write_count_(0) { + EXPECT_NE(fifo_.get(), nullptr); + } + + ~FifoAudioStream() { Flush(); } + + // Allocate new memory, copy |num_frames| samples from |source| into memory + // and add pointer to the memory location to end of the list. + // Increases the size of the FIFO by one element. + void Write(const void* source, size_t num_frames) override { + ASSERT_EQ(num_frames, frames_per_buffer_); + PRINTD("+"); + if (write_count_++ < kNumIgnoreFirstCallbacks) { + return; + } + int16_t* memory = new int16_t[frames_per_buffer_]; + memcpy(static_cast<int16_t*>(&memory[0]), source, bytes_per_buffer_); + rtc::CritScope lock(&lock_); + fifo_->push_back(memory); + const size_t size = fifo_->size(); + if (size > largest_size_) { + largest_size_ = size; + PRINTD("(%" PRIuS ")", largest_size_); + } + total_written_elements_ += size; + } + + // Read pointer to data buffer from front of list, copy |num_frames| of stored + // data into |destination| and delete the utilized memory allocation. + // Decreases the size of the FIFO by one element. + void Read(void* destination, size_t num_frames) override { + ASSERT_EQ(num_frames, frames_per_buffer_); + PRINTD("-"); + rtc::CritScope lock(&lock_); + if (fifo_->empty()) { + memset(destination, 0, bytes_per_buffer_); + } else { + int16_t* memory = fifo_->front(); + fifo_->pop_front(); + memcpy(destination, static_cast<int16_t*>(&memory[0]), bytes_per_buffer_); + delete memory; + } + } + + size_t size() const { return fifo_->size(); } + + size_t largest_size() const { return largest_size_; } + + size_t average_size() const { + return (total_written_elements_ == 0) + ? 0.0 + : 0.5 + + static_cast<float>(total_written_elements_) / + (write_count_ - kNumIgnoreFirstCallbacks); + } + + private: + void Flush() { + for (auto it = fifo_->begin(); it != fifo_->end(); ++it) { + delete *it; + } + fifo_->clear(); + } + + using AudioBufferList = std::list<int16_t*>; + rtc::CriticalSection lock_; + const size_t frames_per_buffer_; + const size_t bytes_per_buffer_; + std::unique_ptr<AudioBufferList> fifo_; + size_t largest_size_; + size_t total_written_elements_; + size_t write_count_; +}; + +// Inserts periodic impulses and measures the latency between the time of +// transmission and time of receiving the same impulse. +// Usage requires a special hardware called Audio Loopback Dongle. +// See http://source.android.com/devices/audio/loopback.html for details. +class LatencyMeasuringAudioStream : public AudioStreamInterface { + public: + explicit LatencyMeasuringAudioStream(size_t frames_per_buffer) + : frames_per_buffer_(frames_per_buffer), + bytes_per_buffer_(frames_per_buffer_ * sizeof(int16_t)), + play_count_(0), + rec_count_(0), + pulse_time_(0) {} + + // Insert periodic impulses in first two samples of |destination|. + void Read(void* destination, size_t num_frames) override { + ASSERT_EQ(num_frames, frames_per_buffer_); + if (play_count_ == 0) { + PRINT("["); + } + play_count_++; + memset(destination, 0, bytes_per_buffer_); + if (play_count_ % (kNumCallbacksPerSecond / kImpulseFrequencyInHz) == 0) { + if (pulse_time_ == 0) { + pulse_time_ = rtc::TimeMillis(); + } + PRINT("."); + const int16_t impulse = std::numeric_limits<int16_t>::max(); + int16_t* ptr16 = static_cast<int16_t*>(destination); + for (size_t i = 0; i < 2; ++i) { + ptr16[i] = impulse; + } + } + } + + // Detect received impulses in |source|, derive time between transmission and + // detection and add the calculated delay to list of latencies. + void Write(const void* source, size_t num_frames) override { + ASSERT_EQ(num_frames, frames_per_buffer_); + rec_count_++; + if (pulse_time_ == 0) { + // Avoid detection of new impulse response until a new impulse has + // been transmitted (sets |pulse_time_| to value larger than zero). + return; + } + const int16_t* ptr16 = static_cast<const int16_t*>(source); + std::vector<int16_t> vec(ptr16, ptr16 + num_frames); + // Find max value in the audio buffer. + int max = *std::max_element(vec.begin(), vec.end()); + // Find index (element position in vector) of the max element. + int index_of_max = + std::distance(vec.begin(), std::find(vec.begin(), vec.end(), max)); + if (max > kImpulseThreshold) { + PRINTD("(%d,%d)", max, index_of_max); + int64_t now_time = rtc::TimeMillis(); + int extra_delay = IndexToMilliseconds(static_cast<double>(index_of_max)); + PRINTD("[%d]", static_cast<int>(now_time - pulse_time_)); + PRINTD("[%d]", extra_delay); + // Total latency is the difference between transmit time and detection + // tome plus the extra delay within the buffer in which we detected the + // received impulse. It is transmitted at sample 0 but can be received + // at sample N where N > 0. The term |extra_delay| accounts for N and it + // is a value between 0 and 10ms. + latencies_.push_back(now_time - pulse_time_ + extra_delay); + pulse_time_ = 0; + } else { + PRINTD("-"); + } + } + + size_t num_latency_values() const { return latencies_.size(); } + + int min_latency() const { + if (latencies_.empty()) + return 0; + return *std::min_element(latencies_.begin(), latencies_.end()); + } + + int max_latency() const { + if (latencies_.empty()) + return 0; + return *std::max_element(latencies_.begin(), latencies_.end()); + } + + int average_latency() const { + if (latencies_.empty()) + return 0; + return 0.5 + + static_cast<double>( + std::accumulate(latencies_.begin(), latencies_.end(), 0)) / + latencies_.size(); + } + + void PrintResults() const { + PRINT("] "); + for (auto it = latencies_.begin(); it != latencies_.end(); ++it) { + PRINT("%d ", *it); + } + PRINT("\n"); + PRINT("%s[min, max, avg]=[%d, %d, %d] ms\n", kTag, min_latency(), + max_latency(), average_latency()); + } + + int IndexToMilliseconds(double index) const { + return 10.0 * (index / frames_per_buffer_) + 0.5; + } + + private: + const size_t frames_per_buffer_; + const size_t bytes_per_buffer_; + size_t play_count_; + size_t rec_count_; + int64_t pulse_time_; + std::vector<int> latencies_; +}; +// Mocks the AudioTransport object and proxies actions for the two callbacks +// (RecordedDataIsAvailable and NeedMorePlayData) to different implementations +// of AudioStreamInterface. +class MockAudioTransportIOS : public test::MockAudioTransport { + public: + explicit MockAudioTransportIOS(int type) + : num_callbacks_(0), + type_(type), + play_count_(0), + rec_count_(0), + audio_stream_(nullptr) {} + + virtual ~MockAudioTransportIOS() {} + + // Set default actions of the mock object. We are delegating to fake + // implementations (of AudioStreamInterface) here. + void HandleCallbacks(rtc::Event* test_is_done, + AudioStreamInterface* audio_stream, + size_t num_callbacks) { + test_is_done_ = test_is_done; + audio_stream_ = audio_stream; + num_callbacks_ = num_callbacks; + if (play_mode()) { + ON_CALL(*this, NeedMorePlayData(_, _, _, _, _, _, _, _)) + .WillByDefault( + Invoke(this, &MockAudioTransportIOS::RealNeedMorePlayData)); + } + if (rec_mode()) { + ON_CALL(*this, RecordedDataIsAvailable(_, _, _, _, _, _, _, _, _, _)) + .WillByDefault(Invoke( + this, &MockAudioTransportIOS::RealRecordedDataIsAvailable)); + } + } + + int32_t RealRecordedDataIsAvailable(const void* audioSamples, + const size_t nSamples, + const size_t nBytesPerSample, + const size_t nChannels, + const uint32_t samplesPerSec, + const uint32_t totalDelayMS, + const int32_t clockDrift, + const uint32_t currentMicLevel, + const bool keyPressed, + uint32_t& newMicLevel) { + EXPECT_TRUE(rec_mode()) << "No test is expecting these callbacks."; + rec_count_++; + // Process the recorded audio stream if an AudioStreamInterface + // implementation exists. + if (audio_stream_) { + audio_stream_->Write(audioSamples, nSamples); + } + if (ReceivedEnoughCallbacks()) { + if (test_is_done_) { + test_is_done_->Set(); + } + } + return 0; + } + + int32_t RealNeedMorePlayData(const size_t nSamples, + const size_t nBytesPerSample, + const size_t nChannels, + const uint32_t samplesPerSec, + void* audioSamples, + size_t& nSamplesOut, + int64_t* elapsed_time_ms, + int64_t* ntp_time_ms) { + EXPECT_TRUE(play_mode()) << "No test is expecting these callbacks."; + play_count_++; + nSamplesOut = nSamples; + // Read (possibly processed) audio stream samples to be played out if an + // AudioStreamInterface implementation exists. + if (audio_stream_) { + audio_stream_->Read(audioSamples, nSamples); + } else { + memset(audioSamples, 0, nSamples * nBytesPerSample); + } + if (ReceivedEnoughCallbacks()) { + if (test_is_done_) { + test_is_done_->Set(); + } + } + return 0; + } + + bool ReceivedEnoughCallbacks() { + bool recording_done = false; + if (rec_mode()) + recording_done = rec_count_ >= num_callbacks_; + else + recording_done = true; + + bool playout_done = false; + if (play_mode()) + playout_done = play_count_ >= num_callbacks_; + else + playout_done = true; + + return recording_done && playout_done; + } + + bool play_mode() const { return type_ & kPlayout; } + bool rec_mode() const { return type_ & kRecording; } + + private: + rtc::Event* test_is_done_; + size_t num_callbacks_; + int type_; + size_t play_count_; + size_t rec_count_; + AudioStreamInterface* audio_stream_; +}; + +// AudioDeviceTest test fixture. +class AudioDeviceTest : public ::testing::Test { + protected: + AudioDeviceTest() { + old_sev_ = rtc::LogMessage::GetLogToDebug(); + // Set suitable logging level here. Change to rtc::LS_INFO for more verbose + // output. See webrtc/rtc_base/logging.h for complete list of options. + rtc::LogMessage::LogToDebug(rtc::LS_INFO); + // Add extra logging fields here (timestamps and thread id). + // rtc::LogMessage::LogTimestamps(); + rtc::LogMessage::LogThreads(); + // Creates an audio device using a default audio layer. + audio_device_ = CreateAudioDevice(AudioDeviceModule::kPlatformDefaultAudio); + EXPECT_NE(audio_device_.get(), nullptr); + EXPECT_EQ(0, audio_device_->Init()); + EXPECT_EQ(0, + audio_device()->GetPlayoutAudioParameters(&playout_parameters_)); + EXPECT_EQ(0, audio_device()->GetRecordAudioParameters(&record_parameters_)); + } + virtual ~AudioDeviceTest() { + EXPECT_EQ(0, audio_device_->Terminate()); + rtc::LogMessage::LogToDebug(old_sev_); + } + + int playout_sample_rate() const { return playout_parameters_.sample_rate(); } + int record_sample_rate() const { return record_parameters_.sample_rate(); } + int playout_channels() const { return playout_parameters_.channels(); } + int record_channels() const { return record_parameters_.channels(); } + size_t playout_frames_per_10ms_buffer() const { + return playout_parameters_.frames_per_10ms_buffer(); + } + size_t record_frames_per_10ms_buffer() const { + return record_parameters_.frames_per_10ms_buffer(); + } + + rtc::scoped_refptr<AudioDeviceModule> audio_device() const { + return audio_device_; + } + + AudioDeviceModuleImpl* audio_device_impl() const { + return static_cast<AudioDeviceModuleImpl*>(audio_device_.get()); + } + + AudioDeviceBuffer* audio_device_buffer() const { + return audio_device_impl()->GetAudioDeviceBuffer(); + } + + rtc::scoped_refptr<AudioDeviceModule> CreateAudioDevice( + AudioDeviceModule::AudioLayer audio_layer) { + rtc::scoped_refptr<AudioDeviceModule> module(AudioDeviceModule::Create(audio_layer)); + return module; + } + + // Returns file name relative to the resource root given a sample rate. + std::string GetFileName(int sample_rate) { + EXPECT_TRUE(sample_rate == 48000 || sample_rate == 44100 || + sample_rate == 16000); + char fname[64]; + snprintf(fname, sizeof(fname), "audio_device/audio_short%d", + sample_rate / 1000); + std::string file_name(webrtc::test::ResourcePath(fname, "pcm")); + EXPECT_TRUE(test::FileExists(file_name)); +#ifdef ENABLE_DEBUG_PRINTF + PRINTD("file name: %s\n", file_name.c_str()); + const size_t bytes = test::GetFileSize(file_name); + PRINTD("file size: %" PRIuS " [bytes]\n", bytes); + PRINTD("file size: %" PRIuS " [samples]\n", bytes / kBytesPerSample); + const int seconds = + static_cast<int>(bytes / (sample_rate * kBytesPerSample)); + PRINTD("file size: %d [secs]\n", seconds); + PRINTD("file size: %" PRIuS " [callbacks]\n", + seconds * kNumCallbacksPerSecond); +#endif + return file_name; + } + + void StartPlayout() { + EXPECT_FALSE(audio_device()->Playing()); + EXPECT_EQ(0, audio_device()->InitPlayout()); + EXPECT_TRUE(audio_device()->PlayoutIsInitialized()); + EXPECT_EQ(0, audio_device()->StartPlayout()); + EXPECT_TRUE(audio_device()->Playing()); + } + + void StopPlayout() { + EXPECT_EQ(0, audio_device()->StopPlayout()); + EXPECT_FALSE(audio_device()->Playing()); + } + + void StartRecording() { + EXPECT_FALSE(audio_device()->Recording()); + EXPECT_EQ(0, audio_device()->InitRecording()); + EXPECT_TRUE(audio_device()->RecordingIsInitialized()); + EXPECT_EQ(0, audio_device()->StartRecording()); + EXPECT_TRUE(audio_device()->Recording()); + } + + void StopRecording() { + EXPECT_EQ(0, audio_device()->StopRecording()); + EXPECT_FALSE(audio_device()->Recording()); + } + + rtc::Event test_is_done_; + rtc::scoped_refptr<AudioDeviceModule> audio_device_; + AudioParameters playout_parameters_; + AudioParameters record_parameters_; + rtc::LoggingSeverity old_sev_; +}; + +TEST_F(AudioDeviceTest, ConstructDestruct) { + // Using the test fixture to create and destruct the audio device module. +} + +TEST_F(AudioDeviceTest, InitTerminate) { + // Initialization is part of the test fixture. + EXPECT_TRUE(audio_device()->Initialized()); + EXPECT_EQ(0, audio_device()->Terminate()); + EXPECT_FALSE(audio_device()->Initialized()); +} + +// Tests that playout can be initiated, started and stopped. No audio callback +// is registered in this test. +// Failing when running on real iOS devices: bugs.webrtc.org/6889. +TEST_F(AudioDeviceTest, DISABLED_StartStopPlayout) { + StartPlayout(); + StopPlayout(); + StartPlayout(); + StopPlayout(); +} + +// Tests that recording can be initiated, started and stopped. No audio callback +// is registered in this test. +// Can sometimes fail when running on real devices: bugs.webrtc.org/7888. +TEST_F(AudioDeviceTest, DISABLED_StartStopRecording) { + StartRecording(); + StopRecording(); + StartRecording(); + StopRecording(); +} + +// Verify that calling StopPlayout() will leave us in an uninitialized state +// which will require a new call to InitPlayout(). This test does not call +// StartPlayout() while being uninitialized since doing so will hit a +// RTC_DCHECK. +TEST_F(AudioDeviceTest, StopPlayoutRequiresInitToRestart) { + EXPECT_EQ(0, audio_device()->InitPlayout()); + EXPECT_EQ(0, audio_device()->StartPlayout()); + EXPECT_EQ(0, audio_device()->StopPlayout()); + EXPECT_FALSE(audio_device()->PlayoutIsInitialized()); +} + +// Verify that we can create two ADMs and start playing on the second ADM. +// Only the first active instance shall activate an audio session and the +// last active instance shall deactivate the audio session. The test does not +// explicitly verify correct audio session calls but instead focuses on +// ensuring that audio starts for both ADMs. + +// Failing when running on real iOS devices: bugs.webrtc.org/6889. +TEST_F(AudioDeviceTest, DISABLED_StartPlayoutOnTwoInstances) { + // Create and initialize a second/extra ADM instance. The default ADM is + // created by the test harness. + rtc::scoped_refptr<AudioDeviceModule> second_audio_device = + CreateAudioDevice(AudioDeviceModule::kPlatformDefaultAudio); + EXPECT_NE(second_audio_device.get(), nullptr); + EXPECT_EQ(0, second_audio_device->Init()); + + // Start playout for the default ADM but don't wait here. Instead use the + // upcoming second stream for that. We set the same expectation on number + // of callbacks as for the second stream. + NiceMock<MockAudioTransportIOS> mock(kPlayout); + mock.HandleCallbacks(nullptr, nullptr, 0); + EXPECT_CALL( + mock, NeedMorePlayData(playout_frames_per_10ms_buffer(), kBytesPerSample, + playout_channels(), playout_sample_rate(), + NotNull(), _, _, _)) + .Times(AtLeast(kNumCallbacks)); + EXPECT_EQ(0, audio_device()->RegisterAudioCallback(&mock)); + StartPlayout(); + + // Initialize playout for the second ADM. If all is OK, the second ADM shall + // reuse the audio session activated when the first ADM started playing. + // This call will also ensure that we avoid a problem related to initializing + // two different audio unit instances back to back (see webrtc:5166 for + // details). + EXPECT_EQ(0, second_audio_device->InitPlayout()); + EXPECT_TRUE(second_audio_device->PlayoutIsInitialized()); + + // Start playout for the second ADM and verify that it starts as intended. + // Passing this test ensures that initialization of the second audio unit + // has been done successfully and that there is no conflict with the already + // playing first ADM. + MockAudioTransportIOS mock2(kPlayout); + mock2.HandleCallbacks(&test_is_done_, nullptr, kNumCallbacks); + EXPECT_CALL( + mock2, NeedMorePlayData(playout_frames_per_10ms_buffer(), kBytesPerSample, + playout_channels(), playout_sample_rate(), + NotNull(), _, _, _)) + .Times(AtLeast(kNumCallbacks)); + EXPECT_EQ(0, second_audio_device->RegisterAudioCallback(&mock2)); + EXPECT_EQ(0, second_audio_device->StartPlayout()); + EXPECT_TRUE(second_audio_device->Playing()); + test_is_done_.Wait(kTestTimeOutInMilliseconds); + EXPECT_EQ(0, second_audio_device->StopPlayout()); + EXPECT_FALSE(second_audio_device->Playing()); + EXPECT_FALSE(second_audio_device->PlayoutIsInitialized()); + + EXPECT_EQ(0, second_audio_device->Terminate()); +} + +// Start playout and verify that the native audio layer starts asking for real +// audio samples to play out using the NeedMorePlayData callback. +TEST_F(AudioDeviceTest, StartPlayoutVerifyCallbacks) { + MockAudioTransportIOS mock(kPlayout); + mock.HandleCallbacks(&test_is_done_, nullptr, kNumCallbacks); + EXPECT_CALL(mock, NeedMorePlayData(playout_frames_per_10ms_buffer(), + kBytesPerSample, playout_channels(), + playout_sample_rate(), NotNull(), _, _, _)) + .Times(AtLeast(kNumCallbacks)); + EXPECT_EQ(0, audio_device()->RegisterAudioCallback(&mock)); + StartPlayout(); + test_is_done_.Wait(kTestTimeOutInMilliseconds); + StopPlayout(); +} + +// Start recording and verify that the native audio layer starts feeding real +// audio samples via the RecordedDataIsAvailable callback. +TEST_F(AudioDeviceTest, StartRecordingVerifyCallbacks) { + MockAudioTransportIOS mock(kRecording); + mock.HandleCallbacks(&test_is_done_, nullptr, kNumCallbacks); + EXPECT_CALL(mock, + RecordedDataIsAvailable( + NotNull(), record_frames_per_10ms_buffer(), kBytesPerSample, + record_channels(), record_sample_rate(), + _, // TODO(henrika): fix delay + 0, 0, false, _)).Times(AtLeast(kNumCallbacks)); + + EXPECT_EQ(0, audio_device()->RegisterAudioCallback(&mock)); + StartRecording(); + test_is_done_.Wait(kTestTimeOutInMilliseconds); + StopRecording(); +} + +// Start playout and recording (full-duplex audio) and verify that audio is +// active in both directions. +TEST_F(AudioDeviceTest, StartPlayoutAndRecordingVerifyCallbacks) { + MockAudioTransportIOS mock(kPlayout | kRecording); + mock.HandleCallbacks(&test_is_done_, nullptr, kNumCallbacks); + EXPECT_CALL(mock, NeedMorePlayData(playout_frames_per_10ms_buffer(), + kBytesPerSample, playout_channels(), + playout_sample_rate(), NotNull(), _, _, _)) + .Times(AtLeast(kNumCallbacks)); + EXPECT_CALL(mock, + RecordedDataIsAvailable( + NotNull(), record_frames_per_10ms_buffer(), kBytesPerSample, + record_channels(), record_sample_rate(), + _, // TODO(henrika): fix delay + 0, 0, false, _)).Times(AtLeast(kNumCallbacks)); + EXPECT_EQ(0, audio_device()->RegisterAudioCallback(&mock)); + StartPlayout(); + StartRecording(); + test_is_done_.Wait(kTestTimeOutInMilliseconds); + StopRecording(); + StopPlayout(); +} + +// Start playout and read audio from an external PCM file when the audio layer +// asks for data to play out. Real audio is played out in this test but it does +// not contain any explicit verification that the audio quality is perfect. +TEST_F(AudioDeviceTest, RunPlayoutWithFileAsSource) { + // TODO(henrika): extend test when mono output is supported. + EXPECT_EQ(1, playout_channels()); + NiceMock<MockAudioTransportIOS> mock(kPlayout); + const int num_callbacks = kFilePlayTimeInSec * kNumCallbacksPerSecond; + std::string file_name = GetFileName(playout_sample_rate()); + std::unique_ptr<FileAudioStream> file_audio_stream( + new FileAudioStream(num_callbacks, file_name, playout_sample_rate())); + mock.HandleCallbacks(&test_is_done_, file_audio_stream.get(), num_callbacks); + // SetMaxPlayoutVolume(); + EXPECT_EQ(0, audio_device()->RegisterAudioCallback(&mock)); + StartPlayout(); + test_is_done_.Wait(kTestTimeOutInMilliseconds); + StopPlayout(); +} + +TEST_F(AudioDeviceTest, Devices) { + // Device enumeration is not supported. Verify fixed values only. + EXPECT_EQ(1, audio_device()->PlayoutDevices()); + EXPECT_EQ(1, audio_device()->RecordingDevices()); +} + +// Start playout and recording and store recorded data in an intermediate FIFO +// buffer from which the playout side then reads its samples in the same order +// as they were stored. Under ideal circumstances, a callback sequence would +// look like: ...+-+-+-+-+-+-+-..., where '+' means 'packet recorded' and '-' +// means 'packet played'. Under such conditions, the FIFO would only contain +// one packet on average. However, under more realistic conditions, the size +// of the FIFO will vary more due to an unbalance between the two sides. +// This test tries to verify that the device maintains a balanced callback- +// sequence by running in loopback for ten seconds while measuring the size +// (max and average) of the FIFO. The size of the FIFO is increased by the +// recording side and decreased by the playout side. +// TODO(henrika): tune the final test parameters after running tests on several +// different devices. +TEST_F(AudioDeviceTest, RunPlayoutAndRecordingInFullDuplex) { + EXPECT_EQ(record_channels(), playout_channels()); + EXPECT_EQ(record_sample_rate(), playout_sample_rate()); + NiceMock<MockAudioTransportIOS> mock(kPlayout | kRecording); + std::unique_ptr<FifoAudioStream> fifo_audio_stream( + new FifoAudioStream(playout_frames_per_10ms_buffer())); + mock.HandleCallbacks( + &test_is_done_, fifo_audio_stream.get(), kFullDuplexTimeInSec * kNumCallbacksPerSecond); + // SetMaxPlayoutVolume(); + EXPECT_EQ(0, audio_device()->RegisterAudioCallback(&mock)); + StartRecording(); + StartPlayout(); + test_is_done_.Wait(std::max(kTestTimeOutInMilliseconds, 1000 * kFullDuplexTimeInSec)); + StopPlayout(); + StopRecording(); + EXPECT_LE(fifo_audio_stream->average_size(), 10u); + EXPECT_LE(fifo_audio_stream->largest_size(), 20u); +} + +// Measures loopback latency and reports the min, max and average values for +// a full duplex audio session. +// The latency is measured like so: +// - Insert impulses periodically on the output side. +// - Detect the impulses on the input side. +// - Measure the time difference between the transmit time and receive time. +// - Store time differences in a vector and calculate min, max and average. +// This test requires a special hardware called Audio Loopback Dongle. +// See http://source.android.com/devices/audio/loopback.html for details. +TEST_F(AudioDeviceTest, DISABLED_MeasureLoopbackLatency) { + EXPECT_EQ(record_channels(), playout_channels()); + EXPECT_EQ(record_sample_rate(), playout_sample_rate()); + NiceMock<MockAudioTransportIOS> mock(kPlayout | kRecording); + std::unique_ptr<LatencyMeasuringAudioStream> latency_audio_stream( + new LatencyMeasuringAudioStream(playout_frames_per_10ms_buffer())); + mock.HandleCallbacks(&test_is_done_, + latency_audio_stream.get(), + kMeasureLatencyTimeInSec * kNumCallbacksPerSecond); + EXPECT_EQ(0, audio_device()->RegisterAudioCallback(&mock)); + // SetMaxPlayoutVolume(); + // DisableBuiltInAECIfAvailable(); + StartRecording(); + StartPlayout(); + test_is_done_.Wait(std::max(kTestTimeOutInMilliseconds, 1000 * kMeasureLatencyTimeInSec)); + StopPlayout(); + StopRecording(); + // Verify that the correct number of transmitted impulses are detected. + EXPECT_EQ(latency_audio_stream->num_latency_values(), + static_cast<size_t>( + kImpulseFrequencyInHz * kMeasureLatencyTimeInSec - 1)); + latency_audio_stream->PrintResults(); +} + +// Verifies that the AudioDeviceIOS is_interrupted_ flag is reset correctly +// after an iOS AVAudioSessionInterruptionTypeEnded notification event. +// AudioDeviceIOS listens to RTCAudioSession interrupted notifications by: +// - In AudioDeviceIOS.InitPlayOrRecord registers its audio_session_observer_ +// callback with RTCAudioSession's delegate list. +// - When RTCAudioSession receives an iOS audio interrupted notification, it +// passes the notification to callbacks in its delegate list which sets +// AudioDeviceIOS's is_interrupted_ flag to true. +// - When AudioDeviceIOS.ShutdownPlayOrRecord is called, its +// audio_session_observer_ callback is removed from RTCAudioSessions's +// delegate list. +// So if RTCAudioSession receives an iOS end audio interruption notification, +// AudioDeviceIOS is not notified as its callback is not in RTCAudioSession's +// delegate list. This causes AudioDeviceIOS's is_interrupted_ flag to be in +// the wrong (true) state and the audio session will ignore audio changes. +// As RTCAudioSession keeps its own interrupted state, the fix is to initialize +// AudioDeviceIOS's is_interrupted_ flag to RTCAudioSession's isInterrupted +// flag in AudioDeviceIOS.InitPlayOrRecord. +TEST_F(AudioDeviceTest, testInterruptedAudioSession) { + RTCAudioSession *session = [RTCAudioSession sharedInstance]; + std::unique_ptr<webrtc::AudioDeviceIOS> audio_device; + audio_device.reset(new webrtc::AudioDeviceIOS()); + std::unique_ptr<webrtc::AudioDeviceBuffer> audio_buffer; + audio_buffer.reset(new webrtc::AudioDeviceBuffer()); + audio_device->AttachAudioBuffer(audio_buffer.get()); + audio_device->Init(); + audio_device->InitPlayout(); + // Force interruption. + [session notifyDidBeginInterruption]; + + // Wait for notification to propagate. + rtc::MessageQueueManager::ProcessAllMessageQueuesForTesting(); + EXPECT_TRUE(audio_device->is_interrupted_); + + // Force it for testing. + audio_device->playing_ = false; + audio_device->ShutdownPlayOrRecord(); + // Force it for testing. + audio_device->audio_is_initialized_ = false; + + [session notifyDidEndInterruptionWithShouldResumeSession:YES]; + // Wait for notification to propagate. + rtc::MessageQueueManager::ProcessAllMessageQueuesForTesting(); + EXPECT_TRUE(audio_device->is_interrupted_); + + audio_device->Init(); + audio_device->InitPlayout(); + EXPECT_FALSE(audio_device->is_interrupted_); +} + +} // namespace webrtc diff --git a/modules/audio_device/ios/audio_session_observer.h b/modules/audio_device/ios/audio_session_observer.h new file mode 100644 index 0000000000..c79cdd1436 --- /dev/null +++ b/modules/audio_device/ios/audio_session_observer.h @@ -0,0 +1,42 @@ +/* + * Copyright 2016 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_DEVICE_IOS_AUDIO_SESSION_OBSERVER_H_ +#define MODULES_AUDIO_DEVICE_IOS_AUDIO_SESSION_OBSERVER_H_ + +#include "rtc_base/async_invoker.h" +#include "rtc_base/thread.h" + +namespace webrtc { + +// Observer interface for listening to AVAudioSession events. +class AudioSessionObserver { + public: + // Called when audio session interruption begins. + virtual void OnInterruptionBegin() = 0; + + // Called when audio session interruption ends. + virtual void OnInterruptionEnd() = 0; + + // Called when audio route changes. + virtual void OnValidRouteChange() = 0; + + // Called when the ability to play or record changes. + virtual void OnCanPlayOrRecordChange(bool can_play_or_record) = 0; + + virtual void OnChangedOutputVolume() = 0; + + protected: + virtual ~AudioSessionObserver() {} +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_DEVICE_IOS_AUDIO_SESSION_OBSERVER_H_ diff --git a/modules/audio_device/ios/objc/RTCAudioSession.h b/modules/audio_device/ios/objc/RTCAudioSession.h new file mode 100644 index 0000000000..23abc3d6b1 --- /dev/null +++ b/modules/audio_device/ios/objc/RTCAudioSession.h @@ -0,0 +1,11 @@ +/* + * Copyright 2017 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "sdk/objc/components/audio/RTCAudioSession.h" diff --git a/modules/audio_device/ios/objc/RTCAudioSessionConfiguration.h b/modules/audio_device/ios/objc/RTCAudioSessionConfiguration.h new file mode 100644 index 0000000000..2584053054 --- /dev/null +++ b/modules/audio_device/ios/objc/RTCAudioSessionConfiguration.h @@ -0,0 +1,11 @@ +/* + * Copyright 2017 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "sdk/objc/components/audio/RTCAudioSessionConfiguration.h" diff --git a/modules/audio_device/ios/objc/RTCAudioSessionDelegateAdapter.h b/modules/audio_device/ios/objc/RTCAudioSessionDelegateAdapter.h new file mode 100644 index 0000000000..54f4c26557 --- /dev/null +++ b/modules/audio_device/ios/objc/RTCAudioSessionDelegateAdapter.h @@ -0,0 +1,29 @@ +/* + * Copyright 2016 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "sdk/objc/components/audio/RTCAudioSession.h" + +namespace webrtc { +class AudioSessionObserver; +} + +/** Adapter that forwards RTCAudioSessionDelegate calls to the appropriate + * methods on the AudioSessionObserver. + */ +@interface RTCAudioSessionDelegateAdapter : NSObject <RTCAudioSessionDelegate> + +- (instancetype)init NS_UNAVAILABLE; + +/** |observer| is a raw pointer and should be kept alive + * for this object's lifetime. + */ +- (instancetype)initWithObserver:(webrtc::AudioSessionObserver *)observer NS_DESIGNATED_INITIALIZER; + +@end diff --git a/modules/audio_device/ios/objc/RTCAudioSessionDelegateAdapter.mm b/modules/audio_device/ios/objc/RTCAudioSessionDelegateAdapter.mm new file mode 100644 index 0000000000..818e077a6a --- /dev/null +++ b/modules/audio_device/ios/objc/RTCAudioSessionDelegateAdapter.mm @@ -0,0 +1,89 @@ +/* + * Copyright 2016 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "modules/audio_device/ios/objc/RTCAudioSessionDelegateAdapter.h" + +#include "modules/audio_device/ios/audio_session_observer.h" + +#import "sdk/objc/base/RTCLogging.h" + +@implementation RTCAudioSessionDelegateAdapter { + webrtc::AudioSessionObserver *_observer; +} + +- (instancetype)initWithObserver:(webrtc::AudioSessionObserver *)observer { + NSParameterAssert(observer); + if (self = [super init]) { + _observer = observer; + } + return self; +} + +#pragma mark - RTCAudioSessionDelegate + +- (void)audioSessionDidBeginInterruption:(RTCAudioSession *)session { + _observer->OnInterruptionBegin(); +} + +- (void)audioSessionDidEndInterruption:(RTCAudioSession *)session + shouldResumeSession:(BOOL)shouldResumeSession { + _observer->OnInterruptionEnd(); +} + +- (void)audioSessionDidChangeRoute:(RTCAudioSession *)session + reason:(AVAudioSessionRouteChangeReason)reason + previousRoute:(AVAudioSessionRouteDescription *)previousRoute { + switch (reason) { + case AVAudioSessionRouteChangeReasonUnknown: + case AVAudioSessionRouteChangeReasonNewDeviceAvailable: + case AVAudioSessionRouteChangeReasonOldDeviceUnavailable: + case AVAudioSessionRouteChangeReasonCategoryChange: + // It turns out that we see a category change (at least in iOS 9.2) + // when making a switch from a BT device to e.g. Speaker using the + // iOS Control Center and that we therefore must check if the sample + // rate has changed. And if so is the case, restart the audio unit. + case AVAudioSessionRouteChangeReasonOverride: + case AVAudioSessionRouteChangeReasonWakeFromSleep: + case AVAudioSessionRouteChangeReasonNoSuitableRouteForCategory: + _observer->OnValidRouteChange(); + break; + case AVAudioSessionRouteChangeReasonRouteConfigurationChange: + // The set of input and output ports has not changed, but their + // configuration has, e.g., a port’s selected data source has + // changed. Ignore this type of route change since we are focusing + // on detecting headset changes. + RTCLog(@"Ignoring RouteConfigurationChange"); + break; + } +} + +- (void)audioSessionMediaServerTerminated:(RTCAudioSession *)session { +} + +- (void)audioSessionMediaServerReset:(RTCAudioSession *)session { +} + +- (void)audioSession:(RTCAudioSession *)session + didChangeCanPlayOrRecord:(BOOL)canPlayOrRecord { + _observer->OnCanPlayOrRecordChange(canPlayOrRecord); +} + +- (void)audioSessionDidStartPlayOrRecord:(RTCAudioSession *)session { +} + +- (void)audioSessionDidStopPlayOrRecord:(RTCAudioSession *)session { +} + +- (void)audioSession:(RTCAudioSession *)audioSession + didChangeOutputVolume:(float)outputVolume { + _observer->OnChangedOutputVolume(); +} + +@end diff --git a/modules/audio_device/ios/voice_processing_audio_unit.h b/modules/audio_device/ios/voice_processing_audio_unit.h new file mode 100644 index 0000000000..3105c129bd --- /dev/null +++ b/modules/audio_device/ios/voice_processing_audio_unit.h @@ -0,0 +1,137 @@ +/* + * Copyright 2016 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_DEVICE_IOS_VOICE_PROCESSING_AUDIO_UNIT_H_ +#define MODULES_AUDIO_DEVICE_IOS_VOICE_PROCESSING_AUDIO_UNIT_H_ + +#include <AudioUnit/AudioUnit.h> + +namespace webrtc { + +class VoiceProcessingAudioUnitObserver { + public: + // Callback function called on a real-time priority I/O thread from the audio + // unit. This method is used to signal that recorded audio is available. + virtual OSStatus OnDeliverRecordedData(AudioUnitRenderActionFlags* flags, + const AudioTimeStamp* time_stamp, + UInt32 bus_number, + UInt32 num_frames, + AudioBufferList* io_data) = 0; + + // Callback function called on a real-time priority I/O thread from the audio + // unit. This method is used to provide audio samples to the audio unit. + virtual OSStatus OnGetPlayoutData(AudioUnitRenderActionFlags* io_action_flags, + const AudioTimeStamp* time_stamp, + UInt32 bus_number, + UInt32 num_frames, + AudioBufferList* io_data) = 0; + + protected: + ~VoiceProcessingAudioUnitObserver() {} +}; + +// Convenience class to abstract away the management of a Voice Processing +// I/O Audio Unit. The Voice Processing I/O unit has the same characteristics +// as the Remote I/O unit (supports full duplex low-latency audio input and +// output) and adds AEC for for two-way duplex communication. It also adds AGC, +// adjustment of voice-processing quality, and muting. Hence, ideal for +// VoIP applications. +class VoiceProcessingAudioUnit { + public: + explicit VoiceProcessingAudioUnit(VoiceProcessingAudioUnitObserver* observer); + ~VoiceProcessingAudioUnit(); + + // TODO(tkchin): enum for state and state checking. + enum State : int32_t { + // Init() should be called. + kInitRequired, + // Audio unit created but not initialized. + kUninitialized, + // Initialized but not started. Equivalent to stopped. + kInitialized, + // Initialized and started. + kStarted, + }; + + // Number of bytes per audio sample for 16-bit signed integer representation. + static const UInt32 kBytesPerSample; + + // Initializes this class by creating the underlying audio unit instance. + // Creates a Voice-Processing I/O unit and configures it for full-duplex + // audio. The selected stream format is selected to avoid internal resampling + // and to match the 10ms callback rate for WebRTC as well as possible. + // Does not intialize the audio unit. + bool Init(); + + VoiceProcessingAudioUnit::State GetState() const; + + // Initializes the underlying audio unit with the given sample rate. + bool Initialize(Float64 sample_rate); + + // Starts the underlying audio unit. + bool Start(); + + // Stops the underlying audio unit. + bool Stop(); + + // Uninitializes the underlying audio unit. + bool Uninitialize(); + + // Calls render on the underlying audio unit. + OSStatus Render(AudioUnitRenderActionFlags* flags, + const AudioTimeStamp* time_stamp, + UInt32 output_bus_number, + UInt32 num_frames, + AudioBufferList* io_data); + + private: + // The C API used to set callbacks requires static functions. When these are + // called, they will invoke the relevant instance method by casting + // in_ref_con to VoiceProcessingAudioUnit*. + static OSStatus OnGetPlayoutData(void* in_ref_con, + AudioUnitRenderActionFlags* flags, + const AudioTimeStamp* time_stamp, + UInt32 bus_number, + UInt32 num_frames, + AudioBufferList* io_data); + static OSStatus OnDeliverRecordedData(void* in_ref_con, + AudioUnitRenderActionFlags* flags, + const AudioTimeStamp* time_stamp, + UInt32 bus_number, + UInt32 num_frames, + AudioBufferList* io_data); + + // Notifies observer that samples are needed for playback. + OSStatus NotifyGetPlayoutData(AudioUnitRenderActionFlags* flags, + const AudioTimeStamp* time_stamp, + UInt32 bus_number, + UInt32 num_frames, + AudioBufferList* io_data); + // Notifies observer that recorded samples are available for render. + OSStatus NotifyDeliverRecordedData(AudioUnitRenderActionFlags* flags, + const AudioTimeStamp* time_stamp, + UInt32 bus_number, + UInt32 num_frames, + AudioBufferList* io_data); + + // Returns the predetermined format with a specific sample rate. See + // implementation file for details on format. + AudioStreamBasicDescription GetFormat(Float64 sample_rate) const; + + // Deletes the underlying audio unit. + void DisposeAudioUnit(); + + VoiceProcessingAudioUnitObserver* observer_; + AudioUnit vpio_unit_; + VoiceProcessingAudioUnit::State state_; +}; +} // namespace webrtc + +#endif // MODULES_AUDIO_DEVICE_IOS_VOICE_PROCESSING_AUDIO_UNIT_H_ diff --git a/modules/audio_device/ios/voice_processing_audio_unit.mm b/modules/audio_device/ios/voice_processing_audio_unit.mm new file mode 100644 index 0000000000..41477d1f67 --- /dev/null +++ b/modules/audio_device/ios/voice_processing_audio_unit.mm @@ -0,0 +1,468 @@ +/* + * Copyright 2016 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "modules/audio_device/ios/voice_processing_audio_unit.h" + +#include "rtc_base/checks.h" +#include "rtc_base/system/fallthrough.h" +#include "system_wrappers/include/metrics.h" + +#import "sdk/objc/base//RTCLogging.h" +#import "sdk/objc/components/audio/RTCAudioSessionConfiguration.h" + +#if !defined(NDEBUG) +static void LogStreamDescription(AudioStreamBasicDescription description) { + char formatIdString[5]; + UInt32 formatId = CFSwapInt32HostToBig(description.mFormatID); + bcopy(&formatId, formatIdString, 4); + formatIdString[4] = '\0'; + RTCLog(@"AudioStreamBasicDescription: {\n" + " mSampleRate: %.2f\n" + " formatIDString: %s\n" + " mFormatFlags: 0x%X\n" + " mBytesPerPacket: %u\n" + " mFramesPerPacket: %u\n" + " mBytesPerFrame: %u\n" + " mChannelsPerFrame: %u\n" + " mBitsPerChannel: %u\n" + " mReserved: %u\n}", + description.mSampleRate, formatIdString, + static_cast<unsigned int>(description.mFormatFlags), + static_cast<unsigned int>(description.mBytesPerPacket), + static_cast<unsigned int>(description.mFramesPerPacket), + static_cast<unsigned int>(description.mBytesPerFrame), + static_cast<unsigned int>(description.mChannelsPerFrame), + static_cast<unsigned int>(description.mBitsPerChannel), + static_cast<unsigned int>(description.mReserved)); +} +#endif + +namespace webrtc { + +// Calls to AudioUnitInitialize() can fail if called back-to-back on different +// ADM instances. A fall-back solution is to allow multiple sequential calls +// with as small delay between each. This factor sets the max number of allowed +// initialization attempts. +static const int kMaxNumberOfAudioUnitInitializeAttempts = 5; +// A VP I/O unit's bus 1 connects to input hardware (microphone). +static const AudioUnitElement kInputBus = 1; +// A VP I/O unit's bus 0 connects to output hardware (speaker). +static const AudioUnitElement kOutputBus = 0; + +// Returns the automatic gain control (AGC) state on the processed microphone +// signal. Should be on by default for Voice Processing audio units. +static OSStatus GetAGCState(AudioUnit audio_unit, UInt32* enabled) { + RTC_DCHECK(audio_unit); + UInt32 size = sizeof(*enabled); + OSStatus result = AudioUnitGetProperty(audio_unit, + kAUVoiceIOProperty_VoiceProcessingEnableAGC, + kAudioUnitScope_Global, + kInputBus, + enabled, + &size); + RTCLog(@"VPIO unit AGC: %u", static_cast<unsigned int>(*enabled)); + return result; +} + +VoiceProcessingAudioUnit::VoiceProcessingAudioUnit( + VoiceProcessingAudioUnitObserver* observer) + : observer_(observer), vpio_unit_(nullptr), state_(kInitRequired) { + RTC_DCHECK(observer); +} + +VoiceProcessingAudioUnit::~VoiceProcessingAudioUnit() { + DisposeAudioUnit(); +} + +const UInt32 VoiceProcessingAudioUnit::kBytesPerSample = 2; + +bool VoiceProcessingAudioUnit::Init() { + RTC_DCHECK_EQ(state_, kInitRequired); + + // Create an audio component description to identify the Voice Processing + // I/O audio unit. + AudioComponentDescription vpio_unit_description; + vpio_unit_description.componentType = kAudioUnitType_Output; + vpio_unit_description.componentSubType = kAudioUnitSubType_VoiceProcessingIO; + vpio_unit_description.componentManufacturer = kAudioUnitManufacturer_Apple; + vpio_unit_description.componentFlags = 0; + vpio_unit_description.componentFlagsMask = 0; + + // Obtain an audio unit instance given the description. + AudioComponent found_vpio_unit_ref = + AudioComponentFindNext(nullptr, &vpio_unit_description); + + // Create a Voice Processing IO audio unit. + OSStatus result = noErr; + result = AudioComponentInstanceNew(found_vpio_unit_ref, &vpio_unit_); + if (result != noErr) { + vpio_unit_ = nullptr; + RTCLogError(@"AudioComponentInstanceNew failed. Error=%ld.", (long)result); + return false; + } + + // Enable input on the input scope of the input element. + UInt32 enable_input = 1; + result = AudioUnitSetProperty(vpio_unit_, kAudioOutputUnitProperty_EnableIO, + kAudioUnitScope_Input, kInputBus, &enable_input, + sizeof(enable_input)); + if (result != noErr) { + DisposeAudioUnit(); + RTCLogError(@"Failed to enable input on input scope of input element. " + "Error=%ld.", + (long)result); + return false; + } + + // Enable output on the output scope of the output element. + UInt32 enable_output = 1; + result = AudioUnitSetProperty(vpio_unit_, kAudioOutputUnitProperty_EnableIO, + kAudioUnitScope_Output, kOutputBus, + &enable_output, sizeof(enable_output)); + if (result != noErr) { + DisposeAudioUnit(); + RTCLogError(@"Failed to enable output on output scope of output element. " + "Error=%ld.", + (long)result); + return false; + } + + // Specify the callback function that provides audio samples to the audio + // unit. + AURenderCallbackStruct render_callback; + render_callback.inputProc = OnGetPlayoutData; + render_callback.inputProcRefCon = this; + result = AudioUnitSetProperty( + vpio_unit_, kAudioUnitProperty_SetRenderCallback, kAudioUnitScope_Input, + kOutputBus, &render_callback, sizeof(render_callback)); + if (result != noErr) { + DisposeAudioUnit(); + RTCLogError(@"Failed to specify the render callback on the output bus. " + "Error=%ld.", + (long)result); + return false; + } + + // Disable AU buffer allocation for the recorder, we allocate our own. + // TODO(henrika): not sure that it actually saves resource to make this call. + UInt32 flag = 0; + result = AudioUnitSetProperty( + vpio_unit_, kAudioUnitProperty_ShouldAllocateBuffer, + kAudioUnitScope_Output, kInputBus, &flag, sizeof(flag)); + if (result != noErr) { + DisposeAudioUnit(); + RTCLogError(@"Failed to disable buffer allocation on the input bus. " + "Error=%ld.", + (long)result); + return false; + } + + // Specify the callback to be called by the I/O thread to us when input audio + // is available. The recorded samples can then be obtained by calling the + // AudioUnitRender() method. + AURenderCallbackStruct input_callback; + input_callback.inputProc = OnDeliverRecordedData; + input_callback.inputProcRefCon = this; + result = AudioUnitSetProperty(vpio_unit_, + kAudioOutputUnitProperty_SetInputCallback, + kAudioUnitScope_Global, kInputBus, + &input_callback, sizeof(input_callback)); + if (result != noErr) { + DisposeAudioUnit(); + RTCLogError(@"Failed to specify the input callback on the input bus. " + "Error=%ld.", + (long)result); + return false; + } + + state_ = kUninitialized; + return true; +} + +VoiceProcessingAudioUnit::State VoiceProcessingAudioUnit::GetState() const { + return state_; +} + +bool VoiceProcessingAudioUnit::Initialize(Float64 sample_rate) { + RTC_DCHECK_GE(state_, kUninitialized); + RTCLog(@"Initializing audio unit with sample rate: %f", sample_rate); + + OSStatus result = noErr; + AudioStreamBasicDescription format = GetFormat(sample_rate); + UInt32 size = sizeof(format); +#if !defined(NDEBUG) + LogStreamDescription(format); +#endif + + // Set the format on the output scope of the input element/bus. + result = + AudioUnitSetProperty(vpio_unit_, kAudioUnitProperty_StreamFormat, + kAudioUnitScope_Output, kInputBus, &format, size); + if (result != noErr) { + RTCLogError(@"Failed to set format on output scope of input bus. " + "Error=%ld.", + (long)result); + return false; + } + + // Set the format on the input scope of the output element/bus. + result = + AudioUnitSetProperty(vpio_unit_, kAudioUnitProperty_StreamFormat, + kAudioUnitScope_Input, kOutputBus, &format, size); + if (result != noErr) { + RTCLogError(@"Failed to set format on input scope of output bus. " + "Error=%ld.", + (long)result); + return false; + } + + // Initialize the Voice Processing I/O unit instance. + // Calls to AudioUnitInitialize() can fail if called back-to-back on + // different ADM instances. The error message in this case is -66635 which is + // undocumented. Tests have shown that calling AudioUnitInitialize a second + // time, after a short sleep, avoids this issue. + // See webrtc:5166 for details. + int failed_initalize_attempts = 0; + result = AudioUnitInitialize(vpio_unit_); + while (result != noErr) { + RTCLogError(@"Failed to initialize the Voice Processing I/O unit. " + "Error=%ld.", + (long)result); + ++failed_initalize_attempts; + if (failed_initalize_attempts == kMaxNumberOfAudioUnitInitializeAttempts) { + // Max number of initialization attempts exceeded, hence abort. + RTCLogError(@"Too many initialization attempts."); + return false; + } + RTCLog(@"Pause 100ms and try audio unit initialization again..."); + [NSThread sleepForTimeInterval:0.1f]; + result = AudioUnitInitialize(vpio_unit_); + } + if (result == noErr) { + RTCLog(@"Voice Processing I/O unit is now initialized."); + } + + // AGC should be enabled by default for Voice Processing I/O units but it is + // checked below and enabled explicitly if needed. This scheme is used + // to be absolutely sure that the AGC is enabled since we have seen cases + // where only zeros are recorded and a disabled AGC could be one of the + // reasons why it happens. + int agc_was_enabled_by_default = 0; + UInt32 agc_is_enabled = 0; + result = GetAGCState(vpio_unit_, &agc_is_enabled); + if (result != noErr) { + RTCLogError(@"Failed to get AGC state (1st attempt). " + "Error=%ld.", + (long)result); + // Example of error code: kAudioUnitErr_NoConnection (-10876). + // All error codes related to audio units are negative and are therefore + // converted into a postive value to match the UMA APIs. + RTC_HISTOGRAM_COUNTS_SPARSE_100000( + "WebRTC.Audio.GetAGCStateErrorCode1", (-1) * result); + } else if (agc_is_enabled) { + // Remember that the AGC was enabled by default. Will be used in UMA. + agc_was_enabled_by_default = 1; + } else { + // AGC was initially disabled => try to enable it explicitly. + UInt32 enable_agc = 1; + result = + AudioUnitSetProperty(vpio_unit_, + kAUVoiceIOProperty_VoiceProcessingEnableAGC, + kAudioUnitScope_Global, kInputBus, &enable_agc, + sizeof(enable_agc)); + if (result != noErr) { + RTCLogError(@"Failed to enable the built-in AGC. " + "Error=%ld.", + (long)result); + RTC_HISTOGRAM_COUNTS_SPARSE_100000( + "WebRTC.Audio.SetAGCStateErrorCode", (-1) * result); + } + result = GetAGCState(vpio_unit_, &agc_is_enabled); + if (result != noErr) { + RTCLogError(@"Failed to get AGC state (2nd attempt). " + "Error=%ld.", + (long)result); + RTC_HISTOGRAM_COUNTS_SPARSE_100000( + "WebRTC.Audio.GetAGCStateErrorCode2", (-1) * result); + } + } + + // Track if the built-in AGC was enabled by default (as it should) or not. + RTC_HISTOGRAM_BOOLEAN("WebRTC.Audio.BuiltInAGCWasEnabledByDefault", + agc_was_enabled_by_default); + RTCLog(@"WebRTC.Audio.BuiltInAGCWasEnabledByDefault: %d", + agc_was_enabled_by_default); + // As a final step, add an UMA histogram for tracking the AGC state. + // At this stage, the AGC should be enabled, and if it is not, more work is + // needed to find out the root cause. + RTC_HISTOGRAM_BOOLEAN("WebRTC.Audio.BuiltInAGCIsEnabled", agc_is_enabled); + RTCLog(@"WebRTC.Audio.BuiltInAGCIsEnabled: %u", + static_cast<unsigned int>(agc_is_enabled)); + + state_ = kInitialized; + return true; +} + +bool VoiceProcessingAudioUnit::Start() { + RTC_DCHECK_GE(state_, kUninitialized); + RTCLog(@"Starting audio unit."); + + OSStatus result = AudioOutputUnitStart(vpio_unit_); + if (result != noErr) { + RTCLogError(@"Failed to start audio unit. Error=%ld", (long)result); + return false; + } else { + RTCLog(@"Started audio unit"); + } + state_ = kStarted; + return true; +} + +bool VoiceProcessingAudioUnit::Stop() { + RTC_DCHECK_GE(state_, kUninitialized); + RTCLog(@"Stopping audio unit."); + + OSStatus result = AudioOutputUnitStop(vpio_unit_); + if (result != noErr) { + RTCLogError(@"Failed to stop audio unit. Error=%ld", (long)result); + return false; + } else { + RTCLog(@"Stopped audio unit"); + } + + state_ = kInitialized; + return true; +} + +bool VoiceProcessingAudioUnit::Uninitialize() { + RTC_DCHECK_GE(state_, kUninitialized); + RTCLog(@"Unintializing audio unit."); + + OSStatus result = AudioUnitUninitialize(vpio_unit_); + if (result != noErr) { + RTCLogError(@"Failed to uninitialize audio unit. Error=%ld", (long)result); + return false; + } else { + RTCLog(@"Uninitialized audio unit."); + } + + state_ = kUninitialized; + return true; +} + +OSStatus VoiceProcessingAudioUnit::Render(AudioUnitRenderActionFlags* flags, + const AudioTimeStamp* time_stamp, + UInt32 output_bus_number, + UInt32 num_frames, + AudioBufferList* io_data) { + RTC_DCHECK(vpio_unit_) << "Init() not called."; + + OSStatus result = AudioUnitRender(vpio_unit_, flags, time_stamp, + output_bus_number, num_frames, io_data); + if (result != noErr) { + RTCLogError(@"Failed to render audio unit. Error=%ld", (long)result); + } + return result; +} + +OSStatus VoiceProcessingAudioUnit::OnGetPlayoutData( + void* in_ref_con, + AudioUnitRenderActionFlags* flags, + const AudioTimeStamp* time_stamp, + UInt32 bus_number, + UInt32 num_frames, + AudioBufferList* io_data) { + VoiceProcessingAudioUnit* audio_unit = + static_cast<VoiceProcessingAudioUnit*>(in_ref_con); + return audio_unit->NotifyGetPlayoutData(flags, time_stamp, bus_number, + num_frames, io_data); +} + +OSStatus VoiceProcessingAudioUnit::OnDeliverRecordedData( + void* in_ref_con, + AudioUnitRenderActionFlags* flags, + const AudioTimeStamp* time_stamp, + UInt32 bus_number, + UInt32 num_frames, + AudioBufferList* io_data) { + VoiceProcessingAudioUnit* audio_unit = + static_cast<VoiceProcessingAudioUnit*>(in_ref_con); + return audio_unit->NotifyDeliverRecordedData(flags, time_stamp, bus_number, + num_frames, io_data); +} + +OSStatus VoiceProcessingAudioUnit::NotifyGetPlayoutData( + AudioUnitRenderActionFlags* flags, + const AudioTimeStamp* time_stamp, + UInt32 bus_number, + UInt32 num_frames, + AudioBufferList* io_data) { + return observer_->OnGetPlayoutData(flags, time_stamp, bus_number, num_frames, + io_data); +} + +OSStatus VoiceProcessingAudioUnit::NotifyDeliverRecordedData( + AudioUnitRenderActionFlags* flags, + const AudioTimeStamp* time_stamp, + UInt32 bus_number, + UInt32 num_frames, + AudioBufferList* io_data) { + return observer_->OnDeliverRecordedData(flags, time_stamp, bus_number, + num_frames, io_data); +} + +AudioStreamBasicDescription VoiceProcessingAudioUnit::GetFormat( + Float64 sample_rate) const { + // Set the application formats for input and output: + // - use same format in both directions + // - avoid resampling in the I/O unit by using the hardware sample rate + // - linear PCM => noncompressed audio data format with one frame per packet + // - no need to specify interleaving since only mono is supported + AudioStreamBasicDescription format; + RTC_DCHECK_EQ(1, kRTCAudioSessionPreferredNumberOfChannels); + format.mSampleRate = sample_rate; + format.mFormatID = kAudioFormatLinearPCM; + format.mFormatFlags = + kLinearPCMFormatFlagIsSignedInteger | kLinearPCMFormatFlagIsPacked; + format.mBytesPerPacket = kBytesPerSample; + format.mFramesPerPacket = 1; // uncompressed. + format.mBytesPerFrame = kBytesPerSample; + format.mChannelsPerFrame = kRTCAudioSessionPreferredNumberOfChannels; + format.mBitsPerChannel = 8 * kBytesPerSample; + return format; +} + +void VoiceProcessingAudioUnit::DisposeAudioUnit() { + if (vpio_unit_) { + switch (state_) { + case kStarted: + Stop(); + // Fall through. + RTC_FALLTHROUGH(); + case kInitialized: + Uninitialize(); + break; + case kUninitialized: + RTC_FALLTHROUGH(); + case kInitRequired: + break; + } + + RTCLog(@"Disposing audio unit."); + OSStatus result = AudioComponentInstanceDispose(vpio_unit_); + if (result != noErr) { + RTCLogError(@"AudioComponentInstanceDispose failed. Error=%ld.", + (long)result); + } + vpio_unit_ = nullptr; + } +} + +} // namespace webrtc |