diff options
Diffstat (limited to 'audio')
65 files changed, 3454 insertions, 1980 deletions
diff --git a/audio/BUILD.gn b/audio/BUILD.gn index 78f6affe84..d2ba68459d 100644 --- a/audio/BUILD.gn +++ b/audio/BUILD.gn @@ -33,8 +33,6 @@ rtc_library("audio") { "channel_send_frame_transformer_delegate.cc", "channel_send_frame_transformer_delegate.h", "conversion.h", - "null_audio_poller.cc", - "null_audio_poller.h", "remix_resample.cc", "remix_resample.h", ] @@ -42,14 +40,17 @@ rtc_library("audio") { deps = [ "../api:array_view", "../api:call_api", + "../api:field_trials_view", "../api:frame_transformer_interface", "../api:function_view", "../api:rtp_headers", "../api:rtp_parameters", "../api:scoped_refptr", + "../api:sequence_checker", "../api:transport_api", "../api/audio:aec3_factory", "../api/audio:audio_frame_api", + "../api/audio:audio_frame_processor", "../api/audio:audio_mixer_api", "../api/audio_codecs:audio_codecs_api", "../api/crypto:frame_decryptor_interface", @@ -58,7 +59,9 @@ rtc_library("audio") { "../api/neteq:neteq_api", "../api/rtc_event_log", "../api/task_queue", + "../api/task_queue:pending_task_safety_flag", "../api/transport/rtp:rtp_source", + "../api/units:time_delta", "../call:audio_sender_interface", "../call:bitrate_allocator", "../call:call_interfaces", @@ -67,6 +70,8 @@ rtc_library("audio") { "../common_audio:common_audio_c", "../logging:rtc_event_audio", "../logging:rtc_stream_config", + "../media:rtc_media_base", + "../modules/async_audio_processing", "../modules/audio_coding", "../modules/audio_coding:audio_coding_module_typedefs", "../modules/audio_coding:audio_encoder_cng", @@ -78,28 +83,39 @@ rtc_library("audio") { "../modules/audio_processing:audio_frame_proxies", "../modules/audio_processing:rms_level", "../modules/pacing", - "../modules/remote_bitrate_estimator", "../modules/rtp_rtcp", "../modules/rtp_rtcp:rtp_rtcp_format", - "../modules/utility", - "../rtc_base", + "../modules/utility:utility", "../rtc_base:audio_format_to_string", + "../rtc_base:buffer", "../rtc_base:checks", + "../rtc_base:event_tracer", + "../rtc_base:logging", + "../rtc_base:macromagic", + "../rtc_base:race_checker", "../rtc_base:rate_limiter", - "../rtc_base:rtc_base_approved", + "../rtc_base:refcount", + "../rtc_base:rtc_event", "../rtc_base:rtc_task_queue", + "../rtc_base:safe_conversions", "../rtc_base:safe_minmax", + "../rtc_base:stringutils", + "../rtc_base:threading", + "../rtc_base:timeutils", + "../rtc_base/containers:flat_set", "../rtc_base/experiments:field_trial_parser", "../rtc_base/synchronization:mutex", - "../rtc_base/synchronization:sequence_checker", - "../rtc_base/task_utils:to_queued_task", + "../rtc_base/system:no_unique_address", + "../rtc_base/task_utils:repeating_task", "../system_wrappers", "../system_wrappers:field_trial", "../system_wrappers:metrics", "utility:audio_frame_operations", ] absl_deps = [ + "//third_party/abseil-cpp/absl/functional:any_invocable", "//third_party/abseil-cpp/absl/memory", + "//third_party/abseil-cpp/absl/strings", "//third_party/abseil-cpp/absl/types:optional", ] } @@ -136,6 +152,8 @@ if (rtc_include_tests) { "mock_voe_channel_proxy.h", "remix_resample_unittest.cc", "test/audio_stats_test.cc", + "test/nack_test.cc", + "test/non_sender_rtt_test.cc", ] deps = [ ":audio", @@ -148,8 +166,10 @@ if (rtc_include_tests) { "../api/audio_codecs:audio_codecs_api", "../api/audio_codecs/opus:audio_decoder_opus", "../api/audio_codecs/opus:audio_encoder_opus", + "../api/crypto:frame_decryptor_interface", "../api/rtc_event_log", "../api/task_queue:default_task_queue_factory", + "../api/task_queue/test:mock_task_queue_base", "../api/units:time_delta", "../call:mock_bitrate_allocator", "../call:mock_call_interfaces", @@ -168,9 +188,10 @@ if (rtc_include_tests) { "../modules/pacing", "../modules/rtp_rtcp:mock_rtp_rtcp", "../modules/rtp_rtcp:rtp_rtcp_format", - "../modules/utility", + "../modules/utility:utility", "../rtc_base:checks", - "../rtc_base:rtc_base_approved", + "../rtc_base:macromagic", + "../rtc_base:refcount", "../rtc_base:rtc_base_tests_utils", "../rtc_base:safe_compare", "../rtc_base:task_queue_for_test", @@ -182,14 +203,16 @@ if (rtc_include_tests) { "../test:mock_transformable_frame", "../test:mock_transport", "../test:rtp_test_utils", + "../test:scoped_key_value_config", "../test:test_common", "../test:test_support", + "../test/time_controller:time_controller", "utility:utility_tests", "//testing/gtest", ] } - if (rtc_enable_protobuf) { + if (rtc_enable_protobuf && !build_with_chromium) { rtc_test("low_bandwidth_audio_test") { testonly = true @@ -207,20 +230,34 @@ if (rtc_include_tests) { "../api:peer_connection_quality_test_fixture_api", "../api:simulated_network_api", "../api:time_controller", + "../api/test/metrics:chrome_perf_dashboard_metrics_exporter", + "../api/test/metrics:global_metrics_logger_and_exporter", + "../api/test/metrics:metrics_exporter", + "../api/test/metrics:stdout_metrics_exporter", + "../api/test/pclf:media_configuration", + "../api/test/pclf:media_quality_test_params", + "../api/test/pclf:peer_configurer", "../call:simulated_network", "../common_audio", "../system_wrappers", "../test:fileutils", - "../test:perf_test", "../test:test_common", "../test:test_main", "../test:test_support", "../test/pc/e2e:network_quality_metrics_reporter", "//testing/gtest", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/flags:flag", + "//third_party/abseil-cpp/absl/strings", ] if (is_android) { - deps += [ "//testing/android/native_test:native_test_native_code" ] + use_default_launcher = false + deps += [ + "//build/android/gtest_apk:native_test_instrumentation_test_runner_java", + "//testing/android/native_test:native_test_java", + "//testing/android/native_test:native_test_support", + ] } data = [ "../resources/voice_engine/audio_tiny16.wav", @@ -254,7 +291,7 @@ if (rtc_include_tests) { data += [ "${root_out_dir}/low_bandwidth_audio_test" ] } - if (is_linux || is_android) { + if (is_linux || is_chromeos || is_android || is_fuchsia) { data += [ "../tools_webrtc/audio_quality/linux/PolqaOem64", "../tools_webrtc/audio_quality/linux/pesq", @@ -271,35 +308,34 @@ if (rtc_include_tests) { if (is_mac) { data += [ "../tools_webrtc/audio_quality/mac/pesq" ] } - - write_runtime_deps = "${root_out_dir}/${target_name}.runtime_deps" } } - rtc_library("audio_perf_tests") { - testonly = true - - sources = [ - "test/audio_bwe_integration_test.cc", - "test/audio_bwe_integration_test.h", - ] - deps = [ - "../api:simulated_network_api", - "../api/task_queue", - "../call:fake_network", - "../call:simulated_network", - "../common_audio", - "../rtc_base:rtc_base_approved", - "../rtc_base:task_queue_for_test", - "../system_wrappers", - "../test:field_trial", - "../test:fileutils", - "../test:test_common", - "../test:test_main", - "../test:test_support", - "//testing/gtest", - ] + if (!build_with_chromium) { + rtc_library("audio_perf_tests") { + testonly = true - data = [ "//resources/voice_engine/audio_dtx16.wav" ] + sources = [ + "test/audio_bwe_integration_test.cc", + "test/audio_bwe_integration_test.h", + ] + deps = [ + "../api:simulated_network_api", + "../api/task_queue", + "../call:fake_network", + "../call:simulated_network", + "../common_audio", + "../rtc_base:task_queue_for_test", + "../system_wrappers", + "../test:field_trial", + "../test:fileutils", + "../test:test_common", + "../test:test_main", + "../test:test_support", + "//testing/gtest", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/functional:any_invocable" ] + data = [ "//resources/voice_engine/audio_dtx16.wav" ] + } } } diff --git a/audio/DEPS b/audio/DEPS index 8bb1f80805..7a0c7e7ce6 100644 --- a/audio/DEPS +++ b/audio/DEPS @@ -2,6 +2,8 @@ include_rules = [ "+call", "+common_audio", "+logging/rtc_event_log", + "+media/base", + "+modules/async_audio_processing", "+modules/audio_coding", "+modules/audio_device", "+modules/audio_mixer", @@ -10,7 +12,6 @@ include_rules = [ "+modules/bitrate_controller", "+modules/congestion_controller", "+modules/pacing", - "+modules/remote_bitrate_estimator", "+modules/rtp_rtcp", "+modules/utility", "+system_wrappers", diff --git a/audio/OWNERS.webrtc b/audio/OWNERS.webrtc index c0255e4d5f..e629bc1815 100644 --- a/audio/OWNERS.webrtc +++ b/audio/OWNERS.webrtc @@ -1,3 +1,5 @@ +alessiob@webrtc.org gustaf@webrtc.org +henrik.lundin@webrtc.org +jakobi@webrtc.org peah@webrtc.org -saza@webrtc.org diff --git a/audio/audio_receive_stream.cc b/audio/audio_receive_stream.cc index 6bc0d4137e..168d214ecd 100644 --- a/audio/audio_receive_stream.cc +++ b/audio/audio_receive_stream.cc @@ -18,12 +18,14 @@ #include "api/audio_codecs/audio_format.h" #include "api/call/audio_sink.h" #include "api/rtp_parameters.h" +#include "api/sequence_checker.h" #include "audio/audio_send_stream.h" #include "audio/audio_state.h" #include "audio/channel_receive.h" #include "audio/conversion.h" #include "call/rtp_config.h" #include "call/rtp_stream_receiver_controller_interface.h" +#include "modules/rtp_rtcp/source/rtp_packet_received.h" #include "rtc_base/checks.h" #include "rtc_base/logging.h" #include "rtc_base/strings/string_builder.h" @@ -31,7 +33,7 @@ namespace webrtc { -std::string AudioReceiveStream::Config::Rtp::ToString() const { +std::string AudioReceiveStreamInterface::Config::Rtp::ToString() const { char ss_buf[1024]; rtc::SimpleStringBuilder ss(ss_buf); ss << "{remote_ssrc: " << remote_ssrc; @@ -50,7 +52,7 @@ std::string AudioReceiveStream::Config::Rtp::ToString() const { return ss.str(); } -std::string AudioReceiveStream::Config::ToString() const { +std::string AudioReceiveStreamInterface::Config::ToString() const { char ss_buf[1024]; rtc::SimpleStringBuilder ss(ss_buf); ss << "{rtp: " << rtp.ToString(); @@ -63,97 +65,131 @@ std::string AudioReceiveStream::Config::ToString() const { return ss.str(); } -namespace internal { namespace { std::unique_ptr<voe::ChannelReceiveInterface> CreateChannelReceive( Clock* clock, webrtc::AudioState* audio_state, - ProcessThread* module_process_thread, NetEqFactory* neteq_factory, - const webrtc::AudioReceiveStream::Config& config, + const webrtc::AudioReceiveStreamInterface::Config& config, RtcEventLog* event_log) { RTC_DCHECK(audio_state); internal::AudioState* internal_audio_state = static_cast<internal::AudioState*>(audio_state); return voe::CreateChannelReceive( - clock, module_process_thread, neteq_factory, - internal_audio_state->audio_device_module(), config.rtcp_send_transport, - event_log, config.rtp.local_ssrc, config.rtp.remote_ssrc, - config.jitter_buffer_max_packets, config.jitter_buffer_fast_accelerate, - config.jitter_buffer_min_delay_ms, - config.jitter_buffer_enable_rtx_handling, config.decoder_factory, - config.codec_pair_id, config.frame_decryptor, config.crypto_options, - std::move(config.frame_transformer)); + clock, neteq_factory, internal_audio_state->audio_device_module(), + config.rtcp_send_transport, event_log, config.rtp.local_ssrc, + config.rtp.remote_ssrc, config.jitter_buffer_max_packets, + config.jitter_buffer_fast_accelerate, config.jitter_buffer_min_delay_ms, + config.enable_non_sender_rtt, config.decoder_factory, + config.codec_pair_id, std::move(config.frame_decryptor), + config.crypto_options, std::move(config.frame_transformer)); } } // namespace -AudioReceiveStream::AudioReceiveStream( +AudioReceiveStreamImpl::AudioReceiveStreamImpl( Clock* clock, - RtpStreamReceiverControllerInterface* receiver_controller, PacketRouter* packet_router, - ProcessThread* module_process_thread, NetEqFactory* neteq_factory, - const webrtc::AudioReceiveStream::Config& config, + const webrtc::AudioReceiveStreamInterface::Config& config, const rtc::scoped_refptr<webrtc::AudioState>& audio_state, webrtc::RtcEventLog* event_log) - : AudioReceiveStream(clock, - receiver_controller, - packet_router, - config, - audio_state, - event_log, - CreateChannelReceive(clock, - audio_state.get(), - module_process_thread, - neteq_factory, - config, - event_log)) {} - -AudioReceiveStream::AudioReceiveStream( + : AudioReceiveStreamImpl(clock, + packet_router, + config, + audio_state, + event_log, + CreateChannelReceive(clock, + audio_state.get(), + neteq_factory, + config, + event_log)) {} + +AudioReceiveStreamImpl::AudioReceiveStreamImpl( Clock* clock, - RtpStreamReceiverControllerInterface* receiver_controller, PacketRouter* packet_router, - const webrtc::AudioReceiveStream::Config& config, + const webrtc::AudioReceiveStreamInterface::Config& config, const rtc::scoped_refptr<webrtc::AudioState>& audio_state, webrtc::RtcEventLog* event_log, std::unique_ptr<voe::ChannelReceiveInterface> channel_receive) - : audio_state_(audio_state), - channel_receive_(std::move(channel_receive)), - source_tracker_(clock) { - RTC_LOG(LS_INFO) << "AudioReceiveStream: " << config.rtp.remote_ssrc; + : config_(config), + audio_state_(audio_state), + source_tracker_(clock), + channel_receive_(std::move(channel_receive)) { + RTC_LOG(LS_INFO) << "AudioReceiveStreamImpl: " << config.rtp.remote_ssrc; RTC_DCHECK(config.decoder_factory); RTC_DCHECK(config.rtcp_send_transport); RTC_DCHECK(audio_state_); RTC_DCHECK(channel_receive_); - module_process_thread_checker_.Detach(); + packet_sequence_checker_.Detach(); - RTC_DCHECK(receiver_controller); RTC_DCHECK(packet_router); // Configure bandwidth estimation. channel_receive_->RegisterReceiverCongestionControlObjects(packet_router); - // Register with transport. - rtp_stream_receiver_ = receiver_controller->CreateReceiver( - config.rtp.remote_ssrc, channel_receive_.get()); - ConfigureStream(this, config, true); + // When output is muted, ChannelReceive will directly notify the source + // tracker of "delivered" frames, so RtpReceiver information will continue to + // be updated. + channel_receive_->SetSourceTracker(&source_tracker_); + + // Complete configuration. + // TODO(solenberg): Config NACK history window (which is a packet count), + // using the actual packet size for the configured codec. + channel_receive_->SetNACKStatus(config.rtp.nack.rtp_history_ms != 0, + config.rtp.nack.rtp_history_ms / 20); + channel_receive_->SetReceiveCodecs(config.decoder_map); + // `frame_transformer` and `frame_decryptor` have been given to + // `channel_receive_` already. } -AudioReceiveStream::~AudioReceiveStream() { +AudioReceiveStreamImpl::~AudioReceiveStreamImpl() { RTC_DCHECK_RUN_ON(&worker_thread_checker_); - RTC_LOG(LS_INFO) << "~AudioReceiveStream: " << config_.rtp.remote_ssrc; + RTC_LOG(LS_INFO) << "~AudioReceiveStreamImpl: " << remote_ssrc(); Stop(); channel_receive_->SetAssociatedSendChannel(nullptr); channel_receive_->ResetReceiverCongestionControlObjects(); } -void AudioReceiveStream::Reconfigure( - const webrtc::AudioReceiveStream::Config& config) { - RTC_DCHECK(worker_thread_checker_.IsCurrent()); - ConfigureStream(this, config, false); +void AudioReceiveStreamImpl::RegisterWithTransport( + RtpStreamReceiverControllerInterface* receiver_controller) { + RTC_DCHECK_RUN_ON(&packet_sequence_checker_); + RTC_DCHECK(!rtp_stream_receiver_); + rtp_stream_receiver_ = receiver_controller->CreateReceiver( + remote_ssrc(), channel_receive_.get()); +} + +void AudioReceiveStreamImpl::UnregisterFromTransport() { + RTC_DCHECK_RUN_ON(&packet_sequence_checker_); + rtp_stream_receiver_.reset(); } -void AudioReceiveStream::Start() { +void AudioReceiveStreamImpl::ReconfigureForTesting( + const webrtc::AudioReceiveStreamInterface::Config& config) { + RTC_DCHECK_RUN_ON(&worker_thread_checker_); + + // SSRC can't be changed mid-stream. + RTC_DCHECK_EQ(remote_ssrc(), config.rtp.remote_ssrc); + RTC_DCHECK_EQ(local_ssrc(), config.rtp.local_ssrc); + + // Configuration parameters which cannot be changed. + RTC_DCHECK_EQ(config_.rtcp_send_transport, config.rtcp_send_transport); + // Decoder factory cannot be changed because it is configured at + // voe::Channel construction time. + RTC_DCHECK_EQ(config_.decoder_factory, config.decoder_factory); + + // TODO(solenberg): Config NACK history window (which is a packet count), + // using the actual packet size for the configured codec. + RTC_DCHECK_EQ(config_.rtp.nack.rtp_history_ms, config.rtp.nack.rtp_history_ms) + << "Use SetUseTransportCcAndNackHistory"; + + RTC_DCHECK(config_.decoder_map == config.decoder_map) << "Use SetDecoderMap"; + RTC_DCHECK_EQ(config_.frame_transformer, config.frame_transformer) + << "Use SetDepacketizerToDecoderFrameTransformer"; + + config_ = config; +} + +void AudioReceiveStreamImpl::Start() { RTC_DCHECK_RUN_ON(&worker_thread_checker_); if (playing_) { return; @@ -163,7 +199,7 @@ void AudioReceiveStream::Start() { audio_state()->AddReceivingStream(this); } -void AudioReceiveStream::Stop() { +void AudioReceiveStreamImpl::Stop() { RTC_DCHECK_RUN_ON(&worker_thread_checker_); if (!playing_) { return; @@ -173,10 +209,85 @@ void AudioReceiveStream::Stop() { audio_state()->RemoveReceivingStream(this); } -webrtc::AudioReceiveStream::Stats AudioReceiveStream::GetStats() const { +bool AudioReceiveStreamImpl::transport_cc() const { + RTC_DCHECK_RUN_ON(&packet_sequence_checker_); + return config_.rtp.transport_cc; +} + +void AudioReceiveStreamImpl::SetTransportCc(bool transport_cc) { + RTC_DCHECK_RUN_ON(&packet_sequence_checker_); + config_.rtp.transport_cc = transport_cc; +} + +bool AudioReceiveStreamImpl::IsRunning() const { + RTC_DCHECK_RUN_ON(&worker_thread_checker_); + return playing_; +} + +void AudioReceiveStreamImpl::SetDepacketizerToDecoderFrameTransformer( + rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer) { + RTC_DCHECK_RUN_ON(&worker_thread_checker_); + channel_receive_->SetDepacketizerToDecoderFrameTransformer( + std::move(frame_transformer)); +} + +void AudioReceiveStreamImpl::SetDecoderMap( + std::map<int, SdpAudioFormat> decoder_map) { + RTC_DCHECK_RUN_ON(&worker_thread_checker_); + config_.decoder_map = std::move(decoder_map); + channel_receive_->SetReceiveCodecs(config_.decoder_map); +} + +void AudioReceiveStreamImpl::SetNackHistory(int history_ms) { + RTC_DCHECK_RUN_ON(&worker_thread_checker_); + RTC_DCHECK_GE(history_ms, 0); + + if (config_.rtp.nack.rtp_history_ms == history_ms) + return; + + config_.rtp.nack.rtp_history_ms = history_ms; + // TODO(solenberg): Config NACK history window (which is a packet count), + // using the actual packet size for the configured codec. + channel_receive_->SetNACKStatus(history_ms != 0, history_ms / 20); +} + +void AudioReceiveStreamImpl::SetNonSenderRttMeasurement(bool enabled) { + RTC_DCHECK_RUN_ON(&worker_thread_checker_); + config_.enable_non_sender_rtt = enabled; + channel_receive_->SetNonSenderRttMeasurement(enabled); +} + +void AudioReceiveStreamImpl::SetFrameDecryptor( + rtc::scoped_refptr<webrtc::FrameDecryptorInterface> frame_decryptor) { + // TODO(bugs.webrtc.org/11993): This is called via WebRtcAudioReceiveStream, + // expect to be called on the network thread. + RTC_DCHECK_RUN_ON(&worker_thread_checker_); + channel_receive_->SetFrameDecryptor(std::move(frame_decryptor)); +} + +void AudioReceiveStreamImpl::SetRtpExtensions( + std::vector<RtpExtension> extensions) { + // TODO(bugs.webrtc.org/11993): This is called via WebRtcAudioReceiveStream, + // expect to be called on the network thread. + RTC_DCHECK_RUN_ON(&worker_thread_checker_); + config_.rtp.extensions = std::move(extensions); +} + +const std::vector<RtpExtension>& AudioReceiveStreamImpl::GetRtpExtensions() + const { + RTC_DCHECK_RUN_ON(&worker_thread_checker_); + return config_.rtp.extensions; +} + +RtpHeaderExtensionMap AudioReceiveStreamImpl::GetRtpExtensionMap() const { + return RtpHeaderExtensionMap(config_.rtp.extensions); +} + +webrtc::AudioReceiveStreamInterface::Stats AudioReceiveStreamImpl::GetStats( + bool get_and_clear_legacy_stats) const { RTC_DCHECK_RUN_ON(&worker_thread_checker_); - webrtc::AudioReceiveStream::Stats stats; - stats.remote_ssrc = config_.rtp.remote_ssrc; + webrtc::AudioReceiveStreamInterface::Stats stats; + stats.remote_ssrc = remote_ssrc(); webrtc::CallReceiveStatistics call_stats = channel_receive_->GetRTCPStatistics(); @@ -192,6 +303,7 @@ webrtc::AudioReceiveStream::Stats AudioReceiveStream::GetStats() const { call_stats.header_and_padding_bytes_rcvd; stats.packets_rcvd = call_stats.packetsReceived; stats.packets_lost = call_stats.cumulativeLost; + stats.nacks_sent = call_stats.nacks_sent; stats.capture_start_ntp_time_ms = call_stats.capture_start_ntp_time_ms_; stats.last_packet_received_timestamp_ms = call_stats.last_packet_received_timestamp_ms; @@ -210,7 +322,8 @@ webrtc::AudioReceiveStream::Stats AudioReceiveStream::GetStats() const { rtc::TimeMillis()); // Get jitter buffer and total delay (alg + jitter + playout) stats. - auto ns = channel_receive_->GetNetworkStatistics(); + auto ns = channel_receive_->GetNetworkStatistics(get_and_clear_legacy_stats); + stats.packets_discarded = ns.packetsDiscarded; stats.fec_packets_received = ns.fecPacketsReceived; stats.fec_packets_discarded = ns.fecPacketsDiscarded; stats.jitter_buffer_ms = ns.currentBufferSize; @@ -226,6 +339,9 @@ webrtc::AudioReceiveStream::Stats AudioReceiveStream::GetStats() const { stats.jitter_buffer_target_delay_seconds = static_cast<double>(ns.jitterBufferTargetDelayMs) / static_cast<double>(rtc::kNumMillisecsPerSec); + stats.jitter_buffer_minimum_delay_seconds = + static_cast<double>(ns.jitterBufferMinimumDelayMs) / + static_cast<double>(rtc::kNumMillisecsPerSec); stats.inserted_samples_for_deceleration = ns.insertedSamplesForDeceleration; stats.removed_samples_for_acceleration = ns.removedSamplesForAcceleration; stats.expand_rate = Q14ToFloat(ns.currentExpandRate); @@ -252,37 +368,48 @@ webrtc::AudioReceiveStream::Stats AudioReceiveStream::GetStats() const { stats.decoding_plc_cng = ds.decoded_plc_cng; stats.decoding_muted_output = ds.decoded_muted_output; + stats.last_sender_report_timestamp_ms = + call_stats.last_sender_report_timestamp_ms; + stats.last_sender_report_remote_timestamp_ms = + call_stats.last_sender_report_remote_timestamp_ms; + stats.sender_reports_packets_sent = call_stats.sender_reports_packets_sent; + stats.sender_reports_bytes_sent = call_stats.sender_reports_bytes_sent; + stats.sender_reports_reports_count = call_stats.sender_reports_reports_count; + stats.round_trip_time = call_stats.round_trip_time; + stats.round_trip_time_measurements = call_stats.round_trip_time_measurements; + stats.total_round_trip_time = call_stats.total_round_trip_time; + return stats; } -void AudioReceiveStream::SetSink(AudioSinkInterface* sink) { +void AudioReceiveStreamImpl::SetSink(AudioSinkInterface* sink) { RTC_DCHECK_RUN_ON(&worker_thread_checker_); channel_receive_->SetSink(sink); } -void AudioReceiveStream::SetGain(float gain) { +void AudioReceiveStreamImpl::SetGain(float gain) { RTC_DCHECK_RUN_ON(&worker_thread_checker_); channel_receive_->SetChannelOutputVolumeScaling(gain); } -bool AudioReceiveStream::SetBaseMinimumPlayoutDelayMs(int delay_ms) { +bool AudioReceiveStreamImpl::SetBaseMinimumPlayoutDelayMs(int delay_ms) { RTC_DCHECK_RUN_ON(&worker_thread_checker_); return channel_receive_->SetBaseMinimumPlayoutDelayMs(delay_ms); } -int AudioReceiveStream::GetBaseMinimumPlayoutDelayMs() const { +int AudioReceiveStreamImpl::GetBaseMinimumPlayoutDelayMs() const { RTC_DCHECK_RUN_ON(&worker_thread_checker_); return channel_receive_->GetBaseMinimumPlayoutDelayMs(); } -std::vector<RtpSource> AudioReceiveStream::GetSources() const { +std::vector<RtpSource> AudioReceiveStreamImpl::GetSources() const { RTC_DCHECK_RUN_ON(&worker_thread_checker_); return source_tracker_.GetSources(); } -AudioMixer::Source::AudioFrameInfo AudioReceiveStream::GetAudioFrameWithInfo( - int sample_rate_hz, - AudioFrame* audio_frame) { +AudioMixer::Source::AudioFrameInfo +AudioReceiveStreamImpl::GetAudioFrameWithInfo(int sample_rate_hz, + AudioFrame* audio_frame) { AudioMixer::Source::AudioFrameInfo audio_frame_info = channel_receive_->GetAudioFrameWithInfo(sample_rate_hz, audio_frame); if (audio_frame_info != AudioMixer::Source::AudioFrameInfo::kError) { @@ -291,37 +418,33 @@ AudioMixer::Source::AudioFrameInfo AudioReceiveStream::GetAudioFrameWithInfo( return audio_frame_info; } -int AudioReceiveStream::Ssrc() const { - return config_.rtp.remote_ssrc; +int AudioReceiveStreamImpl::Ssrc() const { + return remote_ssrc(); } -int AudioReceiveStream::PreferredSampleRate() const { +int AudioReceiveStreamImpl::PreferredSampleRate() const { return channel_receive_->PreferredSampleRate(); } -uint32_t AudioReceiveStream::id() const { +uint32_t AudioReceiveStreamImpl::id() const { RTC_DCHECK_RUN_ON(&worker_thread_checker_); - return config_.rtp.remote_ssrc; + return remote_ssrc(); } -absl::optional<Syncable::Info> AudioReceiveStream::GetInfo() const { - RTC_DCHECK_RUN_ON(&module_process_thread_checker_); - absl::optional<Syncable::Info> info = channel_receive_->GetSyncInfo(); - - if (!info) - return absl::nullopt; - - info->current_delay_ms = channel_receive_->GetDelayEstimate(); - return info; +absl::optional<Syncable::Info> AudioReceiveStreamImpl::GetInfo() const { + // TODO(bugs.webrtc.org/11993): This is called via RtpStreamsSynchronizer, + // expect to be called on the network thread. + RTC_DCHECK_RUN_ON(&worker_thread_checker_); + return channel_receive_->GetSyncInfo(); } -bool AudioReceiveStream::GetPlayoutRtpTimestamp(uint32_t* rtp_timestamp, - int64_t* time_ms) const { +bool AudioReceiveStreamImpl::GetPlayoutRtpTimestamp(uint32_t* rtp_timestamp, + int64_t* time_ms) const { // Called on video capture thread. return channel_receive_->GetPlayoutRtpTimestamp(rtp_timestamp, time_ms); } -void AudioReceiveStream::SetEstimatedPlayoutNtpTimestampMs( +void AudioReceiveStreamImpl::SetEstimatedPlayoutNtpTimestampMs( int64_t ntp_timestamp_ms, int64_t time_ms) { // Called on video capture thread. @@ -329,19 +452,22 @@ void AudioReceiveStream::SetEstimatedPlayoutNtpTimestampMs( time_ms); } -void AudioReceiveStream::SetMinimumPlayoutDelay(int delay_ms) { - RTC_DCHECK_RUN_ON(&module_process_thread_checker_); +bool AudioReceiveStreamImpl::SetMinimumPlayoutDelay(int delay_ms) { + // TODO(bugs.webrtc.org/11993): This is called via RtpStreamsSynchronizer, + // expect to be called on the network thread. + RTC_DCHECK_RUN_ON(&worker_thread_checker_); return channel_receive_->SetMinimumPlayoutDelay(delay_ms); } -void AudioReceiveStream::AssociateSendStream(AudioSendStream* send_stream) { - RTC_DCHECK_RUN_ON(&worker_thread_checker_); +void AudioReceiveStreamImpl::AssociateSendStream( + internal::AudioSendStream* send_stream) { + RTC_DCHECK_RUN_ON(&packet_sequence_checker_); channel_receive_->SetAssociatedSendChannel( send_stream ? send_stream->GetChannel() : nullptr); associated_send_stream_ = send_stream; } -void AudioReceiveStream::DeliverRtcp(const uint8_t* packet, size_t length) { +void AudioReceiveStreamImpl::DeliverRtcp(const uint8_t* packet, size_t length) { // TODO(solenberg): Tests call this function on a network thread, libjingle // calls on the worker thread. We should move towards always using a network // thread. Then this check can be enabled. @@ -349,74 +475,38 @@ void AudioReceiveStream::DeliverRtcp(const uint8_t* packet, size_t length) { channel_receive_->ReceivedRTCPPacket(packet, length); } -void AudioReceiveStream::OnRtpPacket(const RtpPacketReceived& packet) { - // TODO(solenberg): Tests call this function on a network thread, libjingle - // calls on the worker thread. We should move towards always using a network - // thread. Then this check can be enabled. - // RTC_DCHECK(!thread_checker_.IsCurrent()); - channel_receive_->OnRtpPacket(packet); +void AudioReceiveStreamImpl::SetSyncGroup(absl::string_view sync_group) { + RTC_DCHECK_RUN_ON(&packet_sequence_checker_); + config_.sync_group = std::string(sync_group); } -const webrtc::AudioReceiveStream::Config& AudioReceiveStream::config() const { - RTC_DCHECK_RUN_ON(&worker_thread_checker_); - return config_; +void AudioReceiveStreamImpl::SetLocalSsrc(uint32_t local_ssrc) { + RTC_DCHECK_RUN_ON(&packet_sequence_checker_); + // TODO(tommi): Consider storing local_ssrc in one place. + config_.rtp.local_ssrc = local_ssrc; + channel_receive_->OnLocalSsrcChange(local_ssrc); } -const AudioSendStream* AudioReceiveStream::GetAssociatedSendStreamForTesting() - const { - RTC_DCHECK_RUN_ON(&worker_thread_checker_); +uint32_t AudioReceiveStreamImpl::local_ssrc() const { + RTC_DCHECK_RUN_ON(&packet_sequence_checker_); + RTC_DCHECK_EQ(config_.rtp.local_ssrc, channel_receive_->GetLocalSsrc()); + return config_.rtp.local_ssrc; +} + +const std::string& AudioReceiveStreamImpl::sync_group() const { + RTC_DCHECK_RUN_ON(&packet_sequence_checker_); + return config_.sync_group; +} + +const AudioSendStream* +AudioReceiveStreamImpl::GetAssociatedSendStreamForTesting() const { + RTC_DCHECK_RUN_ON(&packet_sequence_checker_); return associated_send_stream_; } -internal::AudioState* AudioReceiveStream::audio_state() const { +internal::AudioState* AudioReceiveStreamImpl::audio_state() const { auto* audio_state = static_cast<internal::AudioState*>(audio_state_.get()); RTC_DCHECK(audio_state); return audio_state; } - -void AudioReceiveStream::ConfigureStream(AudioReceiveStream* stream, - const Config& new_config, - bool first_time) { - RTC_LOG(LS_INFO) << "AudioReceiveStream::ConfigureStream: " - << new_config.ToString(); - RTC_DCHECK(stream); - const auto& channel_receive = stream->channel_receive_; - const auto& old_config = stream->config_; - - // Configuration parameters which cannot be changed. - RTC_DCHECK(first_time || - old_config.rtp.remote_ssrc == new_config.rtp.remote_ssrc); - RTC_DCHECK(first_time || - old_config.rtcp_send_transport == new_config.rtcp_send_transport); - // Decoder factory cannot be changed because it is configured at - // voe::Channel construction time. - RTC_DCHECK(first_time || - old_config.decoder_factory == new_config.decoder_factory); - - if (!first_time) { - // SSRC can't be changed mid-stream. - RTC_DCHECK_EQ(old_config.rtp.local_ssrc, new_config.rtp.local_ssrc); - RTC_DCHECK_EQ(old_config.rtp.remote_ssrc, new_config.rtp.remote_ssrc); - } - - // TODO(solenberg): Config NACK history window (which is a packet count), - // using the actual packet size for the configured codec. - if (first_time || old_config.rtp.nack.rtp_history_ms != - new_config.rtp.nack.rtp_history_ms) { - channel_receive->SetNACKStatus(new_config.rtp.nack.rtp_history_ms != 0, - new_config.rtp.nack.rtp_history_ms / 20); - } - if (first_time || old_config.decoder_map != new_config.decoder_map) { - channel_receive->SetReceiveCodecs(new_config.decoder_map); - } - - if (first_time || - old_config.frame_transformer != new_config.frame_transformer) { - channel_receive->SetDepacketizerToDecoderFrameTransformer( - new_config.frame_transformer); - } - - stream->config_ = new_config; -} -} // namespace internal } // namespace webrtc diff --git a/audio/audio_receive_stream.h b/audio/audio_receive_stream.h index c197aa8833..427077fd94 100644 --- a/audio/audio_receive_stream.h +++ b/audio/audio_receive_stream.h @@ -11,25 +11,26 @@ #ifndef AUDIO_AUDIO_RECEIVE_STREAM_H_ #define AUDIO_AUDIO_RECEIVE_STREAM_H_ +#include <map> #include <memory> +#include <string> #include <vector> +#include "absl/strings/string_view.h" #include "api/audio/audio_mixer.h" #include "api/neteq/neteq_factory.h" #include "api/rtp_headers.h" +#include "api/sequence_checker.h" #include "audio/audio_state.h" #include "call/audio_receive_stream.h" #include "call/syncable.h" #include "modules/rtp_rtcp/source/source_tracker.h" -#include "rtc_base/constructor_magic.h" -#include "rtc_base/thread_checker.h" +#include "rtc_base/system/no_unique_address.h" #include "system_wrappers/include/clock.h" namespace webrtc { class PacketRouter; -class ProcessThread; class RtcEventLog; -class RtpPacketReceived; class RtpStreamReceiverControllerInterface; class RtpStreamReceiverInterface; @@ -39,47 +40,74 @@ class ChannelReceiveInterface; namespace internal { class AudioSendStream; +} // namespace internal -class AudioReceiveStream final : public webrtc::AudioReceiveStream, - public AudioMixer::Source, - public Syncable { +class AudioReceiveStreamImpl final : public webrtc::AudioReceiveStreamInterface, + public AudioMixer::Source, + public Syncable { public: - AudioReceiveStream(Clock* clock, - RtpStreamReceiverControllerInterface* receiver_controller, - PacketRouter* packet_router, - ProcessThread* module_process_thread, - NetEqFactory* neteq_factory, - const webrtc::AudioReceiveStream::Config& config, - const rtc::scoped_refptr<webrtc::AudioState>& audio_state, - webrtc::RtcEventLog* event_log); + AudioReceiveStreamImpl( + Clock* clock, + PacketRouter* packet_router, + NetEqFactory* neteq_factory, + const webrtc::AudioReceiveStreamInterface::Config& config, + const rtc::scoped_refptr<webrtc::AudioState>& audio_state, + webrtc::RtcEventLog* event_log); // For unit tests, which need to supply a mock channel receive. - AudioReceiveStream( + AudioReceiveStreamImpl( Clock* clock, - RtpStreamReceiverControllerInterface* receiver_controller, PacketRouter* packet_router, - const webrtc::AudioReceiveStream::Config& config, + const webrtc::AudioReceiveStreamInterface::Config& config, const rtc::scoped_refptr<webrtc::AudioState>& audio_state, webrtc::RtcEventLog* event_log, std::unique_ptr<voe::ChannelReceiveInterface> channel_receive); - ~AudioReceiveStream() override; - // webrtc::AudioReceiveStream implementation. - void Reconfigure(const webrtc::AudioReceiveStream::Config& config) override; + AudioReceiveStreamImpl() = delete; + AudioReceiveStreamImpl(const AudioReceiveStreamImpl&) = delete; + AudioReceiveStreamImpl& operator=(const AudioReceiveStreamImpl&) = delete; + + // Destruction happens on the worker thread. Prior to destruction the caller + // must ensure that a registration with the transport has been cleared. See + // `RegisterWithTransport` for details. + // TODO(tommi): As a further improvement to this, performing the full + // destruction on the network thread could be made the default. + ~AudioReceiveStreamImpl() override; + + // Called on the network thread to register/unregister with the network + // transport. + void RegisterWithTransport( + RtpStreamReceiverControllerInterface* receiver_controller); + // If registration has previously been done (via `RegisterWithTransport`) then + // `UnregisterFromTransport` must be called prior to destruction, on the + // network thread. + void UnregisterFromTransport(); + + // webrtc::AudioReceiveStreamInterface implementation. void Start() override; void Stop() override; - webrtc::AudioReceiveStream::Stats GetStats() const override; + bool transport_cc() const override; + void SetTransportCc(bool transport_cc) override; + bool IsRunning() const override; + void SetDepacketizerToDecoderFrameTransformer( + rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer) + override; + void SetDecoderMap(std::map<int, SdpAudioFormat> decoder_map) override; + void SetNackHistory(int history_ms) override; + void SetNonSenderRttMeasurement(bool enabled) override; + void SetFrameDecryptor(rtc::scoped_refptr<webrtc::FrameDecryptorInterface> + frame_decryptor) override; + void SetRtpExtensions(std::vector<RtpExtension> extensions) override; + const std::vector<RtpExtension>& GetRtpExtensions() const override; + RtpHeaderExtensionMap GetRtpExtensionMap() const override; + + webrtc::AudioReceiveStreamInterface::Stats GetStats( + bool get_and_clear_legacy_stats) const override; void SetSink(AudioSinkInterface* sink) override; void SetGain(float gain) override; bool SetBaseMinimumPlayoutDelayMs(int delay_ms) override; int GetBaseMinimumPlayoutDelayMs() const override; std::vector<webrtc::RtpSource> GetSources() const override; - // TODO(nisse): We don't formally implement RtpPacketSinkInterface, and this - // method shouldn't be needed. But it's currently used by the - // AudioReceiveStreamTest.ReceiveRtpPacket unittest. Figure out if that test - // shuld be refactored or deleted, and then delete this method. - void OnRtpPacket(const RtpPacketReceived& packet); - // AudioMixer::Source AudioFrameInfo GetAudioFrameWithInfo(int sample_rate_hz, AudioFrame* audio_frame) override; @@ -93,35 +121,57 @@ class AudioReceiveStream final : public webrtc::AudioReceiveStream, int64_t* time_ms) const override; void SetEstimatedPlayoutNtpTimestampMs(int64_t ntp_timestamp_ms, int64_t time_ms) override; - void SetMinimumPlayoutDelay(int delay_ms) override; + bool SetMinimumPlayoutDelay(int delay_ms) override; - void AssociateSendStream(AudioSendStream* send_stream); + void AssociateSendStream(internal::AudioSendStream* send_stream); void DeliverRtcp(const uint8_t* packet, size_t length); - const webrtc::AudioReceiveStream::Config& config() const; - const AudioSendStream* GetAssociatedSendStreamForTesting() const; - private: - static void ConfigureStream(AudioReceiveStream* stream, - const Config& new_config, - bool first_time); + void SetSyncGroup(absl::string_view sync_group); + + void SetLocalSsrc(uint32_t local_ssrc); + + uint32_t local_ssrc() const; - AudioState* audio_state() const; + uint32_t remote_ssrc() const override { + // The remote_ssrc member variable of config_ will never change and can be + // considered const. + return config_.rtp.remote_ssrc; + } - rtc::ThreadChecker worker_thread_checker_; - rtc::ThreadChecker module_process_thread_checker_; - webrtc::AudioReceiveStream::Config config_; + // Returns a reference to the currently set sync group of the stream. + // Must be called on the packet delivery thread. + const std::string& sync_group() const; + + const AudioSendStream* GetAssociatedSendStreamForTesting() const; + + // TODO(tommi): Remove this method. + void ReconfigureForTesting( + const webrtc::AudioReceiveStreamInterface::Config& config); + + private: + internal::AudioState* audio_state() const; + + RTC_NO_UNIQUE_ADDRESS SequenceChecker worker_thread_checker_; + // TODO(bugs.webrtc.org/11993): This checker conceptually represents + // operations that belong to the network thread. The Call class is currently + // moving towards handling network packets on the network thread and while + // that work is ongoing, this checker may in practice represent the worker + // thread, but still serves as a mechanism of grouping together concepts + // that belong to the network thread. Once the packets are fully delivered + // on the network thread, this comment will be deleted. + RTC_NO_UNIQUE_ADDRESS SequenceChecker packet_sequence_checker_; + webrtc::AudioReceiveStreamInterface::Config config_; rtc::scoped_refptr<webrtc::AudioState> audio_state_; - const std::unique_ptr<voe::ChannelReceiveInterface> channel_receive_; SourceTracker source_tracker_; - AudioSendStream* associated_send_stream_ = nullptr; + const std::unique_ptr<voe::ChannelReceiveInterface> channel_receive_; + AudioSendStream* associated_send_stream_ + RTC_GUARDED_BY(packet_sequence_checker_) = nullptr; bool playing_ RTC_GUARDED_BY(worker_thread_checker_) = false; - std::unique_ptr<RtpStreamReceiverInterface> rtp_stream_receiver_; - - RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(AudioReceiveStream); + std::unique_ptr<RtpStreamReceiverInterface> rtp_stream_receiver_ + RTC_GUARDED_BY(packet_sequence_checker_); }; -} // namespace internal } // namespace webrtc #endif // AUDIO_AUDIO_RECEIVE_STREAM_H_ diff --git a/audio/audio_receive_stream_unittest.cc b/audio/audio_receive_stream_unittest.cc index 7759dd1e72..75129acf48 100644 --- a/audio/audio_receive_stream_unittest.cc +++ b/audio/audio_receive_stream_unittest.cc @@ -36,6 +36,7 @@ namespace { using ::testing::_; using ::testing::FloatEq; +using ::testing::NiceMock; using ::testing::Return; AudioDecodingCallStats MakeAudioDecodeStatsForTest() { @@ -53,8 +54,6 @@ AudioDecodingCallStats MakeAudioDecodeStatsForTest() { const uint32_t kRemoteSsrc = 1234; const uint32_t kLocalSsrc = 5678; -const size_t kOneByteExtensionHeaderLength = 4; -const size_t kOneByteExtensionLength = 4; const int kAudioLevelId = 3; const int kTransportSequenceNumberId = 4; const int kJitterBufferDelay = -7; @@ -69,14 +68,40 @@ const std::pair<int, SdpAudioFormat> kReceiveCodec = { 123, {"codec_name_recv", 96000, 0}}; const NetworkStatistics kNetworkStats = { - 123, 456, false, 789012, 3456, 123, 456, 789, 543, 123, - 432, 321, 123, 101, 0, {}, 789, 12, 345, 678, - 901, 0, -1, -1, -1, -1, 0, 0, 0, 0}; + /*currentBufferSize=*/123, + /*preferredBufferSize=*/456, + /*jitterPeaksFound=*/false, + /*totalSamplesReceived=*/789012, + /*concealedSamples=*/3456, + /*silentConcealedSamples=*/123, + /*concealmentEvents=*/456, + /*jitterBufferDelayMs=*/789, + /*jitterBufferEmittedCount=*/543, + /*jitterBufferTargetDelayMs=*/123, + /*jitterBufferMinimumDelayMs=*/222, + /*insertedSamplesForDeceleration=*/432, + /*removedSamplesForAcceleration=*/321, + /*fecPacketsReceived=*/123, + /*fecPacketsDiscarded=*/101, + /*packetsDiscarded=*/989, + /*currentExpandRate=*/789, + /*currentSpeechExpandRate=*/12, + /*currentPreemptiveRate=*/345, + /*currentAccelerateRate =*/678, + /*currentSecondaryDecodedRate=*/901, + /*currentSecondaryDiscardedRate=*/0, + /*meanWaitingTimeMs=*/-1, + /*maxWaitingTimeMs=*/-1, + /*packetBufferFlushes=*/0, + /*delayedPacketOutageSamples=*/0, + /*relativePacketArrivalDelayMs=*/135, + /*interruptionCount=*/-1, + /*totalInterruptionDurationMs=*/-1}; const AudioDecodingCallStats kAudioDecodeStats = MakeAudioDecodeStatsForTest(); struct ConfigHelper { explicit ConfigHelper(bool use_null_audio_processing) - : ConfigHelper(new rtc::RefCountedObject<MockAudioMixer>(), + : ConfigHelper(rtc::make_ref_counted<MockAudioMixer>(), use_null_audio_processing) {} ConfigHelper(rtc::scoped_refptr<MockAudioMixer> audio_mixer, @@ -89,9 +114,9 @@ struct ConfigHelper { config.audio_processing = use_null_audio_processing ? nullptr - : new rtc::RefCountedObject<MockAudioProcessing>(); + : rtc::make_ref_counted<NiceMock<MockAudioProcessing>>(); config.audio_device_module = - new rtc::RefCountedObject<testing::NiceMock<MockAudioDeviceModule>>(); + rtc::make_ref_counted<testing::NiceMock<MockAudioDeviceModule>>(); audio_state_ = AudioState::Create(config); channel_receive_ = new ::testing::StrictMock<MockChannelReceive>(); @@ -106,8 +131,9 @@ struct ConfigHelper { .WillRepeatedly(Invoke([](const std::map<int, SdpAudioFormat>& codecs) { EXPECT_THAT(codecs, ::testing::IsEmpty()); })); - EXPECT_CALL(*channel_receive_, SetDepacketizerToDecoderFrameTransformer(_)) - .Times(1); + EXPECT_CALL(*channel_receive_, SetSourceTracker(_)); + EXPECT_CALL(*channel_receive_, GetLocalSsrc()) + .WillRepeatedly(Return(kLocalSsrc)); stream_config_.rtp.local_ssrc = kLocalSsrc; stream_config_.rtp.remote_ssrc = kRemoteSsrc; @@ -118,18 +144,19 @@ struct ConfigHelper { RtpExtension::kTransportSequenceNumberUri, kTransportSequenceNumberId)); stream_config_.rtcp_send_transport = &rtcp_send_transport_; stream_config_.decoder_factory = - new rtc::RefCountedObject<MockAudioDecoderFactory>; + rtc::make_ref_counted<MockAudioDecoderFactory>(); } - std::unique_ptr<internal::AudioReceiveStream> CreateAudioReceiveStream() { - return std::unique_ptr<internal::AudioReceiveStream>( - new internal::AudioReceiveStream( - Clock::GetRealTimeClock(), &rtp_stream_receiver_controller_, - &packet_router_, stream_config_, audio_state_, &event_log_, - std::unique_ptr<voe::ChannelReceiveInterface>(channel_receive_))); + std::unique_ptr<AudioReceiveStreamImpl> CreateAudioReceiveStream() { + auto ret = std::make_unique<AudioReceiveStreamImpl>( + Clock::GetRealTimeClock(), &packet_router_, stream_config_, + audio_state_, &event_log_, + std::unique_ptr<voe::ChannelReceiveInterface>(channel_receive_)); + ret->RegisterWithTransport(&rtp_stream_receiver_controller_); + return ret; } - AudioReceiveStream::Config& config() { return stream_config_; } + AudioReceiveStreamInterface::Config& config() { return stream_config_; } rtc::scoped_refptr<MockAudioMixer> audio_mixer() { return audio_mixer_; } MockChannelReceive* channel_receive() { return channel_receive_; } @@ -148,7 +175,7 @@ struct ConfigHelper { .WillOnce(Return(kTotalOutputEnergy)); EXPECT_CALL(*channel_receive_, GetTotalOutputDuration()) .WillOnce(Return(kTotalOutputDuration)); - EXPECT_CALL(*channel_receive_, GetNetworkStatistics()) + EXPECT_CALL(*channel_receive_, GetNetworkStatistics(_)) .WillOnce(Return(kNetworkStats)); EXPECT_CALL(*channel_receive_, GetDecodingCallStatistics()) .WillOnce(Return(kAudioDecodeStats)); @@ -163,51 +190,12 @@ struct ConfigHelper { MockRtcEventLog event_log_; rtc::scoped_refptr<AudioState> audio_state_; rtc::scoped_refptr<MockAudioMixer> audio_mixer_; - AudioReceiveStream::Config stream_config_; + AudioReceiveStreamInterface::Config stream_config_; ::testing::StrictMock<MockChannelReceive>* channel_receive_ = nullptr; RtpStreamReceiverController rtp_stream_receiver_controller_; MockTransport rtcp_send_transport_; }; -void BuildOneByteExtension(std::vector<uint8_t>::iterator it, - int id, - uint32_t extension_value, - size_t value_length) { - const uint16_t kRtpOneByteHeaderExtensionId = 0xBEDE; - ByteWriter<uint16_t>::WriteBigEndian(&(*it), kRtpOneByteHeaderExtensionId); - it += 2; - - ByteWriter<uint16_t>::WriteBigEndian(&(*it), kOneByteExtensionLength / 4); - it += 2; - const size_t kExtensionDataLength = kOneByteExtensionLength - 1; - uint32_t shifted_value = extension_value - << (8 * (kExtensionDataLength - value_length)); - *it = (id << 4) + (static_cast<uint8_t>(value_length) - 1); - ++it; - ByteWriter<uint32_t, kExtensionDataLength>::WriteBigEndian(&(*it), - shifted_value); -} - -const std::vector<uint8_t> CreateRtpHeaderWithOneByteExtension( - int extension_id, - uint32_t extension_value, - size_t value_length) { - std::vector<uint8_t> header; - header.resize(webrtc::kRtpHeaderSize + kOneByteExtensionHeaderLength + - kOneByteExtensionLength); - header[0] = 0x80; // Version 2. - header[0] |= 0x10; // Set extension bit. - header[1] = 100; // Payload type. - header[1] |= 0x80; // Marker bit is set. - ByteWriter<uint16_t>::WriteBigEndian(&header[2], 0x1234); // Sequence number. - ByteWriter<uint32_t>::WriteBigEndian(&header[4], 0x5678); // Timestamp. - ByteWriter<uint32_t>::WriteBigEndian(&header[8], 0x4321); // SSRC. - - BuildOneByteExtension(header.begin() + webrtc::kRtpHeaderSize, extension_id, - extension_value, value_length); - return header; -} - const std::vector<uint8_t> CreateRtcpSenderReport() { std::vector<uint8_t> packet; const size_t kRtcpSrLength = 28; // In bytes. @@ -222,7 +210,7 @@ const std::vector<uint8_t> CreateRtcpSenderReport() { } // namespace TEST(AudioReceiveStreamTest, ConfigToString) { - AudioReceiveStream::Config config; + AudioReceiveStreamInterface::Config config; config.rtp.remote_ssrc = kRemoteSsrc; config.rtp.local_ssrc = kLocalSsrc; config.rtp.extensions.push_back( @@ -239,27 +227,7 @@ TEST(AudioReceiveStreamTest, ConstructDestruct) { for (bool use_null_audio_processing : {false, true}) { ConfigHelper helper(use_null_audio_processing); auto recv_stream = helper.CreateAudioReceiveStream(); - } -} - -TEST(AudioReceiveStreamTest, ReceiveRtpPacket) { - for (bool use_null_audio_processing : {false, true}) { - ConfigHelper helper(use_null_audio_processing); - helper.config().rtp.transport_cc = true; - auto recv_stream = helper.CreateAudioReceiveStream(); - const int kTransportSequenceNumberValue = 1234; - std::vector<uint8_t> rtp_packet = CreateRtpHeaderWithOneByteExtension( - kTransportSequenceNumberId, kTransportSequenceNumberValue, 2); - constexpr int64_t packet_time_us = 5678000; - - RtpPacketReceived parsed_packet; - ASSERT_TRUE(parsed_packet.Parse(&rtp_packet[0], rtp_packet.size())); - parsed_packet.set_arrival_time_ms((packet_time_us + 500) / 1000); - - EXPECT_CALL(*helper.channel_receive(), - OnRtpPacket(::testing::Ref(parsed_packet))); - - recv_stream->OnRtpPacket(parsed_packet); + recv_stream->UnregisterFromTransport(); } } @@ -273,6 +241,7 @@ TEST(AudioReceiveStreamTest, ReceiveRtcpPacket) { ReceivedRTCPPacket(&rtcp_packet[0], rtcp_packet.size())) .WillOnce(Return()); recv_stream->DeliverRtcp(&rtcp_packet[0], rtcp_packet.size()); + recv_stream->UnregisterFromTransport(); } } @@ -281,7 +250,8 @@ TEST(AudioReceiveStreamTest, GetStats) { ConfigHelper helper(use_null_audio_processing); auto recv_stream = helper.CreateAudioReceiveStream(); helper.SetupMockForGetStats(); - AudioReceiveStream::Stats stats = recv_stream->GetStats(); + AudioReceiveStreamInterface::Stats stats = + recv_stream->GetStats(/*get_and_clear_legacy_stats=*/true); EXPECT_EQ(kRemoteSsrc, stats.remote_ssrc); EXPECT_EQ(kCallStats.payload_bytes_rcvd, stats.payload_bytes_rcvd); EXPECT_EQ(kCallStats.header_and_padding_bytes_rcvd, @@ -312,6 +282,16 @@ TEST(AudioReceiveStreamTest, GetStats) { EXPECT_EQ(static_cast<double>(kNetworkStats.jitterBufferTargetDelayMs) / static_cast<double>(rtc::kNumMillisecsPerSec), stats.jitter_buffer_target_delay_seconds); + EXPECT_EQ(static_cast<double>(kNetworkStats.jitterBufferMinimumDelayMs) / + static_cast<double>(rtc::kNumMillisecsPerSec), + stats.jitter_buffer_minimum_delay_seconds); + EXPECT_EQ(kNetworkStats.insertedSamplesForDeceleration, + stats.inserted_samples_for_deceleration); + EXPECT_EQ(kNetworkStats.removedSamplesForAcceleration, + stats.removed_samples_for_acceleration); + EXPECT_EQ(kNetworkStats.fecPacketsReceived, stats.fec_packets_received); + EXPECT_EQ(kNetworkStats.fecPacketsDiscarded, stats.fec_packets_discarded); + EXPECT_EQ(kNetworkStats.packetsDiscarded, stats.packets_discarded); EXPECT_EQ(Q14ToFloat(kNetworkStats.currentExpandRate), stats.expand_rate); EXPECT_EQ(Q14ToFloat(kNetworkStats.currentSpeechExpandRate), stats.speech_expand_rate); @@ -323,6 +303,16 @@ TEST(AudioReceiveStreamTest, GetStats) { stats.accelerate_rate); EXPECT_EQ(Q14ToFloat(kNetworkStats.currentPreemptiveRate), stats.preemptive_expand_rate); + EXPECT_EQ(kNetworkStats.packetBufferFlushes, stats.jitter_buffer_flushes); + EXPECT_EQ(kNetworkStats.delayedPacketOutageSamples, + stats.delayed_packet_outage_samples); + EXPECT_EQ(static_cast<double>(kNetworkStats.relativePacketArrivalDelayMs) / + static_cast<double>(rtc::kNumMillisecsPerSec), + stats.relative_packet_arrival_delay_seconds); + EXPECT_EQ(kNetworkStats.interruptionCount, stats.interruption_count); + EXPECT_EQ(kNetworkStats.totalInterruptionDurationMs, + stats.total_interruption_duration_ms); + EXPECT_EQ(kAudioDecodeStats.calls_to_silence_generator, stats.decoding_calls_to_silence_generator); EXPECT_EQ(kAudioDecodeStats.calls_to_neteq, stats.decoding_calls_to_neteq); @@ -336,6 +326,7 @@ TEST(AudioReceiveStreamTest, GetStats) { EXPECT_EQ(kCallStats.capture_start_ntp_time_ms_, stats.capture_start_ntp_time_ms); EXPECT_EQ(kPlayoutNtpTimestampMs, stats.estimated_playout_ntp_timestamp_ms); + recv_stream->UnregisterFromTransport(); } } @@ -346,6 +337,7 @@ TEST(AudioReceiveStreamTest, SetGain) { EXPECT_CALL(*helper.channel_receive(), SetChannelOutputVolumeScaling(FloatEq(0.765f))); recv_stream->SetGain(0.765f); + recv_stream->UnregisterFromTransport(); } } @@ -377,14 +369,9 @@ TEST(AudioReceiveStreamTest, StreamsShouldBeAddedToMixerOnceOnStart) { // Stop stream before it is being destructed. recv_stream2->Stop(); - } -} -TEST(AudioReceiveStreamTest, ReconfigureWithSameConfig) { - for (bool use_null_audio_processing : {false, true}) { - ConfigHelper helper(use_null_audio_processing); - auto recv_stream = helper.CreateAudioReceiveStream(); - recv_stream->Reconfigure(helper.config()); + recv_stream1->UnregisterFromTransport(); + recv_stream2->UnregisterFromTransport(); } } @@ -394,20 +381,32 @@ TEST(AudioReceiveStreamTest, ReconfigureWithUpdatedConfig) { auto recv_stream = helper.CreateAudioReceiveStream(); auto new_config = helper.config(); - new_config.rtp.nack.rtp_history_ms = 300 + 20; + new_config.rtp.extensions.clear(); new_config.rtp.extensions.push_back( RtpExtension(RtpExtension::kAudioLevelUri, kAudioLevelId + 1)); new_config.rtp.extensions.push_back( RtpExtension(RtpExtension::kTransportSequenceNumberUri, kTransportSequenceNumberId + 1)); - new_config.decoder_map.emplace(1, SdpAudioFormat("foo", 8000, 1)); MockChannelReceive& channel_receive = *helper.channel_receive(); - EXPECT_CALL(channel_receive, SetNACKStatus(true, 15 + 1)).Times(1); + + // TODO(tommi, nisse): This applies new extensions to the internal config, + // but there's nothing that actually verifies that the changes take effect. + // In fact Call manages the extensions separately in Call::ReceiveRtpConfig + // and changing this config value (there seem to be a few copies), doesn't + // affect that logic. + recv_stream->ReconfigureForTesting(new_config); + + new_config.decoder_map.emplace(1, SdpAudioFormat("foo", 8000, 1)); EXPECT_CALL(channel_receive, SetReceiveCodecs(new_config.decoder_map)); + recv_stream->SetDecoderMap(new_config.decoder_map); + + EXPECT_CALL(channel_receive, SetNACKStatus(true, 15 + 1)).Times(1); + recv_stream->SetTransportCc(new_config.rtp.transport_cc); + recv_stream->SetNackHistory(300 + 20); - recv_stream->Reconfigure(new_config); + recv_stream->UnregisterFromTransport(); } } @@ -418,17 +417,23 @@ TEST(AudioReceiveStreamTest, ReconfigureWithFrameDecryptor) { auto new_config_0 = helper.config(); rtc::scoped_refptr<FrameDecryptorInterface> mock_frame_decryptor_0( - new rtc::RefCountedObject<MockFrameDecryptor>()); + rtc::make_ref_counted<MockFrameDecryptor>()); new_config_0.frame_decryptor = mock_frame_decryptor_0; - recv_stream->Reconfigure(new_config_0); + // TODO(tommi): While this changes the internal config value, it doesn't + // actually change what frame_decryptor is used. WebRtcAudioReceiveStream + // recreates the whole instance in order to change this value. + // So, it's not clear if changing this post initialization needs to be + // supported. + recv_stream->ReconfigureForTesting(new_config_0); auto new_config_1 = helper.config(); rtc::scoped_refptr<FrameDecryptorInterface> mock_frame_decryptor_1( - new rtc::RefCountedObject<MockFrameDecryptor>()); + rtc::make_ref_counted<MockFrameDecryptor>()); new_config_1.frame_decryptor = mock_frame_decryptor_1; new_config_1.crypto_options.sframe.require_frame_encryption = true; - recv_stream->Reconfigure(new_config_1); + recv_stream->ReconfigureForTesting(new_config_1); + recv_stream->UnregisterFromTransport(); } } diff --git a/audio/audio_send_stream.cc b/audio/audio_send_stream.cc index 1856902d5e..7d6ec794d4 100644 --- a/audio/audio_send_stream.cc +++ b/audio/audio_send_stream.cc @@ -22,6 +22,7 @@ #include "api/crypto/frame_encryptor_interface.h" #include "api/function_view.h" #include "api/rtc_event_log/rtc_event_log.h" +#include "api/task_queue/task_queue_base.h" #include "audio/audio_state.h" #include "audio/channel_send.h" #include "audio/conversion.h" @@ -30,16 +31,15 @@ #include "common_audio/vad/include/vad.h" #include "logging/rtc_event_log/events/rtc_event_audio_send_stream_config.h" #include "logging/rtc_event_log/rtc_stream_config.h" +#include "media/base/media_channel.h" #include "modules/audio_coding/codecs/cng/audio_encoder_cng.h" #include "modules/audio_coding/codecs/red/audio_encoder_copy_red.h" #include "modules/audio_processing/include/audio_processing.h" #include "modules/rtp_rtcp/source/rtp_header_extensions.h" #include "rtc_base/checks.h" -#include "rtc_base/event.h" #include "rtc_base/logging.h" #include "rtc_base/strings/audio_format_to_string.h" -#include "rtc_base/task_queue.h" -#include "system_wrappers/include/field_trial.h" +#include "rtc_base/trace_event.h" namespace webrtc { namespace { @@ -75,6 +75,7 @@ void UpdateEventLogStreamConfig(RtcEventLog* event_log, event_log->Log(std::make_unique<RtcEventAudioSendStreamConfig>( std::move(rtclog_config))); } + } // namespace constexpr char AudioAllocationConfig::kKey[]; @@ -88,8 +89,9 @@ std::unique_ptr<StructParametersParser> AudioAllocationConfig::Parser() { "rate_prio", &bitrate_priority); } -AudioAllocationConfig::AudioAllocationConfig() { - Parser()->Parse(field_trial::FindFullName(kKey)); +AudioAllocationConfig::AudioAllocationConfig( + const FieldTrialsView& field_trials) { + Parser()->Parse(field_trials.Lookup(kKey)); if (priority_bitrate_raw && !priority_bitrate.IsZero()) { RTC_LOG(LS_WARNING) << "'priority_bitrate' and '_raw' are mutually " "exclusive but both were configured."; @@ -102,34 +104,35 @@ AudioSendStream::AudioSendStream( const webrtc::AudioSendStream::Config& config, const rtc::scoped_refptr<webrtc::AudioState>& audio_state, TaskQueueFactory* task_queue_factory, - ProcessThread* module_process_thread, RtpTransportControllerSendInterface* rtp_transport, BitrateAllocatorInterface* bitrate_allocator, RtcEventLog* event_log, RtcpRttStats* rtcp_rtt_stats, - const absl::optional<RtpState>& suspended_rtp_state) - : AudioSendStream(clock, - config, - audio_state, - task_queue_factory, - rtp_transport, - bitrate_allocator, - event_log, - suspended_rtp_state, - voe::CreateChannelSend( - clock, - task_queue_factory, - module_process_thread, - config.send_transport, - rtcp_rtt_stats, - event_log, - config.frame_encryptor, - config.crypto_options, - config.rtp.extmap_allow_mixed, - config.rtcp_report_interval_ms, - config.rtp.ssrc, - config.frame_transformer, - rtp_transport->transport_feedback_observer())) {} + const absl::optional<RtpState>& suspended_rtp_state, + const FieldTrialsView& field_trials) + : AudioSendStream( + clock, + config, + audio_state, + task_queue_factory, + rtp_transport, + bitrate_allocator, + event_log, + suspended_rtp_state, + voe::CreateChannelSend(clock, + task_queue_factory, + config.send_transport, + rtcp_rtt_stats, + event_log, + config.frame_encryptor.get(), + config.crypto_options, + config.rtp.extmap_allow_mixed, + config.rtcp_report_interval_ms, + config.rtp.ssrc, + config.frame_transformer, + rtp_transport->transport_feedback_observer(), + field_trials), + field_trials) {} AudioSendStream::AudioSendStream( Clock* clock, @@ -140,28 +143,28 @@ AudioSendStream::AudioSendStream( BitrateAllocatorInterface* bitrate_allocator, RtcEventLog* event_log, const absl::optional<RtpState>& suspended_rtp_state, - std::unique_ptr<voe::ChannelSendInterface> channel_send) + std::unique_ptr<voe::ChannelSendInterface> channel_send, + const FieldTrialsView& field_trials) : clock_(clock), - worker_queue_(rtp_transport->GetWorkerQueue()), - audio_send_side_bwe_(field_trial::IsEnabled("WebRTC-Audio-SendSideBwe")), + field_trials_(field_trials), + rtp_transport_queue_(rtp_transport->GetWorkerQueue()), allocate_audio_without_feedback_( - field_trial::IsEnabled("WebRTC-Audio-ABWENoTWCC")), + field_trials_.IsEnabled("WebRTC-Audio-ABWENoTWCC")), enable_audio_alr_probing_( - !field_trial::IsDisabled("WebRTC-Audio-AlrProbing")), - send_side_bwe_with_overhead_( - field_trial::IsEnabled("WebRTC-SendSideBwe-WithOverhead")), + !field_trials_.IsDisabled("WebRTC-Audio-AlrProbing")), + allocation_settings_(field_trials_), config_(Config(/*send_transport=*/nullptr)), audio_state_(audio_state), channel_send_(std::move(channel_send)), event_log_(event_log), use_legacy_overhead_calculation_( - field_trial::IsEnabled("WebRTC-Audio-LegacyOverhead")), + field_trials_.IsEnabled("WebRTC-Audio-LegacyOverhead")), bitrate_allocator_(bitrate_allocator), rtp_transport_(rtp_transport), rtp_rtcp_module_(channel_send_->GetRtpRtcp()), suspended_rtp_state_(suspended_rtp_state) { RTC_LOG(LS_INFO) << "AudioSendStream: " << config.rtp.ssrc; - RTC_DCHECK(worker_queue_); + RTC_DCHECK(rtp_transport_queue_); RTC_DCHECK(audio_state_); RTC_DCHECK(channel_send_); RTC_DCHECK(bitrate_allocator_); @@ -169,32 +172,32 @@ AudioSendStream::AudioSendStream( RTC_DCHECK(rtp_rtcp_module_); - ConfigureStream(config, true); - - pacer_thread_checker_.Detach(); + RTC_DCHECK_RUN_ON(&worker_thread_checker_); + ConfigureStream(config, true, nullptr); + UpdateCachedTargetAudioBitrateConstraints(); } AudioSendStream::~AudioSendStream() { - RTC_DCHECK(worker_thread_checker_.IsCurrent()); + RTC_DCHECK_RUN_ON(&worker_thread_checker_); RTC_LOG(LS_INFO) << "~AudioSendStream: " << config_.rtp.ssrc; RTC_DCHECK(!sending_); channel_send_->ResetSenderCongestionControlObjects(); + // Blocking call to synchronize state with worker queue to ensure that there // are no pending tasks left that keeps references to audio. - rtc::Event thread_sync_event; - worker_queue_->PostTask([&] { thread_sync_event.Set(); }); - thread_sync_event.Wait(rtc::Event::kForever); + rtp_transport_queue_->RunSynchronous([] {}); } const webrtc::AudioSendStream::Config& AudioSendStream::GetConfig() const { - RTC_DCHECK(worker_thread_checker_.IsCurrent()); + RTC_DCHECK_RUN_ON(&worker_thread_checker_); return config_; } void AudioSendStream::Reconfigure( - const webrtc::AudioSendStream::Config& new_config) { - RTC_DCHECK(worker_thread_checker_.IsCurrent()); - ConfigureStream(new_config, false); + const webrtc::AudioSendStream::Config& new_config, + SetParametersCallback callback) { + RTC_DCHECK_RUN_ON(&worker_thread_checker_); + ConfigureStream(new_config, false, std::move(callback)); } AudioSendStream::ExtensionIds AudioSendStream::FindExtensionIds( @@ -226,7 +229,8 @@ int AudioSendStream::TransportSeqNumId(const AudioSendStream::Config& config) { void AudioSendStream::ConfigureStream( const webrtc::AudioSendStream::Config& new_config, - bool first_time) { + bool first_time, + SetParametersCallback callback) { RTC_LOG(LS_INFO) << "AudioSendStream::ConfigureStream: " << new_config.ToString(); UpdateEventLogStreamConfig(event_log_, new_config, @@ -271,11 +275,10 @@ void AudioSendStream::ConfigureStream( } if (first_time || new_ids.abs_send_time != old_ids.abs_send_time) { - rtp_rtcp_module_->DeregisterSendRtpHeaderExtension( - kRtpExtensionAbsoluteSendTime); + absl::string_view uri = AbsoluteSendTime::Uri(); + rtp_rtcp_module_->DeregisterSendRtpHeaderExtension(uri); if (new_ids.abs_send_time) { - rtp_rtcp_module_->RegisterRtpHeaderExtension(AbsoluteSendTime::kUri, - new_ids.abs_send_time); + rtp_rtcp_module_->RegisterRtpHeaderExtension(uri, new_ids.abs_send_time); } } @@ -289,10 +292,10 @@ void AudioSendStream::ConfigureStream( RtcpBandwidthObserver* bandwidth_observer = nullptr; - if (audio_send_side_bwe_ && !allocate_audio_without_feedback_ && + if (!allocate_audio_without_feedback_ && new_ids.transport_sequence_number != 0) { rtp_rtcp_module_->RegisterRtpHeaderExtension( - TransportSequenceNumber::kUri, new_ids.transport_sequence_number); + TransportSequenceNumber::Uri(), new_ids.transport_sequence_number); // Probing in application limited region is only used in combination with // send side congestion control, wich depends on feedback packets which // requires transport sequence numbers to be enabled. @@ -310,39 +313,25 @@ void AudioSendStream::ConfigureStream( if ((first_time || new_ids.mid != old_ids.mid || new_config.rtp.mid != old_config.rtp.mid) && new_ids.mid != 0 && !new_config.rtp.mid.empty()) { - rtp_rtcp_module_->RegisterRtpHeaderExtension(RtpMid::kUri, new_ids.mid); + rtp_rtcp_module_->RegisterRtpHeaderExtension(RtpMid::Uri(), new_ids.mid); rtp_rtcp_module_->SetMid(new_config.rtp.mid); } - // RID RTP header extension - if ((first_time || new_ids.rid != old_ids.rid || - new_ids.repaired_rid != old_ids.repaired_rid || - new_config.rtp.rid != old_config.rtp.rid)) { - if (new_ids.rid != 0 || new_ids.repaired_rid != 0) { - if (new_config.rtp.rid.empty()) { - rtp_rtcp_module_->DeregisterSendRtpHeaderExtension(RtpStreamId::kUri); - } else if (new_ids.repaired_rid != 0) { - rtp_rtcp_module_->RegisterRtpHeaderExtension(RtpStreamId::kUri, - new_ids.repaired_rid); - } else { - rtp_rtcp_module_->RegisterRtpHeaderExtension(RtpStreamId::kUri, - new_ids.rid); - } - } - rtp_rtcp_module_->SetRid(new_config.rtp.rid); - } - if (first_time || new_ids.abs_capture_time != old_ids.abs_capture_time) { - rtp_rtcp_module_->DeregisterSendRtpHeaderExtension( - kRtpExtensionAbsoluteCaptureTime); + absl::string_view uri = AbsoluteCaptureTimeExtension::Uri(); + rtp_rtcp_module_->DeregisterSendRtpHeaderExtension(uri); if (new_ids.abs_capture_time) { - rtp_rtcp_module_->RegisterRtpHeaderExtension( - AbsoluteCaptureTimeExtension::kUri, new_ids.abs_capture_time); + rtp_rtcp_module_->RegisterRtpHeaderExtension(uri, + new_ids.abs_capture_time); } } if (!ReconfigureSendCodec(new_config)) { RTC_LOG(LS_ERROR) << "Failed to set up send codec state."; + + webrtc::InvokeSetParametersCallback( + callback, webrtc::RTCError(webrtc::RTCErrorType::INTERNAL_ERROR, + "Failed to set up send codec state.")); } // Set currently known overhead (used in ANA, opus only). @@ -352,20 +341,24 @@ void AudioSendStream::ConfigureStream( } channel_send_->CallEncoder([this](AudioEncoder* encoder) { + RTC_DCHECK_RUN_ON(&worker_thread_checker_); if (!encoder) { return; } - worker_queue_->PostTask( - [this, length_range = encoder->GetFrameLengthRange()] { - RTC_DCHECK_RUN_ON(worker_queue_); - frame_length_range_ = length_range; - }); + frame_length_range_ = encoder->GetFrameLengthRange(); + UpdateCachedTargetAudioBitrateConstraints(); }); if (sending_) { ReconfigureBitrateObserver(new_config); } + config_ = new_config; + if (!first_time) { + UpdateCachedTargetAudioBitrateConstraints(); + } + + webrtc::InvokeSetParametersCallback(callback, webrtc::RTCError::OK()); } void AudioSendStream::Start() { @@ -377,16 +370,9 @@ void AudioSendStream::Start() { config_.max_bitrate_bps != -1 && (allocate_audio_without_feedback_ || TransportSeqNumId(config_) != 0)) { rtp_transport_->AccountForAudioPacketsInPacedSender(true); - if (send_side_bwe_with_overhead_) - rtp_transport_->IncludeOverheadInPacedSender(); + rtp_transport_->IncludeOverheadInPacedSender(); rtp_rtcp_module_->SetAsPartOfAllocation(true); - rtc::Event thread_sync_event; - worker_queue_->PostTask([&] { - RTC_DCHECK_RUN_ON(worker_queue_); - ConfigureBitrateObserver(); - thread_sync_event.Set(); - }); - thread_sync_event.Wait(rtc::Event::kForever); + ConfigureBitrateObserver(); } else { rtp_rtcp_module_->SetAsPartOfAllocation(false); } @@ -397,7 +383,7 @@ void AudioSendStream::Start() { } void AudioSendStream::Stop() { - RTC_DCHECK(worker_thread_checker_.IsCurrent()); + RTC_DCHECK_RUN_ON(&worker_thread_checker_); if (!sending_) { return; } @@ -411,6 +397,7 @@ void AudioSendStream::Stop() { void AudioSendStream::SendAudioData(std::unique_ptr<AudioFrame> audio_frame) { RTC_CHECK_RUNS_SERIALIZED(&audio_capture_race_checker_); RTC_DCHECK_GT(audio_frame->sample_rate_hz_, 0); + TRACE_EVENT0("webrtc", "AudioSendStream::SendAudioData"); double duration = static_cast<double>(audio_frame->samples_per_channel_) / audio_frame->sample_rate_hz_; { @@ -432,14 +419,14 @@ bool AudioSendStream::SendTelephoneEvent(int payload_type, int payload_frequency, int event, int duration_ms) { - RTC_DCHECK(worker_thread_checker_.IsCurrent()); + RTC_DCHECK_RUN_ON(&worker_thread_checker_); channel_send_->SetSendTelephoneEventPayloadType(payload_type, payload_frequency); return channel_send_->SendTelephoneEventOutband(event, duration_ms); } void AudioSendStream::SetMuted(bool muted) { - RTC_DCHECK(worker_thread_checker_.IsCurrent()); + RTC_DCHECK_RUN_ON(&worker_thread_checker_); channel_send_->SetInputMute(muted); } @@ -449,10 +436,10 @@ webrtc::AudioSendStream::Stats AudioSendStream::GetStats() const { webrtc::AudioSendStream::Stats AudioSendStream::GetStats( bool has_remote_tracks) const { - RTC_DCHECK(worker_thread_checker_.IsCurrent()); + RTC_DCHECK_RUN_ON(&worker_thread_checker_); webrtc::AudioSendStream::Stats stats; stats.local_ssrc = config_.rtp.ssrc; - stats.target_bitrate_bps = channel_send_->GetBitrate(); + stats.target_bitrate_bps = channel_send_->GetTargetBitrate(); webrtc::CallSendStatistics call_stats = channel_send_->GetRTCPStatistics(); stats.payload_bytes_sent = call_stats.payload_bytes_sent; @@ -460,6 +447,7 @@ webrtc::AudioSendStream::Stats AudioSendStream::GetStats( call_stats.header_and_padding_bytes_sent; stats.retransmitted_bytes_sent = call_stats.retransmitted_bytes_sent; stats.packets_sent = call_stats.packetsSent; + stats.total_packet_send_delay = call_stats.total_packet_send_delay; stats.retransmitted_packets_sent = call_stats.retransmitted_packets_sent; // RTT isn't known until a RTCP report is received. Until then, VoiceEngine // returns 0 to indicate an error value. @@ -494,7 +482,6 @@ webrtc::AudioSendStream::Stats AudioSendStream::GetStats( stats.total_input_duration = audio_level_.TotalDuration(); } - stats.typing_noise_detected = audio_state()->typing_noise_detected(); stats.ana_statistics = channel_send_->GetANAStatistics(); AudioProcessing* ap = audio_state_->audio_processing(); @@ -504,29 +491,35 @@ webrtc::AudioSendStream::Stats AudioSendStream::GetStats( stats.report_block_datas = std::move(call_stats.report_block_datas); + stats.nacks_rcvd = call_stats.nacks_rcvd; + return stats; } void AudioSendStream::DeliverRtcp(const uint8_t* packet, size_t length) { RTC_DCHECK_RUN_ON(&worker_thread_checker_); channel_send_->ReceivedRTCPPacket(packet, length); - worker_queue_->PostTask([&]() { + + { // Poll if overhead has changed, which it can do if ack triggers us to stop // sending mid/rid. MutexLock lock(&overhead_per_packet_lock_); UpdateOverheadForEncoder(); - }); + } + UpdateCachedTargetAudioBitrateConstraints(); } uint32_t AudioSendStream::OnBitrateUpdated(BitrateAllocationUpdate update) { - RTC_DCHECK_RUN_ON(worker_queue_); + RTC_DCHECK_RUN_ON(rtp_transport_queue_); // Pick a target bitrate between the constraints. Overrules the allocator if // it 1) allocated a bitrate of zero to disable the stream or 2) allocated a // higher than max to allow for e.g. extra FEC. - auto constraints = GetMinMaxBitrateConstraints(); - update.target_bitrate.Clamp(constraints.min, constraints.max); - update.stable_target_bitrate.Clamp(constraints.min, constraints.max); + RTC_DCHECK(cached_constraints_.has_value()); + update.target_bitrate.Clamp(cached_constraints_->min, + cached_constraints_->max); + update.stable_target_bitrate.Clamp(cached_constraints_->min, + cached_constraints_->max); channel_send_->OnBitrateAllocation(update); @@ -537,13 +530,17 @@ uint32_t AudioSendStream::OnBitrateUpdated(BitrateAllocationUpdate update) { void AudioSendStream::SetTransportOverhead( int transport_overhead_per_packet_bytes) { - RTC_DCHECK(worker_thread_checker_.IsCurrent()); - MutexLock lock(&overhead_per_packet_lock_); - transport_overhead_per_packet_bytes_ = transport_overhead_per_packet_bytes; - UpdateOverheadForEncoder(); + RTC_DCHECK_RUN_ON(&worker_thread_checker_); + { + MutexLock lock(&overhead_per_packet_lock_); + transport_overhead_per_packet_bytes_ = transport_overhead_per_packet_bytes; + UpdateOverheadForEncoder(); + } + UpdateCachedTargetAudioBitrateConstraints(); } void AudioSendStream::UpdateOverheadForEncoder() { + RTC_DCHECK_RUN_ON(&worker_thread_checker_); size_t overhead_per_packet_bytes = GetPerPacketOverheadBytes(); if (overhead_per_packet_ == overhead_per_packet_bytes) { return; @@ -553,19 +550,11 @@ void AudioSendStream::UpdateOverheadForEncoder() { channel_send_->CallEncoder([&](AudioEncoder* encoder) { encoder->OnReceivedOverhead(overhead_per_packet_bytes); }); - auto update_task = [this, overhead_per_packet_bytes] { - RTC_DCHECK_RUN_ON(worker_queue_); - if (total_packet_overhead_bytes_ != overhead_per_packet_bytes) { - total_packet_overhead_bytes_ = overhead_per_packet_bytes; - if (registered_with_allocator_) { - ConfigureBitrateObserver(); - } + if (total_packet_overhead_bytes_ != overhead_per_packet_bytes) { + total_packet_overhead_bytes_ = overhead_per_packet_bytes; + if (registered_with_allocator_) { + ConfigureBitrateObserver(); } - }; - if (worker_queue_->IsCurrent()) { - update_task(); - } else { - worker_queue_->PostTask(update_task); } } @@ -603,7 +592,6 @@ const internal::AudioState* AudioSendStream::audio_state() const { void AudioSendStream::StoreEncoderProperties(int sample_rate_hz, size_t num_channels) { - RTC_DCHECK(worker_thread_checker_.IsCurrent()); encoder_sample_rate_hz_ = sample_rate_hz; encoder_num_channels_ = num_channels; if (sending_) { @@ -638,11 +626,11 @@ bool AudioSendStream::SetupSendCodec(const Config& new_config) { if (new_config.audio_network_adaptor_config) { if (encoder->EnableAudioNetworkAdaptor( *new_config.audio_network_adaptor_config, event_log_)) { - RTC_DLOG(LS_INFO) << "Audio network adaptor enabled on SSRC " - << new_config.rtp.ssrc; + RTC_LOG(LS_INFO) << "Audio network adaptor enabled on SSRC " + << new_config.rtp.ssrc; } else { - RTC_DLOG(LS_INFO) << "Failed to enable Audio network adaptor on SSRC " - << new_config.rtp.ssrc; + RTC_LOG(LS_INFO) << "Failed to enable Audio network adaptor on SSRC " + << new_config.rtp.ssrc; } } @@ -664,7 +652,8 @@ bool AudioSendStream::SetupSendCodec(const Config& new_config) { AudioEncoderCopyRed::Config red_config; red_config.payload_type = *spec.red_payload_type; red_config.speech_encoder = std::move(encoder); - encoder = std::make_unique<AudioEncoderCopyRed>(std::move(red_config)); + encoder = std::make_unique<AudioEncoderCopyRed>(std::move(red_config), + field_trials_); } // Set currently known overhead (used in ANA, opus only). @@ -706,7 +695,9 @@ bool AudioSendStream::ReconfigureSendCodec(const Config& new_config) { new_config.send_codec_spec->format != old_config.send_codec_spec->format || new_config.send_codec_spec->payload_type != - old_config.send_codec_spec->payload_type) { + old_config.send_codec_spec->payload_type || + new_config.send_codec_spec->red_payload_type != + old_config.send_codec_spec->red_payload_type) { return SetupSendCodec(new_config); } @@ -734,21 +725,29 @@ void AudioSendStream::ReconfigureANA(const Config& new_config) { return; } if (new_config.audio_network_adaptor_config) { + // This lock needs to be acquired before CallEncoder, since it aquires + // another lock and we need to maintain the same order at all call sites to + // avoid deadlock. + MutexLock lock(&overhead_per_packet_lock_); + size_t overhead = GetPerPacketOverheadBytes(); channel_send_->CallEncoder([&](AudioEncoder* encoder) { if (encoder->EnableAudioNetworkAdaptor( *new_config.audio_network_adaptor_config, event_log_)) { - RTC_DLOG(LS_INFO) << "Audio network adaptor enabled on SSRC " - << new_config.rtp.ssrc; + RTC_LOG(LS_INFO) << "Audio network adaptor enabled on SSRC " + << new_config.rtp.ssrc; + if (overhead > 0) { + encoder->OnReceivedOverhead(overhead); + } } else { - RTC_DLOG(LS_INFO) << "Failed to enable Audio network adaptor on SSRC " - << new_config.rtp.ssrc; + RTC_LOG(LS_INFO) << "Failed to enable Audio network adaptor on SSRC " + << new_config.rtp.ssrc; } }); } else { channel_send_->CallEncoder( [&](AudioEncoder* encoder) { encoder->DisableAudioNetworkAdaptor(); }); - RTC_DLOG(LS_INFO) << "Audio network adaptor disabled on SSRC " - << new_config.rtp.ssrc; + RTC_LOG(LS_INFO) << "Audio network adaptor disabled on SSRC " + << new_config.rtp.ssrc; } } @@ -793,7 +792,6 @@ void AudioSendStream::ReconfigureCNG(const Config& new_config) { void AudioSendStream::ReconfigureBitrateObserver( const webrtc::AudioSendStream::Config& new_config) { - RTC_DCHECK_RUN_ON(&worker_thread_checker_); // Since the Config's default is for both of these to be -1, this test will // allow us to configure the bitrate observer if the new config has bitrate // limits set, but would only have us call RemoveBitrateObserver if we were @@ -801,8 +799,7 @@ void AudioSendStream::ReconfigureBitrateObserver( if (config_.min_bitrate_bps == new_config.min_bitrate_bps && config_.max_bitrate_bps == new_config.max_bitrate_bps && config_.bitrate_priority == new_config.bitrate_priority && - (TransportSeqNumId(config_) == TransportSeqNumId(new_config) || - !audio_send_side_bwe_) && + TransportSeqNumId(config_) == TransportSeqNumId(new_config) && config_.audio_network_adaptor_config == new_config.audio_network_adaptor_config) { return; @@ -811,22 +808,14 @@ void AudioSendStream::ReconfigureBitrateObserver( if (!new_config.has_dscp && new_config.min_bitrate_bps != -1 && new_config.max_bitrate_bps != -1 && TransportSeqNumId(new_config) != 0) { rtp_transport_->AccountForAudioPacketsInPacedSender(true); - if (send_side_bwe_with_overhead_) - rtp_transport_->IncludeOverheadInPacedSender(); - rtc::Event thread_sync_event; - worker_queue_->PostTask([&] { - RTC_DCHECK_RUN_ON(worker_queue_); - // We may get a callback immediately as the observer is registered, so - // make - // sure the bitrate limits in config_ are up-to-date. - config_.min_bitrate_bps = new_config.min_bitrate_bps; - config_.max_bitrate_bps = new_config.max_bitrate_bps; - - config_.bitrate_priority = new_config.bitrate_priority; - ConfigureBitrateObserver(); - thread_sync_event.Set(); - }); - thread_sync_event.Wait(rtc::Event::kForever); + rtp_transport_->IncludeOverheadInPacedSender(); + // We may get a callback immediately as the observer is registered, so + // make sure the bitrate limits in config_ are up-to-date. + config_.min_bitrate_bps = new_config.min_bitrate_bps; + config_.max_bitrate_bps = new_config.max_bitrate_bps; + + config_.bitrate_priority = new_config.bitrate_priority; + ConfigureBitrateObserver(); rtp_rtcp_module_->SetAsPartOfAllocation(true); } else { rtp_transport_->AccountForAudioPacketsInPacedSender(false); @@ -839,51 +828,59 @@ void AudioSendStream::ConfigureBitrateObserver() { // This either updates the current observer or adds a new observer. // TODO(srte): Add overhead compensation here. auto constraints = GetMinMaxBitrateConstraints(); + RTC_DCHECK(constraints.has_value()); DataRate priority_bitrate = allocation_settings_.priority_bitrate; - if (send_side_bwe_with_overhead_) { - if (use_legacy_overhead_calculation_) { - // OverheadPerPacket = Ipv4(20B) + UDP(8B) + SRTP(10B) + RTP(12) - constexpr int kOverheadPerPacket = 20 + 8 + 10 + 12; - const TimeDelta kMinPacketDuration = TimeDelta::Millis(20); - DataRate max_overhead = - DataSize::Bytes(kOverheadPerPacket) / kMinPacketDuration; - priority_bitrate += max_overhead; - } else { - RTC_DCHECK(frame_length_range_); - const DataSize overhead_per_packet = - DataSize::Bytes(total_packet_overhead_bytes_); - DataRate min_overhead = overhead_per_packet / frame_length_range_->second; - priority_bitrate += min_overhead; - } + if (use_legacy_overhead_calculation_) { + // OverheadPerPacket = Ipv4(20B) + UDP(8B) + SRTP(10B) + RTP(12) + constexpr int kOverheadPerPacket = 20 + 8 + 10 + 12; + const TimeDelta kMinPacketDuration = TimeDelta::Millis(20); + DataRate max_overhead = + DataSize::Bytes(kOverheadPerPacket) / kMinPacketDuration; + priority_bitrate += max_overhead; + } else { + RTC_DCHECK(frame_length_range_); + const DataSize overhead_per_packet = + DataSize::Bytes(total_packet_overhead_bytes_); + DataRate min_overhead = overhead_per_packet / frame_length_range_->second; + priority_bitrate += min_overhead; } + if (allocation_settings_.priority_bitrate_raw) priority_bitrate = *allocation_settings_.priority_bitrate_raw; - bitrate_allocator_->AddObserver( - this, - MediaStreamAllocationConfig{ - constraints.min.bps<uint32_t>(), constraints.max.bps<uint32_t>(), 0, - priority_bitrate.bps(), true, - allocation_settings_.bitrate_priority.value_or( - config_.bitrate_priority)}); + rtp_transport_queue_->RunOrPost([this, constraints, priority_bitrate, + config_bitrate_priority = + config_.bitrate_priority] { + RTC_DCHECK_RUN_ON(rtp_transport_queue_); + bitrate_allocator_->AddObserver( + this, + MediaStreamAllocationConfig{ + constraints->min.bps<uint32_t>(), constraints->max.bps<uint32_t>(), + 0, priority_bitrate.bps(), true, + allocation_settings_.bitrate_priority.value_or( + config_bitrate_priority)}); + }); registered_with_allocator_ = true; } void AudioSendStream::RemoveBitrateObserver() { - RTC_DCHECK(worker_thread_checker_.IsCurrent()); - rtc::Event thread_sync_event; - worker_queue_->PostTask([this, &thread_sync_event] { - RTC_DCHECK_RUN_ON(worker_queue_); - registered_with_allocator_ = false; + registered_with_allocator_ = false; + rtp_transport_queue_->RunSynchronous([this] { + RTC_DCHECK_RUN_ON(rtp_transport_queue_); bitrate_allocator_->RemoveObserver(this); - thread_sync_event.Set(); }); - thread_sync_event.Wait(rtc::Event::kForever); } -AudioSendStream::TargetAudioBitrateConstraints +absl::optional<AudioSendStream::TargetAudioBitrateConstraints> AudioSendStream::GetMinMaxBitrateConstraints() const { + if (config_.min_bitrate_bps < 0 || config_.max_bitrate_bps < 0) { + RTC_LOG(LS_WARNING) << "Config is invalid: min_bitrate_bps=" + << config_.min_bitrate_bps + << "; max_bitrate_bps=" << config_.max_bitrate_bps + << "; both expected greater or equal to 0"; + return absl::nullopt; + } TargetAudioBitrateConstraints constraints{ DataRate::BitsPerSec(config_.min_bitrate_bps), DataRate::BitsPerSec(config_.max_bitrate_bps)}; @@ -896,23 +893,28 @@ AudioSendStream::GetMinMaxBitrateConstraints() const { RTC_DCHECK_GE(constraints.min, DataRate::Zero()); RTC_DCHECK_GE(constraints.max, DataRate::Zero()); - RTC_DCHECK_GE(constraints.max, constraints.min); - if (send_side_bwe_with_overhead_) { - if (use_legacy_overhead_calculation_) { - // OverheadPerPacket = Ipv4(20B) + UDP(8B) + SRTP(10B) + RTP(12) - const DataSize kOverheadPerPacket = DataSize::Bytes(20 + 8 + 10 + 12); - const TimeDelta kMaxFrameLength = - TimeDelta::Millis(60); // Based on Opus spec - const DataRate kMinOverhead = kOverheadPerPacket / kMaxFrameLength; - constraints.min += kMinOverhead; - constraints.max += kMinOverhead; - } else { - RTC_DCHECK(frame_length_range_); - const DataSize kOverheadPerPacket = - DataSize::Bytes(total_packet_overhead_bytes_); - constraints.min += kOverheadPerPacket / frame_length_range_->second; - constraints.max += kOverheadPerPacket / frame_length_range_->first; + if (constraints.max < constraints.min) { + RTC_LOG(LS_WARNING) << "TargetAudioBitrateConstraints::max is less than " + << "TargetAudioBitrateConstraints::min"; + return absl::nullopt; + } + if (use_legacy_overhead_calculation_) { + // OverheadPerPacket = Ipv4(20B) + UDP(8B) + SRTP(10B) + RTP(12) + const DataSize kOverheadPerPacket = DataSize::Bytes(20 + 8 + 10 + 12); + const TimeDelta kMaxFrameLength = + TimeDelta::Millis(60); // Based on Opus spec + const DataRate kMinOverhead = kOverheadPerPacket / kMaxFrameLength; + constraints.min += kMinOverhead; + constraints.max += kMinOverhead; + } else { + if (!frame_length_range_.has_value()) { + RTC_LOG(LS_WARNING) << "frame_length_range_ is not set"; + return absl::nullopt; } + const DataSize kOverheadPerPacket = + DataSize::Bytes(total_packet_overhead_bytes_); + constraints.min += kOverheadPerPacket / frame_length_range_->second; + constraints.max += kOverheadPerPacket / frame_length_range_->first; } return constraints; } @@ -921,5 +923,18 @@ void AudioSendStream::RegisterCngPayloadType(int payload_type, int clockrate_hz) { channel_send_->RegisterCngPayloadType(payload_type, clockrate_hz); } + +void AudioSendStream::UpdateCachedTargetAudioBitrateConstraints() { + absl::optional<AudioSendStream::TargetAudioBitrateConstraints> + new_constraints = GetMinMaxBitrateConstraints(); + if (!new_constraints.has_value()) { + return; + } + rtp_transport_queue_->RunOrPost([this, new_constraints]() { + RTC_DCHECK_RUN_ON(rtp_transport_queue_); + cached_constraints_ = new_constraints; + }); +} + } // namespace internal } // namespace webrtc diff --git a/audio/audio_send_stream.h b/audio/audio_send_stream.h index 7bc3183123..42be43afb9 100644 --- a/audio/audio_send_stream.h +++ b/audio/audio_send_stream.h @@ -15,18 +15,21 @@ #include <utility> #include <vector> +#include "absl/functional/any_invocable.h" +#include "api/field_trials_view.h" +#include "api/sequence_checker.h" +#include "api/task_queue/task_queue_base.h" #include "audio/audio_level.h" #include "audio/channel_send.h" #include "call/audio_send_stream.h" #include "call/audio_state.h" #include "call/bitrate_allocator.h" #include "modules/rtp_rtcp/source/rtp_rtcp_interface.h" -#include "rtc_base/constructor_magic.h" +#include "modules/utility/maybe_worker_thread.h" #include "rtc_base/experiments/struct_parameters_parser.h" #include "rtc_base/race_checker.h" #include "rtc_base/synchronization/mutex.h" #include "rtc_base/task_queue.h" -#include "rtc_base/thread_checker.h" namespace webrtc { class RtcEventLog; @@ -47,7 +50,7 @@ struct AudioAllocationConfig { absl::optional<double> bitrate_priority; std::unique_ptr<StructParametersParser> Parser(); - AudioAllocationConfig(); + explicit AudioAllocationConfig(const FieldTrialsView& field_trials); }; namespace internal { class AudioState; @@ -59,12 +62,12 @@ class AudioSendStream final : public webrtc::AudioSendStream, const webrtc::AudioSendStream::Config& config, const rtc::scoped_refptr<webrtc::AudioState>& audio_state, TaskQueueFactory* task_queue_factory, - ProcessThread* module_process_thread, RtpTransportControllerSendInterface* rtp_transport, BitrateAllocatorInterface* bitrate_allocator, RtcEventLog* event_log, RtcpRttStats* rtcp_rtt_stats, - const absl::optional<RtpState>& suspended_rtp_state); + const absl::optional<RtpState>& suspended_rtp_state, + const FieldTrialsView& field_trials); // For unit tests, which need to supply a mock ChannelSend. AudioSendStream(Clock* clock, const webrtc::AudioSendStream::Config& config, @@ -74,12 +77,19 @@ class AudioSendStream final : public webrtc::AudioSendStream, BitrateAllocatorInterface* bitrate_allocator, RtcEventLog* event_log, const absl::optional<RtpState>& suspended_rtp_state, - std::unique_ptr<voe::ChannelSendInterface> channel_send); + std::unique_ptr<voe::ChannelSendInterface> channel_send, + const FieldTrialsView& field_trials); + + AudioSendStream() = delete; + AudioSendStream(const AudioSendStream&) = delete; + AudioSendStream& operator=(const AudioSendStream&) = delete; + ~AudioSendStream() override; // webrtc::AudioSendStream implementation. const webrtc::AudioSendStream::Config& GetConfig() const override; - void Reconfigure(const webrtc::AudioSendStream::Config& config) override; + void Reconfigure(const webrtc::AudioSendStream::Config& config, + SetParametersCallback callback) override; void Start() override; void Stop() override; void SendAudioData(std::unique_ptr<AudioFrame> audio_frame) override; @@ -117,22 +127,31 @@ class AudioSendStream final : public webrtc::AudioSendStream, internal::AudioState* audio_state(); const internal::AudioState* audio_state() const; - void StoreEncoderProperties(int sample_rate_hz, size_t num_channels); - - void ConfigureStream(const Config& new_config, bool first_time); - bool SetupSendCodec(const Config& new_config); - bool ReconfigureSendCodec(const Config& new_config); - void ReconfigureANA(const Config& new_config); - void ReconfigureCNG(const Config& new_config); - void ReconfigureBitrateObserver(const Config& new_config); - - void ConfigureBitrateObserver() RTC_RUN_ON(worker_queue_); - void RemoveBitrateObserver(); + void StoreEncoderProperties(int sample_rate_hz, size_t num_channels) + RTC_RUN_ON(worker_thread_checker_); + + void ConfigureStream(const Config& new_config, + bool first_time, + SetParametersCallback callback) + RTC_RUN_ON(worker_thread_checker_); + bool SetupSendCodec(const Config& new_config) + RTC_RUN_ON(worker_thread_checker_); + bool ReconfigureSendCodec(const Config& new_config) + RTC_RUN_ON(worker_thread_checker_); + void ReconfigureANA(const Config& new_config) + RTC_RUN_ON(worker_thread_checker_); + void ReconfigureCNG(const Config& new_config) + RTC_RUN_ON(worker_thread_checker_); + void ReconfigureBitrateObserver(const Config& new_config) + RTC_RUN_ON(worker_thread_checker_); + + void ConfigureBitrateObserver() RTC_RUN_ON(worker_thread_checker_); + void RemoveBitrateObserver() RTC_RUN_ON(worker_thread_checker_); // Returns bitrate constraints, maybe including overhead when enabled by // field trial. - TargetAudioBitrateConstraints GetMinMaxBitrateConstraints() const - RTC_RUN_ON(worker_queue_); + absl::optional<TargetAudioBitrateConstraints> GetMinMaxBitrateConstraints() + const RTC_RUN_ON(worker_thread_checker_); // Sets per-packet overhead on encoded (for ANA) based on current known values // of transport and packetization overheads. @@ -143,37 +162,44 @@ class AudioSendStream final : public webrtc::AudioSendStream, size_t GetPerPacketOverheadBytes() const RTC_EXCLUSIVE_LOCKS_REQUIRED(overhead_per_packet_lock_); - void RegisterCngPayloadType(int payload_type, int clockrate_hz); + void RegisterCngPayloadType(int payload_type, int clockrate_hz) + RTC_RUN_ON(worker_thread_checker_); + + void UpdateCachedTargetAudioBitrateConstraints() + RTC_RUN_ON(worker_thread_checker_); + Clock* clock_; + const FieldTrialsView& field_trials_; - rtc::ThreadChecker worker_thread_checker_; - rtc::ThreadChecker pacer_thread_checker_; + SequenceChecker worker_thread_checker_; rtc::RaceChecker audio_capture_race_checker_; - rtc::TaskQueue* worker_queue_; + MaybeWorkerThread* rtp_transport_queue_; - const bool audio_send_side_bwe_; const bool allocate_audio_without_feedback_; const bool force_no_audio_feedback_ = allocate_audio_without_feedback_; const bool enable_audio_alr_probing_; - const bool send_side_bwe_with_overhead_; const AudioAllocationConfig allocation_settings_; - webrtc::AudioSendStream::Config config_; + webrtc::AudioSendStream::Config config_ + RTC_GUARDED_BY(worker_thread_checker_); rtc::scoped_refptr<webrtc::AudioState> audio_state_; const std::unique_ptr<voe::ChannelSendInterface> channel_send_; RtcEventLog* const event_log_; const bool use_legacy_overhead_calculation_; - int encoder_sample_rate_hz_ = 0; - size_t encoder_num_channels_ = 0; - bool sending_ = false; + int encoder_sample_rate_hz_ RTC_GUARDED_BY(worker_thread_checker_) = 0; + size_t encoder_num_channels_ RTC_GUARDED_BY(worker_thread_checker_) = 0; + bool sending_ RTC_GUARDED_BY(worker_thread_checker_) = false; mutable Mutex audio_level_lock_; // Keeps track of audio level, total audio energy and total samples duration. // https://w3c.github.io/webrtc-stats/#dom-rtcaudiohandlerstats-totalaudioenergy webrtc::voe::AudioLevel audio_level_ RTC_GUARDED_BY(audio_level_lock_); BitrateAllocatorInterface* const bitrate_allocator_ - RTC_GUARDED_BY(worker_queue_); + RTC_GUARDED_BY(rtp_transport_queue_); + // Constrains cached to be accessed from `rtp_transport_queue_`. + absl::optional<AudioSendStream::TargetAudioBitrateConstraints> + cached_constraints_ RTC_GUARDED_BY(rtp_transport_queue_) = absl::nullopt; RtpTransportControllerSendInterface* const rtp_transport_; RtpRtcpInterface* const rtp_rtcp_module_; @@ -202,12 +228,12 @@ class AudioSendStream final : public webrtc::AudioSendStream, size_t transport_overhead_per_packet_bytes_ RTC_GUARDED_BY(overhead_per_packet_lock_) = 0; - bool registered_with_allocator_ RTC_GUARDED_BY(worker_queue_) = false; - size_t total_packet_overhead_bytes_ RTC_GUARDED_BY(worker_queue_) = 0; + bool registered_with_allocator_ RTC_GUARDED_BY(worker_thread_checker_) = + false; + size_t total_packet_overhead_bytes_ RTC_GUARDED_BY(worker_thread_checker_) = + 0; absl::optional<std::pair<TimeDelta, TimeDelta>> frame_length_range_ - RTC_GUARDED_BY(worker_queue_); - - RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(AudioSendStream); + RTC_GUARDED_BY(worker_thread_checker_); }; } // namespace internal } // namespace webrtc diff --git a/audio/audio_send_stream_tests.cc b/audio/audio_send_stream_tests.cc index d2ea99ce08..2ec7229bfb 100644 --- a/audio/audio_send_stream_tests.cc +++ b/audio/audio_send_stream_tests.cc @@ -31,7 +31,7 @@ enum : int { // The first valid value is 1. class AudioSendTest : public SendTest { public: - AudioSendTest() : SendTest(CallTest::kDefaultTimeoutMs) {} + AudioSendTest() : SendTest(CallTest::kDefaultTimeout) {} size_t GetNumVideoStreams() const override { return 0; } size_t GetNumAudioStreams() const override { return 1; } @@ -61,9 +61,9 @@ TEST_F(AudioSendStreamCallTest, SupportsCName) { return SEND_PACKET; } - void ModifyAudioConfigs( - AudioSendStream::Config* send_config, - std::vector<AudioReceiveStream::Config>* receive_configs) override { + void ModifyAudioConfigs(AudioSendStream::Config* send_config, + std::vector<AudioReceiveStreamInterface::Config>* + receive_configs) override { send_config->rtp.c_name = kCName; } @@ -90,9 +90,9 @@ TEST_F(AudioSendStreamCallTest, NoExtensionsByDefault) { return SEND_PACKET; } - void ModifyAudioConfigs( - AudioSendStream::Config* send_config, - std::vector<AudioReceiveStream::Config>* receive_configs) override { + void ModifyAudioConfigs(AudioSendStream::Config* send_config, + std::vector<AudioReceiveStreamInterface::Config>* + receive_configs) override { send_config->rtp.extensions.clear(); } @@ -129,9 +129,9 @@ TEST_F(AudioSendStreamCallTest, SupportsAudioLevel) { return SEND_PACKET; } - void ModifyAudioConfigs( - AudioSendStream::Config* send_config, - std::vector<AudioReceiveStream::Config>* receive_configs) override { + void ModifyAudioConfigs(AudioSendStream::Config* send_config, + std::vector<AudioReceiveStreamInterface::Config>* + receive_configs) override { send_config->rtp.extensions.clear(); send_config->rtp.extensions.push_back( RtpExtension(RtpExtension::kAudioLevelUri, kAudioLevelExtensionId)); @@ -171,9 +171,9 @@ class TransportWideSequenceNumberObserver : public AudioSendTest { return SEND_PACKET; } - void ModifyAudioConfigs( - AudioSendStream::Config* send_config, - std::vector<AudioReceiveStream::Config>* receive_configs) override { + void ModifyAudioConfigs(AudioSendStream::Config* send_config, + std::vector<AudioReceiveStreamInterface::Config>* + receive_configs) override { send_config->rtp.extensions.clear(); send_config->rtp.extensions.push_back( RtpExtension(RtpExtension::kTransportSequenceNumberUri, @@ -188,17 +188,10 @@ class TransportWideSequenceNumberObserver : public AudioSendTest { }; TEST_F(AudioSendStreamCallTest, SendsTransportWideSequenceNumbersInFieldTrial) { - ScopedFieldTrials field_trials("WebRTC-Audio-SendSideBwe/Enabled/"); TransportWideSequenceNumberObserver test(/*expect_sequence_number=*/true); RunBaseTest(&test); } -TEST_F(AudioSendStreamCallTest, - DoesNotSendTransportWideSequenceNumbersPerDefault) { - TransportWideSequenceNumberObserver test(/*expect_sequence_number=*/false); - RunBaseTest(&test); -} - TEST_F(AudioSendStreamCallTest, SendDtmf) { static const uint8_t kDtmfPayloadType = 120; static const int kDtmfPayloadFrequency = 8000; @@ -230,9 +223,9 @@ TEST_F(AudioSendStreamCallTest, SendDtmf) { return SEND_PACKET; } - void OnAudioStreamsCreated( - AudioSendStream* send_stream, - const std::vector<AudioReceiveStream*>& receive_streams) override { + void OnAudioStreamsCreated(AudioSendStream* send_stream, + const std::vector<AudioReceiveStreamInterface*>& + receive_streams) override { // Need to start stream here, else DTMF events are dropped. send_stream->Start(); for (int event = kDtmfEventFirst; event <= kDtmfEventLast; ++event) { diff --git a/audio/audio_send_stream_unittest.cc b/audio/audio_send_stream_unittest.cc index d094198721..a81b40cbe7 100644 --- a/audio/audio_send_stream_unittest.cc +++ b/audio/audio_send_stream_unittest.cc @@ -30,12 +30,13 @@ #include "modules/audio_processing/include/mock_audio_processing.h" #include "modules/rtp_rtcp/mocks/mock_rtcp_bandwidth_observer.h" #include "modules/rtp_rtcp/mocks/mock_rtp_rtcp.h" -#include "rtc_base/task_queue_for_test.h" +#include "modules/utility/maybe_worker_thread.h" #include "system_wrappers/include/clock.h" -#include "test/field_trial.h" #include "test/gtest.h" #include "test/mock_audio_encoder.h" #include "test/mock_audio_encoder_factory.h" +#include "test/scoped_key_value_config.h" +#include "test/time_controller/real_time_controller.h" namespace webrtc { namespace test { @@ -45,8 +46,10 @@ using ::testing::_; using ::testing::AnyNumber; using ::testing::Eq; using ::testing::Field; +using ::testing::InSequence; using ::testing::Invoke; using ::testing::Ne; +using ::testing::NiceMock; using ::testing::Return; using ::testing::StrEq; @@ -119,7 +122,7 @@ std::unique_ptr<MockAudioEncoder> SetupAudioEncoderMock( rtc::scoped_refptr<MockAudioEncoderFactory> SetupEncoderFactoryMock() { rtc::scoped_refptr<MockAudioEncoderFactory> factory = - new rtc::RefCountedObject<MockAudioEncoderFactory>(); + rtc::make_ref_counted<MockAudioEncoderFactory>(); ON_CALL(*factory.get(), GetSupportedEncoders()) .WillByDefault(Return(std::vector<AudioCodecSpec>( std::begin(kCodecSpecs), std::end(kCodecSpecs)))); @@ -146,32 +149,29 @@ struct ConfigHelper { ConfigHelper(bool audio_bwe_enabled, bool expect_set_encoder_call, bool use_null_audio_processing) - : clock_(1000000), - task_queue_factory_(CreateDefaultTaskQueueFactory()), - stream_config_(/*send_transport=*/nullptr), + : stream_config_(/*send_transport=*/nullptr), audio_processing_( use_null_audio_processing ? nullptr - : new rtc::RefCountedObject<MockAudioProcessing>()), + : rtc::make_ref_counted<NiceMock<MockAudioProcessing>>()), bitrate_allocator_(&limit_observer_), - worker_queue_(task_queue_factory_->CreateTaskQueue( - "ConfigHelper_worker_queue", - TaskQueueFactory::Priority::NORMAL)), + worker_queue_(field_trials, + "ConfigHelper_worker_queue", + time_controller_.GetTaskQueueFactory()), audio_encoder_(nullptr) { using ::testing::Invoke; AudioState::Config config; config.audio_mixer = AudioMixerImpl::Create(); config.audio_processing = audio_processing_; - config.audio_device_module = - new rtc::RefCountedObject<MockAudioDeviceModule>(); + config.audio_device_module = rtc::make_ref_counted<MockAudioDeviceModule>(); audio_state_ = AudioState::Create(config); SetupDefaultChannelSend(audio_bwe_enabled); SetupMockForSetupSendCodec(expect_set_encoder_call); SetupMockForCallEncoder(); - // Use ISAC as default codec so as to prevent unnecessary |channel_proxy_| + // Use ISAC as default codec so as to prevent unnecessary `channel_proxy_` // calls from the default ctor behavior. stream_config_.send_codec_spec = AudioSendStream::Config::SendCodecSpec(kIsacPayloadType, kIsacFormat); @@ -192,10 +192,11 @@ struct ConfigHelper { .WillRepeatedly(Return(&worker_queue_)); return std::unique_ptr<internal::AudioSendStream>( new internal::AudioSendStream( - Clock::GetRealTimeClock(), stream_config_, audio_state_, - task_queue_factory_.get(), &rtp_transport_, &bitrate_allocator_, - &event_log_, absl::nullopt, - std::unique_ptr<voe::ChannelSendInterface>(channel_send_))); + time_controller_.GetClock(), stream_config_, audio_state_, + time_controller_.GetTaskQueueFactory(), &rtp_transport_, + &bitrate_allocator_, &event_log_, absl::nullopt, + std::unique_ptr<voe::ChannelSendInterface>(channel_send_), + field_trials)); } AudioSendStream::Config& config() { return stream_config_; } @@ -232,7 +233,7 @@ struct ConfigHelper { .WillRepeatedly(Return(&bandwidth_observer_)); if (audio_bwe_enabled) { EXPECT_CALL(rtp_rtcp_, - RegisterRtpHeaderExtension(TransportSequenceNumber::kUri, + RegisterRtpHeaderExtension(TransportSequenceNumber::Uri(), kTransportSequenceNumberId)) .Times(1); EXPECT_CALL(*channel_send_, @@ -245,7 +246,6 @@ struct ConfigHelper { .Times(1); } EXPECT_CALL(*channel_send_, ResetSenderCongestionControlObjects()).Times(1); - EXPECT_CALL(rtp_rtcp_, SetRid(std::string())).Times(1); } void SetupMockForSetupSendCodec(bool expect_set_encoder_call) { @@ -300,7 +300,7 @@ struct ConfigHelper { .WillRepeatedly(Return(report_blocks)); EXPECT_CALL(*channel_send_, GetANAStatistics()) .WillRepeatedly(Return(ANAStats())); - EXPECT_CALL(*channel_send_, GetBitrate()).WillRepeatedly(Return(0)); + EXPECT_CALL(*channel_send_, GetTargetBitrate()).WillRepeatedly(Return(0)); audio_processing_stats_.echo_return_loss = kEchoReturnLoss; audio_processing_stats_.echo_return_loss_enhancement = @@ -319,11 +319,12 @@ struct ConfigHelper { } } - TaskQueueForTest* worker() { return &worker_queue_; } + MaybeWorkerThread* worker() { return &worker_queue_; } + + test::ScopedKeyValueConfig field_trials; private: - SimulatedClock clock_; - std::unique_ptr<TaskQueueFactory> task_queue_factory_; + RealTimeController time_controller_; rtc::scoped_refptr<AudioState> audio_state_; AudioSendStream::Config stream_config_; ::testing::StrictMock<MockChannelSend>* channel_send_ = nullptr; @@ -335,9 +336,9 @@ struct ConfigHelper { ::testing::NiceMock<MockRtpRtcpInterface> rtp_rtcp_; ::testing::NiceMock<MockLimitObserver> limit_observer_; BitrateAllocator bitrate_allocator_; - // |worker_queue| is defined last to ensure all pending tasks are cancelled + // `worker_queue` is defined last to ensure all pending tasks are cancelled // and deleted before any other members. - TaskQueueForTest worker_queue_; + MaybeWorkerThread worker_queue_; std::unique_ptr<AudioEncoder> audio_encoder_; }; @@ -366,6 +367,7 @@ TEST(AudioSendStreamTest, ConfigToString) { config.rtp.c_name = kCName; config.min_bitrate_bps = 12000; config.max_bitrate_bps = 34000; + config.has_dscp = true; config.send_codec_spec = AudioSendStream::Config::SendCodecSpec(kIsacPayloadType, kIsacFormat); config.send_codec_spec->nack_enabled = true; @@ -382,9 +384,11 @@ TEST(AudioSendStreamTest, ConfigToString) { "urn:ietf:params:rtp-hdrext:ssrc-audio-level, id: 2}], " "c_name: foo_name}, rtcp_report_interval_ms: 2500, " "send_transport: null, " - "min_bitrate_bps: 12000, max_bitrate_bps: 34000, " + "min_bitrate_bps: 12000, max_bitrate_bps: 34000, has " + "audio_network_adaptor_config: false, has_dscp: true, " "send_codec_spec: {nack_enabled: true, transport_cc_enabled: false, " - "cng_payload_type: 42, red_payload_type: 43, payload_type: 103, " + "enable_non_sender_rtt: false, cng_payload_type: 42, " + "red_payload_type: 43, payload_type: 103, " "format: {name: isac, clockrate_hz: 16000, num_channels: 1, " "parameters: {}}}}", config.ToString()); @@ -418,7 +422,6 @@ TEST(AudioSendStreamTest, SetMuted) { } TEST(AudioSendStreamTest, AudioBweCorrectObjectsOnChannelProxy) { - ScopedFieldTrials field_trials("WebRTC-Audio-SendSideBwe/Enabled/"); for (bool use_null_audio_processing : {false, true}) { ConfigHelper helper(true, true, use_null_audio_processing); auto send_stream = helper.CreateAudioSendStream(); @@ -467,7 +470,6 @@ TEST(AudioSendStreamTest, GetStats) { stats.apm_statistics.residual_echo_likelihood); EXPECT_EQ(kResidualEchoLikelihoodMax, stats.apm_statistics.residual_echo_likelihood_recent_max); - EXPECT_FALSE(stats.typing_noise_detected); } } } @@ -520,14 +522,12 @@ TEST(AudioSendStreamTest, GetStatsAudioLevel) { TEST(AudioSendStreamTest, SendCodecAppliesAudioNetworkAdaptor) { for (bool use_null_audio_processing : {false, true}) { - ConfigHelper helper(false, true, use_null_audio_processing); + ConfigHelper helper(true, true, use_null_audio_processing); helper.config().send_codec_spec = AudioSendStream::Config::SendCodecSpec(0, kOpusFormat); const std::string kAnaConfigString = "abcde"; const std::string kAnaReconfigString = "12345"; - helper.config().rtp.extensions.push_back(RtpExtension( - RtpExtension::kTransportSequenceNumberUri, kTransportSequenceNumberId)); helper.config().audio_network_adaptor_config = kAnaConfigString; EXPECT_CALL(helper.mock_encoder_factory(), MakeAudioEncoderMock(_, _, _, _)) @@ -550,7 +550,47 @@ TEST(AudioSendStreamTest, SendCodecAppliesAudioNetworkAdaptor) { auto stream_config = helper.config(); stream_config.audio_network_adaptor_config = kAnaReconfigString; - send_stream->Reconfigure(stream_config); + send_stream->Reconfigure(stream_config, nullptr); + } +} + +TEST(AudioSendStreamTest, AudioNetworkAdaptorReceivesOverhead) { + for (bool use_null_audio_processing : {false, true}) { + ConfigHelper helper(true, true, use_null_audio_processing); + helper.config().send_codec_spec = + AudioSendStream::Config::SendCodecSpec(0, kOpusFormat); + const std::string kAnaConfigString = "abcde"; + + EXPECT_CALL(helper.mock_encoder_factory(), MakeAudioEncoderMock(_, _, _, _)) + .WillOnce(Invoke( + [&kAnaConfigString](int payload_type, const SdpAudioFormat& format, + absl::optional<AudioCodecPairId> codec_pair_id, + std::unique_ptr<AudioEncoder>* return_value) { + auto mock_encoder = SetupAudioEncoderMock(payload_type, format); + InSequence s; + EXPECT_CALL( + *mock_encoder, + OnReceivedOverhead(Eq(kOverheadPerPacket.bytes<size_t>()))) + .Times(2); + EXPECT_CALL(*mock_encoder, + EnableAudioNetworkAdaptor(StrEq(kAnaConfigString), _)) + .WillOnce(Return(true)); + // Note: Overhead is received AFTER ANA has been enabled. + EXPECT_CALL( + *mock_encoder, + OnReceivedOverhead(Eq(kOverheadPerPacket.bytes<size_t>()))) + .WillOnce(Return()); + *return_value = std::move(mock_encoder); + })); + EXPECT_CALL(*helper.rtp_rtcp(), ExpectedPerPacketOverhead) + .WillRepeatedly(Return(kOverheadPerPacket.bytes<size_t>())); + + auto send_stream = helper.CreateAudioSendStream(); + + auto stream_config = helper.config(); + stream_config.audio_network_adaptor_config = kAnaConfigString; + + send_stream->Reconfigure(stream_config, nullptr); } } @@ -596,13 +636,12 @@ TEST(AudioSendStreamTest, DoesNotPassHigherBitrateThanMaxBitrate) { update.packet_loss_ratio = 0; update.round_trip_time = TimeDelta::Millis(50); update.bwe_period = TimeDelta::Millis(6000); - helper.worker()->SendTask([&] { send_stream->OnBitrateUpdated(update); }, - RTC_FROM_HERE); + helper.worker()->RunSynchronous( + [&] { send_stream->OnBitrateUpdated(update); }); } } TEST(AudioSendStreamTest, SSBweTargetInRangeRespected) { - ScopedFieldTrials field_trials("WebRTC-Audio-SendSideBwe/Enabled/"); for (bool use_null_audio_processing : {false, true}) { ConfigHelper helper(true, true, use_null_audio_processing); auto send_stream = helper.CreateAudioSendStream(); @@ -614,17 +653,16 @@ TEST(AudioSendStreamTest, SSBweTargetInRangeRespected) { BitrateAllocationUpdate update; update.target_bitrate = DataRate::BitsPerSec(helper.config().max_bitrate_bps - 5000); - helper.worker()->SendTask([&] { send_stream->OnBitrateUpdated(update); }, - RTC_FROM_HERE); + helper.worker()->RunSynchronous( + [&] { send_stream->OnBitrateUpdated(update); }); } } TEST(AudioSendStreamTest, SSBweFieldTrialMinRespected) { - ScopedFieldTrials field_trials( - "WebRTC-Audio-SendSideBwe/Enabled/" - "WebRTC-Audio-Allocation/min:6kbps,max:64kbps/"); for (bool use_null_audio_processing : {false, true}) { ConfigHelper helper(true, true, use_null_audio_processing); + ScopedKeyValueConfig field_trials( + helper.field_trials, "WebRTC-Audio-Allocation/min:6kbps,max:64kbps/"); auto send_stream = helper.CreateAudioSendStream(); EXPECT_CALL( *helper.channel_send(), @@ -632,17 +670,16 @@ TEST(AudioSendStreamTest, SSBweFieldTrialMinRespected) { Eq(DataRate::KilobitsPerSec(6))))); BitrateAllocationUpdate update; update.target_bitrate = DataRate::KilobitsPerSec(1); - helper.worker()->SendTask([&] { send_stream->OnBitrateUpdated(update); }, - RTC_FROM_HERE); + helper.worker()->RunSynchronous( + [&] { send_stream->OnBitrateUpdated(update); }); } } TEST(AudioSendStreamTest, SSBweFieldTrialMaxRespected) { - ScopedFieldTrials field_trials( - "WebRTC-Audio-SendSideBwe/Enabled/" - "WebRTC-Audio-Allocation/min:6kbps,max:64kbps/"); for (bool use_null_audio_processing : {false, true}) { ConfigHelper helper(true, true, use_null_audio_processing); + ScopedKeyValueConfig field_trials( + helper.field_trials, "WebRTC-Audio-Allocation/min:6kbps,max:64kbps/"); auto send_stream = helper.CreateAudioSendStream(); EXPECT_CALL( *helper.channel_send(), @@ -650,18 +687,16 @@ TEST(AudioSendStreamTest, SSBweFieldTrialMaxRespected) { Eq(DataRate::KilobitsPerSec(64))))); BitrateAllocationUpdate update; update.target_bitrate = DataRate::KilobitsPerSec(128); - helper.worker()->SendTask([&] { send_stream->OnBitrateUpdated(update); }, - RTC_FROM_HERE); + helper.worker()->RunSynchronous( + [&] { send_stream->OnBitrateUpdated(update); }); } } TEST(AudioSendStreamTest, SSBweWithOverhead) { - ScopedFieldTrials field_trials( - "WebRTC-Audio-SendSideBwe/Enabled/" - "WebRTC-SendSideBwe-WithOverhead/Enabled/" - "WebRTC-Audio-LegacyOverhead/Disabled/"); for (bool use_null_audio_processing : {false, true}) { ConfigHelper helper(true, true, use_null_audio_processing); + ScopedKeyValueConfig field_trials(helper.field_trials, + "WebRTC-Audio-LegacyOverhead/Disabled/"); EXPECT_CALL(*helper.rtp_rtcp(), ExpectedPerPacketOverhead) .WillRepeatedly(Return(kOverheadPerPacket.bytes<size_t>())); auto send_stream = helper.CreateAudioSendStream(); @@ -673,19 +708,18 @@ TEST(AudioSendStreamTest, SSBweWithOverhead) { &BitrateAllocationUpdate::target_bitrate, Eq(bitrate)))); BitrateAllocationUpdate update; update.target_bitrate = bitrate; - helper.worker()->SendTask([&] { send_stream->OnBitrateUpdated(update); }, - RTC_FROM_HERE); + helper.worker()->RunSynchronous( + [&] { send_stream->OnBitrateUpdated(update); }); } } TEST(AudioSendStreamTest, SSBweWithOverheadMinRespected) { - ScopedFieldTrials field_trials( - "WebRTC-Audio-SendSideBwe/Enabled/" - "WebRTC-SendSideBwe-WithOverhead/Enabled/" - "WebRTC-Audio-LegacyOverhead/Disabled/" - "WebRTC-Audio-Allocation/min:6kbps,max:64kbps/"); for (bool use_null_audio_processing : {false, true}) { ConfigHelper helper(true, true, use_null_audio_processing); + ScopedKeyValueConfig field_trials( + helper.field_trials, + "WebRTC-Audio-LegacyOverhead/Disabled/" + "WebRTC-Audio-Allocation/min:6kbps,max:64kbps/"); EXPECT_CALL(*helper.rtp_rtcp(), ExpectedPerPacketOverhead) .WillRepeatedly(Return(kOverheadPerPacket.bytes<size_t>())); auto send_stream = helper.CreateAudioSendStream(); @@ -695,19 +729,18 @@ TEST(AudioSendStreamTest, SSBweWithOverheadMinRespected) { &BitrateAllocationUpdate::target_bitrate, Eq(bitrate)))); BitrateAllocationUpdate update; update.target_bitrate = DataRate::KilobitsPerSec(1); - helper.worker()->SendTask([&] { send_stream->OnBitrateUpdated(update); }, - RTC_FROM_HERE); + helper.worker()->RunSynchronous( + [&] { send_stream->OnBitrateUpdated(update); }); } } TEST(AudioSendStreamTest, SSBweWithOverheadMaxRespected) { - ScopedFieldTrials field_trials( - "WebRTC-Audio-SendSideBwe/Enabled/" - "WebRTC-SendSideBwe-WithOverhead/Enabled/" - "WebRTC-Audio-LegacyOverhead/Disabled/" - "WebRTC-Audio-Allocation/min:6kbps,max:64kbps/"); for (bool use_null_audio_processing : {false, true}) { ConfigHelper helper(true, true, use_null_audio_processing); + ScopedKeyValueConfig field_trials( + helper.field_trials, + "WebRTC-Audio-LegacyOverhead/Disabled/" + "WebRTC-Audio-Allocation/min:6kbps,max:64kbps/"); EXPECT_CALL(*helper.rtp_rtcp(), ExpectedPerPacketOverhead) .WillRepeatedly(Return(kOverheadPerPacket.bytes<size_t>())); auto send_stream = helper.CreateAudioSendStream(); @@ -717,8 +750,8 @@ TEST(AudioSendStreamTest, SSBweWithOverheadMaxRespected) { &BitrateAllocationUpdate::target_bitrate, Eq(bitrate)))); BitrateAllocationUpdate update; update.target_bitrate = DataRate::KilobitsPerSec(128); - helper.worker()->SendTask([&] { send_stream->OnBitrateUpdated(update); }, - RTC_FROM_HERE); + helper.worker()->RunSynchronous( + [&] { send_stream->OnBitrateUpdated(update); }); } } @@ -736,8 +769,8 @@ TEST(AudioSendStreamTest, ProbingIntervalOnBitrateUpdated) { update.packet_loss_ratio = 0; update.round_trip_time = TimeDelta::Millis(50); update.bwe_period = TimeDelta::Millis(5000); - helper.worker()->SendTask([&] { send_stream->OnBitrateUpdated(update); }, - RTC_FROM_HERE); + helper.worker()->RunSynchronous( + [&] { send_stream->OnBitrateUpdated(update); }); } } @@ -758,12 +791,11 @@ TEST(AudioSendStreamTest, DontRecreateEncoder) { AudioSendStream::Config::SendCodecSpec(9, kG722Format); helper.config().send_codec_spec->cng_payload_type = 105; auto send_stream = helper.CreateAudioSendStream(); - send_stream->Reconfigure(helper.config()); + send_stream->Reconfigure(helper.config(), nullptr); } } TEST(AudioSendStreamTest, ReconfigureTransportCcResetsFirst) { - ScopedFieldTrials field_trials("WebRTC-Audio-SendSideBwe/Enabled/"); for (bool use_null_audio_processing : {false, true}) { ConfigHelper helper(false, true, use_null_audio_processing); auto send_stream = helper.CreateAudioSendStream(); @@ -771,7 +803,7 @@ TEST(AudioSendStreamTest, ReconfigureTransportCcResetsFirst) { ConfigHelper::AddBweToConfig(&new_config); EXPECT_CALL(*helper.rtp_rtcp(), - RegisterRtpHeaderExtension(TransportSequenceNumber::kUri, + RegisterRtpHeaderExtension(TransportSequenceNumber::Uri(), kTransportSequenceNumberId)) .Times(1); { @@ -784,7 +816,7 @@ TEST(AudioSendStreamTest, ReconfigureTransportCcResetsFirst) { .Times(1); } - send_stream->Reconfigure(new_config); + send_stream->Reconfigure(new_config, nullptr); } } @@ -840,8 +872,8 @@ TEST(AudioSendStreamTest, AudioOverheadChanged) { DataRate::BitsPerSec(helper.config().max_bitrate_bps) + kMaxOverheadRate; EXPECT_CALL(*helper.channel_send(), OnBitrateAllocation); - helper.worker()->SendTask([&] { send_stream->OnBitrateUpdated(update); }, - RTC_FROM_HERE); + helper.worker()->RunSynchronous( + [&] { send_stream->OnBitrateUpdated(update); }); EXPECT_EQ(audio_overhead_per_packet_bytes, send_stream->TestOnlyGetPerPacketOverheadBytes()); @@ -849,8 +881,8 @@ TEST(AudioSendStreamTest, AudioOverheadChanged) { EXPECT_CALL(*helper.rtp_rtcp(), ExpectedPerPacketOverhead) .WillRepeatedly(Return(audio_overhead_per_packet_bytes + 20)); EXPECT_CALL(*helper.channel_send(), OnBitrateAllocation); - helper.worker()->SendTask([&] { send_stream->OnBitrateUpdated(update); }, - RTC_FROM_HERE); + helper.worker()->RunSynchronous( + [&] { send_stream->OnBitrateUpdated(update); }); EXPECT_EQ(audio_overhead_per_packet_bytes + 20, send_stream->TestOnlyGetPerPacketOverheadBytes()); @@ -874,8 +906,8 @@ TEST(AudioSendStreamTest, OnAudioAndTransportOverheadChanged) { DataRate::BitsPerSec(helper.config().max_bitrate_bps) + kMaxOverheadRate; EXPECT_CALL(*helper.channel_send(), OnBitrateAllocation); - helper.worker()->SendTask([&] { send_stream->OnBitrateUpdated(update); }, - RTC_FROM_HERE); + helper.worker()->RunSynchronous( + [&] { send_stream->OnBitrateUpdated(update); }); EXPECT_EQ( transport_overhead_per_packet_bytes + audio_overhead_per_packet_bytes, @@ -892,25 +924,25 @@ TEST(AudioSendStreamTest, ReconfigureWithFrameEncryptor) { auto new_config = helper.config(); rtc::scoped_refptr<FrameEncryptorInterface> mock_frame_encryptor_0( - new rtc::RefCountedObject<MockFrameEncryptor>()); + rtc::make_ref_counted<MockFrameEncryptor>()); new_config.frame_encryptor = mock_frame_encryptor_0; EXPECT_CALL(*helper.channel_send(), SetFrameEncryptor(Ne(nullptr))) .Times(1); - send_stream->Reconfigure(new_config); + send_stream->Reconfigure(new_config, nullptr); // Not updating the frame encryptor shouldn't force it to reconfigure. EXPECT_CALL(*helper.channel_send(), SetFrameEncryptor(_)).Times(0); - send_stream->Reconfigure(new_config); + send_stream->Reconfigure(new_config, nullptr); // Updating frame encryptor to a new object should force a call to the // proxy. rtc::scoped_refptr<FrameEncryptorInterface> mock_frame_encryptor_1( - new rtc::RefCountedObject<MockFrameEncryptor>()); + rtc::make_ref_counted<MockFrameEncryptor>()); new_config.frame_encryptor = mock_frame_encryptor_1; new_config.crypto_options.sframe.require_frame_encryption = true; EXPECT_CALL(*helper.channel_send(), SetFrameEncryptor(Ne(nullptr))) .Times(1); - send_stream->Reconfigure(new_config); + send_stream->Reconfigure(new_config, nullptr); } } } // namespace test diff --git a/audio/audio_state.cc b/audio/audio_state.cc index 73366e20a8..76ff152eea 100644 --- a/audio/audio_state.cc +++ b/audio/audio_state.cc @@ -15,29 +15,33 @@ #include <utility> #include <vector> +#include "api/sequence_checker.h" +#include "api/task_queue/task_queue_base.h" +#include "api/units/time_delta.h" #include "audio/audio_receive_stream.h" #include "audio/audio_send_stream.h" #include "modules/audio_device/include/audio_device.h" #include "rtc_base/checks.h" #include "rtc_base/logging.h" -#include "rtc_base/ref_counted_object.h" -#include "rtc_base/thread.h" namespace webrtc { namespace internal { AudioState::AudioState(const AudioState::Config& config) : config_(config), - audio_transport_(config_.audio_mixer, config_.audio_processing.get()) { + audio_transport_(config_.audio_mixer.get(), + config_.audio_processing.get(), + config_.async_audio_processing_factory.get()) { process_thread_checker_.Detach(); RTC_DCHECK(config_.audio_mixer); RTC_DCHECK(config_.audio_device_module); } AudioState::~AudioState() { - RTC_DCHECK(thread_checker_.IsCurrent()); + RTC_DCHECK_RUN_ON(&thread_checker_); RTC_DCHECK(receiving_streams_.empty()); RTC_DCHECK(sending_streams_.empty()); + RTC_DCHECK(!null_audio_poller_.Running()); } AudioProcessing* AudioState::audio_processing() { @@ -48,17 +52,13 @@ AudioTransport* AudioState::audio_transport() { return &audio_transport_; } -bool AudioState::typing_noise_detected() const { - RTC_DCHECK(thread_checker_.IsCurrent()); - return audio_transport_.typing_noise_detected(); -} - -void AudioState::AddReceivingStream(webrtc::AudioReceiveStream* stream) { - RTC_DCHECK(thread_checker_.IsCurrent()); +void AudioState::AddReceivingStream( + webrtc::AudioReceiveStreamInterface* stream) { + RTC_DCHECK_RUN_ON(&thread_checker_); RTC_DCHECK_EQ(0, receiving_streams_.count(stream)); receiving_streams_.insert(stream); if (!config_.audio_mixer->AddSource( - static_cast<internal::AudioReceiveStream*>(stream))) { + static_cast<AudioReceiveStreamImpl*>(stream))) { RTC_DLOG(LS_ERROR) << "Failed to add source to mixer."; } @@ -76,12 +76,13 @@ void AudioState::AddReceivingStream(webrtc::AudioReceiveStream* stream) { } } -void AudioState::RemoveReceivingStream(webrtc::AudioReceiveStream* stream) { - RTC_DCHECK(thread_checker_.IsCurrent()); +void AudioState::RemoveReceivingStream( + webrtc::AudioReceiveStreamInterface* stream) { + RTC_DCHECK_RUN_ON(&thread_checker_); auto count = receiving_streams_.erase(stream); RTC_DCHECK_EQ(1, count); config_.audio_mixer->RemoveSource( - static_cast<internal::AudioReceiveStream*>(stream)); + static_cast<AudioReceiveStreamImpl*>(stream)); UpdateNullAudioPollerState(); if (receiving_streams_.empty()) { config_.audio_device_module->StopPlayout(); @@ -91,7 +92,7 @@ void AudioState::RemoveReceivingStream(webrtc::AudioReceiveStream* stream) { void AudioState::AddSendingStream(webrtc::AudioSendStream* stream, int sample_rate_hz, size_t num_channels) { - RTC_DCHECK(thread_checker_.IsCurrent()); + RTC_DCHECK_RUN_ON(&thread_checker_); auto& properties = sending_streams_[stream]; properties.sample_rate_hz = sample_rate_hz; properties.num_channels = num_channels; @@ -111,7 +112,7 @@ void AudioState::AddSendingStream(webrtc::AudioSendStream* stream, } void AudioState::RemoveSendingStream(webrtc::AudioSendStream* stream) { - RTC_DCHECK(thread_checker_.IsCurrent()); + RTC_DCHECK_RUN_ON(&thread_checker_); auto count = sending_streams_.erase(stream); RTC_DCHECK_EQ(1, count); UpdateAudioTransportWithSendingStreams(); @@ -121,8 +122,8 @@ void AudioState::RemoveSendingStream(webrtc::AudioSendStream* stream) { } void AudioState::SetPlayout(bool enabled) { - RTC_LOG(INFO) << "SetPlayout(" << enabled << ")"; - RTC_DCHECK(thread_checker_.IsCurrent()); + RTC_LOG(LS_INFO) << "SetPlayout(" << enabled << ")"; + RTC_DCHECK_RUN_ON(&thread_checker_); if (playout_enabled_ != enabled) { playout_enabled_ = enabled; if (enabled) { @@ -138,8 +139,8 @@ void AudioState::SetPlayout(bool enabled) { } void AudioState::SetRecording(bool enabled) { - RTC_LOG(INFO) << "SetRecording(" << enabled << ")"; - RTC_DCHECK(thread_checker_.IsCurrent()); + RTC_LOG(LS_INFO) << "SetRecording(" << enabled << ")"; + RTC_DCHECK_RUN_ON(&thread_checker_); if (recording_enabled_ != enabled) { recording_enabled_ = enabled; if (enabled) { @@ -175,16 +176,38 @@ void AudioState::UpdateNullAudioPollerState() { // Run NullAudioPoller when there are receiving streams and playout is // disabled. if (!receiving_streams_.empty() && !playout_enabled_) { - if (!null_audio_poller_) - null_audio_poller_ = std::make_unique<NullAudioPoller>(&audio_transport_); + if (!null_audio_poller_.Running()) { + AudioTransport* audio_transport = &audio_transport_; + null_audio_poller_ = RepeatingTaskHandle::Start( + TaskQueueBase::Current(), [audio_transport] { + static constexpr size_t kNumChannels = 1; + static constexpr uint32_t kSamplesPerSecond = 48'000; + // 10ms of samples + static constexpr size_t kNumSamples = kSamplesPerSecond / 100; + + // Buffer to hold the audio samples. + int16_t buffer[kNumSamples * kNumChannels]; + + // Output variables from `NeedMorePlayData`. + size_t n_samples; + int64_t elapsed_time_ms; + int64_t ntp_time_ms; + audio_transport->NeedMorePlayData( + kNumSamples, sizeof(int16_t), kNumChannels, kSamplesPerSecond, + buffer, n_samples, &elapsed_time_ms, &ntp_time_ms); + + // Reschedule the next poll iteration. + return TimeDelta::Millis(10); + }); + } } else { - null_audio_poller_.reset(); + null_audio_poller_.Stop(); } } } // namespace internal rtc::scoped_refptr<AudioState> AudioState::Create( const AudioState::Config& config) { - return new rtc::RefCountedObject<internal::AudioState>(config); + return rtc::make_ref_counted<internal::AudioState>(config); } } // namespace webrtc diff --git a/audio/audio_state.h b/audio/audio_state.h index 70c7208320..6c2b7aa453 100644 --- a/audio/audio_state.h +++ b/audio/audio_state.h @@ -13,25 +13,30 @@ #include <map> #include <memory> -#include <unordered_set> +#include "api/sequence_checker.h" #include "audio/audio_transport_impl.h" -#include "audio/null_audio_poller.h" #include "call/audio_state.h" -#include "rtc_base/constructor_magic.h" +#include "rtc_base/containers/flat_set.h" #include "rtc_base/ref_count.h" -#include "rtc_base/thread_checker.h" +#include "rtc_base/task_utils/repeating_task.h" +#include "rtc_base/thread_annotations.h" namespace webrtc { class AudioSendStream; -class AudioReceiveStream; +class AudioReceiveStreamInterface; namespace internal { class AudioState : public webrtc::AudioState { public: explicit AudioState(const AudioState::Config& config); + + AudioState() = delete; + AudioState(const AudioState&) = delete; + AudioState& operator=(const AudioState&) = delete; + ~AudioState() override; AudioProcessing* audio_processing() override; @@ -47,10 +52,8 @@ class AudioState : public webrtc::AudioState { return config_.audio_device_module.get(); } - bool typing_noise_detected() const; - - void AddReceivingStream(webrtc::AudioReceiveStream* stream); - void RemoveReceivingStream(webrtc::AudioReceiveStream* stream); + void AddReceivingStream(webrtc::AudioReceiveStreamInterface* stream); + void RemoveReceivingStream(webrtc::AudioReceiveStreamInterface* stream); void AddSendingStream(webrtc::AudioSendStream* stream, int sample_rate_hz, @@ -59,10 +62,10 @@ class AudioState : public webrtc::AudioState { private: void UpdateAudioTransportWithSendingStreams(); - void UpdateNullAudioPollerState(); + void UpdateNullAudioPollerState() RTC_RUN_ON(&thread_checker_); - rtc::ThreadChecker thread_checker_; - rtc::ThreadChecker process_thread_checker_; + SequenceChecker thread_checker_; + SequenceChecker process_thread_checker_; const webrtc::AudioState::Config config_; bool recording_enabled_ = true; bool playout_enabled_ = true; @@ -74,16 +77,14 @@ class AudioState : public webrtc::AudioState { // Null audio poller is used to continue polling the audio streams if audio // playout is disabled so that audio processing still happens and the audio // stats are still updated. - std::unique_ptr<NullAudioPoller> null_audio_poller_; + RepeatingTaskHandle null_audio_poller_ RTC_GUARDED_BY(&thread_checker_); - std::unordered_set<webrtc::AudioReceiveStream*> receiving_streams_; + webrtc::flat_set<webrtc::AudioReceiveStreamInterface*> receiving_streams_; struct StreamProperties { int sample_rate_hz = 0; size_t num_channels = 0; }; std::map<webrtc::AudioSendStream*, StreamProperties> sending_streams_; - - RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(AudioState); }; } // namespace internal } // namespace webrtc diff --git a/audio/audio_state_unittest.cc b/audio/audio_state_unittest.cc index 2bbe0fb0b7..4426a782d7 100644 --- a/audio/audio_state_unittest.cc +++ b/audio/audio_state_unittest.cc @@ -11,13 +11,14 @@ #include "audio/audio_state.h" #include <memory> +#include <utility> #include <vector> +#include "api/task_queue/test/mock_task_queue_base.h" #include "call/test/mock_audio_send_stream.h" #include "modules/audio_device/include/mock_audio_device.h" #include "modules/audio_mixer/audio_mixer_impl.h" #include "modules/audio_processing/include/mock_audio_processing.h" -#include "rtc_base/ref_counted_object.h" #include "test/gtest.h" namespace webrtc { @@ -26,28 +27,101 @@ namespace { using ::testing::_; using ::testing::Matcher; +using ::testing::NiceMock; +using ::testing::StrictMock; +using ::testing::Values; constexpr int kSampleRate = 16000; constexpr int kNumberOfChannels = 1; +struct FakeAsyncAudioProcessingHelper { + class FakeTaskQueue : public StrictMock<MockTaskQueueBase> { + public: + FakeTaskQueue() = default; + + void Delete() override { delete this; } + void PostTask(absl::AnyInvocable<void() &&> task) override { + std::move(task)(); + } + }; + + class FakeTaskQueueFactory : public TaskQueueFactory { + public: + FakeTaskQueueFactory() = default; + ~FakeTaskQueueFactory() override = default; + std::unique_ptr<TaskQueueBase, TaskQueueDeleter> CreateTaskQueue( + absl::string_view name, + Priority priority) const override { + return std::unique_ptr<webrtc::TaskQueueBase, webrtc::TaskQueueDeleter>( + new FakeTaskQueue()); + } + }; + + class MockAudioFrameProcessor : public AudioFrameProcessor { + public: + ~MockAudioFrameProcessor() override = default; + + MOCK_METHOD(void, ProcessCalled, ()); + MOCK_METHOD(void, SinkSet, ()); + MOCK_METHOD(void, SinkCleared, ()); + + void Process(std::unique_ptr<AudioFrame> frame) override { + ProcessCalled(); + sink_callback_(std::move(frame)); + } + + void SetSink(OnAudioFrameCallback sink_callback) override { + sink_callback_ = std::move(sink_callback); + if (sink_callback_ == nullptr) + SinkCleared(); + else + SinkSet(); + } + + private: + OnAudioFrameCallback sink_callback_; + }; + + NiceMock<MockAudioFrameProcessor> audio_frame_processor_; + FakeTaskQueueFactory task_queue_factory_; + + rtc::scoped_refptr<AsyncAudioProcessing::Factory> CreateFactory() { + return rtc::make_ref_counted<AsyncAudioProcessing::Factory>( + audio_frame_processor_, task_queue_factory_); + } +}; + struct ConfigHelper { - explicit ConfigHelper(bool use_null_audio_processing) + struct Params { + bool use_null_audio_processing; + bool use_async_audio_processing; + }; + + explicit ConfigHelper(const Params& params) : audio_mixer(AudioMixerImpl::Create()) { audio_state_config.audio_mixer = audio_mixer; audio_state_config.audio_processing = - use_null_audio_processing + params.use_null_audio_processing ? nullptr - : new rtc::RefCountedObject< - testing::NiceMock<MockAudioProcessing>>(); + : rtc::make_ref_counted<testing::NiceMock<MockAudioProcessing>>(); audio_state_config.audio_device_module = - new rtc::RefCountedObject<MockAudioDeviceModule>(); + rtc::make_ref_counted<NiceMock<MockAudioDeviceModule>>(); + if (params.use_async_audio_processing) { + audio_state_config.async_audio_processing_factory = + async_audio_processing_helper_.CreateFactory(); + } } AudioState::Config& config() { return audio_state_config; } rtc::scoped_refptr<AudioMixer> mixer() { return audio_mixer; } + NiceMock<FakeAsyncAudioProcessingHelper::MockAudioFrameProcessor>& + mock_audio_frame_processor() { + return async_audio_processing_helper_.audio_frame_processor_; + } private: AudioState::Config audio_state_config; rtc::scoped_refptr<AudioMixer> audio_mixer; + FakeAsyncAudioProcessingHelper async_audio_processing_helper_; }; class FakeAudioSource : public AudioMixer::Source { @@ -93,184 +167,200 @@ std::vector<uint32_t> ComputeChannelLevels(AudioFrame* audio_frame) { } } // namespace -TEST(AudioStateTest, Create) { - for (bool use_null_audio_processing : {false, true}) { - ConfigHelper helper(use_null_audio_processing); - auto audio_state = AudioState::Create(helper.config()); - EXPECT_TRUE(audio_state.get()); - } +class AudioStateTest : public ::testing::TestWithParam<ConfigHelper::Params> {}; + +TEST_P(AudioStateTest, Create) { + ConfigHelper helper(GetParam()); + auto audio_state = AudioState::Create(helper.config()); + EXPECT_TRUE(audio_state.get()); } -TEST(AudioStateTest, ConstructDestruct) { - for (bool use_null_audio_processing : {false, true}) { - ConfigHelper helper(use_null_audio_processing); - rtc::scoped_refptr<internal::AudioState> audio_state( - new rtc::RefCountedObject<internal::AudioState>(helper.config())); - } +TEST_P(AudioStateTest, ConstructDestruct) { + ConfigHelper helper(GetParam()); + rtc::scoped_refptr<internal::AudioState> audio_state( + rtc::make_ref_counted<internal::AudioState>(helper.config())); } -TEST(AudioStateTest, RecordedAudioArrivesAtSingleStream) { - for (bool use_null_audio_processing : {false, true}) { - ConfigHelper helper(use_null_audio_processing); - rtc::scoped_refptr<internal::AudioState> audio_state( - new rtc::RefCountedObject<internal::AudioState>(helper.config())); - - MockAudioSendStream stream; - audio_state->AddSendingStream(&stream, 8000, 2); - - EXPECT_CALL( - stream, - SendAudioDataForMock(::testing::AllOf( - ::testing::Field(&AudioFrame::sample_rate_hz_, ::testing::Eq(8000)), - ::testing::Field(&AudioFrame::num_channels_, ::testing::Eq(2u))))) - .WillOnce( - // Verify that channels are not swapped by default. - ::testing::Invoke([](AudioFrame* audio_frame) { - auto levels = ComputeChannelLevels(audio_frame); - EXPECT_LT(0u, levels[0]); - EXPECT_EQ(0u, levels[1]); - })); - MockAudioProcessing* ap = use_null_audio_processing - ? nullptr - : static_cast<MockAudioProcessing*>( - audio_state->audio_processing()); - if (ap) { - EXPECT_CALL(*ap, set_stream_delay_ms(0)); - EXPECT_CALL(*ap, set_stream_key_pressed(false)); - EXPECT_CALL(*ap, ProcessStream(_, _, _, Matcher<int16_t*>(_))); - } +TEST_P(AudioStateTest, RecordedAudioArrivesAtSingleStream) { + ConfigHelper helper(GetParam()); - constexpr int kSampleRate = 16000; - constexpr size_t kNumChannels = 2; - auto audio_data = Create10msTestData(kSampleRate, kNumChannels); - uint32_t new_mic_level = 667; - audio_state->audio_transport()->RecordedDataIsAvailable( - &audio_data[0], kSampleRate / 100, kNumChannels * 2, kNumChannels, - kSampleRate, 0, 0, 0, false, new_mic_level); - EXPECT_EQ(667u, new_mic_level); + if (GetParam().use_async_audio_processing) { + EXPECT_CALL(helper.mock_audio_frame_processor(), SinkSet); + EXPECT_CALL(helper.mock_audio_frame_processor(), ProcessCalled); + EXPECT_CALL(helper.mock_audio_frame_processor(), SinkCleared); + } - audio_state->RemoveSendingStream(&stream); + rtc::scoped_refptr<internal::AudioState> audio_state( + rtc::make_ref_counted<internal::AudioState>(helper.config())); + + MockAudioSendStream stream; + audio_state->AddSendingStream(&stream, 8000, 2); + + EXPECT_CALL( + stream, + SendAudioDataForMock(::testing::AllOf( + ::testing::Field(&AudioFrame::sample_rate_hz_, ::testing::Eq(8000)), + ::testing::Field(&AudioFrame::num_channels_, ::testing::Eq(2u))))) + .WillOnce( + // Verify that channels are not swapped by default. + ::testing::Invoke([](AudioFrame* audio_frame) { + auto levels = ComputeChannelLevels(audio_frame); + EXPECT_LT(0u, levels[0]); + EXPECT_EQ(0u, levels[1]); + })); + MockAudioProcessing* ap = + GetParam().use_null_audio_processing + ? nullptr + : static_cast<MockAudioProcessing*>(audio_state->audio_processing()); + if (ap) { + EXPECT_CALL(*ap, set_stream_delay_ms(0)); + EXPECT_CALL(*ap, set_stream_key_pressed(false)); + EXPECT_CALL(*ap, ProcessStream(_, _, _, Matcher<int16_t*>(_))); } + + constexpr int kSampleRate = 16000; + constexpr size_t kNumChannels = 2; + auto audio_data = Create10msTestData(kSampleRate, kNumChannels); + uint32_t new_mic_level = 667; + audio_state->audio_transport()->RecordedDataIsAvailable( + &audio_data[0], kSampleRate / 100, kNumChannels * 2, kNumChannels, + kSampleRate, 0, 0, 0, false, new_mic_level); + EXPECT_EQ(667u, new_mic_level); + + audio_state->RemoveSendingStream(&stream); } -TEST(AudioStateTest, RecordedAudioArrivesAtMultipleStreams) { - for (bool use_null_audio_processing : {false, true}) { - ConfigHelper helper(use_null_audio_processing); - rtc::scoped_refptr<internal::AudioState> audio_state( - new rtc::RefCountedObject<internal::AudioState>(helper.config())); - - MockAudioSendStream stream_1; - MockAudioSendStream stream_2; - audio_state->AddSendingStream(&stream_1, 8001, 2); - audio_state->AddSendingStream(&stream_2, 32000, 1); - - EXPECT_CALL( - stream_1, - SendAudioDataForMock(::testing::AllOf( - ::testing::Field(&AudioFrame::sample_rate_hz_, - ::testing::Eq(16000)), - ::testing::Field(&AudioFrame::num_channels_, ::testing::Eq(1u))))) - .WillOnce( - // Verify that there is output signal. - ::testing::Invoke([](AudioFrame* audio_frame) { - auto levels = ComputeChannelLevels(audio_frame); - EXPECT_LT(0u, levels[0]); - })); - EXPECT_CALL( - stream_2, - SendAudioDataForMock(::testing::AllOf( - ::testing::Field(&AudioFrame::sample_rate_hz_, - ::testing::Eq(16000)), - ::testing::Field(&AudioFrame::num_channels_, ::testing::Eq(1u))))) - .WillOnce( - // Verify that there is output signal. - ::testing::Invoke([](AudioFrame* audio_frame) { - auto levels = ComputeChannelLevels(audio_frame); - EXPECT_LT(0u, levels[0]); - })); - MockAudioProcessing* ap = - static_cast<MockAudioProcessing*>(audio_state->audio_processing()); - if (ap) { - EXPECT_CALL(*ap, set_stream_delay_ms(5)); - EXPECT_CALL(*ap, set_stream_key_pressed(true)); - EXPECT_CALL(*ap, ProcessStream(_, _, _, Matcher<int16_t*>(_))); - } +TEST_P(AudioStateTest, RecordedAudioArrivesAtMultipleStreams) { + ConfigHelper helper(GetParam()); + + if (GetParam().use_async_audio_processing) { + EXPECT_CALL(helper.mock_audio_frame_processor(), SinkSet); + EXPECT_CALL(helper.mock_audio_frame_processor(), ProcessCalled); + EXPECT_CALL(helper.mock_audio_frame_processor(), SinkCleared); + } - constexpr int kSampleRate = 16000; - constexpr size_t kNumChannels = 1; - auto audio_data = Create10msTestData(kSampleRate, kNumChannels); - uint32_t new_mic_level = 667; - audio_state->audio_transport()->RecordedDataIsAvailable( - &audio_data[0], kSampleRate / 100, kNumChannels * 2, kNumChannels, - kSampleRate, 5, 0, 0, true, new_mic_level); - EXPECT_EQ(667u, new_mic_level); - - audio_state->RemoveSendingStream(&stream_1); - audio_state->RemoveSendingStream(&stream_2); + rtc::scoped_refptr<internal::AudioState> audio_state( + rtc::make_ref_counted<internal::AudioState>(helper.config())); + + MockAudioSendStream stream_1; + MockAudioSendStream stream_2; + audio_state->AddSendingStream(&stream_1, 8001, 2); + audio_state->AddSendingStream(&stream_2, 32000, 1); + + EXPECT_CALL( + stream_1, + SendAudioDataForMock(::testing::AllOf( + ::testing::Field(&AudioFrame::sample_rate_hz_, ::testing::Eq(16000)), + ::testing::Field(&AudioFrame::num_channels_, ::testing::Eq(1u))))) + .WillOnce( + // Verify that there is output signal. + ::testing::Invoke([](AudioFrame* audio_frame) { + auto levels = ComputeChannelLevels(audio_frame); + EXPECT_LT(0u, levels[0]); + })); + EXPECT_CALL( + stream_2, + SendAudioDataForMock(::testing::AllOf( + ::testing::Field(&AudioFrame::sample_rate_hz_, ::testing::Eq(16000)), + ::testing::Field(&AudioFrame::num_channels_, ::testing::Eq(1u))))) + .WillOnce( + // Verify that there is output signal. + ::testing::Invoke([](AudioFrame* audio_frame) { + auto levels = ComputeChannelLevels(audio_frame); + EXPECT_LT(0u, levels[0]); + })); + MockAudioProcessing* ap = + static_cast<MockAudioProcessing*>(audio_state->audio_processing()); + if (ap) { + EXPECT_CALL(*ap, set_stream_delay_ms(5)); + EXPECT_CALL(*ap, set_stream_key_pressed(true)); + EXPECT_CALL(*ap, ProcessStream(_, _, _, Matcher<int16_t*>(_))); } + + constexpr int kSampleRate = 16000; + constexpr size_t kNumChannels = 1; + auto audio_data = Create10msTestData(kSampleRate, kNumChannels); + uint32_t new_mic_level = 667; + audio_state->audio_transport()->RecordedDataIsAvailable( + &audio_data[0], kSampleRate / 100, kNumChannels * 2, kNumChannels, + kSampleRate, 5, 0, 0, true, new_mic_level); + EXPECT_EQ(667u, new_mic_level); + + audio_state->RemoveSendingStream(&stream_1); + audio_state->RemoveSendingStream(&stream_2); } -TEST(AudioStateTest, EnableChannelSwap) { +TEST_P(AudioStateTest, EnableChannelSwap) { constexpr int kSampleRate = 16000; constexpr size_t kNumChannels = 2; - for (bool use_null_audio_processing : {false, true}) { - ConfigHelper helper(use_null_audio_processing); - rtc::scoped_refptr<internal::AudioState> audio_state( - new rtc::RefCountedObject<internal::AudioState>(helper.config())); - - audio_state->SetStereoChannelSwapping(true); - - MockAudioSendStream stream; - audio_state->AddSendingStream(&stream, kSampleRate, kNumChannels); - - EXPECT_CALL(stream, SendAudioDataForMock(_)) - .WillOnce( - // Verify that channels are swapped. - ::testing::Invoke([](AudioFrame* audio_frame) { - auto levels = ComputeChannelLevels(audio_frame); - EXPECT_EQ(0u, levels[0]); - EXPECT_LT(0u, levels[1]); - })); - - auto audio_data = Create10msTestData(kSampleRate, kNumChannels); - uint32_t new_mic_level = 667; - audio_state->audio_transport()->RecordedDataIsAvailable( - &audio_data[0], kSampleRate / 100, kNumChannels * 2, kNumChannels, - kSampleRate, 0, 0, 0, false, new_mic_level); - EXPECT_EQ(667u, new_mic_level); - - audio_state->RemoveSendingStream(&stream); + ConfigHelper helper(GetParam()); + + if (GetParam().use_async_audio_processing) { + EXPECT_CALL(helper.mock_audio_frame_processor(), SinkSet); + EXPECT_CALL(helper.mock_audio_frame_processor(), ProcessCalled); + EXPECT_CALL(helper.mock_audio_frame_processor(), SinkCleared); } + + rtc::scoped_refptr<internal::AudioState> audio_state( + rtc::make_ref_counted<internal::AudioState>(helper.config())); + + audio_state->SetStereoChannelSwapping(true); + + MockAudioSendStream stream; + audio_state->AddSendingStream(&stream, kSampleRate, kNumChannels); + + EXPECT_CALL(stream, SendAudioDataForMock(_)) + .WillOnce( + // Verify that channels are swapped. + ::testing::Invoke([](AudioFrame* audio_frame) { + auto levels = ComputeChannelLevels(audio_frame); + EXPECT_EQ(0u, levels[0]); + EXPECT_LT(0u, levels[1]); + })); + + auto audio_data = Create10msTestData(kSampleRate, kNumChannels); + uint32_t new_mic_level = 667; + audio_state->audio_transport()->RecordedDataIsAvailable( + &audio_data[0], kSampleRate / 100, kNumChannels * 2, kNumChannels, + kSampleRate, 0, 0, 0, false, new_mic_level); + EXPECT_EQ(667u, new_mic_level); + + audio_state->RemoveSendingStream(&stream); } -TEST(AudioStateTest, - QueryingTransportForAudioShouldResultInGetAudioCallOnMixerSource) { - for (bool use_null_audio_processing : {false, true}) { - ConfigHelper helper(use_null_audio_processing); - auto audio_state = AudioState::Create(helper.config()); - - FakeAudioSource fake_source; - helper.mixer()->AddSource(&fake_source); - - EXPECT_CALL(fake_source, GetAudioFrameWithInfo(_, _)) - .WillOnce( - ::testing::Invoke([](int sample_rate_hz, AudioFrame* audio_frame) { - audio_frame->sample_rate_hz_ = sample_rate_hz; - audio_frame->samples_per_channel_ = sample_rate_hz / 100; - audio_frame->num_channels_ = kNumberOfChannels; - return AudioMixer::Source::AudioFrameInfo::kNormal; - })); - - int16_t audio_buffer[kSampleRate / 100 * kNumberOfChannels]; - size_t n_samples_out; - int64_t elapsed_time_ms; - int64_t ntp_time_ms; - audio_state->audio_transport()->NeedMorePlayData( - kSampleRate / 100, kNumberOfChannels * 2, kNumberOfChannels, - kSampleRate, audio_buffer, n_samples_out, &elapsed_time_ms, - &ntp_time_ms); - } +TEST_P(AudioStateTest, + QueryingTransportForAudioShouldResultInGetAudioCallOnMixerSource) { + ConfigHelper helper(GetParam()); + auto audio_state = AudioState::Create(helper.config()); + + FakeAudioSource fake_source; + helper.mixer()->AddSource(&fake_source); + + EXPECT_CALL(fake_source, GetAudioFrameWithInfo(_, _)) + .WillOnce( + ::testing::Invoke([](int sample_rate_hz, AudioFrame* audio_frame) { + audio_frame->sample_rate_hz_ = sample_rate_hz; + audio_frame->samples_per_channel_ = sample_rate_hz / 100; + audio_frame->num_channels_ = kNumberOfChannels; + return AudioMixer::Source::AudioFrameInfo::kNormal; + })); + + int16_t audio_buffer[kSampleRate / 100 * kNumberOfChannels]; + size_t n_samples_out; + int64_t elapsed_time_ms; + int64_t ntp_time_ms; + audio_state->audio_transport()->NeedMorePlayData( + kSampleRate / 100, kNumberOfChannels * 2, kNumberOfChannels, kSampleRate, + audio_buffer, n_samples_out, &elapsed_time_ms, &ntp_time_ms); } + +INSTANTIATE_TEST_SUITE_P(AudioStateTest, + AudioStateTest, + Values(ConfigHelper::Params({false, false}), + ConfigHelper::Params({true, false}), + ConfigHelper::Params({false, true}), + ConfigHelper::Params({true, true}))); + } // namespace test } // namespace webrtc diff --git a/audio/audio_transport_impl.cc b/audio/audio_transport_impl.cc index 11b37ffcf1..9f2823bcda 100644 --- a/audio/audio_transport_impl.cc +++ b/audio/audio_transport_impl.cc @@ -17,8 +17,10 @@ #include "audio/remix_resample.h" #include "audio/utility/audio_frame_operations.h" #include "call/audio_sender.h" +#include "modules/async_audio_processing/async_audio_processing.h" #include "modules/audio_processing/include/audio_frame_proxies.h" #include "rtc_base/checks.h" +#include "rtc_base/trace_event.h" namespace webrtc { @@ -63,12 +65,14 @@ void ProcessCaptureFrame(uint32_t delay_ms, } } -// Resample audio in |frame| to given sample rate preserving the -// channel count and place the result in |destination|. +// Resample audio in `frame` to given sample rate preserving the +// channel count and place the result in `destination`. int Resample(const AudioFrame& frame, const int destination_sample_rate, PushResampler<int16_t>* resampler, int16_t* destination) { + TRACE_EVENT2("webrtc", "Resample", "frame sample rate", frame.sample_rate_hz_, + "destination_sample_rate", destination_sample_rate); const int number_of_channels = static_cast<int>(frame.num_channels_); const int target_number_of_samples_per_channel = destination_sample_rate / 100; @@ -83,14 +87,41 @@ int Resample(const AudioFrame& frame, } } // namespace -AudioTransportImpl::AudioTransportImpl(AudioMixer* mixer, - AudioProcessing* audio_processing) - : audio_processing_(audio_processing), mixer_(mixer) { +AudioTransportImpl::AudioTransportImpl( + AudioMixer* mixer, + AudioProcessing* audio_processing, + AsyncAudioProcessing::Factory* async_audio_processing_factory) + : audio_processing_(audio_processing), + async_audio_processing_( + async_audio_processing_factory + ? async_audio_processing_factory->CreateAsyncAudioProcessing( + [this](std::unique_ptr<AudioFrame> frame) { + this->SendProcessedData(std::move(frame)); + }) + : nullptr), + mixer_(mixer) { RTC_DCHECK(mixer); } AudioTransportImpl::~AudioTransportImpl() {} +int32_t AudioTransportImpl::RecordedDataIsAvailable( + const void* audio_data, + const size_t number_of_frames, + const size_t bytes_per_sample, + const size_t number_of_channels, + const uint32_t sample_rate, + const uint32_t audio_delay_milliseconds, + const int32_t clock_drift, + const uint32_t volume, + const bool key_pressed, + uint32_t& new_mic_volume) { // NOLINT: to avoid changing APIs + return RecordedDataIsAvailable( + audio_data, number_of_frames, bytes_per_sample, number_of_channels, + sample_rate, audio_delay_milliseconds, clock_drift, volume, key_pressed, + new_mic_volume, /* estimated_capture_time_ns */ 0); +} + // Not used in Chromium. Process captured audio and distribute to all sending // streams, and try to do this at the lowest possible sample rate. int32_t AudioTransportImpl::RecordedDataIsAvailable( @@ -103,7 +134,9 @@ int32_t AudioTransportImpl::RecordedDataIsAvailable( const int32_t /*clock_drift*/, const uint32_t /*volume*/, const bool key_pressed, - uint32_t& /*new_mic_volume*/) { // NOLINT: to avoid changing APIs + uint32_t& /*new_mic_volume*/, + const int64_t + estimated_capture_time_ns) { // NOLINT: to avoid changing APIs RTC_DCHECK(audio_data); RTC_DCHECK_GE(number_of_channels, 1); RTC_DCHECK_LE(number_of_channels, 2); @@ -133,41 +166,36 @@ int32_t AudioTransportImpl::RecordedDataIsAvailable( ProcessCaptureFrame(audio_delay_milliseconds, key_pressed, swap_stereo_channels, audio_processing_, audio_frame.get()); + audio_frame->set_absolute_capture_timestamp_ms(estimated_capture_time_ns / + 1000000); - // Typing detection (utilizes the APM/VAD decision). We let the VAD determine - // if we're using this feature or not. - // TODO(solenberg): GetConfig() takes a lock. Work around that. - bool typing_detected = false; - if (audio_processing_ && - audio_processing_->GetConfig().voice_detection.enabled) { - if (audio_frame->vad_activity_ != AudioFrame::kVadUnknown) { - bool vad_active = audio_frame->vad_activity_ == AudioFrame::kVadActive; - typing_detected = typing_detection_.Process(key_pressed, vad_active); - } - } - - // Copy frame and push to each sending stream. The copy is required since an - // encoding task will be posted internally to each stream. - { - MutexLock lock(&capture_lock_); - typing_noise_detected_ = typing_detected; - - RTC_DCHECK_GT(audio_frame->samples_per_channel_, 0); - if (!audio_senders_.empty()) { - auto it = audio_senders_.begin(); - while (++it != audio_senders_.end()) { - std::unique_ptr<AudioFrame> audio_frame_copy(new AudioFrame()); - audio_frame_copy->CopyFrom(*audio_frame); - (*it)->SendAudioData(std::move(audio_frame_copy)); - } - // Send the original frame to the first stream w/o copying. - (*audio_senders_.begin())->SendAudioData(std::move(audio_frame)); - } - } + RTC_DCHECK_GT(audio_frame->samples_per_channel_, 0); + if (async_audio_processing_) + async_audio_processing_->Process(std::move(audio_frame)); + else + SendProcessedData(std::move(audio_frame)); return 0; } +void AudioTransportImpl::SendProcessedData( + std::unique_ptr<AudioFrame> audio_frame) { + TRACE_EVENT0("webrtc", "AudioTransportImpl::SendProcessedData"); + RTC_DCHECK_GT(audio_frame->samples_per_channel_, 0); + MutexLock lock(&capture_lock_); + if (audio_senders_.empty()) + return; + + auto it = audio_senders_.begin(); + while (++it != audio_senders_.end()) { + auto audio_frame_copy = std::make_unique<AudioFrame>(); + audio_frame_copy->CopyFrom(*audio_frame); + (*it)->SendAudioData(std::move(audio_frame_copy)); + } + // Send the original frame to the first stream w/o copying. + (*audio_senders_.begin())->SendAudioData(std::move(audio_frame)); +} + // Mix all received streams, feed the result to the AudioProcessing module, then // resample the result to the requested output rate. int32_t AudioTransportImpl::NeedMorePlayData(const size_t nSamples, @@ -178,6 +206,7 @@ int32_t AudioTransportImpl::NeedMorePlayData(const size_t nSamples, size_t& nSamplesOut, int64_t* elapsed_time_ms, int64_t* ntp_time_ms) { + TRACE_EVENT0("webrtc", "AudioTransportImpl::SendProcessedData"); RTC_DCHECK_EQ(sizeof(int16_t) * nChannels, nBytesPerSample); RTC_DCHECK_GE(nChannels, 1); RTC_DCHECK_LE(nChannels, 2); @@ -215,6 +244,8 @@ void AudioTransportImpl::PullRenderData(int bits_per_sample, void* audio_data, int64_t* elapsed_time_ms, int64_t* ntp_time_ms) { + TRACE_EVENT2("webrtc", "AudioTransportImpl::PullRenderData", "sample_rate", + sample_rate, "number_of_frames", number_of_frames); RTC_DCHECK_EQ(bits_per_sample, 16); RTC_DCHECK_GE(number_of_channels, 1); RTC_DCHECK_GE(sample_rate, AudioProcessing::NativeRate::kSampleRate8kHz); @@ -248,8 +279,4 @@ void AudioTransportImpl::SetStereoChannelSwapping(bool enable) { swap_stereo_channels_ = enable; } -bool AudioTransportImpl::typing_noise_detected() const { - MutexLock lock(&capture_lock_); - return typing_noise_detected_; -} } // namespace webrtc diff --git a/audio/audio_transport_impl.h b/audio/audio_transport_impl.h index 1643a29970..ba067de99d 100644 --- a/audio/audio_transport_impl.h +++ b/audio/audio_transport_impl.h @@ -11,15 +11,15 @@ #ifndef AUDIO_AUDIO_TRANSPORT_IMPL_H_ #define AUDIO_AUDIO_TRANSPORT_IMPL_H_ +#include <memory> #include <vector> #include "api/audio/audio_mixer.h" #include "api/scoped_refptr.h" #include "common_audio/resampler/include/push_resampler.h" +#include "modules/async_audio_processing/async_audio_processing.h" #include "modules/audio_device/include/audio_device.h" #include "modules/audio_processing/include/audio_processing.h" -#include "modules/audio_processing/typing_detection.h" -#include "rtc_base/constructor_magic.h" #include "rtc_base/synchronization/mutex.h" #include "rtc_base/thread_annotations.h" @@ -29,24 +29,45 @@ class AudioSender; class AudioTransportImpl : public AudioTransport { public: - AudioTransportImpl(AudioMixer* mixer, AudioProcessing* audio_processing); + AudioTransportImpl( + AudioMixer* mixer, + AudioProcessing* audio_processing, + AsyncAudioProcessing::Factory* async_audio_processing_factory); + + AudioTransportImpl() = delete; + AudioTransportImpl(const AudioTransportImpl&) = delete; + AudioTransportImpl& operator=(const AudioTransportImpl&) = delete; + ~AudioTransportImpl() override; + // TODO(bugs.webrtc.org/13620) Deprecate this function int32_t RecordedDataIsAvailable(const void* audioSamples, - const size_t nSamples, - const size_t nBytesPerSample, - const size_t nChannels, - const uint32_t samplesPerSec, - const uint32_t totalDelayMS, - const int32_t clockDrift, - const uint32_t currentMicLevel, - const bool keyPressed, + size_t nSamples, + size_t nBytesPerSample, + size_t nChannels, + uint32_t samplesPerSec, + uint32_t totalDelayMS, + int32_t clockDrift, + uint32_t currentMicLevel, + bool keyPressed, uint32_t& newMicLevel) override; - int32_t NeedMorePlayData(const size_t nSamples, - const size_t nBytesPerSample, - const size_t nChannels, - const uint32_t samplesPerSec, + int32_t RecordedDataIsAvailable(const void* audioSamples, + size_t nSamples, + size_t nBytesPerSample, + size_t nChannels, + uint32_t samplesPerSec, + uint32_t totalDelayMS, + int32_t clockDrift, + uint32_t currentMicLevel, + bool keyPressed, + uint32_t& newMicLevel, + int64_t estimated_capture_time_ns) override; + + int32_t NeedMorePlayData(size_t nSamples, + size_t nBytesPerSample, + size_t nChannels, + uint32_t samplesPerSec, void* audioSamples, size_t& nSamplesOut, int64_t* elapsed_time_ms, @@ -64,29 +85,31 @@ class AudioTransportImpl : public AudioTransport { int send_sample_rate_hz, size_t send_num_channels); void SetStereoChannelSwapping(bool enable); - bool typing_noise_detected() const; private: + void SendProcessedData(std::unique_ptr<AudioFrame> audio_frame); + // Shared. AudioProcessing* audio_processing_ = nullptr; // Capture side. + + // Thread-safe. + const std::unique_ptr<AsyncAudioProcessing> async_audio_processing_; + mutable Mutex capture_lock_; std::vector<AudioSender*> audio_senders_ RTC_GUARDED_BY(capture_lock_); int send_sample_rate_hz_ RTC_GUARDED_BY(capture_lock_) = 8000; size_t send_num_channels_ RTC_GUARDED_BY(capture_lock_) = 1; - bool typing_noise_detected_ RTC_GUARDED_BY(capture_lock_) = false; bool swap_stereo_channels_ RTC_GUARDED_BY(capture_lock_) = false; PushResampler<int16_t> capture_resampler_; - TypingDetection typing_detection_; // Render side. + rtc::scoped_refptr<AudioMixer> mixer_; AudioFrame mixed_frame_; // Converts mixed audio to the audio device output rate. PushResampler<int16_t> render_resampler_; - - RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(AudioTransportImpl); }; } // namespace webrtc diff --git a/audio/channel_receive.cc b/audio/channel_receive.cc index 9cbaabbbb0..363eebf0a1 100644 --- a/audio/channel_receive.cc +++ b/audio/channel_receive.cc @@ -10,8 +10,6 @@ #include "audio/channel_receive.h" -#include <assert.h> - #include <algorithm> #include <map> #include <memory> @@ -22,6 +20,10 @@ #include "api/crypto/frame_decryptor_interface.h" #include "api/frame_transformer_interface.h" #include "api/rtc_event_log/rtc_event_log.h" +#include "api/sequence_checker.h" +#include "api/task_queue/pending_task_safety_flag.h" +#include "api/task_queue/task_queue_base.h" +#include "api/units/time_delta.h" #include "audio/audio_level.h" #include "audio/channel_receive_frame_transformer_delegate.h" #include "audio/channel_send.h" @@ -33,22 +35,22 @@ #include "modules/pacing/packet_router.h" #include "modules/rtp_rtcp/include/receive_statistics.h" #include "modules/rtp_rtcp/include/remote_ntp_time_estimator.h" -#include "modules/rtp_rtcp/source/absolute_capture_time_receiver.h" +#include "modules/rtp_rtcp/source/absolute_capture_time_interpolator.h" +#include "modules/rtp_rtcp/source/capture_clock_offset_updater.h" #include "modules/rtp_rtcp/source/rtp_header_extensions.h" #include "modules/rtp_rtcp/source/rtp_packet_received.h" #include "modules/rtp_rtcp/source/rtp_rtcp_config.h" #include "modules/rtp_rtcp/source/rtp_rtcp_impl2.h" -#include "modules/utility/include/process_thread.h" #include "rtc_base/checks.h" -#include "rtc_base/format_macros.h" -#include "rtc_base/location.h" #include "rtc_base/logging.h" #include "rtc_base/numerics/safe_minmax.h" #include "rtc_base/race_checker.h" #include "rtc_base/synchronization/mutex.h" -#include "rtc_base/thread_checker.h" +#include "rtc_base/system/no_unique_address.h" #include "rtc_base/time_utils.h" +#include "rtc_base/trace_event.h" #include "system_wrappers/include/metrics.h" +#include "system_wrappers/include/ntp_time.h" namespace webrtc { namespace voe { @@ -78,12 +80,12 @@ AudioCodingModule::Config AcmConfig( return acm_config; } -class ChannelReceive : public ChannelReceiveInterface { +class ChannelReceive : public ChannelReceiveInterface, + public RtcpPacketTypeCounterObserver { public: // Used for receive streams. ChannelReceive( Clock* clock, - ProcessThread* module_process_thread, NetEqFactory* neteq_factory, AudioDeviceModule* audio_device_module, Transport* rtcp_send_transport, @@ -93,7 +95,7 @@ class ChannelReceive : public ChannelReceiveInterface { size_t jitter_buffer_max_packets, bool jitter_buffer_fast_playout, int jitter_buffer_min_delay_ms, - bool jitter_buffer_enable_rtx_handling, + bool enable_non_sender_rtt, rtc::scoped_refptr<AudioDecoderFactory> decoder_factory, absl::optional<AudioCodecPairId> codec_pair_id, rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor, @@ -128,12 +130,13 @@ class ChannelReceive : public ChannelReceiveInterface { double GetTotalOutputDuration() const override; // Stats. - NetworkStatistics GetNetworkStatistics() const override; + NetworkStatistics GetNetworkStatistics( + bool get_and_clear_legacy_stats) const override; AudioDecodingCallStats GetDecodingCallStatistics() const override; // Audio+Video Sync. uint32_t GetDelayEstimate() const override; - void SetMinimumPlayoutDelay(int delayMs) override; + bool SetMinimumPlayoutDelay(int delayMs) override; bool GetPlayoutRtpTimestamp(uint32_t* rtp_timestamp, int64_t* time_ms) const override; void SetEstimatedPlayoutNtpTimestampMs(int64_t ntp_timestamp_ms, @@ -154,6 +157,7 @@ class ChannelReceive : public ChannelReceiveInterface { CallReceiveStatistics GetRTCPStatistics() const override; void SetNACKStatus(bool enable, int maxNumberOfPackets) override; + void SetNonSenderRttMeasurement(bool enabled) override; AudioMixer::Source::AudioFrameInfo GetAudioFrameWithInfo( int sample_rate_hz, @@ -161,6 +165,8 @@ class ChannelReceive : public ChannelReceiveInterface { int PreferredSampleRate() const override; + void SetSourceTracker(SourceTracker* source_tracker) override; + // Associate to a send channel. // Used for obtaining RTT for a receive-only channel. void SetAssociatedSendChannel(const ChannelSendInterface* channel) override; @@ -171,44 +177,54 @@ class ChannelReceive : public ChannelReceiveInterface { rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer) override; + void SetFrameDecryptor(rtc::scoped_refptr<webrtc::FrameDecryptorInterface> + frame_decryptor) override; + + void OnLocalSsrcChange(uint32_t local_ssrc) override; + uint32_t GetLocalSsrc() const override; + + void RtcpPacketTypesCounterUpdated( + uint32_t ssrc, + const RtcpPacketTypeCounter& packet_counter) override; + private: void ReceivePacket(const uint8_t* packet, size_t packet_length, - const RTPHeader& header); + const RTPHeader& header) + RTC_RUN_ON(worker_thread_checker_); int ResendPackets(const uint16_t* sequence_numbers, int length); - void UpdatePlayoutTimestamp(bool rtcp, int64_t now_ms); + void UpdatePlayoutTimestamp(bool rtcp, int64_t now_ms) + RTC_RUN_ON(worker_thread_checker_); int GetRtpTimestampRateHz() const; - int64_t GetRTT() const; void OnReceivedPayloadData(rtc::ArrayView<const uint8_t> payload, - const RTPHeader& rtpHeader); + const RTPHeader& rtpHeader) + RTC_RUN_ON(worker_thread_checker_); void InitFrameTransformerDelegate( - rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer); - - bool Playing() const { - MutexLock lock(&playing_lock_); - return playing_; - } + rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer) + RTC_RUN_ON(worker_thread_checker_); // Thread checkers document and lock usage of some methods to specific threads // we know about. The goal is to eventually split up voe::ChannelReceive into // parts with single-threaded semantics, and thereby reduce the need for // locks. - rtc::ThreadChecker worker_thread_checker_; - rtc::ThreadChecker module_process_thread_checker_; + RTC_NO_UNIQUE_ADDRESS SequenceChecker worker_thread_checker_; + RTC_NO_UNIQUE_ADDRESS SequenceChecker network_thread_checker_; + + TaskQueueBase* const worker_thread_; + ScopedTaskSafety worker_safety_; + // Methods accessed from audio and video threads are checked for sequential- // only access. We don't necessarily own and control these threads, so thread // checkers cannot be used. E.g. Chromium may transfer "ownership" from one // audio thread to another, but access is still sequential. rtc::RaceChecker audio_thread_race_checker_; - rtc::RaceChecker video_capture_thread_race_checker_; Mutex callback_mutex_; Mutex volume_settings_mutex_; - mutable Mutex playing_lock_; - bool playing_ RTC_GUARDED_BY(&playing_lock_) = false; + bool playing_ RTC_GUARDED_BY(worker_thread_checker_) = false; RtcEventLog* const event_log_; @@ -218,34 +234,34 @@ class ChannelReceive : public ChannelReceiveInterface { std::unique_ptr<ReceiveStatistics> rtp_receive_statistics_; std::unique_ptr<ModuleRtpRtcpImpl2> rtp_rtcp_; const uint32_t remote_ssrc_; + SourceTracker* source_tracker_ = nullptr; // Info for GetSyncInfo is updated on network or worker thread, and queried on // the worker thread. - mutable Mutex sync_info_lock_; absl::optional<uint32_t> last_received_rtp_timestamp_ - RTC_GUARDED_BY(&sync_info_lock_); + RTC_GUARDED_BY(&worker_thread_checker_); absl::optional<int64_t> last_received_rtp_system_time_ms_ - RTC_GUARDED_BY(&sync_info_lock_); + RTC_GUARDED_BY(&worker_thread_checker_); // The AcmReceiver is thread safe, using its own lock. acm2::AcmReceiver acm_receiver_; AudioSinkInterface* audio_sink_ = nullptr; AudioLevel _outputAudioLevel; + Clock* const clock_; RemoteNtpTimeEstimator ntp_estimator_ RTC_GUARDED_BY(ts_stats_lock_); // Timestamp of the audio pulled from NetEq. absl::optional<uint32_t> jitter_buffer_playout_timestamp_; - mutable Mutex video_sync_lock_; - uint32_t playout_timestamp_rtp_ RTC_GUARDED_BY(video_sync_lock_); + uint32_t playout_timestamp_rtp_ RTC_GUARDED_BY(worker_thread_checker_); absl::optional<int64_t> playout_timestamp_rtp_time_ms_ - RTC_GUARDED_BY(video_sync_lock_); - uint32_t playout_delay_ms_ RTC_GUARDED_BY(video_sync_lock_); + RTC_GUARDED_BY(worker_thread_checker_); + uint32_t playout_delay_ms_ RTC_GUARDED_BY(worker_thread_checker_); absl::optional<int64_t> playout_timestamp_ntp_ - RTC_GUARDED_BY(video_sync_lock_); + RTC_GUARDED_BY(worker_thread_checker_); absl::optional<int64_t> playout_timestamp_ntp_time_ms_ - RTC_GUARDED_BY(video_sync_lock_); + RTC_GUARDED_BY(worker_thread_checker_); mutable Mutex ts_stats_lock_; @@ -256,36 +272,64 @@ class ChannelReceive : public ChannelReceiveInterface { // frame. int64_t capture_start_ntp_time_ms_ RTC_GUARDED_BY(ts_stats_lock_); - // uses - ProcessThread* _moduleProcessThreadPtr; AudioDeviceModule* _audioDeviceModulePtr; float _outputGain RTC_GUARDED_BY(volume_settings_mutex_); - // An associated send channel. - mutable Mutex assoc_send_channel_lock_; const ChannelSendInterface* associated_send_channel_ - RTC_GUARDED_BY(assoc_send_channel_lock_); + RTC_GUARDED_BY(network_thread_checker_); PacketRouter* packet_router_ = nullptr; - rtc::ThreadChecker construction_thread_; + SequenceChecker construction_thread_; // E2EE Audio Frame Decryption - rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor_; + rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor_ + RTC_GUARDED_BY(worker_thread_checker_); webrtc::CryptoOptions crypto_options_; - webrtc::AbsoluteCaptureTimeReceiver absolute_capture_time_receiver_; + webrtc::AbsoluteCaptureTimeInterpolator absolute_capture_time_interpolator_ + RTC_GUARDED_BY(worker_thread_checker_); + + webrtc::CaptureClockOffsetUpdater capture_clock_offset_updater_; rtc::scoped_refptr<ChannelReceiveFrameTransformerDelegate> frame_transformer_delegate_; + + // Counter that's used to control the frequency of reporting histograms + // from the `GetAudioFrameWithInfo` callback. + int audio_frame_interval_count_ RTC_GUARDED_BY(audio_thread_race_checker_) = + 0; + // Controls how many callbacks we let pass by before reporting callback stats. + // A value of 100 means 100 callbacks, each one of which represents 10ms worth + // of data, so the stats reporting frequency will be 1Hz (modulo failures). + constexpr static int kHistogramReportingInterval = 100; + + mutable Mutex rtcp_counter_mutex_; + RtcpPacketTypeCounter rtcp_packet_type_counter_ + RTC_GUARDED_BY(rtcp_counter_mutex_); }; void ChannelReceive::OnReceivedPayloadData( rtc::ArrayView<const uint8_t> payload, const RTPHeader& rtpHeader) { - if (!Playing()) { + if (!playing_) { // Avoid inserting into NetEQ when we are not playing. Count the // packet as discarded. + + // If we have a source_tracker_, tell it that the frame has been + // "delivered". Normally, this happens in AudioReceiveStreamInterface when + // audio frames are pulled out, but when playout is muted, nothing is + // pulling frames. The downside of this approach is that frames delivered + // this way won't be delayed for playout, and therefore will be + // unsynchronized with (a) audio delay when playing and (b) any audio/video + // synchronization. But the alternative is that muting playout also stops + // the SourceTracker from updating RtpSource information. + if (source_tracker_) { + RtpPacketInfos::vector_type packet_vector = { + RtpPacketInfo(rtpHeader, clock_->CurrentTime())}; + source_tracker_->OnFrameDelivered(RtpPacketInfos(packet_vector)); + } + return; } @@ -297,7 +341,8 @@ void ChannelReceive::OnReceivedPayloadData( } int64_t round_trip_time = 0; - rtp_rtcp_->RTT(remote_ssrc_, &round_trip_time, NULL, NULL, NULL); + rtp_rtcp_->RTT(remote_ssrc_, &round_trip_time, /*avg_rtt=*/nullptr, + /*min_rtt=*/nullptr, /*max_rtt=*/nullptr); std::vector<uint16_t> nack_list = acm_receiver_.GetNackList(round_trip_time); if (!nack_list.empty()) { @@ -311,24 +356,28 @@ void ChannelReceive::InitFrameTransformerDelegate( rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer) { RTC_DCHECK(frame_transformer); RTC_DCHECK(!frame_transformer_delegate_); + RTC_DCHECK(worker_thread_->IsCurrent()); // Pass a callback to ChannelReceive::OnReceivedPayloadData, to be called by // the delegate to receive transformed audio. ChannelReceiveFrameTransformerDelegate::ReceiveFrameCallback receive_audio_callback = [this](rtc::ArrayView<const uint8_t> packet, const RTPHeader& header) { + RTC_DCHECK_RUN_ON(&worker_thread_checker_); OnReceivedPayloadData(packet, header); }; frame_transformer_delegate_ = - new rtc::RefCountedObject<ChannelReceiveFrameTransformerDelegate>( + rtc::make_ref_counted<ChannelReceiveFrameTransformerDelegate>( std::move(receive_audio_callback), std::move(frame_transformer), - rtc::Thread::Current()); + worker_thread_); frame_transformer_delegate_->Init(); } AudioMixer::Source::AudioFrameInfo ChannelReceive::GetAudioFrameWithInfo( int sample_rate_hz, AudioFrame* audio_frame) { + TRACE_EVENT_BEGIN1("webrtc", "ChannelReceive::GetAudioFrameWithInfo", + "sample_rate_hz", sample_rate_hz); RTC_DCHECK_RUNS_SERIALIZED(&audio_thread_race_checker_); audio_frame->sample_rate_hz_ = sample_rate_hz; @@ -344,6 +393,9 @@ AudioMixer::Source::AudioFrameInfo ChannelReceive::GetAudioFrameWithInfo( // error so that the audio mixer module doesn't add it to the mix. As // a result, it won't be played out and the actions skipped here are // irrelevant. + + TRACE_EVENT_END1("webrtc", "ChannelReceive::GetAudioFrameWithInfo", "error", + 1); return AudioMixer::Source::AudioFrameInfo::kError; } @@ -382,8 +434,8 @@ AudioMixer::Source::AudioFrameInfo ChannelReceive::GetAudioFrameWithInfo( } // Measure audio level (0-9) - // TODO(henrik.lundin) Use the |muted| information here too. - // TODO(deadbeef): Use RmsLevel for |_outputAudioLevel| (see + // TODO(henrik.lundin) Use the `muted` information here too. + // TODO(deadbeef): Use RmsLevel for `_outputAudioLevel` (see // https://crbug.com/webrtc/7517). _outputAudioLevel.ComputeLevel(*audio_frame, kAudioSampleDurationSeconds); @@ -394,7 +446,6 @@ AudioMixer::Source::AudioFrameInfo ChannelReceive::GetAudioFrameWithInfo( if (capture_start_rtp_time_stamp_ >= 0) { // audio_frame.timestamp_ should be valid from now on. - // Compute elapsed time. int64_t unwrap_timestamp = rtp_ts_wraparound_handler_->Unwrap(audio_frame->timestamp_); @@ -407,29 +458,56 @@ AudioMixer::Source::AudioFrameInfo ChannelReceive::GetAudioFrameWithInfo( // Compute ntp time. audio_frame->ntp_time_ms_ = ntp_estimator_.Estimate(audio_frame->timestamp_); - // |ntp_time_ms_| won't be valid until at least 2 RTCP SRs are received. + // `ntp_time_ms_` won't be valid until at least 2 RTCP SRs are received. if (audio_frame->ntp_time_ms_ > 0) { - // Compute |capture_start_ntp_time_ms_| so that - // |capture_start_ntp_time_ms_| + |elapsed_time_ms_| == |ntp_time_ms_| + // Compute `capture_start_ntp_time_ms_` so that + // `capture_start_ntp_time_ms_` + `elapsed_time_ms_` == `ntp_time_ms_` capture_start_ntp_time_ms_ = audio_frame->ntp_time_ms_ - audio_frame->elapsed_time_ms_; } } } - { - RTC_HISTOGRAM_COUNTS_1000("WebRTC.Audio.TargetJitterBufferDelayMs", - acm_receiver_.TargetDelayMs()); - const int jitter_buffer_delay = acm_receiver_.FilteredCurrentDelayMs(); - MutexLock lock(&video_sync_lock_); - RTC_HISTOGRAM_COUNTS_1000("WebRTC.Audio.ReceiverDelayEstimateMs", - jitter_buffer_delay + playout_delay_ms_); - RTC_HISTOGRAM_COUNTS_1000("WebRTC.Audio.ReceiverJitterBufferDelayMs", - jitter_buffer_delay); - RTC_HISTOGRAM_COUNTS_1000("WebRTC.Audio.ReceiverDeviceDelayMs", - playout_delay_ms_); + // Fill in local capture clock offset in `audio_frame->packet_infos_`. + RtpPacketInfos::vector_type packet_infos; + for (auto& packet_info : audio_frame->packet_infos_) { + absl::optional<int64_t> local_capture_clock_offset_q32x32; + if (packet_info.absolute_capture_time().has_value()) { + local_capture_clock_offset_q32x32 = + capture_clock_offset_updater_.AdjustEstimatedCaptureClockOffset( + packet_info.absolute_capture_time() + ->estimated_capture_clock_offset); + } + RtpPacketInfo new_packet_info(packet_info); + absl::optional<TimeDelta> local_capture_clock_offset; + if (local_capture_clock_offset_q32x32.has_value()) { + local_capture_clock_offset = TimeDelta::Millis( + UQ32x32ToInt64Ms(*local_capture_clock_offset_q32x32)); + } + new_packet_info.set_local_capture_clock_offset(local_capture_clock_offset); + packet_infos.push_back(std::move(new_packet_info)); + } + audio_frame->packet_infos_ = RtpPacketInfos(packet_infos); + + ++audio_frame_interval_count_; + if (audio_frame_interval_count_ >= kHistogramReportingInterval) { + audio_frame_interval_count_ = 0; + worker_thread_->PostTask(SafeTask(worker_safety_.flag(), [this]() { + RTC_DCHECK_RUN_ON(&worker_thread_checker_); + RTC_HISTOGRAM_COUNTS_1000("WebRTC.Audio.TargetJitterBufferDelayMs", + acm_receiver_.TargetDelayMs()); + const int jitter_buffer_delay = acm_receiver_.FilteredCurrentDelayMs(); + RTC_HISTOGRAM_COUNTS_1000("WebRTC.Audio.ReceiverDelayEstimateMs", + jitter_buffer_delay + playout_delay_ms_); + RTC_HISTOGRAM_COUNTS_1000("WebRTC.Audio.ReceiverJitterBufferDelayMs", + jitter_buffer_delay); + RTC_HISTOGRAM_COUNTS_1000("WebRTC.Audio.ReceiverDeviceDelayMs", + playout_delay_ms_); + })); } + TRACE_EVENT_END2("webrtc", "ChannelReceive::GetAudioFrameWithInfo", "gain", + output_gain, "muted", muted); return muted ? AudioMixer::Source::AudioFrameInfo::kMuted : AudioMixer::Source::AudioFrameInfo::kNormal; } @@ -441,9 +519,12 @@ int ChannelReceive::PreferredSampleRate() const { acm_receiver_.last_output_sample_rate_hz()); } +void ChannelReceive::SetSourceTracker(SourceTracker* source_tracker) { + source_tracker_ = source_tracker; +} + ChannelReceive::ChannelReceive( Clock* clock, - ProcessThread* module_process_thread, NetEqFactory* neteq_factory, AudioDeviceModule* audio_device_module, Transport* rtcp_send_transport, @@ -453,13 +534,14 @@ ChannelReceive::ChannelReceive( size_t jitter_buffer_max_packets, bool jitter_buffer_fast_playout, int jitter_buffer_min_delay_ms, - bool jitter_buffer_enable_rtx_handling, + bool enable_non_sender_rtt, rtc::scoped_refptr<AudioDecoderFactory> decoder_factory, absl::optional<AudioCodecPairId> codec_pair_id, rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor, const webrtc::CryptoOptions& crypto_options, rtc::scoped_refptr<FrameTransformerInterface> frame_transformer) - : event_log_(rtc_event_log), + : worker_thread_(TaskQueueBase::Current()), + event_log_(rtc_event_log), rtp_receive_statistics_(ReceiveStatistics::Create(clock)), remote_ssrc_(remote_ssrc), acm_receiver_(AcmConfig(neteq_factory, @@ -468,25 +550,23 @@ ChannelReceive::ChannelReceive( jitter_buffer_max_packets, jitter_buffer_fast_playout)), _outputAudioLevel(), + clock_(clock), ntp_estimator_(clock), playout_timestamp_rtp_(0), playout_delay_ms_(0), rtp_ts_wraparound_handler_(new rtc::TimestampWrapAroundHandler()), capture_start_rtp_time_stamp_(-1), capture_start_ntp_time_ms_(-1), - _moduleProcessThreadPtr(module_process_thread), _audioDeviceModulePtr(audio_device_module), _outputGain(1.0f), associated_send_channel_(nullptr), frame_decryptor_(frame_decryptor), crypto_options_(crypto_options), - absolute_capture_time_receiver_(clock) { - // TODO(nisse): Use _moduleProcessThreadPtr instead? - module_process_thread_checker_.Detach(); - - RTC_DCHECK(module_process_thread); + absolute_capture_time_interpolator_(clock) { RTC_DCHECK(audio_device_module); + network_thread_checker_.Detach(); + acm_receiver_.ResetInitialDelay(); acm_receiver_.SetMinimumDelay(0); acm_receiver_.SetMaximumDelay(0); @@ -503,61 +583,55 @@ ChannelReceive::ChannelReceive( configuration.receive_statistics = rtp_receive_statistics_.get(); configuration.event_log = event_log_; configuration.local_media_ssrc = local_ssrc; + configuration.rtcp_packet_type_counter_observer = this; + configuration.non_sender_rtt_measurement = enable_non_sender_rtt; if (frame_transformer) InitFrameTransformerDelegate(std::move(frame_transformer)); rtp_rtcp_ = ModuleRtpRtcpImpl2::Create(configuration); - rtp_rtcp_->SetSendingMediaStatus(false); rtp_rtcp_->SetRemoteSSRC(remote_ssrc_); - _moduleProcessThreadPtr->RegisterModule(rtp_rtcp_.get(), RTC_FROM_HERE); - // Ensure that RTCP is enabled for the created channel. rtp_rtcp_->SetRTCPStatus(RtcpMode::kCompound); } ChannelReceive::~ChannelReceive() { - RTC_DCHECK(construction_thread_.IsCurrent()); + RTC_DCHECK_RUN_ON(&construction_thread_); // Resets the delegate's callback to ChannelReceive::OnReceivedPayloadData. if (frame_transformer_delegate_) frame_transformer_delegate_->Reset(); StopPlayout(); - - if (_moduleProcessThreadPtr) - _moduleProcessThreadPtr->DeRegisterModule(rtp_rtcp_.get()); } void ChannelReceive::SetSink(AudioSinkInterface* sink) { - RTC_DCHECK(worker_thread_checker_.IsCurrent()); + RTC_DCHECK_RUN_ON(&worker_thread_checker_); MutexLock lock(&callback_mutex_); audio_sink_ = sink; } void ChannelReceive::StartPlayout() { - RTC_DCHECK(worker_thread_checker_.IsCurrent()); - MutexLock lock(&playing_lock_); + RTC_DCHECK_RUN_ON(&worker_thread_checker_); playing_ = true; } void ChannelReceive::StopPlayout() { - RTC_DCHECK(worker_thread_checker_.IsCurrent()); - MutexLock lock(&playing_lock_); + RTC_DCHECK_RUN_ON(&worker_thread_checker_); playing_ = false; _outputAudioLevel.ResetLevelFullRange(); } absl::optional<std::pair<int, SdpAudioFormat>> ChannelReceive::GetReceiveCodec() const { - RTC_DCHECK(worker_thread_checker_.IsCurrent()); + RTC_DCHECK_RUN_ON(&worker_thread_checker_); return acm_receiver_.LastDecoder(); } void ChannelReceive::SetReceiveCodecs( const std::map<int, SdpAudioFormat>& codecs) { - RTC_DCHECK(worker_thread_checker_.IsCurrent()); + RTC_DCHECK_RUN_ON(&worker_thread_checker_); for (const auto& kv : codecs) { RTC_DCHECK_GE(kv.second.clockrate_hz, 1000); payload_type_frequencies_[kv.first] = kv.second.clockrate_hz; @@ -565,15 +639,15 @@ void ChannelReceive::SetReceiveCodecs( acm_receiver_.SetCodecs(codecs); } -// May be called on either worker thread or network thread. void ChannelReceive::OnRtpPacket(const RtpPacketReceived& packet) { + RTC_DCHECK_RUN_ON(&worker_thread_checker_); + // TODO(bugs.webrtc.org/11993): Expect to be called exclusively on the + // network thread. Once that's done, the same applies to + // UpdatePlayoutTimestamp and int64_t now_ms = rtc::TimeMillis(); - { - MutexLock lock(&sync_info_lock_); - last_received_rtp_timestamp_ = packet.Timestamp(); - last_received_rtp_system_time_ms_ = now_ms; - } + last_received_rtp_timestamp_ = packet.Timestamp(); + last_received_rtp_system_time_ms_ = now_ms; // Store playout timestamp for the received RTP packet UpdatePlayoutTimestamp(false, now_ms); @@ -581,7 +655,8 @@ void ChannelReceive::OnRtpPacket(const RtpPacketReceived& packet) { const auto& it = payload_type_frequencies_.find(packet.PayloadType()); if (it == payload_type_frequencies_.end()) return; - // TODO(nisse): Set payload_type_frequency earlier, when packet is parsed. + // TODO(bugs.webrtc.org/7135): Set payload_type_frequency earlier, when packet + // is parsed. RtpPacketReceived packet_copy(packet); packet_copy.set_payload_type_frequency(it->second); @@ -592,9 +667,9 @@ void ChannelReceive::OnRtpPacket(const RtpPacketReceived& packet) { // Interpolates absolute capture timestamp RTP header extension. header.extension.absolute_capture_time = - absolute_capture_time_receiver_.OnReceivePacket( - AbsoluteCaptureTimeReceiver::GetSource(header.ssrc, - header.arrOfCSRCs), + absolute_capture_time_interpolator_.OnReceivePacket( + AbsoluteCaptureTimeInterpolator::GetSource(header.ssrc, + header.arrOfCSRCs), header.timestamp, rtc::saturated_cast<uint32_t>(packet_copy.payload_type_frequency()), header.extension.absolute_capture_time); @@ -606,7 +681,7 @@ void ChannelReceive::ReceivePacket(const uint8_t* packet, size_t packet_length, const RTPHeader& header) { const uint8_t* payload = packet + header.headerLength; - assert(packet_length >= header.headerLength); + RTC_DCHECK_GE(packet_length, header.headerLength); size_t payload_length = packet_length - header.headerLength; size_t payload_data_length = payload_length - header.paddingLength; @@ -653,15 +728,20 @@ void ChannelReceive::ReceivePacket(const uint8_t* packet, } } -// May be called on either worker thread or network thread. void ChannelReceive::ReceivedRTCPPacket(const uint8_t* data, size_t length) { + RTC_DCHECK_RUN_ON(&worker_thread_checker_); + // TODO(bugs.webrtc.org/11993): Expect to be called exclusively on the + // network thread. + // Store playout timestamp for the received RTCP packet UpdatePlayoutTimestamp(true, rtc::TimeMillis()); // Deliver RTCP packet to RTP/RTCP module for parsing rtp_rtcp_->IncomingRtcpPacket(data, length); - int64_t rtt = GetRTT(); + int64_t rtt = 0; + rtp_rtcp_->RTT(remote_ssrc_, &rtt, /*avg_rtt=*/nullptr, /*min_rtt=*/nullptr, + /*max_rtt=*/nullptr); if (rtt == 0) { // Waiting for valid RTT. return; @@ -670,42 +750,51 @@ void ChannelReceive::ReceivedRTCPPacket(const uint8_t* data, size_t length) { uint32_t ntp_secs = 0; uint32_t ntp_frac = 0; uint32_t rtp_timestamp = 0; - if (0 != - rtp_rtcp_->RemoteNTP(&ntp_secs, &ntp_frac, NULL, NULL, &rtp_timestamp)) { + if (rtp_rtcp_->RemoteNTP(&ntp_secs, &ntp_frac, + /*rtcp_arrival_time_secs=*/nullptr, + /*rtcp_arrival_time_frac=*/nullptr, + &rtp_timestamp) != 0) { // Waiting for RTCP. return; } { MutexLock lock(&ts_stats_lock_); - ntp_estimator_.UpdateRtcpTimestamp(rtt, ntp_secs, ntp_frac, rtp_timestamp); + ntp_estimator_.UpdateRtcpTimestamp( + TimeDelta::Millis(rtt), NtpTime(ntp_secs, ntp_frac), rtp_timestamp); + absl::optional<int64_t> remote_to_local_clock_offset = + ntp_estimator_.EstimateRemoteToLocalClockOffset(); + if (remote_to_local_clock_offset.has_value()) { + capture_clock_offset_updater_.SetRemoteToLocalClockOffset( + *remote_to_local_clock_offset); + } } } int ChannelReceive::GetSpeechOutputLevelFullRange() const { - RTC_DCHECK(worker_thread_checker_.IsCurrent()); + RTC_DCHECK_RUN_ON(&worker_thread_checker_); return _outputAudioLevel.LevelFullRange(); } double ChannelReceive::GetTotalOutputEnergy() const { - RTC_DCHECK(worker_thread_checker_.IsCurrent()); + RTC_DCHECK_RUN_ON(&worker_thread_checker_); return _outputAudioLevel.TotalEnergy(); } double ChannelReceive::GetTotalOutputDuration() const { - RTC_DCHECK(worker_thread_checker_.IsCurrent()); + RTC_DCHECK_RUN_ON(&worker_thread_checker_); return _outputAudioLevel.TotalDuration(); } void ChannelReceive::SetChannelOutputVolumeScaling(float scaling) { - RTC_DCHECK(worker_thread_checker_.IsCurrent()); + RTC_DCHECK_RUN_ON(&worker_thread_checker_); MutexLock lock(&volume_settings_mutex_); _outputGain = scaling; } void ChannelReceive::RegisterReceiverCongestionControlObjects( PacketRouter* packet_router) { - RTC_DCHECK(worker_thread_checker_.IsCurrent()); + RTC_DCHECK_RUN_ON(&worker_thread_checker_); RTC_DCHECK(packet_router); RTC_DCHECK(!packet_router_); constexpr bool remb_candidate = false; @@ -714,19 +803,18 @@ void ChannelReceive::RegisterReceiverCongestionControlObjects( } void ChannelReceive::ResetReceiverCongestionControlObjects() { - RTC_DCHECK(worker_thread_checker_.IsCurrent()); + RTC_DCHECK_RUN_ON(&worker_thread_checker_); RTC_DCHECK(packet_router_); packet_router_->RemoveReceiveRtpModule(rtp_rtcp_.get()); packet_router_ = nullptr; } CallReceiveStatistics ChannelReceive::GetRTCPStatistics() const { - RTC_DCHECK(worker_thread_checker_.IsCurrent()); - // --- RtcpStatistics + RTC_DCHECK_RUN_ON(&worker_thread_checker_); CallReceiveStatistics stats; - // The jitter statistics is updated for each received RTP packet and is - // based on received packets. + // The jitter statistics is updated for each received RTP packet and is based + // on received packets. RtpReceiveStats rtp_stats; StreamStatistician* statistician = rtp_receive_statistics_->GetStatistician(remote_ssrc_); @@ -737,10 +825,7 @@ CallReceiveStatistics ChannelReceive::GetRTCPStatistics() const { stats.cumulativeLost = rtp_stats.packets_lost; stats.jitterSamples = rtp_stats.jitter; - // --- RTT - stats.rttMs = GetRTT(); - - // --- Data counters + // Data counters. if (statistician) { stats.payload_bytes_rcvd = rtp_stats.packet_counter.payload_bytes; @@ -757,16 +842,44 @@ CallReceiveStatistics ChannelReceive::GetRTCPStatistics() const { stats.last_packet_received_timestamp_ms = absl::nullopt; } - // --- Timestamps + { + MutexLock lock(&rtcp_counter_mutex_); + stats.nacks_sent = rtcp_packet_type_counter_.nack_packets; + } + + // Timestamps. { MutexLock lock(&ts_stats_lock_); stats.capture_start_ntp_time_ms_ = capture_start_ntp_time_ms_; } + + absl::optional<RtpRtcpInterface::SenderReportStats> rtcp_sr_stats = + rtp_rtcp_->GetSenderReportStats(); + if (rtcp_sr_stats.has_value()) { + stats.last_sender_report_timestamp_ms = + rtcp_sr_stats->last_arrival_timestamp.ToMs() - + rtc::kNtpJan1970Millisecs; + stats.last_sender_report_remote_timestamp_ms = + rtcp_sr_stats->last_remote_timestamp.ToMs() - rtc::kNtpJan1970Millisecs; + stats.sender_reports_packets_sent = rtcp_sr_stats->packets_sent; + stats.sender_reports_bytes_sent = rtcp_sr_stats->bytes_sent; + stats.sender_reports_reports_count = rtcp_sr_stats->reports_count; + } + + absl::optional<RtpRtcpInterface::NonSenderRttStats> non_sender_rtt_stats = + rtp_rtcp_->GetNonSenderRttStats(); + if (non_sender_rtt_stats.has_value()) { + stats.round_trip_time = non_sender_rtt_stats->round_trip_time; + stats.round_trip_time_measurements = + non_sender_rtt_stats->round_trip_time_measurements; + stats.total_round_trip_time = non_sender_rtt_stats->total_round_trip_time; + } + return stats; } void ChannelReceive::SetNACKStatus(bool enable, int max_packets) { - RTC_DCHECK(worker_thread_checker_.IsCurrent()); + RTC_DCHECK_RUN_ON(&worker_thread_checker_); // None of these functions can fail. if (enable) { rtp_receive_statistics_->SetMaxReorderingThreshold(max_packets); @@ -778,52 +891,91 @@ void ChannelReceive::SetNACKStatus(bool enable, int max_packets) { } } +void ChannelReceive::SetNonSenderRttMeasurement(bool enabled) { + RTC_DCHECK_RUN_ON(&worker_thread_checker_); + rtp_rtcp_->SetNonSenderRttMeasurement(enabled); +} + // Called when we are missing one or more packets. int ChannelReceive::ResendPackets(const uint16_t* sequence_numbers, int length) { return rtp_rtcp_->SendNACK(sequence_numbers, length); } +void ChannelReceive::RtcpPacketTypesCounterUpdated( + uint32_t ssrc, + const RtcpPacketTypeCounter& packet_counter) { + if (ssrc != remote_ssrc_) { + return; + } + MutexLock lock(&rtcp_counter_mutex_); + rtcp_packet_type_counter_ = packet_counter; +} + void ChannelReceive::SetAssociatedSendChannel( const ChannelSendInterface* channel) { - RTC_DCHECK(worker_thread_checker_.IsCurrent()); - MutexLock lock(&assoc_send_channel_lock_); + RTC_DCHECK_RUN_ON(&network_thread_checker_); associated_send_channel_ = channel; } void ChannelReceive::SetDepacketizerToDecoderFrameTransformer( rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer) { - RTC_DCHECK(worker_thread_checker_.IsCurrent()); + RTC_DCHECK_RUN_ON(&worker_thread_checker_); // Depending on when the channel is created, the transformer might be set // twice. Don't replace the delegate if it was already initialized. - if (!frame_transformer || frame_transformer_delegate_) + if (!frame_transformer || frame_transformer_delegate_) { + RTC_DCHECK_NOTREACHED() << "Not setting the transformer?"; return; + } + InitFrameTransformerDelegate(std::move(frame_transformer)); } -NetworkStatistics ChannelReceive::GetNetworkStatistics() const { - RTC_DCHECK(worker_thread_checker_.IsCurrent()); +void ChannelReceive::SetFrameDecryptor( + rtc::scoped_refptr<webrtc::FrameDecryptorInterface> frame_decryptor) { + // TODO(bugs.webrtc.org/11993): Expect to be called on the network thread. + RTC_DCHECK_RUN_ON(&worker_thread_checker_); + frame_decryptor_ = std::move(frame_decryptor); +} + +void ChannelReceive::OnLocalSsrcChange(uint32_t local_ssrc) { + // TODO(bugs.webrtc.org/11993): Expect to be called on the network thread. + RTC_DCHECK_RUN_ON(&worker_thread_checker_); + rtp_rtcp_->SetLocalSsrc(local_ssrc); +} + +uint32_t ChannelReceive::GetLocalSsrc() const { + // TODO(bugs.webrtc.org/11993): Expect to be called on the network thread. + RTC_DCHECK_RUN_ON(&worker_thread_checker_); + return rtp_rtcp_->local_media_ssrc(); +} + +NetworkStatistics ChannelReceive::GetNetworkStatistics( + bool get_and_clear_legacy_stats) const { + RTC_DCHECK_RUN_ON(&worker_thread_checker_); NetworkStatistics stats; - acm_receiver_.GetNetworkStatistics(&stats); + acm_receiver_.GetNetworkStatistics(&stats, get_and_clear_legacy_stats); return stats; } AudioDecodingCallStats ChannelReceive::GetDecodingCallStatistics() const { - RTC_DCHECK(worker_thread_checker_.IsCurrent()); + RTC_DCHECK_RUN_ON(&worker_thread_checker_); AudioDecodingCallStats stats; acm_receiver_.GetDecodingCallStatistics(&stats); return stats; } uint32_t ChannelReceive::GetDelayEstimate() const { - RTC_DCHECK(worker_thread_checker_.IsCurrent() || - module_process_thread_checker_.IsCurrent()); - MutexLock lock(&video_sync_lock_); + RTC_DCHECK_RUN_ON(&worker_thread_checker_); + // Return the current jitter buffer delay + playout delay. return acm_receiver_.FilteredCurrentDelayMs() + playout_delay_ms_; } -void ChannelReceive::SetMinimumPlayoutDelay(int delay_ms) { - RTC_DCHECK(module_process_thread_checker_.IsCurrent()); +bool ChannelReceive::SetMinimumPlayoutDelay(int delay_ms) { + // TODO(bugs.webrtc.org/11993): This should run on the network thread. + // We get here via RtpStreamsSynchronizer. Once that's done, many (all?) of + // these locks aren't needed. + RTC_DCHECK_RUN_ON(&worker_thread_checker_); // Limit to range accepted by both VoE and ACM, so we're at least getting as // close as possible, instead of failing. delay_ms = rtc::SafeClamp(delay_ms, kVoiceEngineMinMinPlayoutDelayMs, @@ -831,34 +983,31 @@ void ChannelReceive::SetMinimumPlayoutDelay(int delay_ms) { if (acm_receiver_.SetMinimumDelay(delay_ms) != 0) { RTC_DLOG(LS_ERROR) << "SetMinimumPlayoutDelay() failed to set min playout delay"; + return false; } + return true; } bool ChannelReceive::GetPlayoutRtpTimestamp(uint32_t* rtp_timestamp, int64_t* time_ms) const { - RTC_DCHECK_RUNS_SERIALIZED(&video_capture_thread_race_checker_); - { - MutexLock lock(&video_sync_lock_); - if (!playout_timestamp_rtp_time_ms_) - return false; - *rtp_timestamp = playout_timestamp_rtp_; - *time_ms = playout_timestamp_rtp_time_ms_.value(); - return true; - } + RTC_DCHECK_RUN_ON(&worker_thread_checker_); + if (!playout_timestamp_rtp_time_ms_) + return false; + *rtp_timestamp = playout_timestamp_rtp_; + *time_ms = playout_timestamp_rtp_time_ms_.value(); + return true; } void ChannelReceive::SetEstimatedPlayoutNtpTimestampMs(int64_t ntp_timestamp_ms, int64_t time_ms) { - RTC_DCHECK_RUNS_SERIALIZED(&video_capture_thread_race_checker_); - MutexLock lock(&video_sync_lock_); + RTC_DCHECK_RUN_ON(&worker_thread_checker_); playout_timestamp_ntp_ = ntp_timestamp_ms; playout_timestamp_ntp_time_ms_ = time_ms; } absl::optional<int64_t> ChannelReceive::GetCurrentEstimatedPlayoutNtpTimestampMs(int64_t now_ms) const { - RTC_DCHECK(worker_thread_checker_.IsCurrent()); - MutexLock lock(&video_sync_lock_); + RTC_DCHECK_RUN_ON(&worker_thread_checker_); if (!playout_timestamp_ntp_ || !playout_timestamp_ntp_time_ms_) return absl::nullopt; @@ -875,25 +1024,36 @@ int ChannelReceive::GetBaseMinimumPlayoutDelayMs() const { } absl::optional<Syncable::Info> ChannelReceive::GetSyncInfo() const { - RTC_DCHECK(module_process_thread_checker_.IsCurrent()); + // TODO(bugs.webrtc.org/11993): This should run on the network thread. + // We get here via RtpStreamsSynchronizer. Once that's done, many of + // these locks aren't needed. + RTC_DCHECK_RUN_ON(&worker_thread_checker_); Syncable::Info info; if (rtp_rtcp_->RemoteNTP(&info.capture_time_ntp_secs, - &info.capture_time_ntp_frac, nullptr, nullptr, + &info.capture_time_ntp_frac, + /*rtcp_arrival_time_secs=*/nullptr, + /*rtcp_arrival_time_frac=*/nullptr, &info.capture_time_source_clock) != 0) { return absl::nullopt; } - { - MutexLock lock(&sync_info_lock_); - if (!last_received_rtp_timestamp_ || !last_received_rtp_system_time_ms_) { - return absl::nullopt; - } - info.latest_received_capture_timestamp = *last_received_rtp_timestamp_; - info.latest_receive_time_ms = *last_received_rtp_system_time_ms_; + + if (!last_received_rtp_timestamp_ || !last_received_rtp_system_time_ms_) { + return absl::nullopt; } + info.latest_received_capture_timestamp = *last_received_rtp_timestamp_; + info.latest_receive_time_ms = *last_received_rtp_system_time_ms_; + + int jitter_buffer_delay = acm_receiver_.FilteredCurrentDelayMs(); + info.current_delay_ms = jitter_buffer_delay + playout_delay_ms_; + return info; } void ChannelReceive::UpdatePlayoutTimestamp(bool rtcp, int64_t now_ms) { + RTC_DCHECK_RUN_ON(&worker_thread_checker_); + // TODO(bugs.webrtc.org/11993): Expect to be called exclusively on the + // network thread. Once that's done, we won't need video_sync_lock_. + jitter_buffer_playout_timestamp_ = acm_receiver_.GetPlayoutTimestamp(); if (!jitter_buffer_playout_timestamp_) { @@ -916,14 +1076,11 @@ void ChannelReceive::UpdatePlayoutTimestamp(bool rtcp, int64_t now_ms) { // Remove the playout delay. playout_timestamp -= (delay_ms * (GetRtpTimestampRateHz() / 1000)); - { - MutexLock lock(&video_sync_lock_); - if (!rtcp && playout_timestamp != playout_timestamp_rtp_) { - playout_timestamp_rtp_ = playout_timestamp; - playout_timestamp_rtp_time_ms_ = now_ms; - } - playout_delay_ms_ = delay_ms; + if (!rtcp && playout_timestamp != playout_timestamp_rtp_) { + playout_timestamp_rtp_ = playout_timestamp; + playout_timestamp_rtp_time_ms_ = now_ms; } + playout_delay_ms_ = delay_ms; } int ChannelReceive::GetRtpTimestampRateHz() const { @@ -940,38 +1097,10 @@ int ChannelReceive::GetRtpTimestampRateHz() const { : acm_receiver_.last_output_sample_rate_hz(); } -int64_t ChannelReceive::GetRTT() const { - std::vector<RTCPReportBlock> report_blocks; - rtp_rtcp_->RemoteRTCPStat(&report_blocks); - - // TODO(nisse): Could we check the return value from the ->RTT() call below, - // instead of checking if we have any report blocks? - if (report_blocks.empty()) { - MutexLock lock(&assoc_send_channel_lock_); - // Tries to get RTT from an associated channel. - if (!associated_send_channel_) { - return 0; - } - return associated_send_channel_->GetRTT(); - } - - int64_t rtt = 0; - int64_t avg_rtt = 0; - int64_t max_rtt = 0; - int64_t min_rtt = 0; - // TODO(nisse): This method computes RTT based on sender reports, even though - // a receive stream is not supposed to do that. - if (rtp_rtcp_->RTT(remote_ssrc_, &rtt, &avg_rtt, &min_rtt, &max_rtt) != 0) { - return 0; - } - return rtt; -} - } // namespace std::unique_ptr<ChannelReceiveInterface> CreateChannelReceive( Clock* clock, - ProcessThread* module_process_thread, NetEqFactory* neteq_factory, AudioDeviceModule* audio_device_module, Transport* rtcp_send_transport, @@ -981,19 +1110,18 @@ std::unique_ptr<ChannelReceiveInterface> CreateChannelReceive( size_t jitter_buffer_max_packets, bool jitter_buffer_fast_playout, int jitter_buffer_min_delay_ms, - bool jitter_buffer_enable_rtx_handling, + bool enable_non_sender_rtt, rtc::scoped_refptr<AudioDecoderFactory> decoder_factory, absl::optional<AudioCodecPairId> codec_pair_id, rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor, const webrtc::CryptoOptions& crypto_options, rtc::scoped_refptr<FrameTransformerInterface> frame_transformer) { return std::make_unique<ChannelReceive>( - clock, module_process_thread, neteq_factory, audio_device_module, - rtcp_send_transport, rtc_event_log, local_ssrc, remote_ssrc, - jitter_buffer_max_packets, jitter_buffer_fast_playout, - jitter_buffer_min_delay_ms, jitter_buffer_enable_rtx_handling, - decoder_factory, codec_pair_id, frame_decryptor, crypto_options, - std::move(frame_transformer)); + clock, neteq_factory, audio_device_module, rtcp_send_transport, + rtc_event_log, local_ssrc, remote_ssrc, jitter_buffer_max_packets, + jitter_buffer_fast_playout, jitter_buffer_min_delay_ms, + enable_non_sender_rtt, decoder_factory, codec_pair_id, + std::move(frame_decryptor), crypto_options, std::move(frame_transformer)); } } // namespace voe diff --git a/audio/channel_receive.h b/audio/channel_receive.h index bc02ff3023..b47a4b5b97 100644 --- a/audio/channel_receive.h +++ b/audio/channel_receive.h @@ -28,6 +28,7 @@ #include "call/rtp_packet_sink_interface.h" #include "call/syncable.h" #include "modules/audio_coding/include/audio_coding_module_typedefs.h" +#include "modules/rtp_rtcp/source/source_tracker.h" #include "system_wrappers/include/clock.h" // TODO(solenberg, nisse): This file contains a few NOLINT marks, to silence @@ -43,7 +44,6 @@ namespace webrtc { class AudioDeviceModule; class FrameDecryptorInterface; class PacketRouter; -class ProcessThread; class RateLimiter; class ReceiveStatistics; class RtcEventLog; @@ -51,26 +51,38 @@ class RtpPacketReceived; class RtpRtcp; struct CallReceiveStatistics { - unsigned int cumulativeLost; + int cumulativeLost; unsigned int jitterSamples; - int64_t rttMs; int64_t payload_bytes_rcvd = 0; int64_t header_and_padding_bytes_rcvd = 0; int packetsReceived; - // The capture ntp time (in local timebase) of the first played out audio + uint32_t nacks_sent = 0; + // The capture NTP time (in local timebase) of the first played out audio // frame. int64_t capture_start_ntp_time_ms_; // The timestamp at which the last packet was received, i.e. the time of the // local clock when it was received - not the RTP timestamp of that packet. // https://w3c.github.io/webrtc-stats/#dom-rtcinboundrtpstreamstats-lastpacketreceivedtimestamp absl::optional<int64_t> last_packet_received_timestamp_ms; + // Remote outbound stats derived by the received RTCP sender reports. + // Note that the timestamps below correspond to the time elapsed since the + // Unix epoch. + // https://w3c.github.io/webrtc-stats/#remoteoutboundrtpstats-dict* + absl::optional<int64_t> last_sender_report_timestamp_ms; + absl::optional<int64_t> last_sender_report_remote_timestamp_ms; + uint32_t sender_reports_packets_sent = 0; + uint64_t sender_reports_bytes_sent = 0; + uint64_t sender_reports_reports_count = 0; + absl::optional<TimeDelta> round_trip_time; + TimeDelta total_round_trip_time = TimeDelta::Zero(); + int round_trip_time_measurements; }; namespace voe { class ChannelSendInterface; -// Interface class needed for AudioReceiveStream tests that use a +// Interface class needed for AudioReceiveStreamInterface tests that use a // MockChannelReceive. class ChannelReceiveInterface : public RtpPacketSinkInterface { @@ -99,12 +111,13 @@ class ChannelReceiveInterface : public RtpPacketSinkInterface { virtual double GetTotalOutputDuration() const = 0; // Stats. - virtual NetworkStatistics GetNetworkStatistics() const = 0; + virtual NetworkStatistics GetNetworkStatistics( + bool get_and_clear_legacy_stats) const = 0; virtual AudioDecodingCallStats GetDecodingCallStatistics() const = 0; // Audio+Video Sync. virtual uint32_t GetDelayEstimate() const = 0; - virtual void SetMinimumPlayoutDelay(int delay_ms) = 0; + virtual bool SetMinimumPlayoutDelay(int delay_ms) = 0; virtual bool GetPlayoutRtpTimestamp(uint32_t* rtp_timestamp, int64_t* time_ms) const = 0; virtual void SetEstimatedPlayoutNtpTimestampMs(int64_t ntp_timestamp_ms, @@ -127,6 +140,7 @@ class ChannelReceiveInterface : public RtpPacketSinkInterface { virtual CallReceiveStatistics GetRTCPStatistics() const = 0; virtual void SetNACKStatus(bool enable, int max_packets) = 0; + virtual void SetNonSenderRttMeasurement(bool enabled) = 0; virtual AudioMixer::Source::AudioFrameInfo GetAudioFrameWithInfo( int sample_rate_hz, @@ -134,6 +148,10 @@ class ChannelReceiveInterface : public RtpPacketSinkInterface { virtual int PreferredSampleRate() const = 0; + // Sets the source tracker to notify about "delivered" packets when output is + // muted. + virtual void SetSourceTracker(SourceTracker* source_tracker) = 0; + // Associate to a send channel. // Used for obtaining RTT for a receive-only channel. virtual void SetAssociatedSendChannel( @@ -144,11 +162,16 @@ class ChannelReceiveInterface : public RtpPacketSinkInterface { virtual void SetDepacketizerToDecoderFrameTransformer( rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer) = 0; + + virtual void SetFrameDecryptor( + rtc::scoped_refptr<webrtc::FrameDecryptorInterface> frame_decryptor) = 0; + + virtual void OnLocalSsrcChange(uint32_t local_ssrc) = 0; + virtual uint32_t GetLocalSsrc() const = 0; }; std::unique_ptr<ChannelReceiveInterface> CreateChannelReceive( Clock* clock, - ProcessThread* module_process_thread, NetEqFactory* neteq_factory, AudioDeviceModule* audio_device_module, Transport* rtcp_send_transport, @@ -158,7 +181,7 @@ std::unique_ptr<ChannelReceiveInterface> CreateChannelReceive( size_t jitter_buffer_max_packets, bool jitter_buffer_fast_playout, int jitter_buffer_min_delay_ms, - bool jitter_buffer_enable_rtx_handling, + bool enable_non_sender_rtt, rtc::scoped_refptr<AudioDecoderFactory> decoder_factory, absl::optional<AudioCodecPairId> codec_pair_id, rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor, diff --git a/audio/channel_receive_frame_transformer_delegate.cc b/audio/channel_receive_frame_transformer_delegate.cc index 261afbb100..e8ba6ded47 100644 --- a/audio/channel_receive_frame_transformer_delegate.cc +++ b/audio/channel_receive_frame_transformer_delegate.cc @@ -13,29 +13,34 @@ #include <utility> #include "rtc_base/buffer.h" -#include "rtc_base/task_utils/to_queued_task.h" namespace webrtc { namespace { -class TransformableAudioFrame : public TransformableAudioFrameInterface { +class TransformableIncomingAudioFrame + : public TransformableAudioFrameInterface { public: - TransformableAudioFrame(rtc::ArrayView<const uint8_t> payload, - const RTPHeader& header, - uint32_t ssrc) + TransformableIncomingAudioFrame(rtc::ArrayView<const uint8_t> payload, + const RTPHeader& header, + uint32_t ssrc) : payload_(payload.data(), payload.size()), header_(header), ssrc_(ssrc) {} - ~TransformableAudioFrame() override = default; + ~TransformableIncomingAudioFrame() override = default; rtc::ArrayView<const uint8_t> GetData() const override { return payload_; } void SetData(rtc::ArrayView<const uint8_t> data) override { payload_.SetData(data.data(), data.size()); } - uint32_t GetTimestamp() const override { return header_.timestamp; } + uint8_t GetPayloadType() const override { return header_.payloadType; } uint32_t GetSsrc() const override { return ssrc_; } + uint32_t GetTimestamp() const override { return header_.timestamp; } const RTPHeader& GetHeader() const override { return header_; } + rtc::ArrayView<const uint32_t> GetContributingSources() const override { + return rtc::ArrayView<const uint32_t>(header_.arrOfCSRCs, header_.numCSRCs); + } + Direction GetDirection() const override { return Direction::kReceiver; } private: rtc::Buffer payload_; @@ -47,7 +52,7 @@ class TransformableAudioFrame : public TransformableAudioFrameInterface { ChannelReceiveFrameTransformerDelegate::ChannelReceiveFrameTransformerDelegate( ReceiveFrameCallback receive_frame_callback, rtc::scoped_refptr<FrameTransformerInterface> frame_transformer, - rtc::Thread* channel_receive_thread) + TaskQueueBase* channel_receive_thread) : receive_frame_callback_(receive_frame_callback), frame_transformer_(std::move(frame_transformer)), channel_receive_thread_(channel_receive_thread) {} @@ -71,16 +76,16 @@ void ChannelReceiveFrameTransformerDelegate::Transform( uint32_t ssrc) { RTC_DCHECK_RUN_ON(&sequence_checker_); frame_transformer_->Transform( - std::make_unique<TransformableAudioFrame>(packet, header, ssrc)); + std::make_unique<TransformableIncomingAudioFrame>(packet, header, ssrc)); } void ChannelReceiveFrameTransformerDelegate::OnTransformedFrame( std::unique_ptr<TransformableFrameInterface> frame) { - rtc::scoped_refptr<ChannelReceiveFrameTransformerDelegate> delegate = this; - channel_receive_thread_->PostTask(ToQueuedTask( + rtc::scoped_refptr<ChannelReceiveFrameTransformerDelegate> delegate(this); + channel_receive_thread_->PostTask( [delegate = std::move(delegate), frame = std::move(frame)]() mutable { delegate->ReceiveFrame(std::move(frame)); - })); + }); } void ChannelReceiveFrameTransformerDelegate::ReceiveFrame( @@ -88,7 +93,10 @@ void ChannelReceiveFrameTransformerDelegate::ReceiveFrame( RTC_DCHECK_RUN_ON(&sequence_checker_); if (!receive_frame_callback_) return; - auto* transformed_frame = static_cast<TransformableAudioFrame*>(frame.get()); + RTC_CHECK_EQ(frame->GetDirection(), + TransformableFrameInterface::Direction::kReceiver); + auto* transformed_frame = + static_cast<TransformableIncomingAudioFrame*>(frame.get()); receive_frame_callback_(transformed_frame->GetData(), transformed_frame->GetHeader()); } diff --git a/audio/channel_receive_frame_transformer_delegate.h b/audio/channel_receive_frame_transformer_delegate.h index 73112d10e3..04ad7c4695 100644 --- a/audio/channel_receive_frame_transformer_delegate.h +++ b/audio/channel_receive_frame_transformer_delegate.h @@ -14,7 +14,8 @@ #include <memory> #include "api/frame_transformer_interface.h" -#include "rtc_base/synchronization/sequence_checker.h" +#include "api/sequence_checker.h" +#include "rtc_base/system/no_unique_address.h" #include "rtc_base/task_queue.h" #include "rtc_base/thread.h" @@ -22,7 +23,7 @@ namespace webrtc { // Delegates calls to FrameTransformerInterface to transform frames, and to // ChannelReceive to receive the transformed frames using the -// |receive_frame_callback_| on the |channel_receive_thread_|. +// `receive_frame_callback_` on the `channel_receive_thread_`. class ChannelReceiveFrameTransformerDelegate : public TransformedFrameCallback { public: using ReceiveFrameCallback = @@ -31,14 +32,14 @@ class ChannelReceiveFrameTransformerDelegate : public TransformedFrameCallback { ChannelReceiveFrameTransformerDelegate( ReceiveFrameCallback receive_frame_callback, rtc::scoped_refptr<FrameTransformerInterface> frame_transformer, - rtc::Thread* channel_receive_thread); + TaskQueueBase* channel_receive_thread); - // Registers |this| as callback for |frame_transformer_|, to get the + // Registers `this` as callback for `frame_transformer_`, to get the // transformed frames. void Init(); - // Unregisters and releases the |frame_transformer_| reference, and resets - // |receive_frame_callback_| on |channel_receive_thread_|. Called from + // Unregisters and releases the `frame_transformer_` reference, and resets + // `receive_frame_callback_` on `channel_receive_thread_`. Called from // ChannelReceive destructor to prevent running the callback on a dangling // channel. void Reset(); @@ -54,19 +55,19 @@ class ChannelReceiveFrameTransformerDelegate : public TransformedFrameCallback { std::unique_ptr<TransformableFrameInterface> frame) override; // Delegates the call to ChannelReceive::OnReceivedPayloadData on the - // |channel_receive_thread_|, by calling |receive_frame_callback_|. + // `channel_receive_thread_`, by calling `receive_frame_callback_`. void ReceiveFrame(std::unique_ptr<TransformableFrameInterface> frame) const; protected: ~ChannelReceiveFrameTransformerDelegate() override = default; private: - SequenceChecker sequence_checker_; + RTC_NO_UNIQUE_ADDRESS SequenceChecker sequence_checker_; ReceiveFrameCallback receive_frame_callback_ RTC_GUARDED_BY(sequence_checker_); rtc::scoped_refptr<FrameTransformerInterface> frame_transformer_ RTC_GUARDED_BY(sequence_checker_); - rtc::Thread* channel_receive_thread_; + TaskQueueBase* const channel_receive_thread_; }; } // namespace webrtc diff --git a/audio/channel_receive_frame_transformer_delegate_unittest.cc b/audio/channel_receive_frame_transformer_delegate_unittest.cc index e7f5a454b8..e31dd9f876 100644 --- a/audio/channel_receive_frame_transformer_delegate_unittest.cc +++ b/audio/channel_receive_frame_transformer_delegate_unittest.cc @@ -13,7 +13,6 @@ #include <memory> #include <utility> -#include "rtc_base/ref_counted_object.h" #include "test/gmock.h" #include "test/gtest.h" #include "test/mock_frame_transformer.h" @@ -41,9 +40,9 @@ class MockChannelReceive { TEST(ChannelReceiveFrameTransformerDelegateTest, RegisterTransformedFrameCallbackOnInit) { rtc::scoped_refptr<MockFrameTransformer> mock_frame_transformer = - new rtc::RefCountedObject<MockFrameTransformer>(); + rtc::make_ref_counted<MockFrameTransformer>(); rtc::scoped_refptr<ChannelReceiveFrameTransformerDelegate> delegate = - new rtc::RefCountedObject<ChannelReceiveFrameTransformerDelegate>( + rtc::make_ref_counted<ChannelReceiveFrameTransformerDelegate>( ChannelReceiveFrameTransformerDelegate::ReceiveFrameCallback(), mock_frame_transformer, nullptr); EXPECT_CALL(*mock_frame_transformer, RegisterTransformedFrameCallback); @@ -55,9 +54,9 @@ TEST(ChannelReceiveFrameTransformerDelegateTest, TEST(ChannelReceiveFrameTransformerDelegateTest, UnregisterTransformedFrameCallbackOnReset) { rtc::scoped_refptr<MockFrameTransformer> mock_frame_transformer = - new rtc::RefCountedObject<MockFrameTransformer>(); + rtc::make_ref_counted<MockFrameTransformer>(); rtc::scoped_refptr<ChannelReceiveFrameTransformerDelegate> delegate = - new rtc::RefCountedObject<ChannelReceiveFrameTransformerDelegate>( + rtc::make_ref_counted<ChannelReceiveFrameTransformerDelegate>( ChannelReceiveFrameTransformerDelegate::ReceiveFrameCallback(), mock_frame_transformer, nullptr); EXPECT_CALL(*mock_frame_transformer, UnregisterTransformedFrameCallback); @@ -68,11 +67,12 @@ TEST(ChannelReceiveFrameTransformerDelegateTest, // transformer, it passes it to the channel using the ReceiveFrameCallback. TEST(ChannelReceiveFrameTransformerDelegateTest, TransformRunsChannelReceiveCallback) { + rtc::AutoThread main_thread; rtc::scoped_refptr<MockFrameTransformer> mock_frame_transformer = - new rtc::RefCountedObject<NiceMock<MockFrameTransformer>>(); + rtc::make_ref_counted<NiceMock<MockFrameTransformer>>(); MockChannelReceive mock_channel; rtc::scoped_refptr<ChannelReceiveFrameTransformerDelegate> delegate = - new rtc::RefCountedObject<ChannelReceiveFrameTransformerDelegate>( + rtc::make_ref_counted<ChannelReceiveFrameTransformerDelegate>( mock_channel.callback(), mock_frame_transformer, rtc::Thread::Current()); rtc::scoped_refptr<TransformedFrameCallback> callback; @@ -99,11 +99,12 @@ TEST(ChannelReceiveFrameTransformerDelegateTest, // after resetting the delegate. TEST(ChannelReceiveFrameTransformerDelegateTest, OnTransformedDoesNotRunChannelReceiveCallbackAfterReset) { + rtc::AutoThread main_thread; rtc::scoped_refptr<MockFrameTransformer> mock_frame_transformer = - new rtc::RefCountedObject<testing::NiceMock<MockFrameTransformer>>(); + rtc::make_ref_counted<testing::NiceMock<MockFrameTransformer>>(); MockChannelReceive mock_channel; rtc::scoped_refptr<ChannelReceiveFrameTransformerDelegate> delegate = - new rtc::RefCountedObject<ChannelReceiveFrameTransformerDelegate>( + rtc::make_ref_counted<ChannelReceiveFrameTransformerDelegate>( mock_channel.callback(), mock_frame_transformer, rtc::Thread::Current()); diff --git a/audio/channel_send.cc b/audio/channel_send.cc index 80e7ab2f47..d2604061b8 100644 --- a/audio/channel_send.cc +++ b/audio/channel_send.cc @@ -21,6 +21,7 @@ #include "api/call/transport.h" #include "api/crypto/frame_encryptor_interface.h" #include "api/rtc_event_log/rtc_event_log.h" +#include "api/sequence_checker.h" #include "audio/channel_send_frame_transformer_delegate.h" #include "audio/utility/audio_frame_operations.h" #include "call/rtp_transport_controller_send_interface.h" @@ -30,21 +31,17 @@ #include "modules/audio_processing/rms_level.h" #include "modules/pacing/packet_router.h" #include "modules/rtp_rtcp/source/rtp_rtcp_impl2.h" -#include "modules/utility/include/process_thread.h" #include "rtc_base/checks.h" #include "rtc_base/event.h" -#include "rtc_base/format_macros.h" -#include "rtc_base/location.h" #include "rtc_base/logging.h" #include "rtc_base/numerics/safe_conversions.h" #include "rtc_base/race_checker.h" #include "rtc_base/rate_limiter.h" #include "rtc_base/synchronization/mutex.h" #include "rtc_base/task_queue.h" -#include "rtc_base/thread_checker.h" #include "rtc_base/time_utils.h" +#include "rtc_base/trace_event.h" #include "system_wrappers/include/clock.h" -#include "system_wrappers/include/field_trial.h" #include "system_wrappers/include/metrics.h" namespace webrtc { @@ -60,16 +57,12 @@ class TransportSequenceNumberProxy; class VoERtcpObserver; class ChannelSend : public ChannelSendInterface, - public AudioPacketizationCallback { // receive encoded - // packets from the ACM + public AudioPacketizationCallback, // receive encoded + // packets from the ACM + public RtcpPacketTypeCounterObserver { public: - // TODO(nisse): Make OnUplinkPacketLossRate public, and delete friend - // declaration. - friend class VoERtcpObserver; - ChannelSend(Clock* clock, TaskQueueFactory* task_queue_factory, - ProcessThread* module_process_thread, Transport* rtp_transport, RtcpRttStats* rtcp_rtt_stats, RtcEventLog* rtc_event_log, @@ -79,7 +72,8 @@ class ChannelSend : public ChannelSendInterface, int rtcp_report_interval_ms, uint32_t ssrc, rtc::scoped_refptr<FrameTransformerInterface> frame_transformer, - TransportFeedbackObserver* feedback_observer); + TransportFeedbackObserver* feedback_observer, + const FieldTrialsView& field_trials); ~ChannelSend() override; @@ -96,7 +90,7 @@ class ChannelSend : public ChannelSendInterface, // Codecs void OnBitrateAllocation(BitrateAllocationUpdate update) override; - int GetBitrate() const override; + int GetTargetBitrate() const override; // Network void ReceivedRTCPPacket(const uint8_t* data, size_t length) override; @@ -151,6 +145,13 @@ class ChannelSend : public ChannelSendInterface, rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer) override; + // RtcpPacketTypeCounterObserver. + void RtcpPacketTypesCounterUpdated( + uint32_t ssrc, + const RtcpPacketTypeCounter& packet_counter) override; + + void OnUplinkPacketLossRate(float packet_loss_rate); + private: // From AudioPacketizationCallback in the ACM int32_t SendData(AudioFrameType frameType, @@ -160,7 +161,6 @@ class ChannelSend : public ChannelSendInterface, size_t payloadSize, int64_t absolute_capture_timestamp_ms) override; - void OnUplinkPacketLossRate(float packet_loss_rate); bool InputMute() const; int32_t SendRtpAudio(AudioFrameType frameType, @@ -179,8 +179,7 @@ class ChannelSend : public ChannelSendInterface, // specific threads we know about. The goal is to eventually split up // voe::Channel into parts with single-threaded semantics, and thereby reduce // the need for locks. - rtc::ThreadChecker worker_thread_checker_; - rtc::ThreadChecker module_process_thread_checker_; + SequenceChecker worker_thread_checker_; // Methods accessed from audio and video threads are checked for sequential- // only access. We don't necessarily own and control these threads, so thread // checkers cannot be used. E.g. Chromium may transfer "ownership" from one @@ -189,6 +188,7 @@ class ChannelSend : public ChannelSendInterface, mutable Mutex volume_settings_mutex_; + const uint32_t ssrc_; bool sending_ RTC_GUARDED_BY(&worker_thread_checker_) = false; RtcEventLog* const event_log_; @@ -200,7 +200,6 @@ class ChannelSend : public ChannelSendInterface, uint32_t _timeStamp RTC_GUARDED_BY(encoder_queue_); // uses - ProcessThread* const _moduleProcessThreadPtr; RmsLevel rms_level_ RTC_GUARDED_BY(encoder_queue_); bool input_mute_ RTC_GUARDED_BY(volume_settings_mutex_); bool previous_frame_muted_ RTC_GUARDED_BY(encoder_queue_); @@ -218,8 +217,7 @@ class ChannelSend : public ChannelSendInterface, const std::unique_ptr<RtpPacketSenderProxy> rtp_packet_pacer_proxy_; const std::unique_ptr<RateLimiter> retransmission_rate_limiter_; - rtc::ThreadChecker construction_thread_; - + SequenceChecker construction_thread_; bool encoder_queue_is_active_ RTC_GUARDED_BY(encoder_queue_) = false; @@ -235,8 +233,11 @@ class ChannelSend : public ChannelSendInterface, rtc::scoped_refptr<ChannelSendFrameTransformerDelegate> frame_transformer_delegate_ RTC_GUARDED_BY(encoder_queue_); - mutable Mutex bitrate_mutex_; - int configured_bitrate_bps_ RTC_GUARDED_BY(bitrate_mutex_) = 0; + const bool fixing_timestamp_stall_; + + mutable Mutex rtcp_counter_mutex_; + RtcpPacketTypeCounter rtcp_packet_type_counter_ + RTC_GUARDED_BY(rtcp_counter_mutex_); // Defined last to ensure that there are no running tasks when the other // members are destroyed. @@ -262,7 +263,7 @@ class RtpPacketSenderProxy : public RtpPacketSender { } private: - rtc::ThreadChecker thread_checker_; + SequenceChecker thread_checker_; Mutex mutex_; RtpPacketSender* rtp_packet_pacer_ RTC_GUARDED_BY(&mutex_); }; @@ -444,7 +445,6 @@ int32_t ChannelSend::SendRtpAudio(AudioFrameType frameType, ChannelSend::ChannelSend( Clock* clock, TaskQueueFactory* task_queue_factory, - ProcessThread* module_process_thread, Transport* rtp_transport, RtcpRttStats* rtcp_rtt_stats, RtcEventLog* rtc_event_log, @@ -454,11 +454,12 @@ ChannelSend::ChannelSend( int rtcp_report_interval_ms, uint32_t ssrc, rtc::scoped_refptr<FrameTransformerInterface> frame_transformer, - TransportFeedbackObserver* feedback_observer) - : event_log_(rtc_event_log), + TransportFeedbackObserver* feedback_observer, + const FieldTrialsView& field_trials) + : ssrc_(ssrc), + event_log_(rtc_event_log), _timeStamp(0), // This is just an offset, RTP module will add it's own // random offset - _moduleProcessThreadPtr(module_process_thread), input_mute_(false), previous_frame_muted_(false), _includeAudioLevelIndication(false), @@ -469,12 +470,11 @@ ChannelSend::ChannelSend( new RateLimiter(clock, kMaxRetransmissionWindowMs)), frame_encryptor_(frame_encryptor), crypto_options_(crypto_options), + fixing_timestamp_stall_( + field_trials.IsDisabled("WebRTC-Audio-FixTimestampStall")), encoder_queue_(task_queue_factory->CreateTaskQueue( "AudioEncoder", TaskQueueFactory::Priority::NORMAL)) { - RTC_DCHECK(module_process_thread); - module_process_thread_checker_.Detach(); - audio_coding_.reset(AudioCodingModule::Create(AudioCodingModule::Config())); RtpRtcpInterface::Configuration configuration; @@ -492,6 +492,7 @@ ChannelSend::ChannelSend( retransmission_rate_limiter_.get(); configuration.extmap_allow_mixed = extmap_allow_mixed; configuration.rtcp_report_interval_ms = rtcp_report_interval_ms; + configuration.rtcp_packet_type_counter_observer = this; configuration.local_media_ssrc = ssrc; @@ -501,8 +502,6 @@ ChannelSend::ChannelSend( rtp_sender_audio_ = std::make_unique<RTPSenderAudio>(configuration.clock, rtp_rtcp_->RtpSender()); - _moduleProcessThreadPtr->RegisterModule(rtp_rtcp_.get(), RTC_FROM_HERE); - // Ensure that RTCP is enabled by default for the created channel. rtp_rtcp_->SetRTCPStatus(RtcpMode::kCompound); @@ -522,9 +521,6 @@ ChannelSend::~ChannelSend() { StopSend(); int error = audio_coding_->RegisterTransportCallback(NULL); RTC_DCHECK_EQ(0, error); - - if (_moduleProcessThreadPtr) - _moduleProcessThreadPtr->DeRegisterModule(rtp_rtcp_.get()); } void ChannelSend::StartSend() { @@ -532,9 +528,12 @@ void ChannelSend::StartSend() { RTC_DCHECK(!sending_); sending_ = true; + RTC_DCHECK(packet_router_); + packet_router_->AddSendRtpModule(rtp_rtcp_.get(), /*remb_candidate=*/false); rtp_rtcp_->SetSendingMediaStatus(true); int ret = rtp_rtcp_->SetSendingStatus(true); RTC_DCHECK_EQ(0, ret); + // It is now OK to start processing on the encoder task queue. encoder_queue_.PostTask([this] { RTC_DCHECK_RUN_ON(&encoder_queue_); @@ -563,6 +562,9 @@ void ChannelSend::StopSend() { RTC_DLOG(LS_ERROR) << "StartSend() RTP/RTCP failed to stop sending"; } rtp_rtcp_->SetSendingMediaStatus(false); + + RTC_DCHECK(packet_router_); + packet_router_->RemoveSendRtpModule(rtp_rtcp_.get()); } void ChannelSend::SetEncoder(int payload_type, @@ -607,18 +609,14 @@ void ChannelSend::OnBitrateAllocation(BitrateAllocationUpdate update) { // rules. // RTC_DCHECK(worker_thread_checker_.IsCurrent() || // module_process_thread_checker_.IsCurrent()); - MutexLock lock(&bitrate_mutex_); - CallEncoder([&](AudioEncoder* encoder) { encoder->OnReceivedUplinkAllocation(update); }); retransmission_rate_limiter_->SetMaxRate(update.target_bitrate.bps()); - configured_bitrate_bps_ = update.target_bitrate.bps(); } -int ChannelSend::GetBitrate() const { - MutexLock lock(&bitrate_mutex_); - return configured_bitrate_bps_; +int ChannelSend::GetTargetBitrate() const { + return audio_coding_->GetTargetBitrate(); } void ChannelSend::OnUplinkPacketLossRate(float packet_loss_rate) { @@ -699,9 +697,9 @@ void ChannelSend::SetSendAudioLevelIndicationStatus(bool enable, int id) { RTC_DCHECK_RUN_ON(&worker_thread_checker_); _includeAudioLevelIndication = enable; if (enable) { - rtp_rtcp_->RegisterRtpHeaderExtension(AudioLevel::kUri, id); + rtp_rtcp_->RegisterRtpHeaderExtension(AudioLevel::Uri(), id); } else { - rtp_rtcp_->DeregisterSendRtpHeaderExtension(AudioLevel::kUri); + rtp_rtcp_->DeregisterSendRtpHeaderExtension(AudioLevel::Uri()); } } @@ -718,8 +716,6 @@ void ChannelSend::RegisterSenderCongestionControlObjects( rtcp_observer_->SetBandwidthObserver(bandwidth_observer); rtp_packet_pacer_proxy_->SetPacketPacer(rtp_packet_pacer); rtp_rtcp_->SetStorePacketsStatus(true, 600); - constexpr bool remb_candidate = false; - packet_router->AddSendRtpModule(rtp_rtcp_.get(), remb_candidate); packet_router_ = packet_router; } @@ -728,7 +724,6 @@ void ChannelSend::ResetSenderCongestionControlObjects() { RTC_DCHECK(packet_router_); rtp_rtcp_->SetStorePacketsStatus(false, 600); rtcp_observer_->SetBandwidthObserver(nullptr); - packet_router_->RemoveSendRtpModule(rtp_rtcp_.get()); packet_router_ = nullptr; rtp_packet_pacer_proxy_->SetPacketPacer(nullptr); } @@ -746,25 +741,20 @@ std::vector<ReportBlock> ChannelSend::GetRemoteRTCPReportBlocks() const { // Get the report blocks from the latest received RTCP Sender or Receiver // Report. Each element in the vector contains the sender's SSRC and a // report block according to RFC 3550. - std::vector<RTCPReportBlock> rtcp_report_blocks; - - int ret = rtp_rtcp_->RemoteRTCPStat(&rtcp_report_blocks); - RTC_DCHECK_EQ(0, ret); - std::vector<ReportBlock> report_blocks; - - std::vector<RTCPReportBlock>::const_iterator it = rtcp_report_blocks.begin(); - for (; it != rtcp_report_blocks.end(); ++it) { + for (const ReportBlockData& data : rtp_rtcp_->GetLatestReportBlockData()) { ReportBlock report_block; - report_block.sender_SSRC = it->sender_ssrc; - report_block.source_SSRC = it->source_ssrc; - report_block.fraction_lost = it->fraction_lost; - report_block.cumulative_num_packets_lost = it->packets_lost; + report_block.sender_SSRC = data.report_block().sender_ssrc; + report_block.source_SSRC = data.report_block().source_ssrc; + report_block.fraction_lost = data.report_block().fraction_lost; + report_block.cumulative_num_packets_lost = data.report_block().packets_lost; report_block.extended_highest_sequence_number = - it->extended_highest_sequence_number; - report_block.interarrival_jitter = it->jitter; - report_block.last_SR_timestamp = it->last_sender_report_timestamp; - report_block.delay_since_last_SR = it->delay_since_last_sender_report; + data.report_block().extended_highest_sequence_number; + report_block.interarrival_jitter = data.report_block().jitter; + report_block.last_SR_timestamp = + data.report_block().last_sender_report_timestamp; + report_block.delay_since_last_SR = + data.report_block().delay_since_last_sender_report; report_blocks.push_back(report_block); } return report_blocks; @@ -789,14 +779,32 @@ CallSendStatistics ChannelSend::GetRTCPStatistics() const { stats.retransmitted_bytes_sent = rtp_stats.retransmitted.payload_bytes; stats.packetsSent = rtp_stats.transmitted.packets + rtx_stats.transmitted.packets; + stats.total_packet_send_delay = rtp_stats.transmitted.total_packet_delay; stats.retransmitted_packets_sent = rtp_stats.retransmitted.packets; stats.report_block_datas = rtp_rtcp_->GetLatestReportBlockData(); + { + MutexLock lock(&rtcp_counter_mutex_); + stats.nacks_rcvd = rtcp_packet_type_counter_.nack_packets; + } + return stats; } +void ChannelSend::RtcpPacketTypesCounterUpdated( + uint32_t ssrc, + const RtcpPacketTypeCounter& packet_counter) { + if (ssrc != ssrc_) { + return; + } + MutexLock lock(&rtcp_counter_mutex_); + rtcp_packet_type_counter_ = packet_counter; +} + void ChannelSend::ProcessAndEncodeAudio( std::unique_ptr<AudioFrame> audio_frame) { + TRACE_EVENT0("webrtc", "ChannelSend::ProcessAndEncodeAudio"); + RTC_DCHECK_RUNS_SERIALIZED(&audio_thread_race_checker_); RTC_DCHECK_GT(audio_frame->samples_per_channel_, 0); RTC_DCHECK_LE(audio_frame->num_channels_, 8); @@ -808,6 +816,10 @@ void ChannelSend::ProcessAndEncodeAudio( [this, audio_frame = std::move(audio_frame)]() mutable { RTC_DCHECK_RUN_ON(&encoder_queue_); if (!encoder_queue_is_active_) { + if (fixing_timestamp_stall_) { + _timeStamp += + static_cast<uint32_t>(audio_frame->samples_per_channel_); + } return; } // Measure time between when the audio frame is added to the task queue @@ -856,29 +868,19 @@ ANAStats ChannelSend::GetANAStatistics() const { } RtpRtcpInterface* ChannelSend::GetRtpRtcp() const { - RTC_DCHECK(module_process_thread_checker_.IsCurrent()); return rtp_rtcp_.get(); } int64_t ChannelSend::GetRTT() const { - std::vector<RTCPReportBlock> report_blocks; - rtp_rtcp_->RemoteRTCPStat(&report_blocks); - + std::vector<ReportBlockData> report_blocks = + rtp_rtcp_->GetLatestReportBlockData(); if (report_blocks.empty()) { return 0; } - int64_t rtt = 0; - int64_t avg_rtt = 0; - int64_t max_rtt = 0; - int64_t min_rtt = 0; // We don't know in advance the remote ssrc used by the other end's receiver - // reports, so use the SSRC of the first report block for calculating the RTT. - if (rtp_rtcp_->RTT(report_blocks[0].sender_ssrc, &rtt, &avg_rtt, &min_rtt, - &max_rtt) != 0) { - return 0; - } - return rtt; + // reports, so use the first report block for the RTT. + return report_blocks.front().last_rtt_ms(); } void ChannelSend::SetFrameEncryptor( @@ -926,7 +928,7 @@ void ChannelSend::InitFrameTransformerDelegate( absolute_capture_timestamp_ms); }; frame_transformer_delegate_ = - new rtc::RefCountedObject<ChannelSendFrameTransformerDelegate>( + rtc::make_ref_counted<ChannelSendFrameTransformerDelegate>( std::move(send_audio_callback), std::move(frame_transformer), &encoder_queue_); frame_transformer_delegate_->Init(); @@ -937,7 +939,6 @@ void ChannelSend::InitFrameTransformerDelegate( std::unique_ptr<ChannelSendInterface> CreateChannelSend( Clock* clock, TaskQueueFactory* task_queue_factory, - ProcessThread* module_process_thread, Transport* rtp_transport, RtcpRttStats* rtcp_rtt_stats, RtcEventLog* rtc_event_log, @@ -947,12 +948,13 @@ std::unique_ptr<ChannelSendInterface> CreateChannelSend( int rtcp_report_interval_ms, uint32_t ssrc, rtc::scoped_refptr<FrameTransformerInterface> frame_transformer, - TransportFeedbackObserver* feedback_observer) { + TransportFeedbackObserver* feedback_observer, + const FieldTrialsView& field_trials) { return std::make_unique<ChannelSend>( - clock, task_queue_factory, module_process_thread, rtp_transport, - rtcp_rtt_stats, rtc_event_log, frame_encryptor, crypto_options, - extmap_allow_mixed, rtcp_report_interval_ms, ssrc, - std::move(frame_transformer), feedback_observer); + clock, task_queue_factory, rtp_transport, rtcp_rtt_stats, rtc_event_log, + frame_encryptor, crypto_options, extmap_allow_mixed, + rtcp_report_interval_ms, ssrc, std::move(frame_transformer), + feedback_observer, field_trials); } } // namespace voe diff --git a/audio/channel_send.h b/audio/channel_send.h index 2e23ef5d2d..cf9a273f70 100644 --- a/audio/channel_send.h +++ b/audio/channel_send.h @@ -18,6 +18,7 @@ #include "api/audio/audio_frame.h" #include "api/audio_codecs/audio_encoder.h" #include "api/crypto/crypto_options.h" +#include "api/field_trials_view.h" #include "api/frame_transformer_interface.h" #include "api/function_view.h" #include "api/task_queue/task_queue_factory.h" @@ -28,7 +29,6 @@ namespace webrtc { class FrameEncryptorInterface; -class ProcessThread; class RtcEventLog; class RtpTransportControllerSendInterface; @@ -39,6 +39,8 @@ struct CallSendStatistics { // https://w3c.github.io/webrtc-stats/#dom-rtcoutboundrtpstreamstats-retransmittedbytessent uint64_t retransmitted_bytes_sent; int packetsSent; + // https://w3c.github.io/webrtc-stats/#dom-rtcoutboundrtpstreamstats-totalpacketsenddelay + TimeDelta total_packet_send_delay = TimeDelta::Zero(); // https://w3c.github.io/webrtc-stats/#dom-rtcoutboundrtpstreamstats-retransmittedpacketssent uint64_t retransmitted_packets_sent; // A snapshot of Report Blocks with additional data of interest to statistics. @@ -46,6 +48,7 @@ struct CallSendStatistics { // ReportBlockData represents the latest Report Block that was received for // that pair. std::vector<ReportBlockData> report_block_datas; + uint32_t nacks_rcvd; }; // See section 6.4.2 in http://www.ietf.org/rfc/rfc3550.txt for details. @@ -91,19 +94,19 @@ class ChannelSendInterface { int payload_frequency) = 0; virtual bool SendTelephoneEventOutband(int event, int duration_ms) = 0; virtual void OnBitrateAllocation(BitrateAllocationUpdate update) = 0; - virtual int GetBitrate() const = 0; + virtual int GetTargetBitrate() const = 0; virtual void SetInputMute(bool muted) = 0; virtual void ProcessAndEncodeAudio( std::unique_ptr<AudioFrame> audio_frame) = 0; virtual RtpRtcpInterface* GetRtpRtcp() const = 0; - // In RTP we currently rely on RTCP packets (|ReceivedRTCPPacket|) to inform + // In RTP we currently rely on RTCP packets (`ReceivedRTCPPacket`) to inform // about RTT. // In media transport we rely on the TargetTransferRateObserver instead. // In other words, if you are using RTP, you should expect - // |ReceivedRTCPPacket| to be called, if you are using media transport, - // |OnTargetTransferRate| will be called. + // `ReceivedRTCPPacket` to be called, if you are using media transport, + // `OnTargetTransferRate` will be called. // // In future, RTP media will move to the media transport implementation and // these conditions will be removed. @@ -126,7 +129,6 @@ class ChannelSendInterface { std::unique_ptr<ChannelSendInterface> CreateChannelSend( Clock* clock, TaskQueueFactory* task_queue_factory, - ProcessThread* module_process_thread, Transport* rtp_transport, RtcpRttStats* rtcp_rtt_stats, RtcEventLog* rtc_event_log, @@ -136,7 +138,8 @@ std::unique_ptr<ChannelSendInterface> CreateChannelSend( int rtcp_report_interval_ms, uint32_t ssrc, rtc::scoped_refptr<FrameTransformerInterface> frame_transformer, - TransportFeedbackObserver* feedback_observer); + TransportFeedbackObserver* feedback_observer, + const FieldTrialsView& field_trials); } // namespace voe } // namespace webrtc diff --git a/audio/channel_send_frame_transformer_delegate.cc b/audio/channel_send_frame_transformer_delegate.cc index 72a459d897..29bb0b81d8 100644 --- a/audio/channel_send_frame_transformer_delegate.cc +++ b/audio/channel_send_frame_transformer_delegate.cc @@ -15,16 +15,16 @@ namespace webrtc { namespace { -class TransformableAudioFrame : public TransformableFrameInterface { +class TransformableOutgoingAudioFrame : public TransformableFrameInterface { public: - TransformableAudioFrame(AudioFrameType frame_type, - uint8_t payload_type, - uint32_t rtp_timestamp, - uint32_t rtp_start_timestamp, - const uint8_t* payload_data, - size_t payload_size, - int64_t absolute_capture_timestamp_ms, - uint32_t ssrc) + TransformableOutgoingAudioFrame(AudioFrameType frame_type, + uint8_t payload_type, + uint32_t rtp_timestamp, + uint32_t rtp_start_timestamp, + const uint8_t* payload_data, + size_t payload_size, + int64_t absolute_capture_timestamp_ms, + uint32_t ssrc) : frame_type_(frame_type), payload_type_(payload_type), rtp_timestamp_(rtp_timestamp), @@ -32,7 +32,7 @@ class TransformableAudioFrame : public TransformableFrameInterface { payload_(payload_data, payload_size), absolute_capture_timestamp_ms_(absolute_capture_timestamp_ms), ssrc_(ssrc) {} - ~TransformableAudioFrame() override = default; + ~TransformableOutgoingAudioFrame() override = default; rtc::ArrayView<const uint8_t> GetData() const override { return payload_; } void SetData(rtc::ArrayView<const uint8_t> data) override { payload_.SetData(data.data(), data.size()); @@ -44,10 +44,11 @@ class TransformableAudioFrame : public TransformableFrameInterface { uint32_t GetSsrc() const override { return ssrc_; } AudioFrameType GetFrameType() const { return frame_type_; } - uint8_t GetPayloadType() const { return payload_type_; } + uint8_t GetPayloadType() const override { return payload_type_; } int64_t GetAbsoluteCaptureTimestampMs() const { return absolute_capture_timestamp_ms_; } + Direction GetDirection() const override { return Direction::kSender; } private: AudioFrameType frame_type_; @@ -90,9 +91,10 @@ void ChannelSendFrameTransformerDelegate::Transform( size_t payload_size, int64_t absolute_capture_timestamp_ms, uint32_t ssrc) { - frame_transformer_->Transform(std::make_unique<TransformableAudioFrame>( - frame_type, payload_type, rtp_timestamp, rtp_start_timestamp, - payload_data, payload_size, absolute_capture_timestamp_ms, ssrc)); + frame_transformer_->Transform( + std::make_unique<TransformableOutgoingAudioFrame>( + frame_type, payload_type, rtp_timestamp, rtp_start_timestamp, + payload_data, payload_size, absolute_capture_timestamp_ms, ssrc)); } void ChannelSendFrameTransformerDelegate::OnTransformedFrame( @@ -100,7 +102,7 @@ void ChannelSendFrameTransformerDelegate::OnTransformedFrame( MutexLock lock(&send_lock_); if (!send_frame_callback_) return; - rtc::scoped_refptr<ChannelSendFrameTransformerDelegate> delegate = this; + rtc::scoped_refptr<ChannelSendFrameTransformerDelegate> delegate(this); encoder_queue_->PostTask( [delegate = std::move(delegate), frame = std::move(frame)]() mutable { delegate->SendFrame(std::move(frame)); @@ -111,9 +113,12 @@ void ChannelSendFrameTransformerDelegate::SendFrame( std::unique_ptr<TransformableFrameInterface> frame) const { MutexLock lock(&send_lock_); RTC_DCHECK_RUN_ON(encoder_queue_); + RTC_CHECK_EQ(frame->GetDirection(), + TransformableFrameInterface::Direction::kSender); if (!send_frame_callback_) return; - auto* transformed_frame = static_cast<TransformableAudioFrame*>(frame.get()); + auto* transformed_frame = + static_cast<TransformableOutgoingAudioFrame*>(frame.get()); send_frame_callback_(transformed_frame->GetFrameType(), transformed_frame->GetPayloadType(), transformed_frame->GetTimestamp() - diff --git a/audio/channel_send_frame_transformer_delegate.h b/audio/channel_send_frame_transformer_delegate.h index 531d1bc110..6d9f0a8613 100644 --- a/audio/channel_send_frame_transformer_delegate.h +++ b/audio/channel_send_frame_transformer_delegate.h @@ -14,17 +14,17 @@ #include <memory> #include "api/frame_transformer_interface.h" +#include "api/sequence_checker.h" #include "modules/audio_coding/include/audio_coding_module_typedefs.h" #include "rtc_base/buffer.h" #include "rtc_base/synchronization/mutex.h" -#include "rtc_base/synchronization/sequence_checker.h" #include "rtc_base/task_queue.h" namespace webrtc { // Delegates calls to FrameTransformerInterface to transform frames, and to -// ChannelSend to send the transformed frames using |send_frame_callback_| on -// the |encoder_queue_|. +// ChannelSend to send the transformed frames using `send_frame_callback_` on +// the `encoder_queue_`. // OnTransformedFrame() can be called from any thread, the delegate ensures // thread-safe access to the ChannelSend callback. class ChannelSendFrameTransformerDelegate : public TransformedFrameCallback { @@ -40,12 +40,12 @@ class ChannelSendFrameTransformerDelegate : public TransformedFrameCallback { rtc::scoped_refptr<FrameTransformerInterface> frame_transformer, rtc::TaskQueue* encoder_queue); - // Registers |this| as callback for |frame_transformer_|, to get the + // Registers `this` as callback for `frame_transformer_`, to get the // transformed frames. void Init(); - // Unregisters and releases the |frame_transformer_| reference, and resets - // |send_frame_callback_| under lock. Called from ChannelSend destructor to + // Unregisters and releases the `frame_transformer_` reference, and resets + // `send_frame_callback_` under lock. Called from ChannelSend destructor to // prevent running the callback on a dangling channel. void Reset(); @@ -64,8 +64,8 @@ class ChannelSendFrameTransformerDelegate : public TransformedFrameCallback { void OnTransformedFrame( std::unique_ptr<TransformableFrameInterface> frame) override; - // Delegates the call to ChannelSend::SendRtpAudio on the |encoder_queue_|, - // by calling |send_audio_callback_|. + // Delegates the call to ChannelSend::SendRtpAudio on the `encoder_queue_`, + // by calling `send_audio_callback_`. void SendFrame(std::unique_ptr<TransformableFrameInterface> frame) const; protected: diff --git a/audio/channel_send_frame_transformer_delegate_unittest.cc b/audio/channel_send_frame_transformer_delegate_unittest.cc index e2f3647c0a..9196bcb41f 100644 --- a/audio/channel_send_frame_transformer_delegate_unittest.cc +++ b/audio/channel_send_frame_transformer_delegate_unittest.cc @@ -13,7 +13,6 @@ #include <memory> #include <utility> -#include "rtc_base/ref_counted_object.h" #include "rtc_base/task_queue_for_test.h" #include "test/gmock.h" #include "test/gtest.h" @@ -53,9 +52,9 @@ class MockChannelSend { TEST(ChannelSendFrameTransformerDelegateTest, RegisterTransformedFrameCallbackOnInit) { rtc::scoped_refptr<MockFrameTransformer> mock_frame_transformer = - new rtc::RefCountedObject<MockFrameTransformer>(); + rtc::make_ref_counted<MockFrameTransformer>(); rtc::scoped_refptr<ChannelSendFrameTransformerDelegate> delegate = - new rtc::RefCountedObject<ChannelSendFrameTransformerDelegate>( + rtc::make_ref_counted<ChannelSendFrameTransformerDelegate>( ChannelSendFrameTransformerDelegate::SendFrameCallback(), mock_frame_transformer, nullptr); EXPECT_CALL(*mock_frame_transformer, RegisterTransformedFrameCallback); @@ -67,9 +66,9 @@ TEST(ChannelSendFrameTransformerDelegateTest, TEST(ChannelSendFrameTransformerDelegateTest, UnregisterTransformedFrameCallbackOnReset) { rtc::scoped_refptr<MockFrameTransformer> mock_frame_transformer = - new rtc::RefCountedObject<MockFrameTransformer>(); + rtc::make_ref_counted<MockFrameTransformer>(); rtc::scoped_refptr<ChannelSendFrameTransformerDelegate> delegate = - new rtc::RefCountedObject<ChannelSendFrameTransformerDelegate>( + rtc::make_ref_counted<ChannelSendFrameTransformerDelegate>( ChannelSendFrameTransformerDelegate::SendFrameCallback(), mock_frame_transformer, nullptr); EXPECT_CALL(*mock_frame_transformer, UnregisterTransformedFrameCallback); @@ -82,10 +81,10 @@ TEST(ChannelSendFrameTransformerDelegateTest, TransformRunsChannelSendCallback) { TaskQueueForTest channel_queue("channel_queue"); rtc::scoped_refptr<MockFrameTransformer> mock_frame_transformer = - new rtc::RefCountedObject<NiceMock<MockFrameTransformer>>(); + rtc::make_ref_counted<NiceMock<MockFrameTransformer>>(); MockChannelSend mock_channel; rtc::scoped_refptr<ChannelSendFrameTransformerDelegate> delegate = - new rtc::RefCountedObject<ChannelSendFrameTransformerDelegate>( + rtc::make_ref_counted<ChannelSendFrameTransformerDelegate>( mock_channel.callback(), mock_frame_transformer, &channel_queue); rtc::scoped_refptr<TransformedFrameCallback> callback; EXPECT_CALL(*mock_frame_transformer, RegisterTransformedFrameCallback) @@ -112,10 +111,10 @@ TEST(ChannelSendFrameTransformerDelegateTest, OnTransformedDoesNotRunChannelSendCallbackAfterReset) { TaskQueueForTest channel_queue("channel_queue"); rtc::scoped_refptr<MockFrameTransformer> mock_frame_transformer = - new rtc::RefCountedObject<testing::NiceMock<MockFrameTransformer>>(); + rtc::make_ref_counted<testing::NiceMock<MockFrameTransformer>>(); MockChannelSend mock_channel; rtc::scoped_refptr<ChannelSendFrameTransformerDelegate> delegate = - new rtc::RefCountedObject<ChannelSendFrameTransformerDelegate>( + rtc::make_ref_counted<ChannelSendFrameTransformerDelegate>( mock_channel.callback(), mock_frame_transformer, &channel_queue); delegate->Reset(); diff --git a/audio/conversion.h b/audio/conversion.h index 920aa3a434..dd71942f6a 100644 --- a/audio/conversion.h +++ b/audio/conversion.h @@ -11,6 +11,9 @@ #ifndef AUDIO_CONVERSION_H_ #define AUDIO_CONVERSION_H_ +#include <stddef.h> +#include <stdint.h> + namespace webrtc { // Convert fixed point number with 8 bit fractional part, to floating point. diff --git a/audio/mock_voe_channel_proxy.h b/audio/mock_voe_channel_proxy.h index 542358f687..a02bee38ad 100644 --- a/audio/mock_voe_channel_proxy.h +++ b/audio/mock_voe_channel_proxy.h @@ -17,6 +17,7 @@ #include <utility> #include <vector> +#include "api/crypto/frame_decryptor_interface.h" #include "api/test/mock_frame_encryptor.h" #include "audio/channel_receive.h" #include "audio/channel_send.h" @@ -29,13 +30,17 @@ namespace test { class MockChannelReceive : public voe::ChannelReceiveInterface { public: MOCK_METHOD(void, SetNACKStatus, (bool enable, int max_packets), (override)); + MOCK_METHOD(void, SetNonSenderRttMeasurement, (bool enabled), (override)); MOCK_METHOD(void, RegisterReceiverCongestionControlObjects, (PacketRouter*), (override)); MOCK_METHOD(void, ResetReceiverCongestionControlObjects, (), (override)); MOCK_METHOD(CallReceiveStatistics, GetRTCPStatistics, (), (const, override)); - MOCK_METHOD(NetworkStatistics, GetNetworkStatistics, (), (const, override)); + MOCK_METHOD(NetworkStatistics, + GetNetworkStatistics, + (bool), + (const, override)); MOCK_METHOD(AudioDecodingCallStats, GetDecodingCallStatistics, (), @@ -56,6 +61,7 @@ class MockChannelReceive : public voe::ChannelReceiveInterface { (int sample_rate_hz, AudioFrame*), (override)); MOCK_METHOD(int, PreferredSampleRate, (), (const, override)); + MOCK_METHOD(void, SetSourceTracker, (SourceTracker*), (override)); MOCK_METHOD(void, SetAssociatedSendChannel, (const voe::ChannelSendInterface*), @@ -76,7 +82,7 @@ class MockChannelReceive : public voe::ChannelReceiveInterface { GetSyncInfo, (), (const, override)); - MOCK_METHOD(void, SetMinimumPlayoutDelay, (int delay_ms), (override)); + MOCK_METHOD(bool, SetMinimumPlayoutDelay, (int delay_ms), (override)); MOCK_METHOD(bool, SetBaseMinimumPlayoutDelayMs, (int delay_ms), (override)); MOCK_METHOD(int, GetBaseMinimumPlayoutDelayMs, (), (const, override)); MOCK_METHOD((absl::optional<std::pair<int, SdpAudioFormat>>), @@ -94,6 +100,13 @@ class MockChannelReceive : public voe::ChannelReceiveInterface { SetDepacketizerToDecoderFrameTransformer, (rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer), (override)); + MOCK_METHOD( + void, + SetFrameDecryptor, + (rtc::scoped_refptr<webrtc::FrameDecryptorInterface> frame_decryptor), + (override)); + MOCK_METHOD(void, OnLocalSsrcChange, (uint32_t local_ssrc), (override)); + MOCK_METHOD(uint32_t, GetLocalSsrc, (), (const, override)); }; class MockChannelSend : public voe::ChannelSendInterface { @@ -153,7 +166,7 @@ class MockChannelSend : public voe::ChannelSendInterface { (std::unique_ptr<AudioFrame>), (override)); MOCK_METHOD(RtpRtcpInterface*, GetRtpRtcp, (), (const, override)); - MOCK_METHOD(int, GetBitrate, (), (const, override)); + MOCK_METHOD(int, GetTargetBitrate, (), (const, override)); MOCK_METHOD(int64_t, GetRTT, (), (const, override)); MOCK_METHOD(void, StartSend, (), (override)); MOCK_METHOD(void, StopSend, (), (override)); diff --git a/audio/null_audio_poller.cc b/audio/null_audio_poller.cc deleted file mode 100644 index 22f575d8bb..0000000000 --- a/audio/null_audio_poller.cc +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "audio/null_audio_poller.h" - -#include <stddef.h> - -#include "rtc_base/checks.h" -#include "rtc_base/location.h" -#include "rtc_base/thread.h" -#include "rtc_base/time_utils.h" - -namespace webrtc { -namespace internal { - -namespace { - -constexpr int64_t kPollDelayMs = 10; // WebRTC uses 10ms by default - -constexpr size_t kNumChannels = 1; -constexpr uint32_t kSamplesPerSecond = 48000; // 48kHz -constexpr size_t kNumSamples = kSamplesPerSecond / 100; // 10ms of samples - -} // namespace - -NullAudioPoller::NullAudioPoller(AudioTransport* audio_transport) - : audio_transport_(audio_transport), - reschedule_at_(rtc::TimeMillis() + kPollDelayMs) { - RTC_DCHECK(audio_transport); - OnMessage(nullptr); // Start the poll loop. -} - -NullAudioPoller::~NullAudioPoller() { - RTC_DCHECK(thread_checker_.IsCurrent()); - rtc::Thread::Current()->Clear(this); -} - -void NullAudioPoller::OnMessage(rtc::Message* msg) { - RTC_DCHECK(thread_checker_.IsCurrent()); - - // Buffer to hold the audio samples. - int16_t buffer[kNumSamples * kNumChannels]; - // Output variables from |NeedMorePlayData|. - size_t n_samples; - int64_t elapsed_time_ms; - int64_t ntp_time_ms; - audio_transport_->NeedMorePlayData(kNumSamples, sizeof(int16_t), kNumChannels, - kSamplesPerSecond, buffer, n_samples, - &elapsed_time_ms, &ntp_time_ms); - - // Reschedule the next poll iteration. If, for some reason, the given - // reschedule time has already passed, reschedule as soon as possible. - int64_t now = rtc::TimeMillis(); - if (reschedule_at_ < now) { - reschedule_at_ = now; - } - rtc::Thread::Current()->PostAt(RTC_FROM_HERE, reschedule_at_, this, 0); - - // Loop after next will be kPollDelayMs later. - reschedule_at_ += kPollDelayMs; -} - -} // namespace internal -} // namespace webrtc diff --git a/audio/null_audio_poller.h b/audio/null_audio_poller.h deleted file mode 100644 index 97cd2c7e6c..0000000000 --- a/audio/null_audio_poller.h +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#ifndef AUDIO_NULL_AUDIO_POLLER_H_ -#define AUDIO_NULL_AUDIO_POLLER_H_ - -#include <stdint.h> - -#include "modules/audio_device/include/audio_device_defines.h" -#include "rtc_base/message_handler.h" -#include "rtc_base/thread_checker.h" - -namespace webrtc { -namespace internal { - -class NullAudioPoller final : public rtc::MessageHandler { - public: - explicit NullAudioPoller(AudioTransport* audio_transport); - ~NullAudioPoller() override; - - protected: - void OnMessage(rtc::Message* msg) override; - - private: - rtc::ThreadChecker thread_checker_; - AudioTransport* const audio_transport_; - int64_t reschedule_at_; -}; - -} // namespace internal -} // namespace webrtc - -#endif // AUDIO_NULL_AUDIO_POLLER_H_ diff --git a/audio/remix_resample.cc b/audio/remix_resample.cc index 3694d34e40..178af622a1 100644 --- a/audio/remix_resample.cc +++ b/audio/remix_resample.cc @@ -56,9 +56,10 @@ void RemixAndResample(const int16_t* src_data, if (resampler->InitializeIfNeeded(sample_rate_hz, dst_frame->sample_rate_hz_, audio_ptr_num_channels) == -1) { - FATAL() << "InitializeIfNeeded failed: sample_rate_hz = " << sample_rate_hz - << ", dst_frame->sample_rate_hz_ = " << dst_frame->sample_rate_hz_ - << ", audio_ptr_num_channels = " << audio_ptr_num_channels; + RTC_FATAL() << "InitializeIfNeeded failed: sample_rate_hz = " + << sample_rate_hz << ", dst_frame->sample_rate_hz_ = " + << dst_frame->sample_rate_hz_ + << ", audio_ptr_num_channels = " << audio_ptr_num_channels; } // TODO(yujo): for muted input frames, don't resample. Either 1) allow @@ -70,9 +71,10 @@ void RemixAndResample(const int16_t* src_data, resampler->Resample(audio_ptr, src_length, dst_frame->mutable_data(), AudioFrame::kMaxDataSizeSamples); if (out_length == -1) { - FATAL() << "Resample failed: audio_ptr = " << audio_ptr - << ", src_length = " << src_length - << ", dst_frame->mutable_data() = " << dst_frame->mutable_data(); + RTC_FATAL() << "Resample failed: audio_ptr = " << audio_ptr + << ", src_length = " << src_length + << ", dst_frame->mutable_data() = " + << dst_frame->mutable_data(); } dst_frame->samples_per_channel_ = out_length / audio_ptr_num_channels; diff --git a/audio/remix_resample.h b/audio/remix_resample.h index a45270b39a..bd8da76c6a 100644 --- a/audio/remix_resample.h +++ b/audio/remix_resample.h @@ -17,19 +17,19 @@ namespace webrtc { namespace voe { -// Upmix or downmix and resample the audio to |dst_frame|. Expects |dst_frame| +// Upmix or downmix and resample the audio to `dst_frame`. Expects `dst_frame` // to have its sample rate and channels members set to the desired values. -// Updates the |samples_per_channel_| member accordingly. +// Updates the `samples_per_channel_` member accordingly. // -// This version has an AudioFrame |src_frame| as input and sets the output -// |timestamp_|, |elapsed_time_ms_| and |ntp_time_ms_| members equals to the +// This version has an AudioFrame `src_frame` as input and sets the output +// `timestamp_`, `elapsed_time_ms_` and `ntp_time_ms_` members equals to the // input ones. void RemixAndResample(const AudioFrame& src_frame, PushResampler<int16_t>* resampler, AudioFrame* dst_frame); -// This version has a pointer to the samples |src_data| as input and receives -// |samples_per_channel|, |num_channels| and |sample_rate_hz| of the data as +// This version has a pointer to the samples `src_data` as input and receives +// `samples_per_channel`, `num_channels` and `sample_rate_hz` of the data as // parameters. void RemixAndResample(const int16_t* src_data, size_t samples_per_channel, diff --git a/audio/remix_resample_unittest.cc b/audio/remix_resample_unittest.cc index d2155a64f0..31dcfac1fe 100644 --- a/audio/remix_resample_unittest.cc +++ b/audio/remix_resample_unittest.cc @@ -15,13 +15,16 @@ #include "common_audio/resampler/include/push_resampler.h" #include "rtc_base/arraysize.h" #include "rtc_base/checks.h" -#include "rtc_base/format_macros.h" #include "test/gtest.h" namespace webrtc { namespace voe { namespace { +int GetFrameSize(int sample_rate_hz) { + return sample_rate_hz / 100; +} + class UtilityTest : public ::testing::Test { protected: UtilityTest() { @@ -43,14 +46,14 @@ class UtilityTest : public ::testing::Test { AudioFrame golden_frame_; }; -// Sets the signal value to increase by |data| with every sample. Floats are +// Sets the signal value to increase by `data` with every sample. Floats are // used so non-integer values result in rounding error, but not an accumulating // error. void SetMonoFrame(float data, int sample_rate_hz, AudioFrame* frame) { frame->Mute(); frame->num_channels_ = 1; frame->sample_rate_hz_ = sample_rate_hz; - frame->samples_per_channel_ = rtc::CheckedDivExact(sample_rate_hz, 100); + frame->samples_per_channel_ = GetFrameSize(sample_rate_hz); int16_t* frame_data = frame->mutable_data(); for (size_t i = 0; i < frame->samples_per_channel_; i++) { frame_data[i] = static_cast<int16_t>(data * i); @@ -62,7 +65,7 @@ void SetMonoFrame(float data, AudioFrame* frame) { SetMonoFrame(data, frame->sample_rate_hz_, frame); } -// Sets the signal value to increase by |left| and |right| with every sample in +// Sets the signal value to increase by `left` and `right` with every sample in // each channel respectively. void SetStereoFrame(float left, float right, @@ -71,7 +74,7 @@ void SetStereoFrame(float left, frame->Mute(); frame->num_channels_ = 2; frame->sample_rate_hz_ = sample_rate_hz; - frame->samples_per_channel_ = rtc::CheckedDivExact(sample_rate_hz, 100); + frame->samples_per_channel_ = GetFrameSize(sample_rate_hz); int16_t* frame_data = frame->mutable_data(); for (size_t i = 0; i < frame->samples_per_channel_; i++) { frame_data[i * 2] = static_cast<int16_t>(left * i); @@ -84,7 +87,7 @@ void SetStereoFrame(float left, float right, AudioFrame* frame) { SetStereoFrame(left, right, frame->sample_rate_hz_, frame); } -// Sets the signal value to increase by |ch1|, |ch2|, |ch3|, |ch4| with every +// Sets the signal value to increase by `ch1`, `ch2`, `ch3`, `ch4` with every // sample in each channel respectively. void SetQuadFrame(float ch1, float ch2, @@ -95,7 +98,7 @@ void SetQuadFrame(float ch1, frame->Mute(); frame->num_channels_ = 4; frame->sample_rate_hz_ = sample_rate_hz; - frame->samples_per_channel_ = rtc::CheckedDivExact(sample_rate_hz, 100); + frame->samples_per_channel_ = GetFrameSize(sample_rate_hz); int16_t* frame_data = frame->mutable_data(); for (size_t i = 0; i < frame->samples_per_channel_; i++) { frame_data[i * 4] = static_cast<int16_t>(ch1 * i); @@ -111,8 +114,8 @@ void VerifyParams(const AudioFrame& ref_frame, const AudioFrame& test_frame) { EXPECT_EQ(ref_frame.sample_rate_hz_, test_frame.sample_rate_hz_); } -// Computes the best SNR based on the error between |ref_frame| and -// |test_frame|. It allows for up to a |max_delay| in samples between the +// Computes the best SNR based on the error between `ref_frame` and +// `test_frame`. It allows for up to a `max_delay` in samples between the // signals to compensate for the resampling delay. float ComputeSNR(const AudioFrame& ref_frame, const AudioFrame& test_frame, @@ -140,7 +143,7 @@ float ComputeSNR(const AudioFrame& ref_frame, best_delay = delay; } } - printf("SNR=%.1f dB at delay=%" RTC_PRIuS "\n", best_snr, best_delay); + printf("SNR=%.1f dB at delay=%zu\n", best_snr, best_delay); return best_snr; } @@ -212,7 +215,7 @@ void UtilityTest::RunResampleTest(int src_channels, src_channels, src_sample_rate_hz, dst_channels, dst_sample_rate_hz); RemixAndResample(src_frame_, &resampler, &dst_frame_); - if (src_sample_rate_hz == 96000 && dst_sample_rate_hz == 8000) { + if (src_sample_rate_hz == 96000 && dst_sample_rate_hz <= 11025) { // The sinc resampler gives poor SNR at this extreme conversion, but we // expect to see this rarely in practice. EXPECT_GT(ComputeSNR(golden_frame_, dst_frame_, max_delay), 14.0f); @@ -252,20 +255,16 @@ TEST_F(UtilityTest, RemixAndResampleMixingOnlySucceeds) { } TEST_F(UtilityTest, RemixAndResampleSucceeds) { - const int kSampleRates[] = {8000, 16000, 32000, 44100, 48000, 96000}; - const int kSampleRatesSize = arraysize(kSampleRates); + const int kSampleRates[] = {8000, 11025, 16000, 22050, + 32000, 44100, 48000, 96000}; const int kSrcChannels[] = {1, 2, 4}; - const int kSrcChannelsSize = arraysize(kSrcChannels); const int kDstChannels[] = {1, 2}; - const int kDstChannelsSize = arraysize(kDstChannels); - for (int src_rate = 0; src_rate < kSampleRatesSize; src_rate++) { - for (int dst_rate = 0; dst_rate < kSampleRatesSize; dst_rate++) { - for (int src_channel = 0; src_channel < kSrcChannelsSize; src_channel++) { - for (int dst_channel = 0; dst_channel < kDstChannelsSize; - dst_channel++) { - RunResampleTest(kSrcChannels[src_channel], kSampleRates[src_rate], - kDstChannels[dst_channel], kSampleRates[dst_rate]); + for (int src_rate : kSampleRates) { + for (int dst_rate : kSampleRates) { + for (size_t src_channels : kSrcChannels) { + for (size_t dst_channels : kDstChannels) { + RunResampleTest(src_channels, src_rate, dst_channels, dst_rate); } } } diff --git a/audio/test/OWNERS.webrtc b/audio/test/OWNERS.webrtc new file mode 100644 index 0000000000..3754d4823a --- /dev/null +++ b/audio/test/OWNERS.webrtc @@ -0,0 +1,3 @@ +# Script to launch low_bandwidth_audio_test. +per-file low_bandwidth_audio_test.py=mbonadei@webrtc.org +per-file low_bandwidth_audio_test.py=jleconte@webrtc.org diff --git a/audio/test/audio_bwe_integration_test.cc b/audio/test/audio_bwe_integration_test.cc index eed7acb8de..a5faf23860 100644 --- a/audio/test/audio_bwe_integration_test.cc +++ b/audio/test/audio_bwe_integration_test.cc @@ -12,7 +12,7 @@ #include <memory> -#include "api/task_queue/queued_task.h" +#include "absl/functional/any_invocable.h" #include "api/task_queue/task_queue_base.h" #include "call/fake_network_pipe.h" #include "call/simulated_network.h" @@ -35,7 +35,7 @@ enum : int { // The first valid value is 1. constexpr int kExtraProcessTimeMs = 1000; } // namespace -AudioBweTest::AudioBweTest() : EndToEndTest(CallTest::kDefaultTimeoutMs) {} +AudioBweTest::AudioBweTest() : EndToEndTest(CallTest::kDefaultTimeout) {} size_t AudioBweTest::GetNumVideoStreams() const { return 0; @@ -84,30 +84,24 @@ void AudioBweTest::PerformTest() { SleepMs(GetNetworkPipeConfig().queue_delay_ms + kExtraProcessTimeMs); } -class StatsPollTask : public QueuedTask { - public: - explicit StatsPollTask(Call* sender_call) : sender_call_(sender_call) {} - - private: - bool Run() override { - RTC_CHECK(sender_call_); - Call::Stats call_stats = sender_call_->GetStats(); +absl::AnyInvocable<void() &&> StatsPollTask(Call* sender_call) { + RTC_CHECK(sender_call); + return [sender_call] { + Call::Stats call_stats = sender_call->GetStats(); EXPECT_GT(call_stats.send_bandwidth_bps, 25000); - TaskQueueBase::Current()->PostDelayedTask(std::unique_ptr<QueuedTask>(this), - 100); - return false; - } - Call* sender_call_; -}; + TaskQueueBase::Current()->PostDelayedTask(StatsPollTask(sender_call), + TimeDelta::Millis(100)); + }; +} class NoBandwidthDropAfterDtx : public AudioBweTest { public: NoBandwidthDropAfterDtx() : sender_call_(nullptr), stats_poller_("stats poller task queue") {} - void ModifyAudioConfigs( - AudioSendStream::Config* send_config, - std::vector<AudioReceiveStream::Config>* receive_configs) override { + void ModifyAudioConfigs(AudioSendStream::Config* send_config, + std::vector<AudioReceiveStreamInterface::Config>* + receive_configs) override { send_config->send_codec_spec = AudioSendStream::Config::SendCodecSpec( test::CallTest::kAudioSendPayloadType, {"OPUS", @@ -120,7 +114,7 @@ class NoBandwidthDropAfterDtx : public AudioBweTest { send_config->rtp.extensions.push_back( RtpExtension(RtpExtension::kTransportSequenceNumberUri, kTransportSequenceNumberExtensionId)); - for (AudioReceiveStream::Config& recv_config : *receive_configs) { + for (AudioReceiveStreamInterface::Config& recv_config : *receive_configs) { recv_config.rtp.transport_cc = true; recv_config.rtp.extensions = send_config->rtp.extensions; recv_config.rtp.remote_ssrc = send_config->rtp.ssrc; @@ -144,8 +138,8 @@ class NoBandwidthDropAfterDtx : public AudioBweTest { } void PerformTest() override { - stats_poller_.PostDelayedTask(std::make_unique<StatsPollTask>(sender_call_), - 100); + stats_poller_.PostDelayedTask(StatsPollTask(sender_call_), + TimeDelta::Millis(100)); sender_call_->OnAudioTransportOverheadChanged(0); AudioBweTest::PerformTest(); } @@ -160,9 +154,6 @@ using AudioBweIntegrationTest = CallTest; // TODO(tschumim): This test is flaky when run on android and mac. Re-enable the // test for when the issue is fixed. TEST_F(AudioBweIntegrationTest, DISABLED_NoBandwidthDropAfterDtx) { - webrtc::test::ScopedFieldTrials override_field_trials( - "WebRTC-Audio-SendSideBwe/Enabled/" - "WebRTC-SendSideBwe-WithOverhead/Enabled/"); NoBandwidthDropAfterDtx test; RunBaseTest(&test); } diff --git a/audio/test/audio_end_to_end_test.cc b/audio/test/audio_end_to_end_test.cc index 896b0f2dae..de9cf7d56f 100644 --- a/audio/test/audio_end_to_end_test.cc +++ b/audio/test/audio_end_to_end_test.cc @@ -29,7 +29,7 @@ constexpr int kSampleRate = 48000; } // namespace AudioEndToEndTest::AudioEndToEndTest() - : EndToEndTest(CallTest::kDefaultTimeoutMs) {} + : EndToEndTest(CallTest::kDefaultTimeout) {} BuiltInNetworkBehaviorConfig AudioEndToEndTest::GetNetworkPipeConfig() const { return BuiltInNetworkBehaviorConfig(); @@ -86,17 +86,19 @@ AudioEndToEndTest::CreateReceiveTransport(TaskQueueBase* task_queue) { void AudioEndToEndTest::ModifyAudioConfigs( AudioSendStream::Config* send_config, - std::vector<AudioReceiveStream::Config>* receive_configs) { + std::vector<AudioReceiveStreamInterface::Config>* receive_configs) { // Large bitrate by default. const webrtc::SdpAudioFormat kDefaultFormat("opus", 48000, 2, {{"stereo", "1"}}); send_config->send_codec_spec = AudioSendStream::Config::SendCodecSpec( test::CallTest::kAudioSendPayloadType, kDefaultFormat); + send_config->min_bitrate_bps = 32000; + send_config->max_bitrate_bps = 32000; } void AudioEndToEndTest::OnAudioStreamsCreated( AudioSendStream* send_stream, - const std::vector<AudioReceiveStream*>& receive_streams) { + const std::vector<AudioReceiveStreamInterface*>& receive_streams) { ASSERT_NE(nullptr, send_stream); ASSERT_EQ(1u, receive_streams.size()); ASSERT_NE(nullptr, receive_streams[0]); diff --git a/audio/test/audio_end_to_end_test.h b/audio/test/audio_end_to_end_test.h index c47cb47076..6afa0baea3 100644 --- a/audio/test/audio_end_to_end_test.h +++ b/audio/test/audio_end_to_end_test.h @@ -28,7 +28,9 @@ class AudioEndToEndTest : public test::EndToEndTest { protected: TestAudioDeviceModule* send_audio_device() { return send_audio_device_; } const AudioSendStream* send_stream() const { return send_stream_; } - const AudioReceiveStream* receive_stream() const { return receive_stream_; } + const AudioReceiveStreamInterface* receive_stream() const { + return receive_stream_; + } virtual BuiltInNetworkBehaviorConfig GetNetworkPipeConfig() const; @@ -49,19 +51,19 @@ class AudioEndToEndTest : public test::EndToEndTest { std::unique_ptr<test::PacketTransport> CreateReceiveTransport( TaskQueueBase* task_queue) override; - void ModifyAudioConfigs( - AudioSendStream::Config* send_config, - std::vector<AudioReceiveStream::Config>* receive_configs) override; - void OnAudioStreamsCreated( - AudioSendStream* send_stream, - const std::vector<AudioReceiveStream*>& receive_streams) override; + void ModifyAudioConfigs(AudioSendStream::Config* send_config, + std::vector<AudioReceiveStreamInterface::Config>* + receive_configs) override; + void OnAudioStreamsCreated(AudioSendStream* send_stream, + const std::vector<AudioReceiveStreamInterface*>& + receive_streams) override; void PerformTest() override; private: TestAudioDeviceModule* send_audio_device_ = nullptr; AudioSendStream* send_stream_ = nullptr; - AudioReceiveStream* receive_stream_ = nullptr; + AudioReceiveStreamInterface* receive_stream_ = nullptr; }; } // namespace test diff --git a/audio/test/audio_stats_test.cc b/audio/test/audio_stats_test.cc index c91183c66b..c637bff94e 100644 --- a/audio/test/audio_stats_test.cc +++ b/audio/test/audio_stats_test.cc @@ -63,12 +63,12 @@ class NoLossTest : public AudioEndToEndTest { EXPECT_FALSE(send_stats.apm_statistics.echo_return_loss_enhancement); EXPECT_FALSE(send_stats.apm_statistics.residual_echo_likelihood); EXPECT_FALSE(send_stats.apm_statistics.residual_echo_likelihood_recent_max); - EXPECT_EQ(false, send_stats.typing_noise_detected); - AudioReceiveStream::Stats recv_stats = receive_stream()->GetStats(); + AudioReceiveStreamInterface::Stats recv_stats = + receive_stream()->GetStats(/*get_and_clear_legacy_stats=*/true); EXPECT_PRED2(IsNear, kBytesSent, recv_stats.payload_bytes_rcvd); EXPECT_PRED2(IsNear, kPacketsSent, recv_stats.packets_rcvd); - EXPECT_EQ(0u, recv_stats.packets_lost); + EXPECT_EQ(0, recv_stats.packets_lost); EXPECT_EQ("opus", send_stats.codec_name); // recv_stats.jitter_ms // recv_stats.jitter_buffer_ms diff --git a/audio/test/low_bandwidth_audio_test.cc b/audio/test/low_bandwidth_audio_test.cc index 50cf499920..948dcbc8f2 100644 --- a/audio/test/low_bandwidth_audio_test.cc +++ b/audio/test/low_bandwidth_audio_test.cc @@ -73,9 +73,9 @@ class AudioQualityTest : public AudioEndToEndTest { }; class Mobile2GNetworkTest : public AudioQualityTest { - void ModifyAudioConfigs( - AudioSendStream::Config* send_config, - std::vector<AudioReceiveStream::Config>* receive_configs) override { + void ModifyAudioConfigs(AudioSendStream::Config* send_config, + std::vector<AudioReceiveStreamInterface::Config>* + receive_configs) override { send_config->send_codec_spec = AudioSendStream::Config::SendCodecSpec( test::CallTest::kAudioSendPayloadType, {"OPUS", diff --git a/audio/test/low_bandwidth_audio_test.py b/audio/test/low_bandwidth_audio_test.py index 51273f7486..07065e2c8d 100755 --- a/audio/test/low_bandwidth_audio_test.py +++ b/audio/test/low_bandwidth_audio_test.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env vpython3 # Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. # # Use of this source code is governed by a BSD-style license @@ -6,7 +6,6 @@ # tree. An additional intellectual property rights grant can be found # in the file PATENTS. All contributing project authors may # be found in the AUTHORS file in the root of the source tree. - """ This script is the wrapper that runs the low-bandwidth audio test. @@ -16,6 +15,7 @@ output files will be performed. import argparse import collections +import json import logging import os import re @@ -23,21 +23,20 @@ import shutil import subprocess import sys - SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) SRC_DIR = os.path.normpath(os.path.join(SCRIPT_DIR, os.pardir, os.pardir)) NO_TOOLS_ERROR_MESSAGE = ( - 'Could not find PESQ or POLQA at %s.\n' - '\n' - 'To fix this run:\n' - ' python %s %s\n' - '\n' - 'Note that these tools are Google-internal due to licensing, so in order to ' - 'use them you will have to get your own license and manually put them in the ' - 'right location.\n' - 'See https://cs.chromium.org/chromium/src/third_party/webrtc/tools_webrtc/' - 'download_tools.py?rcl=bbceb76f540159e2dba0701ac03c514f01624130&l=13') + 'Could not find PESQ or POLQA at %s.\n' + '\n' + 'To fix this run:\n' + ' python %s %s\n' + '\n' + 'Note that these tools are Google-internal due to licensing, so in order ' + 'to use them you will have to get your own license and manually put them ' + 'in the right location.\n' + 'See https://cs.chromium.org/chromium/src/third_party/webrtc/tools_webrtc/' + 'download_tools.py?rcl=bbceb76f540159e2dba0701ac03c514f01624130&l=13') def _LogCommand(command): @@ -48,34 +47,38 @@ def _LogCommand(command): def _ParseArgs(): parser = argparse.ArgumentParser(description='Run low-bandwidth audio tests.') parser.add_argument('build_dir', - help='Path to the build directory (e.g. out/Release).') - parser.add_argument('--remove', action='store_true', - help='Remove output audio files after testing.') - parser.add_argument('--android', action='store_true', + help='Path to the build directory (e.g. out/Release).') + parser.add_argument('--remove', + action='store_true', + help='Remove output audio files after testing.') + parser.add_argument( + '--android', + action='store_true', help='Perform the test on a connected Android device instead.') parser.add_argument('--adb-path', help='Path to adb binary.', default='adb') - parser.add_argument('--num-retries', default='0', + parser.add_argument('--num-retries', + default='0', help='Number of times to retry the test on Android.') - parser.add_argument('--isolated-script-test-perf-output', default=None, + parser.add_argument( + '--isolated-script-test-perf-output', + default=None, help='Path to store perf results in histogram proto format.') - parser.add_argument('--extra-test-args', default=[], action='append', - help='Extra args to path to the test binary.') + parser.add_argument( + '--isolated-script-test-output', + default=None, + help='Path to output an empty JSON file which Chromium infra requires.') - # Ignore Chromium-specific flags - parser.add_argument('--test-launcher-summary-output', - type=str, default=None) - args = parser.parse_args() - - return args + return parser.parse_known_args() def _GetPlatform(): if sys.platform == 'win32': return 'win' - elif sys.platform == 'darwin': + if sys.platform == 'darwin': return 'mac' - elif sys.platform.startswith('linux'): + if sys.platform.startswith('linux'): return 'linux' + raise AssertionError('Unknown platform %s' % sys.platform) def _GetExtension(): @@ -98,10 +101,8 @@ def _GetPathToTools(): polqa_path = None if (platform != 'mac' and not polqa_path) or not pesq_path: - logging.error(NO_TOOLS_ERROR_MESSAGE, - toolchain_dir, - os.path.join(tools_dir, 'download_tools.py'), - toolchain_dir) + logging.error(NO_TOOLS_ERROR_MESSAGE, toolchain_dir, + os.path.join(tools_dir, 'download_tools.py'), toolchain_dir) return pesq_path, polqa_path @@ -126,8 +127,11 @@ def ExtractTestRuns(lines, echo=False): yield match.groups() -def _GetFile(file_path, out_dir, move=False, - android=False, adb_prefix=('adb',)): +def _GetFile(file_path, + out_dir, + move=False, + android=False, + adb_prefix=('adb', )): out_file_name = os.path.basename(file_path) out_file_path = os.path.join(out_dir, out_file_name) @@ -148,39 +152,48 @@ def _GetFile(file_path, out_dir, move=False, return out_file_path -def _RunPesq(executable_path, reference_file, degraded_file, +def _RunPesq(executable_path, + reference_file, + degraded_file, sample_rate_hz=16000): directory = os.path.dirname(reference_file) assert os.path.dirname(degraded_file) == directory # Analyze audio. - command = [executable_path, '+%d' % sample_rate_hz, - os.path.basename(reference_file), - os.path.basename(degraded_file)] + command = [ + executable_path, + '+%d' % sample_rate_hz, + os.path.basename(reference_file), + os.path.basename(degraded_file) + ] # Need to provide paths in the current directory due to a bug in PESQ: # On Mac, for some 'path/to/file.wav', if 'file.wav' is longer than # 'path/to', PESQ crashes. out = subprocess.check_output(_LogCommand(command), - cwd=directory, stderr=subprocess.STDOUT) + cwd=directory, + universal_newlines=True, + stderr=subprocess.STDOUT) # Find the scores in stdout of PESQ. match = re.search( r'Prediction \(Raw MOS, MOS-LQO\):\s+=\s+([\d.]+)\s+([\d.]+)', out) if match: raw_mos, _ = match.groups() - return {'pesq_mos': (raw_mos, 'unitless')} - else: - logging.error('PESQ: %s', out.splitlines()[-1]) - return {} + logging.error('PESQ: %s', out.splitlines()[-1]) + return {} def _RunPolqa(executable_path, reference_file, degraded_file): # Analyze audio. - command = [executable_path, '-q', '-LC', 'NB', - '-Ref', reference_file, '-Test', degraded_file] + command = [ + executable_path, '-q', '-LC', 'NB', '-Ref', reference_file, '-Test', + degraded_file + ] process = subprocess.Popen(_LogCommand(command), - stdout=subprocess.PIPE, stderr=subprocess.PIPE) + universal_newlines=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) out, err = process.communicate() # Find the scores in stdout of POLQA. @@ -212,46 +225,47 @@ def _MergeInPerfResultsFromCcTests(histograms, run_perf_results_file): histograms.Merge(cc_histograms) -Analyzer = collections.namedtuple('Analyzer', ['name', 'func', 'executable', - 'sample_rate_hz']) +Analyzer = collections.namedtuple( + 'Analyzer', ['name', 'func', 'executable', 'sample_rate_hz']) def _ConfigurePythonPath(args): script_dir = os.path.dirname(os.path.realpath(__file__)) - checkout_root = os.path.abspath( - os.path.join(script_dir, os.pardir, os.pardir)) - - # TODO(https://crbug.com/1029452): Use a copy rule and add these from the out - # dir like for the third_party/protobuf code. - sys.path.insert(0, os.path.join(checkout_root, 'third_party', 'catapult', - 'tracing')) - - # The low_bandwidth_audio_perf_test gn rule will build the protobuf stub for - # python, so put it in the path for this script before we attempt to import - # it. - histogram_proto_path = os.path.join( - os.path.abspath(args.build_dir), 'pyproto', 'tracing', 'tracing', 'proto') + checkout_root = os.path.abspath(os.path.join(script_dir, os.pardir, + os.pardir)) + + # TODO(https://crbug.com/1029452): Use a copy rule and add these from the + # out dir like for the third_party/protobuf code. + sys.path.insert( + 0, os.path.join(checkout_root, 'third_party', 'catapult', 'tracing')) + + # The low_bandwidth_audio_perf_test gn rule will build the protobuf stub + # for python, so put it in the path for this script before we attempt to + # import it. + histogram_proto_path = os.path.join(os.path.abspath(args.build_dir), + 'pyproto', 'tracing', 'tracing', 'proto') sys.path.insert(0, histogram_proto_path) proto_stub_path = os.path.join(os.path.abspath(args.build_dir), 'pyproto') sys.path.insert(0, proto_stub_path) # Fail early in case the proto hasn't been built. try: + #pylint: disable=unused-import import histogram_pb2 except ImportError as e: - logging.exception(e) raise ImportError('Could not import histogram_pb2. You need to build the ' 'low_bandwidth_audio_perf_test target before invoking ' 'this script. Expected to find ' - 'histogram_pb2.py in %s.' % histogram_proto_path) + 'histogram_pb2.py in %s.' % histogram_proto_path) from e def main(): - # pylint: disable=W0101 - logging.basicConfig(level=logging.INFO) + logging.basicConfig(format='%(asctime)s %(levelname)-8s %(message)s', + level=logging.INFO, + datefmt='%Y-%m-%d %H:%M:%S') logging.info('Invoked with %s', str(sys.argv)) - args = _ParseArgs() + args, extra_test_args = _ParseArgs() _ConfigurePythonPath(args) @@ -266,28 +280,30 @@ def main(): out_dir = os.path.join(args.build_dir, '..') if args.android: - test_command = [os.path.join(args.build_dir, 'bin', - 'run_low_bandwidth_audio_test'), - '-v', '--num-retries', args.num_retries] + test_command = [ + os.path.join(args.build_dir, 'bin', 'run_low_bandwidth_audio_test'), + '-v', '--num-retries', args.num_retries + ] else: test_command = [os.path.join(args.build_dir, 'low_bandwidth_audio_test')] analyzers = [Analyzer('pesq', _RunPesq, pesq_path, 16000)] # Check if POLQA can run at all, or skip the 48 kHz tests entirely. - example_path = os.path.join(SRC_DIR, 'resources', - 'voice_engine', 'audio_tiny48.wav') + example_path = os.path.join(SRC_DIR, 'resources', 'voice_engine', + 'audio_tiny48.wav') if polqa_path and _RunPolqa(polqa_path, example_path, example_path): analyzers.append(Analyzer('polqa', _RunPolqa, polqa_path, 48000)) histograms = histogram_set.HistogramSet() for analyzer in analyzers: # Start the test executable that produces audio files. - test_process = subprocess.Popen( - _LogCommand(test_command + [ - '--sample_rate_hz=%d' % analyzer.sample_rate_hz, - '--test_case_prefix=%s' % analyzer.name, - ] + args.extra_test_args), - stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + test_process = subprocess.Popen(_LogCommand(test_command + [ + '--sample_rate_hz=%d' % analyzer.sample_rate_hz, + '--test_case_prefix=%s' % analyzer.name, + ] + extra_test_args), + universal_newlines=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) perf_results_file = None try: lines = iter(test_process.stdout.readline, '') @@ -295,24 +311,29 @@ def main(): (android_device, test_name, reference_file, degraded_file, perf_results_file) = result - adb_prefix = (args.adb_path,) + adb_prefix = (args.adb_path, ) if android_device: adb_prefix += ('-s', android_device) - reference_file = _GetFile(reference_file, out_dir, - android=args.android, adb_prefix=adb_prefix) - degraded_file = _GetFile(degraded_file, out_dir, move=True, - android=args.android, adb_prefix=adb_prefix) - - analyzer_results = analyzer.func(analyzer.executable, - reference_file, degraded_file) - for metric, (value, units) in analyzer_results.items(): + reference_file = _GetFile(reference_file, + out_dir, + android=args.android, + adb_prefix=adb_prefix) + degraded_file = _GetFile(degraded_file, + out_dir, + move=True, + android=args.android, + adb_prefix=adb_prefix) + + analyzer_results = analyzer.func(analyzer.executable, reference_file, + degraded_file) + for metric, (value, units) in list(analyzer_results.items()): hist = histograms.CreateHistogram(metric, units, [value]) user_story = generic_set.GenericSet([test_name]) hist.diagnostics[reserved_infos.STORIES.name] = user_story # Output human readable results. - print 'RESULT %s: %s= %s %s' % (metric, test_name, value, units) + print('RESULT %s: %s= %s %s' % (metric, test_name, value, units)) if args.remove: os.remove(reference_file) @@ -320,8 +341,11 @@ def main(): finally: test_process.terminate() if perf_results_file: - perf_results_file = _GetFile(perf_results_file, out_dir, move=True, - android=args.android, adb_prefix=adb_prefix) + perf_results_file = _GetFile(perf_results_file, + out_dir, + move=True, + android=args.android, + adb_prefix=adb_prefix) _MergeInPerfResultsFromCcTests(histograms, perf_results_file) if args.remove: os.remove(perf_results_file) @@ -330,6 +354,10 @@ def main(): with open(args.isolated_script_test_perf_output, 'wb') as f: f.write(histograms.AsProto().SerializeToString()) + if args.isolated_script_test_output: + with open(args.isolated_script_test_output, 'w') as f: + json.dump({"version": 3}, f) + return test_process.wait() diff --git a/audio/test/nack_test.cc b/audio/test/nack_test.cc new file mode 100644 index 0000000000..f383627dbe --- /dev/null +++ b/audio/test/nack_test.cc @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "audio/test/audio_end_to_end_test.h" +#include "system_wrappers/include/sleep.h" +#include "test/gtest.h" + +namespace webrtc { +namespace test { + +using NackTest = CallTest; + +TEST_F(NackTest, ShouldNackInLossyNetwork) { + class NackTest : public AudioEndToEndTest { + public: + const int kTestDurationMs = 2000; + const int64_t kRttMs = 30; + const int64_t kLossPercent = 30; + const int kNackHistoryMs = 1000; + + BuiltInNetworkBehaviorConfig GetNetworkPipeConfig() const override { + BuiltInNetworkBehaviorConfig pipe_config; + pipe_config.queue_delay_ms = kRttMs / 2; + pipe_config.loss_percent = kLossPercent; + return pipe_config; + } + + void ModifyAudioConfigs(AudioSendStream::Config* send_config, + std::vector<AudioReceiveStreamInterface::Config>* + receive_configs) override { + ASSERT_EQ(receive_configs->size(), 1U); + (*receive_configs)[0].rtp.nack.rtp_history_ms = kNackHistoryMs; + AudioEndToEndTest::ModifyAudioConfigs(send_config, receive_configs); + } + + void PerformTest() override { SleepMs(kTestDurationMs); } + + void OnStreamsStopped() override { + AudioReceiveStreamInterface::Stats recv_stats = + receive_stream()->GetStats(/*get_and_clear_legacy_stats=*/true); + EXPECT_GT(recv_stats.nacks_sent, 0U); + AudioSendStream::Stats send_stats = send_stream()->GetStats(); + EXPECT_GT(send_stats.retransmitted_packets_sent, 0U); + EXPECT_GT(send_stats.nacks_rcvd, 0U); + } + } test; + + RunBaseTest(&test); +} + +} // namespace test +} // namespace webrtc diff --git a/audio/test/non_sender_rtt_test.cc b/audio/test/non_sender_rtt_test.cc new file mode 100644 index 0000000000..07de99ac37 --- /dev/null +++ b/audio/test/non_sender_rtt_test.cc @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "audio/test/audio_end_to_end_test.h" +#include "system_wrappers/include/sleep.h" +#include "test/gtest.h" + +namespace webrtc { +namespace test { + +using NonSenderRttTest = CallTest; + +TEST_F(NonSenderRttTest, NonSenderRttStats) { + class NonSenderRttTest : public AudioEndToEndTest { + public: + const int kTestDurationMs = 10000; + const int64_t kRttMs = 30; + + BuiltInNetworkBehaviorConfig GetNetworkPipeConfig() const override { + BuiltInNetworkBehaviorConfig pipe_config; + pipe_config.queue_delay_ms = kRttMs / 2; + return pipe_config; + } + + void ModifyAudioConfigs(AudioSendStream::Config* send_config, + std::vector<AudioReceiveStreamInterface::Config>* + receive_configs) override { + ASSERT_EQ(receive_configs->size(), 1U); + (*receive_configs)[0].enable_non_sender_rtt = true; + AudioEndToEndTest::ModifyAudioConfigs(send_config, receive_configs); + send_config->send_codec_spec->enable_non_sender_rtt = true; + } + + void PerformTest() override { SleepMs(kTestDurationMs); } + + void OnStreamsStopped() override { + AudioReceiveStreamInterface::Stats recv_stats = + receive_stream()->GetStats(/*get_and_clear_legacy_stats=*/true); + EXPECT_GT(recv_stats.round_trip_time_measurements, 0); + ASSERT_TRUE(recv_stats.round_trip_time.has_value()); + EXPECT_GT(recv_stats.round_trip_time->ms(), 0); + EXPECT_GE(recv_stats.total_round_trip_time.ms(), + recv_stats.round_trip_time->ms()); + } + } test; + + RunBaseTest(&test); +} + +} // namespace test +} // namespace webrtc diff --git a/audio/test/pc_low_bandwidth_audio_test.cc b/audio/test/pc_low_bandwidth_audio_test.cc index 95a32238c5..8b733d578d 100644 --- a/audio/test/pc_low_bandwidth_audio_test.cc +++ b/audio/test/pc_low_bandwidth_audio_test.cc @@ -12,9 +12,17 @@ #include "absl/flags/declare.h" #include "absl/flags/flag.h" +#include "absl/strings/string_view.h" #include "api/test/create_network_emulation_manager.h" #include "api/test/create_peerconnection_quality_test_fixture.h" +#include "api/test/metrics/chrome_perf_dashboard_metrics_exporter.h" +#include "api/test/metrics/global_metrics_logger_and_exporter.h" +#include "api/test/metrics/metrics_exporter.h" +#include "api/test/metrics/stdout_metrics_exporter.h" #include "api/test/network_emulation_manager.h" +#include "api/test/pclf/media_configuration.h" +#include "api/test/pclf/media_quality_test_params.h" +#include "api/test/pclf/peer_configurer.h" #include "api/test/peerconnection_quality_test_fixture.h" #include "api/test/simulated_network.h" #include "api/test/time_controller.h" @@ -22,7 +30,6 @@ #include "test/gtest.h" #include "test/pc/e2e/network_quality_metrics_reporter.h" #include "test/testsupport/file_utils.h" -#include "test/testsupport/perf_test.h" ABSL_DECLARE_FLAG(std::string, test_case_prefix); ABSL_DECLARE_FLAG(int, sample_rate_hz); @@ -31,11 +38,9 @@ ABSL_DECLARE_FLAG(bool, quick); namespace webrtc { namespace test { -using PeerConfigurer = - webrtc_pc_e2e::PeerConnectionE2EQualityTestFixture::PeerConfigurer; -using RunParams = webrtc_pc_e2e::PeerConnectionE2EQualityTestFixture::RunParams; -using AudioConfig = - webrtc_pc_e2e::PeerConnectionE2EQualityTestFixture::AudioConfig; +using ::webrtc::webrtc_pc_e2e::AudioConfig; +using ::webrtc::webrtc_pc_e2e::PeerConfigurer; +using ::webrtc::webrtc_pc_e2e::RunParams; namespace { @@ -52,41 +57,29 @@ std::string GetMetricTestCaseName() { return test_case_prefix + "_" + test_info->name(); } -std::pair<EmulatedNetworkManagerInterface*, EmulatedNetworkManagerInterface*> -CreateTwoNetworkLinks(NetworkEmulationManager* emulation, - const BuiltInNetworkBehaviorConfig& config) { - auto* alice_node = emulation->CreateEmulatedNode(config); - auto* bob_node = emulation->CreateEmulatedNode(config); - - auto* alice_endpoint = emulation->CreateEndpoint(EmulatedEndpointConfig()); - auto* bob_endpoint = emulation->CreateEndpoint(EmulatedEndpointConfig()); - - emulation->CreateRoute(alice_endpoint, {alice_node}, bob_endpoint); - emulation->CreateRoute(bob_endpoint, {bob_node}, alice_endpoint); - - return { - emulation->CreateEmulatedNetworkManagerInterface({alice_endpoint}), - emulation->CreateEmulatedNetworkManagerInterface({bob_endpoint}), - }; -} - std::unique_ptr<webrtc_pc_e2e::PeerConnectionE2EQualityTestFixture> -CreateTestFixture(const std::string& test_case_name, +CreateTestFixture(absl::string_view test_case_name, TimeController& time_controller, std::pair<EmulatedNetworkManagerInterface*, EmulatedNetworkManagerInterface*> network_links, rtc::FunctionView<void(PeerConfigurer*)> alice_configurer, rtc::FunctionView<void(PeerConfigurer*)> bob_configurer) { auto fixture = webrtc_pc_e2e::CreatePeerConnectionE2EQualityTestFixture( - test_case_name, time_controller, /*audio_quality_analyzer=*/nullptr, + std::string(test_case_name), time_controller, + /*audio_quality_analyzer=*/nullptr, /*video_quality_analyzer=*/nullptr); - fixture->AddPeer(network_links.first->network_thread(), - network_links.first->network_manager(), alice_configurer); - fixture->AddPeer(network_links.second->network_thread(), - network_links.second->network_manager(), bob_configurer); + auto alice = std::make_unique<PeerConfigurer>( + network_links.first->network_dependencies()); + auto bob = std::make_unique<PeerConfigurer>( + network_links.second->network_dependencies()); + alice_configurer(alice.get()); + bob_configurer(bob.get()); + fixture->AddPeer(std::move(alice)); + fixture->AddPeer(std::move(bob)); fixture->AddQualityMetricsReporter( std::make_unique<webrtc_pc_e2e::NetworkQualityMetricsReporter>( - network_links.first, network_links.second)); + network_links.first, network_links.second, + test::GetGlobalMetricsLogger())); return fixture; } @@ -113,7 +106,12 @@ std::string PerfResultsOutputFile() { void LogTestResults() { std::string perf_results_output_file = PerfResultsOutputFile(); - EXPECT_TRUE(webrtc::test::WritePerfResults(perf_results_output_file)); + std::vector<std::unique_ptr<MetricsExporter>> exporters; + exporters.push_back(std::make_unique<StdoutMetricsExporter>()); + exporters.push_back(std::make_unique<ChromePerfDashboardMetricsExporter>( + perf_results_output_file)); + EXPECT_TRUE( + ExportPerfMetric(*GetGlobalMetricsLogger(), std::move(exporters))); const ::testing::TestInfo* const test_info = ::testing::UnitTest::GetInstance()->current_test_info(); @@ -131,8 +129,8 @@ TEST(PCLowBandwidthAudioTest, PCGoodNetworkHighBitrate) { CreateNetworkEmulationManager(); auto fixture = CreateTestFixture( GetMetricTestCaseName(), *network_emulation_manager->time_controller(), - CreateTwoNetworkLinks(network_emulation_manager.get(), - BuiltInNetworkBehaviorConfig()), + network_emulation_manager->CreateEndpointPairWithTwoWayRoutes( + BuiltInNetworkBehaviorConfig()), [](PeerConfigurer* alice) { AudioConfig audio; audio.stream_label = "alice-audio"; @@ -158,7 +156,7 @@ TEST(PCLowBandwidthAudioTest, PC40kbpsNetwork) { config.loss_percent = 1; auto fixture = CreateTestFixture( GetMetricTestCaseName(), *network_emulation_manager->time_controller(), - CreateTwoNetworkLinks(network_emulation_manager.get(), config), + network_emulation_manager->CreateEndpointPairWithTwoWayRoutes(config), [](PeerConfigurer* alice) { AudioConfig audio; audio.stream_label = "alice-audio"; diff --git a/audio/test/unittests/low_bandwidth_audio_test_test.py b/audio/test/unittests/low_bandwidth_audio_test_test.py index 7403663cd4..be72fcbfdd 100755 --- a/audio/test/unittests/low_bandwidth_audio_test_test.py +++ b/audio/test/unittests/low_bandwidth_audio_test_test.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. # # Use of this source code is governed by a BSD-style license @@ -7,11 +7,11 @@ # in the file PATENTS. All contributing project authors may # be found in the AUTHORS file in the root of the source tree. +from __future__ import absolute_import import os import unittest import sys - SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) PARENT_DIR = os.path.join(SCRIPT_DIR, os.pardir) sys.path.append(PARENT_DIR) @@ -25,7 +25,8 @@ class TestExtractTestRuns(unittest.TestCase): expected) def testLinux(self): - self._TestLog(LINUX_LOG, + self._TestLog( + LINUX_LOG, (None, 'GoodNetworkHighBitrate', '/webrtc/src/resources/voice_engine/audio_tiny16.wav', '/webrtc/src/out/LowBandwidth_GoodNetworkHighBitrate.wav', None), @@ -42,7 +43,8 @@ class TestExtractTestRuns(unittest.TestCase): '/webrtc/src/out/PCLowBandwidth_perf_48.json')) def testAndroid(self): - self._TestLog(ANDROID_LOG, + self._TestLog( + ANDROID_LOG, ('ddfa6149', 'Mobile2GNetwork', '/sdcard/chromium_tests_root/resources/voice_engine/audio_tiny16.wav', '/sdcard/chromium_tests_root/LowBandwidth_Mobile2GNetwork.wav', None), @@ -57,7 +59,7 @@ class TestExtractTestRuns(unittest.TestCase): ('TA99205CNO', 'PCGoodNetworkHighBitrate', '/sdcard/chromium_tests_root/resources/voice_engine/audio_tiny16.wav', ('/sdcard/chromium_tests_root/' - 'PCLowBandwidth_PCGoodNetworkHighBitrate.wav'), + 'PCLowBandwidth_PCGoodNetworkHighBitrate.wav'), '/sdcard/chromium_tests_root/PCLowBandwidth_perf_48.json')) @@ -233,6 +235,5 @@ I 16.608s tear_down_device(ddfa6149) Wrote device cache: /webrtc/src/out/debu I 16.608s tear_down_device(TA99205CNO) Wrote device cache: /webrtc/src/out/debug-android/device_cache_TA99305CMO.json ''' - if __name__ == "__main__": unittest.main() diff --git a/audio/utility/BUILD.gn b/audio/utility/BUILD.gn index 54ca04698d..983b6286e4 100644 --- a/audio/utility/BUILD.gn +++ b/audio/utility/BUILD.gn @@ -26,10 +26,11 @@ rtc_library("audio_frame_operations") { "../../api/audio:audio_frame_api", "../../common_audio", "../../rtc_base:checks", - "../../rtc_base:deprecation", - "../../rtc_base:rtc_base_approved", + "../../rtc_base:logging", + "../../rtc_base:safe_conversions", "../../system_wrappers:field_trial", ] + absl_deps = [ "//third_party/abseil-cpp/absl/base:core_headers" ] } if (rtc_include_tests) { @@ -44,7 +45,9 @@ if (rtc_include_tests) { ":audio_frame_operations", "../../api/audio:audio_frame_api", "../../rtc_base:checks", - "../../rtc_base:rtc_base_approved", + "../../rtc_base:logging", + "../../rtc_base:macromagic", + "../../rtc_base:stringutils", "../../test:field_trial", "../../test:test_support", "//testing/gtest", diff --git a/audio/utility/audio_frame_operations.cc b/audio/utility/audio_frame_operations.cc index a9d2cf1632..1b936c239b 100644 --- a/audio/utility/audio_frame_operations.cc +++ b/audio/utility/audio_frame_operations.cc @@ -131,8 +131,8 @@ void AudioFrameOperations::DownmixChannels(const int16_t* src_audio, return; } - RTC_NOTREACHED() << "src_channels: " << src_channels - << ", dst_channels: " << dst_channels; + RTC_DCHECK_NOTREACHED() << "src_channels: " << src_channels + << ", dst_channels: " << dst_channels; } void AudioFrameOperations::DownmixChannels(size_t dst_channels, @@ -149,8 +149,8 @@ void AudioFrameOperations::DownmixChannels(size_t dst_channels, int err = QuadToStereo(frame); RTC_DCHECK_EQ(err, 0); } else { - RTC_NOTREACHED() << "src_channels: " << frame->num_channels_ - << ", dst_channels: " << dst_channels; + RTC_DCHECK_NOTREACHED() << "src_channels: " << frame->num_channels_ + << ", dst_channels: " << dst_channels; } } @@ -169,10 +169,10 @@ void AudioFrameOperations::UpmixChannels(size_t target_number_of_channels, if (!frame->muted()) { // Up-mixing done in place. Going backwards through the frame ensure nothing // is irrevocably overwritten. + int16_t* frame_data = frame->mutable_data(); for (int i = frame->samples_per_channel_ - 1; i >= 0; i--) { for (size_t j = 0; j < target_number_of_channels; ++j) { - frame->mutable_data()[target_number_of_channels * i + j] = - frame->data()[i]; + frame_data[target_number_of_channels * i + j] = frame_data[i]; } } } @@ -222,14 +222,14 @@ void AudioFrameOperations::Mute(AudioFrame* frame, size_t end = count; float start_g = 0.0f; if (current_frame_muted) { - // Fade out the last |count| samples of frame. + // Fade out the last `count` samples of frame. RTC_DCHECK(!previous_frame_muted); start = frame->samples_per_channel_ - count; end = frame->samples_per_channel_; start_g = 1.0f; inc = -inc; } else { - // Fade in the first |count| samples of frame. + // Fade in the first `count` samples of frame. RTC_DCHECK(previous_frame_muted); } diff --git a/audio/utility/audio_frame_operations.h b/audio/utility/audio_frame_operations.h index 65c310c489..2a5f29f4f5 100644 --- a/audio/utility/audio_frame_operations.h +++ b/audio/utility/audio_frame_operations.h @@ -14,8 +14,8 @@ #include <stddef.h> #include <stdint.h> +#include "absl/base/attributes.h" #include "api/audio/audio_frame.h" -#include "rtc_base/deprecation.h" namespace webrtc { @@ -24,38 +24,40 @@ namespace webrtc { // than a class. class AudioFrameOperations { public: - // Add samples in |frame_to_add| with samples in |result_frame| - // putting the results in |results_frame|. The fields - // |vad_activity_| and |speech_type_| of the result frame are - // updated. If |result_frame| is empty (|samples_per_channel_|==0), - // the samples in |frame_to_add| are added to it. The number of + // Add samples in `frame_to_add` with samples in `result_frame` + // putting the results in `results_frame`. The fields + // `vad_activity_` and `speech_type_` of the result frame are + // updated. If `result_frame` is empty (`samples_per_channel_`==0), + // the samples in `frame_to_add` are added to it. The number of // channels and number of samples per channel must match except when - // |result_frame| is empty. + // `result_frame` is empty. static void Add(const AudioFrame& frame_to_add, AudioFrame* result_frame); - // |frame.num_channels_| will be updated. This version checks for sufficient - // buffer size and that |num_channels_| is mono. Use UpmixChannels + // `frame.num_channels_` will be updated. This version checks for sufficient + // buffer size and that `num_channels_` is mono. Use UpmixChannels // instead. TODO(bugs.webrtc.org/8649): remove. - RTC_DEPRECATED static int MonoToStereo(AudioFrame* frame); + ABSL_DEPRECATED("bugs.webrtc.org/8649") + static int MonoToStereo(AudioFrame* frame); - // |frame.num_channels_| will be updated. This version checks that - // |num_channels_| is stereo. Use DownmixChannels + // `frame.num_channels_` will be updated. This version checks that + // `num_channels_` is stereo. Use DownmixChannels // instead. TODO(bugs.webrtc.org/8649): remove. - RTC_DEPRECATED static int StereoToMono(AudioFrame* frame); + ABSL_DEPRECATED("bugs.webrtc.org/8649") + static int StereoToMono(AudioFrame* frame); - // Downmixes 4 channels |src_audio| to stereo |dst_audio|. This is an in-place - // operation, meaning |src_audio| and |dst_audio| may point to the same + // Downmixes 4 channels `src_audio` to stereo `dst_audio`. This is an in-place + // operation, meaning `src_audio` and `dst_audio` may point to the same // buffer. static void QuadToStereo(const int16_t* src_audio, size_t samples_per_channel, int16_t* dst_audio); - // |frame.num_channels_| will be updated. This version checks that - // |num_channels_| is 4 channels. + // `frame.num_channels_` will be updated. This version checks that + // `num_channels_` is 4 channels. static int QuadToStereo(AudioFrame* frame); - // Downmixes |src_channels| |src_audio| to |dst_channels| |dst_audio|. - // This is an in-place operation, meaning |src_audio| and |dst_audio| + // Downmixes `src_channels` `src_audio` to `dst_channels` `dst_audio`. + // This is an in-place operation, meaning `src_audio` and `dst_audio` // may point to the same buffer. Supported channel combinations are // Stereo to Mono, Quad to Mono, and Quad to Stereo. static void DownmixChannels(const int16_t* src_audio, @@ -64,27 +66,27 @@ class AudioFrameOperations { size_t dst_channels, int16_t* dst_audio); - // |frame.num_channels_| will be updated. This version checks that - // |num_channels_| and |dst_channels| are valid and performs relevant downmix. + // `frame.num_channels_` will be updated. This version checks that + // `num_channels_` and `dst_channels` are valid and performs relevant downmix. // Supported channel combinations are N channels to Mono, and Quad to Stereo. static void DownmixChannels(size_t dst_channels, AudioFrame* frame); - // |frame.num_channels_| will be updated. This version checks that - // |num_channels_| and |dst_channels| are valid and performs relevant + // `frame.num_channels_` will be updated. This version checks that + // `num_channels_` and `dst_channels` are valid and performs relevant // downmix. Supported channel combinations are Mono to N // channels. The single channel is replicated. static void UpmixChannels(size_t target_number_of_channels, AudioFrame* frame); - // Swap the left and right channels of |frame|. Fails silently if |frame| is + // Swap the left and right channels of `frame`. Fails silently if `frame` is // not stereo. static void SwapStereoChannels(AudioFrame* frame); - // Conditionally zero out contents of |frame| for implementing audio mute: - // |previous_frame_muted| && |current_frame_muted| - Zero out whole frame. - // |previous_frame_muted| && !|current_frame_muted| - Fade-in at frame start. - // !|previous_frame_muted| && |current_frame_muted| - Fade-out at frame end. - // !|previous_frame_muted| && !|current_frame_muted| - Leave frame untouched. + // Conditionally zero out contents of `frame` for implementing audio mute: + // `previous_frame_muted` && `current_frame_muted` - Zero out whole frame. + // `previous_frame_muted` && !`current_frame_muted` - Fade-in at frame start. + // !`previous_frame_muted` && `current_frame_muted` - Fade-out at frame end. + // !`previous_frame_muted` && !`current_frame_muted` - Leave frame untouched. static void Mute(AudioFrame* frame, bool previous_frame_muted, bool current_frame_muted); @@ -92,7 +94,7 @@ class AudioFrameOperations { // Zero out contents of frame. static void Mute(AudioFrame* frame); - // Halve samples in |frame|. + // Halve samples in `frame`. static void ApplyHalfGain(AudioFrame* frame); static int Scale(float left, float right, AudioFrame* frame); diff --git a/audio/utility/channel_mixer.cc b/audio/utility/channel_mixer.cc index 8867a3eed4..0f1e663873 100644 --- a/audio/utility/channel_mixer.cc +++ b/audio/utility/channel_mixer.cc @@ -90,7 +90,7 @@ void ChannelMixer::Transform(AudioFrame* frame) { frame->num_channels_ = output_channels_; frame->channel_layout_ = output_layout_; - // Copy the output result to the audio frame in |frame|. + // Copy the output result to the audio frame in `frame`. memcpy( frame->mutable_data(), out_audio, sizeof(int16_t) * frame->samples_per_channel() * frame->num_channels()); diff --git a/audio/utility/channel_mixer.h b/audio/utility/channel_mixer.h index 8b6b7f517d..2dea8eb45b 100644 --- a/audio/utility/channel_mixer.h +++ b/audio/utility/channel_mixer.h @@ -38,8 +38,8 @@ class ChannelMixer { ChannelMixer(ChannelLayout input_layout, ChannelLayout output_layout); ~ChannelMixer(); - // Transforms all input channels corresponding to the selected |input_layout| - // to the number of channels in the selected |output_layout|. + // Transforms all input channels corresponding to the selected `input_layout` + // to the number of channels in the selected `output_layout`. // Example usage (downmix from stereo to mono): // // ChannelMixer mixer(CHANNEL_LAYOUT_STEREO, CHANNEL_LAYOUT_MONO); @@ -69,11 +69,11 @@ class ChannelMixer { // 1D array used as temporary storage during the transformation. std::unique_ptr<int16_t[]> audio_vector_; - // Number of elements allocated for |audio_vector_|. + // Number of elements allocated for `audio_vector_`. size_t audio_vector_size_ = 0; // Optimization case for when we can simply remap the input channels to output - // channels, i.e., when all scaling factors in |matrix_| equals 1.0. + // channels, i.e., when all scaling factors in `matrix_` equals 1.0. bool remapping_; // Delete the copy constructor and assignment operator. diff --git a/audio/utility/channel_mixing_matrix.cc b/audio/utility/channel_mixing_matrix.cc index 4baff8bfba..1244653f63 100644 --- a/audio/utility/channel_mixing_matrix.cc +++ b/audio/utility/channel_mixing_matrix.cc @@ -274,7 +274,7 @@ bool ChannelMixingMatrix::CreateTransformationMatrix( // All channels should now be accounted for. RTC_DCHECK(unaccounted_inputs_.empty()); - // See if the output |matrix_| is simply a remapping matrix. If each input + // See if the output `matrix_` is simply a remapping matrix. If each input // channel maps to a single output channel we can simply remap. Doing this // programmatically is less fragile than logic checks on channel mappings. for (int output_ch = 0; output_ch < output_channels_; ++output_ch) { @@ -287,7 +287,7 @@ bool ChannelMixingMatrix::CreateTransformationMatrix( } } - // If we've gotten here, |matrix_| is simply a remapping. + // If we've gotten here, `matrix_` is simply a remapping. return true; } diff --git a/audio/utility/channel_mixing_matrix.h b/audio/utility/channel_mixing_matrix.h index 7aef47b3b2..ee00860846 100644 --- a/audio/utility/channel_mixing_matrix.h +++ b/audio/utility/channel_mixing_matrix.h @@ -29,7 +29,7 @@ class ChannelMixingMatrix { // Create the transformation matrix of input channels to output channels. // Updates the empty matrix with the transformation, and returns true // if the transformation is just a remapping of channels (no mixing). - // The size of |matrix| is |output_channels| x |input_channels|, i.e., the + // The size of `matrix` is `output_channels` x `input_channels`, i.e., the // number of rows equals the number of output channels and the number of // columns corresponds to the number of input channels. // This file is derived from Chromium's media/base/channel_mixing_matrix.h. @@ -55,14 +55,14 @@ class ChannelMixingMatrix { void AccountFor(Channels ch); bool IsUnaccounted(Channels ch) const; - // Helper methods for checking if |ch| exists in either |input_layout_| or - // |output_layout_| respectively. + // Helper methods for checking if `ch` exists in either `input_layout_` or + // `output_layout_` respectively. bool HasInputChannel(Channels ch) const; bool HasOutputChannel(Channels ch) const; - // Helper methods for updating |matrix_| with the proper value for - // mixing |input_ch| into |output_ch|. MixWithoutAccounting() does not - // remove the channel from |unaccounted_inputs_|. + // Helper methods for updating `matrix_` with the proper value for + // mixing `input_ch` into `output_ch`. MixWithoutAccounting() does not + // remove the channel from `unaccounted_inputs_`. void Mix(Channels input_ch, Channels output_ch, float scale); void MixWithoutAccounting(Channels input_ch, Channels output_ch, float scale); diff --git a/audio/voip/BUILD.gn b/audio/voip/BUILD.gn index 52f9d07f17..1b8c29e5a7 100644 --- a/audio/voip/BUILD.gn +++ b/audio/voip/BUILD.gn @@ -23,7 +23,6 @@ rtc_library("voip_core") { "../../modules/audio_device:audio_device_api", "../../modules/audio_mixer:audio_mixer_impl", "../../modules/audio_processing:api", - "../../modules/utility:utility", "../../rtc_base:criticalsection", "../../rtc_base:logging", "../../rtc_base/synchronization:mutex", @@ -46,11 +45,9 @@ rtc_library("audio_channel") { "../../modules/audio_device:audio_device_api", "../../modules/rtp_rtcp", "../../modules/rtp_rtcp:rtp_rtcp_format", - "../../modules/utility", "../../rtc_base:criticalsection", "../../rtc_base:logging", "../../rtc_base:refcount", - "../../rtc_base:rtc_base_approved", ] } @@ -67,10 +64,10 @@ rtc_library("audio_ingress") { "../../api:transport_api", "../../api/audio:audio_mixer_api", "../../api/audio_codecs:audio_codecs_api", + "../../api/voip:voip_api", "../../modules/audio_coding", "../../modules/rtp_rtcp", "../../modules/rtp_rtcp:rtp_rtcp_format", - "../../modules/utility", "../../rtc_base:criticalsection", "../../rtc_base:logging", "../../rtc_base:safe_minmax", @@ -78,6 +75,7 @@ rtc_library("audio_ingress") { "../../rtc_base/synchronization:mutex", "../utility:audio_frame_operations", ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] } rtc_library("audio_egress") { @@ -87,6 +85,7 @@ rtc_library("audio_egress") { ] deps = [ "..:audio", + "../../api:sequence_checker", "../../api/audio_codecs:audio_codecs_api", "../../api/task_queue", "../../call:audio_sender_interface", @@ -95,9 +94,9 @@ rtc_library("audio_egress") { "../../modules/rtp_rtcp:rtp_rtcp_format", "../../rtc_base:logging", "../../rtc_base:rtc_task_queue", - "../../rtc_base:thread_checker", "../../rtc_base:timeutils", "../../rtc_base/synchronization:mutex", + "../../rtc_base/system:no_unique_address", "../utility:audio_frame_operations", ] } diff --git a/audio/voip/audio_channel.cc b/audio/voip/audio_channel.cc index d9c89fcdc4..a70e33ec38 100644 --- a/audio/voip/audio_channel.cc +++ b/audio/voip/audio_channel.cc @@ -17,7 +17,6 @@ #include "api/task_queue/task_queue_factory.h" #include "modules/rtp_rtcp/include/receive_statistics.h" #include "modules/rtp_rtcp/source/rtp_rtcp_impl2.h" -#include "rtc_base/location.h" #include "rtc_base/logging.h" namespace webrtc { @@ -32,12 +31,10 @@ AudioChannel::AudioChannel( Transport* transport, uint32_t local_ssrc, TaskQueueFactory* task_queue_factory, - ProcessThread* process_thread, AudioMixer* audio_mixer, rtc::scoped_refptr<AudioDecoderFactory> decoder_factory) - : audio_mixer_(audio_mixer), process_thread_(process_thread) { + : audio_mixer_(audio_mixer) { RTC_DCHECK(task_queue_factory); - RTC_DCHECK(process_thread); RTC_DCHECK(audio_mixer); Clock* clock = Clock::GetRealTimeClock(); @@ -56,9 +53,6 @@ AudioChannel::AudioChannel( rtp_rtcp_->SetSendingMediaStatus(false); rtp_rtcp_->SetRTCPStatus(RtcpMode::kCompound); - // ProcessThread periodically services RTP stack for RTCP. - process_thread_->RegisterModule(rtp_rtcp_.get(), RTC_FROM_HERE); - ingress_ = std::make_unique<AudioIngress>(rtp_rtcp_.get(), clock, receive_statistics_.get(), std::move(decoder_factory)); @@ -79,48 +73,101 @@ AudioChannel::~AudioChannel() { } audio_mixer_->RemoveSource(ingress_.get()); - process_thread_->DeRegisterModule(rtp_rtcp_.get()); + + // TODO(bugs.webrtc.org/11581): unclear if we still need to clear `egress_` + // here. + egress_.reset(); + ingress_.reset(); } -void AudioChannel::StartSend() { - egress_->StartSend(); +bool AudioChannel::StartSend() { + // If encoder has not been set, return false. + if (!egress_->StartSend()) { + return false; + } // Start sending with RTP stack if it has not been sending yet. - if (!rtp_rtcp_->Sending() && rtp_rtcp_->SetSendingStatus(true) != 0) { - RTC_DLOG(LS_ERROR) << "StartSend() RTP/RTCP failed to start sending"; + if (!rtp_rtcp_->Sending()) { + rtp_rtcp_->SetSendingStatus(true); } + return true; } void AudioChannel::StopSend() { egress_->StopSend(); - // If the channel is not playing and RTP stack is active then deactivate RTP - // stack. SetSendingStatus(false) triggers the transmission of RTCP BYE + // Deactivate RTP stack when both sending and receiving are stopped. + // SetSendingStatus(false) triggers the transmission of RTCP BYE // message to remote endpoint. - if (!IsPlaying() && rtp_rtcp_->Sending() && - rtp_rtcp_->SetSendingStatus(false) != 0) { - RTC_DLOG(LS_ERROR) << "StopSend() RTP/RTCP failed to stop sending"; + if (!ingress_->IsPlaying() && rtp_rtcp_->Sending()) { + rtp_rtcp_->SetSendingStatus(false); } } -void AudioChannel::StartPlay() { - ingress_->StartPlay(); +bool AudioChannel::StartPlay() { + // If decoders have not been set, return false. + if (!ingress_->StartPlay()) { + return false; + } // If RTP stack is not sending then start sending as in recv-only mode, RTCP // receiver report is expected. - if (!rtp_rtcp_->Sending() && rtp_rtcp_->SetSendingStatus(true) != 0) { - RTC_DLOG(LS_ERROR) << "StartPlay() RTP/RTCP failed to start sending"; + if (!rtp_rtcp_->Sending()) { + rtp_rtcp_->SetSendingStatus(true); } + return true; } void AudioChannel::StopPlay() { ingress_->StopPlay(); // Deactivate RTP stack only when both sending and receiving are stopped. - if (!IsSendingMedia() && rtp_rtcp_->Sending() && - rtp_rtcp_->SetSendingStatus(false) != 0) { - RTC_DLOG(LS_ERROR) << "StopPlay() RTP/RTCP failed to stop sending"; + if (!rtp_rtcp_->SendingMedia() && rtp_rtcp_->Sending()) { + rtp_rtcp_->SetSendingStatus(false); } } +IngressStatistics AudioChannel::GetIngressStatistics() { + IngressStatistics ingress_stats; + NetworkStatistics stats = ingress_->GetNetworkStatistics(); + ingress_stats.neteq_stats.total_samples_received = stats.totalSamplesReceived; + ingress_stats.neteq_stats.concealed_samples = stats.concealedSamples; + ingress_stats.neteq_stats.concealment_events = stats.concealmentEvents; + ingress_stats.neteq_stats.jitter_buffer_delay_ms = stats.jitterBufferDelayMs; + ingress_stats.neteq_stats.jitter_buffer_emitted_count = + stats.jitterBufferEmittedCount; + ingress_stats.neteq_stats.jitter_buffer_target_delay_ms = + stats.jitterBufferTargetDelayMs; + ingress_stats.neteq_stats.inserted_samples_for_deceleration = + stats.insertedSamplesForDeceleration; + ingress_stats.neteq_stats.removed_samples_for_acceleration = + stats.removedSamplesForAcceleration; + ingress_stats.neteq_stats.silent_concealed_samples = + stats.silentConcealedSamples; + ingress_stats.neteq_stats.fec_packets_received = stats.fecPacketsReceived; + ingress_stats.neteq_stats.fec_packets_discarded = stats.fecPacketsDiscarded; + ingress_stats.neteq_stats.delayed_packet_outage_samples = + stats.delayedPacketOutageSamples; + ingress_stats.neteq_stats.relative_packet_arrival_delay_ms = + stats.relativePacketArrivalDelayMs; + ingress_stats.neteq_stats.interruption_count = stats.interruptionCount; + ingress_stats.neteq_stats.total_interruption_duration_ms = + stats.totalInterruptionDurationMs; + ingress_stats.total_duration = ingress_->GetOutputTotalDuration(); + return ingress_stats; +} + +ChannelStatistics AudioChannel::GetChannelStatistics() { + ChannelStatistics channel_stat = ingress_->GetChannelStatistics(); + + StreamDataCounters rtp_stats, rtx_stats; + rtp_rtcp_->GetSendStreamDataCounters(&rtp_stats, &rtx_stats); + channel_stat.bytes_sent = + rtp_stats.transmitted.payload_bytes + rtx_stats.transmitted.payload_bytes; + channel_stat.packets_sent = + rtp_stats.transmitted.packets + rtx_stats.transmitted.packets; + + return channel_stat; +} + } // namespace webrtc diff --git a/audio/voip/audio_channel.h b/audio/voip/audio_channel.h index 659e990c30..7338d9faab 100644 --- a/audio/voip/audio_channel.h +++ b/audio/voip/audio_channel.h @@ -18,10 +18,10 @@ #include "api/task_queue/task_queue_factory.h" #include "api/voip/voip_base.h" +#include "api/voip/voip_statistics.h" #include "audio/voip/audio_egress.h" #include "audio/voip/audio_ingress.h" #include "modules/rtp_rtcp/source/rtp_rtcp_impl2.h" -#include "modules/utility/include/process_thread.h" #include "rtc_base/ref_count.h" namespace webrtc { @@ -34,7 +34,6 @@ class AudioChannel : public rtc::RefCountInterface { AudioChannel(Transport* transport, uint32_t local_ssrc, TaskQueueFactory* task_queue_factory, - ProcessThread* process_thread, AudioMixer* audio_mixer, rtc::scoped_refptr<AudioDecoderFactory> decoder_factory); ~AudioChannel() override; @@ -45,9 +44,11 @@ class AudioChannel : public rtc::RefCountInterface { ChannelId GetId() const { return id_; } // APIs to start/stop audio channel on each direction. - void StartSend(); + // StartSend/StartPlay returns false if encoder/decoders + // have not been set, respectively. + bool StartSend(); void StopSend(); - void StartPlay(); + bool StartPlay(); void StopPlay(); // APIs relayed to AudioEgress. @@ -61,6 +62,13 @@ class AudioChannel : public rtc::RefCountInterface { absl::optional<SdpAudioFormat> GetEncoderFormat() const { return egress_->GetEncoderFormat(); } + void RegisterTelephoneEventType(int rtp_payload_type, int sample_rate_hz) { + egress_->RegisterTelephoneEventType(rtp_payload_type, sample_rate_hz); + } + bool SendTelephoneEvent(int dtmf_event, int duration_ms) { + return egress_->SendTelephoneEvent(dtmf_event, duration_ms); + } + void SetMute(bool enable) { egress_->SetMute(enable); } // APIs relayed to AudioIngress. bool IsPlaying() const { return ingress_->IsPlaying(); } @@ -73,6 +81,35 @@ class AudioChannel : public rtc::RefCountInterface { void SetReceiveCodecs(const std::map<int, SdpAudioFormat>& codecs) { ingress_->SetReceiveCodecs(codecs); } + IngressStatistics GetIngressStatistics(); + ChannelStatistics GetChannelStatistics(); + + // See comments on the methods used from AudioEgress and AudioIngress. + // Conversion to double is following what is done in + // DoubleAudioLevelFromIntAudioLevel method in rtc_stats_collector.cc to be + // consistent. + double GetInputAudioLevel() const { + return egress_->GetInputAudioLevel() / 32767.0; + } + double GetInputTotalEnergy() const { return egress_->GetInputTotalEnergy(); } + double GetInputTotalDuration() const { + return egress_->GetInputTotalDuration(); + } + double GetOutputAudioLevel() const { + return ingress_->GetOutputAudioLevel() / 32767.0; + } + double GetOutputTotalEnergy() const { + return ingress_->GetOutputTotalEnergy(); + } + double GetOutputTotalDuration() const { + return ingress_->GetOutputTotalDuration(); + } + + // Internal API for testing purpose. + void SendRTCPReportForTesting(RTCPPacketType type) { + int32_t result = rtp_rtcp_->SendRTCP(type); + RTC_DCHECK(result == 0); + } private: // ChannelId that this audio channel belongs for logging purpose. @@ -81,9 +118,6 @@ class AudioChannel : public rtc::RefCountInterface { // Synchronization is handled internally by AudioMixer. AudioMixer* audio_mixer_; - // Synchronization is handled internally by ProcessThread. - ProcessThread* process_thread_; - // Listed in order for safe destruction of AudioChannel object. // Synchronization for these are handled internally. std::unique_ptr<ReceiveStatistics> receive_statistics_; diff --git a/audio/voip/audio_egress.cc b/audio/voip/audio_egress.cc index 305f712624..1162824c9e 100644 --- a/audio/voip/audio_egress.cc +++ b/audio/voip/audio_egress.cc @@ -56,8 +56,13 @@ void AudioEgress::SetEncoder(int payload_type, audio_coding_->SetEncoder(std::move(encoder)); } -void AudioEgress::StartSend() { +bool AudioEgress::StartSend() { + if (!GetEncoderFormat()) { + RTC_DLOG(LS_WARNING) << "Send codec has not been set yet"; + return false; + } rtp_rtcp_->SetSendingMediaStatus(true); + return true; } void AudioEgress::StopSend() { @@ -75,6 +80,12 @@ void AudioEgress::SendAudioData(std::unique_ptr<AudioFrame> audio_frame) { return; } + double duration_seconds = + static_cast<double>(audio_frame->samples_per_channel_) / + audio_frame->sample_rate_hz_; + + input_audio_level_.ComputeLevel(*audio_frame, duration_seconds); + AudioFrameOperations::Mute(audio_frame.get(), encoder_context_.previously_muted_, encoder_context_.mute_); diff --git a/audio/voip/audio_egress.h b/audio/voip/audio_egress.h index 8ec048f915..989e5bda59 100644 --- a/audio/voip/audio_egress.h +++ b/audio/voip/audio_egress.h @@ -15,7 +15,9 @@ #include <string> #include "api/audio_codecs/audio_format.h" +#include "api/sequence_checker.h" #include "api/task_queue/task_queue_factory.h" +#include "audio/audio_level.h" #include "audio/utility/audio_frame_operations.h" #include "call/audio_sender.h" #include "modules/audio_coding/include/audio_coding_module.h" @@ -24,7 +26,6 @@ #include "modules/rtp_rtcp/source/rtp_sender_audio.h" #include "rtc_base/synchronization/mutex.h" #include "rtc_base/task_queue.h" -#include "rtc_base/thread_checker.h" #include "rtc_base/time_utils.h" namespace webrtc { @@ -51,7 +52,7 @@ class AudioEgress : public AudioSender, public AudioPacketizationCallback { // Set the encoder format and payload type for AudioCodingModule. // It's possible to change the encoder type during its active usage. - // |payload_type| must be the type that is negotiated with peer through + // `payload_type` must be the type that is negotiated with peer through // offer/answer. void SetEncoder(int payload_type, const SdpAudioFormat& encoder_format, @@ -59,8 +60,9 @@ class AudioEgress : public AudioSender, public AudioPacketizationCallback { // Start or stop sending operation of AudioEgress. This will start/stop // the RTP stack also causes encoder queue thread to start/stop - // processing input audio samples. - void StartSend(); + // processing input audio samples. StartSend will return false if + // a send codec has not been set. + bool StartSend(); void StopSend(); // Query the state of the RTP stack. This returns true if StartSend() @@ -82,12 +84,22 @@ class AudioEgress : public AudioSender, public AudioPacketizationCallback { // Send DTMF named event as specified by // https://tools.ietf.org/html/rfc4733#section-3.2 - // |duration_ms| specifies the duration of DTMF packets that will be emitted + // `duration_ms` specifies the duration of DTMF packets that will be emitted // in place of real RTP packets instead. // This will return true when requested dtmf event is successfully scheduled // otherwise false when the dtmf queue reached maximum of 20 events. bool SendTelephoneEvent(int dtmf_event, int duration_ms); + // See comments on LevelFullRange, TotalEnergy, TotalDuration from + // audio/audio_level.h. + int GetInputAudioLevel() const { return input_audio_level_.LevelFullRange(); } + double GetInputTotalEnergy() const { + return input_audio_level_.TotalEnergy(); + } + double GetInputTotalDuration() const { + return input_audio_level_.TotalDuration(); + } + // Implementation of AudioSender interface. void SendAudioData(std::unique_ptr<AudioFrame> audio_frame) override; @@ -118,13 +130,16 @@ class AudioEgress : public AudioSender, public AudioPacketizationCallback { // Synchronization is handled internally by AudioCodingModule. const std::unique_ptr<AudioCodingModule> audio_coding_; + // Synchronization is handled internally by voe::AudioLevel. + voe::AudioLevel input_audio_level_; + // Struct that holds all variables used by encoder task queue. struct EncoderContext { // Offset used to mark rtp timestamp in sample rate unit in // newly received audio frame from AudioTransport. uint32_t frame_rtp_timestamp_ = 0; - // Flag to track mute state from caller. |previously_muted_| is used to + // Flag to track mute state from caller. `previously_muted_` is used to // track previous state as part of input to AudioFrameOperations::Mute // to implement fading effect when (un)mute is invoked. bool mute_ = false; diff --git a/audio/voip/audio_ingress.cc b/audio/voip/audio_ingress.cc index 560055d4f4..71026e84e0 100644 --- a/audio/voip/audio_ingress.cc +++ b/audio/voip/audio_ingress.cc @@ -17,6 +17,10 @@ #include "api/audio_codecs/audio_format.h" #include "audio/utility/audio_frame_operations.h" #include "modules/audio_coding/include/audio_coding_module.h" +#include "modules/rtp_rtcp/source/byte_io.h" +#include "modules/rtp_rtcp/source/rtcp_packet/common_header.h" +#include "modules/rtp_rtcp/source/rtcp_packet/receiver_report.h" +#include "modules/rtp_rtcp/source/rtcp_packet/sender_report.h" #include "rtc_base/logging.h" #include "rtc_base/numerics/safe_minmax.h" @@ -73,6 +77,12 @@ AudioMixer::Source::AudioFrameInfo AudioIngress::GetAudioFrameWithInfo( constexpr double kAudioSampleDurationSeconds = 0.01; output_audio_level_.ComputeLevel(*audio_frame, kAudioSampleDurationSeconds); + // If caller invoked StopPlay(), then mute the frame. + if (!playing_) { + AudioFrameOperations::Mute(audio_frame); + muted = true; + } + // Set first rtp timestamp with first audio frame with valid timestamp. if (first_rtp_timestamp_ < 0 && audio_frame->timestamp_ != 0) { first_rtp_timestamp_ = audio_frame->timestamp_; @@ -103,6 +113,18 @@ AudioMixer::Source::AudioFrameInfo AudioIngress::GetAudioFrameWithInfo( : AudioMixer::Source::AudioFrameInfo::kNormal; } +bool AudioIngress::StartPlay() { + { + MutexLock lock(&lock_); + if (receive_codec_info_.empty()) { + RTC_DLOG(LS_WARNING) << "Receive codecs have not been set yet"; + return false; + } + } + playing_ = true; + return true; +} + void AudioIngress::SetReceiveCodecs( const std::map<int, SdpAudioFormat>& codecs) { { @@ -115,10 +137,6 @@ void AudioIngress::SetReceiveCodecs( } void AudioIngress::ReceivedRTPPacket(rtc::ArrayView<const uint8_t> rtp_packet) { - if (!IsPlaying()) { - return; - } - RtpPacketReceived rtp_packet_received; rtp_packet_received.Parse(rtp_packet.data(), rtp_packet.size()); @@ -139,6 +157,12 @@ void AudioIngress::ReceivedRTPPacket(rtc::ArrayView<const uint8_t> rtp_packet) { rtp_packet_received.set_payload_type_frequency(it->second); } + // Track current remote SSRC. + if (rtp_packet_received.Ssrc() != remote_ssrc_) { + rtp_rtcp_->SetRemoteSSRC(rtp_packet_received.Ssrc()); + remote_ssrc_.store(rtp_packet_received.Ssrc()); + } + rtp_receive_statistics_->OnRtpPacket(rtp_packet_received); RTPHeader header; @@ -167,11 +191,28 @@ void AudioIngress::ReceivedRTPPacket(rtc::ArrayView<const uint8_t> rtp_packet) { void AudioIngress::ReceivedRTCPPacket( rtc::ArrayView<const uint8_t> rtcp_packet) { - // Deliver RTCP packet to RTP/RTCP module for parsing. + rtcp::CommonHeader rtcp_header; + if (rtcp_header.Parse(rtcp_packet.data(), rtcp_packet.size()) && + (rtcp_header.type() == rtcp::SenderReport::kPacketType || + rtcp_header.type() == rtcp::ReceiverReport::kPacketType)) { + RTC_DCHECK_GE(rtcp_packet.size(), 8); + + uint32_t sender_ssrc = + ByteReader<uint32_t>::ReadBigEndian(rtcp_packet.data() + 4); + + // If we don't have remote ssrc at this point, it's likely that remote + // endpoint is receive-only or it could have restarted the media. + if (sender_ssrc != remote_ssrc_) { + rtp_rtcp_->SetRemoteSSRC(sender_ssrc); + remote_ssrc_.store(sender_ssrc); + } + } + + // Deliver RTCP packet to RTP/RTCP module for parsing and processing. rtp_rtcp_->IncomingRtcpPacket(rtcp_packet.data(), rtcp_packet.size()); - int64_t rtt = GetRoundTripTime(); - if (rtt == -1) { + int64_t rtt = 0; + if (rtp_rtcp_->RTT(remote_ssrc_, &rtt, nullptr, nullptr, nullptr) != 0) { // Waiting for valid RTT. return; } @@ -185,34 +226,70 @@ void AudioIngress::ReceivedRTCPPacket( { MutexLock lock(&lock_); - ntp_estimator_.UpdateRtcpTimestamp(rtt, ntp_secs, ntp_frac, rtp_timestamp); + ntp_estimator_.UpdateRtcpTimestamp( + TimeDelta::Millis(rtt), NtpTime(ntp_secs, ntp_frac), rtp_timestamp); } } -int64_t AudioIngress::GetRoundTripTime() { - const std::vector<ReportBlockData>& report_data = - rtp_rtcp_->GetLatestReportBlockData(); +ChannelStatistics AudioIngress::GetChannelStatistics() { + ChannelStatistics channel_stats; - // If we do not have report block which means remote RTCP hasn't be received - // yet, return -1 as to indicate uninitialized value. - if (report_data.empty()) { - return -1; + // Get clockrate for current decoder ahead of jitter calculation. + uint32_t clockrate_hz = 0; + absl::optional<std::pair<int, SdpAudioFormat>> decoder = + acm_receiver_.LastDecoder(); + if (decoder) { + clockrate_hz = decoder->second.clockrate_hz; } - // We don't know in advance the remote SSRC used by the other end's receiver - // reports, so use the SSRC of the first report block as remote SSRC for now. - // TODO(natim@webrtc.org): handle the case where remote end is changing ssrc - // and update accordingly here. - const ReportBlockData& block_data = report_data[0]; - - const uint32_t sender_ssrc = block_data.report_block().sender_ssrc; + StreamStatistician* statistician = + rtp_receive_statistics_->GetStatistician(remote_ssrc_); + if (statistician) { + RtpReceiveStats stats = statistician->GetStats(); + channel_stats.packets_lost = stats.packets_lost; + channel_stats.packets_received = stats.packet_counter.packets; + channel_stats.bytes_received = stats.packet_counter.payload_bytes; + channel_stats.remote_ssrc = remote_ssrc_; + if (clockrate_hz > 0) { + channel_stats.jitter = static_cast<double>(stats.jitter) / clockrate_hz; + } + } - if (sender_ssrc != remote_ssrc_.load()) { - remote_ssrc_.store(sender_ssrc); - rtp_rtcp_->SetRemoteSSRC(sender_ssrc); + // Get RTCP report using remote SSRC. + const std::vector<ReportBlockData>& report_data = + rtp_rtcp_->GetLatestReportBlockData(); + for (const ReportBlockData& block_data : report_data) { + const RTCPReportBlock& rtcp_report = block_data.report_block(); + if (rtp_rtcp_->SSRC() != rtcp_report.source_ssrc || + remote_ssrc_ != rtcp_report.sender_ssrc) { + continue; + } + RemoteRtcpStatistics remote_stat; + remote_stat.packets_lost = rtcp_report.packets_lost; + remote_stat.fraction_lost = + static_cast<double>(rtcp_report.fraction_lost) / (1 << 8); + if (clockrate_hz > 0) { + remote_stat.jitter = + static_cast<double>(rtcp_report.jitter) / clockrate_hz; + } + if (block_data.has_rtt()) { + remote_stat.round_trip_time = + static_cast<double>(block_data.last_rtt_ms()) / + rtc::kNumMillisecsPerSec; + } + remote_stat.last_report_received_timestamp_ms = + block_data.report_block_timestamp_utc_us() / + rtc::kNumMicrosecsPerMillisec; + channel_stats.remote_rtcp = remote_stat; + + // Receive only channel won't send any RTP packets. + if (!channel_stats.remote_ssrc.has_value()) { + channel_stats.remote_ssrc = remote_ssrc_; + } + break; } - return (block_data.has_rtt() ? block_data.last_rtt_ms() : -1); + return channel_stats; } } // namespace webrtc diff --git a/audio/voip/audio_ingress.h b/audio/voip/audio_ingress.h index 5a8df21f7a..9a36a46563 100644 --- a/audio/voip/audio_ingress.h +++ b/audio/voip/audio_ingress.h @@ -17,10 +17,12 @@ #include <memory> #include <utility> +#include "absl/types/optional.h" #include "api/array_view.h" #include "api/audio/audio_mixer.h" #include "api/rtp_headers.h" #include "api/scoped_refptr.h" +#include "api/voip/voip_statistics.h" #include "audio/audio_level.h" #include "modules/audio_coding/acm2/acm_receiver.h" #include "modules/audio_coding/include/audio_coding_module.h" @@ -51,7 +53,7 @@ class AudioIngress : public AudioMixer::Source { ~AudioIngress() override; // Start or stop receiving operation of AudioIngress. - void StartPlay() { playing_ = true; } + bool StartPlay(); void StopPlay() { playing_ = false; output_audio_level_.ResetLevelFullRange(); @@ -68,29 +70,25 @@ class AudioIngress : public AudioMixer::Source { void ReceivedRTPPacket(rtc::ArrayView<const uint8_t> rtp_packet); void ReceivedRTCPPacket(rtc::ArrayView<const uint8_t> rtcp_packet); - // Retrieve highest speech output level in last 100 ms. Note that - // this isn't RMS but absolute raw audio level on int16_t sample unit. - // Therefore, the return value will vary between 0 ~ 0xFFFF. This type of - // value may be useful to be used for measuring active speaker gauge. - int GetSpeechOutputLevelFullRange() const { + // See comments on LevelFullRange, TotalEnergy, TotalDuration from + // audio/audio_level.h. + int GetOutputAudioLevel() const { return output_audio_level_.LevelFullRange(); } - - // Returns network round trip time (RTT) measued by RTCP exchange with - // remote media endpoint. RTT value -1 indicates that it's not initialized. - int64_t GetRoundTripTime(); + double GetOutputTotalEnergy() { return output_audio_level_.TotalEnergy(); } + double GetOutputTotalDuration() { + return output_audio_level_.TotalDuration(); + } NetworkStatistics GetNetworkStatistics() const { NetworkStatistics stats; - acm_receiver_.GetNetworkStatistics(&stats); - return stats; - } - AudioDecodingCallStats GetDecodingStatistics() const { - AudioDecodingCallStats stats; - acm_receiver_.GetDecodingCallStatistics(&stats); + acm_receiver_.GetNetworkStatistics(&stats, + /*get_and_clear_legacy_stats=*/false); return stats; } + ChannelStatistics GetChannelStatistics(); + // Implementation of AudioMixer::Source interface. AudioMixer::Source::AudioFrameInfo GetAudioFrameWithInfo( int sampling_rate, diff --git a/audio/voip/test/BUILD.gn b/audio/voip/test/BUILD.gn index d698b3321d..e89f2b001a 100644 --- a/audio/voip/test/BUILD.gn +++ b/audio/voip/test/BUILD.gn @@ -9,41 +9,55 @@ import("../../../webrtc.gni") if (rtc_include_tests) { - rtc_library("voip_core_unittests") { + rtc_source_set("mock_task_queue") { testonly = true - sources = [ "voip_core_unittest.cc" ] + visibility = [ "*" ] + sources = [ "mock_task_queue.h" ] deps = [ - "..:voip_core", - "../../../api/audio_codecs:builtin_audio_decoder_factory", - "../../../api/audio_codecs:builtin_audio_encoder_factory", - "../../../api/task_queue:default_task_queue_factory", - "../../../modules/audio_device:mock_audio_device", - "../../../modules/audio_processing:mocks", - "../../../test:audio_codec_mocks", - "../../../test:mock_transport", + "../../../api/task_queue:task_queue", + "../../../api/task_queue/test:mock_task_queue_base", "../../../test:test_support", ] } + if (!build_with_chromium) { + rtc_library("voip_core_unittests") { + testonly = true + sources = [ "voip_core_unittest.cc" ] + deps = [ + "..:voip_core", + "../../../api/audio_codecs:builtin_audio_decoder_factory", + "../../../api/audio_codecs:builtin_audio_encoder_factory", + "../../../api/task_queue:default_task_queue_factory", + "../../../modules/audio_device:mock_audio_device", + "../../../modules/audio_processing:mocks", + "../../../test:audio_codec_mocks", + "../../../test:mock_transport", + "../../../test:run_loop", + "../../../test:test_support", + ] + } + } + rtc_library("audio_channel_unittests") { testonly = true sources = [ "audio_channel_unittest.cc" ] deps = [ + ":mock_task_queue", "..:audio_channel", "../../../api:transport_api", "../../../api/audio_codecs:builtin_audio_decoder_factory", "../../../api/audio_codecs:builtin_audio_encoder_factory", - "../../../api/task_queue:default_task_queue_factory", + "../../../api/task_queue:task_queue", "../../../modules/audio_mixer:audio_mixer_impl", "../../../modules/audio_mixer:audio_mixer_test_utils", "../../../modules/rtp_rtcp:rtp_rtcp", "../../../modules/rtp_rtcp:rtp_rtcp_format", - "../../../modules/utility", "../../../rtc_base:logging", - "../../../rtc_base:rtc_event", "../../../test:mock_transport", "../../../test:test_support", ] + absl_deps = [ "//third_party/abseil-cpp/absl/functional:any_invocable" ] } rtc_library("audio_ingress_unittests") { @@ -61,6 +75,7 @@ if (rtc_include_tests) { "../../../rtc_base:logging", "../../../rtc_base:rtc_event", "../../../test:mock_transport", + "../../../test:run_loop", "../../../test:test_support", ] } @@ -79,6 +94,7 @@ if (rtc_include_tests) { "../../../rtc_base:logging", "../../../rtc_base:rtc_event", "../../../test:mock_transport", + "../../../test:run_loop", "../../../test:test_support", ] } diff --git a/audio/voip/test/audio_channel_unittest.cc b/audio/voip/test/audio_channel_unittest.cc index ce557823cb..8955810429 100644 --- a/audio/voip/test/audio_channel_unittest.cc +++ b/audio/voip/test/audio_channel_unittest.cc @@ -9,15 +9,16 @@ */ #include "audio/voip/audio_channel.h" + +#include "absl/functional/any_invocable.h" #include "api/audio_codecs/builtin_audio_decoder_factory.h" #include "api/audio_codecs/builtin_audio_encoder_factory.h" #include "api/call/transport.h" -#include "api/task_queue/default_task_queue_factory.h" +#include "api/task_queue/task_queue_factory.h" +#include "audio/voip/test/mock_task_queue.h" #include "modules/audio_mixer/audio_mixer_impl.h" #include "modules/audio_mixer/sine_wave_generator.h" #include "modules/rtp_rtcp/source/rtp_packet_received.h" -#include "modules/utility/include/process_thread.h" -#include "rtc_base/event.h" #include "rtc_base/logging.h" #include "test/gmock.h" #include "test/gtest.h" @@ -28,6 +29,7 @@ namespace { using ::testing::Invoke; using ::testing::NiceMock; +using ::testing::Return; using ::testing::Unused; constexpr uint64_t kStartTime = 123456789; @@ -41,30 +43,38 @@ class AudioChannelTest : public ::testing::Test { AudioChannelTest() : fake_clock_(kStartTime), wave_generator_(1000.0, kAudioLevel) { - process_thread_ = ProcessThread::Create("ModuleProcessThread"); + task_queue_factory_ = std::make_unique<MockTaskQueueFactory>(&task_queue_); audio_mixer_ = AudioMixerImpl::Create(); - task_queue_factory_ = CreateDefaultTaskQueueFactory(); encoder_factory_ = CreateBuiltinAudioEncoderFactory(); decoder_factory_ = CreateBuiltinAudioDecoderFactory(); - } - void SetUp() override { - audio_channel_ = new rtc::RefCountedObject<AudioChannel>( - &transport_, kLocalSsrc, task_queue_factory_.get(), - process_thread_.get(), audio_mixer_.get(), decoder_factory_); - - audio_channel_->SetEncoder(kPcmuPayload, kPcmuFormat, - encoder_factory_->MakeAudioEncoder( - kPcmuPayload, kPcmuFormat, absl::nullopt)); - audio_channel_->SetReceiveCodecs({{kPcmuPayload, kPcmuFormat}}); - audio_channel_->StartSend(); - audio_channel_->StartPlay(); + // By default, run the queued task immediately. + ON_CALL(task_queue_, PostTask) + .WillByDefault( + [](absl::AnyInvocable<void() &&> task) { std::move(task)(); }); } - void TearDown() override { - audio_channel_->StopSend(); - audio_channel_->StopPlay(); - audio_channel_ = nullptr; + void SetUp() override { audio_channel_ = CreateAudioChannel(kLocalSsrc); } + + void TearDown() override { audio_channel_ = nullptr; } + + rtc::scoped_refptr<AudioChannel> CreateAudioChannel(uint32_t ssrc) { + // Use same audio mixer here for simplicity sake as we are not checking + // audio activity of RTP in our testcases. If we need to do test on audio + // signal activity then we need to assign audio mixer for each channel. + // Also this uses the same transport object for different audio channel to + // simplify network routing logic. + rtc::scoped_refptr<AudioChannel> audio_channel = + rtc::make_ref_counted<AudioChannel>( + &transport_, ssrc, task_queue_factory_.get(), audio_mixer_.get(), + decoder_factory_); + audio_channel->SetEncoder(kPcmuPayload, kPcmuFormat, + encoder_factory_->MakeAudioEncoder( + kPcmuPayload, kPcmuFormat, absl::nullopt)); + audio_channel->SetReceiveCodecs({{kPcmuPayload, kPcmuFormat}}); + audio_channel->StartSend(); + audio_channel->StartPlay(); + return audio_channel; } std::unique_ptr<AudioFrame> GetAudioFrame(int order) { @@ -80,11 +90,11 @@ class AudioChannelTest : public ::testing::Test { SimulatedClock fake_clock_; SineWaveGenerator wave_generator_; NiceMock<MockTransport> transport_; + NiceMock<MockTaskQueue> task_queue_; std::unique_ptr<TaskQueueFactory> task_queue_factory_; rtc::scoped_refptr<AudioMixer> audio_mixer_; rtc::scoped_refptr<AudioDecoderFactory> decoder_factory_; rtc::scoped_refptr<AudioEncoderFactory> encoder_factory_; - std::unique_ptr<ProcessThread> process_thread_; rtc::scoped_refptr<AudioChannel> audio_channel_; }; @@ -92,11 +102,9 @@ class AudioChannelTest : public ::testing::Test { // Resulted RTP packet is looped back into AudioChannel and gets decoded into // audio frame to see if it has some signal to indicate its validity. TEST_F(AudioChannelTest, PlayRtpByLocalLoop) { - rtc::Event event; auto loop_rtp = [&](const uint8_t* packet, size_t length, Unused) { audio_channel_->ReceivedRTPPacket( rtc::ArrayView<const uint8_t>(packet, length)); - event.Set(); return true; }; EXPECT_CALL(transport_, SendRtp).WillOnce(Invoke(loop_rtp)); @@ -105,8 +113,6 @@ TEST_F(AudioChannelTest, PlayRtpByLocalLoop) { audio_sender->SendAudioData(GetAudioFrame(0)); audio_sender->SendAudioData(GetAudioFrame(1)); - event.Wait(/*ms=*/1000); - AudioFrame empty_frame, audio_frame; empty_frame.Mute(); empty_frame.mutable_data(); // This will zero out the data. @@ -122,10 +128,8 @@ TEST_F(AudioChannelTest, PlayRtpByLocalLoop) { // Validate assigned local SSRC is resulted in RTP packet. TEST_F(AudioChannelTest, VerifyLocalSsrcAsAssigned) { RtpPacketReceived rtp; - rtc::Event event; auto loop_rtp = [&](const uint8_t* packet, size_t length, Unused) { rtp.Parse(packet, length); - event.Set(); return true; }; EXPECT_CALL(transport_, SendRtp).WillOnce(Invoke(loop_rtp)); @@ -134,10 +138,220 @@ TEST_F(AudioChannelTest, VerifyLocalSsrcAsAssigned) { audio_sender->SendAudioData(GetAudioFrame(0)); audio_sender->SendAudioData(GetAudioFrame(1)); - event.Wait(/*ms=*/1000); - EXPECT_EQ(rtp.Ssrc(), kLocalSsrc); } +// Check metrics after processing an RTP packet. +TEST_F(AudioChannelTest, TestIngressStatistics) { + auto loop_rtp = [&](const uint8_t* packet, size_t length, Unused) { + audio_channel_->ReceivedRTPPacket( + rtc::ArrayView<const uint8_t>(packet, length)); + return true; + }; + EXPECT_CALL(transport_, SendRtp).WillRepeatedly(Invoke(loop_rtp)); + + auto audio_sender = audio_channel_->GetAudioSender(); + audio_sender->SendAudioData(GetAudioFrame(0)); + audio_sender->SendAudioData(GetAudioFrame(1)); + + AudioFrame audio_frame; + audio_mixer_->Mix(/*number_of_channels=*/1, &audio_frame); + audio_mixer_->Mix(/*number_of_channels=*/1, &audio_frame); + + absl::optional<IngressStatistics> ingress_stats = + audio_channel_->GetIngressStatistics(); + EXPECT_TRUE(ingress_stats); + EXPECT_EQ(ingress_stats->neteq_stats.total_samples_received, 160ULL); + EXPECT_EQ(ingress_stats->neteq_stats.concealed_samples, 0ULL); + EXPECT_EQ(ingress_stats->neteq_stats.concealment_events, 0ULL); + EXPECT_EQ(ingress_stats->neteq_stats.inserted_samples_for_deceleration, 0ULL); + EXPECT_EQ(ingress_stats->neteq_stats.removed_samples_for_acceleration, 0ULL); + EXPECT_EQ(ingress_stats->neteq_stats.silent_concealed_samples, 0ULL); + // To extract the jitter buffer length in millisecond, jitter_buffer_delay_ms + // needs to be divided by jitter_buffer_emitted_count (number of samples). + EXPECT_EQ(ingress_stats->neteq_stats.jitter_buffer_delay_ms, 1600ULL); + EXPECT_EQ(ingress_stats->neteq_stats.jitter_buffer_emitted_count, 160ULL); + EXPECT_GT(ingress_stats->neteq_stats.jitter_buffer_target_delay_ms, 0ULL); + EXPECT_EQ(ingress_stats->neteq_stats.interruption_count, 0); + EXPECT_EQ(ingress_stats->neteq_stats.total_interruption_duration_ms, 0); + EXPECT_DOUBLE_EQ(ingress_stats->total_duration, 0.02); + + // Now without any RTP pending in jitter buffer pull more. + audio_mixer_->Mix(/*number_of_channels=*/1, &audio_frame); + audio_mixer_->Mix(/*number_of_channels=*/1, &audio_frame); + + // Send another RTP packet to intentionally break PLC. + audio_sender->SendAudioData(GetAudioFrame(2)); + audio_sender->SendAudioData(GetAudioFrame(3)); + + ingress_stats = audio_channel_->GetIngressStatistics(); + EXPECT_TRUE(ingress_stats); + EXPECT_EQ(ingress_stats->neteq_stats.total_samples_received, 320ULL); + EXPECT_EQ(ingress_stats->neteq_stats.concealed_samples, 168ULL); + EXPECT_EQ(ingress_stats->neteq_stats.concealment_events, 1ULL); + EXPECT_EQ(ingress_stats->neteq_stats.inserted_samples_for_deceleration, 0ULL); + EXPECT_EQ(ingress_stats->neteq_stats.removed_samples_for_acceleration, 0ULL); + EXPECT_EQ(ingress_stats->neteq_stats.silent_concealed_samples, 0ULL); + EXPECT_EQ(ingress_stats->neteq_stats.jitter_buffer_delay_ms, 1600ULL); + EXPECT_EQ(ingress_stats->neteq_stats.jitter_buffer_emitted_count, 160ULL); + EXPECT_GT(ingress_stats->neteq_stats.jitter_buffer_target_delay_ms, 0ULL); + EXPECT_EQ(ingress_stats->neteq_stats.interruption_count, 0); + EXPECT_EQ(ingress_stats->neteq_stats.total_interruption_duration_ms, 0); + EXPECT_DOUBLE_EQ(ingress_stats->total_duration, 0.04); + + // Pull the last RTP packet. + audio_mixer_->Mix(/*number_of_channels=*/1, &audio_frame); + audio_mixer_->Mix(/*number_of_channels=*/1, &audio_frame); + + ingress_stats = audio_channel_->GetIngressStatistics(); + EXPECT_TRUE(ingress_stats); + EXPECT_EQ(ingress_stats->neteq_stats.total_samples_received, 480ULL); + EXPECT_EQ(ingress_stats->neteq_stats.concealed_samples, 168ULL); + EXPECT_EQ(ingress_stats->neteq_stats.concealment_events, 1ULL); + EXPECT_EQ(ingress_stats->neteq_stats.inserted_samples_for_deceleration, 0ULL); + EXPECT_EQ(ingress_stats->neteq_stats.removed_samples_for_acceleration, 0ULL); + EXPECT_EQ(ingress_stats->neteq_stats.silent_concealed_samples, 0ULL); + EXPECT_EQ(ingress_stats->neteq_stats.jitter_buffer_delay_ms, 3200ULL); + EXPECT_EQ(ingress_stats->neteq_stats.jitter_buffer_emitted_count, 320ULL); + EXPECT_GT(ingress_stats->neteq_stats.jitter_buffer_target_delay_ms, 0ULL); + EXPECT_EQ(ingress_stats->neteq_stats.interruption_count, 0); + EXPECT_EQ(ingress_stats->neteq_stats.total_interruption_duration_ms, 0); + EXPECT_DOUBLE_EQ(ingress_stats->total_duration, 0.06); +} + +// Check ChannelStatistics metric after processing RTP and RTCP packets. +TEST_F(AudioChannelTest, TestChannelStatistics) { + auto loop_rtp = [&](const uint8_t* packet, size_t length, Unused) { + audio_channel_->ReceivedRTPPacket( + rtc::ArrayView<const uint8_t>(packet, length)); + return true; + }; + auto loop_rtcp = [&](const uint8_t* packet, size_t length) { + audio_channel_->ReceivedRTCPPacket( + rtc::ArrayView<const uint8_t>(packet, length)); + return true; + }; + EXPECT_CALL(transport_, SendRtp).WillRepeatedly(Invoke(loop_rtp)); + EXPECT_CALL(transport_, SendRtcp).WillRepeatedly(Invoke(loop_rtcp)); + + // Simulate microphone giving audio frame (10 ms). This will trigger tranport + // to send RTP as handled in loop_rtp above. + auto audio_sender = audio_channel_->GetAudioSender(); + audio_sender->SendAudioData(GetAudioFrame(0)); + audio_sender->SendAudioData(GetAudioFrame(1)); + + // Simulate speaker requesting audio frame (10 ms). This will trigger VoIP + // engine to fetch audio samples from RTP packets stored in jitter buffer. + AudioFrame audio_frame; + audio_mixer_->Mix(/*number_of_channels=*/1, &audio_frame); + audio_mixer_->Mix(/*number_of_channels=*/1, &audio_frame); + + // Force sending RTCP SR report in order to have remote_rtcp field available + // in channel statistics. This will trigger tranport to send RTCP as handled + // in loop_rtcp above. + audio_channel_->SendRTCPReportForTesting(kRtcpSr); + + absl::optional<ChannelStatistics> channel_stats = + audio_channel_->GetChannelStatistics(); + EXPECT_TRUE(channel_stats); + + EXPECT_EQ(channel_stats->packets_sent, 1ULL); + EXPECT_EQ(channel_stats->bytes_sent, 160ULL); + + EXPECT_EQ(channel_stats->packets_received, 1ULL); + EXPECT_EQ(channel_stats->bytes_received, 160ULL); + EXPECT_EQ(channel_stats->jitter, 0); + EXPECT_EQ(channel_stats->packets_lost, 0); + EXPECT_EQ(channel_stats->remote_ssrc.value(), kLocalSsrc); + + EXPECT_TRUE(channel_stats->remote_rtcp.has_value()); + + EXPECT_EQ(channel_stats->remote_rtcp->jitter, 0); + EXPECT_EQ(channel_stats->remote_rtcp->packets_lost, 0); + EXPECT_EQ(channel_stats->remote_rtcp->fraction_lost, 0); + EXPECT_GT(channel_stats->remote_rtcp->last_report_received_timestamp_ms, 0); + EXPECT_FALSE(channel_stats->remote_rtcp->round_trip_time.has_value()); +} + +// Check ChannelStatistics RTT metric after processing RTP and RTCP packets +// using three audio channels where each represents media endpoint. +// +// 1) AC1 <- RTP/RTCP -> AC2 +// 2) AC1 <- RTP/RTCP -> AC3 +// +// During step 1), AC1 should be able to check RTT from AC2's SSRC. +// During step 2), AC1 should be able to check RTT from AC3's SSRC. +TEST_F(AudioChannelTest, RttIsAvailableAfterChangeOfRemoteSsrc) { + // Create AC2 and AC3. + constexpr uint32_t kAc2Ssrc = 0xdeadbeef; + constexpr uint32_t kAc3Ssrc = 0xdeafbeef; + + auto ac_2 = CreateAudioChannel(kAc2Ssrc); + auto ac_3 = CreateAudioChannel(kAc3Ssrc); + + auto send_recv_rtp = [&](rtc::scoped_refptr<AudioChannel> rtp_sender, + rtc::scoped_refptr<AudioChannel> rtp_receiver) { + // Setup routing logic via transport_. + auto route_rtp = [&](const uint8_t* packet, size_t length, Unused) { + rtp_receiver->ReceivedRTPPacket(rtc::MakeArrayView(packet, length)); + return true; + }; + ON_CALL(transport_, SendRtp).WillByDefault(route_rtp); + + // This will trigger route_rtp callback via transport_. + rtp_sender->GetAudioSender()->SendAudioData(GetAudioFrame(0)); + rtp_sender->GetAudioSender()->SendAudioData(GetAudioFrame(1)); + + // Process received RTP in receiver. + AudioFrame audio_frame; + audio_mixer_->Mix(/*number_of_channels=*/1, &audio_frame); + audio_mixer_->Mix(/*number_of_channels=*/1, &audio_frame); + + // Revert to default to avoid using reference in route_rtp lambda. + ON_CALL(transport_, SendRtp).WillByDefault(Return(true)); + }; + + auto send_recv_rtcp = [&](rtc::scoped_refptr<AudioChannel> rtcp_sender, + rtc::scoped_refptr<AudioChannel> rtcp_receiver) { + // Setup routing logic via transport_. + auto route_rtcp = [&](const uint8_t* packet, size_t length) { + rtcp_receiver->ReceivedRTCPPacket(rtc::MakeArrayView(packet, length)); + return true; + }; + ON_CALL(transport_, SendRtcp).WillByDefault(route_rtcp); + + // This will trigger route_rtcp callback via transport_. + rtcp_sender->SendRTCPReportForTesting(kRtcpSr); + + // Revert to default to avoid using reference in route_rtcp lambda. + ON_CALL(transport_, SendRtcp).WillByDefault(Return(true)); + }; + + // AC1 <-- RTP/RTCP --> AC2 + send_recv_rtp(audio_channel_, ac_2); + send_recv_rtp(ac_2, audio_channel_); + send_recv_rtcp(audio_channel_, ac_2); + send_recv_rtcp(ac_2, audio_channel_); + + absl::optional<ChannelStatistics> channel_stats = + audio_channel_->GetChannelStatistics(); + ASSERT_TRUE(channel_stats); + EXPECT_EQ(channel_stats->remote_ssrc, kAc2Ssrc); + ASSERT_TRUE(channel_stats->remote_rtcp); + EXPECT_GT(channel_stats->remote_rtcp->round_trip_time, 0.0); + + // AC1 <-- RTP/RTCP --> AC3 + send_recv_rtp(audio_channel_, ac_3); + send_recv_rtp(ac_3, audio_channel_); + send_recv_rtcp(audio_channel_, ac_3); + send_recv_rtcp(ac_3, audio_channel_); + + channel_stats = audio_channel_->GetChannelStatistics(); + ASSERT_TRUE(channel_stats); + EXPECT_EQ(channel_stats->remote_ssrc, kAc3Ssrc); + ASSERT_TRUE(channel_stats->remote_rtcp); + EXPECT_GT(channel_stats->remote_rtcp->round_trip_time, 0.0); +} + } // namespace } // namespace webrtc diff --git a/audio/voip/test/audio_egress_unittest.cc b/audio/voip/test/audio_egress_unittest.cc index 70fb6dcf36..34c5585347 100644 --- a/audio/voip/test/audio_egress_unittest.cc +++ b/audio/voip/test/audio_egress_unittest.cc @@ -9,6 +9,7 @@ */ #include "audio/voip/audio_egress.h" + #include "api/audio_codecs/builtin_audio_encoder_factory.h" #include "api/call/transport.h" #include "api/task_queue/default_task_queue_factory.h" @@ -20,6 +21,7 @@ #include "test/gmock.h" #include "test/gtest.h" #include "test/mock_transport.h" +#include "test/run_loop.h" namespace webrtc { namespace { @@ -43,12 +45,13 @@ std::unique_ptr<ModuleRtpRtcpImpl2> CreateRtpStack(Clock* clock, return rtp_rtcp; } +constexpr int16_t kAudioLevel = 3004; // Used for sine wave level. + // AudioEgressTest configures audio egress by using Rtp Stack, fake clock, // and task queue factory. Encoder factory is needed to create codec and // configure the RTP stack in audio egress. class AudioEgressTest : public ::testing::Test { public: - static constexpr int16_t kAudioLevel = 3004; // Used for sine wave level. static constexpr uint16_t kSeqNum = 12345; static constexpr uint64_t kStartTime = 123456789; static constexpr uint32_t kRemoteSsrc = 0xDEADBEEF; @@ -56,7 +59,6 @@ class AudioEgressTest : public ::testing::Test { AudioEgressTest() : fake_clock_(kStartTime), wave_generator_(1000.0, kAudioLevel) { - rtp_rtcp_ = CreateRtpStack(&fake_clock_, &transport_, kRemoteSsrc); task_queue_factory_ = CreateDefaultTaskQueueFactory(); encoder_factory_ = CreateBuiltinAudioEncoderFactory(); } @@ -64,6 +66,7 @@ class AudioEgressTest : public ::testing::Test { // Prepare test on audio egress by using PCMu codec with specific // sequence number and its status to be running. void SetUp() override { + rtp_rtcp_ = CreateRtpStack(&fake_clock_, &transport_, kRemoteSsrc); egress_ = std::make_unique<AudioEgress>(rtp_rtcp_.get(), &fake_clock_, task_queue_factory_.get()); constexpr int kPcmuPayload = 0; @@ -80,6 +83,7 @@ class AudioEgressTest : public ::testing::Test { egress_->StopSend(); rtp_rtcp_->SetSendingStatus(false); egress_.reset(); + rtp_rtcp_.reset(); } // Create an audio frame prepared for pcmu encoding. Timestamp is @@ -96,6 +100,7 @@ class AudioEgressTest : public ::testing::Test { return frame; } + test::RunLoop run_loop_; // SimulatedClock doesn't directly affect this testcase as the the // AudioFrame's timestamp is driven by GetAudioFrame. SimulatedClock fake_clock_; @@ -136,7 +141,7 @@ TEST_F(AudioEgressTest, ProcessAudioWithMute) { fake_clock_.AdvanceTimeMilliseconds(10); } - event.Wait(/*ms=*/1000); + event.Wait(TimeDelta::Seconds(1)); EXPECT_EQ(rtp_count, kExpected); // we expect on pcmu payload to result in 255 for silenced payload @@ -172,7 +177,7 @@ TEST_F(AudioEgressTest, ProcessAudioWithSineWave) { fake_clock_.AdvanceTimeMilliseconds(10); } - event.Wait(/*ms=*/1000); + event.Wait(TimeDelta::Seconds(1)); EXPECT_EQ(rtp_count, kExpected); // we expect on pcmu to result in < 255 for payload with sine wave @@ -206,7 +211,7 @@ TEST_F(AudioEgressTest, SkipAudioEncodingAfterStopSend) { fake_clock_.AdvanceTimeMilliseconds(10); } - event.Wait(/*ms=*/1000); + event.Wait(TimeDelta::Seconds(1)); EXPECT_EQ(rtp_count, kExpected); // Now stop send and yet feed more data. @@ -282,9 +287,41 @@ TEST_F(AudioEgressTest, SendDTMF) { fake_clock_.AdvanceTimeMilliseconds(10); } - event.Wait(/*ms=*/1000); + event.Wait(TimeDelta::Seconds(1)); EXPECT_EQ(dtmf_count, kExpected); } +TEST_F(AudioEgressTest, TestAudioInputLevelAndEnergyDuration) { + // Per audio_level's kUpdateFrequency, we need more than 10 audio samples to + // get audio level from input source. + constexpr int kExpected = 6; + rtc::Event event; + int rtp_count = 0; + auto rtp_sent = [&](const uint8_t* packet, size_t length, Unused) { + if (++rtp_count == kExpected) { + event.Set(); + } + return true; + }; + + EXPECT_CALL(transport_, SendRtp).WillRepeatedly(Invoke(rtp_sent)); + + // Two 10 ms audio frames will result in rtp packet with ptime 20. + for (size_t i = 0; i < kExpected * 2; i++) { + egress_->SendAudioData(GetAudioFrame(i)); + fake_clock_.AdvanceTimeMilliseconds(10); + } + + event.Wait(/*give_up_after=*/TimeDelta::Seconds(1)); + EXPECT_EQ(rtp_count, kExpected); + + constexpr double kExpectedEnergy = 0.00016809565587789564; + constexpr double kExpectedDuration = 0.11999999999999998; + + EXPECT_EQ(egress_->GetInputAudioLevel(), kAudioLevel); + EXPECT_DOUBLE_EQ(egress_->GetInputTotalEnergy(), kExpectedEnergy); + EXPECT_DOUBLE_EQ(egress_->GetInputTotalDuration(), kExpectedDuration); +} + } // namespace } // namespace webrtc diff --git a/audio/voip/test/audio_ingress_unittest.cc b/audio/voip/test/audio_ingress_unittest.cc index 3a2a66a325..3c309dbf82 100644 --- a/audio/voip/test/audio_ingress_unittest.cc +++ b/audio/voip/test/audio_ingress_unittest.cc @@ -9,6 +9,7 @@ */ #include "audio/voip/audio_ingress.h" + #include "api/audio_codecs/builtin_audio_decoder_factory.h" #include "api/audio_codecs/builtin_audio_encoder_factory.h" #include "api/call/transport.h" @@ -21,6 +22,7 @@ #include "test/gmock.h" #include "test/gtest.h" #include "test/mock_transport.h" +#include "test/run_loop.h" namespace webrtc { namespace { @@ -91,6 +93,7 @@ class AudioIngressTest : public ::testing::Test { return frame; } + test::RunLoop run_loop_; SimulatedClock fake_clock_; SineWaveGenerator wave_generator_; NiceMock<MockTransport> transport_; @@ -119,7 +122,7 @@ TEST_F(AudioIngressTest, GetAudioFrameAfterRtpReceived) { EXPECT_CALL(transport_, SendRtp).WillRepeatedly(Invoke(handle_rtp)); egress_->SendAudioData(GetAudioFrame(0)); egress_->SendAudioData(GetAudioFrame(1)); - event.Wait(/*ms=*/1000); + event.Wait(TimeDelta::Seconds(1)); AudioFrame audio_frame; EXPECT_EQ( @@ -134,9 +137,10 @@ TEST_F(AudioIngressTest, GetAudioFrameAfterRtpReceived) { EXPECT_EQ(audio_frame.elapsed_time_ms_, 0); } -TEST_F(AudioIngressTest, GetSpeechOutputLevelFullRange) { - // Per audio_level's kUpdateFrequency, we need 11 RTP to get audio level. - constexpr int kNumRtp = 11; +TEST_F(AudioIngressTest, TestSpeechOutputLevelAndEnergyDuration) { + // Per audio_level's kUpdateFrequency, we need more than 10 audio samples to + // get audio level from output source. + constexpr int kNumRtp = 6; int rtp_count = 0; rtc::Event event; auto handle_rtp = [&](const uint8_t* packet, size_t length, Unused) { @@ -151,15 +155,21 @@ TEST_F(AudioIngressTest, GetSpeechOutputLevelFullRange) { egress_->SendAudioData(GetAudioFrame(i)); fake_clock_.AdvanceTimeMilliseconds(10); } - event.Wait(/*ms=*/1000); + event.Wait(/*give_up_after=*/TimeDelta::Seconds(1)); - for (int i = 0; i < kNumRtp; ++i) { + for (int i = 0; i < kNumRtp * 2; ++i) { AudioFrame audio_frame; EXPECT_EQ( ingress_->GetAudioFrameWithInfo(kPcmuFormat.clockrate_hz, &audio_frame), AudioMixer::Source::AudioFrameInfo::kNormal); } - EXPECT_EQ(ingress_->GetSpeechOutputLevelFullRange(), kAudioLevel); + EXPECT_EQ(ingress_->GetOutputAudioLevel(), kAudioLevel); + + constexpr double kExpectedEnergy = 0.00016809565587789564; + constexpr double kExpectedDuration = 0.11999999999999998; + + EXPECT_DOUBLE_EQ(ingress_->GetOutputTotalEnergy(), kExpectedEnergy); + EXPECT_DOUBLE_EQ(ingress_->GetOutputTotalDuration(), kExpectedDuration); } TEST_F(AudioIngressTest, PreferredSampleRate) { @@ -172,7 +182,7 @@ TEST_F(AudioIngressTest, PreferredSampleRate) { EXPECT_CALL(transport_, SendRtp).WillRepeatedly(Invoke(handle_rtp)); egress_->SendAudioData(GetAudioFrame(0)); egress_->SendAudioData(GetAudioFrame(1)); - event.Wait(/*ms=*/1000); + event.Wait(TimeDelta::Seconds(1)); AudioFrame audio_frame; EXPECT_EQ( @@ -181,5 +191,48 @@ TEST_F(AudioIngressTest, PreferredSampleRate) { EXPECT_EQ(ingress_->PreferredSampleRate(), kPcmuFormat.clockrate_hz); } +// This test highlights the case where caller invokes StopPlay() which then +// AudioIngress should play silence frame afterwards. +TEST_F(AudioIngressTest, GetMutedAudioFrameAfterRtpReceivedAndStopPlay) { + // StopPlay before we start sending RTP packet with sine wave. + ingress_->StopPlay(); + + // Send 6 RTP packets to generate more than 100 ms audio sample to get + // valid speech level. + constexpr int kNumRtp = 6; + int rtp_count = 0; + rtc::Event event; + auto handle_rtp = [&](const uint8_t* packet, size_t length, Unused) { + ingress_->ReceivedRTPPacket(rtc::ArrayView<const uint8_t>(packet, length)); + if (++rtp_count == kNumRtp) { + event.Set(); + } + return true; + }; + EXPECT_CALL(transport_, SendRtp).WillRepeatedly(Invoke(handle_rtp)); + for (int i = 0; i < kNumRtp * 2; i++) { + egress_->SendAudioData(GetAudioFrame(i)); + fake_clock_.AdvanceTimeMilliseconds(10); + } + event.Wait(/*give_up_after=*/TimeDelta::Seconds(1)); + + for (int i = 0; i < kNumRtp * 2; ++i) { + AudioFrame audio_frame; + EXPECT_EQ( + ingress_->GetAudioFrameWithInfo(kPcmuFormat.clockrate_hz, &audio_frame), + AudioMixer::Source::AudioFrameInfo::kMuted); + const int16_t* audio_data = audio_frame.data(); + size_t length = + audio_frame.samples_per_channel_ * audio_frame.num_channels_; + for (size_t j = 0; j < length; ++j) { + EXPECT_EQ(audio_data[j], 0); + } + } + + // Now we should still see valid speech output level as StopPlay won't affect + // the measurement. + EXPECT_EQ(ingress_->GetOutputAudioLevel(), kAudioLevel); +} + } // namespace } // namespace webrtc diff --git a/audio/voip/test/mock_task_queue.h b/audio/voip/test/mock_task_queue.h new file mode 100644 index 0000000000..547b0d3f75 --- /dev/null +++ b/audio/voip/test/mock_task_queue.h @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef AUDIO_VOIP_TEST_MOCK_TASK_QUEUE_H_ +#define AUDIO_VOIP_TEST_MOCK_TASK_QUEUE_H_ + +#include <memory> + +#include "api/task_queue/task_queue_factory.h" +#include "api/task_queue/test/mock_task_queue_base.h" +#include "test/gmock.h" + +namespace webrtc { + +// MockTaskQueue enables immediate task run from global TaskQueueBase. +// It's necessary for some tests depending on TaskQueueBase internally. +class MockTaskQueue : public MockTaskQueueBase { + public: + MockTaskQueue() : current_(this) {} + + // Delete is deliberately defined as no-op as MockTaskQueue is expected to + // hold onto current global TaskQueueBase throughout the testing. + void Delete() override {} + + private: + CurrentTaskQueueSetter current_; +}; + +class MockTaskQueueFactory : public TaskQueueFactory { + public: + explicit MockTaskQueueFactory(MockTaskQueue* task_queue) + : task_queue_(task_queue) {} + + std::unique_ptr<TaskQueueBase, TaskQueueDeleter> CreateTaskQueue( + absl::string_view name, + Priority priority) const override { + // Default MockTaskQueue::Delete is no-op, therefore it's safe to pass the + // raw pointer. + return std::unique_ptr<TaskQueueBase, TaskQueueDeleter>(task_queue_); + } + + private: + MockTaskQueue* task_queue_; +}; + +} // namespace webrtc + +#endif // AUDIO_VOIP_TEST_MOCK_TASK_QUEUE_H_ diff --git a/audio/voip/test/voip_core_unittest.cc b/audio/voip/test/voip_core_unittest.cc index c1969d6ed0..b432506b12 100644 --- a/audio/voip/test/voip_core_unittest.cc +++ b/audio/voip/test/voip_core_unittest.cc @@ -9,6 +9,7 @@ */ #include "audio/voip/voip_core.h" + #include "api/audio_codecs/builtin_audio_decoder_factory.h" #include "api/audio_codecs/builtin_audio_encoder_factory.h" #include "api/task_queue/default_task_queue_factory.h" @@ -16,6 +17,7 @@ #include "modules/audio_processing/include/mock_audio_processing.h" #include "test/gtest.h" #include "test/mock_transport.h" +#include "test/run_loop.h" namespace webrtc { namespace { @@ -24,6 +26,9 @@ using ::testing::NiceMock; using ::testing::Return; constexpr int kPcmuPayload = 0; +constexpr int kPcmuSampleRateHz = 8000; +constexpr int kDtmfEventDurationMs = 1000; +constexpr DtmfEvent kDtmfEventCode = DtmfEvent::kDigitZero; class VoipCoreTest : public ::testing::Test { public: @@ -35,14 +40,15 @@ class VoipCoreTest : public ::testing::Test { auto encoder_factory = CreateBuiltinAudioEncoderFactory(); auto decoder_factory = CreateBuiltinAudioDecoderFactory(); rtc::scoped_refptr<AudioProcessing> audio_processing = - new rtc::RefCountedObject<test::MockAudioProcessing>(); + rtc::make_ref_counted<NiceMock<test::MockAudioProcessing>>(); - voip_core_ = std::make_unique<VoipCore>(); - voip_core_->Init(std::move(encoder_factory), std::move(decoder_factory), - CreateDefaultTaskQueueFactory(), audio_device_, - std::move(audio_processing)); + voip_core_ = std::make_unique<VoipCore>( + std::move(encoder_factory), std::move(decoder_factory), + CreateDefaultTaskQueueFactory(), audio_device_, + std::move(audio_processing)); } + test::RunLoop run_loop_; std::unique_ptr<VoipCore> voip_core_; NiceMock<MockTransport> transport_; rtc::scoped_refptr<test::MockAudioDeviceModule> audio_device_; @@ -60,13 +66,23 @@ TEST_F(VoipCoreTest, BasicVoipCoreOperation) { EXPECT_CALL(*audio_device_, StartPlayout()).WillOnce(Return(0)); auto channel = voip_core_->CreateChannel(&transport_, 0xdeadc0de); - EXPECT_TRUE(channel); - voip_core_->SetSendCodec(*channel, kPcmuPayload, kPcmuFormat); - voip_core_->SetReceiveCodecs(*channel, {{kPcmuPayload, kPcmuFormat}}); + EXPECT_EQ(voip_core_->SetSendCodec(channel, kPcmuPayload, kPcmuFormat), + VoipResult::kOk); + EXPECT_EQ( + voip_core_->SetReceiveCodecs(channel, {{kPcmuPayload, kPcmuFormat}}), + VoipResult::kOk); + + EXPECT_EQ(voip_core_->StartSend(channel), VoipResult::kOk); + EXPECT_EQ(voip_core_->StartPlayout(channel), VoipResult::kOk); - EXPECT_TRUE(voip_core_->StartSend(*channel)); - EXPECT_TRUE(voip_core_->StartPlayout(*channel)); + EXPECT_EQ(voip_core_->RegisterTelephoneEventType(channel, kPcmuPayload, + kPcmuSampleRateHz), + VoipResult::kOk); + + EXPECT_EQ( + voip_core_->SendDtmfEvent(channel, kDtmfEventCode, kDtmfEventDurationMs), + VoipResult::kOk); // Program mock as operational that is ready to be stopped. EXPECT_CALL(*audio_device_, Recording()).WillOnce(Return(true)); @@ -74,26 +90,103 @@ TEST_F(VoipCoreTest, BasicVoipCoreOperation) { EXPECT_CALL(*audio_device_, StopRecording()).WillOnce(Return(0)); EXPECT_CALL(*audio_device_, StopPlayout()).WillOnce(Return(0)); - EXPECT_TRUE(voip_core_->StopSend(*channel)); - EXPECT_TRUE(voip_core_->StopPlayout(*channel)); - voip_core_->ReleaseChannel(*channel); + EXPECT_EQ(voip_core_->StopSend(channel), VoipResult::kOk); + EXPECT_EQ(voip_core_->StopPlayout(channel), VoipResult::kOk); + EXPECT_EQ(voip_core_->ReleaseChannel(channel), VoipResult::kOk); } TEST_F(VoipCoreTest, ExpectFailToUseReleasedChannelId) { auto channel = voip_core_->CreateChannel(&transport_, 0xdeadc0de); - EXPECT_TRUE(channel); // Release right after creation. - voip_core_->ReleaseChannel(*channel); + EXPECT_EQ(voip_core_->ReleaseChannel(channel), VoipResult::kOk); // Now use released channel. - // These should be no-op. - voip_core_->SetSendCodec(*channel, kPcmuPayload, kPcmuFormat); - voip_core_->SetReceiveCodecs(*channel, {{kPcmuPayload, kPcmuFormat}}); + EXPECT_EQ(voip_core_->SetSendCodec(channel, kPcmuPayload, kPcmuFormat), + VoipResult::kInvalidArgument); + EXPECT_EQ( + voip_core_->SetReceiveCodecs(channel, {{kPcmuPayload, kPcmuFormat}}), + VoipResult::kInvalidArgument); + EXPECT_EQ(voip_core_->RegisterTelephoneEventType(channel, kPcmuPayload, + kPcmuSampleRateHz), + VoipResult::kInvalidArgument); + EXPECT_EQ(voip_core_->StartSend(channel), VoipResult::kInvalidArgument); + EXPECT_EQ(voip_core_->StartPlayout(channel), VoipResult::kInvalidArgument); + EXPECT_EQ( + voip_core_->SendDtmfEvent(channel, kDtmfEventCode, kDtmfEventDurationMs), + VoipResult::kInvalidArgument); +} + +TEST_F(VoipCoreTest, SendDtmfEventWithoutRegistering) { + // Program mock as non-operational and ready to start send. + EXPECT_CALL(*audio_device_, Recording()).WillOnce(Return(false)); + EXPECT_CALL(*audio_device_, InitRecording()).WillOnce(Return(0)); + EXPECT_CALL(*audio_device_, StartRecording()).WillOnce(Return(0)); + + auto channel = voip_core_->CreateChannel(&transport_, 0xdeadc0de); + + EXPECT_EQ(voip_core_->SetSendCodec(channel, kPcmuPayload, kPcmuFormat), + VoipResult::kOk); + + EXPECT_EQ(voip_core_->StartSend(channel), VoipResult::kOk); + // Send Dtmf event without registering beforehand, thus payload + // type is not set and kFailedPrecondition is expected. + EXPECT_EQ( + voip_core_->SendDtmfEvent(channel, kDtmfEventCode, kDtmfEventDurationMs), + VoipResult::kFailedPrecondition); + + // Program mock as sending and is ready to be stopped. + EXPECT_CALL(*audio_device_, Recording()).WillOnce(Return(true)); + EXPECT_CALL(*audio_device_, StopRecording()).WillOnce(Return(0)); + + EXPECT_EQ(voip_core_->StopSend(channel), VoipResult::kOk); + EXPECT_EQ(voip_core_->ReleaseChannel(channel), VoipResult::kOk); +} + +TEST_F(VoipCoreTest, SendDtmfEventWithoutStartSend) { + auto channel = voip_core_->CreateChannel(&transport_, 0xdeadc0de); + + EXPECT_EQ(voip_core_->RegisterTelephoneEventType(channel, kPcmuPayload, + kPcmuSampleRateHz), + VoipResult::kOk); + + // Send Dtmf event without calling StartSend beforehand, thus + // Dtmf events cannot be sent and kFailedPrecondition is expected. + EXPECT_EQ( + voip_core_->SendDtmfEvent(channel, kDtmfEventCode, kDtmfEventDurationMs), + VoipResult::kFailedPrecondition); + + EXPECT_EQ(voip_core_->ReleaseChannel(channel), VoipResult::kOk); +} + +TEST_F(VoipCoreTest, StartSendAndPlayoutWithoutSettingCodec) { + auto channel = voip_core_->CreateChannel(&transport_, 0xdeadc0de); + + // Call StartSend and StartPlayout without setting send/receive + // codec. Code should see that codecs aren't set and return false. + EXPECT_EQ(voip_core_->StartSend(channel), VoipResult::kFailedPrecondition); + EXPECT_EQ(voip_core_->StartPlayout(channel), VoipResult::kFailedPrecondition); + + EXPECT_EQ(voip_core_->ReleaseChannel(channel), VoipResult::kOk); +} + +TEST_F(VoipCoreTest, StopSendAndPlayoutWithoutStarting) { + auto channel = voip_core_->CreateChannel(&transport_, 0xdeadc0de); + + EXPECT_EQ(voip_core_->SetSendCodec(channel, kPcmuPayload, kPcmuFormat), + VoipResult::kOk); + EXPECT_EQ( + voip_core_->SetReceiveCodecs(channel, {{kPcmuPayload, kPcmuFormat}}), + VoipResult::kOk); + + // Call StopSend and StopPlayout without starting them in + // the first place. Should see that it is already in the + // stopped state and return true. + EXPECT_EQ(voip_core_->StopSend(channel), VoipResult::kOk); + EXPECT_EQ(voip_core_->StopPlayout(channel), VoipResult::kOk); - EXPECT_FALSE(voip_core_->StartSend(*channel)); - EXPECT_FALSE(voip_core_->StartPlayout(*channel)); + EXPECT_EQ(voip_core_->ReleaseChannel(channel), VoipResult::kOk); } } // namespace diff --git a/audio/voip/voip_core.cc b/audio/voip/voip_core.cc index 7292644648..8df1c594aa 100644 --- a/audio/voip/voip_core.cc +++ b/audio/voip/voip_core.cc @@ -37,29 +37,33 @@ static constexpr int kMaxChannelId = 100000; } // namespace -bool VoipCore::Init(rtc::scoped_refptr<AudioEncoderFactory> encoder_factory, - rtc::scoped_refptr<AudioDecoderFactory> decoder_factory, - std::unique_ptr<TaskQueueFactory> task_queue_factory, - rtc::scoped_refptr<AudioDeviceModule> audio_device_module, - rtc::scoped_refptr<AudioProcessing> audio_processing) { +VoipCore::VoipCore(rtc::scoped_refptr<AudioEncoderFactory> encoder_factory, + rtc::scoped_refptr<AudioDecoderFactory> decoder_factory, + std::unique_ptr<TaskQueueFactory> task_queue_factory, + rtc::scoped_refptr<AudioDeviceModule> audio_device_module, + rtc::scoped_refptr<AudioProcessing> audio_processing) { encoder_factory_ = std::move(encoder_factory); decoder_factory_ = std::move(decoder_factory); task_queue_factory_ = std::move(task_queue_factory); audio_device_module_ = std::move(audio_device_module); - - process_thread_ = ProcessThread::Create("ModuleProcessThread"); + audio_processing_ = std::move(audio_processing); audio_mixer_ = AudioMixerImpl::Create(); - if (audio_processing) { - audio_processing_ = std::move(audio_processing); - AudioProcessing::Config apm_config = audio_processing_->GetConfig(); - apm_config.echo_canceller.enabled = true; - audio_processing_->ApplyConfig(apm_config); - } - // AudioTransportImpl depends on audio mixer and audio processing instances. audio_transport_ = std::make_unique<AudioTransportImpl>( - audio_mixer_.get(), audio_processing_.get()); + audio_mixer_.get(), audio_processing_.get(), nullptr); +} + +bool VoipCore::InitializeIfNeeded() { + // `audio_device_module_` internally owns a lock and the whole logic here + // needs to be executed atomically once using another lock in VoipCore. + // Further changes in this method will need to make sure that no deadlock is + // introduced in the future. + MutexLock lock(&lock_); + + if (initialized_) { + return true; + } // Initialize ADM. if (audio_device_module_->Init() != 0) { @@ -72,7 +76,6 @@ bool VoipCore::Init(rtc::scoped_refptr<AudioEncoderFactory> encoder_factory, // recording device functioning (e.g webinar where only speaker is available). // It's also possible that there are other audio devices available that may // work. - // TODO(natim@webrtc.org): consider moving this part out of initialization. // Initialize default speaker device. if (audio_device_module_->SetPlayoutDevice(kAudioDeviceId) != 0) { @@ -113,13 +116,14 @@ bool VoipCore::Init(rtc::scoped_refptr<AudioEncoderFactory> encoder_factory, RTC_LOG(LS_WARNING) << "Unable to register audio callback."; } + initialized_ = true; + return true; } -absl::optional<ChannelId> VoipCore::CreateChannel( - Transport* transport, - absl::optional<uint32_t> local_ssrc) { - absl::optional<ChannelId> channel; +ChannelId VoipCore::CreateChannel(Transport* transport, + absl::optional<uint32_t> local_ssrc) { + ChannelId channel_id; // Set local ssrc to random if not set by caller. if (!local_ssrc) { @@ -127,16 +131,16 @@ absl::optional<ChannelId> VoipCore::CreateChannel( local_ssrc = random.Rand<uint32_t>(); } - rtc::scoped_refptr<AudioChannel> audio_channel = - new rtc::RefCountedObject<AudioChannel>( - transport, local_ssrc.value(), task_queue_factory_.get(), - process_thread_.get(), audio_mixer_.get(), decoder_factory_); + rtc::scoped_refptr<AudioChannel> channel = + rtc::make_ref_counted<AudioChannel>(transport, local_ssrc.value(), + task_queue_factory_.get(), + audio_mixer_.get(), decoder_factory_); { MutexLock lock(&lock_); - channel = static_cast<ChannelId>(next_channel_id_); - channels_[*channel] = audio_channel; + channel_id = static_cast<ChannelId>(next_channel_id_); + channels_[channel_id] = channel; next_channel_id_++; if (next_channel_id_ >= kMaxChannelId) { next_channel_id_ = 0; @@ -144,41 +148,65 @@ absl::optional<ChannelId> VoipCore::CreateChannel( } // Set ChannelId in audio channel for logging/debugging purpose. - audio_channel->SetId(*channel); + channel->SetId(channel_id); - return channel; + return channel_id; } -void VoipCore::ReleaseChannel(ChannelId channel) { +VoipResult VoipCore::ReleaseChannel(ChannelId channel_id) { // Destroy channel outside of the lock. - rtc::scoped_refptr<AudioChannel> audio_channel; + rtc::scoped_refptr<AudioChannel> channel; + + bool no_channels_after_release = false; + { MutexLock lock(&lock_); - auto iter = channels_.find(channel); + auto iter = channels_.find(channel_id); if (iter != channels_.end()) { - audio_channel = std::move(iter->second); + channel = std::move(iter->second); channels_.erase(iter); } + + no_channels_after_release = channels_.empty(); + } + + VoipResult status_code = VoipResult::kOk; + if (!channel) { + RTC_LOG(LS_WARNING) << "Channel " << channel_id << " not found"; + status_code = VoipResult::kInvalidArgument; } - if (!audio_channel) { - RTC_LOG(LS_WARNING) << "Channel " << channel << " not found"; + + if (no_channels_after_release) { + // TODO(bugs.webrtc.org/11581): unclear if we still need to clear `channel` + // here. + channel = nullptr; + + // Make sure to stop playout on ADM if it is playing. + if (audio_device_module_->Playing()) { + if (audio_device_module_->StopPlayout() != 0) { + RTC_LOG(LS_WARNING) << "StopPlayout failed"; + status_code = VoipResult::kInternal; + } + } } + + return status_code; } -rtc::scoped_refptr<AudioChannel> VoipCore::GetChannel(ChannelId channel) { - rtc::scoped_refptr<AudioChannel> audio_channel; +rtc::scoped_refptr<AudioChannel> VoipCore::GetChannel(ChannelId channel_id) { + rtc::scoped_refptr<AudioChannel> channel; { MutexLock lock(&lock_); - auto iter = channels_.find(channel); + auto iter = channels_.find(channel_id); if (iter != channels_.end()) { - audio_channel = iter->second; + channel = iter->second; } } - if (!audio_channel) { - RTC_LOG(LS_ERROR) << "Channel " << channel << " not found"; + if (!channel) { + RTC_LOG(LS_ERROR) << "Channel " << channel_id << " not found"; } - return audio_channel; + return channel; } bool VoipCore::UpdateAudioTransportWithSenders() { @@ -216,6 +244,11 @@ bool VoipCore::UpdateAudioTransportWithSenders() { // Depending on availability of senders, turn on or off ADM recording. if (!audio_senders.empty()) { + // Initialize audio device module and default device if needed. + if (!InitializeIfNeeded()) { + return false; + } + if (!audio_device_module_->Recording()) { if (audio_device_module_->InitRecording() != 0) { RTC_LOG(LS_ERROR) << "InitRecording failed"; @@ -236,112 +269,232 @@ bool VoipCore::UpdateAudioTransportWithSenders() { return true; } -bool VoipCore::StartSend(ChannelId channel) { - auto audio_channel = GetChannel(channel); - if (!audio_channel) { - return false; +VoipResult VoipCore::StartSend(ChannelId channel_id) { + rtc::scoped_refptr<AudioChannel> channel = GetChannel(channel_id); + + if (!channel) { + return VoipResult::kInvalidArgument; } - audio_channel->StartSend(); + if (!channel->StartSend()) { + return VoipResult::kFailedPrecondition; + } - return UpdateAudioTransportWithSenders(); + return UpdateAudioTransportWithSenders() ? VoipResult::kOk + : VoipResult::kInternal; } -bool VoipCore::StopSend(ChannelId channel) { - auto audio_channel = GetChannel(channel); - if (!audio_channel) { - return false; +VoipResult VoipCore::StopSend(ChannelId channel_id) { + rtc::scoped_refptr<AudioChannel> channel = GetChannel(channel_id); + + if (!channel) { + return VoipResult::kInvalidArgument; } - audio_channel->StopSend(); + channel->StopSend(); - return UpdateAudioTransportWithSenders(); + return UpdateAudioTransportWithSenders() ? VoipResult::kOk + : VoipResult::kInternal; } -bool VoipCore::StartPlayout(ChannelId channel) { - auto audio_channel = GetChannel(channel); - if (!audio_channel) { - return false; +VoipResult VoipCore::StartPlayout(ChannelId channel_id) { + rtc::scoped_refptr<AudioChannel> channel = GetChannel(channel_id); + + if (!channel) { + return VoipResult::kInvalidArgument; } - audio_channel->StartPlay(); + if (channel->IsPlaying()) { + return VoipResult::kOk; + } + + if (!channel->StartPlay()) { + return VoipResult::kFailedPrecondition; + } + + // Initialize audio device module and default device if needed. + if (!InitializeIfNeeded()) { + return VoipResult::kInternal; + } if (!audio_device_module_->Playing()) { if (audio_device_module_->InitPlayout() != 0) { RTC_LOG(LS_ERROR) << "InitPlayout failed"; - return false; + return VoipResult::kInternal; } if (audio_device_module_->StartPlayout() != 0) { RTC_LOG(LS_ERROR) << "StartPlayout failed"; - return false; + return VoipResult::kInternal; } } - return true; + + return VoipResult::kOk; } -bool VoipCore::StopPlayout(ChannelId channel) { - auto audio_channel = GetChannel(channel); - if (!audio_channel) { - return false; +VoipResult VoipCore::StopPlayout(ChannelId channel_id) { + rtc::scoped_refptr<AudioChannel> channel = GetChannel(channel_id); + + if (!channel) { + return VoipResult::kInvalidArgument; } - audio_channel->StopPlay(); + channel->StopPlay(); - bool stop_device = true; - { - MutexLock lock(&lock_); - for (auto kv : channels_) { - rtc::scoped_refptr<AudioChannel>& channel = kv.second; - if (channel->IsPlaying()) { - stop_device = false; - break; - } - } + return VoipResult::kOk; +} + +VoipResult VoipCore::ReceivedRTPPacket( + ChannelId channel_id, + rtc::ArrayView<const uint8_t> rtp_packet) { + rtc::scoped_refptr<AudioChannel> channel = GetChannel(channel_id); + + if (!channel) { + return VoipResult::kInvalidArgument; } - if (stop_device && audio_device_module_->Playing()) { - if (audio_device_module_->StopPlayout() != 0) { - RTC_LOG(LS_ERROR) << "StopPlayout failed"; - return false; - } + channel->ReceivedRTPPacket(rtp_packet); + + return VoipResult::kOk; +} + +VoipResult VoipCore::ReceivedRTCPPacket( + ChannelId channel_id, + rtc::ArrayView<const uint8_t> rtcp_packet) { + rtc::scoped_refptr<AudioChannel> channel = GetChannel(channel_id); + + if (!channel) { + return VoipResult::kInvalidArgument; } - return true; + + channel->ReceivedRTCPPacket(rtcp_packet); + + return VoipResult::kOk; +} + +VoipResult VoipCore::SetSendCodec(ChannelId channel_id, + int payload_type, + const SdpAudioFormat& encoder_format) { + rtc::scoped_refptr<AudioChannel> channel = GetChannel(channel_id); + + if (!channel) { + return VoipResult::kInvalidArgument; + } + + auto encoder = encoder_factory_->MakeAudioEncoder( + payload_type, encoder_format, absl::nullopt); + channel->SetEncoder(payload_type, encoder_format, std::move(encoder)); + + return VoipResult::kOk; } -void VoipCore::ReceivedRTPPacket(ChannelId channel, - rtc::ArrayView<const uint8_t> rtp_packet) { - // Failure to locate channel is logged internally in GetChannel. - if (auto audio_channel = GetChannel(channel)) { - audio_channel->ReceivedRTPPacket(rtp_packet); +VoipResult VoipCore::SetReceiveCodecs( + ChannelId channel_id, + const std::map<int, SdpAudioFormat>& decoder_specs) { + rtc::scoped_refptr<AudioChannel> channel = GetChannel(channel_id); + + if (!channel) { + return VoipResult::kInvalidArgument; } + + channel->SetReceiveCodecs(decoder_specs); + + return VoipResult::kOk; } -void VoipCore::ReceivedRTCPPacket(ChannelId channel, - rtc::ArrayView<const uint8_t> rtcp_packet) { - // Failure to locate channel is logged internally in GetChannel. - if (auto audio_channel = GetChannel(channel)) { - audio_channel->ReceivedRTCPPacket(rtcp_packet); +VoipResult VoipCore::RegisterTelephoneEventType(ChannelId channel_id, + int rtp_payload_type, + int sample_rate_hz) { + rtc::scoped_refptr<AudioChannel> channel = GetChannel(channel_id); + + if (!channel) { + return VoipResult::kInvalidArgument; } + + channel->RegisterTelephoneEventType(rtp_payload_type, sample_rate_hz); + + return VoipResult::kOk; } -void VoipCore::SetSendCodec(ChannelId channel, - int payload_type, - const SdpAudioFormat& encoder_format) { - // Failure to locate channel is logged internally in GetChannel. - if (auto audio_channel = GetChannel(channel)) { - auto encoder = encoder_factory_->MakeAudioEncoder( - payload_type, encoder_format, absl::nullopt); - audio_channel->SetEncoder(payload_type, encoder_format, std::move(encoder)); +VoipResult VoipCore::SendDtmfEvent(ChannelId channel_id, + DtmfEvent dtmf_event, + int duration_ms) { + rtc::scoped_refptr<AudioChannel> channel = GetChannel(channel_id); + + if (!channel) { + return VoipResult::kInvalidArgument; } + + return (channel->SendTelephoneEvent(static_cast<int>(dtmf_event), duration_ms) + ? VoipResult::kOk + : VoipResult::kFailedPrecondition); } -void VoipCore::SetReceiveCodecs( - ChannelId channel, - const std::map<int, SdpAudioFormat>& decoder_specs) { - // Failure to locate channel is logged internally in GetChannel. - if (auto audio_channel = GetChannel(channel)) { - audio_channel->SetReceiveCodecs(decoder_specs); +VoipResult VoipCore::GetIngressStatistics(ChannelId channel_id, + IngressStatistics& ingress_stats) { + rtc::scoped_refptr<AudioChannel> channel = GetChannel(channel_id); + + if (!channel) { + return VoipResult::kInvalidArgument; } + + ingress_stats = channel->GetIngressStatistics(); + + return VoipResult::kOk; +} + +VoipResult VoipCore::GetChannelStatistics(ChannelId channel_id, + ChannelStatistics& channel_stats) { + rtc::scoped_refptr<AudioChannel> channel = GetChannel(channel_id); + + if (!channel) { + return VoipResult::kInvalidArgument; + } + + channel_stats = channel->GetChannelStatistics(); + + return VoipResult::kOk; +} + +VoipResult VoipCore::SetInputMuted(ChannelId channel_id, bool enable) { + rtc::scoped_refptr<AudioChannel> channel = GetChannel(channel_id); + + if (!channel) { + return VoipResult::kInvalidArgument; + } + + channel->SetMute(enable); + + return VoipResult::kOk; +} + +VoipResult VoipCore::GetInputVolumeInfo(ChannelId channel_id, + VolumeInfo& input_volume) { + rtc::scoped_refptr<AudioChannel> channel = GetChannel(channel_id); + + if (!channel) { + return VoipResult::kInvalidArgument; + } + + input_volume.audio_level = channel->GetInputAudioLevel(); + input_volume.total_energy = channel->GetInputTotalEnergy(); + input_volume.total_duration = channel->GetInputTotalDuration(); + + return VoipResult::kOk; +} + +VoipResult VoipCore::GetOutputVolumeInfo(ChannelId channel_id, + VolumeInfo& output_volume) { + rtc::scoped_refptr<AudioChannel> channel = GetChannel(channel_id); + + if (!channel) { + return VoipResult::kInvalidArgument; + } + + output_volume.audio_level = channel->GetOutputAudioLevel(); + output_volume.total_energy = channel->GetOutputTotalEnergy(); + output_volume.total_duration = channel->GetOutputTotalDuration(); + + return VoipResult::kOk; } } // namespace webrtc diff --git a/audio/voip/voip_core.h b/audio/voip/voip_core.h index 22a6559981..6c3aec6fa2 100644 --- a/audio/voip/voip_core.h +++ b/audio/voip/voip_core.h @@ -23,14 +23,16 @@ #include "api/task_queue/task_queue_factory.h" #include "api/voip/voip_base.h" #include "api/voip/voip_codec.h" +#include "api/voip/voip_dtmf.h" #include "api/voip/voip_engine.h" #include "api/voip/voip_network.h" +#include "api/voip/voip_statistics.h" +#include "api/voip/voip_volume_control.h" #include "audio/audio_transport_impl.h" #include "audio/voip/audio_channel.h" #include "modules/audio_device/include/audio_device.h" #include "modules/audio_mixer/audio_mixer_impl.h" #include "modules/audio_processing/include/audio_processing.h" -#include "modules/utility/include/process_thread.h" #include "rtc_base/synchronization/mutex.h" namespace webrtc { @@ -45,53 +47,87 @@ namespace webrtc { class VoipCore : public VoipEngine, public VoipBase, public VoipNetwork, - public VoipCodec { + public VoipCodec, + public VoipDtmf, + public VoipStatistics, + public VoipVolumeControl { public: + // Construct VoipCore with provided arguments. + VoipCore(rtc::scoped_refptr<AudioEncoderFactory> encoder_factory, + rtc::scoped_refptr<AudioDecoderFactory> decoder_factory, + std::unique_ptr<TaskQueueFactory> task_queue_factory, + rtc::scoped_refptr<AudioDeviceModule> audio_device_module, + rtc::scoped_refptr<AudioProcessing> audio_processing); ~VoipCore() override = default; - // Initialize VoipCore components with provided arguments. - // Returns false only when |audio_device_module| fails to initialize which - // would presumably render further processing useless. - // TODO(natim@webrtc.org): Need to report audio device errors to user layer. - bool Init(rtc::scoped_refptr<AudioEncoderFactory> encoder_factory, - rtc::scoped_refptr<AudioDecoderFactory> decoder_factory, - std::unique_ptr<TaskQueueFactory> task_queue_factory, - rtc::scoped_refptr<AudioDeviceModule> audio_device_module, - rtc::scoped_refptr<AudioProcessing> audio_processing); - // Implements VoipEngine interfaces. VoipBase& Base() override { return *this; } VoipNetwork& Network() override { return *this; } VoipCodec& Codec() override { return *this; } + VoipDtmf& Dtmf() override { return *this; } + VoipStatistics& Statistics() override { return *this; } + VoipVolumeControl& VolumeControl() override { return *this; } // Implements VoipBase interfaces. - absl::optional<ChannelId> CreateChannel( - Transport* transport, - absl::optional<uint32_t> local_ssrc) override; - void ReleaseChannel(ChannelId channel) override; - bool StartSend(ChannelId channel) override; - bool StopSend(ChannelId channel) override; - bool StartPlayout(ChannelId channel) override; - bool StopPlayout(ChannelId channel) override; + ChannelId CreateChannel(Transport* transport, + absl::optional<uint32_t> local_ssrc) override; + VoipResult ReleaseChannel(ChannelId channel_id) override; + VoipResult StartSend(ChannelId channel_id) override; + VoipResult StopSend(ChannelId channel_id) override; + VoipResult StartPlayout(ChannelId channel_id) override; + VoipResult StopPlayout(ChannelId channel_id) override; // Implements VoipNetwork interfaces. - void ReceivedRTPPacket(ChannelId channel, - rtc::ArrayView<const uint8_t> rtp_packet) override; - void ReceivedRTCPPacket(ChannelId channel, - rtc::ArrayView<const uint8_t> rtcp_packet) override; + VoipResult ReceivedRTPPacket( + ChannelId channel_id, + rtc::ArrayView<const uint8_t> rtp_packet) override; + VoipResult ReceivedRTCPPacket( + ChannelId channel_id, + rtc::ArrayView<const uint8_t> rtcp_packet) override; // Implements VoipCodec interfaces. - void SetSendCodec(ChannelId channel, - int payload_type, - const SdpAudioFormat& encoder_format) override; - void SetReceiveCodecs( - ChannelId channel, + VoipResult SetSendCodec(ChannelId channel_id, + int payload_type, + const SdpAudioFormat& encoder_format) override; + VoipResult SetReceiveCodecs( + ChannelId channel_id, const std::map<int, SdpAudioFormat>& decoder_specs) override; + // Implements VoipDtmf interfaces. + VoipResult RegisterTelephoneEventType(ChannelId channel_id, + int rtp_payload_type, + int sample_rate_hz) override; + VoipResult SendDtmfEvent(ChannelId channel_id, + DtmfEvent dtmf_event, + int duration_ms) override; + + // Implements VoipStatistics interfaces. + VoipResult GetIngressStatistics(ChannelId channel_id, + IngressStatistics& ingress_stats) override; + VoipResult GetChannelStatistics(ChannelId channe_id, + ChannelStatistics& channel_stats) override; + + // Implements VoipVolumeControl interfaces. + VoipResult SetInputMuted(ChannelId channel_id, bool enable) override; + VoipResult GetInputVolumeInfo(ChannelId channel_id, + VolumeInfo& volume_info) override; + VoipResult GetOutputVolumeInfo(ChannelId channel_id, + VolumeInfo& volume_info) override; + private: - // Fetches the corresponding AudioChannel assigned with given |channel|. + // Initialize ADM and default audio device if needed. + // Returns true if ADM is successfully initialized or already in such state + // (e.g called more than once). Returns false when ADM fails to initialize + // which would presumably render further processing useless. Note that such + // failure won't necessarily succeed in next initialization attempt as it + // would mean changing the ADM implementation. From Android N and onwards, the + // mobile app may not be able to gain microphone access when in background + // mode. Therefore it would be better to delay the logic as late as possible. + bool InitializeIfNeeded(); + + // Fetches the corresponding AudioChannel assigned with given `channel`. // Returns nullptr if not found. - rtc::scoped_refptr<AudioChannel> GetChannel(ChannelId channel); + rtc::scoped_refptr<AudioChannel> GetChannel(ChannelId channel_id); // Updates AudioTransportImpl with a new set of actively sending AudioSender // (AudioEgress). This needs to be invoked whenever StartSend/StopSend is @@ -104,25 +140,21 @@ class VoipCore : public VoipEngine, rtc::scoped_refptr<AudioDecoderFactory> decoder_factory_; std::unique_ptr<TaskQueueFactory> task_queue_factory_; - // Synchronization is handled internally by AudioProessing. - // Must be placed before |audio_device_module_| for proper destruction. + // Synchronization is handled internally by AudioProcessing. + // Must be placed before `audio_device_module_` for proper destruction. rtc::scoped_refptr<AudioProcessing> audio_processing_; // Synchronization is handled internally by AudioMixer. - // Must be placed before |audio_device_module_| for proper destruction. + // Must be placed before `audio_device_module_` for proper destruction. rtc::scoped_refptr<AudioMixer> audio_mixer_; // Synchronization is handled internally by AudioTransportImpl. - // Must be placed before |audio_device_module_| for proper destruction. + // Must be placed before `audio_device_module_` for proper destruction. std::unique_ptr<AudioTransportImpl> audio_transport_; // Synchronization is handled internally by AudioDeviceModule. rtc::scoped_refptr<AudioDeviceModule> audio_device_module_; - // Synchronization is handled internally by ProcessThread. - // Must be placed before |channels_| for proper destruction. - std::unique_ptr<ProcessThread> process_thread_; - Mutex lock_; // Member to track a next ChannelId for new AudioChannel. @@ -132,6 +164,9 @@ class VoipCore : public VoipEngine, // ChannelId. std::unordered_map<ChannelId, rtc::scoped_refptr<AudioChannel>> channels_ RTC_GUARDED_BY(lock_); + + // Boolean flag to ensure initialization only occurs once. + bool initialized_ RTC_GUARDED_BY(lock_) = false; }; } // namespace webrtc |