diff options
author | Jorge E. Moreira <jemoreira@google.com> | 2020-07-17 14:12:45 -0700 |
---|---|---|
committer | Jorge E. Moreira <jemoreira@google.com> | 2020-07-17 14:25:15 -0700 |
commit | 85770cd25bd31d7241fa6588912981dcd110a14b (patch) | |
tree | 93180f3f3e0eabb38ff99f276ce8c6c4f535c22d /media | |
parent | 477c82e10a6e48c0d446b90aaeb9c6dacae5ae3f (diff) | |
parent | cc57b935cdd19bc78352ab7e3091b2b3ad653074 (diff) | |
download | webrtc-85770cd25bd31d7241fa6588912981dcd110a14b.tar.gz |
Merge branch 'upstream-master'
Bug: 153469641
Test: none
Change-Id: Ic33e363deb0d1ac4bb4d57c3c239eb2e45370802
Diffstat (limited to 'media')
114 files changed, 43047 insertions, 0 deletions
diff --git a/media/BUILD.gn b/media/BUILD.gn new file mode 100644 index 0000000000..28a8755615 --- /dev/null +++ b/media/BUILD.gn @@ -0,0 +1,655 @@ +# Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +import("//build/config/linux/pkg_config.gni") +import("../webrtc.gni") + +group("media") { + deps = [] + if (!build_with_mozilla) { + deps += [ + ":rtc_media", + ":rtc_media_base", + ] + } +} + +config("rtc_media_defines_config") { + defines = [ "HAVE_WEBRTC_VIDEO" ] +} + +rtc_library("rtc_h264_profile_id") { + visibility = [ "*" ] + sources = [ + "base/h264_profile_level_id.cc", + "base/h264_profile_level_id.h", + ] + + deps = [ + "..:webrtc_common", + "../rtc_base", + "../rtc_base:checks", + "../rtc_base:rtc_base_approved", + "../rtc_base/system:rtc_export", + "//third_party/abseil-cpp/absl/types:optional", + ] +} + +rtc_source_set("rtc_media_config") { + visibility = [ "*" ] + sources = [ "base/media_config.h" ] +} + +rtc_library("rtc_vp9_profile") { + visibility = [ "*" ] + sources = [ + "base/vp9_profile.cc", + "base/vp9_profile.h", + ] + + deps = [ + "..:webrtc_common", + "../api/video_codecs:video_codecs_api", + "../rtc_base:rtc_base_approved", + "../rtc_base/system:rtc_export", + "//third_party/abseil-cpp/absl/types:optional", + ] +} + +rtc_library("rtc_sdp_fmtp_utils") { + visibility = [ "*" ] + sources = [ + "base/sdp_fmtp_utils.cc", + "base/sdp_fmtp_utils.h", + ] + + deps = [ + "../api/video_codecs:video_codecs_api", + "../rtc_base:stringutils", + "//third_party/abseil-cpp/absl/types:optional", + ] +} + +rtc_library("rtc_media_base") { + visibility = [ "*" ] + defines = [] + libs = [] + deps = [ + ":rtc_h264_profile_id", + ":rtc_media_config", + ":rtc_vp9_profile", + "..:webrtc_common", + "../api:array_view", + "../api:audio_options_api", + "../api:frame_transformer_interface", + "../api:media_stream_interface", + "../api:rtc_error", + "../api:rtp_parameters", + "../api:scoped_refptr", + "../api/audio_codecs:audio_codecs_api", + "../api/crypto:frame_decryptor_interface", + "../api/crypto:frame_encryptor_interface", + "../api/crypto:options", + "../api/transport:stun_types", + "../api/transport/media:media_transport_interface", + "../api/transport/rtp:rtp_source", + "../api/video:video_bitrate_allocation", + "../api/video:video_bitrate_allocator_factory", + "../api/video:video_frame", + "../api/video:video_frame_i420", + "../api/video:video_rtp_headers", + "../api/video_codecs:video_codecs_api", + "../call:call_interfaces", + "../call:video_stream_api", + "../common_video", + "../modules/audio_processing:audio_processing_statistics", + "../modules/rtp_rtcp:rtp_rtcp_format", + "../rtc_base", + "../rtc_base:checks", + "../rtc_base:rtc_base_approved", + "../rtc_base:rtc_base_approved", + "../rtc_base:rtc_task_queue", + "../rtc_base:sanitizer", + "../rtc_base:stringutils", + "../rtc_base/synchronization:sequence_checker", + "../rtc_base/system:file_wrapper", + "../rtc_base/system:rtc_export", + "../rtc_base/third_party/sigslot", + "../system_wrappers:field_trial", + "//third_party/abseil-cpp/absl/algorithm:container", + "//third_party/abseil-cpp/absl/strings", + "//third_party/abseil-cpp/absl/types:optional", + ] + sources = [ + "base/adapted_video_track_source.cc", + "base/adapted_video_track_source.h", + "base/audio_source.h", + "base/codec.cc", + "base/codec.h", + "base/delayable.h", + "base/media_channel.cc", + "base/media_channel.h", + "base/media_constants.cc", + "base/media_constants.h", + "base/media_engine.cc", + "base/media_engine.h", + "base/rid_description.cc", + "base/rid_description.h", + "base/rtp_data_engine.cc", + "base/rtp_data_engine.h", + "base/rtp_utils.cc", + "base/rtp_utils.h", + "base/stream_params.cc", + "base/stream_params.h", + "base/turn_utils.cc", + "base/turn_utils.h", + "base/video_adapter.cc", + "base/video_adapter.h", + "base/video_broadcaster.cc", + "base/video_broadcaster.h", + "base/video_common.cc", + "base/video_common.h", + "base/video_source_base.cc", + "base/video_source_base.h", + ] +} + +rtc_library("rtc_constants") { + defines = [] + libs = [] + deps = [] + sources = [ + "engine/constants.cc", + "engine/constants.h", + ] +} + +rtc_library("rtc_simulcast_encoder_adapter") { + visibility = [ "*" ] + defines = [] + libs = [] + sources = [ + "engine/simulcast_encoder_adapter.cc", + "engine/simulcast_encoder_adapter.h", + ] + deps = [ + ":rtc_media_base", + "../api:fec_controller_api", + "../api:scoped_refptr", + "../api/video:video_codec_constants", + "../api/video:video_frame", + "../api/video:video_frame_i420", + "../api/video:video_rtp_headers", + "../api/video_codecs:rtc_software_fallback_wrappers", + "../api/video_codecs:video_codecs_api", + "../call:video_stream_api", + "../modules/video_coding:video_codec_interface", + "../modules/video_coding:video_coding_utility", + "../rtc_base:checks", + "../rtc_base:rtc_base_approved", + "../rtc_base/experiments:rate_control_settings", + "../rtc_base/synchronization:sequence_checker", + "../rtc_base/system:rtc_export", + "../system_wrappers", + "../system_wrappers:field_trial", + "//third_party/abseil-cpp/absl/types:optional", + ] +} + +rtc_library("rtc_encoder_simulcast_proxy") { + visibility = [ "*" ] + defines = [] + libs = [] + sources = [ + "engine/encoder_simulcast_proxy.cc", + "engine/encoder_simulcast_proxy.h", + ] + deps = [ + ":rtc_simulcast_encoder_adapter", + "../api/video:video_bitrate_allocation", + "../api/video:video_frame", + "../api/video:video_rtp_headers", + "../api/video_codecs:video_codecs_api", + "../modules/video_coding:video_codec_interface", + "../rtc_base/system:rtc_export", + ] +} + +rtc_library("rtc_internal_video_codecs") { + visibility = [ "*" ] + allow_poison = [ "software_video_codecs" ] + defines = [] + libs = [] + deps = [ + ":rtc_constants", + ":rtc_encoder_simulcast_proxy", + ":rtc_h264_profile_id", + ":rtc_media_base", + ":rtc_simulcast_encoder_adapter", + "../:webrtc_common", + "../api/video:encoded_image", + "../api/video:video_bitrate_allocation", + "../api/video:video_frame", + "../api/video:video_rtp_headers", + "../api/video_codecs:rtc_software_fallback_wrappers", + "../api/video_codecs:video_codecs_api", + "../call:call_interfaces", + "../call:video_stream_api", + "../modules:module_api", + "../modules/video_coding:video_codec_interface", + "../modules/video_coding:webrtc_h264", + "../modules/video_coding:webrtc_multiplex", + "../modules/video_coding:webrtc_vp8", + "../modules/video_coding:webrtc_vp9", + "../modules/video_coding/codecs/av1:libaom_av1_decoder", + "../modules/video_coding/codecs/av1:libaom_av1_encoder", + "../rtc_base:checks", + "../rtc_base:deprecation", + "../rtc_base:rtc_base_approved", + "../rtc_base/system:rtc_export", + "../test:fake_video_codecs", + "//third_party/abseil-cpp/absl/strings", + ] + sources = [ + "engine/fake_video_codec_factory.cc", + "engine/fake_video_codec_factory.h", + "engine/internal_decoder_factory.cc", + "engine/internal_decoder_factory.h", + "engine/internal_encoder_factory.cc", + "engine/internal_encoder_factory.h", + "engine/multiplex_codec_factory.cc", + "engine/multiplex_codec_factory.h", + + # TODO(bugs.webrtc.org/7925): stop exporting this header once downstream + # targets depend on :rtc_encoder_simulcast_proxy directly. + "engine/encoder_simulcast_proxy.h", + ] +} + +rtc_library("rtc_audio_video") { + visibility = [ "*" ] + allow_poison = [ "audio_codecs" ] # TODO(bugs.webrtc.org/8396): Remove. + defines = [] + libs = [] + deps = [ + ":rtc_constants", + ":rtc_media_base", + "..:webrtc_common", + "../api:call_api", + "../api:libjingle_peerconnection_api", + "../api:media_stream_interface", + "../api:rtp_parameters", + "../api:scoped_refptr", + "../api:transport_api", + "../api/audio:audio_mixer_api", + "../api/audio_codecs:audio_codecs_api", + "../api/task_queue", + "../api/transport:bitrate_settings", + "../api/transport:datagram_transport_interface", + "../api/transport/media:media_transport_interface", + "../api/transport/rtp:rtp_source", + "../api/units:data_rate", + "../api/video:video_bitrate_allocation", + "../api/video:video_bitrate_allocator_factory", + "../api/video:video_codec_constants", + "../api/video:video_frame", + "../api/video:video_frame_i420", + "../api/video:video_rtp_headers", + "../api/video_codecs:rtc_software_fallback_wrappers", + "../api/video_codecs:video_codecs_api", + "../call", + "../call:call_interfaces", + "../call:video_stream_api", + "../common_video", + "../modules/audio_device", + "../modules/audio_device:audio_device_impl", + "../modules/audio_mixer:audio_mixer_impl", + "../modules/audio_processing:api", + "../modules/audio_processing/aec_dump", + "../modules/audio_processing/agc:gain_control_interface", + "../modules/video_coding", + "../modules/video_coding:video_codec_interface", + "../modules/video_coding:video_coding_utility", + "../rtc_base", + "../rtc_base:audio_format_to_string", + "../rtc_base:checks", + "../rtc_base:rtc_task_queue", + "../rtc_base:stringutils", + "../rtc_base/experiments:field_trial_parser", + "../rtc_base/experiments:min_video_bitrate_experiment", + "../rtc_base/experiments:normalize_simulcast_size_experiment", + "../rtc_base/experiments:rate_control_settings", + "../rtc_base/system:rtc_export", + "../rtc_base/third_party/base64", + "../system_wrappers", + "../system_wrappers:field_trial", + "../system_wrappers:metrics", + "//third_party/abseil-cpp/absl/algorithm:container", + "//third_party/abseil-cpp/absl/strings", + "//third_party/abseil-cpp/absl/types:optional", + ] + + sources = [ + "engine/adm_helpers.cc", + "engine/adm_helpers.h", + "engine/null_webrtc_video_engine.h", + "engine/payload_type_mapper.cc", + "engine/payload_type_mapper.h", + "engine/simulcast.cc", + "engine/simulcast.h", + "engine/unhandled_packets_buffer.cc", + "engine/unhandled_packets_buffer.h", + "engine/webrtc_media_engine.cc", + "engine/webrtc_media_engine.h", + "engine/webrtc_video_engine.cc", + "engine/webrtc_video_engine.h", + "engine/webrtc_voice_engine.cc", + "engine/webrtc_voice_engine.h", + ] + + public_configs = [] + if (!build_with_chromium) { + public_configs += [ ":rtc_media_defines_config" ] + deps += [ "../modules/video_capture:video_capture_internal_impl" ] + } + if (rtc_enable_protobuf) { + deps += [ "../modules/audio_processing/aec_dump:aec_dump_impl" ] + } else { + deps += [ "../modules/audio_processing/aec_dump:null_aec_dump_factory" ] + } +} + +# Heavy but optional helper for unittests and webrtc users who prefer to use +# defaults factories or do not worry about extra dependencies and binary size. +rtc_library("rtc_media_engine_defaults") { + visibility = [ "*" ] + allow_poison = [ + "audio_codecs", + "default_task_queue", + "software_video_codecs", + ] + sources = [ + "engine/webrtc_media_engine_defaults.cc", + "engine/webrtc_media_engine_defaults.h", + ] + deps = [ + ":rtc_audio_video", + "../api/audio_codecs:builtin_audio_decoder_factory", + "../api/audio_codecs:builtin_audio_encoder_factory", + "../api/task_queue:default_task_queue_factory", + "../api/video:builtin_video_bitrate_allocator_factory", + "../api/video_codecs:builtin_video_decoder_factory", + "../api/video_codecs:builtin_video_encoder_factory", + "../modules/audio_processing:api", + "../rtc_base:checks", + "../rtc_base/system:rtc_export", + ] +} + +rtc_library("rtc_data") { + defines = [ + # "SCTP_DEBUG" # Uncomment for SCTP debugging. + ] + deps = [ + ":rtc_media_base", + "..:webrtc_common", + "../api:call_api", + "../api:transport_api", + "../p2p:rtc_p2p", + "../rtc_base", + "../rtc_base:rtc_base_approved", + "../rtc_base/third_party/sigslot", + "../system_wrappers", + "//third_party/abseil-cpp/absl/algorithm:container", + "//third_party/abseil-cpp/absl/base:core_headers", + "//third_party/abseil-cpp/absl/types:optional", + ] + + if (rtc_enable_sctp) { + sources = [ + "sctp/sctp_transport.cc", + "sctp/sctp_transport.h", + "sctp/sctp_transport_internal.h", + ] + } else { + # libtool on mac does not like empty targets. + sources = [ "sctp/noop.cc" ] + } + + if (rtc_enable_sctp && rtc_build_usrsctp) { + include_dirs = [ + # TODO(jiayl): move this into the public_configs of + # //third_party/usrsctp/BUILD.gn. + "//third_party/usrsctp/usrsctplib", + ] + deps += [ "//third_party/usrsctp" ] + } +} + +rtc_source_set("rtc_media") { + visibility = [ "*" ] + allow_poison = [ "audio_codecs" ] # TODO(bugs.webrtc.org/8396): Remove. + deps = [ + ":rtc_audio_video", + ":rtc_data", + ] +} + +if (rtc_include_tests) { + rtc_library("rtc_media_tests_utils") { + testonly = true + + defines = [] + deps = [ + ":rtc_audio_video", + ":rtc_internal_video_codecs", + ":rtc_media", + ":rtc_media_base", + ":rtc_simulcast_encoder_adapter", + "../api:call_api", + "../api:fec_controller_api", + "../api:scoped_refptr", + "../api/video:encoded_image", + "../api/video:video_bitrate_allocation", + "../api/video:video_frame", + "../api/video:video_frame_i420", + "../api/video:video_rtp_headers", + "../api/video_codecs:video_codecs_api", + "../call:call_interfaces", + "../call:mock_rtp_interfaces", + "../call:video_stream_api", + "../common_video", + "../modules/audio_processing", + "../modules/audio_processing:api", + "../modules/rtp_rtcp:rtp_rtcp_format", + "../modules/video_coding:video_codec_interface", + "../modules/video_coding:video_coding_utility", + "../p2p:rtc_p2p", + "../rtc_base", + "../rtc_base:checks", + "../rtc_base:gunit_helpers", + "../rtc_base:rtc_base_approved", + "../rtc_base:rtc_task_queue", + "../rtc_base:stringutils", + "../rtc_base/third_party/sigslot", + "../test:test_support", + "//testing/gtest", + "//third_party/abseil-cpp/absl/algorithm:container", + "//third_party/abseil-cpp/absl/strings", + ] + sources = [ + "base/fake_frame_source.cc", + "base/fake_frame_source.h", + "base/fake_media_engine.cc", + "base/fake_media_engine.h", + "base/fake_network_interface.h", + "base/fake_rtp.cc", + "base/fake_rtp.h", + "base/fake_video_renderer.cc", + "base/fake_video_renderer.h", + "base/test_utils.cc", + "base/test_utils.h", + "engine/fake_webrtc_call.cc", + "engine/fake_webrtc_call.h", + "engine/fake_webrtc_video_engine.cc", + "engine/fake_webrtc_video_engine.h", + ] + } + + rtc_media_unittests_resources = [ + "../resources/media/captured-320x240-2s-48.frames", + "../resources/media/faces.1280x720_P420.yuv", + "../resources/media/faces_I400.jpg", + "../resources/media/faces_I411.jpg", + "../resources/media/faces_I420.jpg", + "../resources/media/faces_I422.jpg", + "../resources/media/faces_I444.jpg", + ] + + if (is_ios) { + bundle_data("rtc_media_unittests_bundle_data") { + testonly = true + sources = rtc_media_unittests_resources + outputs = [ "{{bundle_resources_dir}}/{{source_file_part}}" ] + } + } + + rtc_test("rtc_media_unittests") { + testonly = true + + defines = [] + deps = [ + ":rtc_audio_video", + ":rtc_constants", + ":rtc_data", + ":rtc_encoder_simulcast_proxy", + ":rtc_internal_video_codecs", + ":rtc_media", + ":rtc_media_base", + ":rtc_media_engine_defaults", + ":rtc_media_tests_utils", + ":rtc_sdp_fmtp_utils", + ":rtc_simulcast_encoder_adapter", + ":rtc_vp9_profile", + "../:webrtc_common", + "../api:create_simulcast_test_fixture_api", + "../api:libjingle_peerconnection_api", + "../api:mock_video_bitrate_allocator", + "../api:mock_video_bitrate_allocator_factory", + "../api:mock_video_codec_factory", + "../api:mock_video_encoder", + "../api:rtp_parameters", + "../api:scoped_refptr", + "../api:simulcast_test_fixture_api", + "../api/audio_codecs:builtin_audio_decoder_factory", + "../api/audio_codecs:builtin_audio_encoder_factory", + "../api/rtc_event_log", + "../api/task_queue", + "../api/task_queue:default_task_queue_factory", + "../api/test/video:function_video_factory", + "../api/transport:field_trial_based_config", + "../api/transport/media:media_transport_interface", + "../api/units:time_delta", + "../api/video:builtin_video_bitrate_allocator_factory", + "../api/video:video_bitrate_allocation", + "../api/video:video_frame", + "../api/video:video_frame_i420", + "../api/video:video_rtp_headers", + "../api/video_codecs:builtin_video_decoder_factory", + "../api/video_codecs:builtin_video_encoder_factory", + "../api/video_codecs:video_codecs_api", + "../audio", + "../call:call_interfaces", + "../common_video", + "../media:rtc_h264_profile_id", + "../modules/audio_device:mock_audio_device", + "../modules/audio_processing", + "../modules/audio_processing:api", + "../modules/audio_processing:mocks", + "../modules/rtp_rtcp", + "../modules/video_coding:simulcast_test_fixture_impl", + "../modules/video_coding:video_codec_interface", + "../modules/video_coding:webrtc_h264", + "../modules/video_coding:webrtc_vp8", + "../modules/video_coding/codecs/av1:libaom_av1_decoder", + "../p2p:p2p_test_utils", + "../rtc_base", + "../rtc_base:checks", + "../rtc_base:gunit_helpers", + "../rtc_base:rtc_base_approved", + "../rtc_base:rtc_base_tests_utils", + "../rtc_base:rtc_task_queue", + "../rtc_base:stringutils", + "../rtc_base/experiments:min_video_bitrate_experiment", + "../rtc_base/third_party/sigslot", + "../test:audio_codec_mocks", + "../test:fake_video_codecs", + "../test:field_trial", + "../test:rtp_test_utils", + "../test:test_main", + "../test:test_support", + "../test:video_test_common", + "//third_party/abseil-cpp/absl/algorithm:container", + "//third_party/abseil-cpp/absl/memory", + "//third_party/abseil-cpp/absl/strings", + "//third_party/abseil-cpp/absl/types:optional", + ] + sources = [ + "base/codec_unittest.cc", + "base/media_engine_unittest.cc", + "base/rtp_data_engine_unittest.cc", + "base/rtp_utils_unittest.cc", + "base/sdp_fmtp_utils_unittest.cc", + "base/stream_params_unittest.cc", + "base/turn_utils_unittest.cc", + "base/video_adapter_unittest.cc", + "base/video_broadcaster_unittest.cc", + "base/video_common_unittest.cc", + "engine/encoder_simulcast_proxy_unittest.cc", + "engine/internal_decoder_factory_unittest.cc", + "engine/multiplex_codec_factory_unittest.cc", + "engine/null_webrtc_video_engine_unittest.cc", + "engine/payload_type_mapper_unittest.cc", + "engine/simulcast_encoder_adapter_unittest.cc", + "engine/simulcast_unittest.cc", + "engine/unhandled_packets_buffer_unittest.cc", + "engine/webrtc_media_engine_unittest.cc", + "engine/webrtc_video_engine_unittest.cc", + ] + + # TODO(kthelgason): Reenable this test on iOS. + # See bugs.webrtc.org/5569 + if (!is_ios) { + sources += [ "engine/webrtc_voice_engine_unittest.cc" ] + } + + if (rtc_enable_sctp) { + sources += [ + "sctp/sctp_transport_reliability_unittest.cc", + "sctp/sctp_transport_unittest.cc", + ] + } + + if (rtc_opus_support_120ms_ptime) { + defines += [ "WEBRTC_OPUS_SUPPORT_120MS_PTIME=1" ] + } else { + defines += [ "WEBRTC_OPUS_SUPPORT_120MS_PTIME=0" ] + } + + data = rtc_media_unittests_resources + + if (is_android) { + deps += [ "//testing/android/native_test:native_test_support" ] + shard_timeout = 900 + } + + if (is_ios) { + deps += [ ":rtc_media_unittests_bundle_data" ] + } + } +} diff --git a/media/DEPS b/media/DEPS new file mode 100644 index 0000000000..1e13c18b16 --- /dev/null +++ b/media/DEPS @@ -0,0 +1,24 @@ +include_rules = [ + "+call", + "+common_video", + "+logging/rtc_event_log", + "+modules/audio_coding", + "+modules/audio_device", + "+modules/audio_mixer", + "+modules/audio_processing", + "+modules/rtp_rtcp", + "+modules/video_capture", + "+modules/video_coding", + "+modules/video_coding/utility", + "+p2p", + "+sound", + "+system_wrappers", + "+usrsctplib", + "+third_party/libyuv", +] + +specific_include_rules = { + "win32devicemanager\.cc": [ + "+third_party/logitech/files/logitechquickcam.h", + ], +} diff --git a/media/OWNERS.webrtc b/media/OWNERS.webrtc new file mode 100644 index 0000000000..b8910326b9 --- /dev/null +++ b/media/OWNERS.webrtc @@ -0,0 +1,10 @@ +nisse@webrtc.org +ilnik@webrtc.org +sprang@webrtc.org +magjed@webrtc.org +mflodman@webrtc.org +perkj@webrtc.org + +# Audio-related changes: +peah@webrtc.org +saza@webrtc.org diff --git a/media/base/adapted_video_track_source.cc b/media/base/adapted_video_track_source.cc new file mode 100644 index 0000000000..c4918725d2 --- /dev/null +++ b/media/base/adapted_video_track_source.cc @@ -0,0 +1,117 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "media/base/adapted_video_track_source.h" + +#include "api/scoped_refptr.h" +#include "api/video/i420_buffer.h" +#include "api/video/video_frame_buffer.h" +#include "api/video/video_rotation.h" +#include "rtc_base/checks.h" +#include "rtc_base/time_utils.h" + +namespace rtc { + +AdaptedVideoTrackSource::AdaptedVideoTrackSource() = default; + +AdaptedVideoTrackSource::AdaptedVideoTrackSource(int required_alignment) + : video_adapter_(required_alignment) {} + +AdaptedVideoTrackSource::~AdaptedVideoTrackSource() = default; + +bool AdaptedVideoTrackSource::GetStats(Stats* stats) { + rtc::CritScope lock(&stats_crit_); + + if (!stats_) { + return false; + } + + *stats = *stats_; + return true; +} + +void AdaptedVideoTrackSource::OnFrame(const webrtc::VideoFrame& frame) { + rtc::scoped_refptr<webrtc::VideoFrameBuffer> buffer( + frame.video_frame_buffer()); + /* Note that this is a "best effort" approach to + wants.rotation_applied; apply_rotation_ can change from false to + true between the check of apply_rotation() and the call to + broadcaster_.OnFrame(), in which case we generate a frame with + pending rotation despite some sink with wants.rotation_applied == + true was just added. The VideoBroadcaster enforces + synchronization for us in this case, by not passing the frame on + to sinks which don't want it. */ + if (apply_rotation() && frame.rotation() != webrtc::kVideoRotation_0 && + buffer->type() == webrtc::VideoFrameBuffer::Type::kI420) { + /* Apply pending rotation. */ + webrtc::VideoFrame rotated_frame(frame); + rotated_frame.set_video_frame_buffer( + webrtc::I420Buffer::Rotate(*buffer->GetI420(), frame.rotation())); + rotated_frame.set_rotation(webrtc::kVideoRotation_0); + broadcaster_.OnFrame(rotated_frame); + } else { + broadcaster_.OnFrame(frame); + } +} + +void AdaptedVideoTrackSource::AddOrUpdateSink( + rtc::VideoSinkInterface<webrtc::VideoFrame>* sink, + const rtc::VideoSinkWants& wants) { + broadcaster_.AddOrUpdateSink(sink, wants); + OnSinkWantsChanged(broadcaster_.wants()); +} + +void AdaptedVideoTrackSource::RemoveSink( + rtc::VideoSinkInterface<webrtc::VideoFrame>* sink) { + broadcaster_.RemoveSink(sink); + OnSinkWantsChanged(broadcaster_.wants()); +} + +bool AdaptedVideoTrackSource::apply_rotation() { + return broadcaster_.wants().rotation_applied; +} + +void AdaptedVideoTrackSource::OnSinkWantsChanged( + const rtc::VideoSinkWants& wants) { + video_adapter_.OnSinkWants(wants); +} + +bool AdaptedVideoTrackSource::AdaptFrame(int width, + int height, + int64_t time_us, + int* out_width, + int* out_height, + int* crop_width, + int* crop_height, + int* crop_x, + int* crop_y) { + { + rtc::CritScope lock(&stats_crit_); + stats_ = Stats{width, height}; + } + + if (!broadcaster_.frame_wanted()) { + return false; + } + + if (!video_adapter_.AdaptFrameResolution( + width, height, time_us * rtc::kNumNanosecsPerMicrosec, crop_width, + crop_height, out_width, out_height)) { + broadcaster_.OnDiscardedFrame(); + // VideoAdapter dropped the frame. + return false; + } + + *crop_x = (width - *crop_width) / 2; + *crop_y = (height - *crop_height) / 2; + return true; +} + +} // namespace rtc diff --git a/media/base/adapted_video_track_source.h b/media/base/adapted_video_track_source.h new file mode 100644 index 0000000000..7dbab540ed --- /dev/null +++ b/media/base/adapted_video_track_source.h @@ -0,0 +1,100 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MEDIA_BASE_ADAPTED_VIDEO_TRACK_SOURCE_H_ +#define MEDIA_BASE_ADAPTED_VIDEO_TRACK_SOURCE_H_ + +#include <stdint.h> + +#include "absl/types/optional.h" +#include "api/media_stream_interface.h" +#include "api/notifier.h" +#include "api/video/video_frame.h" +#include "api/video/video_sink_interface.h" +#include "api/video/video_source_interface.h" +#include "media/base/video_adapter.h" +#include "media/base/video_broadcaster.h" +#include "rtc_base/critical_section.h" +#include "rtc_base/system/rtc_export.h" +#include "rtc_base/thread_annotations.h" + +namespace rtc { + +// Base class for sources which needs video adaptation, e.g., video +// capture sources. Sinks must be added and removed on one and only +// one thread, while AdaptFrame and OnFrame may be called on any +// thread. +class RTC_EXPORT AdaptedVideoTrackSource + : public webrtc::Notifier<webrtc::VideoTrackSourceInterface> { + public: + AdaptedVideoTrackSource(); + ~AdaptedVideoTrackSource() override; + + protected: + // Allows derived classes to initialize |video_adapter_| with a custom + // alignment. + explicit AdaptedVideoTrackSource(int required_alignment); + // Checks the apply_rotation() flag. If the frame needs rotation, and it is a + // plain memory frame, it is rotated. Subclasses producing native frames must + // handle apply_rotation() themselves. + void OnFrame(const webrtc::VideoFrame& frame); + + // Reports the appropriate frame size after adaptation. Returns true + // if a frame is wanted. Returns false if there are no interested + // sinks, or if the VideoAdapter decides to drop the frame. + bool AdaptFrame(int width, + int height, + int64_t time_us, + int* out_width, + int* out_height, + int* crop_width, + int* crop_height, + int* crop_x, + int* crop_y); + + // Returns the current value of the apply_rotation flag, derived + // from the VideoSinkWants of registered sinks. The value is derived + // from sinks' wants, in AddOrUpdateSink and RemoveSink. Beware that + // when using this method from a different thread, the value may + // become stale before it is used. + bool apply_rotation(); + + cricket::VideoAdapter* video_adapter() { return &video_adapter_; } + + private: + // Implements rtc::VideoSourceInterface. + void AddOrUpdateSink(rtc::VideoSinkInterface<webrtc::VideoFrame>* sink, + const rtc::VideoSinkWants& wants) override; + void RemoveSink(rtc::VideoSinkInterface<webrtc::VideoFrame>* sink) override; + + // Part of VideoTrackSourceInterface. + bool GetStats(Stats* stats) override; + + void OnSinkWantsChanged(const rtc::VideoSinkWants& wants); + + // Encoded sinks not implemented for AdaptedVideoTrackSource. + bool SupportsEncodedOutput() const override { return false; } + void GenerateKeyFrame() override {} + void AddEncodedSink( + rtc::VideoSinkInterface<webrtc::RecordableEncodedFrame>* sink) override {} + void RemoveEncodedSink( + rtc::VideoSinkInterface<webrtc::RecordableEncodedFrame>* sink) override {} + + cricket::VideoAdapter video_adapter_; + + rtc::CriticalSection stats_crit_; + absl::optional<Stats> stats_ RTC_GUARDED_BY(stats_crit_); + + VideoBroadcaster broadcaster_; +}; + +} // namespace rtc + +#endif // MEDIA_BASE_ADAPTED_VIDEO_TRACK_SOURCE_H_ diff --git a/media/base/audio_source.h b/media/base/audio_source.h new file mode 100644 index 0000000000..8a8796800b --- /dev/null +++ b/media/base/audio_source.h @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MEDIA_BASE_AUDIO_SOURCE_H_ +#define MEDIA_BASE_AUDIO_SOURCE_H_ + +#include <cstddef> + +#include "absl/types/optional.h" + +namespace cricket { + +// Abstract interface for providing the audio data. +// TODO(deadbeef): Rename this to AudioSourceInterface, and rename +// webrtc::AudioSourceInterface to AudioTrackSourceInterface. +class AudioSource { + public: + class Sink { + public: + // Callback to receive data from the AudioSource. + virtual void OnData( + const void* audio_data, + int bits_per_sample, + int sample_rate, + size_t number_of_channels, + size_t number_of_frames, + absl::optional<int64_t> absolute_capture_timestamp_ms) = 0; + + // Called when the AudioSource is going away. + virtual void OnClose() = 0; + + protected: + virtual ~Sink() {} + }; + + // Sets a sink to the AudioSource. There can be only one sink connected + // to the source at a time. + virtual void SetSink(Sink* sink) = 0; + + protected: + virtual ~AudioSource() {} +}; + +} // namespace cricket + +#endif // MEDIA_BASE_AUDIO_SOURCE_H_ diff --git a/media/base/codec.cc b/media/base/codec.cc new file mode 100644 index 0000000000..6b9a052da3 --- /dev/null +++ b/media/base/codec.cc @@ -0,0 +1,477 @@ +/* + * Copyright (c) 2004 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "media/base/codec.h" + +#include "absl/algorithm/container.h" +#include "absl/strings/match.h" +#include "media/base/h264_profile_level_id.h" +#include "media/base/vp9_profile.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "rtc_base/string_encode.h" +#include "rtc_base/strings/string_builder.h" + +namespace cricket { +namespace { + +std::string GetH264PacketizationModeOrDefault(const CodecParameterMap& params) { + auto it = params.find(kH264FmtpPacketizationMode); + if (it != params.end()) { + return it->second; + } + // If packetization-mode is not present, default to "0". + // https://tools.ietf.org/html/rfc6184#section-6.2 + return "0"; +} + +bool IsSameH264PacketizationMode(const CodecParameterMap& left, + const CodecParameterMap& right) { + return GetH264PacketizationModeOrDefault(left) == + GetH264PacketizationModeOrDefault(right); +} + +// Some (video) codecs are actually families of codecs and rely on parameters +// to distinguish different incompatible family members. +bool IsSameCodecSpecific(const std::string& name1, + const CodecParameterMap& params1, + const std::string& name2, + const CodecParameterMap& params2) { + // The names might not necessarily match, so check both. + auto either_name_matches = [&](const std::string name) { + return absl::EqualsIgnoreCase(name, name1) || + absl::EqualsIgnoreCase(name, name2); + }; + if (either_name_matches(kH264CodecName)) + return webrtc::H264::IsSameH264Profile(params1, params2) && + IsSameH264PacketizationMode(params1, params2); + if (either_name_matches(kVp9CodecName)) + return webrtc::IsSameVP9Profile(params1, params2); + return true; +} + +bool IsCodecInList( + const webrtc::SdpVideoFormat& format, + const std::vector<webrtc::SdpVideoFormat>& existing_formats) { + for (auto existing_format : existing_formats) { + if (IsSameCodec(format.name, format.parameters, existing_format.name, + existing_format.parameters)) { + return true; + } + } + return false; +} + +} // namespace + +FeedbackParams::FeedbackParams() = default; +FeedbackParams::~FeedbackParams() = default; + +bool FeedbackParam::operator==(const FeedbackParam& other) const { + return absl::EqualsIgnoreCase(other.id(), id()) && + absl::EqualsIgnoreCase(other.param(), param()); +} + +bool FeedbackParams::operator==(const FeedbackParams& other) const { + return params_ == other.params_; +} + +bool FeedbackParams::Has(const FeedbackParam& param) const { + return absl::c_linear_search(params_, param); +} + +void FeedbackParams::Add(const FeedbackParam& param) { + if (param.id().empty()) { + return; + } + if (Has(param)) { + // Param already in |this|. + return; + } + params_.push_back(param); + RTC_CHECK(!HasDuplicateEntries()); +} + +void FeedbackParams::Intersect(const FeedbackParams& from) { + std::vector<FeedbackParam>::iterator iter_to = params_.begin(); + while (iter_to != params_.end()) { + if (!from.Has(*iter_to)) { + iter_to = params_.erase(iter_to); + } else { + ++iter_to; + } + } +} + +bool FeedbackParams::HasDuplicateEntries() const { + for (std::vector<FeedbackParam>::const_iterator iter = params_.begin(); + iter != params_.end(); ++iter) { + for (std::vector<FeedbackParam>::const_iterator found = iter + 1; + found != params_.end(); ++found) { + if (*found == *iter) { + return true; + } + } + } + return false; +} + +Codec::Codec(int id, const std::string& name, int clockrate) + : id(id), name(name), clockrate(clockrate) {} + +Codec::Codec() : id(0), clockrate(0) {} + +Codec::Codec(const Codec& c) = default; +Codec::Codec(Codec&& c) = default; +Codec::~Codec() = default; +Codec& Codec::operator=(const Codec& c) = default; +Codec& Codec::operator=(Codec&& c) = default; + +bool Codec::operator==(const Codec& c) const { + return this->id == c.id && // id is reserved in objective-c + name == c.name && clockrate == c.clockrate && params == c.params && + feedback_params == c.feedback_params; +} + +bool Codec::Matches(const Codec& codec) const { + // Match the codec id/name based on the typical static/dynamic name rules. + // Matching is case-insensitive. + const int kMaxStaticPayloadId = 95; + return (id <= kMaxStaticPayloadId || codec.id <= kMaxStaticPayloadId) + ? (id == codec.id) + : (absl::EqualsIgnoreCase(name, codec.name)); +} + +bool Codec::MatchesCapability( + const webrtc::RtpCodecCapability& codec_capability) const { + webrtc::RtpCodecParameters codec_parameters = ToCodecParameters(); + + return codec_parameters.name == codec_capability.name && + codec_parameters.kind == codec_capability.kind && + (codec_parameters.name == cricket::kRtxCodecName || + (codec_parameters.num_channels == codec_capability.num_channels && + codec_parameters.clock_rate == codec_capability.clock_rate && + codec_parameters.parameters == codec_capability.parameters)); +} + +bool Codec::GetParam(const std::string& name, std::string* out) const { + CodecParameterMap::const_iterator iter = params.find(name); + if (iter == params.end()) + return false; + *out = iter->second; + return true; +} + +bool Codec::GetParam(const std::string& name, int* out) const { + CodecParameterMap::const_iterator iter = params.find(name); + if (iter == params.end()) + return false; + return rtc::FromString(iter->second, out); +} + +void Codec::SetParam(const std::string& name, const std::string& value) { + params[name] = value; +} + +void Codec::SetParam(const std::string& name, int value) { + params[name] = rtc::ToString(value); +} + +bool Codec::RemoveParam(const std::string& name) { + return params.erase(name) == 1; +} + +void Codec::AddFeedbackParam(const FeedbackParam& param) { + feedback_params.Add(param); +} + +bool Codec::HasFeedbackParam(const FeedbackParam& param) const { + return feedback_params.Has(param); +} + +void Codec::IntersectFeedbackParams(const Codec& other) { + feedback_params.Intersect(other.feedback_params); +} + +webrtc::RtpCodecParameters Codec::ToCodecParameters() const { + webrtc::RtpCodecParameters codec_params; + codec_params.payload_type = id; + codec_params.name = name; + codec_params.clock_rate = clockrate; + codec_params.parameters.insert(params.begin(), params.end()); + return codec_params; +} + +AudioCodec::AudioCodec(int id, + const std::string& name, + int clockrate, + int bitrate, + size_t channels) + : Codec(id, name, clockrate), bitrate(bitrate), channels(channels) {} + +AudioCodec::AudioCodec() : Codec(), bitrate(0), channels(0) {} + +AudioCodec::AudioCodec(const AudioCodec& c) = default; +AudioCodec::AudioCodec(AudioCodec&& c) = default; +AudioCodec& AudioCodec::operator=(const AudioCodec& c) = default; +AudioCodec& AudioCodec::operator=(AudioCodec&& c) = default; + +bool AudioCodec::operator==(const AudioCodec& c) const { + return bitrate == c.bitrate && channels == c.channels && Codec::operator==(c); +} + +bool AudioCodec::Matches(const AudioCodec& codec) const { + // If a nonzero clockrate is specified, it must match the actual clockrate. + // If a nonzero bitrate is specified, it must match the actual bitrate, + // unless the codec is VBR (0), where we just force the supplied value. + // The number of channels must match exactly, with the exception + // that channels=0 is treated synonymously as channels=1, per RFC + // 4566 section 6: " [The channels] parameter is OPTIONAL and may be + // omitted if the number of channels is one." + // Preference is ignored. + // TODO(juberti): Treat a zero clockrate as 8000Hz, the RTP default clockrate. + return Codec::Matches(codec) && + ((codec.clockrate == 0 /*&& clockrate == 8000*/) || + clockrate == codec.clockrate) && + (codec.bitrate == 0 || bitrate <= 0 || bitrate == codec.bitrate) && + ((codec.channels < 2 && channels < 2) || channels == codec.channels); +} + +std::string AudioCodec::ToString() const { + char buf[256]; + rtc::SimpleStringBuilder sb(buf); + sb << "AudioCodec[" << id << ":" << name << ":" << clockrate << ":" << bitrate + << ":" << channels << "]"; + return sb.str(); +} + +webrtc::RtpCodecParameters AudioCodec::ToCodecParameters() const { + webrtc::RtpCodecParameters codec_params = Codec::ToCodecParameters(); + codec_params.num_channels = static_cast<int>(channels); + codec_params.kind = MEDIA_TYPE_AUDIO; + return codec_params; +} + +std::string VideoCodec::ToString() const { + char buf[256]; + rtc::SimpleStringBuilder sb(buf); + sb << "VideoCodec[" << id << ":" << name << "]"; + return sb.str(); +} + +webrtc::RtpCodecParameters VideoCodec::ToCodecParameters() const { + webrtc::RtpCodecParameters codec_params = Codec::ToCodecParameters(); + codec_params.kind = MEDIA_TYPE_VIDEO; + return codec_params; +} + +VideoCodec::VideoCodec(int id, const std::string& name) + : Codec(id, name, kVideoCodecClockrate) { + SetDefaultParameters(); +} + +VideoCodec::VideoCodec(const std::string& name) : VideoCodec(0 /* id */, name) { + SetDefaultParameters(); +} + +VideoCodec::VideoCodec() : Codec() { + clockrate = kVideoCodecClockrate; +} + +VideoCodec::VideoCodec(const webrtc::SdpVideoFormat& c) + : Codec(0 /* id */, c.name, kVideoCodecClockrate) { + params = c.parameters; +} + +VideoCodec::VideoCodec(const VideoCodec& c) = default; +VideoCodec::VideoCodec(VideoCodec&& c) = default; +VideoCodec& VideoCodec::operator=(const VideoCodec& c) = default; +VideoCodec& VideoCodec::operator=(VideoCodec&& c) = default; + +void VideoCodec::SetDefaultParameters() { + if (absl::EqualsIgnoreCase(kH264CodecName, name)) { + // This default is set for all H.264 codecs created because + // that was the default before packetization mode support was added. + // TODO(hta): Move this to the places that create VideoCodecs from + // SDP or from knowledge of implementation capabilities. + SetParam(kH264FmtpPacketizationMode, "1"); + } +} + +bool VideoCodec::operator==(const VideoCodec& c) const { + return Codec::operator==(c) && packetization == c.packetization; +} + +bool VideoCodec::Matches(const VideoCodec& other) const { + return Codec::Matches(other) && + IsSameCodecSpecific(name, params, other.name, other.params); +} + +absl::optional<std::string> VideoCodec::IntersectPacketization( + const VideoCodec& local_codec, + const VideoCodec& remote_codec) { + if (local_codec.packetization == remote_codec.packetization) { + return local_codec.packetization; + } + return absl::nullopt; +} + +VideoCodec VideoCodec::CreateRtxCodec(int rtx_payload_type, + int associated_payload_type) { + VideoCodec rtx_codec(rtx_payload_type, kRtxCodecName); + rtx_codec.SetParam(kCodecParamAssociatedPayloadType, associated_payload_type); + return rtx_codec; +} + +VideoCodec::CodecType VideoCodec::GetCodecType() const { + if (absl::EqualsIgnoreCase(name, kRedCodecName)) { + return CODEC_RED; + } + if (absl::EqualsIgnoreCase(name, kUlpfecCodecName)) { + return CODEC_ULPFEC; + } + if (absl::EqualsIgnoreCase(name, kFlexfecCodecName)) { + return CODEC_FLEXFEC; + } + if (absl::EqualsIgnoreCase(name, kRtxCodecName)) { + return CODEC_RTX; + } + + return CODEC_VIDEO; +} + +bool VideoCodec::ValidateCodecFormat() const { + if (id < 0 || id > 127) { + RTC_LOG(LS_ERROR) << "Codec with invalid payload type: " << ToString(); + return false; + } + if (GetCodecType() != CODEC_VIDEO) { + return true; + } + + // Video validation from here on. + int min_bitrate = -1; + int max_bitrate = -1; + if (GetParam(kCodecParamMinBitrate, &min_bitrate) && + GetParam(kCodecParamMaxBitrate, &max_bitrate)) { + if (max_bitrate < min_bitrate) { + RTC_LOG(LS_ERROR) << "Codec with max < min bitrate: " << ToString(); + return false; + } + } + return true; +} + +RtpDataCodec::RtpDataCodec(int id, const std::string& name) + : Codec(id, name, kDataCodecClockrate) {} + +RtpDataCodec::RtpDataCodec() : Codec() { + clockrate = kDataCodecClockrate; +} + +RtpDataCodec::RtpDataCodec(const RtpDataCodec& c) = default; +RtpDataCodec::RtpDataCodec(RtpDataCodec&& c) = default; +RtpDataCodec& RtpDataCodec::operator=(const RtpDataCodec& c) = default; +RtpDataCodec& RtpDataCodec::operator=(RtpDataCodec&& c) = default; + +std::string RtpDataCodec::ToString() const { + char buf[256]; + rtc::SimpleStringBuilder sb(buf); + sb << "RtpDataCodec[" << id << ":" << name << "]"; + return sb.str(); +} + +bool HasLntf(const Codec& codec) { + return codec.HasFeedbackParam( + FeedbackParam(kRtcpFbParamLntf, kParamValueEmpty)); +} + +bool HasNack(const Codec& codec) { + return codec.HasFeedbackParam( + FeedbackParam(kRtcpFbParamNack, kParamValueEmpty)); +} + +bool HasRemb(const Codec& codec) { + return codec.HasFeedbackParam( + FeedbackParam(kRtcpFbParamRemb, kParamValueEmpty)); +} + +bool HasRrtr(const Codec& codec) { + return codec.HasFeedbackParam( + FeedbackParam(kRtcpFbParamRrtr, kParamValueEmpty)); +} + +bool HasTransportCc(const Codec& codec) { + return codec.HasFeedbackParam( + FeedbackParam(kRtcpFbParamTransportCc, kParamValueEmpty)); +} + +const VideoCodec* FindMatchingCodec( + const std::vector<VideoCodec>& supported_codecs, + const VideoCodec& codec) { + for (const VideoCodec& supported_codec : supported_codecs) { + if (IsSameCodec(codec.name, codec.params, supported_codec.name, + supported_codec.params)) { + return &supported_codec; + } + } + return nullptr; +} + +bool IsSameCodec(const std::string& name1, + const CodecParameterMap& params1, + const std::string& name2, + const CodecParameterMap& params2) { + // Two codecs are considered the same if the name matches (case insensitive) + // and certain codec-specific parameters match. + return absl::EqualsIgnoreCase(name1, name2) && + IsSameCodecSpecific(name1, params1, name2, params2); +} + +// If a decoder supports any H264 profile, it is implicitly assumed to also +// support constrained base line even though it's not explicitly listed. +void AddH264ConstrainedBaselineProfileToSupportedFormats( + std::vector<webrtc::SdpVideoFormat>* supported_formats) { + std::vector<webrtc::SdpVideoFormat> cbr_supported_formats; + + // For any H264 supported profile, add the corresponding constrained baseline + // profile. + for (auto it = supported_formats->cbegin(); it != supported_formats->cend(); + ++it) { + if (it->name == cricket::kH264CodecName) { + const absl::optional<webrtc::H264::ProfileLevelId> profile_level_id = + webrtc::H264::ParseSdpProfileLevelId(it->parameters); + if (profile_level_id && profile_level_id->profile != + webrtc::H264::kProfileConstrainedBaseline) { + webrtc::SdpVideoFormat cbp_format = *it; + webrtc::H264::ProfileLevelId cbp_profile = *profile_level_id; + cbp_profile.profile = webrtc::H264::kProfileConstrainedBaseline; + cbp_format.parameters[cricket::kH264FmtpProfileLevelId] = + *webrtc::H264::ProfileLevelIdToString(cbp_profile); + cbr_supported_formats.push_back(cbp_format); + } + } + } + + size_t original_size = supported_formats->size(); + // ...if it's not already in the list. + std::copy_if(cbr_supported_formats.begin(), cbr_supported_formats.end(), + std::back_inserter(*supported_formats), + [supported_formats](const webrtc::SdpVideoFormat& format) { + return !IsCodecInList(format, *supported_formats); + }); + + if (supported_formats->size() > original_size) { + RTC_LOG(LS_WARNING) << "Explicitly added H264 constrained baseline to list " + "of supported formats."; + } +} + +} // namespace cricket diff --git a/media/base/codec.h b/media/base/codec.h new file mode 100644 index 0000000000..fd8a97c5e4 --- /dev/null +++ b/media/base/codec.h @@ -0,0 +1,251 @@ +/* + * Copyright (c) 2004 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MEDIA_BASE_CODEC_H_ +#define MEDIA_BASE_CODEC_H_ + +#include <map> +#include <set> +#include <string> +#include <vector> + +#include "absl/types/optional.h" +#include "api/rtp_parameters.h" +#include "api/video_codecs/sdp_video_format.h" +#include "media/base/media_constants.h" +#include "rtc_base/system/rtc_export.h" + +namespace cricket { + +typedef std::map<std::string, std::string> CodecParameterMap; + +class FeedbackParam { + public: + FeedbackParam() = default; + FeedbackParam(const std::string& id, const std::string& param) + : id_(id), param_(param) {} + explicit FeedbackParam(const std::string& id) + : id_(id), param_(kParamValueEmpty) {} + + bool operator==(const FeedbackParam& other) const; + + const std::string& id() const { return id_; } + const std::string& param() const { return param_; } + + private: + std::string id_; // e.g. "nack", "ccm" + std::string param_; // e.g. "", "rpsi", "fir" +}; + +class FeedbackParams { + public: + FeedbackParams(); + ~FeedbackParams(); + bool operator==(const FeedbackParams& other) const; + + bool Has(const FeedbackParam& param) const; + void Add(const FeedbackParam& param); + + void Intersect(const FeedbackParams& from); + + const std::vector<FeedbackParam>& params() const { return params_; } + + private: + bool HasDuplicateEntries() const; + + std::vector<FeedbackParam> params_; +}; + +struct RTC_EXPORT Codec { + int id; + std::string name; + int clockrate; + CodecParameterMap params; + FeedbackParams feedback_params; + + virtual ~Codec(); + + // Indicates if this codec is compatible with the specified codec. + bool Matches(const Codec& codec) const; + bool MatchesCapability(const webrtc::RtpCodecCapability& capability) const; + + // Find the parameter for |name| and write the value to |out|. + bool GetParam(const std::string& name, std::string* out) const; + bool GetParam(const std::string& name, int* out) const; + + void SetParam(const std::string& name, const std::string& value); + void SetParam(const std::string& name, int value); + + // It is safe to input a non-existent parameter. + // Returns true if the parameter existed, false if it did not exist. + bool RemoveParam(const std::string& name); + + bool HasFeedbackParam(const FeedbackParam& param) const; + void AddFeedbackParam(const FeedbackParam& param); + + // Filter |this| feedbacks params such that only those shared by both |this| + // and |other| are kept. + void IntersectFeedbackParams(const Codec& other); + + virtual webrtc::RtpCodecParameters ToCodecParameters() const; + + Codec& operator=(const Codec& c); + Codec& operator=(Codec&& c); + + bool operator==(const Codec& c) const; + + bool operator!=(const Codec& c) const { return !(*this == c); } + + protected: + // A Codec can't be created without a subclass. + // Creates a codec with the given parameters. + Codec(int id, const std::string& name, int clockrate); + // Creates an empty codec. + Codec(); + Codec(const Codec& c); + Codec(Codec&& c); +}; + +struct AudioCodec : public Codec { + int bitrate; + size_t channels; + + // Creates a codec with the given parameters. + AudioCodec(int id, + const std::string& name, + int clockrate, + int bitrate, + size_t channels); + // Creates an empty codec. + AudioCodec(); + AudioCodec(const AudioCodec& c); + AudioCodec(AudioCodec&& c); + ~AudioCodec() override = default; + + // Indicates if this codec is compatible with the specified codec. + bool Matches(const AudioCodec& codec) const; + + std::string ToString() const; + + webrtc::RtpCodecParameters ToCodecParameters() const override; + + AudioCodec& operator=(const AudioCodec& c); + AudioCodec& operator=(AudioCodec&& c); + + bool operator==(const AudioCodec& c) const; + + bool operator!=(const AudioCodec& c) const { return !(*this == c); } +}; + +struct RTC_EXPORT VideoCodec : public Codec { + absl::optional<std::string> packetization; + + // Creates a codec with the given parameters. + VideoCodec(int id, const std::string& name); + // Creates a codec with the given name and empty id. + explicit VideoCodec(const std::string& name); + // Creates an empty codec. + VideoCodec(); + VideoCodec(const VideoCodec& c); + explicit VideoCodec(const webrtc::SdpVideoFormat& c); + VideoCodec(VideoCodec&& c); + ~VideoCodec() override = default; + + // Indicates if this video codec is the same as the other video codec, e.g. if + // they are both VP8 or VP9, or if they are both H264 with the same H264 + // profile. H264 levels however are not compared. + bool Matches(const VideoCodec& codec) const; + + std::string ToString() const; + + webrtc::RtpCodecParameters ToCodecParameters() const override; + + VideoCodec& operator=(const VideoCodec& c); + VideoCodec& operator=(VideoCodec&& c); + + bool operator==(const VideoCodec& c) const; + + bool operator!=(const VideoCodec& c) const { return !(*this == c); } + + // Return packetization which both |local_codec| and |remote_codec| support. + static absl::optional<std::string> IntersectPacketization( + const VideoCodec& local_codec, + const VideoCodec& remote_codec); + + static VideoCodec CreateRtxCodec(int rtx_payload_type, + int associated_payload_type); + + enum CodecType { + CODEC_VIDEO, + CODEC_RED, + CODEC_ULPFEC, + CODEC_FLEXFEC, + CODEC_RTX, + }; + + CodecType GetCodecType() const; + // Validates a VideoCodec's payload type, dimensions and bitrates etc. If they + // don't make sense (such as max < min bitrate), and error is logged and + // ValidateCodecFormat returns false. + bool ValidateCodecFormat() const; + + private: + void SetDefaultParameters(); +}; + +struct RtpDataCodec : public Codec { + RtpDataCodec(int id, const std::string& name); + RtpDataCodec(); + RtpDataCodec(const RtpDataCodec& c); + RtpDataCodec(RtpDataCodec&& c); + ~RtpDataCodec() override = default; + + RtpDataCodec& operator=(const RtpDataCodec& c); + RtpDataCodec& operator=(RtpDataCodec&& c); + + std::string ToString() const; +}; + +// For backwards compatibility +// TODO(bugs.webrtc.org/10597): Remove when no longer needed. +typedef RtpDataCodec DataCodec; + +// Get the codec setting associated with |payload_type|. If there +// is no codec associated with that payload type it returns nullptr. +template <class Codec> +const Codec* FindCodecById(const std::vector<Codec>& codecs, int payload_type) { + for (const auto& codec : codecs) { + if (codec.id == payload_type) + return &codec; + } + return nullptr; +} + +bool HasLntf(const Codec& codec); +bool HasNack(const Codec& codec); +bool HasRemb(const Codec& codec); +bool HasRrtr(const Codec& codec); +bool HasTransportCc(const Codec& codec); +// Returns the first codec in |supported_codecs| that matches |codec|, or +// nullptr if no codec matches. +const VideoCodec* FindMatchingCodec( + const std::vector<VideoCodec>& supported_codecs, + const VideoCodec& codec); +RTC_EXPORT bool IsSameCodec(const std::string& name1, + const CodecParameterMap& params1, + const std::string& name2, + const CodecParameterMap& params2); + +RTC_EXPORT void AddH264ConstrainedBaselineProfileToSupportedFormats( + std::vector<webrtc::SdpVideoFormat>* supported_formats); + +} // namespace cricket + +#endif // MEDIA_BASE_CODEC_H_ diff --git a/media/base/codec_unittest.cc b/media/base/codec_unittest.cc new file mode 100644 index 0000000000..04130e1642 --- /dev/null +++ b/media/base/codec_unittest.cc @@ -0,0 +1,596 @@ +/* + * Copyright (c) 2009 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "media/base/codec.h" + +#include <tuple> + +#include "common_types.h" // NOLINT(build/include) +#include "media/base/h264_profile_level_id.h" +#include "media/base/vp9_profile.h" +#include "modules/video_coding/codecs/h264/include/h264.h" +#include "rtc_base/gunit.h" + +using cricket::AudioCodec; +using cricket::Codec; +using cricket::DataCodec; +using cricket::FeedbackParam; +using cricket::kCodecParamAssociatedPayloadType; +using cricket::kCodecParamMaxBitrate; +using cricket::kCodecParamMinBitrate; +using cricket::VideoCodec; + +class TestCodec : public Codec { + public: + TestCodec(int id, const std::string& name, int clockrate) + : Codec(id, name, clockrate) {} + TestCodec() : Codec() {} + TestCodec(const TestCodec& c) : Codec(c) {} +}; + +TEST(CodecTest, TestCodecOperators) { + TestCodec c0(96, "D", 1000); + c0.SetParam("a", 1); + + TestCodec c1 = c0; + EXPECT_TRUE(c1 == c0); + + int param_value0; + int param_value1; + EXPECT_TRUE(c0.GetParam("a", ¶m_value0)); + EXPECT_TRUE(c1.GetParam("a", ¶m_value1)); + EXPECT_EQ(param_value0, param_value1); + + c1.id = 86; + EXPECT_TRUE(c0 != c1); + + c1 = c0; + c1.name = "x"; + EXPECT_TRUE(c0 != c1); + + c1 = c0; + c1.clockrate = 2000; + EXPECT_TRUE(c0 != c1); + + c1 = c0; + c1.SetParam("a", 2); + EXPECT_TRUE(c0 != c1); + + TestCodec c5; + TestCodec c6(0, "", 0); + EXPECT_TRUE(c5 == c6); +} + +TEST(CodecTest, TestAudioCodecOperators) { + AudioCodec c0(96, "A", 44100, 20000, 2); + AudioCodec c1(95, "A", 44100, 20000, 2); + AudioCodec c2(96, "x", 44100, 20000, 2); + AudioCodec c3(96, "A", 48000, 20000, 2); + AudioCodec c4(96, "A", 44100, 10000, 2); + AudioCodec c5(96, "A", 44100, 20000, 1); + EXPECT_NE(c0, c1); + EXPECT_NE(c0, c2); + EXPECT_NE(c0, c3); + EXPECT_NE(c0, c4); + EXPECT_NE(c0, c5); + + AudioCodec c7; + AudioCodec c8(0, "", 0, 0, 0); + AudioCodec c9 = c0; + EXPECT_EQ(c8, c7); + EXPECT_NE(c9, c7); + EXPECT_EQ(c9, c0); + + AudioCodec c10(c0); + AudioCodec c11(c0); + AudioCodec c12(c0); + AudioCodec c13(c0); + c10.params["x"] = "abc"; + c11.params["x"] = "def"; + c12.params["y"] = "abc"; + c13.params["x"] = "abc"; + EXPECT_NE(c10, c0); + EXPECT_NE(c11, c0); + EXPECT_NE(c11, c10); + EXPECT_NE(c12, c0); + EXPECT_NE(c12, c10); + EXPECT_NE(c12, c11); + EXPECT_EQ(c13, c10); +} + +TEST(CodecTest, TestAudioCodecMatches) { + // Test a codec with a static payload type. + AudioCodec c0(95, "A", 44100, 20000, 1); + EXPECT_TRUE(c0.Matches(AudioCodec(95, "", 44100, 20000, 1))); + EXPECT_TRUE(c0.Matches(AudioCodec(95, "", 44100, 20000, 0))); + EXPECT_TRUE(c0.Matches(AudioCodec(95, "", 44100, 0, 0))); + EXPECT_TRUE(c0.Matches(AudioCodec(95, "", 0, 0, 0))); + EXPECT_FALSE(c0.Matches(AudioCodec(96, "", 44100, 20000, 1))); + EXPECT_FALSE(c0.Matches(AudioCodec(95, "", 55100, 20000, 1))); + EXPECT_FALSE(c0.Matches(AudioCodec(95, "", 44100, 30000, 1))); + EXPECT_FALSE(c0.Matches(AudioCodec(95, "", 44100, 20000, 2))); + EXPECT_FALSE(c0.Matches(AudioCodec(95, "", 55100, 30000, 2))); + + // Test a codec with a dynamic payload type. + AudioCodec c1(96, "A", 44100, 20000, 1); + EXPECT_TRUE(c1.Matches(AudioCodec(96, "A", 0, 0, 0))); + EXPECT_TRUE(c1.Matches(AudioCodec(97, "A", 0, 0, 0))); + EXPECT_TRUE(c1.Matches(AudioCodec(96, "a", 0, 0, 0))); + EXPECT_TRUE(c1.Matches(AudioCodec(97, "a", 0, 0, 0))); + EXPECT_FALSE(c1.Matches(AudioCodec(95, "A", 0, 0, 0))); + EXPECT_FALSE(c1.Matches(AudioCodec(96, "", 44100, 20000, 2))); + EXPECT_FALSE(c1.Matches(AudioCodec(96, "A", 55100, 30000, 1))); + + // Test a codec with a dynamic payload type, and auto bitrate. + AudioCodec c2(97, "A", 16000, 0, 1); + // Use default bitrate. + EXPECT_TRUE(c2.Matches(AudioCodec(97, "A", 16000, 0, 1))); + EXPECT_TRUE(c2.Matches(AudioCodec(97, "A", 16000, 0, 0))); + // Use explicit bitrate. + EXPECT_TRUE(c2.Matches(AudioCodec(97, "A", 16000, 32000, 1))); + // Backward compatibility with clients that might send "-1" (for default). + EXPECT_TRUE(c2.Matches(AudioCodec(97, "A", 16000, -1, 1))); + + // Stereo doesn't match channels = 0. + AudioCodec c3(96, "A", 44100, 20000, 2); + EXPECT_TRUE(c3.Matches(AudioCodec(96, "A", 44100, 20000, 2))); + EXPECT_FALSE(c3.Matches(AudioCodec(96, "A", 44100, 20000, 1))); + EXPECT_FALSE(c3.Matches(AudioCodec(96, "A", 44100, 20000, 0))); +} + +TEST(CodecTest, TestVideoCodecOperators) { + VideoCodec c0(96, "V"); + VideoCodec c1(95, "V"); + VideoCodec c2(96, "x"); + + EXPECT_TRUE(c0 != c1); + EXPECT_TRUE(c0 != c2); + + VideoCodec c7; + VideoCodec c8(0, ""); + VideoCodec c9 = c0; + EXPECT_TRUE(c8 == c7); + EXPECT_TRUE(c9 != c7); + EXPECT_TRUE(c9 == c0); + + VideoCodec c10(c0); + VideoCodec c11(c0); + VideoCodec c12(c0); + VideoCodec c13(c0); + c10.params["x"] = "abc"; + c11.params["x"] = "def"; + c12.params["y"] = "abc"; + c13.params["x"] = "abc"; + EXPECT_TRUE(c10 != c0); + EXPECT_TRUE(c11 != c0); + EXPECT_TRUE(c11 != c10); + EXPECT_TRUE(c12 != c0); + EXPECT_TRUE(c12 != c10); + EXPECT_TRUE(c12 != c11); + EXPECT_TRUE(c13 == c10); +} + +TEST(CodecTest, TestVideoCodecIntersectPacketization) { + VideoCodec c1; + c1.packetization = "raw"; + VideoCodec c2; + c2.packetization = "raw"; + VideoCodec c3; + + EXPECT_EQ(VideoCodec::IntersectPacketization(c1, c2), "raw"); + EXPECT_EQ(VideoCodec::IntersectPacketization(c1, c3), absl::nullopt); +} + +TEST(CodecTest, TestVideoCodecEqualsWithDifferentPacketization) { + VideoCodec c0(100, cricket::kVp8CodecName); + VideoCodec c1(100, cricket::kVp8CodecName); + VideoCodec c2(100, cricket::kVp8CodecName); + c2.packetization = "raw"; + + EXPECT_EQ(c0, c1); + EXPECT_NE(c0, c2); + EXPECT_NE(c2, c0); + EXPECT_EQ(c2, c2); +} + +TEST(CodecTest, TestVideoCodecMatches) { + // Test a codec with a static payload type. + VideoCodec c0(95, "V"); + EXPECT_TRUE(c0.Matches(VideoCodec(95, ""))); + EXPECT_FALSE(c0.Matches(VideoCodec(96, ""))); + + // Test a codec with a dynamic payload type. + VideoCodec c1(96, "V"); + EXPECT_TRUE(c1.Matches(VideoCodec(96, "V"))); + EXPECT_TRUE(c1.Matches(VideoCodec(97, "V"))); + EXPECT_TRUE(c1.Matches(VideoCodec(96, "v"))); + EXPECT_TRUE(c1.Matches(VideoCodec(97, "v"))); + EXPECT_FALSE(c1.Matches(VideoCodec(96, ""))); + EXPECT_FALSE(c1.Matches(VideoCodec(95, "V"))); +} + +TEST(CodecTest, TestVideoCodecMatchesWithDifferentPacketization) { + VideoCodec c0(100, cricket::kVp8CodecName); + VideoCodec c1(101, cricket::kVp8CodecName); + c1.packetization = "raw"; + + EXPECT_TRUE(c0.Matches(c1)); + EXPECT_TRUE(c1.Matches(c0)); +} + +// VP9 codecs compare profile information. +TEST(CodecTest, TestVP9CodecMatches) { + const char kProfile0[] = "0"; + const char kProfile2[] = "2"; + + VideoCodec c_no_profile(95, cricket::kVp9CodecName); + VideoCodec c_profile0(95, cricket::kVp9CodecName); + c_profile0.params[webrtc::kVP9FmtpProfileId] = kProfile0; + + EXPECT_TRUE(c_profile0.Matches(c_no_profile)); + + { + VideoCodec c_profile0_eq(95, cricket::kVp9CodecName); + c_profile0_eq.params[webrtc::kVP9FmtpProfileId] = kProfile0; + EXPECT_TRUE(c_profile0.Matches(c_profile0_eq)); + } + + { + VideoCodec c_profile2(95, cricket::kVp9CodecName); + c_profile2.params[webrtc::kVP9FmtpProfileId] = kProfile2; + EXPECT_FALSE(c_profile0.Matches(c_profile2)); + EXPECT_FALSE(c_no_profile.Matches(c_profile2)); + } + + { + VideoCodec c_no_profile_eq(95, cricket::kVp9CodecName); + EXPECT_TRUE(c_no_profile.Matches(c_no_profile_eq)); + } +} + +// Matching H264 codecs also need to have matching profile-level-id and +// packetization-mode. +TEST(CodecTest, TestH264CodecMatches) { + const char kProfileLevelId1[] = "42e01f"; + const char kProfileLevelId2[] = "42a01e"; + + VideoCodec pli_1_pm_0(95, "H264"); + pli_1_pm_0.params[cricket::kH264FmtpProfileLevelId] = kProfileLevelId1; + pli_1_pm_0.params[cricket::kH264FmtpPacketizationMode] = "0"; + + { + VideoCodec pli_1_pm_blank(95, "H264"); + pli_1_pm_blank.params[cricket::kH264FmtpProfileLevelId] = kProfileLevelId1; + pli_1_pm_blank.params.erase( + pli_1_pm_blank.params.find(cricket::kH264FmtpPacketizationMode)); + + // Matches since if packetization-mode is not specified it defaults to "0". + EXPECT_TRUE(pli_1_pm_0.Matches(pli_1_pm_blank)); + } + + { + VideoCodec pli_1_pm_1(95, "H264"); + pli_1_pm_1.params[cricket::kH264FmtpProfileLevelId] = kProfileLevelId1; + pli_1_pm_1.params[cricket::kH264FmtpPacketizationMode] = "1"; + + // Does not match since packetization-mode is different. + EXPECT_FALSE(pli_1_pm_0.Matches(pli_1_pm_1)); + } + + { + VideoCodec pli_2_pm_0(95, "H264"); + pli_2_pm_0.params[cricket::kH264FmtpProfileLevelId] = kProfileLevelId2; + pli_2_pm_0.params[cricket::kH264FmtpPacketizationMode] = "0"; + + // Does not match since profile-level-id is different. + EXPECT_FALSE(pli_1_pm_0.Matches(pli_2_pm_0)); + } +} + +TEST(CodecTest, TestDataCodecMatches) { + // Test a codec with a static payload type. + DataCodec c0(95, "D"); + EXPECT_TRUE(c0.Matches(DataCodec(95, ""))); + EXPECT_FALSE(c0.Matches(DataCodec(96, ""))); + + // Test a codec with a dynamic payload type. + DataCodec c1(96, "D"); + EXPECT_TRUE(c1.Matches(DataCodec(96, "D"))); + EXPECT_TRUE(c1.Matches(DataCodec(97, "D"))); + EXPECT_TRUE(c1.Matches(DataCodec(96, "d"))); + EXPECT_TRUE(c1.Matches(DataCodec(97, "d"))); + EXPECT_FALSE(c1.Matches(DataCodec(96, ""))); + EXPECT_FALSE(c1.Matches(DataCodec(95, "D"))); +} + +TEST(CodecTest, TestSetParamGetParamAndRemoveParam) { + AudioCodec codec; + codec.SetParam("a", "1"); + codec.SetParam("b", "x"); + + int int_value = 0; + EXPECT_TRUE(codec.GetParam("a", &int_value)); + EXPECT_EQ(1, int_value); + EXPECT_FALSE(codec.GetParam("b", &int_value)); + EXPECT_FALSE(codec.GetParam("c", &int_value)); + + std::string str_value; + EXPECT_TRUE(codec.GetParam("a", &str_value)); + EXPECT_EQ("1", str_value); + EXPECT_TRUE(codec.GetParam("b", &str_value)); + EXPECT_EQ("x", str_value); + EXPECT_FALSE(codec.GetParam("c", &str_value)); + EXPECT_TRUE(codec.RemoveParam("a")); + EXPECT_FALSE(codec.RemoveParam("c")); +} + +TEST(CodecTest, TestIntersectFeedbackParams) { + const FeedbackParam a1("a", "1"); + const FeedbackParam b2("b", "2"); + const FeedbackParam b3("b", "3"); + const FeedbackParam c3("c", "3"); + TestCodec c1; + c1.AddFeedbackParam(a1); // Only match with c2. + c1.AddFeedbackParam(b2); // Same param different values. + c1.AddFeedbackParam(c3); // Not in c2. + TestCodec c2; + c2.AddFeedbackParam(a1); + c2.AddFeedbackParam(b3); + + c1.IntersectFeedbackParams(c2); + EXPECT_TRUE(c1.HasFeedbackParam(a1)); + EXPECT_FALSE(c1.HasFeedbackParam(b2)); + EXPECT_FALSE(c1.HasFeedbackParam(c3)); +} + +TEST(CodecTest, TestGetCodecType) { + // Codec type comparison should be case insenstive on names. + const VideoCodec codec(96, "V"); + const VideoCodec rtx_codec(96, "rTx"); + const VideoCodec ulpfec_codec(96, "ulpFeC"); + const VideoCodec flexfec_codec(96, "FlExFeC-03"); + const VideoCodec red_codec(96, "ReD"); + EXPECT_EQ(VideoCodec::CODEC_VIDEO, codec.GetCodecType()); + EXPECT_EQ(VideoCodec::CODEC_RTX, rtx_codec.GetCodecType()); + EXPECT_EQ(VideoCodec::CODEC_ULPFEC, ulpfec_codec.GetCodecType()); + EXPECT_EQ(VideoCodec::CODEC_FLEXFEC, flexfec_codec.GetCodecType()); + EXPECT_EQ(VideoCodec::CODEC_RED, red_codec.GetCodecType()); +} + +TEST(CodecTest, TestCreateRtxCodec) { + VideoCodec rtx_codec = VideoCodec::CreateRtxCodec(96, 120); + EXPECT_EQ(96, rtx_codec.id); + EXPECT_EQ(VideoCodec::CODEC_RTX, rtx_codec.GetCodecType()); + int associated_payload_type; + ASSERT_TRUE(rtx_codec.GetParam(kCodecParamAssociatedPayloadType, + &associated_payload_type)); + EXPECT_EQ(120, associated_payload_type); +} + +TEST(CodecTest, TestValidateCodecFormat) { + const VideoCodec codec(96, "V"); + ASSERT_TRUE(codec.ValidateCodecFormat()); + + // Accept 0-127 as payload types. + VideoCodec low_payload_type = codec; + low_payload_type.id = 0; + VideoCodec high_payload_type = codec; + high_payload_type.id = 127; + ASSERT_TRUE(low_payload_type.ValidateCodecFormat()); + EXPECT_TRUE(high_payload_type.ValidateCodecFormat()); + + // Reject negative payloads. + VideoCodec negative_payload_type = codec; + negative_payload_type.id = -1; + EXPECT_FALSE(negative_payload_type.ValidateCodecFormat()); + + // Reject too-high payloads. + VideoCodec too_high_payload_type = codec; + too_high_payload_type.id = 128; + EXPECT_FALSE(too_high_payload_type.ValidateCodecFormat()); + + // Reject codecs with min bitrate > max bitrate. + VideoCodec incorrect_bitrates = codec; + incorrect_bitrates.params[kCodecParamMinBitrate] = "100"; + incorrect_bitrates.params[kCodecParamMaxBitrate] = "80"; + EXPECT_FALSE(incorrect_bitrates.ValidateCodecFormat()); + + // Accept min bitrate == max bitrate. + VideoCodec equal_bitrates = codec; + equal_bitrates.params[kCodecParamMinBitrate] = "100"; + equal_bitrates.params[kCodecParamMaxBitrate] = "100"; + EXPECT_TRUE(equal_bitrates.ValidateCodecFormat()); + + // Accept min bitrate < max bitrate. + VideoCodec different_bitrates = codec; + different_bitrates.params[kCodecParamMinBitrate] = "99"; + different_bitrates.params[kCodecParamMaxBitrate] = "100"; + EXPECT_TRUE(different_bitrates.ValidateCodecFormat()); +} + +TEST(CodecTest, TestToCodecParameters) { + VideoCodec v(96, "V"); + v.SetParam("p1", "v1"); + webrtc::RtpCodecParameters codec_params_1 = v.ToCodecParameters(); + EXPECT_EQ(96, codec_params_1.payload_type); + EXPECT_EQ(cricket::MEDIA_TYPE_VIDEO, codec_params_1.kind); + EXPECT_EQ("V", codec_params_1.name); + EXPECT_EQ(cricket::kVideoCodecClockrate, codec_params_1.clock_rate); + EXPECT_EQ(absl::nullopt, codec_params_1.num_channels); + ASSERT_EQ(1u, codec_params_1.parameters.size()); + EXPECT_EQ("p1", codec_params_1.parameters.begin()->first); + EXPECT_EQ("v1", codec_params_1.parameters.begin()->second); + + AudioCodec a(97, "A", 44100, 20000, 2); + a.SetParam("p1", "a1"); + webrtc::RtpCodecParameters codec_params_2 = a.ToCodecParameters(); + EXPECT_EQ(97, codec_params_2.payload_type); + EXPECT_EQ(cricket::MEDIA_TYPE_AUDIO, codec_params_2.kind); + EXPECT_EQ("A", codec_params_2.name); + EXPECT_EQ(44100, codec_params_2.clock_rate); + EXPECT_EQ(2, codec_params_2.num_channels); + ASSERT_EQ(1u, codec_params_2.parameters.size()); + EXPECT_EQ("p1", codec_params_2.parameters.begin()->first); + EXPECT_EQ("a1", codec_params_2.parameters.begin()->second); +} + +TEST(CodecTest, H264CostrainedBaselineIsAddedIfH264IsSupported) { + const std::vector<webrtc::SdpVideoFormat> kExplicitlySupportedFormats = { + webrtc::CreateH264Format(webrtc::H264::kProfileBaseline, + webrtc::H264::kLevel3_1, "1"), + webrtc::CreateH264Format(webrtc::H264::kProfileBaseline, + webrtc::H264::kLevel3_1, "0")}; + + std::vector<webrtc::SdpVideoFormat> supported_formats = + kExplicitlySupportedFormats; + cricket::AddH264ConstrainedBaselineProfileToSupportedFormats( + &supported_formats); + + const webrtc::SdpVideoFormat kH264ConstrainedBasedlinePacketization1 = + webrtc::CreateH264Format(webrtc::H264::kProfileConstrainedBaseline, + webrtc::H264::kLevel3_1, "1"); + const webrtc::SdpVideoFormat kH264ConstrainedBasedlinePacketization0 = + webrtc::CreateH264Format(webrtc::H264::kProfileConstrainedBaseline, + webrtc::H264::kLevel3_1, "0"); + + EXPECT_EQ(supported_formats[0], kExplicitlySupportedFormats[0]); + EXPECT_EQ(supported_formats[1], kExplicitlySupportedFormats[1]); + EXPECT_EQ(supported_formats[2], kH264ConstrainedBasedlinePacketization1); + EXPECT_EQ(supported_formats[3], kH264ConstrainedBasedlinePacketization0); +} + +TEST(CodecTest, H264CostrainedBaselineIsNotAddedIfH264IsUnsupported) { + const std::vector<webrtc::SdpVideoFormat> kExplicitlySupportedFormats = { + {cricket::kVp9CodecName, + {{webrtc::kVP9FmtpProfileId, + VP9ProfileToString(webrtc::VP9Profile::kProfile0)}}}}; + + std::vector<webrtc::SdpVideoFormat> supported_formats = + kExplicitlySupportedFormats; + cricket::AddH264ConstrainedBaselineProfileToSupportedFormats( + &supported_formats); + + EXPECT_EQ(supported_formats[0], kExplicitlySupportedFormats[0]); + EXPECT_EQ(supported_formats.size(), kExplicitlySupportedFormats.size()); +} + +TEST(CodecTest, H264CostrainedBaselineNotAddedIfAlreadySpecified) { + const std::vector<webrtc::SdpVideoFormat> kExplicitlySupportedFormats = { + webrtc::CreateH264Format(webrtc::H264::kProfileBaseline, + webrtc::H264::kLevel3_1, "1"), + webrtc::CreateH264Format(webrtc::H264::kProfileBaseline, + webrtc::H264::kLevel3_1, "0"), + webrtc::CreateH264Format(webrtc::H264::kProfileConstrainedBaseline, + webrtc::H264::kLevel3_1, "1"), + webrtc::CreateH264Format(webrtc::H264::kProfileConstrainedBaseline, + webrtc::H264::kLevel3_1, "0")}; + + std::vector<webrtc::SdpVideoFormat> supported_formats = + kExplicitlySupportedFormats; + cricket::AddH264ConstrainedBaselineProfileToSupportedFormats( + &supported_formats); + + EXPECT_EQ(supported_formats[0], kExplicitlySupportedFormats[0]); + EXPECT_EQ(supported_formats[1], kExplicitlySupportedFormats[1]); + EXPECT_EQ(supported_formats[2], kExplicitlySupportedFormats[2]); + EXPECT_EQ(supported_formats[3], kExplicitlySupportedFormats[3]); + EXPECT_EQ(supported_formats.size(), kExplicitlySupportedFormats.size()); +} + +// Tests that the helper IsSameCodec returns the correct value for codecs that +// must also be matched on particular parameter values. +using IsSameCodecParamsTestCase = + std::tuple<cricket::CodecParameterMap, cricket::CodecParameterMap>; +class IsSameCodecParamsTest + : public ::testing::TestWithParam< + std::tuple<std::string, bool, IsSameCodecParamsTestCase>> { + protected: + IsSameCodecParamsTest() { + name_ = std::get<0>(GetParam()); + expected_ = std::get<1>(GetParam()); + const auto& test_case = std::get<2>(GetParam()); + params_left_ = std::get<0>(test_case); + params_right_ = std::get<1>(test_case); + } + + std::string name_; + bool expected_; + cricket::CodecParameterMap params_left_; + cricket::CodecParameterMap params_right_; +}; + +TEST_P(IsSameCodecParamsTest, Expected) { + EXPECT_EQ(expected_, + cricket::IsSameCodec(name_, params_left_, name_, params_right_)); +} + +TEST_P(IsSameCodecParamsTest, Commutative) { + EXPECT_EQ(expected_, + cricket::IsSameCodec(name_, params_right_, name_, params_left_)); +} + +IsSameCodecParamsTestCase MakeTestCase(cricket::CodecParameterMap left, + cricket::CodecParameterMap right) { + return std::make_tuple(left, right); +} + +const IsSameCodecParamsTestCase kH264ParamsSameTestCases[] = { + // Both have the same defaults. + MakeTestCase({}, {}), + // packetization-mode: 0 is the default. + MakeTestCase({{cricket::kH264FmtpPacketizationMode, "0"}}, {}), + // Non-default packetization-mode matches. + MakeTestCase({{cricket::kH264FmtpPacketizationMode, "1"}}, + {{cricket::kH264FmtpPacketizationMode, "1"}}), +}; +INSTANTIATE_TEST_SUITE_P( + H264_Same, + IsSameCodecParamsTest, + ::testing::Combine(::testing::Values("H264"), + ::testing::Values(true), + ::testing::ValuesIn(kH264ParamsSameTestCases))); + +const IsSameCodecParamsTestCase kH264ParamsNotSameTestCases[] = { + // packetization-mode does not match the default of "0". + MakeTestCase({{cricket::kH264FmtpPacketizationMode, "1"}}, {}), +}; +INSTANTIATE_TEST_SUITE_P( + H264_NotSame, + IsSameCodecParamsTest, + ::testing::Combine(::testing::Values("H264"), + ::testing::Values(false), + ::testing::ValuesIn(kH264ParamsNotSameTestCases))); + +const IsSameCodecParamsTestCase kVP9ParamsSameTestCases[] = { + // Both have the same defaults. + MakeTestCase({}, {}), + // profile-id: 0 is the default. + MakeTestCase({{webrtc::kVP9FmtpProfileId, "0"}}, {}), + // Non-default profile-id matches. + MakeTestCase({{webrtc::kVP9FmtpProfileId, "2"}}, + {{webrtc::kVP9FmtpProfileId, "2"}}), +}; +INSTANTIATE_TEST_SUITE_P( + VP9_Same, + IsSameCodecParamsTest, + ::testing::Combine(::testing::Values("VP9"), + ::testing::Values(true), + ::testing::ValuesIn(kVP9ParamsSameTestCases))); + +const IsSameCodecParamsTestCase kVP9ParamsNotSameTestCases[] = { + // profile-id missing from right. + MakeTestCase({{webrtc::kVP9FmtpProfileId, "2"}}, {}), +}; +INSTANTIATE_TEST_SUITE_P( + VP9_NotSame, + IsSameCodecParamsTest, + ::testing::Combine(::testing::Values("VP9"), + ::testing::Values(false), + ::testing::ValuesIn(kVP9ParamsNotSameTestCases))); diff --git a/media/base/delayable.h b/media/base/delayable.h new file mode 100644 index 0000000000..90ce5d7089 --- /dev/null +++ b/media/base/delayable.h @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MEDIA_BASE_DELAYABLE_H_ +#define MEDIA_BASE_DELAYABLE_H_ + +#include <stdint.h> + +#include "absl/types/optional.h" + +namespace cricket { + +// Delayable is used by user code through ApplyConstraints algorithm. Its +// methods must take precendence over similar functional in |syncable.h|. +class Delayable { + public: + virtual ~Delayable() {} + // Set base minimum delay of the receive stream with specified ssrc. + // Base minimum delay sets lower bound on minimum delay value which + // determines minimum delay until audio playout. + // Returns false if there is no stream with given ssrc. + virtual bool SetBaseMinimumPlayoutDelayMs(uint32_t ssrc, int delay_ms) = 0; + + // Returns current value of base minimum delay in milliseconds. + virtual absl::optional<int> GetBaseMinimumPlayoutDelayMs( + uint32_t ssrc) const = 0; +}; + +} // namespace cricket + +#endif // MEDIA_BASE_DELAYABLE_H_ diff --git a/media/base/fake_frame_source.cc b/media/base/fake_frame_source.cc new file mode 100644 index 0000000000..8a05536c83 --- /dev/null +++ b/media/base/fake_frame_source.cc @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "media/base/fake_frame_source.h" + +#include "api/scoped_refptr.h" +#include "api/video/i420_buffer.h" +#include "api/video/video_frame_buffer.h" +#include "rtc_base/checks.h" +#include "rtc_base/time_utils.h" + +namespace cricket { + +FakeFrameSource::FakeFrameSource(int width, + int height, + int interval_us, + int64_t timestamp_offset_us) + : width_(width), + height_(height), + interval_us_(interval_us), + next_timestamp_us_(timestamp_offset_us) { + RTC_CHECK_GT(width_, 0); + RTC_CHECK_GT(height_, 0); + RTC_CHECK_GT(interval_us_, 0); + RTC_CHECK_GE(next_timestamp_us_, 0); +} + +FakeFrameSource::FakeFrameSource(int width, int height, int interval_us) + : FakeFrameSource(width, height, interval_us, rtc::TimeMicros()) {} + +webrtc::VideoRotation FakeFrameSource::GetRotation() const { + return rotation_; +} + +void FakeFrameSource::SetRotation(webrtc::VideoRotation rotation) { + rotation_ = rotation; +} + +webrtc::VideoFrame FakeFrameSource::GetFrameRotationApplied() { + switch (rotation_) { + case webrtc::kVideoRotation_0: + case webrtc::kVideoRotation_180: + return GetFrame(width_, height_, webrtc::kVideoRotation_0, interval_us_); + case webrtc::kVideoRotation_90: + case webrtc::kVideoRotation_270: + return GetFrame(height_, width_, webrtc::kVideoRotation_0, interval_us_); + } + RTC_NOTREACHED() << "Invalid rotation value: " << static_cast<int>(rotation_); + // Without this return, the Windows Visual Studio compiler complains + // "not all control paths return a value". + return GetFrame(); +} + +webrtc::VideoFrame FakeFrameSource::GetFrame() { + return GetFrame(width_, height_, rotation_, interval_us_); +} + +webrtc::VideoFrame FakeFrameSource::GetFrame(int width, + int height, + webrtc::VideoRotation rotation, + int interval_us) { + RTC_CHECK_GT(width, 0); + RTC_CHECK_GT(height, 0); + RTC_CHECK_GT(interval_us, 0); + + rtc::scoped_refptr<webrtc::I420Buffer> buffer( + webrtc::I420Buffer::Create(width, height)); + + buffer->InitializeData(); + webrtc::VideoFrame frame = webrtc::VideoFrame::Builder() + .set_video_frame_buffer(buffer) + .set_rotation(rotation) + .set_timestamp_us(next_timestamp_us_) + .build(); + + next_timestamp_us_ += interval_us; + return frame; +} + +} // namespace cricket diff --git a/media/base/fake_frame_source.h b/media/base/fake_frame_source.h new file mode 100644 index 0000000000..4c56204e69 --- /dev/null +++ b/media/base/fake_frame_source.h @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MEDIA_BASE_FAKE_FRAME_SOURCE_H_ +#define MEDIA_BASE_FAKE_FRAME_SOURCE_H_ + +#include "api/video/video_frame.h" +#include "rtc_base/time_utils.h" + +namespace cricket { + +class FakeFrameSource { + public: + FakeFrameSource(int width, + int height, + int interval_us, + int64_t timestamp_offset_us); + FakeFrameSource(int width, int height, int interval_us); + + webrtc::VideoRotation GetRotation() const; + void SetRotation(webrtc::VideoRotation rotation); + + webrtc::VideoFrame GetFrame(); + webrtc::VideoFrame GetFrameRotationApplied(); + + // Override configuration. + webrtc::VideoFrame GetFrame(int width, + int height, + webrtc::VideoRotation rotation, + int interval_us); + + private: + const int width_; + const int height_; + const int interval_us_; + + webrtc::VideoRotation rotation_ = webrtc::kVideoRotation_0; + int64_t next_timestamp_us_; +}; + +} // namespace cricket + +#endif // MEDIA_BASE_FAKE_FRAME_SOURCE_H_ diff --git a/media/base/fake_media_engine.cc b/media/base/fake_media_engine.cc new file mode 100644 index 0000000000..1040757f8e --- /dev/null +++ b/media/base/fake_media_engine.cc @@ -0,0 +1,688 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "media/base/fake_media_engine.h" + +#include <memory> +#include <utility> + +#include "absl/algorithm/container.h" +#include "absl/strings/match.h" +#include "rtc_base/checks.h" + +namespace cricket { + +FakeVoiceMediaChannel::DtmfInfo::DtmfInfo(uint32_t ssrc, + int event_code, + int duration) + : ssrc(ssrc), event_code(event_code), duration(duration) {} + +FakeVoiceMediaChannel::VoiceChannelAudioSink::VoiceChannelAudioSink( + AudioSource* source) + : source_(source) { + source_->SetSink(this); +} +FakeVoiceMediaChannel::VoiceChannelAudioSink::~VoiceChannelAudioSink() { + if (source_) { + source_->SetSink(nullptr); + } +} +void FakeVoiceMediaChannel::VoiceChannelAudioSink::OnData( + const void* audio_data, + int bits_per_sample, + int sample_rate, + size_t number_of_channels, + size_t number_of_frames, + absl::optional<int64_t> absolute_capture_timestamp_ms) {} +void FakeVoiceMediaChannel::VoiceChannelAudioSink::OnClose() { + source_ = nullptr; +} +AudioSource* FakeVoiceMediaChannel::VoiceChannelAudioSink::source() const { + return source_; +} + +FakeVoiceMediaChannel::FakeVoiceMediaChannel(FakeVoiceEngine* engine, + const AudioOptions& options) + : engine_(engine), max_bps_(-1) { + output_scalings_[0] = 1.0; // For default channel. + SetOptions(options); +} +FakeVoiceMediaChannel::~FakeVoiceMediaChannel() { + if (engine_) { + engine_->UnregisterChannel(this); + } +} +const std::vector<AudioCodec>& FakeVoiceMediaChannel::recv_codecs() const { + return recv_codecs_; +} +const std::vector<AudioCodec>& FakeVoiceMediaChannel::send_codecs() const { + return send_codecs_; +} +const std::vector<AudioCodec>& FakeVoiceMediaChannel::codecs() const { + return send_codecs(); +} +const std::vector<FakeVoiceMediaChannel::DtmfInfo>& +FakeVoiceMediaChannel::dtmf_info_queue() const { + return dtmf_info_queue_; +} +const AudioOptions& FakeVoiceMediaChannel::options() const { + return options_; +} +int FakeVoiceMediaChannel::max_bps() const { + return max_bps_; +} +bool FakeVoiceMediaChannel::SetSendParameters( + const AudioSendParameters& params) { + set_send_rtcp_parameters(params.rtcp); + return (SetSendCodecs(params.codecs) && + SetSendExtmapAllowMixed(params.extmap_allow_mixed) && + SetSendRtpHeaderExtensions(params.extensions) && + SetMaxSendBandwidth(params.max_bandwidth_bps) && + SetOptions(params.options)); +} +bool FakeVoiceMediaChannel::SetRecvParameters( + const AudioRecvParameters& params) { + set_recv_rtcp_parameters(params.rtcp); + return (SetRecvCodecs(params.codecs) && + SetRecvRtpHeaderExtensions(params.extensions)); +} +void FakeVoiceMediaChannel::SetPlayout(bool playout) { + set_playout(playout); +} +void FakeVoiceMediaChannel::SetSend(bool send) { + set_sending(send); +} +bool FakeVoiceMediaChannel::SetAudioSend(uint32_t ssrc, + bool enable, + const AudioOptions* options, + AudioSource* source) { + if (!SetLocalSource(ssrc, source)) { + return false; + } + if (!RtpHelper<VoiceMediaChannel>::MuteStream(ssrc, !enable)) { + return false; + } + if (enable && options) { + return SetOptions(*options); + } + return true; +} +bool FakeVoiceMediaChannel::HasSource(uint32_t ssrc) const { + return local_sinks_.find(ssrc) != local_sinks_.end(); +} +bool FakeVoiceMediaChannel::AddRecvStream(const StreamParams& sp) { + if (!RtpHelper<VoiceMediaChannel>::AddRecvStream(sp)) + return false; + output_scalings_[sp.first_ssrc()] = 1.0; + output_delays_[sp.first_ssrc()] = 0; + return true; +} +bool FakeVoiceMediaChannel::RemoveRecvStream(uint32_t ssrc) { + if (!RtpHelper<VoiceMediaChannel>::RemoveRecvStream(ssrc)) + return false; + output_scalings_.erase(ssrc); + output_delays_.erase(ssrc); + return true; +} +bool FakeVoiceMediaChannel::CanInsertDtmf() { + for (std::vector<AudioCodec>::const_iterator it = send_codecs_.begin(); + it != send_codecs_.end(); ++it) { + // Find the DTMF telephone event "codec". + if (absl::EqualsIgnoreCase(it->name, "telephone-event")) { + return true; + } + } + return false; +} +bool FakeVoiceMediaChannel::InsertDtmf(uint32_t ssrc, + int event_code, + int duration) { + dtmf_info_queue_.push_back(DtmfInfo(ssrc, event_code, duration)); + return true; +} +bool FakeVoiceMediaChannel::SetOutputVolume(uint32_t ssrc, double volume) { + if (output_scalings_.find(ssrc) != output_scalings_.end()) { + output_scalings_[ssrc] = volume; + return true; + } + return false; +} +bool FakeVoiceMediaChannel::SetDefaultOutputVolume(double volume) { + for (auto& entry : output_scalings_) { + entry.second = volume; + } + return true; +} +bool FakeVoiceMediaChannel::GetOutputVolume(uint32_t ssrc, double* volume) { + if (output_scalings_.find(ssrc) == output_scalings_.end()) + return false; + *volume = output_scalings_[ssrc]; + return true; +} +bool FakeVoiceMediaChannel::SetBaseMinimumPlayoutDelayMs(uint32_t ssrc, + int delay_ms) { + if (output_delays_.find(ssrc) == output_delays_.end()) { + return false; + } else { + output_delays_[ssrc] = delay_ms; + return true; + } +} +absl::optional<int> FakeVoiceMediaChannel::GetBaseMinimumPlayoutDelayMs( + uint32_t ssrc) const { + const auto it = output_delays_.find(ssrc); + if (it != output_delays_.end()) { + return it->second; + } + return absl::nullopt; +} +bool FakeVoiceMediaChannel::GetStats(VoiceMediaInfo* info) { + return false; +} +void FakeVoiceMediaChannel::SetRawAudioSink( + uint32_t ssrc, + std::unique_ptr<webrtc::AudioSinkInterface> sink) { + sink_ = std::move(sink); +} +void FakeVoiceMediaChannel::SetDefaultRawAudioSink( + std::unique_ptr<webrtc::AudioSinkInterface> sink) { + sink_ = std::move(sink); +} +std::vector<webrtc::RtpSource> FakeVoiceMediaChannel::GetSources( + uint32_t ssrc) const { + return std::vector<webrtc::RtpSource>(); +} +bool FakeVoiceMediaChannel::SetRecvCodecs( + const std::vector<AudioCodec>& codecs) { + if (fail_set_recv_codecs()) { + // Fake the failure in SetRecvCodecs. + return false; + } + recv_codecs_ = codecs; + return true; +} +bool FakeVoiceMediaChannel::SetSendCodecs( + const std::vector<AudioCodec>& codecs) { + if (fail_set_send_codecs()) { + // Fake the failure in SetSendCodecs. + return false; + } + send_codecs_ = codecs; + return true; +} +bool FakeVoiceMediaChannel::SetMaxSendBandwidth(int bps) { + max_bps_ = bps; + return true; +} +bool FakeVoiceMediaChannel::SetOptions(const AudioOptions& options) { + // Does a "merge" of current options and set options. + options_.SetAll(options); + return true; +} +bool FakeVoiceMediaChannel::SetLocalSource(uint32_t ssrc, AudioSource* source) { + auto it = local_sinks_.find(ssrc); + if (source) { + if (it != local_sinks_.end()) { + RTC_CHECK(it->second->source() == source); + } else { + local_sinks_.insert(std::make_pair( + ssrc, std::make_unique<VoiceChannelAudioSink>(source))); + } + } else { + if (it != local_sinks_.end()) { + local_sinks_.erase(it); + } + } + return true; +} + +bool CompareDtmfInfo(const FakeVoiceMediaChannel::DtmfInfo& info, + uint32_t ssrc, + int event_code, + int duration) { + return (info.duration == duration && info.event_code == event_code && + info.ssrc == ssrc); +} + +FakeVideoMediaChannel::FakeVideoMediaChannel(FakeVideoEngine* engine, + const VideoOptions& options) + : engine_(engine), max_bps_(-1) { + SetOptions(options); +} +FakeVideoMediaChannel::~FakeVideoMediaChannel() { + if (engine_) { + engine_->UnregisterChannel(this); + } +} +const std::vector<VideoCodec>& FakeVideoMediaChannel::recv_codecs() const { + return recv_codecs_; +} +const std::vector<VideoCodec>& FakeVideoMediaChannel::send_codecs() const { + return send_codecs_; +} +const std::vector<VideoCodec>& FakeVideoMediaChannel::codecs() const { + return send_codecs(); +} +bool FakeVideoMediaChannel::rendering() const { + return playout(); +} +const VideoOptions& FakeVideoMediaChannel::options() const { + return options_; +} +const std::map<uint32_t, rtc::VideoSinkInterface<webrtc::VideoFrame>*>& +FakeVideoMediaChannel::sinks() const { + return sinks_; +} +int FakeVideoMediaChannel::max_bps() const { + return max_bps_; +} +bool FakeVideoMediaChannel::SetSendParameters( + const VideoSendParameters& params) { + set_send_rtcp_parameters(params.rtcp); + return (SetSendCodecs(params.codecs) && + SetSendExtmapAllowMixed(params.extmap_allow_mixed) && + SetSendRtpHeaderExtensions(params.extensions) && + SetMaxSendBandwidth(params.max_bandwidth_bps)); +} +bool FakeVideoMediaChannel::SetRecvParameters( + const VideoRecvParameters& params) { + set_recv_rtcp_parameters(params.rtcp); + return (SetRecvCodecs(params.codecs) && + SetRecvRtpHeaderExtensions(params.extensions)); +} +bool FakeVideoMediaChannel::AddSendStream(const StreamParams& sp) { + return RtpHelper<VideoMediaChannel>::AddSendStream(sp); +} +bool FakeVideoMediaChannel::RemoveSendStream(uint32_t ssrc) { + return RtpHelper<VideoMediaChannel>::RemoveSendStream(ssrc); +} +bool FakeVideoMediaChannel::GetSendCodec(VideoCodec* send_codec) { + if (send_codecs_.empty()) { + return false; + } + *send_codec = send_codecs_[0]; + return true; +} +bool FakeVideoMediaChannel::SetSink( + uint32_t ssrc, + rtc::VideoSinkInterface<webrtc::VideoFrame>* sink) { + auto it = sinks_.find(ssrc); + if (it == sinks_.end()) { + return false; + } + it->second = sink; + return true; +} +void FakeVideoMediaChannel::SetDefaultSink( + rtc::VideoSinkInterface<webrtc::VideoFrame>* sink) {} +bool FakeVideoMediaChannel::HasSink(uint32_t ssrc) const { + return sinks_.find(ssrc) != sinks_.end() && sinks_.at(ssrc) != nullptr; +} +bool FakeVideoMediaChannel::SetSend(bool send) { + return set_sending(send); +} +bool FakeVideoMediaChannel::SetVideoSend( + uint32_t ssrc, + const VideoOptions* options, + rtc::VideoSourceInterface<webrtc::VideoFrame>* source) { + if (options) { + if (!SetOptions(*options)) { + return false; + } + } + sources_[ssrc] = source; + return true; +} +bool FakeVideoMediaChannel::HasSource(uint32_t ssrc) const { + return sources_.find(ssrc) != sources_.end() && sources_.at(ssrc) != nullptr; +} +bool FakeVideoMediaChannel::AddRecvStream(const StreamParams& sp) { + if (!RtpHelper<VideoMediaChannel>::AddRecvStream(sp)) + return false; + sinks_[sp.first_ssrc()] = NULL; + output_delays_[sp.first_ssrc()] = 0; + return true; +} +bool FakeVideoMediaChannel::RemoveRecvStream(uint32_t ssrc) { + if (!RtpHelper<VideoMediaChannel>::RemoveRecvStream(ssrc)) + return false; + sinks_.erase(ssrc); + output_delays_.erase(ssrc); + return true; +} +void FakeVideoMediaChannel::FillBitrateInfo(BandwidthEstimationInfo* bwe_info) { +} +bool FakeVideoMediaChannel::GetStats(VideoMediaInfo* info) { + return false; +} +std::vector<webrtc::RtpSource> FakeVideoMediaChannel::GetSources( + uint32_t ssrc) const { + return {}; +} +bool FakeVideoMediaChannel::SetBaseMinimumPlayoutDelayMs(uint32_t ssrc, + int delay_ms) { + if (output_delays_.find(ssrc) == output_delays_.end()) { + return false; + } else { + output_delays_[ssrc] = delay_ms; + return true; + } +} +absl::optional<int> FakeVideoMediaChannel::GetBaseMinimumPlayoutDelayMs( + uint32_t ssrc) const { + const auto it = output_delays_.find(ssrc); + if (it != output_delays_.end()) { + return it->second; + } + return absl::nullopt; +} +bool FakeVideoMediaChannel::SetRecvCodecs( + const std::vector<VideoCodec>& codecs) { + if (fail_set_recv_codecs()) { + // Fake the failure in SetRecvCodecs. + return false; + } + recv_codecs_ = codecs; + return true; +} +bool FakeVideoMediaChannel::SetSendCodecs( + const std::vector<VideoCodec>& codecs) { + if (fail_set_send_codecs()) { + // Fake the failure in SetSendCodecs. + return false; + } + send_codecs_ = codecs; + + return true; +} +bool FakeVideoMediaChannel::SetOptions(const VideoOptions& options) { + options_ = options; + return true; +} + +bool FakeVideoMediaChannel::SetMaxSendBandwidth(int bps) { + max_bps_ = bps; + return true; +} + +void FakeVideoMediaChannel::SetRecordableEncodedFrameCallback( + uint32_t ssrc, + std::function<void(const webrtc::RecordableEncodedFrame&)> callback) {} + +void FakeVideoMediaChannel::ClearRecordableEncodedFrameCallback(uint32_t ssrc) { +} + +void FakeVideoMediaChannel::GenerateKeyFrame(uint32_t ssrc) {} + +FakeDataMediaChannel::FakeDataMediaChannel(void* unused, + const DataOptions& options) + : send_blocked_(false), max_bps_(-1) {} +FakeDataMediaChannel::~FakeDataMediaChannel() {} +const std::vector<DataCodec>& FakeDataMediaChannel::recv_codecs() const { + return recv_codecs_; +} +const std::vector<DataCodec>& FakeDataMediaChannel::send_codecs() const { + return send_codecs_; +} +const std::vector<DataCodec>& FakeDataMediaChannel::codecs() const { + return send_codecs(); +} +int FakeDataMediaChannel::max_bps() const { + return max_bps_; +} +bool FakeDataMediaChannel::SetSendParameters(const DataSendParameters& params) { + set_send_rtcp_parameters(params.rtcp); + return (SetSendCodecs(params.codecs) && + SetMaxSendBandwidth(params.max_bandwidth_bps)); +} +bool FakeDataMediaChannel::SetRecvParameters(const DataRecvParameters& params) { + set_recv_rtcp_parameters(params.rtcp); + return SetRecvCodecs(params.codecs); +} +bool FakeDataMediaChannel::SetSend(bool send) { + return set_sending(send); +} +bool FakeDataMediaChannel::SetReceive(bool receive) { + set_playout(receive); + return true; +} +bool FakeDataMediaChannel::AddRecvStream(const StreamParams& sp) { + if (!RtpHelper<DataMediaChannel>::AddRecvStream(sp)) + return false; + return true; +} +bool FakeDataMediaChannel::RemoveRecvStream(uint32_t ssrc) { + if (!RtpHelper<DataMediaChannel>::RemoveRecvStream(ssrc)) + return false; + return true; +} +bool FakeDataMediaChannel::SendData(const SendDataParams& params, + const rtc::CopyOnWriteBuffer& payload, + SendDataResult* result) { + if (send_blocked_) { + *result = SDR_BLOCK; + return false; + } else { + last_sent_data_params_ = params; + last_sent_data_ = std::string(payload.data<char>(), payload.size()); + return true; + } +} +SendDataParams FakeDataMediaChannel::last_sent_data_params() { + return last_sent_data_params_; +} +std::string FakeDataMediaChannel::last_sent_data() { + return last_sent_data_; +} +bool FakeDataMediaChannel::is_send_blocked() { + return send_blocked_; +} +void FakeDataMediaChannel::set_send_blocked(bool blocked) { + send_blocked_ = blocked; +} +bool FakeDataMediaChannel::SetRecvCodecs(const std::vector<DataCodec>& codecs) { + if (fail_set_recv_codecs()) { + // Fake the failure in SetRecvCodecs. + return false; + } + recv_codecs_ = codecs; + return true; +} +bool FakeDataMediaChannel::SetSendCodecs(const std::vector<DataCodec>& codecs) { + if (fail_set_send_codecs()) { + // Fake the failure in SetSendCodecs. + return false; + } + send_codecs_ = codecs; + return true; +} +bool FakeDataMediaChannel::SetMaxSendBandwidth(int bps) { + max_bps_ = bps; + return true; +} + +FakeVoiceEngine::FakeVoiceEngine() : fail_create_channel_(false) { + // Add a fake audio codec. Note that the name must not be "" as there are + // sanity checks against that. + SetCodecs({AudioCodec(101, "fake_audio_codec", 0, 0, 1)}); +} +void FakeVoiceEngine::Init() {} +rtc::scoped_refptr<webrtc::AudioState> FakeVoiceEngine::GetAudioState() const { + return rtc::scoped_refptr<webrtc::AudioState>(); +} +VoiceMediaChannel* FakeVoiceEngine::CreateMediaChannel( + webrtc::Call* call, + const MediaConfig& config, + const AudioOptions& options, + const webrtc::CryptoOptions& crypto_options) { + if (fail_create_channel_) { + return nullptr; + } + + FakeVoiceMediaChannel* ch = new FakeVoiceMediaChannel(this, options); + channels_.push_back(ch); + return ch; +} +FakeVoiceMediaChannel* FakeVoiceEngine::GetChannel(size_t index) { + return (channels_.size() > index) ? channels_[index] : NULL; +} +void FakeVoiceEngine::UnregisterChannel(VoiceMediaChannel* channel) { + channels_.erase(absl::c_find(channels_, channel)); +} +const std::vector<AudioCodec>& FakeVoiceEngine::send_codecs() const { + return send_codecs_; +} +const std::vector<AudioCodec>& FakeVoiceEngine::recv_codecs() const { + return recv_codecs_; +} +void FakeVoiceEngine::SetCodecs(const std::vector<AudioCodec>& codecs) { + send_codecs_ = codecs; + recv_codecs_ = codecs; +} +void FakeVoiceEngine::SetRecvCodecs(const std::vector<AudioCodec>& codecs) { + recv_codecs_ = codecs; +} +void FakeVoiceEngine::SetSendCodecs(const std::vector<AudioCodec>& codecs) { + send_codecs_ = codecs; +} +int FakeVoiceEngine::GetInputLevel() { + return 0; +} +bool FakeVoiceEngine::StartAecDump(webrtc::FileWrapper file, + int64_t max_size_bytes) { + return false; +} +void FakeVoiceEngine::StopAecDump() {} + +std::vector<webrtc::RtpHeaderExtensionCapability> +FakeVoiceEngine::GetRtpHeaderExtensions() const { + return header_extensions_; +} + +void FakeVoiceEngine::SetRtpHeaderExtensions( + std::vector<webrtc::RtpHeaderExtensionCapability> header_extensions) { + header_extensions_ = std::move(header_extensions); +} + +FakeVideoEngine::FakeVideoEngine() + : capture_(false), fail_create_channel_(false) { + // Add a fake video codec. Note that the name must not be "" as there are + // sanity checks against that. + send_codecs_.push_back(VideoCodec(0, "fake_video_codec")); + recv_codecs_.push_back(VideoCodec(0, "fake_video_codec")); +} +bool FakeVideoEngine::SetOptions(const VideoOptions& options) { + options_ = options; + return true; +} +VideoMediaChannel* FakeVideoEngine::CreateMediaChannel( + webrtc::Call* call, + const MediaConfig& config, + const VideoOptions& options, + const webrtc::CryptoOptions& crypto_options, + webrtc::VideoBitrateAllocatorFactory* video_bitrate_allocator_factory) { + if (fail_create_channel_) { + return nullptr; + } + + FakeVideoMediaChannel* ch = new FakeVideoMediaChannel(this, options); + channels_.emplace_back(ch); + return ch; +} +FakeVideoMediaChannel* FakeVideoEngine::GetChannel(size_t index) { + return (channels_.size() > index) ? channels_[index] : nullptr; +} +void FakeVideoEngine::UnregisterChannel(VideoMediaChannel* channel) { + auto it = absl::c_find(channels_, channel); + RTC_DCHECK(it != channels_.end()); + channels_.erase(it); +} +std::vector<VideoCodec> FakeVideoEngine::send_codecs() const { + return send_codecs_; +} + +std::vector<VideoCodec> FakeVideoEngine::recv_codecs() const { + return recv_codecs_; +} + +void FakeVideoEngine::SetSendCodecs(const std::vector<VideoCodec>& codecs) { + send_codecs_ = codecs; +} + +void FakeVideoEngine::SetRecvCodecs(const std::vector<VideoCodec>& codecs) { + recv_codecs_ = codecs; +} + +bool FakeVideoEngine::SetCapture(bool capture) { + capture_ = capture; + return true; +} +std::vector<webrtc::RtpHeaderExtensionCapability> +FakeVideoEngine::GetRtpHeaderExtensions() const { + return header_extensions_; +} +void FakeVideoEngine::SetRtpHeaderExtensions( + std::vector<webrtc::RtpHeaderExtensionCapability> header_extensions) { + header_extensions_ = std::move(header_extensions); +} + +FakeMediaEngine::FakeMediaEngine() + : CompositeMediaEngine(std::make_unique<FakeVoiceEngine>(), + std::make_unique<FakeVideoEngine>()), + voice_(static_cast<FakeVoiceEngine*>(&voice())), + video_(static_cast<FakeVideoEngine*>(&video())) {} +FakeMediaEngine::~FakeMediaEngine() {} +void FakeMediaEngine::SetAudioCodecs(const std::vector<AudioCodec>& codecs) { + voice_->SetCodecs(codecs); +} +void FakeMediaEngine::SetAudioRecvCodecs( + const std::vector<AudioCodec>& codecs) { + voice_->SetRecvCodecs(codecs); +} +void FakeMediaEngine::SetAudioSendCodecs( + const std::vector<AudioCodec>& codecs) { + voice_->SetSendCodecs(codecs); +} +void FakeMediaEngine::SetVideoCodecs(const std::vector<VideoCodec>& codecs) { + video_->SetSendCodecs(codecs); + video_->SetRecvCodecs(codecs); +} + +FakeVoiceMediaChannel* FakeMediaEngine::GetVoiceChannel(size_t index) { + return voice_->GetChannel(index); +} +FakeVideoMediaChannel* FakeMediaEngine::GetVideoChannel(size_t index) { + return video_->GetChannel(index); +} + +void FakeMediaEngine::set_fail_create_channel(bool fail) { + voice_->fail_create_channel_ = fail; + video_->fail_create_channel_ = fail; +} + +DataMediaChannel* FakeDataEngine::CreateChannel(const MediaConfig& config) { + FakeDataMediaChannel* ch = new FakeDataMediaChannel(this, DataOptions()); + channels_.push_back(ch); + return ch; +} +FakeDataMediaChannel* FakeDataEngine::GetChannel(size_t index) { + return (channels_.size() > index) ? channels_[index] : NULL; +} +void FakeDataEngine::UnregisterChannel(DataMediaChannel* channel) { + channels_.erase(absl::c_find(channels_, channel)); +} +void FakeDataEngine::SetDataCodecs(const std::vector<DataCodec>& data_codecs) { + data_codecs_ = data_codecs; +} +const std::vector<DataCodec>& FakeDataEngine::data_codecs() { + return data_codecs_; +} + +} // namespace cricket diff --git a/media/base/fake_media_engine.h b/media/base/fake_media_engine.h new file mode 100644 index 0000000000..338c329aa1 --- /dev/null +++ b/media/base/fake_media_engine.h @@ -0,0 +1,630 @@ +/* + * Copyright (c) 2004 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MEDIA_BASE_FAKE_MEDIA_ENGINE_H_ +#define MEDIA_BASE_FAKE_MEDIA_ENGINE_H_ + +#include <list> +#include <map> +#include <memory> +#include <set> +#include <string> +#include <tuple> +#include <vector> + +#include "absl/algorithm/container.h" +#include "api/call/audio_sink.h" +#include "media/base/audio_source.h" +#include "media/base/media_engine.h" +#include "media/base/rtp_utils.h" +#include "media/base/stream_params.h" +#include "media/engine/webrtc_video_engine.h" +#include "modules/audio_processing/include/audio_processing.h" +#include "rtc_base/copy_on_write_buffer.h" +#include "rtc_base/network_route.h" + +using webrtc::RtpExtension; + +namespace cricket { + +class FakeMediaEngine; +class FakeVideoEngine; +class FakeVoiceEngine; + +// A common helper class that handles sending and receiving RTP/RTCP packets. +template <class Base> +class RtpHelper : public Base { + public: + RtpHelper() + : sending_(false), + playout_(false), + fail_set_send_codecs_(false), + fail_set_recv_codecs_(false), + send_ssrc_(0), + ready_to_send_(false), + transport_overhead_per_packet_(0), + num_network_route_changes_(0) {} + virtual ~RtpHelper() = default; + const std::vector<RtpExtension>& recv_extensions() { + return recv_extensions_; + } + const std::vector<RtpExtension>& send_extensions() { + return send_extensions_; + } + bool sending() const { return sending_; } + bool playout() const { return playout_; } + const std::list<std::string>& rtp_packets() const { return rtp_packets_; } + const std::list<std::string>& rtcp_packets() const { return rtcp_packets_; } + + bool SendRtp(const void* data, + size_t len, + const rtc::PacketOptions& options) { + if (!sending_) { + return false; + } + rtc::CopyOnWriteBuffer packet(reinterpret_cast<const uint8_t*>(data), len, + kMaxRtpPacketLen); + return Base::SendPacket(&packet, options); + } + bool SendRtcp(const void* data, size_t len) { + rtc::CopyOnWriteBuffer packet(reinterpret_cast<const uint8_t*>(data), len, + kMaxRtpPacketLen); + return Base::SendRtcp(&packet, rtc::PacketOptions()); + } + + bool CheckRtp(const void* data, size_t len) { + bool success = !rtp_packets_.empty(); + if (success) { + std::string packet = rtp_packets_.front(); + rtp_packets_.pop_front(); + success = (packet == std::string(static_cast<const char*>(data), len)); + } + return success; + } + bool CheckRtcp(const void* data, size_t len) { + bool success = !rtcp_packets_.empty(); + if (success) { + std::string packet = rtcp_packets_.front(); + rtcp_packets_.pop_front(); + success = (packet == std::string(static_cast<const char*>(data), len)); + } + return success; + } + bool CheckNoRtp() { return rtp_packets_.empty(); } + bool CheckNoRtcp() { return rtcp_packets_.empty(); } + void set_fail_set_send_codecs(bool fail) { fail_set_send_codecs_ = fail; } + void set_fail_set_recv_codecs(bool fail) { fail_set_recv_codecs_ = fail; } + virtual bool AddSendStream(const StreamParams& sp) { + if (absl::c_linear_search(send_streams_, sp)) { + return false; + } + send_streams_.push_back(sp); + rtp_send_parameters_[sp.first_ssrc()] = + CreateRtpParametersWithEncodings(sp); + return true; + } + virtual bool RemoveSendStream(uint32_t ssrc) { + auto parameters_iterator = rtp_send_parameters_.find(ssrc); + if (parameters_iterator != rtp_send_parameters_.end()) { + rtp_send_parameters_.erase(parameters_iterator); + } + return RemoveStreamBySsrc(&send_streams_, ssrc); + } + virtual void ResetUnsignaledRecvStream() {} + + virtual bool AddRecvStream(const StreamParams& sp) { + if (absl::c_linear_search(receive_streams_, sp)) { + return false; + } + receive_streams_.push_back(sp); + rtp_receive_parameters_[sp.first_ssrc()] = + CreateRtpParametersWithEncodings(sp); + return true; + } + virtual bool RemoveRecvStream(uint32_t ssrc) { + auto parameters_iterator = rtp_receive_parameters_.find(ssrc); + if (parameters_iterator != rtp_receive_parameters_.end()) { + rtp_receive_parameters_.erase(parameters_iterator); + } + return RemoveStreamBySsrc(&receive_streams_, ssrc); + } + + virtual webrtc::RtpParameters GetRtpSendParameters(uint32_t ssrc) const { + auto parameters_iterator = rtp_send_parameters_.find(ssrc); + if (parameters_iterator != rtp_send_parameters_.end()) { + return parameters_iterator->second; + } + return webrtc::RtpParameters(); + } + virtual webrtc::RTCError SetRtpSendParameters( + uint32_t ssrc, + const webrtc::RtpParameters& parameters) { + auto parameters_iterator = rtp_send_parameters_.find(ssrc); + if (parameters_iterator != rtp_send_parameters_.end()) { + auto result = CheckRtpParametersInvalidModificationAndValues( + parameters_iterator->second, parameters); + if (!result.ok()) + return result; + + parameters_iterator->second = parameters; + return webrtc::RTCError::OK(); + } + // Replicate the behavior of the real media channel: return false + // when setting parameters for unknown SSRCs. + return webrtc::RTCError(webrtc::RTCErrorType::INTERNAL_ERROR); + } + + virtual webrtc::RtpParameters GetRtpReceiveParameters(uint32_t ssrc) const { + auto parameters_iterator = rtp_receive_parameters_.find(ssrc); + if (parameters_iterator != rtp_receive_parameters_.end()) { + return parameters_iterator->second; + } + return webrtc::RtpParameters(); + } + virtual webrtc::RtpParameters GetDefaultRtpReceiveParameters() const { + return webrtc::RtpParameters(); + } + + bool IsStreamMuted(uint32_t ssrc) const { + bool ret = muted_streams_.find(ssrc) != muted_streams_.end(); + // If |ssrc = 0| check if the first send stream is muted. + if (!ret && ssrc == 0 && !send_streams_.empty()) { + return muted_streams_.find(send_streams_[0].first_ssrc()) != + muted_streams_.end(); + } + return ret; + } + const std::vector<StreamParams>& send_streams() const { + return send_streams_; + } + const std::vector<StreamParams>& recv_streams() const { + return receive_streams_; + } + bool HasRecvStream(uint32_t ssrc) const { + return GetStreamBySsrc(receive_streams_, ssrc) != nullptr; + } + bool HasSendStream(uint32_t ssrc) const { + return GetStreamBySsrc(send_streams_, ssrc) != nullptr; + } + // TODO(perkj): This is to support legacy unit test that only check one + // sending stream. + uint32_t send_ssrc() const { + if (send_streams_.empty()) + return 0; + return send_streams_[0].first_ssrc(); + } + + // TODO(perkj): This is to support legacy unit test that only check one + // sending stream. + const std::string rtcp_cname() { + if (send_streams_.empty()) + return ""; + return send_streams_[0].cname; + } + const RtcpParameters& send_rtcp_parameters() { return send_rtcp_parameters_; } + const RtcpParameters& recv_rtcp_parameters() { return recv_rtcp_parameters_; } + + bool ready_to_send() const { return ready_to_send_; } + + int transport_overhead_per_packet() const { + return transport_overhead_per_packet_; + } + + rtc::NetworkRoute last_network_route() const { return last_network_route_; } + int num_network_route_changes() const { return num_network_route_changes_; } + void set_num_network_route_changes(int changes) { + num_network_route_changes_ = changes; + } + + void OnRtcpPacketReceived(rtc::CopyOnWriteBuffer* packet, + int64_t packet_time_us) { + rtcp_packets_.push_back(std::string(packet->cdata<char>(), packet->size())); + } + + protected: + bool MuteStream(uint32_t ssrc, bool mute) { + if (!HasSendStream(ssrc) && ssrc != 0) { + return false; + } + if (mute) { + muted_streams_.insert(ssrc); + } else { + muted_streams_.erase(ssrc); + } + return true; + } + bool set_sending(bool send) { + sending_ = send; + return true; + } + void set_playout(bool playout) { playout_ = playout; } + bool SetRecvRtpHeaderExtensions(const std::vector<RtpExtension>& extensions) { + recv_extensions_ = extensions; + return true; + } + bool SetSendExtmapAllowMixed(bool extmap_allow_mixed) { + if (Base::ExtmapAllowMixed() != extmap_allow_mixed) { + Base::SetExtmapAllowMixed(extmap_allow_mixed); + } + return true; + } + bool SetSendRtpHeaderExtensions(const std::vector<RtpExtension>& extensions) { + send_extensions_ = extensions; + return true; + } + void set_send_rtcp_parameters(const RtcpParameters& params) { + send_rtcp_parameters_ = params; + } + void set_recv_rtcp_parameters(const RtcpParameters& params) { + recv_rtcp_parameters_ = params; + } + virtual void OnPacketReceived(rtc::CopyOnWriteBuffer packet, + int64_t packet_time_us) { + rtp_packets_.push_back(std::string(packet.cdata<char>(), packet.size())); + } + virtual void OnReadyToSend(bool ready) { ready_to_send_ = ready; } + + virtual void OnNetworkRouteChanged(const std::string& transport_name, + const rtc::NetworkRoute& network_route) { + last_network_route_ = network_route; + ++num_network_route_changes_; + transport_overhead_per_packet_ = network_route.packet_overhead; + } + bool fail_set_send_codecs() const { return fail_set_send_codecs_; } + bool fail_set_recv_codecs() const { return fail_set_recv_codecs_; } + + private: + bool sending_; + bool playout_; + std::vector<RtpExtension> recv_extensions_; + std::vector<RtpExtension> send_extensions_; + std::list<std::string> rtp_packets_; + std::list<std::string> rtcp_packets_; + std::vector<StreamParams> send_streams_; + std::vector<StreamParams> receive_streams_; + RtcpParameters send_rtcp_parameters_; + RtcpParameters recv_rtcp_parameters_; + std::set<uint32_t> muted_streams_; + std::map<uint32_t, webrtc::RtpParameters> rtp_send_parameters_; + std::map<uint32_t, webrtc::RtpParameters> rtp_receive_parameters_; + bool fail_set_send_codecs_; + bool fail_set_recv_codecs_; + uint32_t send_ssrc_; + std::string rtcp_cname_; + bool ready_to_send_; + int transport_overhead_per_packet_; + rtc::NetworkRoute last_network_route_; + int num_network_route_changes_; +}; + +class FakeVoiceMediaChannel : public RtpHelper<VoiceMediaChannel> { + public: + struct DtmfInfo { + DtmfInfo(uint32_t ssrc, int event_code, int duration); + uint32_t ssrc; + int event_code; + int duration; + }; + explicit FakeVoiceMediaChannel(FakeVoiceEngine* engine, + const AudioOptions& options); + ~FakeVoiceMediaChannel(); + const std::vector<AudioCodec>& recv_codecs() const; + const std::vector<AudioCodec>& send_codecs() const; + const std::vector<AudioCodec>& codecs() const; + const std::vector<DtmfInfo>& dtmf_info_queue() const; + const AudioOptions& options() const; + int max_bps() const; + bool SetSendParameters(const AudioSendParameters& params) override; + + bool SetRecvParameters(const AudioRecvParameters& params) override; + + void SetPlayout(bool playout) override; + void SetSend(bool send) override; + bool SetAudioSend(uint32_t ssrc, + bool enable, + const AudioOptions* options, + AudioSource* source) override; + + bool HasSource(uint32_t ssrc) const; + + bool AddRecvStream(const StreamParams& sp) override; + bool RemoveRecvStream(uint32_t ssrc) override; + + bool CanInsertDtmf() override; + bool InsertDtmf(uint32_t ssrc, int event_code, int duration) override; + + bool SetOutputVolume(uint32_t ssrc, double volume) override; + bool SetDefaultOutputVolume(double volume) override; + + bool GetOutputVolume(uint32_t ssrc, double* volume); + + bool SetBaseMinimumPlayoutDelayMs(uint32_t ssrc, int delay_ms) override; + absl::optional<int> GetBaseMinimumPlayoutDelayMs( + uint32_t ssrc) const override; + + bool GetStats(VoiceMediaInfo* info) override; + + void SetRawAudioSink( + uint32_t ssrc, + std::unique_ptr<webrtc::AudioSinkInterface> sink) override; + void SetDefaultRawAudioSink( + std::unique_ptr<webrtc::AudioSinkInterface> sink) override; + + std::vector<webrtc::RtpSource> GetSources(uint32_t ssrc) const override; + + private: + class VoiceChannelAudioSink : public AudioSource::Sink { + public: + explicit VoiceChannelAudioSink(AudioSource* source); + ~VoiceChannelAudioSink() override; + void OnData(const void* audio_data, + int bits_per_sample, + int sample_rate, + size_t number_of_channels, + size_t number_of_frames, + absl::optional<int64_t> absolute_capture_timestamp_ms) override; + void OnClose() override; + AudioSource* source() const; + + private: + AudioSource* source_; + }; + + bool SetRecvCodecs(const std::vector<AudioCodec>& codecs); + bool SetSendCodecs(const std::vector<AudioCodec>& codecs); + bool SetMaxSendBandwidth(int bps); + bool SetOptions(const AudioOptions& options); + bool SetLocalSource(uint32_t ssrc, AudioSource* source); + + FakeVoiceEngine* engine_; + std::vector<AudioCodec> recv_codecs_; + std::vector<AudioCodec> send_codecs_; + std::map<uint32_t, double> output_scalings_; + std::map<uint32_t, int> output_delays_; + std::vector<DtmfInfo> dtmf_info_queue_; + AudioOptions options_; + std::map<uint32_t, std::unique_ptr<VoiceChannelAudioSink>> local_sinks_; + std::unique_ptr<webrtc::AudioSinkInterface> sink_; + int max_bps_; +}; + +// A helper function to compare the FakeVoiceMediaChannel::DtmfInfo. +bool CompareDtmfInfo(const FakeVoiceMediaChannel::DtmfInfo& info, + uint32_t ssrc, + int event_code, + int duration); + +class FakeVideoMediaChannel : public RtpHelper<VideoMediaChannel> { + public: + FakeVideoMediaChannel(FakeVideoEngine* engine, const VideoOptions& options); + + ~FakeVideoMediaChannel(); + + const std::vector<VideoCodec>& recv_codecs() const; + const std::vector<VideoCodec>& send_codecs() const; + const std::vector<VideoCodec>& codecs() const; + bool rendering() const; + const VideoOptions& options() const; + const std::map<uint32_t, rtc::VideoSinkInterface<webrtc::VideoFrame>*>& + sinks() const; + int max_bps() const; + bool SetSendParameters(const VideoSendParameters& params) override; + bool SetRecvParameters(const VideoRecvParameters& params) override; + bool AddSendStream(const StreamParams& sp) override; + bool RemoveSendStream(uint32_t ssrc) override; + + bool GetSendCodec(VideoCodec* send_codec) override; + bool SetSink(uint32_t ssrc, + rtc::VideoSinkInterface<webrtc::VideoFrame>* sink) override; + void SetDefaultSink( + rtc::VideoSinkInterface<webrtc::VideoFrame>* sink) override; + bool HasSink(uint32_t ssrc) const; + + bool SetSend(bool send) override; + bool SetVideoSend( + uint32_t ssrc, + const VideoOptions* options, + rtc::VideoSourceInterface<webrtc::VideoFrame>* source) override; + + bool HasSource(uint32_t ssrc) const; + bool AddRecvStream(const StreamParams& sp) override; + bool RemoveRecvStream(uint32_t ssrc) override; + + void FillBitrateInfo(BandwidthEstimationInfo* bwe_info) override; + bool GetStats(VideoMediaInfo* info) override; + + std::vector<webrtc::RtpSource> GetSources(uint32_t ssrc) const override; + + bool SetBaseMinimumPlayoutDelayMs(uint32_t ssrc, int delay_ms) override; + absl::optional<int> GetBaseMinimumPlayoutDelayMs( + uint32_t ssrc) const override; + + void SetRecordableEncodedFrameCallback( + uint32_t ssrc, + std::function<void(const webrtc::RecordableEncodedFrame&)> callback) + override; + void ClearRecordableEncodedFrameCallback(uint32_t ssrc) override; + void GenerateKeyFrame(uint32_t ssrc) override; + + private: + bool SetRecvCodecs(const std::vector<VideoCodec>& codecs); + bool SetSendCodecs(const std::vector<VideoCodec>& codecs); + bool SetOptions(const VideoOptions& options); + bool SetMaxSendBandwidth(int bps); + + FakeVideoEngine* engine_; + std::vector<VideoCodec> recv_codecs_; + std::vector<VideoCodec> send_codecs_; + std::map<uint32_t, rtc::VideoSinkInterface<webrtc::VideoFrame>*> sinks_; + std::map<uint32_t, rtc::VideoSourceInterface<webrtc::VideoFrame>*> sources_; + std::map<uint32_t, int> output_delays_; + VideoOptions options_; + int max_bps_; +}; + +// Dummy option class, needed for the DataTraits abstraction in +// channel_unittest.c. +class DataOptions {}; + +class FakeDataMediaChannel : public RtpHelper<DataMediaChannel> { + public: + explicit FakeDataMediaChannel(void* unused, const DataOptions& options); + ~FakeDataMediaChannel(); + const std::vector<DataCodec>& recv_codecs() const; + const std::vector<DataCodec>& send_codecs() const; + const std::vector<DataCodec>& codecs() const; + int max_bps() const; + + bool SetSendParameters(const DataSendParameters& params) override; + bool SetRecvParameters(const DataRecvParameters& params) override; + bool SetSend(bool send) override; + bool SetReceive(bool receive) override; + bool AddRecvStream(const StreamParams& sp) override; + bool RemoveRecvStream(uint32_t ssrc) override; + + bool SendData(const SendDataParams& params, + const rtc::CopyOnWriteBuffer& payload, + SendDataResult* result) override; + + SendDataParams last_sent_data_params(); + std::string last_sent_data(); + bool is_send_blocked(); + void set_send_blocked(bool blocked); + + private: + bool SetRecvCodecs(const std::vector<DataCodec>& codecs); + bool SetSendCodecs(const std::vector<DataCodec>& codecs); + bool SetMaxSendBandwidth(int bps); + + std::vector<DataCodec> recv_codecs_; + std::vector<DataCodec> send_codecs_; + SendDataParams last_sent_data_params_; + std::string last_sent_data_; + bool send_blocked_; + int max_bps_; +}; + +class FakeVoiceEngine : public VoiceEngineInterface { + public: + FakeVoiceEngine(); + void Init() override; + rtc::scoped_refptr<webrtc::AudioState> GetAudioState() const override; + + VoiceMediaChannel* CreateMediaChannel( + webrtc::Call* call, + const MediaConfig& config, + const AudioOptions& options, + const webrtc::CryptoOptions& crypto_options) override; + FakeVoiceMediaChannel* GetChannel(size_t index); + void UnregisterChannel(VoiceMediaChannel* channel); + + // TODO(ossu): For proper testing, These should either individually settable + // or the voice engine should reference mockable factories. + const std::vector<AudioCodec>& send_codecs() const override; + const std::vector<AudioCodec>& recv_codecs() const override; + void SetCodecs(const std::vector<AudioCodec>& codecs); + void SetRecvCodecs(const std::vector<AudioCodec>& codecs); + void SetSendCodecs(const std::vector<AudioCodec>& codecs); + int GetInputLevel(); + bool StartAecDump(webrtc::FileWrapper file, int64_t max_size_bytes) override; + void StopAecDump() override; + std::vector<webrtc::RtpHeaderExtensionCapability> GetRtpHeaderExtensions() + const override; + void SetRtpHeaderExtensions( + std::vector<webrtc::RtpHeaderExtensionCapability> header_extensions); + + private: + std::vector<FakeVoiceMediaChannel*> channels_; + std::vector<AudioCodec> recv_codecs_; + std::vector<AudioCodec> send_codecs_; + bool fail_create_channel_; + std::vector<webrtc::RtpHeaderExtensionCapability> header_extensions_; + + friend class FakeMediaEngine; +}; + +class FakeVideoEngine : public VideoEngineInterface { + public: + FakeVideoEngine(); + bool SetOptions(const VideoOptions& options); + VideoMediaChannel* CreateMediaChannel( + webrtc::Call* call, + const MediaConfig& config, + const VideoOptions& options, + const webrtc::CryptoOptions& crypto_options, + webrtc::VideoBitrateAllocatorFactory* video_bitrate_allocator_factory) + override; + FakeVideoMediaChannel* GetChannel(size_t index); + void UnregisterChannel(VideoMediaChannel* channel); + std::vector<VideoCodec> send_codecs() const override; + std::vector<VideoCodec> recv_codecs() const override; + void SetSendCodecs(const std::vector<VideoCodec>& codecs); + void SetRecvCodecs(const std::vector<VideoCodec>& codecs); + bool SetCapture(bool capture); + std::vector<webrtc::RtpHeaderExtensionCapability> GetRtpHeaderExtensions() + const override; + void SetRtpHeaderExtensions( + std::vector<webrtc::RtpHeaderExtensionCapability> header_extensions); + + private: + std::vector<FakeVideoMediaChannel*> channels_; + std::vector<VideoCodec> send_codecs_; + std::vector<VideoCodec> recv_codecs_; + bool capture_; + VideoOptions options_; + bool fail_create_channel_; + std::vector<webrtc::RtpHeaderExtensionCapability> header_extensions_; + + friend class FakeMediaEngine; +}; + +class FakeMediaEngine : public CompositeMediaEngine { + public: + FakeMediaEngine(); + + ~FakeMediaEngine() override; + + void SetAudioCodecs(const std::vector<AudioCodec>& codecs); + void SetAudioRecvCodecs(const std::vector<AudioCodec>& codecs); + void SetAudioSendCodecs(const std::vector<AudioCodec>& codecs); + void SetVideoCodecs(const std::vector<VideoCodec>& codecs); + + FakeVoiceMediaChannel* GetVoiceChannel(size_t index); + FakeVideoMediaChannel* GetVideoChannel(size_t index); + + void set_fail_create_channel(bool fail); + + private: + FakeVoiceEngine* const voice_; + FakeVideoEngine* const video_; +}; + +// Have to come afterwards due to declaration order + +class FakeDataEngine : public DataEngineInterface { + public: + DataMediaChannel* CreateChannel(const MediaConfig& config) override; + + FakeDataMediaChannel* GetChannel(size_t index); + + void UnregisterChannel(DataMediaChannel* channel); + + void SetDataCodecs(const std::vector<DataCodec>& data_codecs); + + const std::vector<DataCodec>& data_codecs() override; + + private: + std::vector<FakeDataMediaChannel*> channels_; + std::vector<DataCodec> data_codecs_; +}; + +} // namespace cricket + +#endif // MEDIA_BASE_FAKE_MEDIA_ENGINE_H_ diff --git a/media/base/fake_network_interface.h b/media/base/fake_network_interface.h new file mode 100644 index 0000000000..7d50ca84bc --- /dev/null +++ b/media/base/fake_network_interface.h @@ -0,0 +1,227 @@ +/* + * Copyright (c) 2004 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MEDIA_BASE_FAKE_NETWORK_INTERFACE_H_ +#define MEDIA_BASE_FAKE_NETWORK_INTERFACE_H_ + +#include <map> +#include <set> +#include <vector> + +#include "media/base/media_channel.h" +#include "media/base/rtp_utils.h" +#include "rtc_base/byte_order.h" +#include "rtc_base/copy_on_write_buffer.h" +#include "rtc_base/critical_section.h" +#include "rtc_base/dscp.h" +#include "rtc_base/message_handler.h" +#include "rtc_base/thread.h" + +namespace cricket { + +// Fake NetworkInterface that sends/receives RTP/RTCP packets. +class FakeNetworkInterface : public MediaChannel::NetworkInterface, + public rtc::MessageHandler { + public: + FakeNetworkInterface() + : thread_(rtc::Thread::Current()), + dest_(NULL), + conf_(false), + sendbuf_size_(-1), + recvbuf_size_(-1), + dscp_(rtc::DSCP_NO_CHANGE) {} + + void SetDestination(MediaChannel* dest) { dest_ = dest; } + + // Conference mode is a mode where instead of simply forwarding the packets, + // the transport will send multiple copies of the packet with the specified + // SSRCs. This allows us to simulate receiving media from multiple sources. + void SetConferenceMode(bool conf, const std::vector<uint32_t>& ssrcs) { + rtc::CritScope cs(&crit_); + conf_ = conf; + conf_sent_ssrcs_ = ssrcs; + } + + int NumRtpBytes() { + rtc::CritScope cs(&crit_); + int bytes = 0; + for (size_t i = 0; i < rtp_packets_.size(); ++i) { + bytes += static_cast<int>(rtp_packets_[i].size()); + } + return bytes; + } + + int NumRtpBytes(uint32_t ssrc) { + rtc::CritScope cs(&crit_); + int bytes = 0; + GetNumRtpBytesAndPackets(ssrc, &bytes, NULL); + return bytes; + } + + int NumRtpPackets() { + rtc::CritScope cs(&crit_); + return static_cast<int>(rtp_packets_.size()); + } + + int NumRtpPackets(uint32_t ssrc) { + rtc::CritScope cs(&crit_); + int packets = 0; + GetNumRtpBytesAndPackets(ssrc, NULL, &packets); + return packets; + } + + int NumSentSsrcs() { + rtc::CritScope cs(&crit_); + return static_cast<int>(sent_ssrcs_.size()); + } + + // Note: callers are responsible for deleting the returned buffer. + const rtc::CopyOnWriteBuffer* GetRtpPacket(int index) { + rtc::CritScope cs(&crit_); + if (index >= NumRtpPackets()) { + return NULL; + } + return new rtc::CopyOnWriteBuffer(rtp_packets_[index]); + } + + int NumRtcpPackets() { + rtc::CritScope cs(&crit_); + return static_cast<int>(rtcp_packets_.size()); + } + + // Note: callers are responsible for deleting the returned buffer. + const rtc::CopyOnWriteBuffer* GetRtcpPacket(int index) { + rtc::CritScope cs(&crit_); + if (index >= NumRtcpPackets()) { + return NULL; + } + return new rtc::CopyOnWriteBuffer(rtcp_packets_[index]); + } + + int sendbuf_size() const { return sendbuf_size_; } + int recvbuf_size() const { return recvbuf_size_; } + rtc::DiffServCodePoint dscp() const { return dscp_; } + rtc::PacketOptions options() const { return options_; } + + protected: + virtual bool SendPacket(rtc::CopyOnWriteBuffer* packet, + const rtc::PacketOptions& options) { + rtc::CritScope cs(&crit_); + + uint32_t cur_ssrc = 0; + if (!GetRtpSsrc(packet->data(), packet->size(), &cur_ssrc)) { + return false; + } + sent_ssrcs_[cur_ssrc]++; + options_ = options; + + rtp_packets_.push_back(*packet); + if (conf_) { + for (size_t i = 0; i < conf_sent_ssrcs_.size(); ++i) { + if (!SetRtpSsrc(packet->data(), packet->size(), conf_sent_ssrcs_[i])) { + return false; + } + PostMessage(ST_RTP, *packet); + } + } else { + PostMessage(ST_RTP, *packet); + } + return true; + } + + virtual bool SendRtcp(rtc::CopyOnWriteBuffer* packet, + const rtc::PacketOptions& options) { + rtc::CritScope cs(&crit_); + rtcp_packets_.push_back(*packet); + options_ = options; + if (!conf_) { + // don't worry about RTCP in conf mode for now + PostMessage(ST_RTCP, *packet); + } + return true; + } + + virtual int SetOption(SocketType type, rtc::Socket::Option opt, int option) { + if (opt == rtc::Socket::OPT_SNDBUF) { + sendbuf_size_ = option; + } else if (opt == rtc::Socket::OPT_RCVBUF) { + recvbuf_size_ = option; + } else if (opt == rtc::Socket::OPT_DSCP) { + dscp_ = static_cast<rtc::DiffServCodePoint>(option); + } + return 0; + } + + void PostMessage(int id, const rtc::CopyOnWriteBuffer& packet) { + thread_->Post(RTC_FROM_HERE, this, id, rtc::WrapMessageData(packet)); + } + + virtual void OnMessage(rtc::Message* msg) { + rtc::TypedMessageData<rtc::CopyOnWriteBuffer>* msg_data = + static_cast<rtc::TypedMessageData<rtc::CopyOnWriteBuffer>*>(msg->pdata); + if (dest_) { + if (msg->message_id == ST_RTP) { + dest_->OnPacketReceived(msg_data->data(), rtc::TimeMicros()); + } else { + RTC_LOG(LS_VERBOSE) << "Dropping RTCP packet, they not handled by " + "MediaChannel anymore."; + } + } + delete msg_data; + } + + private: + void GetNumRtpBytesAndPackets(uint32_t ssrc, int* bytes, int* packets) { + if (bytes) { + *bytes = 0; + } + if (packets) { + *packets = 0; + } + uint32_t cur_ssrc = 0; + for (size_t i = 0; i < rtp_packets_.size(); ++i) { + if (!GetRtpSsrc(rtp_packets_[i].data(), rtp_packets_[i].size(), + &cur_ssrc)) { + return; + } + if (ssrc == cur_ssrc) { + if (bytes) { + *bytes += static_cast<int>(rtp_packets_[i].size()); + } + if (packets) { + ++(*packets); + } + } + } + } + + rtc::Thread* thread_; + MediaChannel* dest_; + bool conf_; + // The ssrcs used in sending out packets in conference mode. + std::vector<uint32_t> conf_sent_ssrcs_; + // Map to track counts of packets that have been sent per ssrc. + // This includes packets that are dropped. + std::map<uint32_t, uint32_t> sent_ssrcs_; + // Map to track packet-number that needs to be dropped per ssrc. + std::map<uint32_t, std::set<uint32_t> > drop_map_; + rtc::CriticalSection crit_; + std::vector<rtc::CopyOnWriteBuffer> rtp_packets_; + std::vector<rtc::CopyOnWriteBuffer> rtcp_packets_; + int sendbuf_size_; + int recvbuf_size_; + rtc::DiffServCodePoint dscp_; + // Options of the most recently sent packet. + rtc::PacketOptions options_; +}; + +} // namespace cricket + +#endif // MEDIA_BASE_FAKE_NETWORK_INTERFACE_H_ diff --git a/media/base/fake_rtp.cc b/media/base/fake_rtp.cc new file mode 100644 index 0000000000..4f42821762 --- /dev/null +++ b/media/base/fake_rtp.cc @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "media/base/fake_rtp.h" + +#include <stdint.h> +#include <string.h> + +#include "absl/algorithm/container.h" +#include "rtc_base/checks.h" +#include "test/gtest.h" + +void CompareHeaderExtensions(const char* packet1, + size_t packet1_size, + const char* packet2, + size_t packet2_size, + const std::vector<int> encrypted_headers, + bool expect_equal) { + // Sanity check: packets must be large enough to contain the RTP header and + // extensions header. + RTC_CHECK_GE(packet1_size, 12 + 4); + RTC_CHECK_GE(packet2_size, 12 + 4); + // RTP extension headers are the same. + EXPECT_EQ(0, memcmp(packet1 + 12, packet2 + 12, 4)); + // Check for one-byte header extensions. + EXPECT_EQ('\xBE', packet1[12]); + EXPECT_EQ('\xDE', packet1[13]); + // Determine position and size of extension headers. + size_t extension_words = packet1[14] << 8 | packet1[15]; + const char* extension_data1 = packet1 + 12 + 4; + const char* extension_end1 = extension_data1 + extension_words * 4; + const char* extension_data2 = packet2 + 12 + 4; + // Sanity check: packets must be large enough to contain the RTP header + // extensions. + RTC_CHECK_GE(packet1_size, 12 + 4 + extension_words * 4); + RTC_CHECK_GE(packet2_size, 12 + 4 + extension_words * 4); + while (extension_data1 < extension_end1) { + uint8_t id = (*extension_data1 & 0xf0) >> 4; + uint8_t len = (*extension_data1 & 0x0f) + 1; + extension_data1++; + extension_data2++; + EXPECT_LE(extension_data1, extension_end1); + if (id == 15) { + // Finished parsing. + break; + } + + // The header extension doesn't get encrypted if the id is not in the + // list of header extensions to encrypt. + if (expect_equal || !absl::c_linear_search(encrypted_headers, id)) { + EXPECT_EQ(0, memcmp(extension_data1, extension_data2, len)); + } else { + EXPECT_NE(0, memcmp(extension_data1, extension_data2, len)); + } + + extension_data1 += len; + extension_data2 += len; + // Skip padding. + while (extension_data1 < extension_end1 && *extension_data1 == 0) { + extension_data1++; + extension_data2++; + } + } +} diff --git a/media/base/fake_rtp.h b/media/base/fake_rtp.h new file mode 100644 index 0000000000..f2578151ed --- /dev/null +++ b/media/base/fake_rtp.h @@ -0,0 +1,301 @@ +/* + * Copyright (c) 2004 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// Fake RTP and RTCP packets to use in unit tests. + +#ifndef MEDIA_BASE_FAKE_RTP_H_ +#define MEDIA_BASE_FAKE_RTP_H_ + +#include <cstddef> // size_t +#include <vector> + +// A typical PCMU RTP packet. +// PT=0, SN=1, TS=0, SSRC=1 +// all data FF +static const unsigned char kPcmuFrame[] = { + 0x80, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, +}; + +static const int kHeaderExtensionIDs[] = {1, 4}; + +// A typical PCMU RTP packet with header extensions. +// PT=0, SN=1, TS=0, SSRC=1 +// all data FF +static const unsigned char kPcmuFrameWithExtensions[] = { + 0x90, + 0x00, + 0x00, + 0x01, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x01, + // RFC 5285, section 4.2. One-Byte Header. + 0xBE, + 0xDE, + // Header extension length 6 * 32 bits. + 0x00, + 0x06, + // 8 bytes header id 1. + 0x17, + 0x41, + 0x42, + 0x73, + 0xA4, + 0x75, + 0x26, + 0x27, + 0x48, + // 3 bytes header id 2. + 0x22, + 0x00, + 0x00, + 0xC8, + // 1 byte header id 3. + 0x30, + 0x8E, + // 7 bytes header id 4. + 0x46, + 0x55, + 0x99, + 0x63, + 0x86, + 0xB3, + 0x95, + 0xFB, + // 1 byte header padding. + 0x00, + // Payload data. + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, +}; + +// A typical Receiver Report RTCP packet. +// PT=RR, LN=1, SSRC=1 +// send SSRC=2, all other fields 0 +static const unsigned char kRtcpReport[] = { + 0x80, 0xc9, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; + +// PT = 97, TS = 0, Seq = 1, SSRC = 2 +// H264 - NRI = 1, Type = 1, bit stream = FF + +static const unsigned char kH264Packet[] = { + 0x80, 0x61, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, + 0x21, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, +}; + +// PT= 101, SN=2, TS=3, SSRC = 4 +static const unsigned char kDataPacket[] = { + 0x80, 0x65, 0x00, 0x02, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x04, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, +}; + +// This expects both packets to be based on kPcmuFrameWithExtensions. +// Header extensions with an id in "encrypted_headers" are expected to be +// different in the packets unless "expect_equal" is set to "true". +void CompareHeaderExtensions(const char* packet1, + size_t packet1_size, + const char* packet2, + size_t packet2_size, + const std::vector<int> encrypted_headers, + bool expect_equal); + +#endif // MEDIA_BASE_FAKE_RTP_H_ diff --git a/media/base/fake_video_renderer.cc b/media/base/fake_video_renderer.cc new file mode 100644 index 0000000000..801f81dec4 --- /dev/null +++ b/media/base/fake_video_renderer.cc @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "media/base/fake_video_renderer.h" + +namespace cricket { + +FakeVideoRenderer::FakeVideoRenderer() = default; + +void FakeVideoRenderer::OnFrame(const webrtc::VideoFrame& frame) { + rtc::CritScope cs(&crit_); + // TODO(zhurunz) Check with VP8 team to see if we can remove this + // tolerance on Y values. Some unit tests produce Y values close + // to 16 rather than close to zero, for supposedly black frames. + // Largest value observed is 34, e.g., running + // PeerConnectionIntegrationTest.SendAndReceive16To9AspectRatio. + black_frame_ = CheckFrameColorYuv(0, 48, 128, 128, 128, 128, &frame); + // Treat unexpected frame size as error. + ++num_rendered_frames_; + width_ = frame.width(); + height_ = frame.height(); + rotation_ = frame.rotation(); + timestamp_us_ = frame.timestamp_us(); + ntp_timestamp_ms_ = frame.ntp_time_ms(); + color_space_ = frame.color_space(); + packet_infos_ = frame.packet_infos(); + frame_rendered_event_.Set(); +} + +bool FakeVideoRenderer::WaitForRenderedFrame(int64_t timeout_ms) { + return frame_rendered_event_.Wait(timeout_ms); +} + +} // namespace cricket diff --git a/media/base/fake_video_renderer.h b/media/base/fake_video_renderer.h new file mode 100644 index 0000000000..ba67bf0332 --- /dev/null +++ b/media/base/fake_video_renderer.h @@ -0,0 +1,151 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MEDIA_BASE_FAKE_VIDEO_RENDERER_H_ +#define MEDIA_BASE_FAKE_VIDEO_RENDERER_H_ + +#include <stdint.h> + +#include "api/scoped_refptr.h" +#include "api/video/video_frame.h" +#include "api/video/video_frame_buffer.h" +#include "api/video/video_rotation.h" +#include "api/video/video_sink_interface.h" +#include "rtc_base/critical_section.h" +#include "rtc_base/event.h" + +namespace cricket { + +// Faked video renderer that has a callback for actions on rendering. +class FakeVideoRenderer : public rtc::VideoSinkInterface<webrtc::VideoFrame> { + public: + FakeVideoRenderer(); + + void OnFrame(const webrtc::VideoFrame& frame) override; + + int errors() const { return errors_; } + + int width() const { + rtc::CritScope cs(&crit_); + return width_; + } + int height() const { + rtc::CritScope cs(&crit_); + return height_; + } + + webrtc::VideoRotation rotation() const { + rtc::CritScope cs(&crit_); + return rotation_; + } + + int64_t timestamp_us() const { + rtc::CritScope cs(&crit_); + return timestamp_us_; + } + + int num_rendered_frames() const { + rtc::CritScope cs(&crit_); + return num_rendered_frames_; + } + + bool black_frame() const { + rtc::CritScope cs(&crit_); + return black_frame_; + } + + int64_t ntp_time_ms() const { + rtc::CritScope cs(&crit_); + return ntp_timestamp_ms_; + } + + absl::optional<webrtc::ColorSpace> color_space() const { + rtc::CritScope cs(&crit_); + return color_space_; + } + + webrtc::RtpPacketInfos packet_infos() const { + rtc::CritScope cs(&crit_); + return packet_infos_; + } + + bool WaitForRenderedFrame(int64_t timeout_ms); + + private: + static bool CheckFrameColorYuv(uint8_t y_min, + uint8_t y_max, + uint8_t u_min, + uint8_t u_max, + uint8_t v_min, + uint8_t v_max, + const webrtc::VideoFrame* frame) { + if (!frame || !frame->video_frame_buffer()) { + return false; + } + rtc::scoped_refptr<const webrtc::I420BufferInterface> i420_buffer = + frame->video_frame_buffer()->ToI420(); + // Y + int y_width = frame->width(); + int y_height = frame->height(); + const uint8_t* y_plane = i420_buffer->DataY(); + const uint8_t* y_pos = y_plane; + int32_t y_pitch = i420_buffer->StrideY(); + for (int i = 0; i < y_height; ++i) { + for (int j = 0; j < y_width; ++j) { + uint8_t y_value = *(y_pos + j); + if (y_value < y_min || y_value > y_max) { + return false; + } + } + y_pos += y_pitch; + } + // U and V + int chroma_width = i420_buffer->ChromaWidth(); + int chroma_height = i420_buffer->ChromaHeight(); + const uint8_t* u_plane = i420_buffer->DataU(); + const uint8_t* v_plane = i420_buffer->DataV(); + const uint8_t* u_pos = u_plane; + const uint8_t* v_pos = v_plane; + int32_t u_pitch = i420_buffer->StrideU(); + int32_t v_pitch = i420_buffer->StrideV(); + for (int i = 0; i < chroma_height; ++i) { + for (int j = 0; j < chroma_width; ++j) { + uint8_t u_value = *(u_pos + j); + if (u_value < u_min || u_value > u_max) { + return false; + } + uint8_t v_value = *(v_pos + j); + if (v_value < v_min || v_value > v_max) { + return false; + } + } + u_pos += u_pitch; + v_pos += v_pitch; + } + return true; + } + + int errors_ = 0; + int width_ = 0; + int height_ = 0; + webrtc::VideoRotation rotation_ = webrtc::kVideoRotation_0; + int64_t timestamp_us_ = 0; + int num_rendered_frames_ = 0; + int64_t ntp_timestamp_ms_ = 0; + bool black_frame_ = false; + rtc::CriticalSection crit_; + rtc::Event frame_rendered_event_; + absl::optional<webrtc::ColorSpace> color_space_; + webrtc::RtpPacketInfos packet_infos_; +}; + +} // namespace cricket + +#endif // MEDIA_BASE_FAKE_VIDEO_RENDERER_H_ diff --git a/media/base/h264_profile_level_id.cc b/media/base/h264_profile_level_id.cc new file mode 100644 index 0000000000..32fa02c143 --- /dev/null +++ b/media/base/h264_profile_level_id.cc @@ -0,0 +1,311 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "media/base/h264_profile_level_id.h" + +#include <cstdio> +#include <cstdlib> +#include <cstring> + +#include "rtc_base/arraysize.h" +#include "rtc_base/checks.h" + +namespace webrtc { +namespace H264 { + +namespace { + +const char kProfileLevelId[] = "profile-level-id"; +const char kLevelAsymmetryAllowed[] = "level-asymmetry-allowed"; + +// For level_idc=11 and profile_idc=0x42, 0x4D, or 0x58, the constraint set3 +// flag specifies if level 1b or level 1.1 is used. +const uint8_t kConstraintSet3Flag = 0x10; + +// Convert a string of 8 characters into a byte where the positions containing +// character c will have their bit set. For example, c = 'x', str = "x1xx0000" +// will return 0b10110000. constexpr is used so that the pattern table in +// kProfilePatterns is statically initialized. +constexpr uint8_t ByteMaskString(char c, const char (&str)[9]) { + return (str[0] == c) << 7 | (str[1] == c) << 6 | (str[2] == c) << 5 | + (str[3] == c) << 4 | (str[4] == c) << 3 | (str[5] == c) << 2 | + (str[6] == c) << 1 | (str[7] == c) << 0; +} + +// Class for matching bit patterns such as "x1xx0000" where 'x' is allowed to be +// either 0 or 1. +class BitPattern { + public: + explicit constexpr BitPattern(const char (&str)[9]) + : mask_(~ByteMaskString('x', str)), + masked_value_(ByteMaskString('1', str)) {} + + bool IsMatch(uint8_t value) const { return masked_value_ == (value & mask_); } + + private: + const uint8_t mask_; + const uint8_t masked_value_; +}; + +// Table for converting between profile_idc/profile_iop to H264::Profile. +struct ProfilePattern { + const uint8_t profile_idc; + const BitPattern profile_iop; + const Profile profile; +}; + +// This is from https://tools.ietf.org/html/rfc6184#section-8.1. +constexpr ProfilePattern kProfilePatterns[] = { + {0x42, BitPattern("x1xx0000"), kProfileConstrainedBaseline}, + {0x4D, BitPattern("1xxx0000"), kProfileConstrainedBaseline}, + {0x58, BitPattern("11xx0000"), kProfileConstrainedBaseline}, + {0x42, BitPattern("x0xx0000"), kProfileBaseline}, + {0x58, BitPattern("10xx0000"), kProfileBaseline}, + {0x4D, BitPattern("0x0x0000"), kProfileMain}, + {0x64, BitPattern("00000000"), kProfileHigh}, + {0x64, BitPattern("00001100"), kProfileConstrainedHigh}}; + +// Compare H264 levels and handle the level 1b case. +bool IsLess(Level a, Level b) { + if (a == kLevel1_b) + return b != kLevel1 && b != kLevel1_b; + if (b == kLevel1_b) + return a == kLevel1; + return a < b; +} + +Level Min(Level a, Level b) { + return IsLess(a, b) ? a : b; +} + +bool IsLevelAsymmetryAllowed(const CodecParameterMap& params) { + const auto it = params.find(kLevelAsymmetryAllowed); + return it != params.end() && strcmp(it->second.c_str(), "1") == 0; +} + +struct LevelConstraint { + const int max_macroblocks_per_second; + const int max_macroblock_frame_size; + const webrtc::H264::Level level; +}; + +// This is from ITU-T H.264 (02/2016) Table A-1 – Level limits. +static constexpr LevelConstraint kLevelConstraints[] = { + {1485, 99, webrtc::H264::kLevel1}, + {1485, 99, webrtc::H264::kLevel1_b}, + {3000, 396, webrtc::H264::kLevel1_1}, + {6000, 396, webrtc::H264::kLevel1_2}, + {11880, 396, webrtc::H264::kLevel1_3}, + {11880, 396, webrtc::H264::kLevel2}, + {19800, 792, webrtc::H264::kLevel2_1}, + {20250, 1620, webrtc::H264::kLevel2_2}, + {40500, 1620, webrtc::H264::kLevel3}, + {108000, 3600, webrtc::H264::kLevel3_1}, + {216000, 5120, webrtc::H264::kLevel3_2}, + {245760, 8192, webrtc::H264::kLevel4}, + {245760, 8192, webrtc::H264::kLevel4_1}, + {522240, 8704, webrtc::H264::kLevel4_2}, + {589824, 22080, webrtc::H264::kLevel5}, + {983040, 36864, webrtc::H264::kLevel5_1}, + {2073600, 36864, webrtc::H264::kLevel5_2}, +}; + +} // anonymous namespace + +absl::optional<ProfileLevelId> ParseProfileLevelId(const char* str) { + // The string should consist of 3 bytes in hexadecimal format. + if (strlen(str) != 6u) + return absl::nullopt; + const uint32_t profile_level_id_numeric = strtol(str, nullptr, 16); + if (profile_level_id_numeric == 0) + return absl::nullopt; + + // Separate into three bytes. + const uint8_t level_idc = + static_cast<uint8_t>(profile_level_id_numeric & 0xFF); + const uint8_t profile_iop = + static_cast<uint8_t>((profile_level_id_numeric >> 8) & 0xFF); + const uint8_t profile_idc = + static_cast<uint8_t>((profile_level_id_numeric >> 16) & 0xFF); + + // Parse level based on level_idc and constraint set 3 flag. + Level level; + switch (level_idc) { + case kLevel1_1: + level = (profile_iop & kConstraintSet3Flag) != 0 ? kLevel1_b : kLevel1_1; + break; + case kLevel1: + case kLevel1_2: + case kLevel1_3: + case kLevel2: + case kLevel2_1: + case kLevel2_2: + case kLevel3: + case kLevel3_1: + case kLevel3_2: + case kLevel4: + case kLevel4_1: + case kLevel4_2: + case kLevel5: + case kLevel5_1: + case kLevel5_2: + level = static_cast<Level>(level_idc); + break; + default: + // Unrecognized level_idc. + return absl::nullopt; + } + + // Parse profile_idc/profile_iop into a Profile enum. + for (const ProfilePattern& pattern : kProfilePatterns) { + if (profile_idc == pattern.profile_idc && + pattern.profile_iop.IsMatch(profile_iop)) { + return ProfileLevelId(pattern.profile, level); + } + } + + // Unrecognized profile_idc/profile_iop combination. + return absl::nullopt; +} + +absl::optional<Level> SupportedLevel(int max_frame_pixel_count, float max_fps) { + static const int kPixelsPerMacroblock = 16 * 16; + + for (int i = arraysize(kLevelConstraints) - 1; i >= 0; --i) { + const LevelConstraint& level_constraint = kLevelConstraints[i]; + if (level_constraint.max_macroblock_frame_size * kPixelsPerMacroblock <= + max_frame_pixel_count && + level_constraint.max_macroblocks_per_second <= + max_fps * level_constraint.max_macroblock_frame_size) { + return level_constraint.level; + } + } + + // No level supported. + return absl::nullopt; +} + +absl::optional<ProfileLevelId> ParseSdpProfileLevelId( + const CodecParameterMap& params) { + // TODO(magjed): The default should really be kProfileBaseline and kLevel1 + // according to the spec: https://tools.ietf.org/html/rfc6184#section-8.1. In + // order to not break backwards compatibility with older versions of WebRTC + // where external codecs don't have any parameters, use + // kProfileConstrainedBaseline kLevel3_1 instead. This workaround will only be + // done in an interim period to allow external clients to update their code. + // http://crbug/webrtc/6337. + static const ProfileLevelId kDefaultProfileLevelId( + kProfileConstrainedBaseline, kLevel3_1); + + const auto profile_level_id_it = params.find(kProfileLevelId); + return (profile_level_id_it == params.end()) + ? kDefaultProfileLevelId + : ParseProfileLevelId(profile_level_id_it->second.c_str()); +} + +absl::optional<std::string> ProfileLevelIdToString( + const ProfileLevelId& profile_level_id) { + // Handle special case level == 1b. + if (profile_level_id.level == kLevel1_b) { + switch (profile_level_id.profile) { + case kProfileConstrainedBaseline: + return {"42f00b"}; + case kProfileBaseline: + return {"42100b"}; + case kProfileMain: + return {"4d100b"}; + // Level 1b is not allowed for other profiles. + default: + return absl::nullopt; + } + } + + const char* profile_idc_iop_string; + switch (profile_level_id.profile) { + case kProfileConstrainedBaseline: + profile_idc_iop_string = "42e0"; + break; + case kProfileBaseline: + profile_idc_iop_string = "4200"; + break; + case kProfileMain: + profile_idc_iop_string = "4d00"; + break; + case kProfileConstrainedHigh: + profile_idc_iop_string = "640c"; + break; + case kProfileHigh: + profile_idc_iop_string = "6400"; + break; + // Unrecognized profile. + default: + return absl::nullopt; + } + + char str[7]; + snprintf(str, 7u, "%s%02x", profile_idc_iop_string, profile_level_id.level); + return {str}; +} + +// Set level according to https://tools.ietf.org/html/rfc6184#section-8.2.2. +void GenerateProfileLevelIdForAnswer( + const CodecParameterMap& local_supported_params, + const CodecParameterMap& remote_offered_params, + CodecParameterMap* answer_params) { + // If both local and remote haven't set profile-level-id, they are both using + // the default profile. In this case, don't set profile-level-id in answer + // either. + if (!local_supported_params.count(kProfileLevelId) && + !remote_offered_params.count(kProfileLevelId)) { + return; + } + + // Parse profile-level-ids. + const absl::optional<ProfileLevelId> local_profile_level_id = + ParseSdpProfileLevelId(local_supported_params); + const absl::optional<ProfileLevelId> remote_profile_level_id = + ParseSdpProfileLevelId(remote_offered_params); + // The local and remote codec must have valid and equal H264 Profiles. + RTC_DCHECK(local_profile_level_id); + RTC_DCHECK(remote_profile_level_id); + RTC_DCHECK_EQ(local_profile_level_id->profile, + remote_profile_level_id->profile); + + // Parse level information. + const bool level_asymmetry_allowed = + IsLevelAsymmetryAllowed(local_supported_params) && + IsLevelAsymmetryAllowed(remote_offered_params); + const Level local_level = local_profile_level_id->level; + const Level remote_level = remote_profile_level_id->level; + const Level min_level = Min(local_level, remote_level); + + // Determine answer level. When level asymmetry is not allowed, level upgrade + // is not allowed, i.e., the level in the answer must be equal to or lower + // than the level in the offer. + const Level answer_level = level_asymmetry_allowed ? local_level : min_level; + + // Set the resulting profile-level-id in the answer parameters. + (*answer_params)[kProfileLevelId] = *ProfileLevelIdToString( + ProfileLevelId(local_profile_level_id->profile, answer_level)); +} + +bool IsSameH264Profile(const CodecParameterMap& params1, + const CodecParameterMap& params2) { + const absl::optional<webrtc::H264::ProfileLevelId> profile_level_id = + webrtc::H264::ParseSdpProfileLevelId(params1); + const absl::optional<webrtc::H264::ProfileLevelId> other_profile_level_id = + webrtc::H264::ParseSdpProfileLevelId(params2); + // Compare H264 profiles, but not levels. + return profile_level_id && other_profile_level_id && + profile_level_id->profile == other_profile_level_id->profile; +} + +} // namespace H264 +} // namespace webrtc diff --git a/media/base/h264_profile_level_id.h b/media/base/h264_profile_level_id.h new file mode 100644 index 0000000000..6a3e7cdb32 --- /dev/null +++ b/media/base/h264_profile_level_id.h @@ -0,0 +1,109 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MEDIA_BASE_H264_PROFILE_LEVEL_ID_H_ +#define MEDIA_BASE_H264_PROFILE_LEVEL_ID_H_ + +#include <map> +#include <string> + +#include "absl/types/optional.h" +#include "common_types.h" // NOLINT(build/include) +#include "rtc_base/system/rtc_export.h" + +namespace webrtc { +namespace H264 { + +// Map containting SDP codec parameters. +typedef std::map<std::string, std::string> CodecParameterMap; + +// All values are equal to ten times the level number, except level 1b which is +// special. +enum Level { + kLevel1_b = 0, + kLevel1 = 10, + kLevel1_1 = 11, + kLevel1_2 = 12, + kLevel1_3 = 13, + kLevel2 = 20, + kLevel2_1 = 21, + kLevel2_2 = 22, + kLevel3 = 30, + kLevel3_1 = 31, + kLevel3_2 = 32, + kLevel4 = 40, + kLevel4_1 = 41, + kLevel4_2 = 42, + kLevel5 = 50, + kLevel5_1 = 51, + kLevel5_2 = 52 +}; + +struct ProfileLevelId { + constexpr ProfileLevelId(Profile profile, Level level) + : profile(profile), level(level) {} + Profile profile; + Level level; +}; + +// Parse profile level id that is represented as a string of 3 hex bytes. +// Nothing will be returned if the string is not a recognized H264 +// profile level id. +absl::optional<ProfileLevelId> ParseProfileLevelId(const char* str); + +// Parse profile level id that is represented as a string of 3 hex bytes +// contained in an SDP key-value map. A default profile level id will be +// returned if the profile-level-id key is missing. Nothing will be returned if +// the key is present but the string is invalid. +RTC_EXPORT absl::optional<ProfileLevelId> ParseSdpProfileLevelId( + const CodecParameterMap& params); + +// Given that a decoder supports up to a given frame size (in pixels) at up to a +// given number of frames per second, return the highest H.264 level where it +// can guarantee that it will be able to support all valid encoded streams that +// are within that level. +RTC_EXPORT absl::optional<Level> SupportedLevel(int max_frame_pixel_count, + float max_fps); + +// Returns canonical string representation as three hex bytes of the profile +// level id, or returns nothing for invalid profile level ids. +RTC_EXPORT absl::optional<std::string> ProfileLevelIdToString( + const ProfileLevelId& profile_level_id); + +// Generate codec parameters that will be used as answer in an SDP negotiation +// based on local supported parameters and remote offered parameters. Both +// |local_supported_params|, |remote_offered_params|, and |answer_params| +// represent sendrecv media descriptions, i.e they are a mix of both encode and +// decode capabilities. In theory, when the profile in |local_supported_params| +// represent a strict superset of the profile in |remote_offered_params|, we +// could limit the profile in |answer_params| to the profile in +// |remote_offered_params|. However, to simplify the code, each supported H264 +// profile should be listed explicitly in the list of local supported codecs, +// even if they are redundant. Then each local codec in the list should be +// tested one at a time against the remote codec, and only when the profiles are +// equal should this function be called. Therefore, this function does not need +// to handle profile intersection, and the profile of |local_supported_params| +// and |remote_offered_params| must be equal before calling this function. The +// parameters that are used when negotiating are the level part of +// profile-level-id and level-asymmetry-allowed. +void GenerateProfileLevelIdForAnswer( + const CodecParameterMap& local_supported_params, + const CodecParameterMap& remote_offered_params, + CodecParameterMap* answer_params); + +// Returns true if the parameters have the same H264 profile, i.e. the same +// H264::Profile (Baseline, High, etc). +bool IsSameH264Profile(const CodecParameterMap& params1, + const CodecParameterMap& params2); + +} // namespace H264 +} // namespace webrtc + +#endif // MEDIA_BASE_H264_PROFILE_LEVEL_ID_H_ diff --git a/media/base/media_channel.cc b/media/base/media_channel.cc new file mode 100644 index 0000000000..2e9bfc3d31 --- /dev/null +++ b/media/base/media_channel.cc @@ -0,0 +1,140 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "media/base/media_channel.h" + +namespace cricket { + +VideoOptions::VideoOptions() + : content_hint(webrtc::VideoTrackInterface::ContentHint::kNone) {} +VideoOptions::~VideoOptions() = default; + +MediaChannel::MediaChannel(const MediaConfig& config) + : enable_dscp_(config.enable_dscp) {} + +MediaChannel::MediaChannel() : enable_dscp_(false) {} + +MediaChannel::~MediaChannel() {} + +void MediaChannel::SetInterface( + NetworkInterface* iface, + const webrtc::MediaTransportConfig& media_transport_config) { + rtc::CritScope cs(&network_interface_crit_); + network_interface_ = iface; + media_transport_config_ = media_transport_config; + UpdateDscp(); +} + +int MediaChannel::GetRtpSendTimeExtnId() const { + return -1; +} + +void MediaChannel::SetFrameEncryptor( + uint32_t ssrc, + rtc::scoped_refptr<webrtc::FrameEncryptorInterface> frame_encryptor) { + // Placeholder should be pure virtual once internal supports it. +} + +void MediaChannel::SetFrameDecryptor( + uint32_t ssrc, + rtc::scoped_refptr<webrtc::FrameDecryptorInterface> frame_decryptor) { + // Placeholder should be pure virtual once internal supports it. +} + +void MediaChannel::SetVideoCodecSwitchingEnabled(bool enabled) {} + +void MediaChannel::SetEncoderToPacketizerFrameTransformer( + uint32_t ssrc, + rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer) {} +void MediaChannel::SetDepacketizerToDecoderFrameTransformer( + uint32_t ssrc, + rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer) {} + +MediaSenderInfo::MediaSenderInfo() = default; +MediaSenderInfo::~MediaSenderInfo() = default; + +MediaReceiverInfo::MediaReceiverInfo() = default; +MediaReceiverInfo::~MediaReceiverInfo() = default; + +VoiceSenderInfo::VoiceSenderInfo() = default; +VoiceSenderInfo::~VoiceSenderInfo() = default; + +VoiceReceiverInfo::VoiceReceiverInfo() = default; +VoiceReceiverInfo::~VoiceReceiverInfo() = default; + +VideoSenderInfo::VideoSenderInfo() = default; +VideoSenderInfo::~VideoSenderInfo() = default; + +VideoReceiverInfo::VideoReceiverInfo() = default; +VideoReceiverInfo::~VideoReceiverInfo() = default; + +VoiceMediaInfo::VoiceMediaInfo() = default; +VoiceMediaInfo::~VoiceMediaInfo() = default; + +VideoMediaInfo::VideoMediaInfo() = default; +VideoMediaInfo::~VideoMediaInfo() = default; + +DataMediaInfo::DataMediaInfo() = default; +DataMediaInfo::~DataMediaInfo() = default; + +AudioSendParameters::AudioSendParameters() = default; +AudioSendParameters::~AudioSendParameters() = default; + +std::map<std::string, std::string> AudioSendParameters::ToStringMap() const { + auto params = RtpSendParameters<AudioCodec>::ToStringMap(); + params["options"] = options.ToString(); + return params; +} + +cricket::MediaType VoiceMediaChannel::media_type() const { + return cricket::MediaType::MEDIA_TYPE_AUDIO; +} + +VideoSendParameters::VideoSendParameters() = default; +VideoSendParameters::~VideoSendParameters() = default; + +std::map<std::string, std::string> VideoSendParameters::ToStringMap() const { + auto params = RtpSendParameters<VideoCodec>::ToStringMap(); + params["conference_mode"] = (conference_mode ? "yes" : "no"); + return params; +} + +cricket::MediaType VideoMediaChannel::media_type() const { + return cricket::MediaType::MEDIA_TYPE_VIDEO; +} + +DataMediaChannel::DataMediaChannel() = default; +DataMediaChannel::DataMediaChannel(const MediaConfig& config) + : MediaChannel(config) {} +DataMediaChannel::~DataMediaChannel() = default; + +webrtc::RtpParameters DataMediaChannel::GetRtpSendParameters( + uint32_t ssrc) const { + // GetRtpSendParameters is not supported for DataMediaChannel. + RTC_NOTREACHED(); + return webrtc::RtpParameters(); +} +webrtc::RTCError DataMediaChannel::SetRtpSendParameters( + uint32_t ssrc, + const webrtc::RtpParameters& parameters) { + // SetRtpSendParameters is not supported for DataMediaChannel. + RTC_NOTREACHED(); + return webrtc::RTCError(webrtc::RTCErrorType::UNSUPPORTED_OPERATION); +} + +cricket::MediaType DataMediaChannel::media_type() const { + return cricket::MediaType::MEDIA_TYPE_DATA; +} + +bool DataMediaChannel::GetStats(DataMediaInfo* info) { + return true; +} + +} // namespace cricket diff --git a/media/base/media_channel.h b/media/base/media_channel.h new file mode 100644 index 0000000000..d71ec9158a --- /dev/null +++ b/media/base/media_channel.h @@ -0,0 +1,1040 @@ +/* + * Copyright (c) 2004 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MEDIA_BASE_MEDIA_CHANNEL_H_ +#define MEDIA_BASE_MEDIA_CHANNEL_H_ + +#include <map> +#include <memory> +#include <string> +#include <utility> +#include <vector> + +#include "absl/types/optional.h" +#include "api/audio_codecs/audio_encoder.h" +#include "api/audio_options.h" +#include "api/crypto/frame_decryptor_interface.h" +#include "api/crypto/frame_encryptor_interface.h" +#include "api/frame_transformer_interface.h" +#include "api/media_stream_interface.h" +#include "api/rtc_error.h" +#include "api/rtp_parameters.h" +#include "api/transport/media/media_transport_config.h" +#include "api/transport/rtp/rtp_source.h" +#include "api/video/video_content_type.h" +#include "api/video/video_sink_interface.h" +#include "api/video/video_source_interface.h" +#include "api/video/video_timing.h" +#include "api/video_codecs/video_encoder_config.h" +#include "call/video_receive_stream.h" +#include "common_video/include/quality_limitation_reason.h" +#include "media/base/codec.h" +#include "media/base/delayable.h" +#include "media/base/media_config.h" +#include "media/base/media_constants.h" +#include "media/base/stream_params.h" +#include "modules/audio_processing/include/audio_processing_statistics.h" +#include "modules/rtp_rtcp/include/report_block_data.h" +#include "rtc_base/async_packet_socket.h" +#include "rtc_base/buffer.h" +#include "rtc_base/callback.h" +#include "rtc_base/copy_on_write_buffer.h" +#include "rtc_base/critical_section.h" +#include "rtc_base/dscp.h" +#include "rtc_base/logging.h" +#include "rtc_base/network_route.h" +#include "rtc_base/socket.h" +#include "rtc_base/string_encode.h" +#include "rtc_base/strings/string_builder.h" +#include "rtc_base/third_party/sigslot/sigslot.h" + +namespace rtc { +class Timing; +} + +namespace webrtc { +class AudioSinkInterface; +class VideoFrame; +} // namespace webrtc + +namespace cricket { + +class AudioSource; +class VideoCapturer; +struct RtpHeader; +struct VideoFormat; + +const int kScreencastDefaultFps = 5; + +template <class T> +static std::string ToStringIfSet(const char* key, + const absl::optional<T>& val) { + std::string str; + if (val) { + str = key; + str += ": "; + str += val ? rtc::ToString(*val) : ""; + str += ", "; + } + return str; +} + +template <class T> +static std::string VectorToString(const std::vector<T>& vals) { + rtc::StringBuilder ost; // no-presubmit-check TODO(webrtc:8982) + ost << "["; + for (size_t i = 0; i < vals.size(); ++i) { + if (i > 0) { + ost << ", "; + } + ost << vals[i].ToString(); + } + ost << "]"; + return ost.Release(); +} + +// Options that can be applied to a VideoMediaChannel or a VideoMediaEngine. +// Used to be flags, but that makes it hard to selectively apply options. +// We are moving all of the setting of options to structs like this, +// but some things currently still use flags. +struct VideoOptions { + VideoOptions(); + ~VideoOptions(); + + void SetAll(const VideoOptions& change) { + SetFrom(&video_noise_reduction, change.video_noise_reduction); + SetFrom(&screencast_min_bitrate_kbps, change.screencast_min_bitrate_kbps); + SetFrom(&is_screencast, change.is_screencast); + } + + bool operator==(const VideoOptions& o) const { + return video_noise_reduction == o.video_noise_reduction && + screencast_min_bitrate_kbps == o.screencast_min_bitrate_kbps && + is_screencast == o.is_screencast; + } + bool operator!=(const VideoOptions& o) const { return !(*this == o); } + + std::string ToString() const { + rtc::StringBuilder ost; + ost << "VideoOptions {"; + ost << ToStringIfSet("noise reduction", video_noise_reduction); + ost << ToStringIfSet("screencast min bitrate kbps", + screencast_min_bitrate_kbps); + ost << ToStringIfSet("is_screencast ", is_screencast); + ost << "}"; + return ost.Release(); + } + + // Enable denoising? This flag comes from the getUserMedia + // constraint 'googNoiseReduction', and WebRtcVideoEngine passes it + // on to the codec options. Disabled by default. + absl::optional<bool> video_noise_reduction; + // Force screencast to use a minimum bitrate. This flag comes from + // the PeerConnection constraint 'googScreencastMinBitrate'. It is + // copied to the encoder config by WebRtcVideoChannel. + absl::optional<int> screencast_min_bitrate_kbps; + // Set by screencast sources. Implies selection of encoding settings + // suitable for screencast. Most likely not the right way to do + // things, e.g., screencast of a text document and screencast of a + // youtube video have different needs. + absl::optional<bool> is_screencast; + webrtc::VideoTrackInterface::ContentHint content_hint; + + private: + template <typename T> + static void SetFrom(absl::optional<T>* s, const absl::optional<T>& o) { + if (o) { + *s = o; + } + } +}; + +// TODO(isheriff): Remove this once client usage is fixed to use RtpExtension. +struct RtpHeaderExtension { + RtpHeaderExtension() : id(0) {} + RtpHeaderExtension(const std::string& uri, int id) : uri(uri), id(id) {} + + std::string ToString() const { + rtc::StringBuilder ost; + ost << "{"; + ost << "uri: " << uri; + ost << ", id: " << id; + ost << "}"; + return ost.Release(); + } + + std::string uri; + int id; +}; + +class MediaChannel : public sigslot::has_slots<> { + public: + class NetworkInterface { + public: + enum SocketType { ST_RTP, ST_RTCP }; + virtual bool SendPacket(rtc::CopyOnWriteBuffer* packet, + const rtc::PacketOptions& options) = 0; + virtual bool SendRtcp(rtc::CopyOnWriteBuffer* packet, + const rtc::PacketOptions& options) = 0; + virtual int SetOption(SocketType type, + rtc::Socket::Option opt, + int option) = 0; + virtual ~NetworkInterface() {} + }; + + explicit MediaChannel(const MediaConfig& config); + MediaChannel(); + ~MediaChannel() override; + + virtual cricket::MediaType media_type() const = 0; + + // Sets the abstract interface class for sending RTP/RTCP data and + // interface for media transport (experimental). If media transport is + // provided, it should be used instead of RTP/RTCP. + // TODO(sukhanov): Currently media transport can co-exist with RTP/RTCP, but + // in the future we will refactor code to send all frames with media + // transport. + virtual void SetInterface( + NetworkInterface* iface, + const webrtc::MediaTransportConfig& media_transport_config); + // Called when a RTP packet is received. + virtual void OnPacketReceived(rtc::CopyOnWriteBuffer packet, + int64_t packet_time_us) = 0; + // Called when the socket's ability to send has changed. + virtual void OnReadyToSend(bool ready) = 0; + // Called when the network route used for sending packets changed. + virtual void OnNetworkRouteChanged( + const std::string& transport_name, + const rtc::NetworkRoute& network_route) = 0; + // Creates a new outgoing media stream with SSRCs and CNAME as described + // by sp. + virtual bool AddSendStream(const StreamParams& sp) = 0; + // Removes an outgoing media stream. + // SSRC must be the first SSRC of the media stream if the stream uses + // multiple SSRCs. In the case of an ssrc of 0, the possibly cached + // StreamParams is removed. + virtual bool RemoveSendStream(uint32_t ssrc) = 0; + // Creates a new incoming media stream with SSRCs, CNAME as described + // by sp. In the case of a sp without SSRCs, the unsignaled sp is cached + // to be used later for unsignaled streams received. + virtual bool AddRecvStream(const StreamParams& sp) = 0; + // Removes an incoming media stream. + // ssrc must be the first SSRC of the media stream if the stream uses + // multiple SSRCs. + virtual bool RemoveRecvStream(uint32_t ssrc) = 0; + // Resets any cached StreamParams for an unsignaled RecvStream. + virtual void ResetUnsignaledRecvStream() = 0; + // Returns the absoulte sendtime extension id value from media channel. + virtual int GetRtpSendTimeExtnId() const; + // Set the frame encryptor to use on all outgoing frames. This is optional. + // This pointers lifetime is managed by the set of RtpSender it is attached + // to. + // TODO(benwright) make pure virtual once internal supports it. + virtual void SetFrameEncryptor( + uint32_t ssrc, + rtc::scoped_refptr<webrtc::FrameEncryptorInterface> frame_encryptor); + // Set the frame decryptor to use on all incoming frames. This is optional. + // This pointers lifetimes is managed by the set of RtpReceivers it is + // attached to. + // TODO(benwright) make pure virtual once internal supports it. + virtual void SetFrameDecryptor( + uint32_t ssrc, + rtc::scoped_refptr<webrtc::FrameDecryptorInterface> frame_decryptor); + + // Enable network condition based codec switching. + virtual void SetVideoCodecSwitchingEnabled(bool enabled); + + // Base method to send packet using NetworkInterface. + bool SendPacket(rtc::CopyOnWriteBuffer* packet, + const rtc::PacketOptions& options) { + return DoSendPacket(packet, false, options); + } + + bool SendRtcp(rtc::CopyOnWriteBuffer* packet, + const rtc::PacketOptions& options) { + return DoSendPacket(packet, true, options); + } + + int SetOption(NetworkInterface::SocketType type, + rtc::Socket::Option opt, + int option) { + rtc::CritScope cs(&network_interface_crit_); + if (!network_interface_) + return -1; + + return network_interface_->SetOption(type, opt, option); + } + + const webrtc::MediaTransportConfig& media_transport_config() const { + return media_transport_config_; + } + + // Corresponds to the SDP attribute extmap-allow-mixed, see RFC8285. + // Set to true if it's allowed to mix one- and two-byte RTP header extensions + // in the same stream. The setter and getter must only be called from + // worker_thread. + void SetExtmapAllowMixed(bool extmap_allow_mixed) { + extmap_allow_mixed_ = extmap_allow_mixed; + } + bool ExtmapAllowMixed() const { return extmap_allow_mixed_; } + + virtual webrtc::RtpParameters GetRtpSendParameters(uint32_t ssrc) const = 0; + virtual webrtc::RTCError SetRtpSendParameters( + uint32_t ssrc, + const webrtc::RtpParameters& parameters) = 0; + + virtual void SetEncoderToPacketizerFrameTransformer( + uint32_t ssrc, + rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer); + virtual void SetDepacketizerToDecoderFrameTransformer( + uint32_t ssrc, + rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer); + + protected: + bool DscpEnabled() const { return enable_dscp_; } + + // This is the DSCP value used for both RTP and RTCP channels if DSCP is + // enabled. It can be changed at any time via |SetPreferredDscp|. + rtc::DiffServCodePoint PreferredDscp() const { + rtc::CritScope cs(&network_interface_crit_); + return preferred_dscp_; + } + + int SetPreferredDscp(rtc::DiffServCodePoint preferred_dscp) { + rtc::CritScope cs(&network_interface_crit_); + if (preferred_dscp == preferred_dscp_) { + return 0; + } + preferred_dscp_ = preferred_dscp; + return UpdateDscp(); + } + + private: + // Apply the preferred DSCP setting to the underlying network interface RTP + // and RTCP channels. If DSCP is disabled, then apply the default DSCP value. + int UpdateDscp() RTC_EXCLUSIVE_LOCKS_REQUIRED(network_interface_crit_) { + rtc::DiffServCodePoint value = + enable_dscp_ ? preferred_dscp_ : rtc::DSCP_DEFAULT; + int ret = SetOption(NetworkInterface::ST_RTP, rtc::Socket::OPT_DSCP, value); + if (ret == 0) { + ret = SetOption(NetworkInterface::ST_RTCP, rtc::Socket::OPT_DSCP, value); + } + return ret; + } + + bool DoSendPacket(rtc::CopyOnWriteBuffer* packet, + bool rtcp, + const rtc::PacketOptions& options) { + rtc::CritScope cs(&network_interface_crit_); + if (!network_interface_) + return false; + + return (!rtcp) ? network_interface_->SendPacket(packet, options) + : network_interface_->SendRtcp(packet, options); + } + + const bool enable_dscp_; + // |network_interface_| can be accessed from the worker_thread and + // from any MediaEngine threads. This critical section is to protect accessing + // of network_interface_ object. + rtc::CriticalSection network_interface_crit_; + NetworkInterface* network_interface_ RTC_GUARDED_BY(network_interface_crit_) = + nullptr; + rtc::DiffServCodePoint preferred_dscp_ + RTC_GUARDED_BY(network_interface_crit_) = rtc::DSCP_DEFAULT; + webrtc::MediaTransportConfig media_transport_config_; + bool extmap_allow_mixed_ = false; +}; + +// The stats information is structured as follows: +// Media are represented by either MediaSenderInfo or MediaReceiverInfo. +// Media contains a vector of SSRC infos that are exclusively used by this +// media. (SSRCs shared between media streams can't be represented.) + +// Information about an SSRC. +// This data may be locally recorded, or received in an RTCP SR or RR. +struct SsrcSenderInfo { + uint32_t ssrc = 0; + double timestamp = 0.0; // NTP timestamp, represented as seconds since epoch. +}; + +struct SsrcReceiverInfo { + uint32_t ssrc = 0; + double timestamp = 0.0; +}; + +struct MediaSenderInfo { + MediaSenderInfo(); + ~MediaSenderInfo(); + void add_ssrc(const SsrcSenderInfo& stat) { local_stats.push_back(stat); } + // Temporary utility function for call sites that only provide SSRC. + // As more info is added into SsrcSenderInfo, this function should go away. + void add_ssrc(uint32_t ssrc) { + SsrcSenderInfo stat; + stat.ssrc = ssrc; + add_ssrc(stat); + } + // Utility accessor for clients that are only interested in ssrc numbers. + std::vector<uint32_t> ssrcs() const { + std::vector<uint32_t> retval; + for (std::vector<SsrcSenderInfo>::const_iterator it = local_stats.begin(); + it != local_stats.end(); ++it) { + retval.push_back(it->ssrc); + } + return retval; + } + // Returns true if the media has been connected. + bool connected() const { return local_stats.size() > 0; } + // Utility accessor for clients that make the assumption only one ssrc + // exists per media. + // This will eventually go away. + // Call sites that compare this to zero should use connected() instead. + // https://bugs.webrtc.org/8694 + uint32_t ssrc() const { + if (connected()) { + return local_stats[0].ssrc; + } else { + return 0; + } + } + // https://w3c.github.io/webrtc-stats/#dom-rtcsentrtpstreamstats-bytessent + int64_t payload_bytes_sent = 0; + // https://w3c.github.io/webrtc-stats/#dom-rtcoutboundrtpstreamstats-headerbytessent + int64_t header_and_padding_bytes_sent = 0; + // https://w3c.github.io/webrtc-stats/#dom-rtcoutboundrtpstreamstats-retransmittedbytessent + uint64_t retransmitted_bytes_sent = 0; + int packets_sent = 0; + // https://w3c.github.io/webrtc-stats/#dom-rtcoutboundrtpstreamstats-retransmittedpacketssent + uint64_t retransmitted_packets_sent = 0; + int packets_lost = 0; + float fraction_lost = 0.0f; + int64_t rtt_ms = 0; + std::string codec_name; + absl::optional<int> codec_payload_type; + std::vector<SsrcSenderInfo> local_stats; + std::vector<SsrcReceiverInfo> remote_stats; + // A snapshot of the most recent Report Block with additional data of interest + // to statistics. Used to implement RTCRemoteInboundRtpStreamStats. Within + // this list, the ReportBlockData::RTCPReportBlock::source_ssrc(), which is + // the SSRC of the corresponding outbound RTP stream, is unique. + std::vector<webrtc::ReportBlockData> report_block_datas; +}; + +struct MediaReceiverInfo { + MediaReceiverInfo(); + ~MediaReceiverInfo(); + void add_ssrc(const SsrcReceiverInfo& stat) { local_stats.push_back(stat); } + // Temporary utility function for call sites that only provide SSRC. + // As more info is added into SsrcSenderInfo, this function should go away. + void add_ssrc(uint32_t ssrc) { + SsrcReceiverInfo stat; + stat.ssrc = ssrc; + add_ssrc(stat); + } + std::vector<uint32_t> ssrcs() const { + std::vector<uint32_t> retval; + for (std::vector<SsrcReceiverInfo>::const_iterator it = local_stats.begin(); + it != local_stats.end(); ++it) { + retval.push_back(it->ssrc); + } + return retval; + } + // Returns true if the media has been connected. + bool connected() const { return local_stats.size() > 0; } + // Utility accessor for clients that make the assumption only one ssrc + // exists per media. + // This will eventually go away. + // Call sites that compare this to zero should use connected(); + // https://bugs.webrtc.org/8694 + uint32_t ssrc() const { + if (connected()) { + return local_stats[0].ssrc; + } else { + return 0; + } + } + + // https://w3c.github.io/webrtc-stats/#dom-rtcinboundrtpstreamstats-bytesreceived + int64_t payload_bytes_rcvd = 0; + // https://w3c.github.io/webrtc-stats/#dom-rtcinboundrtpstreamstats-headerbytesreceived + int64_t header_and_padding_bytes_rcvd = 0; + int packets_rcvd = 0; + int packets_lost = 0; + // The timestamp at which the last packet was received, i.e. the time of the + // local clock when it was received - not the RTP timestamp of that packet. + // https://w3c.github.io/webrtc-stats/#dom-rtcinboundrtpstreamstats-lastpacketreceivedtimestamp + absl::optional<int64_t> last_packet_received_timestamp_ms; + // https://w3c.github.io/webrtc-stats/#dom-rtcinboundrtpstreamstats-estimatedplayouttimestamp + absl::optional<int64_t> estimated_playout_ntp_timestamp_ms; + std::string codec_name; + absl::optional<int> codec_payload_type; + std::vector<SsrcReceiverInfo> local_stats; + std::vector<SsrcSenderInfo> remote_stats; +}; + +struct VoiceSenderInfo : public MediaSenderInfo { + VoiceSenderInfo(); + ~VoiceSenderInfo(); + int jitter_ms = 0; + // Current audio level, expressed linearly [0,32767]. + int audio_level = 0; + // See description of "totalAudioEnergy" in the WebRTC stats spec: + // https://w3c.github.io/webrtc-stats/#dom-rtcmediastreamtrackstats-totalaudioenergy + double total_input_energy = 0.0; + double total_input_duration = 0.0; + bool typing_noise_detected = false; + webrtc::ANAStats ana_statistics; + webrtc::AudioProcessingStats apm_statistics; +}; + +struct VoiceReceiverInfo : public MediaReceiverInfo { + VoiceReceiverInfo(); + ~VoiceReceiverInfo(); + int jitter_ms = 0; + int jitter_buffer_ms = 0; + int jitter_buffer_preferred_ms = 0; + int delay_estimate_ms = 0; + int audio_level = 0; + // Stats below correspond to similarly-named fields in the WebRTC stats spec. + // https://w3c.github.io/webrtc-stats/#dom-rtcmediastreamtrackstats + double total_output_energy = 0.0; + uint64_t total_samples_received = 0; + double total_output_duration = 0.0; + uint64_t concealed_samples = 0; + uint64_t silent_concealed_samples = 0; + uint64_t concealment_events = 0; + double jitter_buffer_delay_seconds = 0.0; + uint64_t jitter_buffer_emitted_count = 0; + double jitter_buffer_target_delay_seconds = 0.0; + uint64_t inserted_samples_for_deceleration = 0; + uint64_t removed_samples_for_acceleration = 0; + uint64_t fec_packets_received = 0; + uint64_t fec_packets_discarded = 0; + // Stats below DO NOT correspond directly to anything in the WebRTC stats + // fraction of synthesized audio inserted through expansion. + float expand_rate = 0.0f; + // fraction of synthesized speech inserted through expansion. + float speech_expand_rate = 0.0f; + // fraction of data out of secondary decoding, including FEC and RED. + float secondary_decoded_rate = 0.0f; + // Fraction of secondary data, including FEC and RED, that is discarded. + // Discarding of secondary data can be caused by the reception of the primary + // data, obsoleting the secondary data. It can also be caused by early + // or late arrival of secondary data. This metric is the percentage of + // discarded secondary data since last query of receiver info. + float secondary_discarded_rate = 0.0f; + // Fraction of data removed through time compression. + float accelerate_rate = 0.0f; + // Fraction of data inserted through time stretching. + float preemptive_expand_rate = 0.0f; + int decoding_calls_to_silence_generator = 0; + int decoding_calls_to_neteq = 0; + int decoding_normal = 0; + // TODO(alexnarest): Consider decoding_neteq_plc for consistency + int decoding_plc = 0; + int decoding_codec_plc = 0; + int decoding_cng = 0; + int decoding_plc_cng = 0; + int decoding_muted_output = 0; + // Estimated capture start time in NTP time in ms. + int64_t capture_start_ntp_time_ms = -1; + // Count of the number of buffer flushes. + uint64_t jitter_buffer_flushes = 0; + // Number of samples expanded due to delayed packets. + uint64_t delayed_packet_outage_samples = 0; + // Arrival delay of received audio packets. + double relative_packet_arrival_delay_seconds = 0.0; + // Count and total duration of audio interruptions (loss-concealement periods + // longer than 150 ms). + int32_t interruption_count = 0; + int32_t total_interruption_duration_ms = 0; +}; + +struct VideoSenderInfo : public MediaSenderInfo { + VideoSenderInfo(); + ~VideoSenderInfo(); + std::vector<SsrcGroup> ssrc_groups; + std::string encoder_implementation_name; + int firs_rcvd = 0; + int plis_rcvd = 0; + int nacks_rcvd = 0; + int send_frame_width = 0; + int send_frame_height = 0; + int framerate_input = 0; + int framerate_sent = 0; + int aggregated_framerate_sent = 0; + int nominal_bitrate = 0; + int adapt_reason = 0; + int adapt_changes = 0; + // https://w3c.github.io/webrtc-stats/#dom-rtcoutboundrtpstreamstats-qualitylimitationreason + webrtc::QualityLimitationReason quality_limitation_reason = + webrtc::QualityLimitationReason::kNone; + // https://w3c.github.io/webrtc-stats/#dom-rtcoutboundrtpstreamstats-qualitylimitationdurations + std::map<webrtc::QualityLimitationReason, int64_t> + quality_limitation_durations_ms; + // https://w3c.github.io/webrtc-stats/#dom-rtcoutboundrtpstreamstats-qualitylimitationresolutionchanges + uint32_t quality_limitation_resolution_changes = 0; + int avg_encode_ms = 0; + int encode_usage_percent = 0; + uint32_t frames_encoded = 0; + uint32_t key_frames_encoded = 0; + // https://w3c.github.io/webrtc-stats/#dom-rtcoutboundrtpstreamstats-totalencodetime + uint64_t total_encode_time_ms = 0; + // https://w3c.github.io/webrtc-stats/#dom-rtcoutboundrtpstreamstats-totalencodedbytestarget + uint64_t total_encoded_bytes_target = 0; + uint64_t total_packet_send_delay_ms = 0; + bool has_entered_low_resolution = false; + absl::optional<uint64_t> qp_sum; + webrtc::VideoContentType content_type = webrtc::VideoContentType::UNSPECIFIED; + uint32_t frames_sent = 0; + // https://w3c.github.io/webrtc-stats/#dom-rtcvideosenderstats-hugeframessent + uint32_t huge_frames_sent = 0; + uint32_t aggregated_huge_frames_sent = 0; + absl::optional<std::string> rid; +}; + +struct VideoReceiverInfo : public MediaReceiverInfo { + VideoReceiverInfo(); + ~VideoReceiverInfo(); + std::vector<SsrcGroup> ssrc_groups; + std::string decoder_implementation_name; + int packets_concealed = 0; + int firs_sent = 0; + int plis_sent = 0; + int nacks_sent = 0; + int frame_width = 0; + int frame_height = 0; + int framerate_rcvd = 0; + int framerate_decoded = 0; + int framerate_output = 0; + // Framerate as sent to the renderer. + int framerate_render_input = 0; + // Framerate that the renderer reports. + int framerate_render_output = 0; + uint32_t frames_received = 0; + uint32_t frames_dropped = 0; + uint32_t frames_decoded = 0; + uint32_t key_frames_decoded = 0; + uint32_t frames_rendered = 0; + absl::optional<uint64_t> qp_sum; + // https://w3c.github.io/webrtc-stats/#dom-rtcinboundrtpstreamstats-totaldecodetime + uint64_t total_decode_time_ms = 0; + double total_inter_frame_delay = 0; + double total_squared_inter_frame_delay = 0; + int64_t interframe_delay_max_ms = -1; + uint32_t freeze_count = 0; + uint32_t pause_count = 0; + uint32_t total_freezes_duration_ms = 0; + uint32_t total_pauses_duration_ms = 0; + uint32_t total_frames_duration_ms = 0; + double sum_squared_frame_durations = 0.0; + + webrtc::VideoContentType content_type = webrtc::VideoContentType::UNSPECIFIED; + + // All stats below are gathered per-VideoReceiver, but some will be correlated + // across MediaStreamTracks. NOTE(hta): when sinking stats into per-SSRC + // structures, reflect this in the new layout. + + // Current frame decode latency. + int decode_ms = 0; + // Maximum observed frame decode latency. + int max_decode_ms = 0; + // Jitter (network-related) latency. + int jitter_buffer_ms = 0; + // Jitter (network-related) latency (cumulative). + // https://w3c.github.io/webrtc-stats/#dom-rtcvideoreceiverstats-jitterbufferdelay + double jitter_buffer_delay_seconds = 0; + // Number of observations for cumulative jitter latency. + // https://w3c.github.io/webrtc-stats/#dom-rtcvideoreceiverstats-jitterbufferemittedcount + uint64_t jitter_buffer_emitted_count = 0; + // Requested minimum playout latency. + int min_playout_delay_ms = 0; + // Requested latency to account for rendering delay. + int render_delay_ms = 0; + // Target overall delay: network+decode+render, accounting for + // min_playout_delay_ms. + int target_delay_ms = 0; + // Current overall delay, possibly ramping towards target_delay_ms. + int current_delay_ms = 0; + + // Estimated capture start time in NTP time in ms. + int64_t capture_start_ntp_time_ms = -1; + + // First frame received to first frame decoded latency. + int64_t first_frame_received_to_decoded_ms = -1; + + // Timing frame info: all important timestamps for a full lifetime of a + // single 'timing frame'. + absl::optional<webrtc::TimingFrameInfo> timing_frame_info; +}; + +struct DataSenderInfo : public MediaSenderInfo { + uint32_t ssrc = 0; +}; + +struct DataReceiverInfo : public MediaReceiverInfo { + uint32_t ssrc = 0; +}; + +struct BandwidthEstimationInfo { + int available_send_bandwidth = 0; + int available_recv_bandwidth = 0; + int target_enc_bitrate = 0; + int actual_enc_bitrate = 0; + int retransmit_bitrate = 0; + int transmit_bitrate = 0; + int64_t bucket_delay = 0; +}; + +// Maps from payload type to |RtpCodecParameters|. +typedef std::map<int, webrtc::RtpCodecParameters> RtpCodecParametersMap; + +struct VoiceMediaInfo { + VoiceMediaInfo(); + ~VoiceMediaInfo(); + void Clear() { + senders.clear(); + receivers.clear(); + send_codecs.clear(); + receive_codecs.clear(); + } + std::vector<VoiceSenderInfo> senders; + std::vector<VoiceReceiverInfo> receivers; + RtpCodecParametersMap send_codecs; + RtpCodecParametersMap receive_codecs; + int32_t device_underrun_count = 0; +}; + +struct VideoMediaInfo { + VideoMediaInfo(); + ~VideoMediaInfo(); + void Clear() { + senders.clear(); + aggregated_senders.clear(); + receivers.clear(); + send_codecs.clear(); + receive_codecs.clear(); + } + // Each sender info represents one "outbound-rtp" stream.In non - simulcast, + // this means one info per RtpSender but if simulcast is used this means + // one info per simulcast layer. + std::vector<VideoSenderInfo> senders; + // Used for legacy getStats() API's "ssrc" stats and modern getStats() API's + // "track" stats. If simulcast is used, instead of having one sender info per + // simulcast layer, the metrics of all layers of an RtpSender are aggregated + // into a single sender info per RtpSender. + std::vector<VideoSenderInfo> aggregated_senders; + std::vector<VideoReceiverInfo> receivers; + RtpCodecParametersMap send_codecs; + RtpCodecParametersMap receive_codecs; +}; + +struct DataMediaInfo { + DataMediaInfo(); + ~DataMediaInfo(); + void Clear() { + senders.clear(); + receivers.clear(); + } + std::vector<DataSenderInfo> senders; + std::vector<DataReceiverInfo> receivers; +}; + +struct RtcpParameters { + bool reduced_size = false; + bool remote_estimate = false; +}; + +template <class Codec> +struct RtpParameters { + virtual ~RtpParameters() = default; + + std::vector<Codec> codecs; + std::vector<webrtc::RtpExtension> extensions; + // For a send stream this is true if we've neogtiated a send direction, + // for a receive stream this is true if we've negotiated a receive direction. + bool is_stream_active = true; + + // TODO(pthatcher): Add streams. + RtcpParameters rtcp; + + std::string ToString() const { + rtc::StringBuilder ost; + ost << "{"; + const char* separator = ""; + for (const auto& entry : ToStringMap()) { + ost << separator << entry.first << ": " << entry.second; + separator = ", "; + } + ost << "}"; + return ost.Release(); + } + + protected: + virtual std::map<std::string, std::string> ToStringMap() const { + return {{"codecs", VectorToString(codecs)}, + {"extensions", VectorToString(extensions)}}; + } +}; + +// TODO(deadbeef): Rename to RtpSenderParameters, since they're intended to +// encapsulate all the parameters needed for an RtpSender. +template <class Codec> +struct RtpSendParameters : RtpParameters<Codec> { + int max_bandwidth_bps = -1; + // This is the value to be sent in the MID RTP header extension (if the header + // extension in included in the list of extensions). + std::string mid; + bool extmap_allow_mixed = false; + + protected: + std::map<std::string, std::string> ToStringMap() const override { + auto params = RtpParameters<Codec>::ToStringMap(); + params["max_bandwidth_bps"] = rtc::ToString(max_bandwidth_bps); + params["mid"] = (mid.empty() ? "<not set>" : mid); + params["extmap-allow-mixed"] = extmap_allow_mixed ? "true" : "false"; + return params; + } +}; + +struct AudioSendParameters : RtpSendParameters<AudioCodec> { + AudioSendParameters(); + ~AudioSendParameters() override; + AudioOptions options; + + protected: + std::map<std::string, std::string> ToStringMap() const override; +}; + +struct AudioRecvParameters : RtpParameters<AudioCodec> {}; + +class VoiceMediaChannel : public MediaChannel, public Delayable { + public: + VoiceMediaChannel() {} + explicit VoiceMediaChannel(const MediaConfig& config) + : MediaChannel(config) {} + ~VoiceMediaChannel() override {} + + cricket::MediaType media_type() const override; + virtual bool SetSendParameters(const AudioSendParameters& params) = 0; + virtual bool SetRecvParameters(const AudioRecvParameters& params) = 0; + // Get the receive parameters for the incoming stream identified by |ssrc|. + virtual webrtc::RtpParameters GetRtpReceiveParameters( + uint32_t ssrc) const = 0; + // Retrieve the receive parameters for the default receive + // stream, which is used when SSRCs are not signaled. + virtual webrtc::RtpParameters GetDefaultRtpReceiveParameters() const = 0; + // Starts or stops playout of received audio. + virtual void SetPlayout(bool playout) = 0; + // Starts or stops sending (and potentially capture) of local audio. + virtual void SetSend(bool send) = 0; + // Configure stream for sending. + virtual bool SetAudioSend(uint32_t ssrc, + bool enable, + const AudioOptions* options, + AudioSource* source) = 0; + // Set speaker output volume of the specified ssrc. + virtual bool SetOutputVolume(uint32_t ssrc, double volume) = 0; + // Set speaker output volume for future unsignaled streams. + virtual bool SetDefaultOutputVolume(double volume) = 0; + // Returns if the telephone-event has been negotiated. + virtual bool CanInsertDtmf() = 0; + // Send a DTMF |event|. The DTMF out-of-band signal will be used. + // The |ssrc| should be either 0 or a valid send stream ssrc. + // The valid value for the |event| are 0 to 15 which corresponding to + // DTMF event 0-9, *, #, A-D. + virtual bool InsertDtmf(uint32_t ssrc, int event, int duration) = 0; + // Gets quality stats for the channel. + virtual bool GetStats(VoiceMediaInfo* info) = 0; + + virtual void SetRawAudioSink( + uint32_t ssrc, + std::unique_ptr<webrtc::AudioSinkInterface> sink) = 0; + virtual void SetDefaultRawAudioSink( + std::unique_ptr<webrtc::AudioSinkInterface> sink) = 0; + + virtual std::vector<webrtc::RtpSource> GetSources(uint32_t ssrc) const = 0; +}; + +// TODO(deadbeef): Rename to VideoSenderParameters, since they're intended to +// encapsulate all the parameters needed for a video RtpSender. +struct VideoSendParameters : RtpSendParameters<VideoCodec> { + VideoSendParameters(); + ~VideoSendParameters() override; + // Use conference mode? This flag comes from the remote + // description's SDP line 'a=x-google-flag:conference', copied over + // by VideoChannel::SetRemoteContent_w, and ultimately used by + // conference mode screencast logic in + // WebRtcVideoChannel::WebRtcVideoSendStream::CreateVideoEncoderConfig. + // The special screencast behaviour is disabled by default. + bool conference_mode = false; + + protected: + std::map<std::string, std::string> ToStringMap() const override; +}; + +// TODO(deadbeef): Rename to VideoReceiverParameters, since they're intended to +// encapsulate all the parameters needed for a video RtpReceiver. +struct VideoRecvParameters : RtpParameters<VideoCodec> {}; + +class VideoMediaChannel : public MediaChannel, public Delayable { + public: + VideoMediaChannel() {} + explicit VideoMediaChannel(const MediaConfig& config) + : MediaChannel(config) {} + ~VideoMediaChannel() override {} + + cricket::MediaType media_type() const override; + virtual bool SetSendParameters(const VideoSendParameters& params) = 0; + virtual bool SetRecvParameters(const VideoRecvParameters& params) = 0; + // Get the receive parameters for the incoming stream identified by |ssrc|. + virtual webrtc::RtpParameters GetRtpReceiveParameters( + uint32_t ssrc) const = 0; + // Retrieve the receive parameters for the default receive + // stream, which is used when SSRCs are not signaled. + virtual webrtc::RtpParameters GetDefaultRtpReceiveParameters() const = 0; + // Gets the currently set codecs/payload types to be used for outgoing media. + virtual bool GetSendCodec(VideoCodec* send_codec) = 0; + // Starts or stops transmission (and potentially capture) of local video. + virtual bool SetSend(bool send) = 0; + // Configure stream for sending and register a source. + // The |ssrc| must correspond to a registered send stream. + virtual bool SetVideoSend( + uint32_t ssrc, + const VideoOptions* options, + rtc::VideoSourceInterface<webrtc::VideoFrame>* source) = 0; + // Sets the sink object to be used for the specified stream. + virtual bool SetSink(uint32_t ssrc, + rtc::VideoSinkInterface<webrtc::VideoFrame>* sink) = 0; + // The sink is used for the 'default' stream. + virtual void SetDefaultSink( + rtc::VideoSinkInterface<webrtc::VideoFrame>* sink) = 0; + // This fills the "bitrate parts" (rtx, video bitrate) of the + // BandwidthEstimationInfo, since that part that isn't possible to get + // through webrtc::Call::GetStats, as they are statistics of the send + // streams. + // TODO(holmer): We should change this so that either BWE graphs doesn't + // need access to bitrates of the streams, or change the (RTC)StatsCollector + // so that it's getting the send stream stats separately by calling + // GetStats(), and merges with BandwidthEstimationInfo by itself. + virtual void FillBitrateInfo(BandwidthEstimationInfo* bwe_info) = 0; + // Gets quality stats for the channel. + virtual bool GetStats(VideoMediaInfo* info) = 0; + // Set recordable encoded frame callback for |ssrc| + virtual void SetRecordableEncodedFrameCallback( + uint32_t ssrc, + std::function<void(const webrtc::RecordableEncodedFrame&)> callback) = 0; + // Clear recordable encoded frame callback for |ssrc| + virtual void ClearRecordableEncodedFrameCallback(uint32_t ssrc) = 0; + // Cause generation of a keyframe for |ssrc| + virtual void GenerateKeyFrame(uint32_t ssrc) = 0; + + virtual std::vector<webrtc::RtpSource> GetSources(uint32_t ssrc) const = 0; +}; + +enum DataMessageType { + // Chrome-Internal use only. See SctpDataMediaChannel for the actual PPID + // values. + DMT_NONE = 0, + DMT_CONTROL = 1, + DMT_BINARY = 2, + DMT_TEXT = 3, +}; + +// Info about data received in DataMediaChannel. For use in +// DataMediaChannel::SignalDataReceived and in all of the signals that +// signal fires, on up the chain. +struct ReceiveDataParams { + // The in-packet stream indentifier. + // RTP data channels use SSRCs, SCTP data channels use SIDs. + union { + uint32_t ssrc; + int sid = 0; + }; + // The type of message (binary, text, or control). + DataMessageType type = DMT_TEXT; + // A per-stream value incremented per packet in the stream. + int seq_num = 0; + // A per-stream value monotonically increasing with time. + int timestamp = 0; +}; + +struct SendDataParams { + // The in-packet stream indentifier. + // RTP data channels use SSRCs, SCTP data channels use SIDs. + union { + uint32_t ssrc; + int sid = 0; + }; + // The type of message (binary, text, or control). + DataMessageType type = DMT_TEXT; + + // TODO(pthatcher): Make |ordered| and |reliable| true by default? + // For SCTP, whether to send messages flagged as ordered or not. + // If false, messages can be received out of order. + bool ordered = false; + // For SCTP, whether the messages are sent reliably or not. + // If false, messages may be lost. + bool reliable = false; + // For SCTP, if reliable == false, provide partial reliability by + // resending up to this many times. Either count or millis + // is supported, not both at the same time. + int max_rtx_count = 0; + // For SCTP, if reliable == false, provide partial reliability by + // resending for up to this many milliseconds. Either count or millis + // is supported, not both at the same time. + int max_rtx_ms = 0; +}; + +enum SendDataResult { SDR_SUCCESS, SDR_ERROR, SDR_BLOCK }; + +struct DataSendParameters : RtpSendParameters<DataCodec> {}; + +struct DataRecvParameters : RtpParameters<DataCodec> {}; + +class DataMediaChannel : public MediaChannel { + public: + DataMediaChannel(); + explicit DataMediaChannel(const MediaConfig& config); + ~DataMediaChannel() override; + + cricket::MediaType media_type() const override; + virtual bool SetSendParameters(const DataSendParameters& params) = 0; + virtual bool SetRecvParameters(const DataRecvParameters& params) = 0; + + // RtpParameter methods are not supported for Data channel. + webrtc::RtpParameters GetRtpSendParameters(uint32_t ssrc) const override; + webrtc::RTCError SetRtpSendParameters( + uint32_t ssrc, + const webrtc::RtpParameters& parameters) override; + + // TODO(pthatcher): Implement this. + virtual bool GetStats(DataMediaInfo* info); + + virtual bool SetSend(bool send) = 0; + virtual bool SetReceive(bool receive) = 0; + + void OnNetworkRouteChanged(const std::string& transport_name, + const rtc::NetworkRoute& network_route) override {} + + virtual bool SendData(const SendDataParams& params, + const rtc::CopyOnWriteBuffer& payload, + SendDataResult* result = NULL) = 0; + // Signals when data is received (params, data, len) + sigslot::signal3<const ReceiveDataParams&, const char*, size_t> + SignalDataReceived; + // Signal when the media channel is ready to send the stream. Arguments are: + // writable(bool) + sigslot::signal1<bool> SignalReadyToSend; +}; + +} // namespace cricket + +#endif // MEDIA_BASE_MEDIA_CHANNEL_H_ diff --git a/media/base/media_config.h b/media/base/media_config.h new file mode 100644 index 0000000000..be314a8aa3 --- /dev/null +++ b/media/base/media_config.h @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MEDIA_BASE_MEDIA_CONFIG_H_ +#define MEDIA_BASE_MEDIA_CONFIG_H_ + +namespace cricket { + +// Construction-time settings, passed on when creating +// MediaChannels. +struct MediaConfig { + // Set DSCP value on packets. This flag comes from the + // PeerConnection constraint 'googDscp'. + bool enable_dscp = false; + + // Video-specific config. + struct Video { + // Enable WebRTC CPU Overuse Detection. This flag comes from the + // PeerConnection constraint 'googCpuOveruseDetection'. + bool enable_cpu_adaptation = true; + + // Enable WebRTC suspension of video. No video frames will be sent + // when the bitrate is below the configured minimum bitrate. This + // flag comes from the PeerConnection constraint + // 'googSuspendBelowMinBitrate', and WebRtcVideoChannel copies it + // to VideoSendStream::Config::suspend_below_min_bitrate. + bool suspend_below_min_bitrate = false; + + // Enable buffering and playout timing smoothing of decoded frames. + // If set to true, then WebRTC will buffer and potentially drop decoded + // frames in order to keep a smooth rendering. + // If set to false, then WebRTC will hand over the frame from the decoder + // to the renderer as soon as possible, meaning that the renderer is + // responsible for smooth rendering. + // Note that even if this flag is set to false, dropping of frames can + // still happen pre-decode, e.g., dropping of higher temporal layers. + // This flag comes from the PeerConnection RtcConfiguration. + bool enable_prerenderer_smoothing = true; + + // Enables periodic bandwidth probing in application-limited region. + bool periodic_alr_bandwidth_probing = false; + + // Enables the new method to estimate the cpu load from encoding, used for + // cpu adaptation. This flag is intended to be controlled primarily by a + // Chrome origin-trial. + // TODO(bugs.webrtc.org/8504): If all goes well, the flag will be removed + // together with the old method of estimation. + bool experiment_cpu_load_estimator = false; + + // Time interval between RTCP report for video + int rtcp_report_interval_ms = 1000; + } video; + + // Audio-specific config. + struct Audio { + // Time interval between RTCP report for audio + int rtcp_report_interval_ms = 5000; + } audio; + + bool operator==(const MediaConfig& o) const { + return enable_dscp == o.enable_dscp && + video.enable_cpu_adaptation == o.video.enable_cpu_adaptation && + video.suspend_below_min_bitrate == + o.video.suspend_below_min_bitrate && + video.enable_prerenderer_smoothing == + o.video.enable_prerenderer_smoothing && + video.periodic_alr_bandwidth_probing == + o.video.periodic_alr_bandwidth_probing && + video.experiment_cpu_load_estimator == + o.video.experiment_cpu_load_estimator && + video.rtcp_report_interval_ms == o.video.rtcp_report_interval_ms && + audio.rtcp_report_interval_ms == o.audio.rtcp_report_interval_ms; + } + + bool operator!=(const MediaConfig& o) const { return !(*this == o); } +}; + +} // namespace cricket + +#endif // MEDIA_BASE_MEDIA_CONFIG_H_ diff --git a/media/base/media_constants.cc b/media/base/media_constants.cc new file mode 100644 index 0000000000..5144a6ea65 --- /dev/null +++ b/media/base/media_constants.cc @@ -0,0 +1,123 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "media/base/media_constants.h" + +namespace cricket { + +const int kVideoCodecClockrate = 90000; +const int kDataCodecClockrate = 90000; +const int kDataMaxBandwidth = 30720; // bps + +const float kHighSystemCpuThreshold = 0.85f; +const float kLowSystemCpuThreshold = 0.65f; +const float kProcessCpuThreshold = 0.10f; + +const char kRtxCodecName[] = "rtx"; +const char kRedCodecName[] = "red"; +const char kUlpfecCodecName[] = "ulpfec"; +const char kMultiplexCodecName[] = "multiplex"; + +// TODO(brandtr): Change this to 'flexfec' when we are confident that the +// header format is not changing anymore. +const char kFlexfecCodecName[] = "flexfec-03"; + +// draft-ietf-payload-flexible-fec-scheme-02.txt +const char kFlexfecFmtpRepairWindow[] = "repair-window"; + +const char kCodecParamAssociatedPayloadType[] = "apt"; +const char kCodecParamAssociatedCodecName[] = "acn"; + +const char kOpusCodecName[] = "opus"; +const char kIsacCodecName[] = "ISAC"; +const char kL16CodecName[] = "L16"; +const char kG722CodecName[] = "G722"; +const char kIlbcCodecName[] = "ILBC"; +const char kPcmuCodecName[] = "PCMU"; +const char kPcmaCodecName[] = "PCMA"; +const char kCnCodecName[] = "CN"; +const char kDtmfCodecName[] = "telephone-event"; + +// draft-spittka-payload-rtp-opus-03.txt +const char kCodecParamPTime[] = "ptime"; +const char kCodecParamMaxPTime[] = "maxptime"; +const char kCodecParamMinPTime[] = "minptime"; +const char kCodecParamSPropStereo[] = "sprop-stereo"; +const char kCodecParamStereo[] = "stereo"; +const char kCodecParamUseInbandFec[] = "useinbandfec"; +const char kCodecParamUseDtx[] = "usedtx"; +const char kCodecParamMaxAverageBitrate[] = "maxaveragebitrate"; +const char kCodecParamMaxPlaybackRate[] = "maxplaybackrate"; + +const char kCodecParamSctpProtocol[] = "protocol"; +const char kCodecParamSctpStreams[] = "streams"; + +const char kParamValueTrue[] = "1"; +const char kParamValueEmpty[] = ""; + +const int kOpusDefaultMaxPTime = 120; +const int kOpusDefaultPTime = 20; +const int kOpusDefaultMinPTime = 3; +const int kOpusDefaultSPropStereo = 0; +const int kOpusDefaultStereo = 0; +const int kOpusDefaultUseInbandFec = 0; +const int kOpusDefaultUseDtx = 0; +const int kOpusDefaultMaxPlaybackRate = 48000; + +const int kPreferredMaxPTime = 120; +const int kPreferredMinPTime = 10; +const int kPreferredSPropStereo = 0; +const int kPreferredStereo = 0; +const int kPreferredUseInbandFec = 0; + +const char kPacketizationParamRaw[] = "raw"; + +const char kRtcpFbParamLntf[] = "goog-lntf"; +const char kRtcpFbParamNack[] = "nack"; +const char kRtcpFbNackParamPli[] = "pli"; +const char kRtcpFbParamRemb[] = "goog-remb"; +const char kRtcpFbParamTransportCc[] = "transport-cc"; + +const char kRtcpFbParamCcm[] = "ccm"; +const char kRtcpFbCcmParamFir[] = "fir"; +const char kRtcpFbParamRrtr[] = "rrtr"; +const char kCodecParamMaxBitrate[] = "x-google-max-bitrate"; +const char kCodecParamMinBitrate[] = "x-google-min-bitrate"; +const char kCodecParamStartBitrate[] = "x-google-start-bitrate"; +const char kCodecParamMaxQuantization[] = "x-google-max-quantization"; +const char kCodecParamPort[] = "x-google-port"; +const char kCodecParamMaxMessageSize[] = "x-google-max-message-size"; + +const int kGoogleRtpDataCodecPlType = 109; +const char kGoogleRtpDataCodecName[] = "google-data"; + +const int kGoogleSctpDataCodecPlType = 108; +const char kGoogleSctpDataCodecName[] = "google-sctp-data"; + +const char kComfortNoiseCodecName[] = "CN"; + +const char kVp8CodecName[] = "VP8"; +const char kVp9CodecName[] = "VP9"; +const char kAv1CodecName[] = "AV1X"; +const char kH264CodecName[] = "H264"; + +// RFC 6184 RTP Payload Format for H.264 video +const char kH264FmtpProfileLevelId[] = "profile-level-id"; +const char kH264FmtpLevelAsymmetryAllowed[] = "level-asymmetry-allowed"; +const char kH264FmtpPacketizationMode[] = "packetization-mode"; +const char kH264FmtpSpropParameterSets[] = "sprop-parameter-sets"; +const char kH264ProfileLevelConstrainedBaseline[] = "42e01f"; + +const int kDefaultVideoMaxFramerate = 60; + +const size_t kConferenceMaxNumSpatialLayers = 3; +const size_t kConferenceMaxNumTemporalLayers = 3; +const size_t kConferenceDefaultNumTemporalLayers = 3; +} // namespace cricket diff --git a/media/base/media_constants.h b/media/base/media_constants.h new file mode 100644 index 0000000000..b9b8a336f7 --- /dev/null +++ b/media/base/media_constants.h @@ -0,0 +1,154 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MEDIA_BASE_MEDIA_CONSTANTS_H_ +#define MEDIA_BASE_MEDIA_CONSTANTS_H_ + +#include <stddef.h> + +#include "rtc_base/system/rtc_export.h" + +// This file contains constants related to media. + +namespace cricket { + +extern const int kVideoCodecClockrate; +extern const int kDataCodecClockrate; +extern const int kDataMaxBandwidth; // bps + +// Default CPU thresholds. +extern const float kHighSystemCpuThreshold; +extern const float kLowSystemCpuThreshold; +extern const float kProcessCpuThreshold; + +extern const char kRtxCodecName[]; +extern const char kRedCodecName[]; +extern const char kUlpfecCodecName[]; +extern const char kFlexfecCodecName[]; +extern const char kMultiplexCodecName[]; + +extern const char kFlexfecFmtpRepairWindow[]; + +// Codec parameters +extern const char kCodecParamAssociatedPayloadType[]; +extern const char kCodecParamAssociatedCodecName[]; + +extern const char kOpusCodecName[]; +extern const char kIsacCodecName[]; +extern const char kL16CodecName[]; +extern const char kG722CodecName[]; +extern const char kIlbcCodecName[]; +extern const char kPcmuCodecName[]; +extern const char kPcmaCodecName[]; +extern const char kCnCodecName[]; +extern const char kDtmfCodecName[]; + +// Attribute parameters +extern const char kCodecParamPTime[]; +extern const char kCodecParamMaxPTime[]; +// fmtp parameters +extern const char kCodecParamMinPTime[]; +extern const char kCodecParamSPropStereo[]; +extern const char kCodecParamStereo[]; +extern const char kCodecParamUseInbandFec[]; +extern const char kCodecParamUseDtx[]; +extern const char kCodecParamMaxAverageBitrate[]; +extern const char kCodecParamMaxPlaybackRate[]; +extern const char kCodecParamSctpProtocol[]; +extern const char kCodecParamSctpStreams[]; + +extern const char kParamValueTrue[]; +// Parameters are stored as parameter/value pairs. For parameters who do not +// have a value, |kParamValueEmpty| should be used as value. +extern const char kParamValueEmpty[]; + +// opus parameters. +// Default value for maxptime according to +// http://tools.ietf.org/html/draft-spittka-payload-rtp-opus-03 +extern const int kOpusDefaultMaxPTime; +extern const int kOpusDefaultPTime; +extern const int kOpusDefaultMinPTime; +extern const int kOpusDefaultSPropStereo; +extern const int kOpusDefaultStereo; +extern const int kOpusDefaultUseInbandFec; +extern const int kOpusDefaultUseDtx; +extern const int kOpusDefaultMaxPlaybackRate; + +// Prefered values in this code base. Note that they may differ from the default +// values in http://tools.ietf.org/html/draft-spittka-payload-rtp-opus-03 +// Only frames larger or equal to 10 ms are currently supported in this code +// base. +extern const int kPreferredMaxPTime; +extern const int kPreferredMinPTime; +extern const int kPreferredSPropStereo; +extern const int kPreferredStereo; +extern const int kPreferredUseInbandFec; + +extern const char kPacketizationParamRaw[]; + +// rtcp-fb message in its first experimental stages. Documentation pending. +extern const char kRtcpFbParamLntf[]; +// rtcp-fb messages according to RFC 4585 +extern const char kRtcpFbParamNack[]; +extern const char kRtcpFbNackParamPli[]; +// rtcp-fb messages according to +// http://tools.ietf.org/html/draft-alvestrand-rmcat-remb-00 +extern const char kRtcpFbParamRemb[]; +// rtcp-fb messages according to +// https://tools.ietf.org/html/draft-holmer-rmcat-transport-wide-cc-extensions-01 +extern const char kRtcpFbParamTransportCc[]; +// ccm submessages according to RFC 5104 +extern const char kRtcpFbParamCcm[]; +extern const char kRtcpFbCcmParamFir[]; +// Receiver reference time report +// https://tools.ietf.org/html/rfc3611 section 4.4 +extern const char kRtcpFbParamRrtr[]; +// Google specific parameters +extern const char kCodecParamMaxBitrate[]; +extern const char kCodecParamMinBitrate[]; +extern const char kCodecParamStartBitrate[]; +extern const char kCodecParamMaxQuantization[]; +extern const char kCodecParamPort[]; +extern const char kCodecParamMaxMessageSize[]; + +// We put the data codec names here so callers of DataEngine::CreateChannel +// don't have to import rtpdataengine.h to get the codec names they want to +// pass in. +extern const int kGoogleRtpDataCodecPlType; +extern const char kGoogleRtpDataCodecName[]; + +// TODO(pthatcher): Find an id that won't conflict with anything. On +// the other hand, it really shouldn't matter since the id won't be +// used on the wire. +extern const int kGoogleSctpDataCodecPlType; +extern const char kGoogleSctpDataCodecName[]; + +extern const char kComfortNoiseCodecName[]; + +RTC_EXPORT extern const char kVp8CodecName[]; +RTC_EXPORT extern const char kVp9CodecName[]; +RTC_EXPORT extern const char kAv1CodecName[]; +RTC_EXPORT extern const char kH264CodecName[]; + +// RFC 6184 RTP Payload Format for H.264 video +RTC_EXPORT extern const char kH264FmtpProfileLevelId[]; +RTC_EXPORT extern const char kH264FmtpLevelAsymmetryAllowed[]; +RTC_EXPORT extern const char kH264FmtpPacketizationMode[]; +extern const char kH264FmtpSpropParameterSets[]; +extern const char kH264ProfileLevelConstrainedBaseline[]; + +extern const int kDefaultVideoMaxFramerate; + +extern const size_t kConferenceMaxNumSpatialLayers; +extern const size_t kConferenceMaxNumTemporalLayers; +extern const size_t kConferenceDefaultNumTemporalLayers; +} // namespace cricket + +#endif // MEDIA_BASE_MEDIA_CONSTANTS_H_ diff --git a/media/base/media_engine.cc b/media/base/media_engine.cc new file mode 100644 index 0000000000..8050258728 --- /dev/null +++ b/media/base/media_engine.cc @@ -0,0 +1,192 @@ +/* + * Copyright (c) 2004 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "media/base/media_engine.h" + +#include <stddef.h> + +#include <cstdint> +#include <string> +#include <utility> + +#include "absl/algorithm/container.h" +#include "api/video/video_bitrate_allocation.h" +#include "rtc_base/checks.h" +#include "rtc_base/string_encode.h" + +namespace cricket { + +RtpCapabilities::RtpCapabilities() = default; +RtpCapabilities::~RtpCapabilities() = default; + +webrtc::RtpParameters CreateRtpParametersWithOneEncoding() { + webrtc::RtpParameters parameters; + webrtc::RtpEncodingParameters encoding; + parameters.encodings.push_back(encoding); + return parameters; +} + +webrtc::RtpParameters CreateRtpParametersWithEncodings(StreamParams sp) { + std::vector<uint32_t> primary_ssrcs; + sp.GetPrimarySsrcs(&primary_ssrcs); + size_t encoding_count = primary_ssrcs.size(); + + std::vector<webrtc::RtpEncodingParameters> encodings(encoding_count); + for (size_t i = 0; i < encodings.size(); ++i) { + encodings[i].ssrc = primary_ssrcs[i]; + } + + const std::vector<RidDescription>& rids = sp.rids(); + RTC_DCHECK(rids.size() == 0 || rids.size() == encoding_count); + for (size_t i = 0; i < rids.size(); ++i) { + encodings[i].rid = rids[i].rid; + } + + webrtc::RtpParameters parameters; + parameters.encodings = encodings; + parameters.rtcp.cname = sp.cname; + return parameters; +} + +std::vector<webrtc::RtpExtension> GetDefaultEnabledRtpHeaderExtensions( + const RtpHeaderExtensionQueryInterface& query_interface) { + std::vector<webrtc::RtpExtension> extensions; + for (const auto& entry : query_interface.GetRtpHeaderExtensions()) { + if (entry.direction != webrtc::RtpTransceiverDirection::kStopped) + extensions.emplace_back(entry.uri, *entry.preferred_id); + } + return extensions; +} + +webrtc::RTCError CheckRtpParametersValues( + const webrtc::RtpParameters& rtp_parameters) { + using webrtc::RTCErrorType; + + for (size_t i = 0; i < rtp_parameters.encodings.size(); ++i) { + if (rtp_parameters.encodings[i].bitrate_priority <= 0) { + LOG_AND_RETURN_ERROR(RTCErrorType::INVALID_RANGE, + "Attempted to set RtpParameters bitrate_priority to " + "an invalid number. bitrate_priority must be > 0."); + } + if (rtp_parameters.encodings[i].scale_resolution_down_by && + *rtp_parameters.encodings[i].scale_resolution_down_by < 1.0) { + LOG_AND_RETURN_ERROR( + RTCErrorType::INVALID_RANGE, + "Attempted to set RtpParameters scale_resolution_down_by to an " + "invalid value. scale_resolution_down_by must be >= 1.0"); + } + if (rtp_parameters.encodings[i].max_framerate && + *rtp_parameters.encodings[i].max_framerate < 0.0) { + LOG_AND_RETURN_ERROR(RTCErrorType::INVALID_RANGE, + "Attempted to set RtpParameters max_framerate to an " + "invalid value. max_framerate must be >= 0.0"); + } + if (rtp_parameters.encodings[i].min_bitrate_bps && + rtp_parameters.encodings[i].max_bitrate_bps) { + if (*rtp_parameters.encodings[i].max_bitrate_bps < + *rtp_parameters.encodings[i].min_bitrate_bps) { + LOG_AND_RETURN_ERROR(webrtc::RTCErrorType::INVALID_RANGE, + "Attempted to set RtpParameters min bitrate " + "larger than max bitrate."); + } + } + if (rtp_parameters.encodings[i].num_temporal_layers) { + if (*rtp_parameters.encodings[i].num_temporal_layers < 1 || + *rtp_parameters.encodings[i].num_temporal_layers > + webrtc::kMaxTemporalStreams) { + LOG_AND_RETURN_ERROR(RTCErrorType::INVALID_RANGE, + "Attempted to set RtpParameters " + "num_temporal_layers to an invalid number."); + } + } + if (i > 0 && (rtp_parameters.encodings[i].num_temporal_layers != + rtp_parameters.encodings[i - 1].num_temporal_layers)) { + LOG_AND_RETURN_ERROR( + RTCErrorType::INVALID_MODIFICATION, + "Attempted to set RtpParameters num_temporal_layers " + "at encoding layer i: " + + rtc::ToString(i) + + " to a different value than other encoding layers."); + } + } + + return webrtc::RTCError::OK(); +} + +webrtc::RTCError CheckRtpParametersInvalidModificationAndValues( + const webrtc::RtpParameters& old_rtp_parameters, + const webrtc::RtpParameters& rtp_parameters) { + using webrtc::RTCErrorType; + if (rtp_parameters.encodings.size() != old_rtp_parameters.encodings.size()) { + LOG_AND_RETURN_ERROR( + RTCErrorType::INVALID_MODIFICATION, + "Attempted to set RtpParameters with different encoding count"); + } + if (rtp_parameters.rtcp != old_rtp_parameters.rtcp) { + LOG_AND_RETURN_ERROR( + RTCErrorType::INVALID_MODIFICATION, + "Attempted to set RtpParameters with modified RTCP parameters"); + } + if (rtp_parameters.header_extensions != + old_rtp_parameters.header_extensions) { + LOG_AND_RETURN_ERROR( + RTCErrorType::INVALID_MODIFICATION, + "Attempted to set RtpParameters with modified header extensions"); + } + if (!absl::c_equal(old_rtp_parameters.encodings, rtp_parameters.encodings, + [](const webrtc::RtpEncodingParameters& encoding1, + const webrtc::RtpEncodingParameters& encoding2) { + return encoding1.rid == encoding2.rid; + })) { + LOG_AND_RETURN_ERROR(RTCErrorType::INVALID_MODIFICATION, + "Attempted to change RID values in the encodings."); + } + if (!absl::c_equal(old_rtp_parameters.encodings, rtp_parameters.encodings, + [](const webrtc::RtpEncodingParameters& encoding1, + const webrtc::RtpEncodingParameters& encoding2) { + return encoding1.ssrc == encoding2.ssrc; + })) { + LOG_AND_RETURN_ERROR(RTCErrorType::INVALID_MODIFICATION, + "Attempted to set RtpParameters with modified SSRC"); + } + + return CheckRtpParametersValues(rtp_parameters); +} + +CompositeMediaEngine::CompositeMediaEngine( + std::unique_ptr<VoiceEngineInterface> voice_engine, + std::unique_ptr<VideoEngineInterface> video_engine) + : voice_engine_(std::move(voice_engine)), + video_engine_(std::move(video_engine)) {} + +CompositeMediaEngine::~CompositeMediaEngine() = default; + +bool CompositeMediaEngine::Init() { + voice().Init(); + return true; +} + +VoiceEngineInterface& CompositeMediaEngine::voice() { + return *voice_engine_.get(); +} + +VideoEngineInterface& CompositeMediaEngine::video() { + return *video_engine_.get(); +} + +const VoiceEngineInterface& CompositeMediaEngine::voice() const { + return *voice_engine_.get(); +} + +const VideoEngineInterface& CompositeMediaEngine::video() const { + return *video_engine_.get(); +} + +} // namespace cricket diff --git a/media/base/media_engine.h b/media/base/media_engine.h new file mode 100644 index 0000000000..be0ae59a04 --- /dev/null +++ b/media/base/media_engine.h @@ -0,0 +1,188 @@ +/* + * Copyright (c) 2004 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MEDIA_BASE_MEDIA_ENGINE_H_ +#define MEDIA_BASE_MEDIA_ENGINE_H_ + +#include <memory> +#include <string> +#include <vector> + +#include "api/audio_codecs/audio_decoder_factory.h" +#include "api/audio_codecs/audio_encoder_factory.h" +#include "api/crypto/crypto_options.h" +#include "api/rtp_parameters.h" +#include "api/video/video_bitrate_allocator_factory.h" +#include "call/audio_state.h" +#include "media/base/codec.h" +#include "media/base/media_channel.h" +#include "media/base/video_common.h" +#include "rtc_base/system/file_wrapper.h" + +namespace webrtc { +class AudioDeviceModule; +class AudioMixer; +class AudioProcessing; +class Call; +} // namespace webrtc + +namespace cricket { + +webrtc::RTCError CheckRtpParametersValues( + const webrtc::RtpParameters& new_parameters); + +webrtc::RTCError CheckRtpParametersInvalidModificationAndValues( + const webrtc::RtpParameters& old_parameters, + const webrtc::RtpParameters& new_parameters); + +struct RtpCapabilities { + RtpCapabilities(); + ~RtpCapabilities(); + std::vector<webrtc::RtpExtension> header_extensions; +}; + +class RtpHeaderExtensionQueryInterface { + public: + virtual ~RtpHeaderExtensionQueryInterface() = default; + + // Returns a vector of RtpHeaderExtensionCapability, whose direction is + // kStopped if the extension is stopped (not used) by default. + virtual std::vector<webrtc::RtpHeaderExtensionCapability> + GetRtpHeaderExtensions() const = 0; +}; + +class VoiceEngineInterface : public RtpHeaderExtensionQueryInterface { + public: + VoiceEngineInterface() = default; + virtual ~VoiceEngineInterface() = default; + RTC_DISALLOW_COPY_AND_ASSIGN(VoiceEngineInterface); + + // Initialization + // Starts the engine. + virtual void Init() = 0; + + // TODO(solenberg): Remove once VoE API refactoring is done. + virtual rtc::scoped_refptr<webrtc::AudioState> GetAudioState() const = 0; + + // MediaChannel creation + // Creates a voice media channel. Returns NULL on failure. + virtual VoiceMediaChannel* CreateMediaChannel( + webrtc::Call* call, + const MediaConfig& config, + const AudioOptions& options, + const webrtc::CryptoOptions& crypto_options) = 0; + + virtual const std::vector<AudioCodec>& send_codecs() const = 0; + virtual const std::vector<AudioCodec>& recv_codecs() const = 0; + + // Starts AEC dump using existing file, a maximum file size in bytes can be + // specified. Logging is stopped just before the size limit is exceeded. + // If max_size_bytes is set to a value <= 0, no limit will be used. + virtual bool StartAecDump(webrtc::FileWrapper file, + int64_t max_size_bytes) = 0; + + // Stops recording AEC dump. + virtual void StopAecDump() = 0; +}; + +class VideoEngineInterface : public RtpHeaderExtensionQueryInterface { + public: + VideoEngineInterface() = default; + virtual ~VideoEngineInterface() = default; + RTC_DISALLOW_COPY_AND_ASSIGN(VideoEngineInterface); + + // Creates a video media channel, paired with the specified voice channel. + // Returns NULL on failure. + virtual VideoMediaChannel* CreateMediaChannel( + webrtc::Call* call, + const MediaConfig& config, + const VideoOptions& options, + const webrtc::CryptoOptions& crypto_options, + webrtc::VideoBitrateAllocatorFactory* + video_bitrate_allocator_factory) = 0; + + virtual std::vector<VideoCodec> send_codecs() const = 0; + virtual std::vector<VideoCodec> recv_codecs() const = 0; +}; + +// MediaEngineInterface is an abstraction of a media engine which can be +// subclassed to support different media componentry backends. +// It supports voice and video operations in the same class to facilitate +// proper synchronization between both media types. +class MediaEngineInterface { + public: + virtual ~MediaEngineInterface() {} + + // Initialization + // Starts the engine. + virtual bool Init() = 0; + virtual VoiceEngineInterface& voice() = 0; + virtual VideoEngineInterface& video() = 0; + virtual const VoiceEngineInterface& voice() const = 0; + virtual const VideoEngineInterface& video() const = 0; +}; + +// CompositeMediaEngine constructs a MediaEngine from separate +// voice and video engine classes. +class CompositeMediaEngine : public MediaEngineInterface { + public: + CompositeMediaEngine(std::unique_ptr<VoiceEngineInterface> audio_engine, + std::unique_ptr<VideoEngineInterface> video_engine); + ~CompositeMediaEngine() override; + bool Init() override; + + VoiceEngineInterface& voice() override; + VideoEngineInterface& video() override; + const VoiceEngineInterface& voice() const override; + const VideoEngineInterface& video() const override; + + private: + std::unique_ptr<VoiceEngineInterface> voice_engine_; + std::unique_ptr<VideoEngineInterface> video_engine_; +}; + +enum DataChannelType { + DCT_NONE = 0, + DCT_RTP = 1, + DCT_SCTP = 2, + + // Data channel transport over media transport. + DCT_MEDIA_TRANSPORT = 3, + + // Data channel transport over datagram transport (with no fallback). This is + // the same behavior as data channel transport over media transport, and is + // usable without DTLS. + DCT_DATA_CHANNEL_TRANSPORT = 4, + + // Data channel transport over datagram transport (with SCTP negotiation + // semantics and a fallback to SCTP). Only usable with DTLS. + DCT_DATA_CHANNEL_TRANSPORT_SCTP = 5, +}; + +class DataEngineInterface { + public: + virtual ~DataEngineInterface() {} + virtual DataMediaChannel* CreateChannel(const MediaConfig& config) = 0; + virtual const std::vector<DataCodec>& data_codecs() = 0; +}; + +webrtc::RtpParameters CreateRtpParametersWithOneEncoding(); +webrtc::RtpParameters CreateRtpParametersWithEncodings(StreamParams sp); + +// Returns a vector of RTP extensions as visible from RtpSender/Receiver +// GetCapabilities(). The returned vector only shows what will definitely be +// offered by default, i.e. the list of extensions returned from +// GetRtpHeaderExtensions() that are not kStopped. +std::vector<webrtc::RtpExtension> GetDefaultEnabledRtpHeaderExtensions( + const RtpHeaderExtensionQueryInterface& query_interface); + +} // namespace cricket + +#endif // MEDIA_BASE_MEDIA_ENGINE_H_ diff --git a/media/base/media_engine_unittest.cc b/media/base/media_engine_unittest.cc new file mode 100644 index 0000000000..f4c6f5f045 --- /dev/null +++ b/media/base/media_engine_unittest.cc @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "media/base/media_engine.h" + +#include "test/gmock.h" + +using ::testing::ElementsAre; +using ::testing::Field; +using ::testing::Return; +using ::testing::StrEq; +using ::webrtc::RtpExtension; +using ::webrtc::RtpHeaderExtensionCapability; +using ::webrtc::RtpTransceiverDirection; + +namespace cricket { +namespace { + +class MockRtpHeaderExtensionQueryInterface + : public RtpHeaderExtensionQueryInterface { + public: + MOCK_CONST_METHOD0(GetRtpHeaderExtensions, + std::vector<RtpHeaderExtensionCapability>()); +}; + +} // namespace + +TEST(MediaEngineTest, ReturnsNotStoppedHeaderExtensions) { + MockRtpHeaderExtensionQueryInterface mock; + std::vector<RtpHeaderExtensionCapability> extensions( + {RtpHeaderExtensionCapability("uri1", 1, + RtpTransceiverDirection::kInactive), + RtpHeaderExtensionCapability("uri2", 2, + RtpTransceiverDirection::kSendRecv), + RtpHeaderExtensionCapability("uri3", 3, + RtpTransceiverDirection::kStopped), + RtpHeaderExtensionCapability("uri4", 4, + RtpTransceiverDirection::kSendOnly), + RtpHeaderExtensionCapability("uri5", 5, + RtpTransceiverDirection::kRecvOnly)}); + EXPECT_CALL(mock, GetRtpHeaderExtensions).WillOnce(Return(extensions)); + EXPECT_THAT(GetDefaultEnabledRtpHeaderExtensions(mock), + ElementsAre(Field(&RtpExtension::uri, StrEq("uri1")), + Field(&RtpExtension::uri, StrEq("uri2")), + Field(&RtpExtension::uri, StrEq("uri4")), + Field(&RtpExtension::uri, StrEq("uri5")))); +} + +} // namespace cricket diff --git a/media/base/rid_description.cc b/media/base/rid_description.cc new file mode 100644 index 0000000000..b3eae272f9 --- /dev/null +++ b/media/base/rid_description.cc @@ -0,0 +1,28 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "media/base/rid_description.h" + +namespace cricket { + +RidDescription::RidDescription() = default; +RidDescription::RidDescription(const std::string& rid, RidDirection direction) + : rid{rid}, direction{direction} {} +RidDescription::RidDescription(const RidDescription& other) = default; +RidDescription::~RidDescription() = default; +RidDescription& RidDescription::operator=(const RidDescription& other) = + default; +bool RidDescription::operator==(const RidDescription& other) const { + return rid == other.rid && direction == other.direction && + payload_types == other.payload_types && + restrictions == other.restrictions; +} + +} // namespace cricket diff --git a/media/base/rid_description.h b/media/base/rid_description.h new file mode 100644 index 0000000000..04c0f3d4bc --- /dev/null +++ b/media/base/rid_description.h @@ -0,0 +1,93 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MEDIA_BASE_RID_DESCRIPTION_H_ +#define MEDIA_BASE_RID_DESCRIPTION_H_ + +#include <map> +#include <string> +#include <vector> + +namespace cricket { + +enum class RidDirection { kSend, kReceive }; + +// Description of a Restriction Id (RID) according to: +// https://tools.ietf.org/html/draft-ietf-mmusic-rid-15 +// A Restriction Identifier serves two purposes: +// 1. Uniquely identifies an RTP stream inside an RTP session. +// When combined with MIDs (https://tools.ietf.org/html/rfc5888), +// RIDs uniquely identify an RTP stream within an RTP session. +// The MID will identify the media section and the RID will identify +// the stream within the section. +// RID identifiers must be unique within the media section. +// 2. Allows indicating further restrictions to the stream. +// These restrictions are added according to the direction specified. +// The direction field identifies the direction of the RTP stream packets +// to which the restrictions apply. The direction is independent of the +// transceiver direction and can be one of {send, recv}. +// The following are some examples of these restrictions: +// a. max-width, max-height, max-fps, max-br, ... +// b. further restricting the codec set (from what m= section specified) +// +// Note: Indicating dependencies between streams (using depend) will not be +// supported, since the WG is adopting a different approach to achieve this. +// As of 2018-12-04, the new SVC (Scalable Video Coder) approach is still not +// mature enough to be implemented as part of this work. +// See: https://w3c.github.io/webrtc-svc/ for more details. +struct RidDescription final { + RidDescription(); + RidDescription(const std::string& rid, RidDirection direction); + RidDescription(const RidDescription& other); + ~RidDescription(); + RidDescription& operator=(const RidDescription& other); + + // This is currently required for unit tests of StreamParams which contains + // RidDescription objects and checks for equality using operator==. + bool operator==(const RidDescription& other) const; + bool operator!=(const RidDescription& other) const { + return !(*this == other); + } + + // The RID identifier that uniquely identifies the stream within the session. + std::string rid; + + // Specifies the direction for which the specified restrictions hold. + // This direction is either send or receive and is independent of the + // direction of the transceiver. + // https://tools.ietf.org/html/draft-ietf-mmusic-rid-15#section-4 : + // The "direction" field identifies the direction of the RTP Stream + // packets to which the indicated restrictions are applied. It may be + // either "send" or "recv". Note that these restriction directions are + // expressed independently of any "inactive", "sendonly", "recvonly", or + // "sendrecv" attributes associated with the media section. It is, for + // example, valid to indicate "recv" restrictions on a "sendonly" + // stream; those restrictions would apply if, at a future point in time, + // the stream were changed to "sendrecv" or "recvonly". + RidDirection direction; + + // The list of codec payload types for this stream. + // It should be a subset of the payloads supported for the media section. + std::vector<int> payload_types; + + // Contains key-value pairs for restrictions. + // The keys are not validated against a known set. + // The meaning to infer for the values depends on each key. + // Examples: + // 1. An entry for max-width will have a value that is interpreted as an int. + // 2. An entry for max-bpp (bits per pixel) will have a float value. + // Interpretation (and validation of value) is left for the implementation. + // I.E. the media engines should validate values for parameters they support. + std::map<std::string, std::string> restrictions; +}; + +} // namespace cricket + +#endif // MEDIA_BASE_RID_DESCRIPTION_H_ diff --git a/media/base/rtp_data_engine.cc b/media/base/rtp_data_engine.cc new file mode 100644 index 0000000000..0303cd33d4 --- /dev/null +++ b/media/base/rtp_data_engine.cc @@ -0,0 +1,338 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "media/base/rtp_data_engine.h" + +#include <map> + +#include "absl/strings/match.h" +#include "media/base/codec.h" +#include "media/base/media_constants.h" +#include "media/base/rtp_utils.h" +#include "media/base/stream_params.h" +#include "rtc_base/copy_on_write_buffer.h" +#include "rtc_base/data_rate_limiter.h" +#include "rtc_base/helpers.h" +#include "rtc_base/logging.h" +#include "rtc_base/sanitizer.h" + +namespace cricket { + +// We want to avoid IP fragmentation. +static const size_t kDataMaxRtpPacketLen = 1200U; +// We reserve space after the RTP header for future wiggle room. +static const unsigned char kReservedSpace[] = {0x00, 0x00, 0x00, 0x00}; + +// Amount of overhead SRTP may take. We need to leave room in the +// buffer for it, otherwise SRTP will fail later. If SRTP ever uses +// more than this, we need to increase this number. +static const size_t kMaxSrtpHmacOverhead = 16; + +RtpDataEngine::RtpDataEngine() { + data_codecs_.push_back( + DataCodec(kGoogleRtpDataCodecPlType, kGoogleRtpDataCodecName)); +} + +DataMediaChannel* RtpDataEngine::CreateChannel(const MediaConfig& config) { + return new RtpDataMediaChannel(config); +} + +static const DataCodec* FindCodecByName(const std::vector<DataCodec>& codecs, + const std::string& name) { + for (const DataCodec& codec : codecs) { + if (absl::EqualsIgnoreCase(name, codec.name)) + return &codec; + } + return nullptr; +} + +RtpDataMediaChannel::RtpDataMediaChannel(const MediaConfig& config) + : DataMediaChannel(config) { + Construct(); + SetPreferredDscp(rtc::DSCP_AF41); +} + +void RtpDataMediaChannel::Construct() { + sending_ = false; + receiving_ = false; + send_limiter_.reset(new rtc::DataRateLimiter(kDataMaxBandwidth / 8, 1.0)); +} + +RtpDataMediaChannel::~RtpDataMediaChannel() { + std::map<uint32_t, RtpClock*>::const_iterator iter; + for (iter = rtp_clock_by_send_ssrc_.begin(); + iter != rtp_clock_by_send_ssrc_.end(); ++iter) { + delete iter->second; + } +} + +void RTC_NO_SANITIZE("float-cast-overflow") // bugs.webrtc.org/8204 + RtpClock::Tick(double now, int* seq_num, uint32_t* timestamp) { + *seq_num = ++last_seq_num_; + *timestamp = timestamp_offset_ + static_cast<uint32_t>(now * clockrate_); + // UBSan: 5.92374e+10 is outside the range of representable values of type + // 'unsigned int' +} + +const DataCodec* FindUnknownCodec(const std::vector<DataCodec>& codecs) { + DataCodec data_codec(kGoogleRtpDataCodecPlType, kGoogleRtpDataCodecName); + std::vector<DataCodec>::const_iterator iter; + for (iter = codecs.begin(); iter != codecs.end(); ++iter) { + if (!iter->Matches(data_codec)) { + return &(*iter); + } + } + return NULL; +} + +const DataCodec* FindKnownCodec(const std::vector<DataCodec>& codecs) { + DataCodec data_codec(kGoogleRtpDataCodecPlType, kGoogleRtpDataCodecName); + std::vector<DataCodec>::const_iterator iter; + for (iter = codecs.begin(); iter != codecs.end(); ++iter) { + if (iter->Matches(data_codec)) { + return &(*iter); + } + } + return NULL; +} + +bool RtpDataMediaChannel::SetRecvCodecs(const std::vector<DataCodec>& codecs) { + const DataCodec* unknown_codec = FindUnknownCodec(codecs); + if (unknown_codec) { + RTC_LOG(LS_WARNING) << "Failed to SetRecvCodecs because of unknown codec: " + << unknown_codec->ToString(); + return false; + } + + recv_codecs_ = codecs; + return true; +} + +bool RtpDataMediaChannel::SetSendCodecs(const std::vector<DataCodec>& codecs) { + const DataCodec* known_codec = FindKnownCodec(codecs); + if (!known_codec) { + RTC_LOG(LS_WARNING) + << "Failed to SetSendCodecs because there is no known codec."; + return false; + } + + send_codecs_ = codecs; + return true; +} + +bool RtpDataMediaChannel::SetSendParameters(const DataSendParameters& params) { + return (SetSendCodecs(params.codecs) && + SetMaxSendBandwidth(params.max_bandwidth_bps)); +} + +bool RtpDataMediaChannel::SetRecvParameters(const DataRecvParameters& params) { + return SetRecvCodecs(params.codecs); +} + +bool RtpDataMediaChannel::AddSendStream(const StreamParams& stream) { + if (!stream.has_ssrcs()) { + return false; + } + + if (GetStreamBySsrc(send_streams_, stream.first_ssrc())) { + RTC_LOG(LS_WARNING) << "Not adding data send stream '" << stream.id + << "' with ssrc=" << stream.first_ssrc() + << " because stream already exists."; + return false; + } + + send_streams_.push_back(stream); + // TODO(pthatcher): This should be per-stream, not per-ssrc. + // And we should probably allow more than one per stream. + rtp_clock_by_send_ssrc_[stream.first_ssrc()] = + new RtpClock(kDataCodecClockrate, rtc::CreateRandomNonZeroId(), + rtc::CreateRandomNonZeroId()); + + RTC_LOG(LS_INFO) << "Added data send stream '" << stream.id + << "' with ssrc=" << stream.first_ssrc(); + return true; +} + +bool RtpDataMediaChannel::RemoveSendStream(uint32_t ssrc) { + if (!GetStreamBySsrc(send_streams_, ssrc)) { + return false; + } + + RemoveStreamBySsrc(&send_streams_, ssrc); + delete rtp_clock_by_send_ssrc_[ssrc]; + rtp_clock_by_send_ssrc_.erase(ssrc); + return true; +} + +bool RtpDataMediaChannel::AddRecvStream(const StreamParams& stream) { + if (!stream.has_ssrcs()) { + return false; + } + + if (GetStreamBySsrc(recv_streams_, stream.first_ssrc())) { + RTC_LOG(LS_WARNING) << "Not adding data recv stream '" << stream.id + << "' with ssrc=" << stream.first_ssrc() + << " because stream already exists."; + return false; + } + + recv_streams_.push_back(stream); + RTC_LOG(LS_INFO) << "Added data recv stream '" << stream.id + << "' with ssrc=" << stream.first_ssrc(); + return true; +} + +bool RtpDataMediaChannel::RemoveRecvStream(uint32_t ssrc) { + RemoveStreamBySsrc(&recv_streams_, ssrc); + return true; +} + +// Not implemented. +void RtpDataMediaChannel::ResetUnsignaledRecvStream() {} + +void RtpDataMediaChannel::OnPacketReceived(rtc::CopyOnWriteBuffer packet, + int64_t /* packet_time_us */) { + RtpHeader header; + if (!GetRtpHeader(packet.cdata(), packet.size(), &header)) { + return; + } + + size_t header_length; + if (!GetRtpHeaderLen(packet.cdata(), packet.size(), &header_length)) { + return; + } + const char* data = + packet.cdata<char>() + header_length + sizeof(kReservedSpace); + size_t data_len = packet.size() - header_length - sizeof(kReservedSpace); + + if (!receiving_) { + RTC_LOG(LS_WARNING) << "Not receiving packet " << header.ssrc << ":" + << header.seq_num << " before SetReceive(true) called."; + return; + } + + if (!FindCodecById(recv_codecs_, header.payload_type)) { + return; + } + + if (!GetStreamBySsrc(recv_streams_, header.ssrc)) { + RTC_LOG(LS_WARNING) << "Received packet for unknown ssrc: " << header.ssrc; + return; + } + + // Uncomment this for easy debugging. + // const auto* found_stream = GetStreamBySsrc(recv_streams_, header.ssrc); + // RTC_LOG(LS_INFO) << "Received packet" + // << " groupid=" << found_stream.groupid + // << ", ssrc=" << header.ssrc + // << ", seqnum=" << header.seq_num + // << ", timestamp=" << header.timestamp + // << ", len=" << data_len; + + ReceiveDataParams params; + params.ssrc = header.ssrc; + params.seq_num = header.seq_num; + params.timestamp = header.timestamp; + SignalDataReceived(params, data, data_len); +} + +bool RtpDataMediaChannel::SetMaxSendBandwidth(int bps) { + if (bps <= 0) { + bps = kDataMaxBandwidth; + } + send_limiter_.reset(new rtc::DataRateLimiter(bps / 8, 1.0)); + RTC_LOG(LS_INFO) << "RtpDataMediaChannel::SetSendBandwidth to " << bps + << "bps."; + return true; +} + +bool RtpDataMediaChannel::SendData(const SendDataParams& params, + const rtc::CopyOnWriteBuffer& payload, + SendDataResult* result) { + if (result) { + // If we return true, we'll set this to SDR_SUCCESS. + *result = SDR_ERROR; + } + if (!sending_) { + RTC_LOG(LS_WARNING) << "Not sending packet with ssrc=" << params.ssrc + << " len=" << payload.size() + << " before SetSend(true)."; + return false; + } + + if (params.type != cricket::DMT_TEXT) { + RTC_LOG(LS_WARNING) + << "Not sending data because binary type is unsupported."; + return false; + } + + const StreamParams* found_stream = + GetStreamBySsrc(send_streams_, params.ssrc); + if (!found_stream) { + RTC_LOG(LS_WARNING) << "Not sending data because ssrc is unknown: " + << params.ssrc; + return false; + } + + const DataCodec* found_codec = + FindCodecByName(send_codecs_, kGoogleRtpDataCodecName); + if (!found_codec) { + RTC_LOG(LS_WARNING) << "Not sending data because codec is unknown: " + << kGoogleRtpDataCodecName; + return false; + } + + size_t packet_len = (kMinRtpPacketLen + sizeof(kReservedSpace) + + payload.size() + kMaxSrtpHmacOverhead); + if (packet_len > kDataMaxRtpPacketLen) { + return false; + } + + double now = + rtc::TimeMicros() / static_cast<double>(rtc::kNumMicrosecsPerSec); + + if (!send_limiter_->CanUse(packet_len, now)) { + RTC_LOG(LS_VERBOSE) << "Dropped data packet of len=" << packet_len + << "; already sent " << send_limiter_->used_in_period() + << "/" << send_limiter_->max_per_period(); + return false; + } + + RtpHeader header; + header.payload_type = found_codec->id; + header.ssrc = params.ssrc; + rtp_clock_by_send_ssrc_[header.ssrc]->Tick(now, &header.seq_num, + &header.timestamp); + + rtc::CopyOnWriteBuffer packet(kMinRtpPacketLen, packet_len); + if (!SetRtpHeader(packet.data(), packet.size(), header)) { + return false; + } + packet.AppendData(kReservedSpace); + packet.AppendData(payload); + + RTC_LOG(LS_VERBOSE) << "Sent RTP data packet: " + " stream=" + << found_stream->id << " ssrc=" << header.ssrc + << ", seqnum=" << header.seq_num + << ", timestamp=" << header.timestamp + << ", len=" << payload.size(); + + rtc::PacketOptions options; + options.info_signaled_after_sent.packet_type = rtc::PacketType::kData; + MediaChannel::SendPacket(&packet, options); + send_limiter_->Use(packet_len, now); + if (result) { + *result = SDR_SUCCESS; + } + return true; +} + +} // namespace cricket diff --git a/media/base/rtp_data_engine.h b/media/base/rtp_data_engine.h new file mode 100644 index 0000000000..e5f071d5a9 --- /dev/null +++ b/media/base/rtp_data_engine.h @@ -0,0 +1,109 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MEDIA_BASE_RTP_DATA_ENGINE_H_ +#define MEDIA_BASE_RTP_DATA_ENGINE_H_ + +#include <map> +#include <memory> +#include <string> +#include <vector> + +#include "media/base/codec.h" +#include "media/base/media_channel.h" +#include "media/base/media_constants.h" +#include "media/base/media_engine.h" + +namespace rtc { +class DataRateLimiter; +} + +namespace cricket { + +class RtpDataEngine : public DataEngineInterface { + public: + RtpDataEngine(); + + virtual DataMediaChannel* CreateChannel(const MediaConfig& config); + + virtual const std::vector<DataCodec>& data_codecs() { return data_codecs_; } + + private: + std::vector<DataCodec> data_codecs_; +}; + +// Keep track of sequence number and timestamp of an RTP stream. The +// sequence number starts with a "random" value and increments. The +// timestamp starts with a "random" value and increases monotonically +// according to the clockrate. +class RtpClock { + public: + RtpClock(int clockrate, uint16_t first_seq_num, uint32_t timestamp_offset) + : clockrate_(clockrate), + last_seq_num_(first_seq_num), + timestamp_offset_(timestamp_offset) {} + + // Given the current time (in number of seconds which must be + // monotonically increasing), Return the next sequence number and + // timestamp. + void Tick(double now, int* seq_num, uint32_t* timestamp); + + private: + int clockrate_; + uint16_t last_seq_num_; + uint32_t timestamp_offset_; +}; + +class RtpDataMediaChannel : public DataMediaChannel { + public: + explicit RtpDataMediaChannel(const MediaConfig& config); + virtual ~RtpDataMediaChannel(); + + virtual bool SetSendParameters(const DataSendParameters& params); + virtual bool SetRecvParameters(const DataRecvParameters& params); + virtual bool AddSendStream(const StreamParams& sp); + virtual bool RemoveSendStream(uint32_t ssrc); + virtual bool AddRecvStream(const StreamParams& sp); + virtual bool RemoveRecvStream(uint32_t ssrc); + virtual void ResetUnsignaledRecvStream(); + virtual bool SetSend(bool send) { + sending_ = send; + return true; + } + virtual bool SetReceive(bool receive) { + receiving_ = receive; + return true; + } + virtual void OnPacketReceived(rtc::CopyOnWriteBuffer packet, + int64_t packet_time_us); + virtual void OnReadyToSend(bool ready) {} + virtual bool SendData(const SendDataParams& params, + const rtc::CopyOnWriteBuffer& payload, + SendDataResult* result); + + private: + void Construct(); + bool SetMaxSendBandwidth(int bps); + bool SetSendCodecs(const std::vector<DataCodec>& codecs); + bool SetRecvCodecs(const std::vector<DataCodec>& codecs); + + bool sending_; + bool receiving_; + std::vector<DataCodec> send_codecs_; + std::vector<DataCodec> recv_codecs_; + std::vector<StreamParams> send_streams_; + std::vector<StreamParams> recv_streams_; + std::map<uint32_t, RtpClock*> rtp_clock_by_send_ssrc_; + std::unique_ptr<rtc::DataRateLimiter> send_limiter_; +}; + +} // namespace cricket + +#endif // MEDIA_BASE_RTP_DATA_ENGINE_H_ diff --git a/media/base/rtp_data_engine_unittest.cc b/media/base/rtp_data_engine_unittest.cc new file mode 100644 index 0000000000..dab4058c33 --- /dev/null +++ b/media/base/rtp_data_engine_unittest.cc @@ -0,0 +1,362 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "media/base/rtp_data_engine.h" + +#include <string.h> + +#include <memory> +#include <string> + +#include "media/base/fake_network_interface.h" +#include "media/base/media_constants.h" +#include "media/base/rtp_utils.h" +#include "rtc_base/copy_on_write_buffer.h" +#include "rtc_base/fake_clock.h" +#include "rtc_base/third_party/sigslot/sigslot.h" +#include "rtc_base/time_utils.h" +#include "test/gtest.h" + +class FakeDataReceiver : public sigslot::has_slots<> { + public: + FakeDataReceiver() : has_received_data_(false) {} + + void OnDataReceived(const cricket::ReceiveDataParams& params, + const char* data, + size_t len) { + has_received_data_ = true; + last_received_data_ = std::string(data, len); + last_received_data_len_ = len; + last_received_data_params_ = params; + } + + bool has_received_data() const { return has_received_data_; } + std::string last_received_data() const { return last_received_data_; } + size_t last_received_data_len() const { return last_received_data_len_; } + cricket::ReceiveDataParams last_received_data_params() const { + return last_received_data_params_; + } + + private: + bool has_received_data_; + std::string last_received_data_; + size_t last_received_data_len_; + cricket::ReceiveDataParams last_received_data_params_; +}; + +class RtpDataMediaChannelTest : public ::testing::Test { + protected: + virtual void SetUp() { + // Seed needed for each test to satisfy expectations. + iface_.reset(new cricket::FakeNetworkInterface()); + dme_.reset(CreateEngine()); + receiver_.reset(new FakeDataReceiver()); + } + + void SetNow(double now) { clock_.SetTime(webrtc::Timestamp::Seconds(now)); } + + cricket::RtpDataEngine* CreateEngine() { + cricket::RtpDataEngine* dme = new cricket::RtpDataEngine(); + return dme; + } + + cricket::RtpDataMediaChannel* CreateChannel() { + return CreateChannel(dme_.get()); + } + + cricket::RtpDataMediaChannel* CreateChannel(cricket::RtpDataEngine* dme) { + cricket::MediaConfig config; + cricket::RtpDataMediaChannel* channel = + static_cast<cricket::RtpDataMediaChannel*>(dme->CreateChannel(config)); + channel->SetInterface(iface_.get(), webrtc::MediaTransportConfig()); + channel->SignalDataReceived.connect(receiver_.get(), + &FakeDataReceiver::OnDataReceived); + return channel; + } + + FakeDataReceiver* receiver() { return receiver_.get(); } + + bool HasReceivedData() { return receiver_->has_received_data(); } + + std::string GetReceivedData() { return receiver_->last_received_data(); } + + size_t GetReceivedDataLen() { return receiver_->last_received_data_len(); } + + cricket::ReceiveDataParams GetReceivedDataParams() { + return receiver_->last_received_data_params(); + } + + bool HasSentData(int count) { return (iface_->NumRtpPackets() > count); } + + std::string GetSentData(int index) { + // Assume RTP header of length 12 + std::unique_ptr<const rtc::CopyOnWriteBuffer> packet( + iface_->GetRtpPacket(index)); + if (packet->size() > 12) { + return std::string(packet->data<char>() + 12, packet->size() - 12); + } else { + return ""; + } + } + + cricket::RtpHeader GetSentDataHeader(int index) { + std::unique_ptr<const rtc::CopyOnWriteBuffer> packet( + iface_->GetRtpPacket(index)); + cricket::RtpHeader header; + GetRtpHeader(packet->data(), packet->size(), &header); + return header; + } + + private: + std::unique_ptr<cricket::RtpDataEngine> dme_; + rtc::ScopedFakeClock clock_; + std::unique_ptr<cricket::FakeNetworkInterface> iface_; + std::unique_ptr<FakeDataReceiver> receiver_; +}; + +TEST_F(RtpDataMediaChannelTest, SetUnknownCodecs) { + std::unique_ptr<cricket::RtpDataMediaChannel> dmc(CreateChannel()); + + cricket::DataCodec known_codec; + known_codec.id = 103; + known_codec.name = "google-data"; + cricket::DataCodec unknown_codec; + unknown_codec.id = 104; + unknown_codec.name = "unknown-data"; + + cricket::DataSendParameters send_parameters_known; + send_parameters_known.codecs.push_back(known_codec); + cricket::DataRecvParameters recv_parameters_known; + recv_parameters_known.codecs.push_back(known_codec); + + cricket::DataSendParameters send_parameters_unknown; + send_parameters_unknown.codecs.push_back(unknown_codec); + cricket::DataRecvParameters recv_parameters_unknown; + recv_parameters_unknown.codecs.push_back(unknown_codec); + + cricket::DataSendParameters send_parameters_mixed; + send_parameters_mixed.codecs.push_back(known_codec); + send_parameters_mixed.codecs.push_back(unknown_codec); + cricket::DataRecvParameters recv_parameters_mixed; + recv_parameters_mixed.codecs.push_back(known_codec); + recv_parameters_mixed.codecs.push_back(unknown_codec); + + EXPECT_TRUE(dmc->SetSendParameters(send_parameters_known)); + EXPECT_FALSE(dmc->SetSendParameters(send_parameters_unknown)); + EXPECT_TRUE(dmc->SetSendParameters(send_parameters_mixed)); + EXPECT_TRUE(dmc->SetRecvParameters(recv_parameters_known)); + EXPECT_FALSE(dmc->SetRecvParameters(recv_parameters_unknown)); + EXPECT_FALSE(dmc->SetRecvParameters(recv_parameters_mixed)); +} + +TEST_F(RtpDataMediaChannelTest, AddRemoveSendStream) { + std::unique_ptr<cricket::RtpDataMediaChannel> dmc(CreateChannel()); + + cricket::StreamParams stream1; + stream1.add_ssrc(41); + EXPECT_TRUE(dmc->AddSendStream(stream1)); + cricket::StreamParams stream2; + stream2.add_ssrc(42); + EXPECT_TRUE(dmc->AddSendStream(stream2)); + + EXPECT_TRUE(dmc->RemoveSendStream(41)); + EXPECT_TRUE(dmc->RemoveSendStream(42)); + EXPECT_FALSE(dmc->RemoveSendStream(43)); +} + +TEST_F(RtpDataMediaChannelTest, AddRemoveRecvStream) { + std::unique_ptr<cricket::RtpDataMediaChannel> dmc(CreateChannel()); + + cricket::StreamParams stream1; + stream1.add_ssrc(41); + EXPECT_TRUE(dmc->AddRecvStream(stream1)); + cricket::StreamParams stream2; + stream2.add_ssrc(42); + EXPECT_TRUE(dmc->AddRecvStream(stream2)); + EXPECT_FALSE(dmc->AddRecvStream(stream2)); + + EXPECT_TRUE(dmc->RemoveRecvStream(41)); + EXPECT_TRUE(dmc->RemoveRecvStream(42)); +} + +TEST_F(RtpDataMediaChannelTest, SendData) { + std::unique_ptr<cricket::RtpDataMediaChannel> dmc(CreateChannel()); + + cricket::SendDataParams params; + params.ssrc = 42; + unsigned char data[] = "food"; + rtc::CopyOnWriteBuffer payload(data, 4); + unsigned char padded_data[] = { + 0x00, 0x00, 0x00, 0x00, 'f', 'o', 'o', 'd', + }; + cricket::SendDataResult result; + + // Not sending + EXPECT_FALSE(dmc->SendData(params, payload, &result)); + EXPECT_EQ(cricket::SDR_ERROR, result); + EXPECT_FALSE(HasSentData(0)); + ASSERT_TRUE(dmc->SetSend(true)); + + // Unknown stream name. + EXPECT_FALSE(dmc->SendData(params, payload, &result)); + EXPECT_EQ(cricket::SDR_ERROR, result); + EXPECT_FALSE(HasSentData(0)); + + cricket::StreamParams stream; + stream.add_ssrc(42); + ASSERT_TRUE(dmc->AddSendStream(stream)); + + // Unknown codec; + EXPECT_FALSE(dmc->SendData(params, payload, &result)); + EXPECT_EQ(cricket::SDR_ERROR, result); + EXPECT_FALSE(HasSentData(0)); + + cricket::DataCodec codec; + codec.id = 103; + codec.name = cricket::kGoogleRtpDataCodecName; + cricket::DataSendParameters parameters; + parameters.codecs.push_back(codec); + ASSERT_TRUE(dmc->SetSendParameters(parameters)); + + // Length too large; + std::string x10000(10000, 'x'); + EXPECT_FALSE(dmc->SendData( + params, rtc::CopyOnWriteBuffer(x10000.data(), x10000.length()), &result)); + EXPECT_EQ(cricket::SDR_ERROR, result); + EXPECT_FALSE(HasSentData(0)); + + // Finally works! + EXPECT_TRUE(dmc->SendData(params, payload, &result)); + EXPECT_EQ(cricket::SDR_SUCCESS, result); + ASSERT_TRUE(HasSentData(0)); + EXPECT_EQ(sizeof(padded_data), GetSentData(0).length()); + EXPECT_EQ(0, memcmp(padded_data, GetSentData(0).data(), sizeof(padded_data))); + cricket::RtpHeader header0 = GetSentDataHeader(0); + EXPECT_NE(0, header0.seq_num); + EXPECT_NE(0U, header0.timestamp); + EXPECT_EQ(header0.ssrc, 42U); + EXPECT_EQ(header0.payload_type, 103); + + // Should bump timestamp by 180000 because the clock rate is 90khz. + SetNow(2); + + EXPECT_TRUE(dmc->SendData(params, payload, &result)); + ASSERT_TRUE(HasSentData(1)); + EXPECT_EQ(sizeof(padded_data), GetSentData(1).length()); + EXPECT_EQ(0, memcmp(padded_data, GetSentData(1).data(), sizeof(padded_data))); + cricket::RtpHeader header1 = GetSentDataHeader(1); + EXPECT_EQ(header1.ssrc, 42U); + EXPECT_EQ(header1.payload_type, 103); + EXPECT_EQ(static_cast<uint16_t>(header0.seq_num + 1), + static_cast<uint16_t>(header1.seq_num)); + EXPECT_EQ(header0.timestamp + 180000, header1.timestamp); +} + +TEST_F(RtpDataMediaChannelTest, SendDataRate) { + std::unique_ptr<cricket::RtpDataMediaChannel> dmc(CreateChannel()); + + ASSERT_TRUE(dmc->SetSend(true)); + + cricket::DataCodec codec; + codec.id = 103; + codec.name = cricket::kGoogleRtpDataCodecName; + cricket::DataSendParameters parameters; + parameters.codecs.push_back(codec); + ASSERT_TRUE(dmc->SetSendParameters(parameters)); + + cricket::StreamParams stream; + stream.add_ssrc(42); + ASSERT_TRUE(dmc->AddSendStream(stream)); + + cricket::SendDataParams params; + params.ssrc = 42; + unsigned char data[] = "food"; + rtc::CopyOnWriteBuffer payload(data, 4); + cricket::SendDataResult result; + + // With rtp overhead of 32 bytes, each one of our packets is 36 + // bytes, or 288 bits. So, a limit of 872bps will allow 3 packets, + // but not four. + parameters.max_bandwidth_bps = 872; + ASSERT_TRUE(dmc->SetSendParameters(parameters)); + + EXPECT_TRUE(dmc->SendData(params, payload, &result)); + EXPECT_TRUE(dmc->SendData(params, payload, &result)); + EXPECT_TRUE(dmc->SendData(params, payload, &result)); + EXPECT_FALSE(dmc->SendData(params, payload, &result)); + EXPECT_FALSE(dmc->SendData(params, payload, &result)); + + SetNow(0.9); + EXPECT_FALSE(dmc->SendData(params, payload, &result)); + + SetNow(1.1); + EXPECT_TRUE(dmc->SendData(params, payload, &result)); + EXPECT_TRUE(dmc->SendData(params, payload, &result)); + SetNow(1.9); + EXPECT_TRUE(dmc->SendData(params, payload, &result)); + + SetNow(2.2); + EXPECT_TRUE(dmc->SendData(params, payload, &result)); + EXPECT_TRUE(dmc->SendData(params, payload, &result)); + EXPECT_TRUE(dmc->SendData(params, payload, &result)); + EXPECT_FALSE(dmc->SendData(params, payload, &result)); +} + +TEST_F(RtpDataMediaChannelTest, ReceiveData) { + // PT= 103, SN=2, TS=3, SSRC = 4, data = "abcde" + unsigned char data[] = {0x80, 0x67, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x03, 0x00, 0x00, 0x00, 0x2A, 0x00, 0x00, + 0x00, 0x00, 'a', 'b', 'c', 'd', 'e'}; + rtc::CopyOnWriteBuffer packet(data, sizeof(data)); + + std::unique_ptr<cricket::RtpDataMediaChannel> dmc(CreateChannel()); + + // SetReceived not called. + dmc->OnPacketReceived(packet, /* packet_time_us */ -1); + EXPECT_FALSE(HasReceivedData()); + + dmc->SetReceive(true); + + // Unknown payload id + dmc->OnPacketReceived(packet, /* packet_time_us */ -1); + EXPECT_FALSE(HasReceivedData()); + + cricket::DataCodec codec; + codec.id = 103; + codec.name = cricket::kGoogleRtpDataCodecName; + cricket::DataRecvParameters parameters; + parameters.codecs.push_back(codec); + ASSERT_TRUE(dmc->SetRecvParameters(parameters)); + + // Unknown stream + dmc->OnPacketReceived(packet, /* packet_time_us */ -1); + EXPECT_FALSE(HasReceivedData()); + + cricket::StreamParams stream; + stream.add_ssrc(42); + ASSERT_TRUE(dmc->AddRecvStream(stream)); + + // Finally works! + dmc->OnPacketReceived(packet, /* packet_time_us */ -1); + EXPECT_TRUE(HasReceivedData()); + EXPECT_EQ("abcde", GetReceivedData()); + EXPECT_EQ(5U, GetReceivedDataLen()); +} + +TEST_F(RtpDataMediaChannelTest, InvalidRtpPackets) { + unsigned char data[] = {0x80, 0x65, 0x00, 0x02}; + rtc::CopyOnWriteBuffer packet(data, sizeof(data)); + + std::unique_ptr<cricket::RtpDataMediaChannel> dmc(CreateChannel()); + + // Too short + dmc->OnPacketReceived(packet, /* packet_time_us */ -1); + EXPECT_FALSE(HasReceivedData()); +} diff --git a/media/base/rtp_utils.cc b/media/base/rtp_utils.cc new file mode 100644 index 0000000000..0b45e69410 --- /dev/null +++ b/media/base/rtp_utils.cc @@ -0,0 +1,521 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "media/base/rtp_utils.h" + +#include <string.h> + +#include <vector> + +// PacketTimeUpdateParams is defined in asyncpacketsocket.h. +// TODO(sergeyu): Find more appropriate place for PacketTimeUpdateParams. +#include "media/base/turn_utils.h" +#include "rtc_base/async_packet_socket.h" +#include "rtc_base/byte_order.h" +#include "rtc_base/checks.h" +#include "rtc_base/message_digest.h" + +namespace cricket { + +static const uint8_t kRtpVersion = 2; +static const size_t kRtpFlagsOffset = 0; +static const size_t kRtpPayloadTypeOffset = 1; +static const size_t kRtpSeqNumOffset = 2; +static const size_t kRtpTimestampOffset = 4; +static const size_t kRtpSsrcOffset = 8; +static const size_t kRtcpPayloadTypeOffset = 1; +static const size_t kRtpExtensionHeaderLen = 4; +static const size_t kAbsSendTimeExtensionLen = 3; +static const size_t kOneByteExtensionHeaderLen = 1; + +namespace { + +// Fake auth tag written by the sender when external authentication is enabled. +// HMAC in packet will be compared against this value before updating packet +// with actual HMAC value. +static const uint8_t kFakeAuthTag[10] = {0xba, 0xdd, 0xba, 0xdd, 0xba, + 0xdd, 0xba, 0xdd, 0xba, 0xdd}; + +void UpdateAbsSendTimeExtensionValue(uint8_t* extension_data, + size_t length, + uint64_t time_us) { + // Absolute send time in RTP streams. + // + // The absolute send time is signaled to the receiver in-band using the + // general mechanism for RTP header extensions [RFC5285]. The payload + // of this extension (the transmitted value) is a 24-bit unsigned integer + // containing the sender's current time in seconds as a fixed point number + // with 18 bits fractional part. + // + // The form of the absolute send time extension block: + // + // 0 1 2 3 + // 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + // | ID | len=2 | absolute send time | + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + if (length != kAbsSendTimeExtensionLen) { + RTC_NOTREACHED(); + return; + } + + // Convert microseconds to a 6.18 fixed point value in seconds. + uint32_t send_time = ((time_us << 18) / 1000000) & 0x00FFFFFF; + extension_data[0] = static_cast<uint8_t>(send_time >> 16); + extension_data[1] = static_cast<uint8_t>(send_time >> 8); + extension_data[2] = static_cast<uint8_t>(send_time); +} + +// Assumes |length| is actual packet length + tag length. Updates HMAC at end of +// the RTP packet. +void UpdateRtpAuthTag(uint8_t* rtp, + size_t length, + const rtc::PacketTimeUpdateParams& packet_time_params) { + // If there is no key, return. + if (packet_time_params.srtp_auth_key.empty()) { + return; + } + + size_t tag_length = packet_time_params.srtp_auth_tag_len; + + // ROC (rollover counter) is at the beginning of the auth tag. + const size_t kRocLength = 4; + if (tag_length < kRocLength || tag_length > length) { + RTC_NOTREACHED(); + return; + } + + uint8_t* auth_tag = rtp + (length - tag_length); + + // We should have a fake HMAC value @ auth_tag. + RTC_DCHECK_EQ(0, memcmp(auth_tag, kFakeAuthTag, tag_length)); + + // Copy ROC after end of rtp packet. + memcpy(auth_tag, &packet_time_params.srtp_packet_index, kRocLength); + // Authentication of a RTP packet will have RTP packet + ROC size. + size_t auth_required_length = length - tag_length + kRocLength; + + uint8_t output[64]; + size_t result = + rtc::ComputeHmac(rtc::DIGEST_SHA_1, &packet_time_params.srtp_auth_key[0], + packet_time_params.srtp_auth_key.size(), rtp, + auth_required_length, output, sizeof(output)); + + if (result < tag_length) { + RTC_NOTREACHED(); + return; + } + + // Copy HMAC from output to packet. This is required as auth tag length + // may not be equal to the actual HMAC length. + memcpy(auth_tag, output, tag_length); +} + +} // namespace + +bool GetUint8(const void* data, size_t offset, int* value) { + if (!data || !value) { + return false; + } + *value = *(static_cast<const uint8_t*>(data) + offset); + return true; +} + +bool GetUint16(const void* data, size_t offset, int* value) { + if (!data || !value) { + return false; + } + *value = static_cast<int>( + rtc::GetBE16(static_cast<const uint8_t*>(data) + offset)); + return true; +} + +bool GetUint32(const void* data, size_t offset, uint32_t* value) { + if (!data || !value) { + return false; + } + *value = rtc::GetBE32(static_cast<const uint8_t*>(data) + offset); + return true; +} + +bool SetUint8(void* data, size_t offset, uint8_t value) { + if (!data) { + return false; + } + rtc::Set8(data, offset, value); + return true; +} + +bool SetUint16(void* data, size_t offset, uint16_t value) { + if (!data) { + return false; + } + rtc::SetBE16(static_cast<uint8_t*>(data) + offset, value); + return true; +} + +bool SetUint32(void* data, size_t offset, uint32_t value) { + if (!data) { + return false; + } + rtc::SetBE32(static_cast<uint8_t*>(data) + offset, value); + return true; +} + +bool GetRtpFlags(const void* data, size_t len, int* value) { + if (len < kMinRtpPacketLen) { + return false; + } + return GetUint8(data, kRtpFlagsOffset, value); +} + +bool GetRtpPayloadType(const void* data, size_t len, int* value) { + if (len < kMinRtpPacketLen) { + return false; + } + if (!GetUint8(data, kRtpPayloadTypeOffset, value)) { + return false; + } + *value &= 0x7F; + return true; +} + +bool GetRtpSeqNum(const void* data, size_t len, int* value) { + if (len < kMinRtpPacketLen) { + return false; + } + return GetUint16(data, kRtpSeqNumOffset, value); +} + +bool GetRtpTimestamp(const void* data, size_t len, uint32_t* value) { + if (len < kMinRtpPacketLen) { + return false; + } + return GetUint32(data, kRtpTimestampOffset, value); +} + +bool GetRtpSsrc(const void* data, size_t len, uint32_t* value) { + if (len < kMinRtpPacketLen) { + return false; + } + return GetUint32(data, kRtpSsrcOffset, value); +} + +bool GetRtpHeaderLen(const void* data, size_t len, size_t* value) { + if (!data || len < kMinRtpPacketLen || !value) + return false; + const uint8_t* header = static_cast<const uint8_t*>(data); + // Get base header size + length of CSRCs (not counting extension yet). + size_t header_size = kMinRtpPacketLen + (header[0] & 0xF) * sizeof(uint32_t); + if (len < header_size) + return false; + // If there's an extension, read and add in the extension size. + if (header[0] & 0x10) { + if (len < header_size + sizeof(uint32_t)) + return false; + header_size += + ((rtc::GetBE16(header + header_size + 2) + 1) * sizeof(uint32_t)); + if (len < header_size) + return false; + } + *value = header_size; + return true; +} + +bool GetRtpHeader(const void* data, size_t len, RtpHeader* header) { + return (GetRtpPayloadType(data, len, &(header->payload_type)) && + GetRtpSeqNum(data, len, &(header->seq_num)) && + GetRtpTimestamp(data, len, &(header->timestamp)) && + GetRtpSsrc(data, len, &(header->ssrc))); +} + +bool GetRtcpType(const void* data, size_t len, int* value) { + if (len < kMinRtcpPacketLen) { + return false; + } + return GetUint8(data, kRtcpPayloadTypeOffset, value); +} + +// This method returns SSRC first of RTCP packet, except if packet is SDES. +// TODO(mallinath) - Fully implement RFC 5506. This standard doesn't restrict +// to send non-compound packets only to feedback messages. +bool GetRtcpSsrc(const void* data, size_t len, uint32_t* value) { + // Packet should be at least of 8 bytes, to get SSRC from a RTCP packet. + if (!data || len < kMinRtcpPacketLen + 4 || !value) + return false; + int pl_type; + if (!GetRtcpType(data, len, &pl_type)) + return false; + // SDES packet parsing is not supported. + if (pl_type == kRtcpTypeSDES) + return false; + *value = rtc::GetBE32(static_cast<const uint8_t*>(data) + 4); + return true; +} + +bool SetRtpSsrc(void* data, size_t len, uint32_t value) { + return SetUint32(data, kRtpSsrcOffset, value); +} + +// Assumes version 2, no padding, no extensions, no csrcs. +bool SetRtpHeader(void* data, size_t len, const RtpHeader& header) { + if (!IsValidRtpPayloadType(header.payload_type) || header.seq_num < 0 || + header.seq_num > static_cast<int>(UINT16_MAX)) { + return false; + } + return (SetUint8(data, kRtpFlagsOffset, kRtpVersion << 6) && + SetUint8(data, kRtpPayloadTypeOffset, header.payload_type & 0x7F) && + SetUint16(data, kRtpSeqNumOffset, + static_cast<uint16_t>(header.seq_num)) && + SetUint32(data, kRtpTimestampOffset, header.timestamp) && + SetRtpSsrc(data, len, header.ssrc)); +} + +static bool HasCorrectRtpVersion(rtc::ArrayView<const uint8_t> packet) { + return packet.data()[0] >> 6 == kRtpVersion; +} + +bool IsRtpPacket(rtc::ArrayView<const char> packet) { + return packet.size() >= kMinRtpPacketLen && + HasCorrectRtpVersion( + rtc::reinterpret_array_view<const uint8_t>(packet)); +} + +// Check the RTP payload type. If 63 < payload type < 96, it's RTCP. +// For additional details, see http://tools.ietf.org/html/rfc5761. +bool IsRtcpPacket(rtc::ArrayView<const char> packet) { + if (packet.size() < kMinRtcpPacketLen || + !HasCorrectRtpVersion( + rtc::reinterpret_array_view<const uint8_t>(packet))) { + return false; + } + + char pt = packet[1] & 0x7F; + return (63 < pt) && (pt < 96); +} + +bool IsValidRtpPayloadType(int payload_type) { + return payload_type >= 0 && payload_type <= 127; +} + +bool IsValidRtpPacketSize(RtpPacketType packet_type, size_t size) { + RTC_DCHECK_NE(RtpPacketType::kUnknown, packet_type); + size_t min_packet_length = packet_type == RtpPacketType::kRtcp + ? kMinRtcpPacketLen + : kMinRtpPacketLen; + return size >= min_packet_length && size <= kMaxRtpPacketLen; +} + +absl::string_view RtpPacketTypeToString(RtpPacketType packet_type) { + switch (packet_type) { + case RtpPacketType::kRtp: + return "RTP"; + case RtpPacketType::kRtcp: + return "RTCP"; + case RtpPacketType::kUnknown: + return "Unknown"; + } +} + +RtpPacketType InferRtpPacketType(rtc::ArrayView<const char> packet) { + // RTCP packets are RTP packets so must check that first. + if (IsRtcpPacket(packet)) { + return RtpPacketType::kRtcp; + } + if (IsRtpPacket(packet)) { + return RtpPacketType::kRtp; + } + return RtpPacketType::kUnknown; +} + +bool ValidateRtpHeader(const uint8_t* rtp, + size_t length, + size_t* header_length) { + if (header_length) { + *header_length = 0; + } + + if (length < kMinRtpPacketLen) { + return false; + } + + size_t cc_count = rtp[0] & 0x0F; + size_t header_length_without_extension = kMinRtpPacketLen + 4 * cc_count; + if (header_length_without_extension > length) { + return false; + } + + // If extension bit is not set, we are done with header processing, as input + // length is verified above. + if (!(rtp[0] & 0x10)) { + if (header_length) + *header_length = header_length_without_extension; + + return true; + } + + rtp += header_length_without_extension; + + if (header_length_without_extension + kRtpExtensionHeaderLen > length) { + return false; + } + + // Getting extension profile length. + // Length is in 32 bit words. + uint16_t extension_length_in_32bits = rtc::GetBE16(rtp + 2); + size_t extension_length = extension_length_in_32bits * 4; + + size_t rtp_header_length = extension_length + + header_length_without_extension + + kRtpExtensionHeaderLen; + + // Verify input length against total header size. + if (rtp_header_length > length) { + return false; + } + + if (header_length) { + *header_length = rtp_header_length; + } + return true; +} + +// ValidateRtpHeader() must be called before this method to make sure, we have +// a sane rtp packet. +bool UpdateRtpAbsSendTimeExtension(uint8_t* rtp, + size_t length, + int extension_id, + uint64_t time_us) { + // 0 1 2 3 + // 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + // |V=2|P|X| CC |M| PT | sequence number | + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + // | timestamp | + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + // | synchronization source (SSRC) identifier | + // +=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+ + // | contributing source (CSRC) identifiers | + // | .... | + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + + // Return if extension bit is not set. + if (!(rtp[0] & 0x10)) { + return true; + } + + size_t cc_count = rtp[0] & 0x0F; + size_t header_length_without_extension = kMinRtpPacketLen + 4 * cc_count; + + rtp += header_length_without_extension; + + // Getting extension profile ID and length. + uint16_t profile_id = rtc::GetBE16(rtp); + // Length is in 32 bit words. + uint16_t extension_length_in_32bits = rtc::GetBE16(rtp + 2); + size_t extension_length = extension_length_in_32bits * 4; + + rtp += kRtpExtensionHeaderLen; // Moving past extension header. + + bool found = false; + // WebRTC is using one byte header extension. + // TODO(mallinath) - Handle two byte header extension. + if (profile_id == 0xBEDE) { // OneByte extension header + // 0 + // 0 1 2 3 4 5 6 7 + // +-+-+-+-+-+-+-+-+ + // | ID |length | + // +-+-+-+-+-+-+-+-+ + + // 0 1 2 3 + // 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + // | 0xBE | 0xDE | length=3 | + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + // | ID | L=0 | data | ID | L=1 | data... + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + // ...data | 0 (pad) | 0 (pad) | ID | L=3 | + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + // | data | + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + const uint8_t* extension_start = rtp; + const uint8_t* extension_end = extension_start + extension_length; + + while (rtp < extension_end) { + const int id = (*rtp & 0xF0) >> 4; + const size_t length = (*rtp & 0x0F) + 1; + if (rtp + kOneByteExtensionHeaderLen + length > extension_end) { + return false; + } + // The 4-bit length is the number minus one of data bytes of this header + // extension element following the one-byte header. + if (id == extension_id) { + UpdateAbsSendTimeExtensionValue(rtp + kOneByteExtensionHeaderLen, + length, time_us); + found = true; + break; + } + rtp += kOneByteExtensionHeaderLen + length; + // Counting padding bytes. + while ((rtp < extension_end) && (*rtp == 0)) { + ++rtp; + } + } + } + return found; +} + +bool ApplyPacketOptions(uint8_t* data, + size_t length, + const rtc::PacketTimeUpdateParams& packet_time_params, + uint64_t time_us) { + RTC_DCHECK(data); + RTC_DCHECK(length); + + // if there is no valid |rtp_sendtime_extension_id| and |srtp_auth_key| in + // PacketOptions, nothing to be updated in this packet. + if (packet_time_params.rtp_sendtime_extension_id == -1 && + packet_time_params.srtp_auth_key.empty()) { + return true; + } + + // If there is a srtp auth key present then the packet must be an RTP packet. + // RTP packet may have been wrapped in a TURN Channel Data or TURN send + // indication. + size_t rtp_start_pos; + size_t rtp_length; + if (!UnwrapTurnPacket(data, length, &rtp_start_pos, &rtp_length)) { + RTC_NOTREACHED(); + return false; + } + + // Making sure we have a valid RTP packet at the end. + auto packet = rtc::MakeArrayView(data + rtp_start_pos, rtp_length); + if (!IsRtpPacket(rtc::reinterpret_array_view<const char>(packet)) || + !ValidateRtpHeader(data + rtp_start_pos, rtp_length, nullptr)) { + RTC_NOTREACHED(); + return false; + } + + uint8_t* start = data + rtp_start_pos; + // If packet option has non default value (-1) for sendtime extension id, + // then we should parse the rtp packet to update the timestamp. Otherwise + // just calculate HMAC and update packet with it. + if (packet_time_params.rtp_sendtime_extension_id != -1) { + UpdateRtpAbsSendTimeExtension(start, rtp_length, + packet_time_params.rtp_sendtime_extension_id, + time_us); + } + + UpdateRtpAuthTag(start, rtp_length, packet_time_params); + return true; +} + +} // namespace cricket diff --git a/media/base/rtp_utils.h b/media/base/rtp_utils.h new file mode 100644 index 0000000000..9ef9f9c7ba --- /dev/null +++ b/media/base/rtp_utils.h @@ -0,0 +1,100 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MEDIA_BASE_RTP_UTILS_H_ +#define MEDIA_BASE_RTP_UTILS_H_ + +#include "absl/strings/string_view.h" +#include "api/array_view.h" +#include "rtc_base/byte_order.h" +#include "rtc_base/system/rtc_export.h" + +namespace rtc { +struct PacketTimeUpdateParams; +} // namespace rtc + +namespace cricket { + +const size_t kMinRtpPacketLen = 12; +const size_t kMaxRtpPacketLen = 2048; +const size_t kMinRtcpPacketLen = 4; + +struct RtpHeader { + int payload_type; + int seq_num; + uint32_t timestamp; + uint32_t ssrc; +}; + +enum RtcpTypes { + kRtcpTypeSR = 200, // Sender report payload type. + kRtcpTypeRR = 201, // Receiver report payload type. + kRtcpTypeSDES = 202, // SDES payload type. + kRtcpTypeBye = 203, // BYE payload type. + kRtcpTypeApp = 204, // APP payload type. + kRtcpTypeRTPFB = 205, // Transport layer Feedback message payload type. + kRtcpTypePSFB = 206, // Payload-specific Feedback message payload type. +}; + +enum class RtpPacketType { + kRtp, + kRtcp, + kUnknown, +}; + +bool GetRtpPayloadType(const void* data, size_t len, int* value); +bool GetRtpSeqNum(const void* data, size_t len, int* value); +bool GetRtpTimestamp(const void* data, size_t len, uint32_t* value); +bool GetRtpSsrc(const void* data, size_t len, uint32_t* value); +bool GetRtpHeaderLen(const void* data, size_t len, size_t* value); +bool GetRtcpType(const void* data, size_t len, int* value); +bool GetRtcpSsrc(const void* data, size_t len, uint32_t* value); +bool GetRtpHeader(const void* data, size_t len, RtpHeader* header); + +bool SetRtpSsrc(void* data, size_t len, uint32_t value); +// Assumes version 2, no padding, no extensions, no csrcs. +bool SetRtpHeader(void* data, size_t len, const RtpHeader& header); + +bool IsRtpPacket(rtc::ArrayView<const char> packet); + +bool IsRtcpPacket(rtc::ArrayView<const char> packet); +// Checks the packet header to determine if it can be an RTP or RTCP packet. +RtpPacketType InferRtpPacketType(rtc::ArrayView<const char> packet); +// True if |payload type| is 0-127. +bool IsValidRtpPayloadType(int payload_type); + +// True if |size| is appropriate for the indicated packet type. +bool IsValidRtpPacketSize(RtpPacketType packet_type, size_t size); + +// Returns "RTCP", "RTP" or "Unknown" according to |packet_type|. +absl::string_view RtpPacketTypeToString(RtpPacketType packet_type); + +// Verifies that a packet has a valid RTP header. +bool RTC_EXPORT ValidateRtpHeader(const uint8_t* rtp, + size_t length, + size_t* header_length); + +// Helper method which updates the absolute send time extension if present. +bool UpdateRtpAbsSendTimeExtension(uint8_t* rtp, + size_t length, + int extension_id, + uint64_t time_us); + +// Applies specified |options| to the packet. It updates the absolute send time +// extension header if it is present present then updates HMAC. +bool RTC_EXPORT +ApplyPacketOptions(uint8_t* data, + size_t length, + const rtc::PacketTimeUpdateParams& packet_time_params, + uint64_t time_us); + +} // namespace cricket + +#endif // MEDIA_BASE_RTP_UTILS_H_ diff --git a/media/base/rtp_utils_unittest.cc b/media/base/rtp_utils_unittest.cc new file mode 100644 index 0000000000..051508cd01 --- /dev/null +++ b/media/base/rtp_utils_unittest.cc @@ -0,0 +1,366 @@ +/* + * Copyright (c) 2004 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "media/base/rtp_utils.h" + +#include <string.h> + +#include <cstdint> +#include <vector> + +#include "media/base/fake_rtp.h" +#include "rtc_base/async_packet_socket.h" +#include "test/gtest.h" + +namespace cricket { + +static const uint8_t kRtpPacketWithMarker[] = { + 0x80, 0x80, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01}; +// 3 CSRCs (0x01020304, 0x12345678, 0xAABBCCDD) +// Extension (0xBEDE, 0x1122334455667788) +static const uint8_t kRtpPacketWithMarkerAndCsrcAndExtension[] = { + 0x93, 0x80, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + 0x01, 0x02, 0x03, 0x04, 0x12, 0x34, 0x56, 0x78, 0xAA, 0xBB, 0xCC, 0xDD, + 0xBE, 0xDE, 0x00, 0x02, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88}; +static const uint8_t kInvalidPacket[] = {0x80, 0x00}; +static const uint8_t kInvalidPacketWithCsrc[] = { + 0x83, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + 0x01, 0x02, 0x03, 0x04, 0x12, 0x34, 0x56, 0x78, 0xAA, 0xBB, 0xCC}; +static const uint8_t kInvalidPacketWithCsrcAndExtension1[] = { + 0x93, 0x80, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x01, 0x01, 0x02, 0x03, 0x04, 0x12, 0x34, + 0x56, 0x78, 0xAA, 0xBB, 0xCC, 0xDD, 0xBE, 0xDE, 0x00}; +static const uint8_t kInvalidPacketWithCsrcAndExtension2[] = { + 0x93, 0x80, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + 0x01, 0x02, 0x03, 0x04, 0x12, 0x34, 0x56, 0x78, 0xAA, 0xBB, 0xCC, 0xDD, + 0xBE, 0xDE, 0x00, 0x02, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77}; + +// PT = 206, FMT = 1, Sender SSRC = 0x1111, Media SSRC = 0x1111 +// No FCI information is needed for PLI. +static const uint8_t kNonCompoundRtcpPliFeedbackPacket[] = { + 0x81, 0xCE, 0x00, 0x0C, 0x00, 0x00, 0x11, 0x11, 0x00, 0x00, 0x11, 0x11}; + +// Packet has only mandatory fixed RTCP header +// PT = 204, SSRC = 0x1111 +static const uint8_t kNonCompoundRtcpAppPacket[] = {0x81, 0xCC, 0x00, 0x0C, + 0x00, 0x00, 0x11, 0x11}; + +// PT = 202, Source count = 0 +static const uint8_t kNonCompoundRtcpSDESPacket[] = {0x80, 0xCA, 0x00, 0x00}; + +static uint8_t kFakeTag[4] = {0xba, 0xdd, 0xba, 0xdd}; +static uint8_t kTestKey[] = "12345678901234567890"; +static uint8_t kTestAstValue[3] = {0xaa, 0xbb, 0xcc}; + +// Valid rtp Message with 2 byte header extension. +static uint8_t kRtpMsgWith2ByteExtnHeader[] = { + // clang-format off + // clang formatting doesn't respect inline comments. + 0x90, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0xAA, 0xBB, 0xCC, 0XDD, // SSRC + 0x10, 0x00, 0x00, 0x01, // 2 Byte header extension + 0x01, 0x00, 0x00, 0x00 + // clang-format on +}; + +// RTP packet with single byte extension header of length 4 bytes. +// Extension id = 3 and length = 3 +static uint8_t kRtpMsgWithAbsSendTimeExtension[] = { + 0x90, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xBE, 0xDE, 0x00, 0x02, 0x22, 0x00, 0x02, 0x1c, 0x32, 0xaa, 0xbb, 0xcc, +}; + +// Index of AbsSendTimeExtn data in message |kRtpMsgWithAbsSendTimeExtension|. +static const int kAstIndexInRtpMsg = 21; + +static const rtc::ArrayView<const char> kPcmuFrameArrayView = + rtc::MakeArrayView(reinterpret_cast<const char*>(kPcmuFrame), + sizeof(kPcmuFrame)); +static const rtc::ArrayView<const char> kRtcpReportArrayView = + rtc::MakeArrayView(reinterpret_cast<const char*>(kRtcpReport), + sizeof(kRtcpReport)); +static const rtc::ArrayView<const char> kInvalidPacketArrayView = + rtc::MakeArrayView(reinterpret_cast<const char*>(kInvalidPacket), + sizeof(kInvalidPacket)); + +TEST(RtpUtilsTest, GetRtp) { + EXPECT_TRUE(IsRtpPacket(kPcmuFrameArrayView)); + + int pt; + EXPECT_TRUE(GetRtpPayloadType(kPcmuFrame, sizeof(kPcmuFrame), &pt)); + EXPECT_EQ(0, pt); + EXPECT_TRUE(GetRtpPayloadType(kRtpPacketWithMarker, + sizeof(kRtpPacketWithMarker), &pt)); + EXPECT_EQ(0, pt); + + int seq_num; + EXPECT_TRUE(GetRtpSeqNum(kPcmuFrame, sizeof(kPcmuFrame), &seq_num)); + EXPECT_EQ(1, seq_num); + + uint32_t ts; + EXPECT_TRUE(GetRtpTimestamp(kPcmuFrame, sizeof(kPcmuFrame), &ts)); + EXPECT_EQ(0u, ts); + + uint32_t ssrc; + EXPECT_TRUE(GetRtpSsrc(kPcmuFrame, sizeof(kPcmuFrame), &ssrc)); + EXPECT_EQ(1u, ssrc); + + RtpHeader header; + EXPECT_TRUE(GetRtpHeader(kPcmuFrame, sizeof(kPcmuFrame), &header)); + EXPECT_EQ(0, header.payload_type); + EXPECT_EQ(1, header.seq_num); + EXPECT_EQ(0u, header.timestamp); + EXPECT_EQ(1u, header.ssrc); + + EXPECT_FALSE(GetRtpPayloadType(kInvalidPacket, sizeof(kInvalidPacket), &pt)); + EXPECT_FALSE(GetRtpSeqNum(kInvalidPacket, sizeof(kInvalidPacket), &seq_num)); + EXPECT_FALSE(GetRtpTimestamp(kInvalidPacket, sizeof(kInvalidPacket), &ts)); + EXPECT_FALSE(GetRtpSsrc(kInvalidPacket, sizeof(kInvalidPacket), &ssrc)); +} + +TEST(RtpUtilsTest, SetRtpHeader) { + uint8_t packet[] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; + + RtpHeader header = {9, 1111, 2222u, 3333u}; + EXPECT_TRUE(SetRtpHeader(packet, sizeof(packet), header)); + + // Bits: 10 0 0 0000 + EXPECT_EQ(128u, packet[0]); + size_t len; + EXPECT_TRUE(GetRtpHeaderLen(packet, sizeof(packet), &len)); + EXPECT_EQ(12U, len); + EXPECT_TRUE(GetRtpHeader(packet, sizeof(packet), &header)); + EXPECT_EQ(9, header.payload_type); + EXPECT_EQ(1111, header.seq_num); + EXPECT_EQ(2222u, header.timestamp); + EXPECT_EQ(3333u, header.ssrc); +} + +TEST(RtpUtilsTest, GetRtpHeaderLen) { + size_t len; + EXPECT_TRUE(GetRtpHeaderLen(kPcmuFrame, sizeof(kPcmuFrame), &len)); + EXPECT_EQ(12U, len); + + EXPECT_TRUE(GetRtpHeaderLen(kRtpPacketWithMarkerAndCsrcAndExtension, + sizeof(kRtpPacketWithMarkerAndCsrcAndExtension), + &len)); + EXPECT_EQ(sizeof(kRtpPacketWithMarkerAndCsrcAndExtension), len); + + EXPECT_FALSE(GetRtpHeaderLen(kInvalidPacket, sizeof(kInvalidPacket), &len)); + EXPECT_FALSE(GetRtpHeaderLen(kInvalidPacketWithCsrc, + sizeof(kInvalidPacketWithCsrc), &len)); + EXPECT_FALSE(GetRtpHeaderLen(kInvalidPacketWithCsrcAndExtension1, + sizeof(kInvalidPacketWithCsrcAndExtension1), + &len)); + EXPECT_FALSE(GetRtpHeaderLen(kInvalidPacketWithCsrcAndExtension2, + sizeof(kInvalidPacketWithCsrcAndExtension2), + &len)); +} + +TEST(RtpUtilsTest, GetRtcp) { + int pt; + EXPECT_TRUE(GetRtcpType(kRtcpReport, sizeof(kRtcpReport), &pt)); + EXPECT_EQ(0xc9, pt); + + EXPECT_FALSE(GetRtcpType(kInvalidPacket, sizeof(kInvalidPacket), &pt)); + + uint32_t ssrc; + EXPECT_TRUE(GetRtcpSsrc(kNonCompoundRtcpPliFeedbackPacket, + sizeof(kNonCompoundRtcpPliFeedbackPacket), &ssrc)); + EXPECT_TRUE(GetRtcpSsrc(kNonCompoundRtcpAppPacket, + sizeof(kNonCompoundRtcpAppPacket), &ssrc)); + EXPECT_FALSE(GetRtcpSsrc(kNonCompoundRtcpSDESPacket, + sizeof(kNonCompoundRtcpSDESPacket), &ssrc)); +} + +// Invalid RTP packets. +TEST(RtpUtilsTest, InvalidRtpHeader) { + // Rtp message with invalid length. + const uint8_t kRtpMsgWithInvalidLength[] = { + // clang-format off + // clang formatting doesn't respect inline comments. + 0x94, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xAA, 0xBB, 0xCC, 0XDD, // SSRC + 0xDD, 0xCC, 0xBB, 0xAA, // Only 1 CSRC, but CC count is 4. + // clang-format on + }; + EXPECT_FALSE(ValidateRtpHeader(kRtpMsgWithInvalidLength, + sizeof(kRtpMsgWithInvalidLength), nullptr)); + + // Rtp message with single byte header extension, invalid extension length. + const uint8_t kRtpMsgWithInvalidExtnLength[] = { + 0x90, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xBE, 0xDE, 0x0A, 0x00, // Extn length - 0x0A00 + }; + EXPECT_FALSE(ValidateRtpHeader(kRtpMsgWithInvalidExtnLength, + sizeof(kRtpMsgWithInvalidExtnLength), + nullptr)); +} + +// Valid RTP packet with a 2byte header extension. +TEST(RtpUtilsTest, Valid2ByteExtnHdrRtpMessage) { + EXPECT_TRUE(ValidateRtpHeader(kRtpMsgWith2ByteExtnHeader, + sizeof(kRtpMsgWith2ByteExtnHeader), nullptr)); +} + +// Valid RTP packet which has 1 byte header AbsSendTime extension in it. +TEST(RtpUtilsTest, ValidRtpPacketWithAbsSendTimeExtension) { + EXPECT_TRUE(ValidateRtpHeader(kRtpMsgWithAbsSendTimeExtension, + sizeof(kRtpMsgWithAbsSendTimeExtension), + nullptr)); +} + +// Verify handling of a 2 byte extension header RTP messsage. Currently these +// messages are not supported. +TEST(RtpUtilsTest, UpdateAbsSendTimeExtensionIn2ByteHeaderExtn) { + std::vector<uint8_t> data( + kRtpMsgWith2ByteExtnHeader, + kRtpMsgWith2ByteExtnHeader + sizeof(kRtpMsgWith2ByteExtnHeader)); + EXPECT_FALSE(UpdateRtpAbsSendTimeExtension(&data[0], data.size(), 3, 0)); +} + +// Verify finding an extension ID in the TURN send indication message. +TEST(RtpUtilsTest, UpdateAbsSendTimeExtensionInTurnSendIndication) { + // A valid STUN indication message with a valid RTP header in data attribute + // payload field and no extension bit set. + uint8_t message_without_extension[] = { + // clang-format off + // clang formatting doesn't respect inline comments. + 0x00, 0x16, 0x00, 0x18, // length of + 0x21, 0x12, 0xA4, 0x42, // magic cookie + '0', '1', '2', '3', // transaction id + '4', '5', '6', '7', + '8', '9', 'a', 'b', + 0x00, 0x20, 0x00, 0x04, // Mapped address. + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x13, 0x00, 0x0C, // Data attribute. + 0x80, 0x00, 0x00, 0x00, // RTP packet. + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + // clang-format on + }; + EXPECT_TRUE(UpdateRtpAbsSendTimeExtension( + message_without_extension, sizeof(message_without_extension), 3, 0)); + + // A valid STUN indication message with a valid RTP header and a extension + // header. + uint8_t message[] = { + // clang-format off + // clang formatting doesn't respect inline comments. + 0x00, 0x16, 0x00, 0x24, // length of + 0x21, 0x12, 0xA4, 0x42, // magic cookie + '0', '1', '2', '3', // transaction id + '4', '5', '6', '7', + '8', '9', 'a', 'b', + 0x00, 0x20, 0x00, 0x04, // Mapped address. + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x13, 0x00, 0x18, // Data attribute. + 0x90, 0x00, 0x00, 0x00, // RTP packet. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xBE, 0xDE, + 0x00, 0x02, 0x22, 0xaa, 0xbb, 0xcc, 0x32, 0xaa, 0xbb, 0xcc, + // clang-format on + }; + EXPECT_TRUE(UpdateRtpAbsSendTimeExtension(message, sizeof(message), 3, 0)); +} + +// Test without any packet options variables set. This method should return +// without HMAC value in the packet. +TEST(RtpUtilsTest, ApplyPacketOptionsWithDefaultValues) { + rtc::PacketTimeUpdateParams packet_time_params; + std::vector<uint8_t> rtp_packet(kRtpMsgWithAbsSendTimeExtension, + kRtpMsgWithAbsSendTimeExtension + + sizeof(kRtpMsgWithAbsSendTimeExtension)); + rtp_packet.insert(rtp_packet.end(), kFakeTag, kFakeTag + sizeof(kFakeTag)); + EXPECT_TRUE(ApplyPacketOptions(&rtp_packet[0], rtp_packet.size(), + packet_time_params, 0)); + + // Making sure HMAC wasn't updated.. + EXPECT_EQ(0, memcmp(&rtp_packet[sizeof(kRtpMsgWithAbsSendTimeExtension)], + kFakeTag, 4)); + + // Verify AbsouluteSendTime extension field wasn't modified. + EXPECT_EQ(0, memcmp(&rtp_packet[kAstIndexInRtpMsg], kTestAstValue, + sizeof(kTestAstValue))); +} + +// Veirfy HMAC is updated when packet option parameters are set. +TEST(RtpUtilsTest, ApplyPacketOptionsWithAuthParams) { + rtc::PacketTimeUpdateParams packet_time_params; + packet_time_params.srtp_auth_key.assign(kTestKey, + kTestKey + sizeof(kTestKey)); + packet_time_params.srtp_auth_tag_len = 4; + + std::vector<uint8_t> rtp_packet(kRtpMsgWithAbsSendTimeExtension, + kRtpMsgWithAbsSendTimeExtension + + sizeof(kRtpMsgWithAbsSendTimeExtension)); + rtp_packet.insert(rtp_packet.end(), kFakeTag, kFakeTag + sizeof(kFakeTag)); + EXPECT_TRUE(ApplyPacketOptions(&rtp_packet[0], rtp_packet.size(), + packet_time_params, 0)); + + uint8_t kExpectedTag[] = {0xc1, 0x7a, 0x8c, 0xa0}; + EXPECT_EQ(0, memcmp(&rtp_packet[sizeof(kRtpMsgWithAbsSendTimeExtension)], + kExpectedTag, sizeof(kExpectedTag))); + + // Verify AbsouluteSendTime extension field is not modified. + EXPECT_EQ(0, memcmp(&rtp_packet[kAstIndexInRtpMsg], kTestAstValue, + sizeof(kTestAstValue))); +} + +// Verify finding an extension ID in a raw rtp message. +TEST(RtpUtilsTest, UpdateAbsSendTimeExtensionInRtpPacket) { + std::vector<uint8_t> rtp_packet(kRtpMsgWithAbsSendTimeExtension, + kRtpMsgWithAbsSendTimeExtension + + sizeof(kRtpMsgWithAbsSendTimeExtension)); + + EXPECT_TRUE(UpdateRtpAbsSendTimeExtension(&rtp_packet[0], rtp_packet.size(), + 3, 51183266)); + + // Verify that the timestamp was updated. + const uint8_t kExpectedTimestamp[3] = {0xcc, 0xbb, 0xaa}; + EXPECT_EQ(0, memcmp(&rtp_packet[kAstIndexInRtpMsg], kExpectedTimestamp, + sizeof(kExpectedTimestamp))); +} + +// Verify we update both AbsSendTime extension header and HMAC. +TEST(RtpUtilsTest, ApplyPacketOptionsWithAuthParamsAndAbsSendTime) { + rtc::PacketTimeUpdateParams packet_time_params; + packet_time_params.srtp_auth_key.assign(kTestKey, + kTestKey + sizeof(kTestKey)); + packet_time_params.srtp_auth_tag_len = 4; + packet_time_params.rtp_sendtime_extension_id = 3; + // 3 is also present in the test message. + + std::vector<uint8_t> rtp_packet(kRtpMsgWithAbsSendTimeExtension, + kRtpMsgWithAbsSendTimeExtension + + sizeof(kRtpMsgWithAbsSendTimeExtension)); + rtp_packet.insert(rtp_packet.end(), kFakeTag, kFakeTag + sizeof(kFakeTag)); + EXPECT_TRUE(ApplyPacketOptions(&rtp_packet[0], rtp_packet.size(), + packet_time_params, 51183266)); + + const uint8_t kExpectedTag[] = {0x81, 0xd1, 0x2c, 0x0e}; + EXPECT_EQ(0, memcmp(&rtp_packet[sizeof(kRtpMsgWithAbsSendTimeExtension)], + kExpectedTag, sizeof(kExpectedTag))); + + // Verify that the timestamp was updated. + const uint8_t kExpectedTimestamp[3] = {0xcc, 0xbb, 0xaa}; + EXPECT_EQ(0, memcmp(&rtp_packet[kAstIndexInRtpMsg], kExpectedTimestamp, + sizeof(kExpectedTimestamp))); +} + +TEST(RtpUtilsTest, InferRtpPacketType) { + EXPECT_EQ(RtpPacketType::kRtp, InferRtpPacketType(kPcmuFrameArrayView)); + EXPECT_EQ(RtpPacketType::kRtcp, InferRtpPacketType(kRtcpReportArrayView)); + EXPECT_EQ(RtpPacketType::kUnknown, + InferRtpPacketType(kInvalidPacketArrayView)); +} + +} // namespace cricket diff --git a/media/base/sdp_fmtp_utils.cc b/media/base/sdp_fmtp_utils.cc new file mode 100644 index 0000000000..4ffc3b9696 --- /dev/null +++ b/media/base/sdp_fmtp_utils.cc @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "media/base/sdp_fmtp_utils.h" + +#include <map> +#include <utility> + +#include "rtc_base/string_to_number.h" + +namespace webrtc { +namespace { +// Max frame rate for VP8 and VP9 video. +const char kVPxFmtpMaxFrameRate[] = "max-fr"; +// Max frame size for VP8 and VP9 video. +const char kVPxFmtpMaxFrameSize[] = "max-fs"; +const int kVPxFmtpFrameSizeSubBlockPixels = 256; + +absl::optional<int> ParsePositiveNumberFromParams( + const SdpVideoFormat::Parameters& params, + const char* parameter_name) { + const auto max_frame_rate_it = params.find(parameter_name); + if (max_frame_rate_it == params.end()) + return absl::nullopt; + + const absl::optional<int> i = + rtc::StringToNumber<int>(max_frame_rate_it->second); + if (!i.has_value() || i.value() <= 0) + return absl::nullopt; + return i; +} + +} // namespace + +absl::optional<int> ParseSdpForVPxMaxFrameRate( + const SdpVideoFormat::Parameters& params) { + return ParsePositiveNumberFromParams(params, kVPxFmtpMaxFrameRate); +} + +absl::optional<int> ParseSdpForVPxMaxFrameSize( + const SdpVideoFormat::Parameters& params) { + const absl::optional<int> i = + ParsePositiveNumberFromParams(params, kVPxFmtpMaxFrameSize); + return i ? absl::make_optional(i.value() * kVPxFmtpFrameSizeSubBlockPixels) + : absl::nullopt; +} + +} // namespace webrtc diff --git a/media/base/sdp_fmtp_utils.h b/media/base/sdp_fmtp_utils.h new file mode 100644 index 0000000000..04e9183614 --- /dev/null +++ b/media/base/sdp_fmtp_utils.h @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MEDIA_BASE_SDP_FMTP_UTILS_H_ +#define MEDIA_BASE_SDP_FMTP_UTILS_H_ + +#include "absl/types/optional.h" +#include "api/video_codecs/sdp_video_format.h" + +namespace webrtc { + +// Parse max frame rate from SDP FMTP line. absl::nullopt is returned if the +// field is missing or not a number. +absl::optional<int> ParseSdpForVPxMaxFrameRate( + const SdpVideoFormat::Parameters& params); + +// Parse max frame size from SDP FMTP line. absl::nullopt is returned if the +// field is missing or not a number. Please note that the value is stored in sub +// blocks but the returned value is in total number of pixels. +absl::optional<int> ParseSdpForVPxMaxFrameSize( + const SdpVideoFormat::Parameters& params); + +} // namespace webrtc + +#endif // MEDIA_BASE_SDP_FMTP_UTILS_H__ diff --git a/media/base/sdp_fmtp_utils_unittest.cc b/media/base/sdp_fmtp_utils_unittest.cc new file mode 100644 index 0000000000..0ff12ffbe1 --- /dev/null +++ b/media/base/sdp_fmtp_utils_unittest.cc @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "media/base/sdp_fmtp_utils.h" + +#include <string.h> +#include <map> +#include <utility> + +#include "rtc_base/string_to_number.h" +#include "test/gtest.h" + +namespace webrtc { +namespace { +// Max frame rate for VP8 and VP9 video. +const char kVPxFmtpMaxFrameRate[] = "max-fr"; +// Max frame size for VP8 and VP9 video. +const char kVPxFmtpMaxFrameSize[] = "max-fs"; +} // namespace + +TEST(SdpFmtpUtilsTest, MaxFrameRateIsMissingOrInvalid) { + SdpVideoFormat::Parameters params; + absl::optional<int> empty = ParseSdpForVPxMaxFrameRate(params); + EXPECT_FALSE(empty); + params[kVPxFmtpMaxFrameRate] = "-1"; + EXPECT_FALSE(ParseSdpForVPxMaxFrameRate(params)); + params[kVPxFmtpMaxFrameRate] = "0"; + EXPECT_FALSE(ParseSdpForVPxMaxFrameRate(params)); + params[kVPxFmtpMaxFrameRate] = "abcde"; + EXPECT_FALSE(ParseSdpForVPxMaxFrameRate(params)); +} + +TEST(SdpFmtpUtilsTest, MaxFrameRateIsSpecified) { + SdpVideoFormat::Parameters params; + params[kVPxFmtpMaxFrameRate] = "30"; + EXPECT_EQ(ParseSdpForVPxMaxFrameRate(params), 30); + params[kVPxFmtpMaxFrameRate] = "60"; + EXPECT_EQ(ParseSdpForVPxMaxFrameRate(params), 60); +} + +TEST(SdpFmtpUtilsTest, MaxFrameSizeIsMissingOrInvalid) { + SdpVideoFormat::Parameters params; + absl::optional<int> empty = ParseSdpForVPxMaxFrameSize(params); + EXPECT_FALSE(empty); + params[kVPxFmtpMaxFrameSize] = "-1"; + EXPECT_FALSE(ParseSdpForVPxMaxFrameSize(params)); + params[kVPxFmtpMaxFrameSize] = "0"; + EXPECT_FALSE(ParseSdpForVPxMaxFrameSize(params)); + params[kVPxFmtpMaxFrameSize] = "abcde"; + EXPECT_FALSE(ParseSdpForVPxMaxFrameSize(params)); +} + +TEST(SdpFmtpUtilsTest, MaxFrameSizeIsSpecified) { + SdpVideoFormat::Parameters params; + params[kVPxFmtpMaxFrameSize] = "8100"; // 1920 x 1080 / (16^2) + EXPECT_EQ(ParseSdpForVPxMaxFrameSize(params), 1920 * 1080); + params[kVPxFmtpMaxFrameSize] = "32400"; // 3840 x 2160 / (16^2) + EXPECT_EQ(ParseSdpForVPxMaxFrameSize(params), 3840 * 2160); +} + +} // namespace webrtc diff --git a/media/base/stream_params.cc b/media/base/stream_params.cc new file mode 100644 index 0000000000..db781acfc7 --- /dev/null +++ b/media/base/stream_params.cc @@ -0,0 +1,236 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "media/base/stream_params.h" + +#include <stdint.h> + +#include <list> + +#include "absl/algorithm/container.h" +#include "api/array_view.h" +#include "rtc_base/strings/string_builder.h" + +namespace cricket { +namespace { + +void AppendSsrcs(rtc::ArrayView<const uint32_t> ssrcs, + rtc::SimpleStringBuilder* sb) { + *sb << "ssrcs:["; + const char* delimiter = ""; + for (uint32_t ssrc : ssrcs) { + *sb << delimiter << ssrc; + delimiter = ","; + } + *sb << "]"; +} + +void AppendSsrcGroups(rtc::ArrayView<const SsrcGroup> ssrc_groups, + rtc::SimpleStringBuilder* sb) { + *sb << "ssrc_groups:"; + const char* delimiter = ""; + for (const SsrcGroup& ssrc_group : ssrc_groups) { + *sb << delimiter << ssrc_group.ToString(); + delimiter = ","; + } +} + +void AppendStreamIds(rtc::ArrayView<const std::string> stream_ids, + rtc::SimpleStringBuilder* sb) { + *sb << "stream_ids:"; + const char* delimiter = ""; + for (const std::string& stream_id : stream_ids) { + *sb << delimiter << stream_id; + delimiter = ","; + } +} + +void AppendRids(rtc::ArrayView<const RidDescription> rids, + rtc::SimpleStringBuilder* sb) { + *sb << "rids:["; + const char* delimiter = ""; + for (const RidDescription& rid : rids) { + *sb << delimiter << rid.rid; + delimiter = ","; + } + *sb << "]"; +} + +} // namespace + +const char kFecSsrcGroupSemantics[] = "FEC"; +const char kFecFrSsrcGroupSemantics[] = "FEC-FR"; +const char kFidSsrcGroupSemantics[] = "FID"; +const char kSimSsrcGroupSemantics[] = "SIM"; + +bool GetStream(const StreamParamsVec& streams, + const StreamSelector& selector, + StreamParams* stream_out) { + const StreamParams* found = GetStream(streams, selector); + if (found && stream_out) + *stream_out = *found; + return found != nullptr; +} + +SsrcGroup::SsrcGroup(const std::string& usage, + const std::vector<uint32_t>& ssrcs) + : semantics(usage), ssrcs(ssrcs) {} +SsrcGroup::SsrcGroup(const SsrcGroup&) = default; +SsrcGroup::SsrcGroup(SsrcGroup&&) = default; +SsrcGroup::~SsrcGroup() = default; + +SsrcGroup& SsrcGroup::operator=(const SsrcGroup&) = default; +SsrcGroup& SsrcGroup::operator=(SsrcGroup&&) = default; + +bool SsrcGroup::has_semantics(const std::string& semantics_in) const { + return (semantics == semantics_in && ssrcs.size() > 0); +} + +std::string SsrcGroup::ToString() const { + char buf[1024]; + rtc::SimpleStringBuilder sb(buf); + sb << "{"; + sb << "semantics:" << semantics << ";"; + AppendSsrcs(ssrcs, &sb); + sb << "}"; + return sb.str(); +} + +StreamParams::StreamParams() = default; +StreamParams::StreamParams(const StreamParams&) = default; +StreamParams::StreamParams(StreamParams&&) = default; +StreamParams::~StreamParams() = default; +StreamParams& StreamParams::operator=(const StreamParams&) = default; +StreamParams& StreamParams::operator=(StreamParams&&) = default; + +bool StreamParams::operator==(const StreamParams& other) const { + return (groupid == other.groupid && id == other.id && ssrcs == other.ssrcs && + ssrc_groups == other.ssrc_groups && cname == other.cname && + stream_ids_ == other.stream_ids_ && + // RIDs are not required to be in the same order for equality. + absl::c_is_permutation(rids_, other.rids_)); +} + +std::string StreamParams::ToString() const { + char buf[2 * 1024]; + rtc::SimpleStringBuilder sb(buf); + sb << "{"; + if (!groupid.empty()) { + sb << "groupid:" << groupid << ";"; + } + if (!id.empty()) { + sb << "id:" << id << ";"; + } + AppendSsrcs(ssrcs, &sb); + sb << ";"; + AppendSsrcGroups(ssrc_groups, &sb); + sb << ";"; + if (!cname.empty()) { + sb << "cname:" << cname << ";"; + } + AppendStreamIds(stream_ids_, &sb); + sb << ";"; + if (!rids_.empty()) { + AppendRids(rids_, &sb); + sb << ";"; + } + sb << "}"; + return sb.str(); +} + +void StreamParams::GenerateSsrcs(int num_layers, + bool generate_fid, + bool generate_fec_fr, + rtc::UniqueRandomIdGenerator* ssrc_generator) { + RTC_DCHECK_GE(num_layers, 0); + RTC_DCHECK(ssrc_generator); + std::vector<uint32_t> primary_ssrcs; + for (int i = 0; i < num_layers; ++i) { + uint32_t ssrc = ssrc_generator->GenerateId(); + primary_ssrcs.push_back(ssrc); + add_ssrc(ssrc); + } + + if (num_layers > 1) { + SsrcGroup simulcast(kSimSsrcGroupSemantics, primary_ssrcs); + ssrc_groups.push_back(simulcast); + } + + if (generate_fid) { + for (uint32_t ssrc : primary_ssrcs) { + AddFidSsrc(ssrc, ssrc_generator->GenerateId()); + } + } + + if (generate_fec_fr) { + for (uint32_t ssrc : primary_ssrcs) { + AddFecFrSsrc(ssrc, ssrc_generator->GenerateId()); + } + } +} + +void StreamParams::GetPrimarySsrcs(std::vector<uint32_t>* ssrcs) const { + const SsrcGroup* sim_group = get_ssrc_group(kSimSsrcGroupSemantics); + if (sim_group == NULL) { + ssrcs->push_back(first_ssrc()); + } else { + ssrcs->insert(ssrcs->end(), sim_group->ssrcs.begin(), + sim_group->ssrcs.end()); + } +} + +void StreamParams::GetFidSsrcs(const std::vector<uint32_t>& primary_ssrcs, + std::vector<uint32_t>* fid_ssrcs) const { + for (uint32_t primary_ssrc : primary_ssrcs) { + uint32_t fid_ssrc; + if (GetFidSsrc(primary_ssrc, &fid_ssrc)) { + fid_ssrcs->push_back(fid_ssrc); + } + } +} + +bool StreamParams::AddSecondarySsrc(const std::string& semantics, + uint32_t primary_ssrc, + uint32_t secondary_ssrc) { + if (!has_ssrc(primary_ssrc)) { + return false; + } + + ssrcs.push_back(secondary_ssrc); + ssrc_groups.push_back(SsrcGroup(semantics, {primary_ssrc, secondary_ssrc})); + return true; +} + +bool StreamParams::GetSecondarySsrc(const std::string& semantics, + uint32_t primary_ssrc, + uint32_t* secondary_ssrc) const { + for (const SsrcGroup& ssrc_group : ssrc_groups) { + if (ssrc_group.has_semantics(semantics) && ssrc_group.ssrcs.size() >= 2 && + ssrc_group.ssrcs[0] == primary_ssrc) { + *secondary_ssrc = ssrc_group.ssrcs[1]; + return true; + } + } + return false; +} + +std::vector<std::string> StreamParams::stream_ids() const { + return stream_ids_; +} + +void StreamParams::set_stream_ids(const std::vector<std::string>& stream_ids) { + stream_ids_ = stream_ids; +} + +std::string StreamParams::first_stream_id() const { + return stream_ids_.empty() ? "" : stream_ids_[0]; +} + +} // namespace cricket diff --git a/media/base/stream_params.h b/media/base/stream_params.h new file mode 100644 index 0000000000..b8c37706df --- /dev/null +++ b/media/base/stream_params.h @@ -0,0 +1,330 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// This file contains structures for describing SSRCs from a media source such +// as a MediaStreamTrack when it is sent across an RTP session. Multiple media +// sources may be sent across the same RTP session, each of them will be +// described by one StreamParams object +// SsrcGroup is used to describe the relationship between the SSRCs that +// are used for this media source. +// E.x: Consider a source that is sent as 3 simulcast streams +// Let the simulcast elements have SSRC 10, 20, 30. +// Let each simulcast element use FEC and let the protection packets have +// SSRC 11,21,31. +// To describe this 4 SsrcGroups are needed, +// StreamParams would then contain ssrc = {10,11,20,21,30,31} and +// ssrc_groups = {{SIM,{10,20,30}, {FEC,{10,11}, {FEC, {20,21}, {FEC {30,31}}} +// Please see RFC 5576. +// A spec-compliant way to achieve this is to use RIDs and Simulcast attribute +// instead of the ssrc-group. In this method, the StreamParam object will +// have multiple RidDescriptions, each corresponding to a simulcast layer +// and the media section will have a simulcast attribute that indicates +// that these layers are for the same source. This also removes the extra +// lines for redundancy streams, as the same RIDs appear in the redundancy +// packets. +// Note: in the spec compliant simulcast scenario, some of the RIDs might be +// alternatives for one another (such as different encodings for same data). +// In the context of the StreamParams class, the notion of alternatives does +// not exist and all the RIDs will describe different layers of the same source. +// When the StreamParams class is used to configure the media engine, simulcast +// considerations will be used to remove the alternative layers outside of this +// class. +// As an example, let the simulcast layers have RID 10, 20, 30. +// StreamParams would contain rid = { 10, 20, 30 }. +// MediaSection would contain SimulcastDescription specifying these rids. +// a=simulcast:send 10;20;30 (or a=simulcast:send 10,20;30 or similar). +// See https://tools.ietf.org/html/draft-ietf-mmusic-sdp-simulcast-13 +// and https://tools.ietf.org/html/draft-ietf-mmusic-rid-15. + +#ifndef MEDIA_BASE_STREAM_PARAMS_H_ +#define MEDIA_BASE_STREAM_PARAMS_H_ + +#include <stddef.h> + +#include <cstdint> +#include <string> +#include <vector> + +#include "absl/algorithm/container.h" +#include "media/base/rid_description.h" +#include "rtc_base/constructor_magic.h" +#include "rtc_base/unique_id_generator.h" + +namespace cricket { + +extern const char kFecSsrcGroupSemantics[]; +extern const char kFecFrSsrcGroupSemantics[]; +extern const char kFidSsrcGroupSemantics[]; +extern const char kSimSsrcGroupSemantics[]; + +struct SsrcGroup { + SsrcGroup(const std::string& usage, const std::vector<uint32_t>& ssrcs); + SsrcGroup(const SsrcGroup&); + SsrcGroup(SsrcGroup&&); + ~SsrcGroup(); + SsrcGroup& operator=(const SsrcGroup&); + SsrcGroup& operator=(SsrcGroup&&); + + bool operator==(const SsrcGroup& other) const { + return (semantics == other.semantics && ssrcs == other.ssrcs); + } + bool operator!=(const SsrcGroup& other) const { return !(*this == other); } + + bool has_semantics(const std::string& semantics) const; + + std::string ToString() const; + + std::string semantics; // e.g FIX, FEC, SIM. + std::vector<uint32_t> ssrcs; // SSRCs of this type. +}; + +// StreamParams is used to represent a sender/track in a SessionDescription. +// In Plan B, this means that multiple StreamParams can exist within one +// MediaContentDescription, while in UnifiedPlan this means that there is one +// StreamParams per MediaContentDescription. +struct StreamParams { + StreamParams(); + StreamParams(const StreamParams&); + StreamParams(StreamParams&&); + ~StreamParams(); + StreamParams& operator=(const StreamParams&); + StreamParams& operator=(StreamParams&&); + + static StreamParams CreateLegacy(uint32_t ssrc) { + StreamParams stream; + stream.ssrcs.push_back(ssrc); + return stream; + } + + bool operator==(const StreamParams& other) const; + bool operator!=(const StreamParams& other) const { return !(*this == other); } + + uint32_t first_ssrc() const { + if (ssrcs.empty()) { + return 0; + } + + return ssrcs[0]; + } + bool has_ssrcs() const { return !ssrcs.empty(); } + bool has_ssrc(uint32_t ssrc) const { + return absl::c_linear_search(ssrcs, ssrc); + } + void add_ssrc(uint32_t ssrc) { ssrcs.push_back(ssrc); } + bool has_ssrc_groups() const { return !ssrc_groups.empty(); } + bool has_ssrc_group(const std::string& semantics) const { + return (get_ssrc_group(semantics) != NULL); + } + const SsrcGroup* get_ssrc_group(const std::string& semantics) const { + for (const SsrcGroup& ssrc_group : ssrc_groups) { + if (ssrc_group.has_semantics(semantics)) { + return &ssrc_group; + } + } + return NULL; + } + + // Convenience function to add an FID ssrc for a primary_ssrc + // that's already been added. + bool AddFidSsrc(uint32_t primary_ssrc, uint32_t fid_ssrc) { + return AddSecondarySsrc(kFidSsrcGroupSemantics, primary_ssrc, fid_ssrc); + } + + // Convenience function to lookup the FID ssrc for a primary_ssrc. + // Returns false if primary_ssrc not found or FID not defined for it. + bool GetFidSsrc(uint32_t primary_ssrc, uint32_t* fid_ssrc) const { + return GetSecondarySsrc(kFidSsrcGroupSemantics, primary_ssrc, fid_ssrc); + } + + // Convenience function to add an FEC-FR ssrc for a primary_ssrc + // that's already been added. + bool AddFecFrSsrc(uint32_t primary_ssrc, uint32_t fecfr_ssrc) { + return AddSecondarySsrc(kFecFrSsrcGroupSemantics, primary_ssrc, fecfr_ssrc); + } + + // Convenience function to lookup the FEC-FR ssrc for a primary_ssrc. + // Returns false if primary_ssrc not found or FEC-FR not defined for it. + bool GetFecFrSsrc(uint32_t primary_ssrc, uint32_t* fecfr_ssrc) const { + return GetSecondarySsrc(kFecFrSsrcGroupSemantics, primary_ssrc, fecfr_ssrc); + } + + // Convenience function to populate the StreamParams with the requested number + // of SSRCs along with accompanying FID and FEC-FR ssrcs if requested. + // SSRCs are generated using the given generator. + void GenerateSsrcs(int num_layers, + bool generate_fid, + bool generate_fec_fr, + rtc::UniqueRandomIdGenerator* ssrc_generator); + + // Convenience to get all the SIM SSRCs if there are SIM ssrcs, or + // the first SSRC otherwise. + void GetPrimarySsrcs(std::vector<uint32_t>* ssrcs) const; + + // Convenience to get all the FID SSRCs for the given primary ssrcs. + // If a given primary SSRC does not have a FID SSRC, the list of FID + // SSRCS will be smaller than the list of primary SSRCs. + void GetFidSsrcs(const std::vector<uint32_t>& primary_ssrcs, + std::vector<uint32_t>* fid_ssrcs) const; + + // Stream ids serialized to SDP. + std::vector<std::string> stream_ids() const; + void set_stream_ids(const std::vector<std::string>& stream_ids); + + // Returns the first stream id or "" if none exist. This method exists only + // as temporary backwards compatibility with the old sync_label. + std::string first_stream_id() const; + + std::string ToString() const; + + // Resource of the MUC jid of the participant of with this stream. + // For 1:1 calls, should be left empty (which means remote streams + // and local streams should not be mixed together). This is not used + // internally and should be deprecated. + std::string groupid; + // A unique identifier of the StreamParams object. When the SDP is created, + // this comes from the track ID of the sender that the StreamParams object + // is associated with. + std::string id; + // There may be no SSRCs stored in unsignaled case when stream_ids are + // signaled with a=msid lines. + std::vector<uint32_t> ssrcs; // All SSRCs for this source + std::vector<SsrcGroup> ssrc_groups; // e.g. FID, FEC, SIM + std::string cname; // RTCP CNAME + + // RID functionality according to + // https://tools.ietf.org/html/draft-ietf-mmusic-rid-15 + // Each layer can be represented by a RID identifier and can also have + // restrictions (such as max-width, max-height, etc.) + // If the track has multiple layers (ex. Simulcast), each layer will be + // represented by a RID. + bool has_rids() const { return !rids_.empty(); } + const std::vector<RidDescription>& rids() const { return rids_; } + void set_rids(const std::vector<RidDescription>& rids) { rids_ = rids; } + + private: + bool AddSecondarySsrc(const std::string& semantics, + uint32_t primary_ssrc, + uint32_t secondary_ssrc); + bool GetSecondarySsrc(const std::string& semantics, + uint32_t primary_ssrc, + uint32_t* secondary_ssrc) const; + + // The stream IDs of the sender that the StreamParams object is associated + // with. In Plan B this should always be size of 1, while in Unified Plan this + // could be none or multiple stream IDs. + std::vector<std::string> stream_ids_; + + std::vector<RidDescription> rids_; +}; + +// A Stream can be selected by either groupid+id or ssrc. +struct StreamSelector { + explicit StreamSelector(uint32_t ssrc) : ssrc(ssrc) {} + + StreamSelector(const std::string& groupid, const std::string& streamid) + : ssrc(0), groupid(groupid), streamid(streamid) {} + + explicit StreamSelector(const std::string& streamid) + : ssrc(0), streamid(streamid) {} + + bool Matches(const StreamParams& stream) const { + if (ssrc == 0) { + return stream.groupid == groupid && stream.id == streamid; + } else { + return stream.has_ssrc(ssrc); + } + } + + uint32_t ssrc; + std::string groupid; + std::string streamid; +}; + +typedef std::vector<StreamParams> StreamParamsVec; + +template <class Condition> +const StreamParams* GetStream(const StreamParamsVec& streams, + Condition condition) { + auto found = absl::c_find_if(streams, condition); + return found == streams.end() ? nullptr : &(*found); +} + +template <class Condition> +StreamParams* GetStream(StreamParamsVec& streams, Condition condition) { + auto found = absl::c_find_if(streams, condition); + return found == streams.end() ? nullptr : &(*found); +} + +inline bool HasStreamWithNoSsrcs(const StreamParamsVec& streams) { + return GetStream(streams, + [](const StreamParams& sp) { return !sp.has_ssrcs(); }); +} + +inline const StreamParams* GetStreamBySsrc(const StreamParamsVec& streams, + uint32_t ssrc) { + return GetStream( + streams, [&ssrc](const StreamParams& sp) { return sp.has_ssrc(ssrc); }); +} + +inline const StreamParams* GetStreamByIds(const StreamParamsVec& streams, + const std::string& groupid, + const std::string& id) { + return GetStream(streams, [&groupid, &id](const StreamParams& sp) { + return sp.groupid == groupid && sp.id == id; + }); +} + +inline StreamParams* GetStreamByIds(StreamParamsVec& streams, + const std::string& groupid, + const std::string& id) { + return GetStream(streams, [&groupid, &id](const StreamParams& sp) { + return sp.groupid == groupid && sp.id == id; + }); +} + +inline const StreamParams* GetStream(const StreamParamsVec& streams, + const StreamSelector& selector) { + return GetStream(streams, [&selector](const StreamParams& sp) { + return selector.Matches(sp); + }); +} + +template <class Condition> +bool RemoveStream(StreamParamsVec* streams, Condition condition) { + auto iter(std::remove_if(streams->begin(), streams->end(), condition)); + if (iter == streams->end()) + return false; + streams->erase(iter, streams->end()); + return true; +} + +// Removes the stream from streams. Returns true if a stream is +// found and removed. +inline bool RemoveStream(StreamParamsVec* streams, + const StreamSelector& selector) { + return RemoveStream(streams, [&selector](const StreamParams& sp) { + return selector.Matches(sp); + }); +} +inline bool RemoveStreamBySsrc(StreamParamsVec* streams, uint32_t ssrc) { + return RemoveStream( + streams, [&ssrc](const StreamParams& sp) { return sp.has_ssrc(ssrc); }); +} +inline bool RemoveStreamByIds(StreamParamsVec* streams, + const std::string& groupid, + const std::string& id) { + return RemoveStream(streams, [&groupid, &id](const StreamParams& sp) { + return sp.groupid == groupid && sp.id == id; + }); +} + +} // namespace cricket + +#endif // MEDIA_BASE_STREAM_PARAMS_H_ diff --git a/media/base/stream_params_unittest.cc b/media/base/stream_params_unittest.cc new file mode 100644 index 0000000000..7adf0f517d --- /dev/null +++ b/media/base/stream_params_unittest.cc @@ -0,0 +1,301 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "media/base/stream_params.h" + +#include <stdint.h> + +#include "media/base/test_utils.h" +#include "rtc_base/arraysize.h" +#include "test/gmock.h" +#include "test/gtest.h" + +using ::testing::Each; +using ::testing::Ne; + +static const uint32_t kSsrcs1[] = {1}; +static const uint32_t kSsrcs2[] = {1, 2}; + +static cricket::StreamParams CreateStreamParamsWithSsrcGroup( + const std::string& semantics, + const uint32_t ssrcs_in[], + size_t len) { + cricket::StreamParams stream; + std::vector<uint32_t> ssrcs(ssrcs_in, ssrcs_in + len); + cricket::SsrcGroup sg(semantics, ssrcs); + stream.ssrcs = ssrcs; + stream.ssrc_groups.push_back(sg); + return stream; +} + +TEST(SsrcGroup, EqualNotEqual) { + cricket::SsrcGroup ssrc_groups[] = { + cricket::SsrcGroup("ABC", MAKE_VECTOR(kSsrcs1)), + cricket::SsrcGroup("ABC", MAKE_VECTOR(kSsrcs2)), + cricket::SsrcGroup("Abc", MAKE_VECTOR(kSsrcs2)), + cricket::SsrcGroup("abc", MAKE_VECTOR(kSsrcs2)), + }; + + for (size_t i = 0; i < arraysize(ssrc_groups); ++i) { + for (size_t j = 0; j < arraysize(ssrc_groups); ++j) { + EXPECT_EQ((ssrc_groups[i] == ssrc_groups[j]), (i == j)); + EXPECT_EQ((ssrc_groups[i] != ssrc_groups[j]), (i != j)); + } + } +} + +TEST(SsrcGroup, HasSemantics) { + cricket::SsrcGroup sg1("ABC", MAKE_VECTOR(kSsrcs1)); + EXPECT_TRUE(sg1.has_semantics("ABC")); + + cricket::SsrcGroup sg2("Abc", MAKE_VECTOR(kSsrcs1)); + EXPECT_FALSE(sg2.has_semantics("ABC")); + + cricket::SsrcGroup sg3("abc", MAKE_VECTOR(kSsrcs1)); + EXPECT_FALSE(sg3.has_semantics("ABC")); +} + +TEST(SsrcGroup, ToString) { + cricket::SsrcGroup sg1("ABC", MAKE_VECTOR(kSsrcs1)); + EXPECT_STREQ("{semantics:ABC;ssrcs:[1]}", sg1.ToString().c_str()); +} + +TEST(StreamParams, CreateLegacy) { + const uint32_t ssrc = 7; + cricket::StreamParams one_sp = cricket::StreamParams::CreateLegacy(ssrc); + EXPECT_EQ(1U, one_sp.ssrcs.size()); + EXPECT_EQ(ssrc, one_sp.first_ssrc()); + EXPECT_TRUE(one_sp.has_ssrcs()); + EXPECT_TRUE(one_sp.has_ssrc(ssrc)); + EXPECT_FALSE(one_sp.has_ssrc(ssrc + 1)); + EXPECT_FALSE(one_sp.has_ssrc_groups()); + EXPECT_EQ(0U, one_sp.ssrc_groups.size()); +} + +TEST(StreamParams, HasSsrcGroup) { + cricket::StreamParams sp = + CreateStreamParamsWithSsrcGroup("XYZ", kSsrcs2, arraysize(kSsrcs2)); + EXPECT_EQ(2U, sp.ssrcs.size()); + EXPECT_EQ(kSsrcs2[0], sp.first_ssrc()); + EXPECT_TRUE(sp.has_ssrcs()); + EXPECT_TRUE(sp.has_ssrc(kSsrcs2[0])); + EXPECT_TRUE(sp.has_ssrc(kSsrcs2[1])); + EXPECT_TRUE(sp.has_ssrc_group("XYZ")); + EXPECT_EQ(1U, sp.ssrc_groups.size()); + EXPECT_EQ(2U, sp.ssrc_groups[0].ssrcs.size()); + EXPECT_EQ(kSsrcs2[0], sp.ssrc_groups[0].ssrcs[0]); + EXPECT_EQ(kSsrcs2[1], sp.ssrc_groups[0].ssrcs[1]); +} + +TEST(StreamParams, GetSsrcGroup) { + cricket::StreamParams sp = + CreateStreamParamsWithSsrcGroup("XYZ", kSsrcs2, arraysize(kSsrcs2)); + EXPECT_EQ(NULL, sp.get_ssrc_group("xyz")); + EXPECT_EQ(&sp.ssrc_groups[0], sp.get_ssrc_group("XYZ")); +} + +TEST(StreamParams, HasStreamWithNoSsrcs) { + cricket::StreamParams sp_1 = cricket::StreamParams::CreateLegacy(kSsrcs1[0]); + cricket::StreamParams sp_2 = cricket::StreamParams::CreateLegacy(kSsrcs2[0]); + std::vector<cricket::StreamParams> streams({sp_1, sp_2}); + EXPECT_FALSE(HasStreamWithNoSsrcs(streams)); + + cricket::StreamParams unsignaled_stream; + streams.push_back(unsignaled_stream); + EXPECT_TRUE(HasStreamWithNoSsrcs(streams)); +} + +TEST(StreamParams, EqualNotEqual) { + cricket::StreamParams l1 = cricket::StreamParams::CreateLegacy(1); + cricket::StreamParams l2 = cricket::StreamParams::CreateLegacy(2); + cricket::StreamParams sg1 = + CreateStreamParamsWithSsrcGroup("ABC", kSsrcs1, arraysize(kSsrcs1)); + cricket::StreamParams sg2 = + CreateStreamParamsWithSsrcGroup("ABC", kSsrcs2, arraysize(kSsrcs2)); + cricket::StreamParams sg3 = + CreateStreamParamsWithSsrcGroup("Abc", kSsrcs2, arraysize(kSsrcs2)); + cricket::StreamParams sg4 = + CreateStreamParamsWithSsrcGroup("abc", kSsrcs2, arraysize(kSsrcs2)); + cricket::StreamParams sps[] = {l1, l2, sg1, sg2, sg3, sg4}; + + for (size_t i = 0; i < arraysize(sps); ++i) { + for (size_t j = 0; j < arraysize(sps); ++j) { + EXPECT_EQ((sps[i] == sps[j]), (i == j)); + EXPECT_EQ((sps[i] != sps[j]), (i != j)); + } + } +} + +TEST(StreamParams, FidFunctions) { + uint32_t fid_ssrc; + + cricket::StreamParams sp = cricket::StreamParams::CreateLegacy(1); + EXPECT_FALSE(sp.AddFidSsrc(10, 20)); + EXPECT_TRUE(sp.AddFidSsrc(1, 2)); + EXPECT_TRUE(sp.GetFidSsrc(1, &fid_ssrc)); + EXPECT_EQ(2u, fid_ssrc); + EXPECT_FALSE(sp.GetFidSsrc(15, &fid_ssrc)); + + sp.add_ssrc(20); + EXPECT_TRUE(sp.AddFidSsrc(20, 30)); + EXPECT_TRUE(sp.GetFidSsrc(20, &fid_ssrc)); + EXPECT_EQ(30u, fid_ssrc); + + // Manually create SsrcGroup to test bounds-checking + // in GetSecondarySsrc. We construct an invalid StreamParams + // for this. + std::vector<uint32_t> fid_vector; + fid_vector.push_back(13); + cricket::SsrcGroup invalid_fid_group(cricket::kFidSsrcGroupSemantics, + fid_vector); + cricket::StreamParams sp_invalid; + sp_invalid.add_ssrc(13); + sp_invalid.ssrc_groups.push_back(invalid_fid_group); + EXPECT_FALSE(sp_invalid.GetFidSsrc(13, &fid_ssrc)); +} + +TEST(StreamParams, GetPrimaryAndFidSsrcs) { + cricket::StreamParams sp; + sp.ssrcs.push_back(1); + sp.ssrcs.push_back(2); + sp.ssrcs.push_back(3); + + std::vector<uint32_t> primary_ssrcs; + sp.GetPrimarySsrcs(&primary_ssrcs); + std::vector<uint32_t> fid_ssrcs; + sp.GetFidSsrcs(primary_ssrcs, &fid_ssrcs); + ASSERT_EQ(1u, primary_ssrcs.size()); + EXPECT_EQ(1u, primary_ssrcs[0]); + ASSERT_EQ(0u, fid_ssrcs.size()); + + sp.ssrc_groups.push_back( + cricket::SsrcGroup(cricket::kSimSsrcGroupSemantics, sp.ssrcs)); + sp.AddFidSsrc(1, 10); + sp.AddFidSsrc(2, 20); + + primary_ssrcs.clear(); + sp.GetPrimarySsrcs(&primary_ssrcs); + fid_ssrcs.clear(); + sp.GetFidSsrcs(primary_ssrcs, &fid_ssrcs); + ASSERT_EQ(3u, primary_ssrcs.size()); + EXPECT_EQ(1u, primary_ssrcs[0]); + EXPECT_EQ(2u, primary_ssrcs[1]); + EXPECT_EQ(3u, primary_ssrcs[2]); + ASSERT_EQ(2u, fid_ssrcs.size()); + EXPECT_EQ(10u, fid_ssrcs[0]); + EXPECT_EQ(20u, fid_ssrcs[1]); +} + +TEST(StreamParams, FecFrFunctions) { + uint32_t fecfr_ssrc; + + cricket::StreamParams sp = cricket::StreamParams::CreateLegacy(1); + EXPECT_FALSE(sp.AddFecFrSsrc(10, 20)); + EXPECT_TRUE(sp.AddFecFrSsrc(1, 2)); + EXPECT_TRUE(sp.GetFecFrSsrc(1, &fecfr_ssrc)); + EXPECT_EQ(2u, fecfr_ssrc); + EXPECT_FALSE(sp.GetFecFrSsrc(15, &fecfr_ssrc)); + + sp.add_ssrc(20); + EXPECT_TRUE(sp.AddFecFrSsrc(20, 30)); + EXPECT_TRUE(sp.GetFecFrSsrc(20, &fecfr_ssrc)); + EXPECT_EQ(30u, fecfr_ssrc); + + // Manually create SsrcGroup to test bounds-checking + // in GetSecondarySsrc. We construct an invalid StreamParams + // for this. + std::vector<uint32_t> fecfr_vector; + fecfr_vector.push_back(13); + cricket::SsrcGroup invalid_fecfr_group(cricket::kFecFrSsrcGroupSemantics, + fecfr_vector); + cricket::StreamParams sp_invalid; + sp_invalid.add_ssrc(13); + sp_invalid.ssrc_groups.push_back(invalid_fecfr_group); + EXPECT_FALSE(sp_invalid.GetFecFrSsrc(13, &fecfr_ssrc)); +} + +TEST(StreamParams, ToString) { + cricket::StreamParams sp = + CreateStreamParamsWithSsrcGroup("XYZ", kSsrcs2, arraysize(kSsrcs2)); + sp.set_stream_ids({"stream_id"}); + EXPECT_STREQ( + "{ssrcs:[1,2];ssrc_groups:{semantics:XYZ;ssrcs:[1,2]};stream_ids:stream_" + "id;}", + sp.ToString().c_str()); +} + +TEST(StreamParams, TestGenerateSsrcs_SingleStreamWithRtxAndFlex) { + rtc::UniqueRandomIdGenerator generator; + cricket::StreamParams stream; + stream.GenerateSsrcs(1, true, true, &generator); + uint32_t primary_ssrc = stream.first_ssrc(); + ASSERT_NE(0u, primary_ssrc); + uint32_t rtx_ssrc = 0; + uint32_t flex_ssrc = 0; + EXPECT_EQ(3u, stream.ssrcs.size()); + EXPECT_TRUE(stream.GetFidSsrc(primary_ssrc, &rtx_ssrc)); + EXPECT_NE(0u, rtx_ssrc); + EXPECT_TRUE(stream.GetFecFrSsrc(primary_ssrc, &flex_ssrc)); + EXPECT_NE(0u, flex_ssrc); + EXPECT_FALSE(stream.has_ssrc_group(cricket::kSimSsrcGroupSemantics)); + EXPECT_TRUE(stream.has_ssrc_group(cricket::kFidSsrcGroupSemantics)); + EXPECT_TRUE(stream.has_ssrc_group(cricket::kFecFrSsrcGroupSemantics)); +} + +TEST(StreamParams, TestGenerateSsrcs_SingleStreamWithRtx) { + rtc::UniqueRandomIdGenerator generator; + cricket::StreamParams stream; + stream.GenerateSsrcs(1, true, false, &generator); + uint32_t primary_ssrc = stream.first_ssrc(); + ASSERT_NE(0u, primary_ssrc); + uint32_t rtx_ssrc = 0; + uint32_t flex_ssrc = 0; + EXPECT_EQ(2u, stream.ssrcs.size()); + EXPECT_TRUE(stream.GetFidSsrc(primary_ssrc, &rtx_ssrc)); + EXPECT_NE(0u, rtx_ssrc); + EXPECT_FALSE(stream.GetFecFrSsrc(primary_ssrc, &flex_ssrc)); + EXPECT_EQ(0u, flex_ssrc); + EXPECT_FALSE(stream.has_ssrc_group(cricket::kSimSsrcGroupSemantics)); + EXPECT_TRUE(stream.has_ssrc_group(cricket::kFidSsrcGroupSemantics)); +} + +TEST(StreamParams, TestGenerateSsrcs_SingleStreamWithFlex) { + rtc::UniqueRandomIdGenerator generator; + cricket::StreamParams stream; + stream.GenerateSsrcs(1, false, true, &generator); + uint32_t primary_ssrc = stream.first_ssrc(); + ASSERT_NE(0u, primary_ssrc); + uint32_t rtx_ssrc = 0; + uint32_t flex_ssrc = 0; + EXPECT_EQ(2u, stream.ssrcs.size()); + EXPECT_FALSE(stream.GetFidSsrc(primary_ssrc, &rtx_ssrc)); + EXPECT_EQ(0u, rtx_ssrc); + EXPECT_TRUE(stream.GetFecFrSsrc(primary_ssrc, &flex_ssrc)); + EXPECT_NE(0u, flex_ssrc); + EXPECT_FALSE(stream.has_ssrc_group(cricket::kSimSsrcGroupSemantics)); + EXPECT_TRUE(stream.has_ssrc_group(cricket::kFecFrSsrcGroupSemantics)); +} + +TEST(StreamParams, TestGenerateSsrcs_SimulcastLayersAndRtx) { + const size_t kNumStreams = 3; + rtc::UniqueRandomIdGenerator generator; + cricket::StreamParams stream; + stream.GenerateSsrcs(kNumStreams, true, false, &generator); + EXPECT_EQ(kNumStreams * 2, stream.ssrcs.size()); + std::vector<uint32_t> primary_ssrcs, rtx_ssrcs; + stream.GetPrimarySsrcs(&primary_ssrcs); + EXPECT_EQ(kNumStreams, primary_ssrcs.size()); + EXPECT_THAT(primary_ssrcs, Each(Ne(0u))); + stream.GetFidSsrcs(primary_ssrcs, &rtx_ssrcs); + EXPECT_EQ(kNumStreams, rtx_ssrcs.size()); + EXPECT_THAT(rtx_ssrcs, Each(Ne(0u))); + EXPECT_TRUE(stream.has_ssrc_group(cricket::kSimSsrcGroupSemantics)); + EXPECT_TRUE(stream.has_ssrc_group(cricket::kFidSsrcGroupSemantics)); +} diff --git a/media/base/test_utils.cc b/media/base/test_utils.cc new file mode 100644 index 0000000000..a6d5f61c17 --- /dev/null +++ b/media/base/test_utils.cc @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2004 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "media/base/test_utils.h" + +#include <cstdint> + +#include "api/video/video_frame.h" +#include "api/video/video_source_interface.h" + +namespace cricket { + +cricket::StreamParams CreateSimStreamParams( + const std::string& cname, + const std::vector<uint32_t>& ssrcs) { + cricket::StreamParams sp; + cricket::SsrcGroup sg(cricket::kSimSsrcGroupSemantics, ssrcs); + sp.ssrcs = ssrcs; + sp.ssrc_groups.push_back(sg); + sp.cname = cname; + return sp; +} + +// There should be an rtx_ssrc per ssrc. +cricket::StreamParams CreateSimWithRtxStreamParams( + const std::string& cname, + const std::vector<uint32_t>& ssrcs, + const std::vector<uint32_t>& rtx_ssrcs) { + cricket::StreamParams sp = CreateSimStreamParams(cname, ssrcs); + for (size_t i = 0; i < ssrcs.size(); ++i) { + sp.ssrcs.push_back(rtx_ssrcs[i]); + std::vector<uint32_t> fid_ssrcs; + fid_ssrcs.push_back(ssrcs[i]); + fid_ssrcs.push_back(rtx_ssrcs[i]); + cricket::SsrcGroup fid_group(cricket::kFidSsrcGroupSemantics, fid_ssrcs); + sp.ssrc_groups.push_back(fid_group); + } + return sp; +} + +cricket::StreamParams CreatePrimaryWithFecFrStreamParams( + const std::string& cname, + uint32_t primary_ssrc, + uint32_t flexfec_ssrc) { + cricket::StreamParams sp; + cricket::SsrcGroup sg(cricket::kFecFrSsrcGroupSemantics, + {primary_ssrc, flexfec_ssrc}); + sp.ssrcs = {primary_ssrc}; + sp.ssrc_groups.push_back(sg); + sp.cname = cname; + return sp; +} + +} // namespace cricket diff --git a/media/base/test_utils.h b/media/base/test_utils.h new file mode 100644 index 0000000000..46783a17f5 --- /dev/null +++ b/media/base/test_utils.h @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2004 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MEDIA_BASE_TEST_UTILS_H_ +#define MEDIA_BASE_TEST_UTILS_H_ + +#include <string> +#include <vector> + +#include "media/base/media_channel.h" +#include "media/base/video_common.h" +#include "rtc_base/arraysize.h" + +namespace webrtc { +class VideoFrame; +} + +namespace cricket { + +// Returns size of 420 image with rounding on chroma for odd sizes. +#define I420_SIZE(w, h) (w * h + (((w + 1) / 2) * ((h + 1) / 2)) * 2) +// Returns size of ARGB image. +#define ARGB_SIZE(w, h) (w * h * 4) + +template <class T> +inline std::vector<T> MakeVector(const T a[], size_t s) { + return std::vector<T>(a, a + s); +} +#define MAKE_VECTOR(a) cricket::MakeVector(a, arraysize(a)) + +// Checks whether |codecs| contains |codec|; checks using Codec::Matches(). +template <class C> +bool ContainsMatchingCodec(const std::vector<C>& codecs, const C& codec) { + typename std::vector<C>::const_iterator it; + for (it = codecs.begin(); it != codecs.end(); ++it) { + if (it->Matches(codec)) { + return true; + } + } + return false; +} + +// Create Simulcast StreamParams with given |ssrcs| and |cname|. +cricket::StreamParams CreateSimStreamParams(const std::string& cname, + const std::vector<uint32_t>& ssrcs); +// Create Simulcast stream with given |ssrcs| and |rtx_ssrcs|. +// The number of |rtx_ssrcs| must match number of |ssrcs|. +cricket::StreamParams CreateSimWithRtxStreamParams( + const std::string& cname, + const std::vector<uint32_t>& ssrcs, + const std::vector<uint32_t>& rtx_ssrcs); + +// Create StreamParams with single primary SSRC and corresponding FlexFEC SSRC. +cricket::StreamParams CreatePrimaryWithFecFrStreamParams( + const std::string& cname, + uint32_t primary_ssrc, + uint32_t flexfec_ssrc); + +} // namespace cricket + +#endif // MEDIA_BASE_TEST_UTILS_H_ diff --git a/media/base/turn_utils.cc b/media/base/turn_utils.cc new file mode 100644 index 0000000000..c413117fb6 --- /dev/null +++ b/media/base/turn_utils.cc @@ -0,0 +1,126 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "media/base/turn_utils.h" + +#include "api/transport/stun.h" +#include "rtc_base/byte_order.h" + +namespace cricket { + +namespace { + +const size_t kTurnChannelHeaderLength = 4; + +bool IsTurnChannelData(const uint8_t* data, size_t length) { + return length >= kTurnChannelHeaderLength && ((*data & 0xC0) == 0x40); +} + +bool IsTurnSendIndicationPacket(const uint8_t* data, size_t length) { + if (length < kStunHeaderSize) { + return false; + } + + uint16_t type = rtc::GetBE16(data); + return (type == TURN_SEND_INDICATION); +} + +} // namespace + +bool UnwrapTurnPacket(const uint8_t* packet, + size_t packet_size, + size_t* content_position, + size_t* content_size) { + if (IsTurnChannelData(packet, packet_size)) { + // Turn Channel Message header format. + // 0 1 2 3 + // 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + // | Channel Number | Length | + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + // | | + // / Application Data / + // / / + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + size_t length = rtc::GetBE16(&packet[2]); + if (length + kTurnChannelHeaderLength > packet_size) { + return false; + } + + *content_position = kTurnChannelHeaderLength; + *content_size = length; + return true; + } + + if (IsTurnSendIndicationPacket(packet, packet_size)) { + // Validate STUN message length. + const size_t stun_message_length = rtc::GetBE16(&packet[2]); + if (stun_message_length + kStunHeaderSize != packet_size) { + return false; + } + + // First skip mandatory stun header which is of 20 bytes. + size_t pos = kStunHeaderSize; + // Loop through STUN attributes until we find STUN DATA attribute. + while (pos < packet_size) { + // Keep reading STUN attributes until we hit DATA attribute. + // Attribute will be a TLV structure. + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + // | Type | Length | + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + // | Value (variable) .... + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + // The value in the length field MUST contain the length of the Value + // part of the attribute, prior to padding, measured in bytes. Since + // STUN aligns attributes on 32-bit boundaries, attributes whose content + // is not a multiple of 4 bytes are padded with 1, 2, or 3 bytes of + // padding so that its value contains a multiple of 4 bytes. The + // padding bits are ignored, and may be any value. + uint16_t attr_type, attr_length; + const int kAttrHeaderLength = sizeof(attr_type) + sizeof(attr_length); + + if (packet_size < pos + kAttrHeaderLength) { + return false; + } + + // Getting attribute type and length. + attr_type = rtc::GetBE16(&packet[pos]); + attr_length = rtc::GetBE16(&packet[pos + sizeof(attr_type)]); + + pos += kAttrHeaderLength; // Skip STUN_DATA_ATTR header. + + // Checking for bogus attribute length. + if (pos + attr_length > packet_size) { + return false; + } + + if (attr_type == STUN_ATTR_DATA) { + *content_position = pos; + *content_size = attr_length; + return true; + } + + pos += attr_length; + if ((attr_length % 4) != 0) { + pos += (4 - (attr_length % 4)); + } + } + + // There is no data attribute present in the message. + return false; + } + + // This is not a TURN packet. + *content_position = 0; + *content_size = packet_size; + return true; +} + +} // namespace cricket diff --git a/media/base/turn_utils.h b/media/base/turn_utils.h new file mode 100644 index 0000000000..ed8e282ba7 --- /dev/null +++ b/media/base/turn_utils.h @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MEDIA_BASE_TURN_UTILS_H_ +#define MEDIA_BASE_TURN_UTILS_H_ + +#include <cstddef> +#include <cstdint> + +#include "rtc_base/system/rtc_export.h" + +namespace cricket { + +struct PacketOptions; + +// Finds data location within a TURN Channel Message or TURN Send Indication +// message. +bool RTC_EXPORT UnwrapTurnPacket(const uint8_t* packet, + size_t packet_size, + size_t* content_position, + size_t* content_size); + +} // namespace cricket + +#endif // MEDIA_BASE_TURN_UTILS_H_ diff --git a/media/base/turn_utils_unittest.cc b/media/base/turn_utils_unittest.cc new file mode 100644 index 0000000000..f7bbf8b8d4 --- /dev/null +++ b/media/base/turn_utils_unittest.cc @@ -0,0 +1,127 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "media/base/turn_utils.h" + +#include "test/gtest.h" + +namespace cricket { + +// Invalid TURN send indication messages. Messages are proper STUN +// messages with incorrect values in attributes. +TEST(TurnUtilsTest, InvalidTurnSendIndicationMessages) { + size_t content_pos = SIZE_MAX; + size_t content_size = SIZE_MAX; + + // Stun Indication message with Zero length + uint8_t kTurnSendIndicationMsgWithNoAttributes[] = { + 0x00, 0x16, 0x00, 0x00, // Zero length + 0x21, 0x12, 0xA4, 0x42, // magic cookie + '0', '1', '2', '3', // transaction id + '4', '5', '6', '7', '8', '9', 'a', 'b', + }; + EXPECT_FALSE(UnwrapTurnPacket(kTurnSendIndicationMsgWithNoAttributes, + sizeof(kTurnSendIndicationMsgWithNoAttributes), + &content_pos, &content_size)); + EXPECT_EQ(SIZE_MAX, content_pos); + EXPECT_EQ(SIZE_MAX, content_size); + + // Stun Send Indication message with invalid length in stun header. + const uint8_t kTurnSendIndicationMsgWithInvalidLength[] = { + 0x00, 0x16, 0xFF, 0x00, // length of 0xFF00 + 0x21, 0x12, 0xA4, 0x42, // magic cookie + '0', '1', '2', '3', // transaction id + '4', '5', '6', '7', '8', '9', 'a', 'b', + }; + EXPECT_FALSE(UnwrapTurnPacket(kTurnSendIndicationMsgWithInvalidLength, + sizeof(kTurnSendIndicationMsgWithInvalidLength), + &content_pos, &content_size)); + EXPECT_EQ(SIZE_MAX, content_pos); + EXPECT_EQ(SIZE_MAX, content_size); + + // Stun Send Indication message with no DATA attribute in message. + const uint8_t kTurnSendIndicatinMsgWithNoDataAttribute[] = { + // clang-format off + // clang formatting doesn't respect inline comments. + 0x00, 0x16, 0x00, 0x08, // length of + 0x21, 0x12, 0xA4, 0x42, // magic cookie + '0', '1', '2', '3', // transaction id + '4', '5', '6', '7', '8', '9', 'a', 'b', + 0x00, 0x20, 0x00, 0x04, // Mapped address. + 0x00, 0x00, 0x00, 0x00, + // clang-format on + }; + EXPECT_FALSE( + UnwrapTurnPacket(kTurnSendIndicatinMsgWithNoDataAttribute, + sizeof(kTurnSendIndicatinMsgWithNoDataAttribute), + &content_pos, &content_size)); + EXPECT_EQ(SIZE_MAX, content_pos); + EXPECT_EQ(SIZE_MAX, content_size); +} + +// Valid TURN Send Indication messages. +TEST(TurnUtilsTest, ValidTurnSendIndicationMessage) { + size_t content_pos = SIZE_MAX; + size_t content_size = SIZE_MAX; + // A valid STUN indication message with a valid RTP header in data attribute + // payload field and no extension bit set. + const uint8_t kTurnSendIndicationMsgWithoutRtpExtension[] = { + // clang-format off + // clang formatting doesn't respect inline comments. + 0x00, 0x16, 0x00, 0x18, // length of + 0x21, 0x12, 0xA4, 0x42, // magic cookie + '0', '1', '2', '3', // transaction id + '4', '5', '6', '7', '8', '9', 'a', 'b', + 0x00, 0x20, 0x00, 0x04, // Mapped address. + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x13, 0x00, 0x0C, // Data attribute. + 0x80, 0x00, 0x00, 0x00, // RTP packet. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + // clang-format on + }; + EXPECT_TRUE( + UnwrapTurnPacket(kTurnSendIndicationMsgWithoutRtpExtension, + sizeof(kTurnSendIndicationMsgWithoutRtpExtension), + &content_pos, &content_size)); + EXPECT_EQ(12U, content_size); + EXPECT_EQ(32U, content_pos); +} + +// Verify that parsing of valid TURN Channel Messages. +TEST(TurnUtilsTest, ValidTurnChannelMessages) { + const uint8_t kTurnChannelMsgWithRtpPacket[] = { + // clang-format off + // clang formatting doesn't respect inline comments. + 0x40, 0x00, 0x00, 0x0C, + 0x80, 0x00, 0x00, 0x00, // RTP packet. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + // clang-format on + }; + + size_t content_pos = 0, content_size = 0; + EXPECT_TRUE(UnwrapTurnPacket(kTurnChannelMsgWithRtpPacket, + sizeof(kTurnChannelMsgWithRtpPacket), + &content_pos, &content_size)); + EXPECT_EQ(12U, content_size); + EXPECT_EQ(4U, content_pos); +} + +TEST(TurnUtilsTest, ChannelMessageZeroLength) { + const uint8_t kTurnChannelMsgWithZeroLength[] = {0x40, 0x00, 0x00, 0x00}; + size_t content_pos = SIZE_MAX; + size_t content_size = SIZE_MAX; + EXPECT_TRUE(UnwrapTurnPacket(kTurnChannelMsgWithZeroLength, + sizeof(kTurnChannelMsgWithZeroLength), + &content_pos, &content_size)); + EXPECT_EQ(4u, content_pos); + EXPECT_EQ(0u, content_size); +} + +} // namespace cricket diff --git a/media/base/video_adapter.cc b/media/base/video_adapter.cc new file mode 100644 index 0000000000..27b82646ac --- /dev/null +++ b/media/base/video_adapter.cc @@ -0,0 +1,354 @@ +/* + * Copyright (c) 2010 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "media/base/video_adapter.h" + +#include <algorithm> +#include <cmath> +#include <cstdlib> +#include <limits> +#include <utility> + +#include "absl/types/optional.h" +#include "media/base/video_common.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "rtc_base/time_utils.h" +#include "system_wrappers/include/field_trial.h" + +namespace { + +struct Fraction { + int numerator; + int denominator; + + void DivideByGcd() { + int g = cricket::GreatestCommonDivisor(numerator, denominator); + numerator /= g; + denominator /= g; + } + + // Determines number of output pixels if both width and height of an input of + // |input_pixels| pixels is scaled with the fraction numerator / denominator. + int scale_pixel_count(int input_pixels) { + return (numerator * numerator * input_pixels) / (denominator * denominator); + } +}; + +// Round |value_to_round| to a multiple of |multiple|. Prefer rounding upwards, +// but never more than |max_value|. +int roundUp(int value_to_round, int multiple, int max_value) { + const int rounded_value = + (value_to_round + multiple - 1) / multiple * multiple; + return rounded_value <= max_value ? rounded_value + : (max_value / multiple * multiple); +} + +// Generates a scale factor that makes |input_pixels| close to |target_pixels|, +// but no higher than |max_pixels|. +Fraction FindScale(int input_width, + int input_height, + int target_pixels, + int max_pixels, + bool variable_start_scale_factor) { + // This function only makes sense for a positive target. + RTC_DCHECK_GT(target_pixels, 0); + RTC_DCHECK_GT(max_pixels, 0); + RTC_DCHECK_GE(max_pixels, target_pixels); + + const int input_pixels = input_width * input_height; + + // Don't scale up original. + if (target_pixels >= input_pixels) + return Fraction{1, 1}; + + Fraction current_scale = Fraction{1, 1}; + Fraction best_scale = Fraction{1, 1}; + + if (variable_start_scale_factor) { + // Start scaling down by 2/3 depending on |input_width| and |input_height|. + if (input_width % 3 == 0 && input_height % 3 == 0) { + // 2/3 (then alternates 3/4, 2/3, 3/4,...). + current_scale = Fraction{6, 6}; + } + if (input_width % 9 == 0 && input_height % 9 == 0) { + // 2/3, 2/3 (then alternates 3/4, 2/3, 3/4,...). + current_scale = Fraction{36, 36}; + } + } + + // The minimum (absolute) difference between the number of output pixels and + // the target pixel count. + int min_pixel_diff = std::numeric_limits<int>::max(); + if (input_pixels <= max_pixels) { + // Start condition for 1/1 case, if it is less than max. + min_pixel_diff = std::abs(input_pixels - target_pixels); + } + + // Alternately scale down by 3/4 and 2/3. This results in fractions which are + // effectively scalable. For instance, starting at 1280x720 will result in + // the series (3/4) => 960x540, (1/2) => 640x360, (3/8) => 480x270, + // (1/4) => 320x180, (3/16) => 240x125, (1/8) => 160x90. + while (current_scale.scale_pixel_count(input_pixels) > target_pixels) { + if (current_scale.numerator % 3 == 0 && + current_scale.denominator % 2 == 0) { + // Multiply by 2/3. + current_scale.numerator /= 3; + current_scale.denominator /= 2; + } else { + // Multiply by 3/4. + current_scale.numerator *= 3; + current_scale.denominator *= 4; + } + + int output_pixels = current_scale.scale_pixel_count(input_pixels); + if (output_pixels <= max_pixels) { + int diff = std::abs(target_pixels - output_pixels); + if (diff < min_pixel_diff) { + min_pixel_diff = diff; + best_scale = current_scale; + } + } + } + best_scale.DivideByGcd(); + + return best_scale; +} +} // namespace + +namespace cricket { + +VideoAdapter::VideoAdapter(int source_resolution_alignment) + : frames_in_(0), + frames_out_(0), + frames_scaled_(0), + adaption_changes_(0), + previous_width_(0), + previous_height_(0), + variable_start_scale_factor_(webrtc::field_trial::IsEnabled( + "WebRTC-Video-VariableStartScaleFactor")), + source_resolution_alignment_(source_resolution_alignment), + resolution_alignment_(source_resolution_alignment), + resolution_request_target_pixel_count_(std::numeric_limits<int>::max()), + resolution_request_max_pixel_count_(std::numeric_limits<int>::max()), + max_framerate_request_(std::numeric_limits<int>::max()) {} + +VideoAdapter::VideoAdapter() : VideoAdapter(1) {} + +VideoAdapter::~VideoAdapter() {} + +bool VideoAdapter::KeepFrame(int64_t in_timestamp_ns) { + rtc::CritScope cs(&critical_section_); + + int max_fps = max_framerate_request_; + if (max_fps_) + max_fps = std::min(max_fps, *max_fps_); + + if (max_fps <= 0) + return false; + + // If |max_framerate_request_| is not set, it will default to maxint, which + // will lead to a frame_interval_ns rounded to 0. + int64_t frame_interval_ns = rtc::kNumNanosecsPerSec / max_fps; + if (frame_interval_ns <= 0) { + // Frame rate throttling not enabled. + return true; + } + + if (next_frame_timestamp_ns_) { + // Time until next frame should be outputted. + const int64_t time_until_next_frame_ns = + (*next_frame_timestamp_ns_ - in_timestamp_ns); + + // Continue if timestamp is within expected range. + if (std::abs(time_until_next_frame_ns) < 2 * frame_interval_ns) { + // Drop if a frame shouldn't be outputted yet. + if (time_until_next_frame_ns > 0) + return false; + // Time to output new frame. + *next_frame_timestamp_ns_ += frame_interval_ns; + return true; + } + } + + // First timestamp received or timestamp is way outside expected range, so + // reset. Set first timestamp target to just half the interval to prefer + // keeping frames in case of jitter. + next_frame_timestamp_ns_ = in_timestamp_ns + frame_interval_ns / 2; + return true; +} + +bool VideoAdapter::AdaptFrameResolution(int in_width, + int in_height, + int64_t in_timestamp_ns, + int* cropped_width, + int* cropped_height, + int* out_width, + int* out_height) { + rtc::CritScope cs(&critical_section_); + ++frames_in_; + + // The max output pixel count is the minimum of the requests from + // OnOutputFormatRequest and OnResolutionFramerateRequest. + int max_pixel_count = resolution_request_max_pixel_count_; + + // Select target aspect ratio and max pixel count depending on input frame + // orientation. + absl::optional<std::pair<int, int>> target_aspect_ratio; + if (in_width > in_height) { + target_aspect_ratio = target_landscape_aspect_ratio_; + if (max_landscape_pixel_count_) + max_pixel_count = std::min(max_pixel_count, *max_landscape_pixel_count_); + } else { + target_aspect_ratio = target_portrait_aspect_ratio_; + if (max_portrait_pixel_count_) + max_pixel_count = std::min(max_pixel_count, *max_portrait_pixel_count_); + } + + int target_pixel_count = + std::min(resolution_request_target_pixel_count_, max_pixel_count); + + // Drop the input frame if necessary. + if (max_pixel_count <= 0 || !KeepFrame(in_timestamp_ns)) { + // Show VAdapt log every 90 frames dropped. (3 seconds) + if ((frames_in_ - frames_out_) % 90 == 0) { + // TODO(fbarchard): Reduce to LS_VERBOSE when adapter info is not needed + // in default calls. + RTC_LOG(LS_INFO) << "VAdapt Drop Frame: scaled " << frames_scaled_ + << " / out " << frames_out_ << " / in " << frames_in_ + << " Changes: " << adaption_changes_ + << " Input: " << in_width << "x" << in_height + << " timestamp: " << in_timestamp_ns + << " Output fps: " << max_framerate_request_ << "/" + << max_fps_.value_or(-1) + << " alignment: " << resolution_alignment_; + } + + // Drop frame. + return false; + } + + // Calculate how the input should be cropped. + if (!target_aspect_ratio || target_aspect_ratio->first <= 0 || + target_aspect_ratio->second <= 0) { + *cropped_width = in_width; + *cropped_height = in_height; + } else { + const float requested_aspect = + target_aspect_ratio->first / + static_cast<float>(target_aspect_ratio->second); + *cropped_width = + std::min(in_width, static_cast<int>(in_height * requested_aspect)); + *cropped_height = + std::min(in_height, static_cast<int>(in_width / requested_aspect)); + } + const Fraction scale = + FindScale(*cropped_width, *cropped_height, target_pixel_count, + max_pixel_count, variable_start_scale_factor_); + // Adjust cropping slightly to get correctly aligned output size and a perfect + // scale factor. + *cropped_width = roundUp(*cropped_width, + scale.denominator * resolution_alignment_, in_width); + *cropped_height = roundUp( + *cropped_height, scale.denominator * resolution_alignment_, in_height); + RTC_DCHECK_EQ(0, *cropped_width % scale.denominator); + RTC_DCHECK_EQ(0, *cropped_height % scale.denominator); + + // Calculate final output size. + *out_width = *cropped_width / scale.denominator * scale.numerator; + *out_height = *cropped_height / scale.denominator * scale.numerator; + RTC_DCHECK_EQ(0, *out_width % resolution_alignment_); + RTC_DCHECK_EQ(0, *out_height % resolution_alignment_); + + ++frames_out_; + if (scale.numerator != scale.denominator) + ++frames_scaled_; + + if (previous_width_ && + (previous_width_ != *out_width || previous_height_ != *out_height)) { + ++adaption_changes_; + RTC_LOG(LS_INFO) << "Frame size changed: scaled " << frames_scaled_ + << " / out " << frames_out_ << " / in " << frames_in_ + << " Changes: " << adaption_changes_ + << " Input: " << in_width << "x" << in_height + << " Scale: " << scale.numerator << "/" + << scale.denominator << " Output: " << *out_width << "x" + << *out_height << " fps: " << max_framerate_request_ << "/" + << max_fps_.value_or(-1) + << " alignment: " << resolution_alignment_; + } + + previous_width_ = *out_width; + previous_height_ = *out_height; + + return true; +} + +void VideoAdapter::OnOutputFormatRequest( + const absl::optional<VideoFormat>& format) { + absl::optional<std::pair<int, int>> target_aspect_ratio; + absl::optional<int> max_pixel_count; + absl::optional<int> max_fps; + if (format) { + target_aspect_ratio = std::make_pair(format->width, format->height); + max_pixel_count = format->width * format->height; + if (format->interval > 0) + max_fps = rtc::kNumNanosecsPerSec / format->interval; + } + OnOutputFormatRequest(target_aspect_ratio, max_pixel_count, max_fps); +} + +void VideoAdapter::OnOutputFormatRequest( + const absl::optional<std::pair<int, int>>& target_aspect_ratio, + const absl::optional<int>& max_pixel_count, + const absl::optional<int>& max_fps) { + absl::optional<std::pair<int, int>> target_landscape_aspect_ratio; + absl::optional<std::pair<int, int>> target_portrait_aspect_ratio; + if (target_aspect_ratio && target_aspect_ratio->first > 0 && + target_aspect_ratio->second > 0) { + // Maintain input orientation. + const int max_side = + std::max(target_aspect_ratio->first, target_aspect_ratio->second); + const int min_side = + std::min(target_aspect_ratio->first, target_aspect_ratio->second); + target_landscape_aspect_ratio = std::make_pair(max_side, min_side); + target_portrait_aspect_ratio = std::make_pair(min_side, max_side); + } + OnOutputFormatRequest(target_landscape_aspect_ratio, max_pixel_count, + target_portrait_aspect_ratio, max_pixel_count, max_fps); +} + +void VideoAdapter::OnOutputFormatRequest( + const absl::optional<std::pair<int, int>>& target_landscape_aspect_ratio, + const absl::optional<int>& max_landscape_pixel_count, + const absl::optional<std::pair<int, int>>& target_portrait_aspect_ratio, + const absl::optional<int>& max_portrait_pixel_count, + const absl::optional<int>& max_fps) { + rtc::CritScope cs(&critical_section_); + target_landscape_aspect_ratio_ = target_landscape_aspect_ratio; + max_landscape_pixel_count_ = max_landscape_pixel_count; + target_portrait_aspect_ratio_ = target_portrait_aspect_ratio; + max_portrait_pixel_count_ = max_portrait_pixel_count; + max_fps_ = max_fps; + next_frame_timestamp_ns_ = absl::nullopt; +} + +void VideoAdapter::OnSinkWants(const rtc::VideoSinkWants& sink_wants) { + rtc::CritScope cs(&critical_section_); + resolution_request_max_pixel_count_ = sink_wants.max_pixel_count; + resolution_request_target_pixel_count_ = + sink_wants.target_pixel_count.value_or( + resolution_request_max_pixel_count_); + max_framerate_request_ = sink_wants.max_framerate_fps; + resolution_alignment_ = cricket::LeastCommonMultiple( + source_resolution_alignment_, sink_wants.resolution_alignment); +} + +} // namespace cricket diff --git a/media/base/video_adapter.h b/media/base/video_adapter.h new file mode 100644 index 0000000000..936cf8917e --- /dev/null +++ b/media/base/video_adapter.h @@ -0,0 +1,145 @@ +/* + * Copyright (c) 2010 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MEDIA_BASE_VIDEO_ADAPTER_H_ +#define MEDIA_BASE_VIDEO_ADAPTER_H_ + +#include <stdint.h> + +#include <utility> + +#include "absl/types/optional.h" +#include "api/video/video_source_interface.h" +#include "media/base/video_common.h" +#include "rtc_base/constructor_magic.h" +#include "rtc_base/critical_section.h" +#include "rtc_base/thread_annotations.h" + +namespace cricket { + +// VideoAdapter adapts an input video frame to an output frame based on the +// specified input and output formats. The adaptation includes dropping frames +// to reduce frame rate and scaling frames. +// VideoAdapter is thread safe. +class VideoAdapter { + public: + VideoAdapter(); + // The source requests output frames whose width and height are divisible + // by |source_resolution_alignment|. + explicit VideoAdapter(int source_resolution_alignment); + virtual ~VideoAdapter(); + + // Return the adapted resolution and cropping parameters given the + // input resolution. The input frame should first be cropped, then + // scaled to the final output resolution. Returns true if the frame + // should be adapted, and false if it should be dropped. + bool AdaptFrameResolution(int in_width, + int in_height, + int64_t in_timestamp_ns, + int* cropped_width, + int* cropped_height, + int* out_width, + int* out_height); + + // DEPRECATED. Please use OnOutputFormatRequest below. + // TODO(asapersson): Remove this once it is no longer used. + // Requests the output frame size and frame interval from + // |AdaptFrameResolution| to not be larger than |format|. Also, the input + // frame size will be cropped to match the requested aspect ratio. The + // requested aspect ratio is orientation agnostic and will be adjusted to + // maintain the input orientation, so it doesn't matter if e.g. 1280x720 or + // 720x1280 is requested. + // Note: Should be called from the source only. + void OnOutputFormatRequest(const absl::optional<VideoFormat>& format); + + // Requests output frame size and frame interval from |AdaptFrameResolution|. + // |target_aspect_ratio|: The input frame size will be cropped to match the + // requested aspect ratio. The aspect ratio is orientation agnostic and will + // be adjusted to maintain the input orientation (i.e. it doesn't matter if + // e.g. <1280,720> or <720,1280> is requested). + // |max_pixel_count|: The maximum output frame size. + // |max_fps|: The maximum output framerate. + // Note: Should be called from the source only. + void OnOutputFormatRequest( + const absl::optional<std::pair<int, int>>& target_aspect_ratio, + const absl::optional<int>& max_pixel_count, + const absl::optional<int>& max_fps); + + // Same as above, but allows setting two different target aspect ratios + // depending on incoming frame orientation. This gives more fine-grained + // control and can e.g. be used to force landscape video to be cropped to + // portrait video. + void OnOutputFormatRequest( + const absl::optional<std::pair<int, int>>& target_landscape_aspect_ratio, + const absl::optional<int>& max_landscape_pixel_count, + const absl::optional<std::pair<int, int>>& target_portrait_aspect_ratio, + const absl::optional<int>& max_portrait_pixel_count, + const absl::optional<int>& max_fps); + + // Requests the output frame size from |AdaptFrameResolution| to have as close + // as possible to |sink_wants.target_pixel_count| pixels (if set) + // but no more than |sink_wants.max_pixel_count|. + // |sink_wants.max_framerate_fps| is essentially analogous to + // |sink_wants.max_pixel_count|, but for framerate rather than resolution. + // Set |sink_wants.max_pixel_count| and/or |sink_wants.max_framerate_fps| to + // std::numeric_limit<int>::max() if no upper limit is desired. + // The sink resolution alignment requirement is given by + // |sink_wants.resolution_alignment|. + // Note: Should be called from the sink only. + void OnSinkWants(const rtc::VideoSinkWants& sink_wants); + + private: + // Determine if frame should be dropped based on input fps and requested fps. + bool KeepFrame(int64_t in_timestamp_ns); + + int frames_in_; // Number of input frames. + int frames_out_; // Number of output frames. + int frames_scaled_; // Number of frames scaled. + int adaption_changes_; // Number of changes in scale factor. + int previous_width_; // Previous adapter output width. + int previous_height_; // Previous adapter output height. + const bool variable_start_scale_factor_; + + // The fixed source resolution alignment requirement. + const int source_resolution_alignment_; + // The currently applied resolution alignment, as given by the requirements: + // - the fixed |source_resolution_alignment_|; and + // - the latest |sink_wants.resolution_alignment|. + int resolution_alignment_ RTC_GUARDED_BY(critical_section_); + + // The target timestamp for the next frame based on requested format. + absl::optional<int64_t> next_frame_timestamp_ns_ + RTC_GUARDED_BY(critical_section_); + + // Max number of pixels/fps requested via calls to OnOutputFormatRequest, + // OnResolutionFramerateRequest respectively. + // The adapted output format is the minimum of these. + absl::optional<std::pair<int, int>> target_landscape_aspect_ratio_ + RTC_GUARDED_BY(critical_section_); + absl::optional<int> max_landscape_pixel_count_ + RTC_GUARDED_BY(critical_section_); + absl::optional<std::pair<int, int>> target_portrait_aspect_ratio_ + RTC_GUARDED_BY(critical_section_); + absl::optional<int> max_portrait_pixel_count_ + RTC_GUARDED_BY(critical_section_); + absl::optional<int> max_fps_ RTC_GUARDED_BY(critical_section_); + int resolution_request_target_pixel_count_ RTC_GUARDED_BY(critical_section_); + int resolution_request_max_pixel_count_ RTC_GUARDED_BY(critical_section_); + int max_framerate_request_ RTC_GUARDED_BY(critical_section_); + + // The critical section to protect the above variables. + rtc::CriticalSection critical_section_; + + RTC_DISALLOW_COPY_AND_ASSIGN(VideoAdapter); +}; + +} // namespace cricket + +#endif // MEDIA_BASE_VIDEO_ADAPTER_H_ diff --git a/media/base/video_adapter_unittest.cc b/media/base/video_adapter_unittest.cc new file mode 100644 index 0000000000..cbea8178ba --- /dev/null +++ b/media/base/video_adapter_unittest.cc @@ -0,0 +1,1389 @@ +/* + * Copyright (c) 2010 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "media/base/video_adapter.h" + +#include <limits> +#include <memory> +#include <string> +#include <utility> + +#include "api/video/video_frame.h" +#include "api/video/video_source_interface.h" +#include "media/base/fake_frame_source.h" +#include "rtc_base/arraysize.h" +#include "rtc_base/time_utils.h" +#include "test/field_trial.h" +#include "test/gtest.h" + +namespace cricket { +namespace { +const int kWidth = 1280; +const int kHeight = 720; +const int kDefaultFps = 30; + +rtc::VideoSinkWants BuildSinkWants(absl::optional<int> target_pixel_count, + int max_pixel_count, + int max_framerate_fps, + int sink_alignment = 1) { + rtc::VideoSinkWants wants; + wants.target_pixel_count = target_pixel_count; + wants.max_pixel_count = max_pixel_count; + wants.max_framerate_fps = max_framerate_fps; + wants.resolution_alignment = sink_alignment; + return wants; +} + +} // namespace + +class VideoAdapterTest : public ::testing::Test, + public ::testing::WithParamInterface<bool> { + public: + VideoAdapterTest() : VideoAdapterTest("", 1) {} + explicit VideoAdapterTest(const std::string& field_trials, + int source_resolution_alignment) + : override_field_trials_(field_trials), + frame_source_(std::make_unique<FakeFrameSource>( + kWidth, + kHeight, + VideoFormat::FpsToInterval(kDefaultFps) / + rtc::kNumNanosecsPerMicrosec)), + adapter_(source_resolution_alignment), + adapter_wrapper_(std::make_unique<VideoAdapterWrapper>(&adapter_)), + use_new_format_request_(GetParam()) {} + + protected: + // Wrap a VideoAdapter and collect stats. + class VideoAdapterWrapper { + public: + struct Stats { + int captured_frames = 0; + int dropped_frames = 0; + bool last_adapt_was_no_op = false; + + int cropped_width = 0; + int cropped_height = 0; + int out_width = 0; + int out_height = 0; + }; + + explicit VideoAdapterWrapper(VideoAdapter* adapter) + : video_adapter_(adapter) {} + + void AdaptFrame(const webrtc::VideoFrame& frame) { + const int in_width = frame.width(); + const int in_height = frame.height(); + int cropped_width; + int cropped_height; + int out_width; + int out_height; + if (video_adapter_->AdaptFrameResolution( + in_width, in_height, + frame.timestamp_us() * rtc::kNumNanosecsPerMicrosec, + &cropped_width, &cropped_height, &out_width, &out_height)) { + stats_.cropped_width = cropped_width; + stats_.cropped_height = cropped_height; + stats_.out_width = out_width; + stats_.out_height = out_height; + stats_.last_adapt_was_no_op = + (in_width == cropped_width && in_height == cropped_height && + in_width == out_width && in_height == out_height); + } else { + ++stats_.dropped_frames; + } + ++stats_.captured_frames; + } + + Stats GetStats() const { return stats_; } + + private: + VideoAdapter* video_adapter_; + Stats stats_; + }; + + void VerifyAdaptedResolution(const VideoAdapterWrapper::Stats& stats, + int cropped_width, + int cropped_height, + int out_width, + int out_height) { + EXPECT_EQ(cropped_width, stats.cropped_width); + EXPECT_EQ(cropped_height, stats.cropped_height); + EXPECT_EQ(out_width, stats.out_width); + EXPECT_EQ(out_height, stats.out_height); + } + + void OnOutputFormatRequest(int width, + int height, + const absl::optional<int>& fps) { + if (use_new_format_request_) { + absl::optional<std::pair<int, int>> target_aspect_ratio = + std::make_pair(width, height); + absl::optional<int> max_pixel_count = width * height; + absl::optional<int> max_fps = fps; + adapter_.OnOutputFormatRequest(target_aspect_ratio, max_pixel_count, + max_fps); + return; + } + adapter_.OnOutputFormatRequest( + VideoFormat(width, height, fps ? VideoFormat::FpsToInterval(*fps) : 0, + cricket::FOURCC_I420)); + } + + webrtc::test::ScopedFieldTrials override_field_trials_; + const std::unique_ptr<FakeFrameSource> frame_source_; + VideoAdapter adapter_; + int cropped_width_; + int cropped_height_; + int out_width_; + int out_height_; + const std::unique_ptr<VideoAdapterWrapper> adapter_wrapper_; + const bool use_new_format_request_; +}; + +class VideoAdapterTestVariableStartScale : public VideoAdapterTest { + public: + VideoAdapterTestVariableStartScale() + : VideoAdapterTest("WebRTC-Video-VariableStartScaleFactor/Enabled/", + /*source_resolution_alignment=*/1) {} +}; + +INSTANTIATE_TEST_SUITE_P(OnOutputFormatRequests, + VideoAdapterTest, + ::testing::Values(true, false)); + +INSTANTIATE_TEST_SUITE_P(OnOutputFormatRequests, + VideoAdapterTestVariableStartScale, + ::testing::Values(true, false)); + +// Do not adapt the frame rate or the resolution. Expect no frame drop, no +// cropping, and no resolution change. +TEST_P(VideoAdapterTest, AdaptNothing) { + for (int i = 0; i < 10; ++i) + adapter_wrapper_->AdaptFrame(frame_source_->GetFrame()); + + // Verify no frame drop and no resolution change. + VideoAdapterWrapper::Stats stats = adapter_wrapper_->GetStats(); + EXPECT_GE(stats.captured_frames, 10); + EXPECT_EQ(0, stats.dropped_frames); + VerifyAdaptedResolution(stats, kWidth, kHeight, kWidth, kHeight); + EXPECT_TRUE(stats.last_adapt_was_no_op); +} + +TEST_P(VideoAdapterTest, AdaptZeroInterval) { + OnOutputFormatRequest(kWidth, kHeight, absl::nullopt); + for (int i = 0; i < 40; ++i) + adapter_wrapper_->AdaptFrame(frame_source_->GetFrame()); + + // Verify no crash and that frames aren't dropped. + VideoAdapterWrapper::Stats stats = adapter_wrapper_->GetStats(); + EXPECT_GE(stats.captured_frames, 40); + EXPECT_EQ(0, stats.dropped_frames); + VerifyAdaptedResolution(stats, kWidth, kHeight, kWidth, kHeight); +} + +// Adapt the frame rate to be half of the capture rate at the beginning. Expect +// the number of dropped frames to be half of the number the captured frames. +TEST_P(VideoAdapterTest, AdaptFramerateToHalf) { + OnOutputFormatRequest(kWidth, kHeight, kDefaultFps / 2); + + // Capture 10 frames and verify that every other frame is dropped. The first + // frame should not be dropped. + adapter_wrapper_->AdaptFrame(frame_source_->GetFrame()); + EXPECT_GE(adapter_wrapper_->GetStats().captured_frames, 1); + EXPECT_EQ(0, adapter_wrapper_->GetStats().dropped_frames); + + adapter_wrapper_->AdaptFrame(frame_source_->GetFrame()); + EXPECT_GE(adapter_wrapper_->GetStats().captured_frames, 2); + EXPECT_EQ(1, adapter_wrapper_->GetStats().dropped_frames); + + adapter_wrapper_->AdaptFrame(frame_source_->GetFrame()); + EXPECT_GE(adapter_wrapper_->GetStats().captured_frames, 3); + EXPECT_EQ(1, adapter_wrapper_->GetStats().dropped_frames); + + adapter_wrapper_->AdaptFrame(frame_source_->GetFrame()); + EXPECT_GE(adapter_wrapper_->GetStats().captured_frames, 4); + EXPECT_EQ(2, adapter_wrapper_->GetStats().dropped_frames); + + adapter_wrapper_->AdaptFrame(frame_source_->GetFrame()); + EXPECT_GE(adapter_wrapper_->GetStats().captured_frames, 5); + EXPECT_EQ(2, adapter_wrapper_->GetStats().dropped_frames); + + adapter_wrapper_->AdaptFrame(frame_source_->GetFrame()); + EXPECT_GE(adapter_wrapper_->GetStats().captured_frames, 6); + EXPECT_EQ(3, adapter_wrapper_->GetStats().dropped_frames); + + adapter_wrapper_->AdaptFrame(frame_source_->GetFrame()); + EXPECT_GE(adapter_wrapper_->GetStats().captured_frames, 7); + EXPECT_EQ(3, adapter_wrapper_->GetStats().dropped_frames); + + adapter_wrapper_->AdaptFrame(frame_source_->GetFrame()); + EXPECT_GE(adapter_wrapper_->GetStats().captured_frames, 8); + EXPECT_EQ(4, adapter_wrapper_->GetStats().dropped_frames); + + adapter_wrapper_->AdaptFrame(frame_source_->GetFrame()); + EXPECT_GE(adapter_wrapper_->GetStats().captured_frames, 9); + EXPECT_EQ(4, adapter_wrapper_->GetStats().dropped_frames); + + adapter_wrapper_->AdaptFrame(frame_source_->GetFrame()); + EXPECT_GE(adapter_wrapper_->GetStats().captured_frames, 10); + EXPECT_EQ(5, adapter_wrapper_->GetStats().dropped_frames); +} + +// Adapt the frame rate to be two thirds of the capture rate at the beginning. +// Expect the number of dropped frames to be one thirds of the number the +// captured frames. +TEST_P(VideoAdapterTest, AdaptFramerateToTwoThirds) { + OnOutputFormatRequest(kWidth, kHeight, kDefaultFps * 2 / 3); + + // Capture 10 frames and verify that every third frame is dropped. The first + // frame should not be dropped. + adapter_wrapper_->AdaptFrame(frame_source_->GetFrame()); + EXPECT_GE(adapter_wrapper_->GetStats().captured_frames, 1); + EXPECT_EQ(0, adapter_wrapper_->GetStats().dropped_frames); + + adapter_wrapper_->AdaptFrame(frame_source_->GetFrame()); + EXPECT_GE(adapter_wrapper_->GetStats().captured_frames, 2); + EXPECT_EQ(0, adapter_wrapper_->GetStats().dropped_frames); + + adapter_wrapper_->AdaptFrame(frame_source_->GetFrame()); + EXPECT_GE(adapter_wrapper_->GetStats().captured_frames, 3); + EXPECT_EQ(1, adapter_wrapper_->GetStats().dropped_frames); + + adapter_wrapper_->AdaptFrame(frame_source_->GetFrame()); + EXPECT_GE(adapter_wrapper_->GetStats().captured_frames, 4); + EXPECT_EQ(1, adapter_wrapper_->GetStats().dropped_frames); + + adapter_wrapper_->AdaptFrame(frame_source_->GetFrame()); + EXPECT_GE(adapter_wrapper_->GetStats().captured_frames, 5); + EXPECT_EQ(1, adapter_wrapper_->GetStats().dropped_frames); + + adapter_wrapper_->AdaptFrame(frame_source_->GetFrame()); + EXPECT_GE(adapter_wrapper_->GetStats().captured_frames, 6); + EXPECT_EQ(2, adapter_wrapper_->GetStats().dropped_frames); + + adapter_wrapper_->AdaptFrame(frame_source_->GetFrame()); + EXPECT_GE(adapter_wrapper_->GetStats().captured_frames, 7); + EXPECT_EQ(2, adapter_wrapper_->GetStats().dropped_frames); + + adapter_wrapper_->AdaptFrame(frame_source_->GetFrame()); + EXPECT_GE(adapter_wrapper_->GetStats().captured_frames, 8); + EXPECT_EQ(2, adapter_wrapper_->GetStats().dropped_frames); + + adapter_wrapper_->AdaptFrame(frame_source_->GetFrame()); + EXPECT_GE(adapter_wrapper_->GetStats().captured_frames, 9); + EXPECT_EQ(3, adapter_wrapper_->GetStats().dropped_frames); + + adapter_wrapper_->AdaptFrame(frame_source_->GetFrame()); + EXPECT_GE(adapter_wrapper_->GetStats().captured_frames, 10); + EXPECT_EQ(3, adapter_wrapper_->GetStats().dropped_frames); +} + +// Request frame rate twice as high as captured frame rate. Expect no frame +// drop. +TEST_P(VideoAdapterTest, AdaptFramerateHighLimit) { + OnOutputFormatRequest(kWidth, kHeight, kDefaultFps * 2); + + for (int i = 0; i < 10; ++i) + adapter_wrapper_->AdaptFrame(frame_source_->GetFrame()); + + // Verify no frame drop. + EXPECT_EQ(0, adapter_wrapper_->GetStats().dropped_frames); +} + +// Adapt the frame rate to be half of the capture rate. No resolution limit set. +// Expect the number of dropped frames to be half of the number the captured +// frames. +TEST_P(VideoAdapterTest, AdaptFramerateToHalfWithNoPixelLimit) { + adapter_.OnOutputFormatRequest(absl::nullopt, absl::nullopt, kDefaultFps / 2); + + // Capture 10 frames and verify that every other frame is dropped. The first + // frame should not be dropped. + int expected_dropped_frames = 0; + for (int i = 0; i < 10; ++i) { + adapter_wrapper_->AdaptFrame(frame_source_->GetFrame()); + EXPECT_GE(adapter_wrapper_->GetStats().captured_frames, i + 1); + if (i % 2 == 1) + ++expected_dropped_frames; + EXPECT_EQ(expected_dropped_frames, + adapter_wrapper_->GetStats().dropped_frames); + VerifyAdaptedResolution(adapter_wrapper_->GetStats(), kWidth, kHeight, + kWidth, kHeight); + } +} + +// After the first timestamp, add a big offset to the timestamps. Expect that +// the adapter is conservative and resets to the new offset and does not drop +// any frame. +TEST_P(VideoAdapterTest, AdaptFramerateTimestampOffset) { + const int64_t capture_interval = VideoFormat::FpsToInterval(kDefaultFps); + OnOutputFormatRequest(640, 480, kDefaultFps); + + const int64_t first_timestamp = 0; + adapter_.AdaptFrameResolution(640, 480, first_timestamp, &cropped_width_, + &cropped_height_, &out_width_, &out_height_); + EXPECT_GT(out_width_, 0); + EXPECT_GT(out_height_, 0); + + const int64_t big_offset = -987654321LL * 1000; + const int64_t second_timestamp = big_offset; + adapter_.AdaptFrameResolution(640, 480, second_timestamp, &cropped_width_, + &cropped_height_, &out_width_, &out_height_); + EXPECT_GT(out_width_, 0); + EXPECT_GT(out_height_, 0); + + const int64_t third_timestamp = big_offset + capture_interval; + adapter_.AdaptFrameResolution(640, 480, third_timestamp, &cropped_width_, + &cropped_height_, &out_width_, &out_height_); + EXPECT_GT(out_width_, 0); + EXPECT_GT(out_height_, 0); +} + +// Request 30 fps and send 30 fps with jitter. Expect that no frame is dropped. +TEST_P(VideoAdapterTest, AdaptFramerateTimestampJitter) { + const int64_t capture_interval = VideoFormat::FpsToInterval(kDefaultFps); + OnOutputFormatRequest(640, 480, kDefaultFps); + + adapter_.AdaptFrameResolution(640, 480, capture_interval * 0 / 10, + &cropped_width_, &cropped_height_, &out_width_, + &out_height_); + EXPECT_GT(out_width_, 0); + EXPECT_GT(out_height_, 0); + + adapter_.AdaptFrameResolution(640, 480, capture_interval * 10 / 10 - 1, + &cropped_width_, &cropped_height_, &out_width_, + &out_height_); + EXPECT_GT(out_width_, 0); + EXPECT_GT(out_height_, 0); + + adapter_.AdaptFrameResolution(640, 480, capture_interval * 25 / 10, + &cropped_width_, &cropped_height_, &out_width_, + &out_height_); + EXPECT_GT(out_width_, 0); + EXPECT_GT(out_height_, 0); + + adapter_.AdaptFrameResolution(640, 480, capture_interval * 30 / 10, + &cropped_width_, &cropped_height_, &out_width_, + &out_height_); + EXPECT_GT(out_width_, 0); + EXPECT_GT(out_height_, 0); + + adapter_.AdaptFrameResolution(640, 480, capture_interval * 35 / 10, + &cropped_width_, &cropped_height_, &out_width_, + &out_height_); + EXPECT_GT(out_width_, 0); + EXPECT_GT(out_height_, 0); + + adapter_.AdaptFrameResolution(640, 480, capture_interval * 50 / 10, + &cropped_width_, &cropped_height_, &out_width_, + &out_height_); + EXPECT_GT(out_width_, 0); + EXPECT_GT(out_height_, 0); +} + +// Adapt the frame rate to be half of the capture rate after capturing no less +// than 10 frames. Expect no frame dropped before adaptation and frame dropped +// after adaptation. +TEST_P(VideoAdapterTest, AdaptFramerateOntheFly) { + OnOutputFormatRequest(kWidth, kHeight, kDefaultFps); + for (int i = 0; i < 10; ++i) + adapter_wrapper_->AdaptFrame(frame_source_->GetFrame()); + + // Verify no frame drop before adaptation. + EXPECT_EQ(0, adapter_wrapper_->GetStats().dropped_frames); + + // Adapt the frame rate. + OnOutputFormatRequest(kWidth, kHeight, kDefaultFps / 2); + for (int i = 0; i < 20; ++i) + adapter_wrapper_->AdaptFrame(frame_source_->GetFrame()); + + // Verify frame drop after adaptation. + EXPECT_GT(adapter_wrapper_->GetStats().dropped_frames, 0); +} + +// Do not adapt the frame rate or the resolution. Expect no frame drop, no +// cropping, and no resolution change. +TEST_P(VideoAdapterTest, AdaptFramerateRequestMax) { + adapter_.OnSinkWants(BuildSinkWants(absl::nullopt, + std::numeric_limits<int>::max(), + std::numeric_limits<int>::max())); + + for (int i = 0; i < 10; ++i) + adapter_wrapper_->AdaptFrame(frame_source_->GetFrame()); + + // Verify no frame drop and no resolution change. + VideoAdapterWrapper::Stats stats = adapter_wrapper_->GetStats(); + EXPECT_GE(stats.captured_frames, 10); + EXPECT_EQ(0, stats.dropped_frames); + VerifyAdaptedResolution(stats, kWidth, kHeight, kWidth, kHeight); + EXPECT_TRUE(stats.last_adapt_was_no_op); +} + +TEST_P(VideoAdapterTest, AdaptFramerateRequestZero) { + adapter_.OnSinkWants( + BuildSinkWants(absl::nullopt, std::numeric_limits<int>::max(), 0)); + for (int i = 0; i < 10; ++i) + adapter_wrapper_->AdaptFrame(frame_source_->GetFrame()); + + // Verify no crash and that frames aren't dropped. + VideoAdapterWrapper::Stats stats = adapter_wrapper_->GetStats(); + EXPECT_GE(stats.captured_frames, 10); + EXPECT_EQ(10, stats.dropped_frames); +} + +// Adapt the frame rate to be half of the capture rate at the beginning. Expect +// the number of dropped frames to be half of the number the captured frames. +TEST_P(VideoAdapterTest, AdaptFramerateRequestHalf) { + adapter_.OnSinkWants(BuildSinkWants( + absl::nullopt, std::numeric_limits<int>::max(), kDefaultFps / 2)); + for (int i = 0; i < 10; ++i) + adapter_wrapper_->AdaptFrame(frame_source_->GetFrame()); + + // Verify no crash and that frames aren't dropped. + VideoAdapterWrapper::Stats stats = adapter_wrapper_->GetStats(); + EXPECT_GE(stats.captured_frames, 10); + EXPECT_EQ(5, stats.dropped_frames); + VerifyAdaptedResolution(stats, kWidth, kHeight, kWidth, kHeight); +} + +// Set a very high output pixel resolution. Expect no cropping or resolution +// change. +TEST_P(VideoAdapterTest, AdaptFrameResolutionHighLimit) { + OnOutputFormatRequest(kWidth * 10, kHeight * 10, kDefaultFps); + EXPECT_TRUE(adapter_.AdaptFrameResolution(kWidth, kHeight, 0, &cropped_width_, + &cropped_height_, &out_width_, + &out_height_)); + EXPECT_EQ(kWidth, cropped_width_); + EXPECT_EQ(kHeight, cropped_height_); + EXPECT_EQ(kWidth, out_width_); + EXPECT_EQ(kHeight, out_height_); +} + +// Adapt the frame resolution to be the same as capture resolution. Expect no +// cropping or resolution change. +TEST_P(VideoAdapterTest, AdaptFrameResolutionIdentical) { + OnOutputFormatRequest(kWidth, kHeight, kDefaultFps); + EXPECT_TRUE(adapter_.AdaptFrameResolution(kWidth, kHeight, 0, &cropped_width_, + &cropped_height_, &out_width_, + &out_height_)); + EXPECT_EQ(kWidth, cropped_width_); + EXPECT_EQ(kHeight, cropped_height_); + EXPECT_EQ(kWidth, out_width_); + EXPECT_EQ(kHeight, out_height_); +} + +// Adapt the frame resolution to be a quarter of the capture resolution. Expect +// no cropping, but a resolution change. +TEST_P(VideoAdapterTest, AdaptFrameResolutionQuarter) { + OnOutputFormatRequest(kWidth / 2, kHeight / 2, kDefaultFps); + EXPECT_TRUE(adapter_.AdaptFrameResolution(kWidth, kHeight, 0, &cropped_width_, + &cropped_height_, &out_width_, + &out_height_)); + EXPECT_EQ(kWidth, cropped_width_); + EXPECT_EQ(kHeight, cropped_height_); + EXPECT_EQ(kWidth / 2, out_width_); + EXPECT_EQ(kHeight / 2, out_height_); +} + +// Adapt the pixel resolution to 0. Expect frame drop. +TEST_P(VideoAdapterTest, AdaptFrameResolutionDrop) { + OnOutputFormatRequest(kWidth * 0, kHeight * 0, kDefaultFps); + EXPECT_FALSE(adapter_.AdaptFrameResolution(kWidth, kHeight, 0, + &cropped_width_, &cropped_height_, + &out_width_, &out_height_)); +} + +// Adapt the frame resolution to be a quarter of the capture resolution at the +// beginning. Expect no cropping but a resolution change. +TEST_P(VideoAdapterTest, AdaptResolution) { + OnOutputFormatRequest(kWidth / 2, kHeight / 2, kDefaultFps); + for (int i = 0; i < 10; ++i) + adapter_wrapper_->AdaptFrame(frame_source_->GetFrame()); + + // Verify no frame drop, no cropping, and resolution change. + VideoAdapterWrapper::Stats stats = adapter_wrapper_->GetStats(); + EXPECT_EQ(0, stats.dropped_frames); + VerifyAdaptedResolution(stats, kWidth, kHeight, kWidth / 2, kHeight / 2); +} + +// Adapt the frame resolution to be a quarter of the capture resolution after +// capturing no less than 10 frames. Expect no resolution change before +// adaptation and resolution change after adaptation. +TEST_P(VideoAdapterTest, AdaptResolutionOnTheFly) { + OnOutputFormatRequest(kWidth, kHeight, kDefaultFps); + for (int i = 0; i < 10; ++i) + adapter_wrapper_->AdaptFrame(frame_source_->GetFrame()); + + // Verify no resolution change before adaptation. + VerifyAdaptedResolution(adapter_wrapper_->GetStats(), kWidth, kHeight, kWidth, + kHeight); + + // Adapt the frame resolution. + OnOutputFormatRequest(kWidth / 2, kHeight / 2, kDefaultFps); + for (int i = 0; i < 10; ++i) + adapter_wrapper_->AdaptFrame(frame_source_->GetFrame()); + + // Verify resolution change after adaptation. + VerifyAdaptedResolution(adapter_wrapper_->GetStats(), kWidth, kHeight, + kWidth / 2, kHeight / 2); +} + +// Drop all frames for resolution 0x0. +TEST_P(VideoAdapterTest, DropAllFrames) { + OnOutputFormatRequest(kWidth * 0, kHeight * 0, kDefaultFps); + for (int i = 0; i < 10; ++i) + adapter_wrapper_->AdaptFrame(frame_source_->GetFrame()); + + // Verify all frames are dropped. + VideoAdapterWrapper::Stats stats = adapter_wrapper_->GetStats(); + EXPECT_GE(stats.captured_frames, 10); + EXPECT_EQ(stats.captured_frames, stats.dropped_frames); +} + +TEST_P(VideoAdapterTest, TestOnOutputFormatRequest) { + EXPECT_TRUE(adapter_.AdaptFrameResolution(640, 400, 0, &cropped_width_, + &cropped_height_, &out_width_, + &out_height_)); + EXPECT_EQ(640, cropped_width_); + EXPECT_EQ(400, cropped_height_); + EXPECT_EQ(640, out_width_); + EXPECT_EQ(400, out_height_); + + // Format request 640x400. + OnOutputFormatRequest(640, 400, absl::nullopt); + EXPECT_TRUE(adapter_.AdaptFrameResolution(640, 400, 0, &cropped_width_, + &cropped_height_, &out_width_, + &out_height_)); + EXPECT_EQ(640, cropped_width_); + EXPECT_EQ(400, cropped_height_); + EXPECT_EQ(640, out_width_); + EXPECT_EQ(400, out_height_); + + // Request 1280x720, higher than input, but aspect 16:9. Expect cropping but + // no scaling. + OnOutputFormatRequest(1280, 720, absl::nullopt); + EXPECT_TRUE(adapter_.AdaptFrameResolution(640, 400, 0, &cropped_width_, + &cropped_height_, &out_width_, + &out_height_)); + EXPECT_EQ(640, cropped_width_); + EXPECT_EQ(360, cropped_height_); + EXPECT_EQ(640, out_width_); + EXPECT_EQ(360, out_height_); + + // Request 0x0. + OnOutputFormatRequest(0, 0, absl::nullopt); + EXPECT_FALSE(adapter_.AdaptFrameResolution(640, 400, 0, &cropped_width_, + &cropped_height_, &out_width_, + &out_height_)); + + // Request 320x200. Expect scaling, but no cropping. + OnOutputFormatRequest(320, 200, absl::nullopt); + EXPECT_TRUE(adapter_.AdaptFrameResolution(640, 400, 0, &cropped_width_, + &cropped_height_, &out_width_, + &out_height_)); + EXPECT_EQ(640, cropped_width_); + EXPECT_EQ(400, cropped_height_); + EXPECT_EQ(320, out_width_); + EXPECT_EQ(200, out_height_); + + // Request resolution close to 2/3 scale. Expect adapt down. Scaling to 2/3 + // is not optimized and not allowed, therefore 1/2 scaling will be used + // instead. + OnOutputFormatRequest(424, 265, absl::nullopt); + EXPECT_TRUE(adapter_.AdaptFrameResolution(640, 400, 0, &cropped_width_, + &cropped_height_, &out_width_, + &out_height_)); + EXPECT_EQ(640, cropped_width_); + EXPECT_EQ(400, cropped_height_); + EXPECT_EQ(320, out_width_); + EXPECT_EQ(200, out_height_); + + // Request resolution of 3 / 8. Expect adapt down. + OnOutputFormatRequest(640 * 3 / 8, 400 * 3 / 8, absl::nullopt); + EXPECT_TRUE(adapter_.AdaptFrameResolution(640, 400, 0, &cropped_width_, + &cropped_height_, &out_width_, + &out_height_)); + EXPECT_EQ(640, cropped_width_); + EXPECT_EQ(400, cropped_height_); + EXPECT_EQ(640 * 3 / 8, out_width_); + EXPECT_EQ(400 * 3 / 8, out_height_); + + // Switch back up. Expect adapt. + OnOutputFormatRequest(320, 200, absl::nullopt); + EXPECT_TRUE(adapter_.AdaptFrameResolution(640, 400, 0, &cropped_width_, + &cropped_height_, &out_width_, + &out_height_)); + EXPECT_EQ(640, cropped_width_); + EXPECT_EQ(400, cropped_height_); + EXPECT_EQ(320, out_width_); + EXPECT_EQ(200, out_height_); + + // Format request 480x300. + OnOutputFormatRequest(480, 300, absl::nullopt); + EXPECT_TRUE(adapter_.AdaptFrameResolution(640, 400, 0, &cropped_width_, + &cropped_height_, &out_width_, + &out_height_)); + EXPECT_EQ(640, cropped_width_); + EXPECT_EQ(400, cropped_height_); + EXPECT_EQ(480, out_width_); + EXPECT_EQ(300, out_height_); +} + +TEST_P(VideoAdapterTest, TestViewRequestPlusCameraSwitch) { + // Start at HD. + EXPECT_TRUE(adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, + &cropped_height_, &out_width_, + &out_height_)); + EXPECT_EQ(1280, cropped_width_); + EXPECT_EQ(720, cropped_height_); + EXPECT_EQ(1280, out_width_); + EXPECT_EQ(720, out_height_); + + // Format request for VGA. + OnOutputFormatRequest(640, 360, absl::nullopt); + EXPECT_TRUE(adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, + &cropped_height_, &out_width_, + &out_height_)); + EXPECT_EQ(1280, cropped_width_); + EXPECT_EQ(720, cropped_height_); + EXPECT_EQ(640, out_width_); + EXPECT_EQ(360, out_height_); + + // Now, the camera reopens at VGA. + // Both the frame and the output format should be 640x360. + EXPECT_TRUE(adapter_.AdaptFrameResolution(640, 360, 0, &cropped_width_, + &cropped_height_, &out_width_, + &out_height_)); + EXPECT_EQ(640, cropped_width_); + EXPECT_EQ(360, cropped_height_); + EXPECT_EQ(640, out_width_); + EXPECT_EQ(360, out_height_); + + // And another view request comes in for 640x360, which should have no + // real impact. + OnOutputFormatRequest(640, 360, absl::nullopt); + EXPECT_TRUE(adapter_.AdaptFrameResolution(640, 360, 0, &cropped_width_, + &cropped_height_, &out_width_, + &out_height_)); + EXPECT_EQ(640, cropped_width_); + EXPECT_EQ(360, cropped_height_); + EXPECT_EQ(640, out_width_); + EXPECT_EQ(360, out_height_); +} + +TEST_P(VideoAdapterTest, TestVgaWidth) { + // Requested output format is 640x360. + OnOutputFormatRequest(640, 360, absl::nullopt); + + EXPECT_TRUE(adapter_.AdaptFrameResolution(640, 480, 0, &cropped_width_, + &cropped_height_, &out_width_, + &out_height_)); + // Expect cropping. + EXPECT_EQ(640, cropped_width_); + EXPECT_EQ(360, cropped_height_); + EXPECT_EQ(640, out_width_); + EXPECT_EQ(360, out_height_); + + // But if frames come in at 640x360, we shouldn't adapt them down. + EXPECT_TRUE(adapter_.AdaptFrameResolution(640, 360, 0, &cropped_width_, + &cropped_height_, &out_width_, + &out_height_)); + EXPECT_EQ(640, cropped_width_); + EXPECT_EQ(360, cropped_height_); + EXPECT_EQ(640, out_width_); + EXPECT_EQ(360, out_height_); + + EXPECT_TRUE(adapter_.AdaptFrameResolution(640, 480, 0, &cropped_width_, + &cropped_height_, &out_width_, + &out_height_)); + EXPECT_EQ(640, cropped_width_); + EXPECT_EQ(360, cropped_height_); + EXPECT_EQ(640, out_width_); + EXPECT_EQ(360, out_height_); +} + +TEST_P(VideoAdapterTest, TestOnResolutionRequestInSmallSteps) { + EXPECT_TRUE(adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, + &cropped_height_, &out_width_, + &out_height_)); + EXPECT_EQ(1280, cropped_width_); + EXPECT_EQ(720, cropped_height_); + EXPECT_EQ(1280, out_width_); + EXPECT_EQ(720, out_height_); + + // Adapt down one step. + adapter_.OnSinkWants(BuildSinkWants(absl::nullopt, 1280 * 720 - 1, + std::numeric_limits<int>::max())); + EXPECT_TRUE(adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, + &cropped_height_, &out_width_, + &out_height_)); + EXPECT_EQ(1280, cropped_width_); + EXPECT_EQ(720, cropped_height_); + EXPECT_EQ(960, out_width_); + EXPECT_EQ(540, out_height_); + + // Adapt down one step more. + adapter_.OnSinkWants(BuildSinkWants(absl::nullopt, 960 * 540 - 1, + std::numeric_limits<int>::max())); + EXPECT_TRUE(adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, + &cropped_height_, &out_width_, + &out_height_)); + EXPECT_EQ(1280, cropped_width_); + EXPECT_EQ(720, cropped_height_); + EXPECT_EQ(640, out_width_); + EXPECT_EQ(360, out_height_); + + // Adapt down one step more. + adapter_.OnSinkWants(BuildSinkWants(absl::nullopt, 640 * 360 - 1, + std::numeric_limits<int>::max())); + EXPECT_TRUE(adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, + &cropped_height_, &out_width_, + &out_height_)); + EXPECT_EQ(1280, cropped_width_); + EXPECT_EQ(720, cropped_height_); + EXPECT_EQ(480, out_width_); + EXPECT_EQ(270, out_height_); + + // Adapt up one step. + adapter_.OnSinkWants( + BuildSinkWants(640 * 360, 960 * 540, std::numeric_limits<int>::max())); + EXPECT_TRUE(adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, + &cropped_height_, &out_width_, + &out_height_)); + EXPECT_EQ(1280, cropped_width_); + EXPECT_EQ(720, cropped_height_); + EXPECT_EQ(640, out_width_); + EXPECT_EQ(360, out_height_); + + // Adapt up one step more. + adapter_.OnSinkWants( + BuildSinkWants(960 * 540, 1280 * 720, std::numeric_limits<int>::max())); + EXPECT_TRUE(adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, + &cropped_height_, &out_width_, + &out_height_)); + EXPECT_EQ(1280, cropped_width_); + EXPECT_EQ(720, cropped_height_); + EXPECT_EQ(960, out_width_); + EXPECT_EQ(540, out_height_); + + // Adapt up one step more. + adapter_.OnSinkWants( + BuildSinkWants(1280 * 720, 1920 * 1080, std::numeric_limits<int>::max())); + EXPECT_TRUE(adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, + &cropped_height_, &out_width_, + &out_height_)); + EXPECT_EQ(1280, cropped_width_); + EXPECT_EQ(720, cropped_height_); + EXPECT_EQ(1280, out_width_); + EXPECT_EQ(720, out_height_); +} + +TEST_P(VideoAdapterTest, TestOnResolutionRequestMaxZero) { + EXPECT_TRUE(adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, + &cropped_height_, &out_width_, + &out_height_)); + EXPECT_EQ(1280, cropped_width_); + EXPECT_EQ(720, cropped_height_); + EXPECT_EQ(1280, out_width_); + EXPECT_EQ(720, out_height_); + + adapter_.OnSinkWants( + BuildSinkWants(absl::nullopt, 0, std::numeric_limits<int>::max())); + EXPECT_FALSE(adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, + &cropped_height_, &out_width_, + &out_height_)); +} + +TEST_P(VideoAdapterTest, TestOnResolutionRequestInLargeSteps) { + // Large step down. + adapter_.OnSinkWants(BuildSinkWants(absl::nullopt, 640 * 360 - 1, + std::numeric_limits<int>::max())); + EXPECT_TRUE(adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, + &cropped_height_, &out_width_, + &out_height_)); + EXPECT_EQ(1280, cropped_width_); + EXPECT_EQ(720, cropped_height_); + EXPECT_EQ(480, out_width_); + EXPECT_EQ(270, out_height_); + + // Large step up. + adapter_.OnSinkWants( + BuildSinkWants(1280 * 720, 1920 * 1080, std::numeric_limits<int>::max())); + EXPECT_TRUE(adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, + &cropped_height_, &out_width_, + &out_height_)); + EXPECT_EQ(1280, cropped_width_); + EXPECT_EQ(720, cropped_height_); + EXPECT_EQ(1280, out_width_); + EXPECT_EQ(720, out_height_); +} + +TEST_P(VideoAdapterTest, TestOnOutputFormatRequestCapsMaxResolution) { + adapter_.OnSinkWants(BuildSinkWants(absl::nullopt, 640 * 360 - 1, + std::numeric_limits<int>::max())); + EXPECT_TRUE(adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, + &cropped_height_, &out_width_, + &out_height_)); + EXPECT_EQ(1280, cropped_width_); + EXPECT_EQ(720, cropped_height_); + EXPECT_EQ(480, out_width_); + EXPECT_EQ(270, out_height_); + + OnOutputFormatRequest(640, 360, absl::nullopt); + EXPECT_TRUE(adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, + &cropped_height_, &out_width_, + &out_height_)); + EXPECT_EQ(1280, cropped_width_); + EXPECT_EQ(720, cropped_height_); + EXPECT_EQ(480, out_width_); + EXPECT_EQ(270, out_height_); + + adapter_.OnSinkWants(BuildSinkWants(absl::nullopt, 960 * 720, + std::numeric_limits<int>::max())); + EXPECT_TRUE(adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, + &cropped_height_, &out_width_, + &out_height_)); + EXPECT_EQ(1280, cropped_width_); + EXPECT_EQ(720, cropped_height_); + EXPECT_EQ(640, out_width_); + EXPECT_EQ(360, out_height_); +} + +TEST_P(VideoAdapterTest, TestOnResolutionRequestReset) { + EXPECT_TRUE(adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, + &cropped_height_, &out_width_, + &out_height_)); + EXPECT_EQ(1280, cropped_width_); + EXPECT_EQ(720, cropped_height_); + EXPECT_EQ(1280, out_width_); + EXPECT_EQ(720, out_height_); + + adapter_.OnSinkWants(BuildSinkWants(absl::nullopt, 640 * 360 - 1, + std::numeric_limits<int>::max())); + EXPECT_TRUE(adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, + &cropped_height_, &out_width_, + &out_height_)); + EXPECT_EQ(1280, cropped_width_); + EXPECT_EQ(720, cropped_height_); + EXPECT_EQ(480, out_width_); + EXPECT_EQ(270, out_height_); + + adapter_.OnSinkWants(BuildSinkWants(absl::nullopt, + std::numeric_limits<int>::max(), + std::numeric_limits<int>::max())); + EXPECT_TRUE(adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, + &cropped_height_, &out_width_, + &out_height_)); + EXPECT_EQ(1280, cropped_width_); + EXPECT_EQ(720, cropped_height_); + EXPECT_EQ(1280, out_width_); + EXPECT_EQ(720, out_height_); +} + +TEST_P(VideoAdapterTest, TestOnOutputFormatRequestResolutionReset) { + EXPECT_TRUE(adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, + &cropped_height_, &out_width_, + &out_height_)); + EXPECT_EQ(1280, cropped_width_); + EXPECT_EQ(720, cropped_height_); + EXPECT_EQ(1280, out_width_); + EXPECT_EQ(720, out_height_); + + adapter_.OnOutputFormatRequest(absl::nullopt, 640 * 360 - 1, absl::nullopt); + EXPECT_TRUE(adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, + &cropped_height_, &out_width_, + &out_height_)); + EXPECT_EQ(1280, cropped_width_); + EXPECT_EQ(720, cropped_height_); + EXPECT_EQ(480, out_width_); + EXPECT_EQ(270, out_height_); + + adapter_.OnOutputFormatRequest(absl::nullopt, absl::nullopt, absl::nullopt); + EXPECT_TRUE(adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, + &cropped_height_, &out_width_, + &out_height_)); + EXPECT_EQ(1280, cropped_width_); + EXPECT_EQ(720, cropped_height_); + EXPECT_EQ(1280, out_width_); + EXPECT_EQ(720, out_height_); +} + +TEST_P(VideoAdapterTest, TestOnOutputFormatRequestFpsReset) { + OnOutputFormatRequest(kWidth, kHeight, kDefaultFps / 2); + for (int i = 0; i < 10; ++i) + adapter_wrapper_->AdaptFrame(frame_source_->GetFrame()); + + // Verify frame drop. + const int dropped_frames = adapter_wrapper_->GetStats().dropped_frames; + EXPECT_GT(dropped_frames, 0); + + // Reset frame rate. + OnOutputFormatRequest(kWidth, kHeight, absl::nullopt); + for (int i = 0; i < 20; ++i) + adapter_wrapper_->AdaptFrame(frame_source_->GetFrame()); + + // Verify no frame drop after reset. + EXPECT_EQ(dropped_frames, adapter_wrapper_->GetStats().dropped_frames); +} + +TEST_P(VideoAdapterTest, RequestAspectRatio) { + // Request aspect ratio 320/180 (16:9), smaller than input, but no resolution + // limit. Expect cropping but no scaling. + adapter_.OnOutputFormatRequest(std::make_pair(320, 180), absl::nullopt, + absl::nullopt); + EXPECT_TRUE(adapter_.AdaptFrameResolution(640, 400, 0, &cropped_width_, + &cropped_height_, &out_width_, + &out_height_)); + EXPECT_EQ(640, cropped_width_); + EXPECT_EQ(360, cropped_height_); + EXPECT_EQ(640, out_width_); + EXPECT_EQ(360, out_height_); +} + +TEST_P(VideoAdapterTest, RequestAspectRatioWithDifferentOrientation) { + // Request 720x1280, higher than input, but aspect 16:9. Orientation should + // not matter, expect cropping but no scaling. + OnOutputFormatRequest(720, 1280, absl::nullopt); + EXPECT_TRUE(adapter_.AdaptFrameResolution(640, 400, 0, &cropped_width_, + &cropped_height_, &out_width_, + &out_height_)); + EXPECT_EQ(640, cropped_width_); + EXPECT_EQ(360, cropped_height_); + EXPECT_EQ(640, out_width_); + EXPECT_EQ(360, out_height_); +} + +TEST_P(VideoAdapterTest, InvalidAspectRatioIgnored) { + // Request aspect ratio 320/0. Expect no cropping. + adapter_.OnOutputFormatRequest(std::make_pair(320, 0), absl::nullopt, + absl::nullopt); + EXPECT_TRUE(adapter_.AdaptFrameResolution(640, 400, 0, &cropped_width_, + &cropped_height_, &out_width_, + &out_height_)); + EXPECT_EQ(640, cropped_width_); + EXPECT_EQ(400, cropped_height_); + EXPECT_EQ(640, out_width_); + EXPECT_EQ(400, out_height_); +} + +TEST_P(VideoAdapterTest, TestCroppingWithResolutionRequest) { + // Ask for 640x360 (16:9 aspect). + OnOutputFormatRequest(640, 360, absl::nullopt); + // Send 640x480 (4:3 aspect). + EXPECT_TRUE(adapter_.AdaptFrameResolution(640, 480, 0, &cropped_width_, + &cropped_height_, &out_width_, + &out_height_)); + // Expect cropping to 16:9 format and no scaling. + EXPECT_EQ(640, cropped_width_); + EXPECT_EQ(360, cropped_height_); + EXPECT_EQ(640, out_width_); + EXPECT_EQ(360, out_height_); + + // Adapt down one step. + adapter_.OnSinkWants(BuildSinkWants(absl::nullopt, 640 * 360 - 1, + std::numeric_limits<int>::max())); + // Expect cropping to 16:9 format and 3/4 scaling. + EXPECT_TRUE(adapter_.AdaptFrameResolution(640, 480, 0, &cropped_width_, + &cropped_height_, &out_width_, + &out_height_)); + EXPECT_EQ(640, cropped_width_); + EXPECT_EQ(360, cropped_height_); + EXPECT_EQ(480, out_width_); + EXPECT_EQ(270, out_height_); + + // Adapt down one step more. + adapter_.OnSinkWants(BuildSinkWants(absl::nullopt, 480 * 270 - 1, + std::numeric_limits<int>::max())); + // Expect cropping to 16:9 format and 1/2 scaling. + EXPECT_TRUE(adapter_.AdaptFrameResolution(640, 480, 0, &cropped_width_, + &cropped_height_, &out_width_, + &out_height_)); + EXPECT_EQ(640, cropped_width_); + EXPECT_EQ(360, cropped_height_); + EXPECT_EQ(320, out_width_); + EXPECT_EQ(180, out_height_); + + // Adapt up one step. + adapter_.OnSinkWants( + BuildSinkWants(480 * 270, 640 * 360, std::numeric_limits<int>::max())); + // Expect cropping to 16:9 format and 3/4 scaling. + EXPECT_TRUE(adapter_.AdaptFrameResolution(640, 480, 0, &cropped_width_, + &cropped_height_, &out_width_, + &out_height_)); + EXPECT_EQ(640, cropped_width_); + EXPECT_EQ(360, cropped_height_); + EXPECT_EQ(480, out_width_); + EXPECT_EQ(270, out_height_); + + // Adapt up one step more. + adapter_.OnSinkWants( + BuildSinkWants(640 * 360, 960 * 540, std::numeric_limits<int>::max())); + // Expect cropping to 16:9 format and no scaling. + EXPECT_TRUE(adapter_.AdaptFrameResolution(640, 480, 0, &cropped_width_, + &cropped_height_, &out_width_, + &out_height_)); + EXPECT_EQ(640, cropped_width_); + EXPECT_EQ(360, cropped_height_); + EXPECT_EQ(640, out_width_); + EXPECT_EQ(360, out_height_); + + // Try to adapt up one step more. + adapter_.OnSinkWants( + BuildSinkWants(960 * 540, 1280 * 720, std::numeric_limits<int>::max())); + // Expect cropping to 16:9 format and no scaling. + EXPECT_TRUE(adapter_.AdaptFrameResolution(640, 480, 0, &cropped_width_, + &cropped_height_, &out_width_, + &out_height_)); + EXPECT_EQ(640, cropped_width_); + EXPECT_EQ(360, cropped_height_); + EXPECT_EQ(640, out_width_); + EXPECT_EQ(360, out_height_); +} + +TEST_P(VideoAdapterTest, TestCroppingOddResolution) { + // Ask for 640x360 (16:9 aspect), with 3/16 scaling. + OnOutputFormatRequest(640, 360, absl::nullopt); + adapter_.OnSinkWants(BuildSinkWants(absl::nullopt, + 640 * 360 * 3 / 16 * 3 / 16, + std::numeric_limits<int>::max())); + + // Send 640x480 (4:3 aspect). + EXPECT_TRUE(adapter_.AdaptFrameResolution(640, 480, 0, &cropped_width_, + &cropped_height_, &out_width_, + &out_height_)); + + // Instead of getting the exact aspect ratio with cropped resolution 640x360, + // the resolution should be adjusted to get a perfect scale factor instead. + EXPECT_EQ(640, cropped_width_); + EXPECT_EQ(368, cropped_height_); + EXPECT_EQ(120, out_width_); + EXPECT_EQ(69, out_height_); +} + +TEST_P(VideoAdapterTest, TestAdaptToVerySmallResolution) { + // Ask for 1920x1080 (16:9 aspect), with 1/16 scaling. + const int w = 1920; + const int h = 1080; + OnOutputFormatRequest(w, h, absl::nullopt); + adapter_.OnSinkWants(BuildSinkWants(absl::nullopt, w * h * 1 / 16 * 1 / 16, + std::numeric_limits<int>::max())); + + // Send 1920x1080 (16:9 aspect). + EXPECT_TRUE(adapter_.AdaptFrameResolution( + w, h, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_)); + + // Instead of getting the exact aspect ratio with cropped resolution 1920x1080 + // the resolution should be adjusted to get a perfect scale factor instead. + EXPECT_EQ(1920, cropped_width_); + EXPECT_EQ(1072, cropped_height_); + EXPECT_EQ(120, out_width_); + EXPECT_EQ(67, out_height_); + + // Adapt back up one step to 3/32. + adapter_.OnSinkWants(BuildSinkWants(w * h * 3 / 32 * 3 / 32, + w * h * 1 / 8 * 1 / 8, + std::numeric_limits<int>::max())); + + // Send 1920x1080 (16:9 aspect). + EXPECT_TRUE(adapter_.AdaptFrameResolution( + w, h, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_)); + + EXPECT_EQ(180, out_width_); + EXPECT_EQ(99, out_height_); +} + +TEST_P(VideoAdapterTest, AdaptFrameResolutionDropWithResolutionRequest) { + OnOutputFormatRequest(0, 0, kDefaultFps); + EXPECT_FALSE(adapter_.AdaptFrameResolution(kWidth, kHeight, 0, + &cropped_width_, &cropped_height_, + &out_width_, &out_height_)); + + adapter_.OnSinkWants(BuildSinkWants(960 * 540, + std::numeric_limits<int>::max(), + std::numeric_limits<int>::max())); + + // Still expect all frames to be dropped + EXPECT_FALSE(adapter_.AdaptFrameResolution(kWidth, kHeight, 0, + &cropped_width_, &cropped_height_, + &out_width_, &out_height_)); + + adapter_.OnSinkWants(BuildSinkWants(absl::nullopt, 640 * 480 - 1, + std::numeric_limits<int>::max())); + + // Still expect all frames to be dropped + EXPECT_FALSE(adapter_.AdaptFrameResolution(kWidth, kHeight, 0, + &cropped_width_, &cropped_height_, + &out_width_, &out_height_)); +} + +// Test that we will adapt to max given a target pixel count close to max. +TEST_P(VideoAdapterTest, TestAdaptToMax) { + OnOutputFormatRequest(640, 360, kDefaultFps); + adapter_.OnSinkWants(BuildSinkWants(640 * 360 - 1 /* target */, + std::numeric_limits<int>::max(), + std::numeric_limits<int>::max())); + + EXPECT_TRUE(adapter_.AdaptFrameResolution(640, 360, 0, &cropped_width_, + &cropped_height_, &out_width_, + &out_height_)); + EXPECT_EQ(640, out_width_); + EXPECT_EQ(360, out_height_); +} + +// Test adjusting to 16:9 in landscape, and 9:16 in portrait. +TEST(VideoAdapterTestMultipleOrientation, TestNormal) { + VideoAdapter video_adapter; + video_adapter.OnOutputFormatRequest(std::make_pair(640, 360), 640 * 360, + std::make_pair(360, 640), 360 * 640, 30); + + int cropped_width; + int cropped_height; + int out_width; + int out_height; + EXPECT_TRUE(video_adapter.AdaptFrameResolution( + /* in_width= */ 640, /* in_height= */ 480, /* in_timestamp_ns= */ 0, + &cropped_width, &cropped_height, &out_width, &out_height)); + EXPECT_EQ(640, cropped_width); + EXPECT_EQ(360, cropped_height); + EXPECT_EQ(640, out_width); + EXPECT_EQ(360, out_height); + + EXPECT_TRUE(video_adapter.AdaptFrameResolution( + /* in_width= */ 480, /* in_height= */ 640, + /* in_timestamp_ns= */ rtc::kNumNanosecsPerSec / 30, &cropped_width, + &cropped_height, &out_width, &out_height)); + EXPECT_EQ(360, cropped_width); + EXPECT_EQ(640, cropped_height); + EXPECT_EQ(360, out_width); + EXPECT_EQ(640, out_height); +} + +// Force output to be 9:16, even for landscape input. +TEST(VideoAdapterTestMultipleOrientation, TestForcePortrait) { + VideoAdapter video_adapter; + video_adapter.OnOutputFormatRequest(std::make_pair(360, 640), 640 * 360, + std::make_pair(360, 640), 360 * 640, 30); + + int cropped_width; + int cropped_height; + int out_width; + int out_height; + EXPECT_TRUE(video_adapter.AdaptFrameResolution( + /* in_width= */ 640, /* in_height= */ 480, /* in_timestamp_ns= */ 0, + &cropped_width, &cropped_height, &out_width, &out_height)); + EXPECT_EQ(270, cropped_width); + EXPECT_EQ(480, cropped_height); + EXPECT_EQ(270, out_width); + EXPECT_EQ(480, out_height); + + EXPECT_TRUE(video_adapter.AdaptFrameResolution( + /* in_width= */ 480, /* in_height= */ 640, + /* in_timestamp_ns= */ rtc::kNumNanosecsPerSec / 30, &cropped_width, + &cropped_height, &out_width, &out_height)); + EXPECT_EQ(360, cropped_width); + EXPECT_EQ(640, cropped_height); + EXPECT_EQ(360, out_width); + EXPECT_EQ(640, out_height); +} + +TEST_P(VideoAdapterTest, AdaptResolutionInSteps) { + const int kWidth = 1280; + const int kHeight = 720; + OnOutputFormatRequest(kWidth, kHeight, absl::nullopt); // 16:9 aspect. + + // Scale factors: 3/4, 2/3, 3/4, 2/3, ... + // Scale : 3/4, 1/2, 3/8, 1/4, 3/16, 1/8. + const int kExpectedWidths[] = {960, 640, 480, 320, 240, 160}; + const int kExpectedHeights[] = {540, 360, 270, 180, 135, 90}; + + int request_width = kWidth; + int request_height = kHeight; + + for (size_t i = 0; i < arraysize(kExpectedWidths); ++i) { + // Adapt down one step. + adapter_.OnSinkWants(BuildSinkWants(absl::nullopt, + request_width * request_height - 1, + std::numeric_limits<int>::max())); + EXPECT_TRUE(adapter_.AdaptFrameResolution(kWidth, kHeight, 0, + &cropped_width_, &cropped_height_, + &out_width_, &out_height_)); + EXPECT_EQ(kExpectedWidths[i], out_width_); + EXPECT_EQ(kExpectedHeights[i], out_height_); + request_width = out_width_; + request_height = out_height_; + } +} + +// Scale factors are 3/4, 2/3, 3/4, 2/3, ... (see test above). +// In VideoAdapterTestVariableStartScale, first scale factor depends on +// resolution. May start with: +// - 2/3 (if width/height multiple of 3) or +// - 2/3, 2/3 (if width/height multiple of 9). +TEST_P(VideoAdapterTestVariableStartScale, AdaptResolutionInStepsFirst3_4) { + const int kWidth = 1280; + const int kHeight = 720; + OnOutputFormatRequest(kWidth, kHeight, absl::nullopt); // 16:9 aspect. + + // Scale factors: 3/4, 2/3, 3/4, 2/3, ... + // Scale : 3/4, 1/2, 3/8, 1/4, 3/16, 1/8. + const int kExpectedWidths[] = {960, 640, 480, 320, 240, 160}; + const int kExpectedHeights[] = {540, 360, 270, 180, 135, 90}; + + int request_width = kWidth; + int request_height = kHeight; + + for (size_t i = 0; i < arraysize(kExpectedWidths); ++i) { + // Adapt down one step. + adapter_.OnSinkWants(BuildSinkWants(absl::nullopt, + request_width * request_height - 1, + std::numeric_limits<int>::max())); + EXPECT_TRUE(adapter_.AdaptFrameResolution(kWidth, kHeight, 0, + &cropped_width_, &cropped_height_, + &out_width_, &out_height_)); + EXPECT_EQ(kExpectedWidths[i], out_width_); + EXPECT_EQ(kExpectedHeights[i], out_height_); + request_width = out_width_; + request_height = out_height_; + } +} + +TEST_P(VideoAdapterTestVariableStartScale, AdaptResolutionInStepsFirst2_3) { + const int kWidth = 1920; + const int kHeight = 1080; + OnOutputFormatRequest(kWidth, kHeight, absl::nullopt); // 16:9 aspect. + + // Scale factors: 2/3, 3/4, 2/3, 3/4, ... + // Scale: 2/3, 1/2, 1/3, 1/4, 1/6, 1/8, 1/12. + const int kExpectedWidths[] = {1280, 960, 640, 480, 320, 240, 160}; + const int kExpectedHeights[] = {720, 540, 360, 270, 180, 135, 90}; + + int request_width = kWidth; + int request_height = kHeight; + + for (size_t i = 0; i < arraysize(kExpectedWidths); ++i) { + // Adapt down one step. + adapter_.OnSinkWants(BuildSinkWants(absl::nullopt, + request_width * request_height - 1, + std::numeric_limits<int>::max())); + EXPECT_TRUE(adapter_.AdaptFrameResolution(kWidth, kHeight, 0, + &cropped_width_, &cropped_height_, + &out_width_, &out_height_)); + EXPECT_EQ(kExpectedWidths[i], out_width_); + EXPECT_EQ(kExpectedHeights[i], out_height_); + request_width = out_width_; + request_height = out_height_; + } +} + +TEST_P(VideoAdapterTestVariableStartScale, AdaptResolutionInStepsFirst2x2_3) { + const int kWidth = 1440; + const int kHeight = 1080; + OnOutputFormatRequest(kWidth, kHeight, absl::nullopt); // 4:3 aspect. + + // Scale factors: 2/3, 2/3, 3/4, 2/3, 3/4, ... + // Scale : 2/3, 4/9, 1/3, 2/9, 1/6, 1/9, 1/12, 1/18, 1/24, 1/36. + const int kExpectedWidths[] = {960, 640, 480, 320, 240, 160, 120, 80, 60, 40}; + const int kExpectedHeights[] = {720, 480, 360, 240, 180, 120, 90, 60, 45, 30}; + + int request_width = kWidth; + int request_height = kHeight; + + for (size_t i = 0; i < arraysize(kExpectedWidths); ++i) { + // Adapt down one step. + adapter_.OnSinkWants(BuildSinkWants(absl::nullopt, + request_width * request_height - 1, + std::numeric_limits<int>::max())); + EXPECT_TRUE(adapter_.AdaptFrameResolution(kWidth, kHeight, 0, + &cropped_width_, &cropped_height_, + &out_width_, &out_height_)); + EXPECT_EQ(kExpectedWidths[i], out_width_); + EXPECT_EQ(kExpectedHeights[i], out_height_); + request_width = out_width_; + request_height = out_height_; + } +} + +TEST_P(VideoAdapterTest, AdaptResolutionWithSinkAlignment) { + constexpr int kSourceWidth = 1280; + constexpr int kSourceHeight = 720; + constexpr int kSourceFramerate = 30; + constexpr int kRequestedWidth = 480; + constexpr int kRequestedHeight = 270; + constexpr int kRequestedFramerate = 30; + + OnOutputFormatRequest(kRequestedWidth, kRequestedHeight, kRequestedFramerate); + + int frame_num = 1; + for (const int sink_alignment : {2, 3, 4, 5}) { + adapter_.OnSinkWants( + BuildSinkWants(absl::nullopt, std::numeric_limits<int>::max(), + std::numeric_limits<int>::max(), sink_alignment)); + EXPECT_TRUE(adapter_.AdaptFrameResolution( + kSourceWidth, kSourceHeight, + frame_num * rtc::kNumNanosecsPerSec / kSourceFramerate, &cropped_width_, + &cropped_height_, &out_width_, &out_height_)); + EXPECT_EQ(out_width_ % sink_alignment, 0); + EXPECT_EQ(out_height_ % sink_alignment, 0); + + ++frame_num; + } +} + +class VideoAdapterWithSourceAlignmentTest : public VideoAdapterTest { + protected: + static constexpr int kSourceResolutionAlignment = 7; + + VideoAdapterWithSourceAlignmentTest() + : VideoAdapterTest(/*field_trials=*/"", kSourceResolutionAlignment) {} +}; + +TEST_P(VideoAdapterWithSourceAlignmentTest, AdaptResolution) { + constexpr int kSourceWidth = 1280; + constexpr int kSourceHeight = 720; + constexpr int kRequestedWidth = 480; + constexpr int kRequestedHeight = 270; + constexpr int kRequestedFramerate = 30; + + OnOutputFormatRequest(kRequestedWidth, kRequestedHeight, kRequestedFramerate); + + EXPECT_TRUE(adapter_.AdaptFrameResolution( + kSourceWidth, kSourceHeight, /*in_timestamp_ns=*/0, &cropped_width_, + &cropped_height_, &out_width_, &out_height_)); + EXPECT_EQ(out_width_ % kSourceResolutionAlignment, 0); + EXPECT_EQ(out_height_ % kSourceResolutionAlignment, 0); +} + +TEST_P(VideoAdapterWithSourceAlignmentTest, AdaptResolutionWithSinkAlignment) { + constexpr int kSourceWidth = 1280; + constexpr int kSourceHeight = 720; + // 7 and 8 neither divide 480 nor 270. + constexpr int kRequestedWidth = 480; + constexpr int kRequestedHeight = 270; + constexpr int kRequestedFramerate = 30; + constexpr int kSinkResolutionAlignment = 8; + + OnOutputFormatRequest(kRequestedWidth, kRequestedHeight, kRequestedFramerate); + + adapter_.OnSinkWants(BuildSinkWants( + absl::nullopt, std::numeric_limits<int>::max(), + std::numeric_limits<int>::max(), kSinkResolutionAlignment)); + EXPECT_TRUE(adapter_.AdaptFrameResolution( + kSourceWidth, kSourceHeight, /*in_timestamp_ns=*/0, &cropped_width_, + &cropped_height_, &out_width_, &out_height_)); + EXPECT_EQ(out_width_ % kSourceResolutionAlignment, 0); + EXPECT_EQ(out_height_ % kSourceResolutionAlignment, 0); + EXPECT_EQ(out_width_ % kSinkResolutionAlignment, 0); + EXPECT_EQ(out_height_ % kSinkResolutionAlignment, 0); +} + +INSTANTIATE_TEST_SUITE_P(OnOutputFormatRequests, + VideoAdapterWithSourceAlignmentTest, + ::testing::Values(true, false)); + +} // namespace cricket diff --git a/media/base/video_broadcaster.cc b/media/base/video_broadcaster.cc new file mode 100644 index 0000000000..700478d4e1 --- /dev/null +++ b/media/base/video_broadcaster.cc @@ -0,0 +1,152 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "media/base/video_broadcaster.h" + +#include <vector> + +#include "absl/types/optional.h" +#include "api/video/i420_buffer.h" +#include "api/video/video_rotation.h" +#include "media/base/video_common.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" + +namespace rtc { + +VideoBroadcaster::VideoBroadcaster() = default; +VideoBroadcaster::~VideoBroadcaster() = default; + +void VideoBroadcaster::AddOrUpdateSink( + VideoSinkInterface<webrtc::VideoFrame>* sink, + const VideoSinkWants& wants) { + RTC_DCHECK(sink != nullptr); + rtc::CritScope cs(&sinks_and_wants_lock_); + if (!FindSinkPair(sink)) { + // |Sink| is a new sink, which didn't receive previous frame. + previous_frame_sent_to_all_sinks_ = false; + } + VideoSourceBase::AddOrUpdateSink(sink, wants); + UpdateWants(); +} + +void VideoBroadcaster::RemoveSink( + VideoSinkInterface<webrtc::VideoFrame>* sink) { + RTC_DCHECK(sink != nullptr); + rtc::CritScope cs(&sinks_and_wants_lock_); + VideoSourceBase::RemoveSink(sink); + UpdateWants(); +} + +bool VideoBroadcaster::frame_wanted() const { + rtc::CritScope cs(&sinks_and_wants_lock_); + return !sink_pairs().empty(); +} + +VideoSinkWants VideoBroadcaster::wants() const { + rtc::CritScope cs(&sinks_and_wants_lock_); + return current_wants_; +} + +void VideoBroadcaster::OnFrame(const webrtc::VideoFrame& frame) { + rtc::CritScope cs(&sinks_and_wants_lock_); + bool current_frame_was_discarded = false; + for (auto& sink_pair : sink_pairs()) { + if (sink_pair.wants.rotation_applied && + frame.rotation() != webrtc::kVideoRotation_0) { + // Calls to OnFrame are not synchronized with changes to the sink wants. + // When rotation_applied is set to true, one or a few frames may get here + // with rotation still pending. Protect sinks that don't expect any + // pending rotation. + RTC_LOG(LS_VERBOSE) << "Discarding frame with unexpected rotation."; + sink_pair.sink->OnDiscardedFrame(); + current_frame_was_discarded = true; + continue; + } + if (sink_pair.wants.black_frames) { + webrtc::VideoFrame black_frame = + webrtc::VideoFrame::Builder() + .set_video_frame_buffer( + GetBlackFrameBuffer(frame.width(), frame.height())) + .set_rotation(frame.rotation()) + .set_timestamp_us(frame.timestamp_us()) + .set_id(frame.id()) + .build(); + sink_pair.sink->OnFrame(black_frame); + } else if (!previous_frame_sent_to_all_sinks_ && frame.has_update_rect()) { + // Since last frame was not sent to some sinks, no reliable update + // information is available, so we need to clear the update rect. + webrtc::VideoFrame copy = frame; + copy.clear_update_rect(); + sink_pair.sink->OnFrame(copy); + } else { + sink_pair.sink->OnFrame(frame); + } + } + previous_frame_sent_to_all_sinks_ = !current_frame_was_discarded; +} + +void VideoBroadcaster::OnDiscardedFrame() { + for (auto& sink_pair : sink_pairs()) { + sink_pair.sink->OnDiscardedFrame(); + } +} + +void VideoBroadcaster::UpdateWants() { + VideoSinkWants wants; + wants.rotation_applied = false; + wants.resolution_alignment = 1; + for (auto& sink : sink_pairs()) { + // wants.rotation_applied == ANY(sink.wants.rotation_applied) + if (sink.wants.rotation_applied) { + wants.rotation_applied = true; + } + // wants.max_pixel_count == MIN(sink.wants.max_pixel_count) + if (sink.wants.max_pixel_count < wants.max_pixel_count) { + wants.max_pixel_count = sink.wants.max_pixel_count; + } + // Select the minimum requested target_pixel_count, if any, of all sinks so + // that we don't over utilize the resources for any one. + // TODO(sprang): Consider using the median instead, since the limit can be + // expressed by max_pixel_count. + if (sink.wants.target_pixel_count && + (!wants.target_pixel_count || + (*sink.wants.target_pixel_count < *wants.target_pixel_count))) { + wants.target_pixel_count = sink.wants.target_pixel_count; + } + // Select the minimum for the requested max framerates. + if (sink.wants.max_framerate_fps < wants.max_framerate_fps) { + wants.max_framerate_fps = sink.wants.max_framerate_fps; + } + wants.resolution_alignment = cricket::LeastCommonMultiple( + wants.resolution_alignment, sink.wants.resolution_alignment); + } + + if (wants.target_pixel_count && + *wants.target_pixel_count >= wants.max_pixel_count) { + wants.target_pixel_count.emplace(wants.max_pixel_count); + } + current_wants_ = wants; +} + +const rtc::scoped_refptr<webrtc::VideoFrameBuffer>& +VideoBroadcaster::GetBlackFrameBuffer(int width, int height) { + if (!black_frame_buffer_ || black_frame_buffer_->width() != width || + black_frame_buffer_->height() != height) { + rtc::scoped_refptr<webrtc::I420Buffer> buffer = + webrtc::I420Buffer::Create(width, height); + webrtc::I420Buffer::SetBlack(buffer.get()); + black_frame_buffer_ = buffer; + } + + return black_frame_buffer_; +} + +} // namespace rtc diff --git a/media/base/video_broadcaster.h b/media/base/video_broadcaster.h new file mode 100644 index 0000000000..898ef2ac9a --- /dev/null +++ b/media/base/video_broadcaster.h @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MEDIA_BASE_VIDEO_BROADCASTER_H_ +#define MEDIA_BASE_VIDEO_BROADCASTER_H_ + +#include "api/scoped_refptr.h" +#include "api/video/video_frame_buffer.h" +#include "api/video/video_source_interface.h" +#include "media/base/video_source_base.h" +#include "rtc_base/critical_section.h" +#include "rtc_base/thread_annotations.h" +#include "rtc_base/thread_checker.h" + +namespace rtc { + +// VideoBroadcaster broadcast video frames to sinks and combines VideoSinkWants +// from its sinks. It does that by implementing rtc::VideoSourceInterface and +// rtc::VideoSinkInterface. The class is threadsafe; methods may be called on +// any thread. This is needed because VideoStreamEncoder calls AddOrUpdateSink +// both on the worker thread and on the encoder task queue. +class VideoBroadcaster : public VideoSourceBase, + public VideoSinkInterface<webrtc::VideoFrame> { + public: + VideoBroadcaster(); + ~VideoBroadcaster() override; + void AddOrUpdateSink(VideoSinkInterface<webrtc::VideoFrame>* sink, + const VideoSinkWants& wants) override; + void RemoveSink(VideoSinkInterface<webrtc::VideoFrame>* sink) override; + + // Returns true if the next frame will be delivered to at least one sink. + bool frame_wanted() const; + + // Returns VideoSinkWants a source is requested to fulfill. They are + // aggregated by all VideoSinkWants from all sinks. + VideoSinkWants wants() const; + + // This method ensures that if a sink sets rotation_applied == true, + // it will never receive a frame with pending rotation. Our caller + // may pass in frames without precise synchronization with changes + // to the VideoSinkWants. + void OnFrame(const webrtc::VideoFrame& frame) override; + + void OnDiscardedFrame() override; + + protected: + void UpdateWants() RTC_EXCLUSIVE_LOCKS_REQUIRED(sinks_and_wants_lock_); + const rtc::scoped_refptr<webrtc::VideoFrameBuffer>& GetBlackFrameBuffer( + int width, + int height) RTC_EXCLUSIVE_LOCKS_REQUIRED(sinks_and_wants_lock_); + + rtc::CriticalSection sinks_and_wants_lock_; + + VideoSinkWants current_wants_ RTC_GUARDED_BY(sinks_and_wants_lock_); + rtc::scoped_refptr<webrtc::VideoFrameBuffer> black_frame_buffer_; + bool previous_frame_sent_to_all_sinks_ RTC_GUARDED_BY(sinks_and_wants_lock_) = + true; +}; + +} // namespace rtc + +#endif // MEDIA_BASE_VIDEO_BROADCASTER_H_ diff --git a/media/base/video_broadcaster_unittest.cc b/media/base/video_broadcaster_unittest.cc new file mode 100644 index 0000000000..b007278547 --- /dev/null +++ b/media/base/video_broadcaster_unittest.cc @@ -0,0 +1,234 @@ +/* + * Copyright 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "media/base/video_broadcaster.h" + +#include <limits> + +#include "absl/types/optional.h" +#include "api/video/i420_buffer.h" +#include "api/video/video_frame.h" +#include "api/video/video_rotation.h" +#include "media/base/fake_video_renderer.h" +#include "test/gtest.h" + +using cricket::FakeVideoRenderer; +using rtc::VideoBroadcaster; +using rtc::VideoSinkWants; + +TEST(VideoBroadcasterTest, frame_wanted) { + VideoBroadcaster broadcaster; + EXPECT_FALSE(broadcaster.frame_wanted()); + + FakeVideoRenderer sink; + broadcaster.AddOrUpdateSink(&sink, rtc::VideoSinkWants()); + EXPECT_TRUE(broadcaster.frame_wanted()); + + broadcaster.RemoveSink(&sink); + EXPECT_FALSE(broadcaster.frame_wanted()); +} + +TEST(VideoBroadcasterTest, OnFrame) { + VideoBroadcaster broadcaster; + + FakeVideoRenderer sink1; + FakeVideoRenderer sink2; + broadcaster.AddOrUpdateSink(&sink1, rtc::VideoSinkWants()); + broadcaster.AddOrUpdateSink(&sink2, rtc::VideoSinkWants()); + static int kWidth = 100; + static int kHeight = 50; + + rtc::scoped_refptr<webrtc::I420Buffer> buffer( + webrtc::I420Buffer::Create(kWidth, kHeight)); + // Initialize, to avoid warnings on use of initialized values. + webrtc::I420Buffer::SetBlack(buffer); + + webrtc::VideoFrame frame = webrtc::VideoFrame::Builder() + .set_video_frame_buffer(buffer) + .set_rotation(webrtc::kVideoRotation_0) + .set_timestamp_us(0) + .build(); + + broadcaster.OnFrame(frame); + EXPECT_EQ(1, sink1.num_rendered_frames()); + EXPECT_EQ(1, sink2.num_rendered_frames()); + + broadcaster.RemoveSink(&sink1); + broadcaster.OnFrame(frame); + EXPECT_EQ(1, sink1.num_rendered_frames()); + EXPECT_EQ(2, sink2.num_rendered_frames()); + + broadcaster.AddOrUpdateSink(&sink1, rtc::VideoSinkWants()); + broadcaster.OnFrame(frame); + EXPECT_EQ(2, sink1.num_rendered_frames()); + EXPECT_EQ(3, sink2.num_rendered_frames()); +} + +TEST(VideoBroadcasterTest, AppliesRotationIfAnySinkWantsRotationApplied) { + VideoBroadcaster broadcaster; + EXPECT_FALSE(broadcaster.wants().rotation_applied); + + FakeVideoRenderer sink1; + VideoSinkWants wants1; + wants1.rotation_applied = false; + + broadcaster.AddOrUpdateSink(&sink1, wants1); + EXPECT_FALSE(broadcaster.wants().rotation_applied); + + FakeVideoRenderer sink2; + VideoSinkWants wants2; + wants2.rotation_applied = true; + + broadcaster.AddOrUpdateSink(&sink2, wants2); + EXPECT_TRUE(broadcaster.wants().rotation_applied); + + broadcaster.RemoveSink(&sink2); + EXPECT_FALSE(broadcaster.wants().rotation_applied); +} + +TEST(VideoBroadcasterTest, AppliesMinOfSinkWantsMaxPixelCount) { + VideoBroadcaster broadcaster; + EXPECT_EQ(std::numeric_limits<int>::max(), + broadcaster.wants().max_pixel_count); + + FakeVideoRenderer sink1; + VideoSinkWants wants1; + wants1.max_pixel_count = 1280 * 720; + + broadcaster.AddOrUpdateSink(&sink1, wants1); + EXPECT_EQ(1280 * 720, broadcaster.wants().max_pixel_count); + + FakeVideoRenderer sink2; + VideoSinkWants wants2; + wants2.max_pixel_count = 640 * 360; + broadcaster.AddOrUpdateSink(&sink2, wants2); + EXPECT_EQ(640 * 360, broadcaster.wants().max_pixel_count); + + broadcaster.RemoveSink(&sink2); + EXPECT_EQ(1280 * 720, broadcaster.wants().max_pixel_count); +} + +TEST(VideoBroadcasterTest, AppliesMinOfSinkWantsMaxAndTargetPixelCount) { + VideoBroadcaster broadcaster; + EXPECT_TRUE(!broadcaster.wants().target_pixel_count); + + FakeVideoRenderer sink1; + VideoSinkWants wants1; + wants1.target_pixel_count = 1280 * 720; + + broadcaster.AddOrUpdateSink(&sink1, wants1); + EXPECT_EQ(1280 * 720, *broadcaster.wants().target_pixel_count); + + FakeVideoRenderer sink2; + VideoSinkWants wants2; + wants2.target_pixel_count = 640 * 360; + broadcaster.AddOrUpdateSink(&sink2, wants2); + EXPECT_EQ(640 * 360, *broadcaster.wants().target_pixel_count); + + broadcaster.RemoveSink(&sink2); + EXPECT_EQ(1280 * 720, *broadcaster.wants().target_pixel_count); +} + +TEST(VideoBroadcasterTest, AppliesMinOfSinkWantsMaxFramerate) { + VideoBroadcaster broadcaster; + EXPECT_EQ(std::numeric_limits<int>::max(), + broadcaster.wants().max_framerate_fps); + + FakeVideoRenderer sink1; + VideoSinkWants wants1; + wants1.max_framerate_fps = 30; + + broadcaster.AddOrUpdateSink(&sink1, wants1); + EXPECT_EQ(30, broadcaster.wants().max_framerate_fps); + + FakeVideoRenderer sink2; + VideoSinkWants wants2; + wants2.max_framerate_fps = 15; + broadcaster.AddOrUpdateSink(&sink2, wants2); + EXPECT_EQ(15, broadcaster.wants().max_framerate_fps); + + broadcaster.RemoveSink(&sink2); + EXPECT_EQ(30, broadcaster.wants().max_framerate_fps); +} + +TEST(VideoBroadcasterTest, + AppliesLeastCommonMultipleOfSinkWantsResolutionAlignment) { + VideoBroadcaster broadcaster; + EXPECT_EQ(broadcaster.wants().resolution_alignment, 1); + + FakeVideoRenderer sink1; + VideoSinkWants wants1; + wants1.resolution_alignment = 2; + broadcaster.AddOrUpdateSink(&sink1, wants1); + EXPECT_EQ(broadcaster.wants().resolution_alignment, 2); + + FakeVideoRenderer sink2; + VideoSinkWants wants2; + wants2.resolution_alignment = 3; + broadcaster.AddOrUpdateSink(&sink2, wants2); + EXPECT_EQ(broadcaster.wants().resolution_alignment, 6); + + FakeVideoRenderer sink3; + VideoSinkWants wants3; + wants3.resolution_alignment = 4; + broadcaster.AddOrUpdateSink(&sink3, wants3); + EXPECT_EQ(broadcaster.wants().resolution_alignment, 12); + + broadcaster.RemoveSink(&sink2); + EXPECT_EQ(broadcaster.wants().resolution_alignment, 4); +} + +TEST(VideoBroadcasterTest, SinkWantsBlackFrames) { + VideoBroadcaster broadcaster; + EXPECT_TRUE(!broadcaster.wants().black_frames); + + FakeVideoRenderer sink1; + VideoSinkWants wants1; + wants1.black_frames = true; + broadcaster.AddOrUpdateSink(&sink1, wants1); + + FakeVideoRenderer sink2; + VideoSinkWants wants2; + wants2.black_frames = false; + broadcaster.AddOrUpdateSink(&sink2, wants2); + + rtc::scoped_refptr<webrtc::I420Buffer> buffer( + webrtc::I420Buffer::Create(100, 200)); + // Makes it not all black. + buffer->InitializeData(); + + webrtc::VideoFrame frame1 = webrtc::VideoFrame::Builder() + .set_video_frame_buffer(buffer) + .set_rotation(webrtc::kVideoRotation_0) + .set_timestamp_us(10) + .build(); + broadcaster.OnFrame(frame1); + EXPECT_TRUE(sink1.black_frame()); + EXPECT_EQ(10, sink1.timestamp_us()); + EXPECT_FALSE(sink2.black_frame()); + EXPECT_EQ(10, sink2.timestamp_us()); + + // Switch the sink wants. + wants1.black_frames = false; + broadcaster.AddOrUpdateSink(&sink1, wants1); + wants2.black_frames = true; + broadcaster.AddOrUpdateSink(&sink2, wants2); + + webrtc::VideoFrame frame2 = webrtc::VideoFrame::Builder() + .set_video_frame_buffer(buffer) + .set_rotation(webrtc::kVideoRotation_0) + .set_timestamp_us(30) + .build(); + broadcaster.OnFrame(frame2); + EXPECT_FALSE(sink1.black_frame()); + EXPECT_EQ(30, sink1.timestamp_us()); + EXPECT_TRUE(sink2.black_frame()); + EXPECT_EQ(30, sink2.timestamp_us()); +} diff --git a/media/base/video_common.cc b/media/base/video_common.cc new file mode 100644 index 0000000000..0ac3b3790e --- /dev/null +++ b/media/base/video_common.cc @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2010 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "media/base/video_common.h" + +#include "api/array_view.h" +#include "rtc_base/arraysize.h" +#include "rtc_base/checks.h" +#include "rtc_base/strings/string_builder.h" + +namespace cricket { + +struct FourCCAliasEntry { + uint32_t alias; + uint32_t canonical; +}; + +static const FourCCAliasEntry kFourCCAliases[] = { + {FOURCC_IYUV, FOURCC_I420}, + {FOURCC_YU16, FOURCC_I422}, + {FOURCC_YU24, FOURCC_I444}, + {FOURCC_YUYV, FOURCC_YUY2}, + {FOURCC_YUVS, FOURCC_YUY2}, + {FOURCC_HDYC, FOURCC_UYVY}, + {FOURCC_2VUY, FOURCC_UYVY}, + {FOURCC_JPEG, FOURCC_MJPG}, // Note: JPEG has DHT while MJPG does not. + {FOURCC_DMB1, FOURCC_MJPG}, + {FOURCC_BA81, FOURCC_BGGR}, + {FOURCC_RGB3, FOURCC_RAW}, + {FOURCC_BGR3, FOURCC_24BG}, + {FOURCC_CM32, FOURCC_BGRA}, + {FOURCC_CM24, FOURCC_RAW}, +}; + +uint32_t CanonicalFourCC(uint32_t fourcc) { + for (uint32_t i = 0; i < arraysize(kFourCCAliases); ++i) { + if (kFourCCAliases[i].alias == fourcc) { + return kFourCCAliases[i].canonical; + } + } + // Not an alias, so return it as-is. + return fourcc; +} + +// The C++ standard requires a namespace-scope definition of static const +// integral types even when they are initialized in the declaration (see +// [class.static.data]/4), but MSVC with /Ze is non-conforming and treats that +// as a multiply defined symbol error. See Also: +// http://msdn.microsoft.com/en-us/library/34h23df8.aspx +#ifndef _MSC_EXTENSIONS +const int64_t VideoFormat::kMinimumInterval; // Initialized in header. +#endif + +std::string VideoFormat::ToString() const { + std::string fourcc_name = GetFourccName(fourcc) + " "; + for (std::string::const_iterator i = fourcc_name.begin(); + i < fourcc_name.end(); ++i) { + // Test character is printable; Avoid isprint() which asserts on negatives. + if (*i < 32 || *i >= 127) { + fourcc_name = ""; + break; + } + } + + char buf[256]; + rtc::SimpleStringBuilder sb(buf); + sb << fourcc_name << width << "x" << height << "x" + << IntervalToFpsFloat(interval); + return sb.str(); +} + +int GreatestCommonDivisor(int a, int b) { + RTC_DCHECK_GE(a, 0); + RTC_DCHECK_GT(b, 0); + int c = a % b; + while (c != 0) { + a = b; + b = c; + c = a % b; + } + return b; +} + +int LeastCommonMultiple(int a, int b) { + RTC_DCHECK_GT(a, 0); + RTC_DCHECK_GT(b, 0); + return a * (b / GreatestCommonDivisor(a, b)); +} + +} // namespace cricket diff --git a/media/base/video_common.h b/media/base/video_common.h new file mode 100644 index 0000000000..e7ad22f9ae --- /dev/null +++ b/media/base/video_common.h @@ -0,0 +1,224 @@ +/* + * Copyright (c) 2004 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// Common definition for video, including fourcc and VideoFormat. + +#ifndef MEDIA_BASE_VIDEO_COMMON_H_ +#define MEDIA_BASE_VIDEO_COMMON_H_ + +#include <stdint.h> + +#include <string> + +#include "rtc_base/system/rtc_export.h" +#include "rtc_base/time_utils.h" + +namespace cricket { + +////////////////////////////////////////////////////////////////////////////// +// Definition of FourCC codes +////////////////////////////////////////////////////////////////////////////// +// Convert four characters to a FourCC code. +// Needs to be a macro otherwise the OS X compiler complains when the kFormat* +// constants are used in a switch. +#define CRICKET_FOURCC(a, b, c, d) \ + ((static_cast<uint32_t>(a)) | (static_cast<uint32_t>(b) << 8) | \ + (static_cast<uint32_t>(c) << 16) | (static_cast<uint32_t>(d) << 24)) +// Some pages discussing FourCC codes: +// http://www.fourcc.org/yuv.php +// http://v4l2spec.bytesex.org/spec/book1.htm +// http://developer.apple.com/quicktime/icefloe/dispatch020.html +// http://msdn.microsoft.com/library/windows/desktop/dd206750.aspx#nv12 +// http://people.xiph.org/~xiphmont/containers/nut/nut4cc.txt + +// FourCC codes grouped according to implementation efficiency. +// Primary formats should convert in 1 efficient step. +// Secondary formats are converted in 2 steps. +// Auxilliary formats call primary converters. +enum FourCC { + // 9 Primary YUV formats: 5 planar, 2 biplanar, 2 packed. + FOURCC_I420 = CRICKET_FOURCC('I', '4', '2', '0'), + FOURCC_I422 = CRICKET_FOURCC('I', '4', '2', '2'), + FOURCC_I444 = CRICKET_FOURCC('I', '4', '4', '4'), + FOURCC_I411 = CRICKET_FOURCC('I', '4', '1', '1'), + FOURCC_I400 = CRICKET_FOURCC('I', '4', '0', '0'), + FOURCC_NV21 = CRICKET_FOURCC('N', 'V', '2', '1'), + FOURCC_NV12 = CRICKET_FOURCC('N', 'V', '1', '2'), + FOURCC_YUY2 = CRICKET_FOURCC('Y', 'U', 'Y', '2'), + FOURCC_UYVY = CRICKET_FOURCC('U', 'Y', 'V', 'Y'), + + // 2 Secondary YUV formats: row biplanar. + FOURCC_M420 = CRICKET_FOURCC('M', '4', '2', '0'), + + // 9 Primary RGB formats: 4 32 bpp, 2 24 bpp, 3 16 bpp. + FOURCC_ARGB = CRICKET_FOURCC('A', 'R', 'G', 'B'), + FOURCC_BGRA = CRICKET_FOURCC('B', 'G', 'R', 'A'), + FOURCC_ABGR = CRICKET_FOURCC('A', 'B', 'G', 'R'), + FOURCC_24BG = CRICKET_FOURCC('2', '4', 'B', 'G'), + FOURCC_RAW = CRICKET_FOURCC('r', 'a', 'w', ' '), + FOURCC_RGBA = CRICKET_FOURCC('R', 'G', 'B', 'A'), + FOURCC_RGBP = CRICKET_FOURCC('R', 'G', 'B', 'P'), // bgr565. + FOURCC_RGBO = CRICKET_FOURCC('R', 'G', 'B', 'O'), // abgr1555. + FOURCC_R444 = CRICKET_FOURCC('R', '4', '4', '4'), // argb4444. + + // 4 Secondary RGB formats: 4 Bayer Patterns. + FOURCC_RGGB = CRICKET_FOURCC('R', 'G', 'G', 'B'), + FOURCC_BGGR = CRICKET_FOURCC('B', 'G', 'G', 'R'), + FOURCC_GRBG = CRICKET_FOURCC('G', 'R', 'B', 'G'), + FOURCC_GBRG = CRICKET_FOURCC('G', 'B', 'R', 'G'), + + // 1 Primary Compressed YUV format. + FOURCC_MJPG = CRICKET_FOURCC('M', 'J', 'P', 'G'), + + // 5 Auxiliary YUV variations: 3 with U and V planes are swapped, 1 Alias. + FOURCC_YV12 = CRICKET_FOURCC('Y', 'V', '1', '2'), + FOURCC_YV16 = CRICKET_FOURCC('Y', 'V', '1', '6'), + FOURCC_YV24 = CRICKET_FOURCC('Y', 'V', '2', '4'), + FOURCC_YU12 = CRICKET_FOURCC('Y', 'U', '1', '2'), // Linux version of I420. + FOURCC_J420 = CRICKET_FOURCC('J', '4', '2', '0'), + FOURCC_J400 = CRICKET_FOURCC('J', '4', '0', '0'), + + // 14 Auxiliary aliases. CanonicalFourCC() maps these to canonical FOURCC. + FOURCC_IYUV = CRICKET_FOURCC('I', 'Y', 'U', 'V'), // Alias for I420. + FOURCC_YU16 = CRICKET_FOURCC('Y', 'U', '1', '6'), // Alias for I422. + FOURCC_YU24 = CRICKET_FOURCC('Y', 'U', '2', '4'), // Alias for I444. + FOURCC_YUYV = CRICKET_FOURCC('Y', 'U', 'Y', 'V'), // Alias for YUY2. + FOURCC_YUVS = CRICKET_FOURCC('y', 'u', 'v', 's'), // Alias for YUY2 on Mac. + FOURCC_HDYC = CRICKET_FOURCC('H', 'D', 'Y', 'C'), // Alias for UYVY. + FOURCC_2VUY = CRICKET_FOURCC('2', 'v', 'u', 'y'), // Alias for UYVY on Mac. + FOURCC_JPEG = CRICKET_FOURCC('J', 'P', 'E', 'G'), // Alias for MJPG. + FOURCC_DMB1 = CRICKET_FOURCC('d', 'm', 'b', '1'), // Alias for MJPG on Mac. + FOURCC_BA81 = CRICKET_FOURCC('B', 'A', '8', '1'), // Alias for BGGR. + FOURCC_RGB3 = CRICKET_FOURCC('R', 'G', 'B', '3'), // Alias for RAW. + FOURCC_BGR3 = CRICKET_FOURCC('B', 'G', 'R', '3'), // Alias for 24BG. + FOURCC_CM32 = CRICKET_FOURCC(0, 0, 0, 32), // BGRA kCMPixelFormat_32ARGB + FOURCC_CM24 = CRICKET_FOURCC(0, 0, 0, 24), // RAW kCMPixelFormat_24RGB + + // 1 Auxiliary compressed YUV format set aside for capturer. + FOURCC_H264 = CRICKET_FOURCC('H', '2', '6', '4'), +}; + +#undef CRICKET_FOURCC + +// Match any fourcc. + +// We move this out of the enum because using it in many places caused +// the compiler to get grumpy, presumably since the above enum is +// backed by an int. +static const uint32_t FOURCC_ANY = 0xFFFFFFFF; + +// Converts fourcc aliases into canonical ones. +uint32_t CanonicalFourCC(uint32_t fourcc); + +// Get FourCC code as a string. +inline std::string GetFourccName(uint32_t fourcc) { + std::string name; + name.push_back(static_cast<char>(fourcc & 0xFF)); + name.push_back(static_cast<char>((fourcc >> 8) & 0xFF)); + name.push_back(static_cast<char>((fourcc >> 16) & 0xFF)); + name.push_back(static_cast<char>((fourcc >> 24) & 0xFF)); + return name; +} + +////////////////////////////////////////////////////////////////////////////// +// Definition of VideoFormat. +////////////////////////////////////////////////////////////////////////////// + +// VideoFormat with Plain Old Data for global variables. +struct VideoFormatPod { + int width; // Number of pixels. + int height; // Number of pixels. + int64_t interval; // Nanoseconds. + uint32_t fourcc; // Color space. FOURCC_ANY means that any color space is OK. +}; + +struct RTC_EXPORT VideoFormat : VideoFormatPod { + static const int64_t kMinimumInterval = + rtc::kNumNanosecsPerSec / 10000; // 10k fps. + + VideoFormat() { Construct(0, 0, 0, 0); } + + VideoFormat(int w, int h, int64_t interval_ns, uint32_t cc) { + Construct(w, h, interval_ns, cc); + } + + explicit VideoFormat(const VideoFormatPod& format) { + Construct(format.width, format.height, format.interval, format.fourcc); + } + + void Construct(int w, int h, int64_t interval_ns, uint32_t cc) { + width = w; + height = h; + interval = interval_ns; + fourcc = cc; + } + + static int64_t FpsToInterval(int fps) { + return fps ? rtc::kNumNanosecsPerSec / fps : kMinimumInterval; + } + + static int IntervalToFps(int64_t interval) { + if (!interval) { + return 0; + } + return static_cast<int>(rtc::kNumNanosecsPerSec / interval); + } + + static float IntervalToFpsFloat(int64_t interval) { + if (!interval) { + return 0.f; + } + return static_cast<float>(rtc::kNumNanosecsPerSec) / + static_cast<float>(interval); + } + + bool operator==(const VideoFormat& format) const { + return width == format.width && height == format.height && + interval == format.interval && fourcc == format.fourcc; + } + + bool operator!=(const VideoFormat& format) const { + return !(*this == format); + } + + bool operator<(const VideoFormat& format) const { + return (fourcc < format.fourcc) || + (fourcc == format.fourcc && width < format.width) || + (fourcc == format.fourcc && width == format.width && + height < format.height) || + (fourcc == format.fourcc && width == format.width && + height == format.height && interval > format.interval); + } + + int framerate() const { return IntervalToFps(interval); } + + // Check if both width and height are 0. + bool IsSize0x0() const { return 0 == width && 0 == height; } + + // Check if this format is less than another one by comparing the resolution + // and frame rate. + bool IsPixelRateLess(const VideoFormat& format) const { + return width * height * framerate() < + format.width * format.height * format.framerate(); + } + + // Get a string presentation in the form of "fourcc width x height x fps" + std::string ToString() const; +}; + +// Returns the largest positive integer that divides both |a| and |b|. +int GreatestCommonDivisor(int a, int b); + +// Returns the smallest positive integer that is divisible by both |a| and |b|. +int LeastCommonMultiple(int a, int b); + +} // namespace cricket + +#endif // MEDIA_BASE_VIDEO_COMMON_H_ diff --git a/media/base/video_common_unittest.cc b/media/base/video_common_unittest.cc new file mode 100644 index 0000000000..3f445c7769 --- /dev/null +++ b/media/base/video_common_unittest.cc @@ -0,0 +1,108 @@ +/* + * Copyright (c) 2008 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "media/base/video_common.h" + +#include "test/gtest.h" + +namespace cricket { + +TEST(VideoCommonTest, TestCanonicalFourCC) { + // Canonical fourccs are not changed. + EXPECT_EQ(FOURCC_I420, CanonicalFourCC(FOURCC_I420)); + // The special FOURCC_ANY value is not changed. + EXPECT_EQ(FOURCC_ANY, CanonicalFourCC(FOURCC_ANY)); + // Aliases are translated to the canonical equivalent. + EXPECT_EQ(FOURCC_I420, CanonicalFourCC(FOURCC_IYUV)); + EXPECT_EQ(FOURCC_I422, CanonicalFourCC(FOURCC_YU16)); + EXPECT_EQ(FOURCC_I444, CanonicalFourCC(FOURCC_YU24)); + EXPECT_EQ(FOURCC_YUY2, CanonicalFourCC(FOURCC_YUYV)); + EXPECT_EQ(FOURCC_YUY2, CanonicalFourCC(FOURCC_YUVS)); + EXPECT_EQ(FOURCC_UYVY, CanonicalFourCC(FOURCC_HDYC)); + EXPECT_EQ(FOURCC_UYVY, CanonicalFourCC(FOURCC_2VUY)); + EXPECT_EQ(FOURCC_MJPG, CanonicalFourCC(FOURCC_JPEG)); + EXPECT_EQ(FOURCC_MJPG, CanonicalFourCC(FOURCC_DMB1)); + EXPECT_EQ(FOURCC_BGGR, CanonicalFourCC(FOURCC_BA81)); + EXPECT_EQ(FOURCC_RAW, CanonicalFourCC(FOURCC_RGB3)); + EXPECT_EQ(FOURCC_24BG, CanonicalFourCC(FOURCC_BGR3)); + EXPECT_EQ(FOURCC_BGRA, CanonicalFourCC(FOURCC_CM32)); + EXPECT_EQ(FOURCC_RAW, CanonicalFourCC(FOURCC_CM24)); +} + +// Test conversion between interval and fps +TEST(VideoCommonTest, TestVideoFormatFps) { + EXPECT_EQ(VideoFormat::kMinimumInterval, VideoFormat::FpsToInterval(0)); + EXPECT_EQ(rtc::kNumNanosecsPerSec / 20, VideoFormat::FpsToInterval(20)); + EXPECT_EQ(20, VideoFormat::IntervalToFps(rtc::kNumNanosecsPerSec / 20)); + EXPECT_EQ(0, VideoFormat::IntervalToFps(0)); +} + +// Test IsSize0x0 +TEST(VideoCommonTest, TestVideoFormatIsSize0x0) { + VideoFormat format; + EXPECT_TRUE(format.IsSize0x0()); + format.width = 320; + EXPECT_FALSE(format.IsSize0x0()); +} + +// Test ToString: print fourcc when it is printable. +TEST(VideoCommonTest, TestVideoFormatToString) { + VideoFormat format; + EXPECT_EQ("0x0x0", format.ToString()); + + format.fourcc = FOURCC_I420; + format.width = 640; + format.height = 480; + format.interval = VideoFormat::FpsToInterval(20); + EXPECT_EQ("I420 640x480x20", format.ToString()); + + format.fourcc = FOURCC_ANY; + format.width = 640; + format.height = 480; + format.interval = VideoFormat::FpsToInterval(20); + EXPECT_EQ("640x480x20", format.ToString()); +} + +// Test comparison. +TEST(VideoCommonTest, TestVideoFormatCompare) { + VideoFormat format(640, 480, VideoFormat::FpsToInterval(20), FOURCC_I420); + VideoFormat format2; + EXPECT_NE(format, format2); + + // Same pixelrate, different fourcc. + format2 = format; + format2.fourcc = FOURCC_YUY2; + EXPECT_NE(format, format2); + EXPECT_FALSE(format.IsPixelRateLess(format2) || + format2.IsPixelRateLess(format2)); + + format2 = format; + format2.interval /= 2; + EXPECT_TRUE(format.IsPixelRateLess(format2)); + + format2 = format; + format2.width *= 2; + EXPECT_TRUE(format.IsPixelRateLess(format2)); +} + +TEST(VideoCommonTest, GreatestCommonDivisor) { + EXPECT_EQ(GreatestCommonDivisor(0, 1000), 1000); + EXPECT_EQ(GreatestCommonDivisor(1, 1), 1); + EXPECT_EQ(GreatestCommonDivisor(8, 12), 4); + EXPECT_EQ(GreatestCommonDivisor(24, 54), 6); +} + +TEST(VideoCommonTest, LeastCommonMultiple) { + EXPECT_EQ(LeastCommonMultiple(1, 1), 1); + EXPECT_EQ(LeastCommonMultiple(2, 3), 6); + EXPECT_EQ(LeastCommonMultiple(16, 32), 32); +} + +} // namespace cricket diff --git a/media/base/video_source_base.cc b/media/base/video_source_base.cc new file mode 100644 index 0000000000..d057a24ad8 --- /dev/null +++ b/media/base/video_source_base.cc @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "media/base/video_source_base.h" + +#include "absl/algorithm/container.h" +#include "rtc_base/checks.h" + +namespace rtc { + +VideoSourceBase::VideoSourceBase() = default; +VideoSourceBase::~VideoSourceBase() = default; + +void VideoSourceBase::AddOrUpdateSink( + VideoSinkInterface<webrtc::VideoFrame>* sink, + const VideoSinkWants& wants) { + RTC_DCHECK(sink != nullptr); + + SinkPair* sink_pair = FindSinkPair(sink); + if (!sink_pair) { + sinks_.push_back(SinkPair(sink, wants)); + } else { + sink_pair->wants = wants; + } +} + +void VideoSourceBase::RemoveSink(VideoSinkInterface<webrtc::VideoFrame>* sink) { + RTC_DCHECK(sink != nullptr); + RTC_DCHECK(FindSinkPair(sink)); + sinks_.erase(std::remove_if(sinks_.begin(), sinks_.end(), + [sink](const SinkPair& sink_pair) { + return sink_pair.sink == sink; + }), + sinks_.end()); +} + +VideoSourceBase::SinkPair* VideoSourceBase::FindSinkPair( + const VideoSinkInterface<webrtc::VideoFrame>* sink) { + auto sink_pair_it = absl::c_find_if( + sinks_, + [sink](const SinkPair& sink_pair) { return sink_pair.sink == sink; }); + if (sink_pair_it != sinks_.end()) { + return &*sink_pair_it; + } + return nullptr; +} + +} // namespace rtc diff --git a/media/base/video_source_base.h b/media/base/video_source_base.h new file mode 100644 index 0000000000..507fa10645 --- /dev/null +++ b/media/base/video_source_base.h @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MEDIA_BASE_VIDEO_SOURCE_BASE_H_ +#define MEDIA_BASE_VIDEO_SOURCE_BASE_H_ + +#include <vector> + +#include "api/video/video_frame.h" +#include "api/video/video_sink_interface.h" +#include "api/video/video_source_interface.h" +#include "rtc_base/thread_checker.h" + +namespace rtc { + +// VideoSourceBase is not thread safe. +class VideoSourceBase : public VideoSourceInterface<webrtc::VideoFrame> { + public: + VideoSourceBase(); + ~VideoSourceBase() override; + void AddOrUpdateSink(VideoSinkInterface<webrtc::VideoFrame>* sink, + const VideoSinkWants& wants) override; + void RemoveSink(VideoSinkInterface<webrtc::VideoFrame>* sink) override; + + protected: + struct SinkPair { + SinkPair(VideoSinkInterface<webrtc::VideoFrame>* sink, VideoSinkWants wants) + : sink(sink), wants(wants) {} + VideoSinkInterface<webrtc::VideoFrame>* sink; + VideoSinkWants wants; + }; + SinkPair* FindSinkPair(const VideoSinkInterface<webrtc::VideoFrame>* sink); + + const std::vector<SinkPair>& sink_pairs() const { return sinks_; } + + private: + std::vector<SinkPair> sinks_; +}; + +} // namespace rtc + +#endif // MEDIA_BASE_VIDEO_SOURCE_BASE_H_ diff --git a/media/base/vp9_profile.cc b/media/base/vp9_profile.cc new file mode 100644 index 0000000000..cfecc5e545 --- /dev/null +++ b/media/base/vp9_profile.cc @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "media/base/vp9_profile.h" + +#include <map> +#include <utility> + +#include "rtc_base/string_to_number.h" + +namespace webrtc { + +// Profile information for VP9 video. +const char kVP9FmtpProfileId[] = "profile-id"; + +std::string VP9ProfileToString(VP9Profile profile) { + switch (profile) { + case VP9Profile::kProfile0: + return "0"; + case VP9Profile::kProfile2: + return "2"; + } + return "0"; +} + +absl::optional<VP9Profile> StringToVP9Profile(const std::string& str) { + const absl::optional<int> i = rtc::StringToNumber<int>(str); + if (!i.has_value()) + return absl::nullopt; + + switch (i.value()) { + case 0: + return VP9Profile::kProfile0; + case 2: + return VP9Profile::kProfile2; + default: + return absl::nullopt; + } + return absl::nullopt; +} + +absl::optional<VP9Profile> ParseSdpForVP9Profile( + const SdpVideoFormat::Parameters& params) { + const auto profile_it = params.find(kVP9FmtpProfileId); + if (profile_it == params.end()) + return VP9Profile::kProfile0; + const std::string& profile_str = profile_it->second; + return StringToVP9Profile(profile_str); +} + +bool IsSameVP9Profile(const SdpVideoFormat::Parameters& params1, + const SdpVideoFormat::Parameters& params2) { + const absl::optional<VP9Profile> profile = ParseSdpForVP9Profile(params1); + const absl::optional<VP9Profile> other_profile = + ParseSdpForVP9Profile(params2); + return profile && other_profile && profile == other_profile; +} + +} // namespace webrtc diff --git a/media/base/vp9_profile.h b/media/base/vp9_profile.h new file mode 100644 index 0000000000..e2bbf19005 --- /dev/null +++ b/media/base/vp9_profile.h @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MEDIA_BASE_VP9_PROFILE_H_ +#define MEDIA_BASE_VP9_PROFILE_H_ + +#include <string> + +#include "absl/types/optional.h" +#include "api/video_codecs/sdp_video_format.h" +#include "rtc_base/system/rtc_export.h" + +namespace webrtc { + +// Profile information for VP9 video. +extern RTC_EXPORT const char kVP9FmtpProfileId[]; + +enum class VP9Profile { + kProfile0, + kProfile2, +}; + +// Helper functions to convert VP9Profile to std::string. Returns "0" by +// default. +RTC_EXPORT std::string VP9ProfileToString(VP9Profile profile); + +// Helper functions to convert std::string to VP9Profile. Returns null if given +// an invalid profile string. +absl::optional<VP9Profile> StringToVP9Profile(const std::string& str); + +// Parse profile that is represented as a string of single digit contained in an +// SDP key-value map. A default profile(kProfile0) will be returned if the +// profile key is missing. Nothing will be returned if the key is present but +// the string is invalid. +RTC_EXPORT absl::optional<VP9Profile> ParseSdpForVP9Profile( + const SdpVideoFormat::Parameters& params); + +// Returns true if the parameters have the same VP9 profile, or neither contains +// VP9 profile. +bool IsSameVP9Profile(const SdpVideoFormat::Parameters& params1, + const SdpVideoFormat::Parameters& params2); + +} // namespace webrtc + +#endif // MEDIA_BASE_VP9_PROFILE_H_ diff --git a/media/engine/adm_helpers.cc b/media/engine/adm_helpers.cc new file mode 100644 index 0000000000..c349b7ce06 --- /dev/null +++ b/media/engine/adm_helpers.cc @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "media/engine/adm_helpers.h" + +#include "modules/audio_device/include/audio_device.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" + +namespace webrtc { +namespace adm_helpers { + +// On Windows Vista and newer, Microsoft introduced the concept of "Default +// Communications Device". This means that there are two types of default +// devices (old Wave Audio style default and Default Communications Device). +// +// On Windows systems which only support Wave Audio style default, uses either +// -1 or 0 to select the default device. +// +// Using a #define for AUDIO_DEVICE since we will call *different* versions of +// the ADM functions, depending on the ID type. +#if defined(WEBRTC_WIN) +#define AUDIO_DEVICE_ID \ + (AudioDeviceModule::WindowsDeviceType::kDefaultCommunicationDevice) +#else +#define AUDIO_DEVICE_ID (0u) +#endif // defined(WEBRTC_WIN) + +void Init(AudioDeviceModule* adm) { + RTC_DCHECK(adm); + + RTC_CHECK_EQ(0, adm->Init()) << "Failed to initialize the ADM."; + + // Playout device. + { + if (adm->SetPlayoutDevice(AUDIO_DEVICE_ID) != 0) { + RTC_LOG(LS_ERROR) << "Unable to set playout device."; + return; + } + if (adm->InitSpeaker() != 0) { + RTC_LOG(LS_ERROR) << "Unable to access speaker."; + } + + // Set number of channels + bool available = false; + if (adm->StereoPlayoutIsAvailable(&available) != 0) { + RTC_LOG(LS_ERROR) << "Failed to query stereo playout."; + } + if (adm->SetStereoPlayout(available) != 0) { + RTC_LOG(LS_ERROR) << "Failed to set stereo playout mode."; + } + } + + // Recording device. + { + if (adm->SetRecordingDevice(AUDIO_DEVICE_ID) != 0) { + RTC_LOG(LS_ERROR) << "Unable to set recording device."; + return; + } + if (adm->InitMicrophone() != 0) { + RTC_LOG(LS_ERROR) << "Unable to access microphone."; + } + + // Set number of channels + bool available = false; + if (adm->StereoRecordingIsAvailable(&available) != 0) { + RTC_LOG(LS_ERROR) << "Failed to query stereo recording."; + } + if (adm->SetStereoRecording(available) != 0) { + RTC_LOG(LS_ERROR) << "Failed to set stereo recording mode."; + } + } +} +} // namespace adm_helpers +} // namespace webrtc diff --git a/media/engine/adm_helpers.h b/media/engine/adm_helpers.h new file mode 100644 index 0000000000..2a35d26b47 --- /dev/null +++ b/media/engine/adm_helpers.h @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MEDIA_ENGINE_ADM_HELPERS_H_ +#define MEDIA_ENGINE_ADM_HELPERS_H_ + +namespace webrtc { + +class AudioDeviceModule; + +namespace adm_helpers { + +void Init(AudioDeviceModule* adm); + +} // namespace adm_helpers +} // namespace webrtc + +#endif // MEDIA_ENGINE_ADM_HELPERS_H_ diff --git a/media/engine/constants.cc b/media/engine/constants.cc new file mode 100644 index 0000000000..12d6ddde5a --- /dev/null +++ b/media/engine/constants.cc @@ -0,0 +1,19 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "media/engine/constants.h" + +namespace cricket { + +const int kVideoMtu = 1200; +const int kVideoRtpSendBufferSize = 65536; +const int kVideoRtpRecvBufferSize = 262144; + +} // namespace cricket diff --git a/media/engine/constants.h b/media/engine/constants.h new file mode 100644 index 0000000000..9a421d9875 --- /dev/null +++ b/media/engine/constants.h @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MEDIA_ENGINE_CONSTANTS_H_ +#define MEDIA_ENGINE_CONSTANTS_H_ + +namespace cricket { + +extern const int kVideoMtu; +extern const int kVideoRtpSendBufferSize; +extern const int kVideoRtpRecvBufferSize; + +} // namespace cricket + +#endif // MEDIA_ENGINE_CONSTANTS_H_ diff --git a/media/engine/encoder_simulcast_proxy.cc b/media/engine/encoder_simulcast_proxy.cc new file mode 100644 index 0000000000..7a6638f56f --- /dev/null +++ b/media/engine/encoder_simulcast_proxy.cc @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "media/engine/encoder_simulcast_proxy.h" + +#include "api/video_codecs/video_encoder.h" +#include "media/engine/simulcast_encoder_adapter.h" +#include "modules/video_coding/include/video_error_codes.h" + +namespace webrtc { + +EncoderSimulcastProxy::EncoderSimulcastProxy(VideoEncoderFactory* factory, + const SdpVideoFormat& format) + : factory_(factory), video_format_(format), callback_(nullptr) { + encoder_ = factory_->CreateVideoEncoder(format); +} + +EncoderSimulcastProxy::EncoderSimulcastProxy(VideoEncoderFactory* factory) + : EncoderSimulcastProxy(factory, SdpVideoFormat("VP8")) {} + +EncoderSimulcastProxy::~EncoderSimulcastProxy() {} + +int EncoderSimulcastProxy::Release() { + return encoder_->Release(); +} + +void EncoderSimulcastProxy::SetFecControllerOverride( + FecControllerOverride* fec_controller_override) { + encoder_->SetFecControllerOverride(fec_controller_override); +} + +// TODO(eladalon): s/inst/codec_settings/g. +int EncoderSimulcastProxy::InitEncode(const VideoCodec* inst, + const VideoEncoder::Settings& settings) { + int ret = encoder_->InitEncode(inst, settings); + if (ret == WEBRTC_VIDEO_CODEC_ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED) { + encoder_.reset(new SimulcastEncoderAdapter(factory_, video_format_)); + if (callback_) { + encoder_->RegisterEncodeCompleteCallback(callback_); + } + ret = encoder_->InitEncode(inst, settings); + } + return ret; +} + +int EncoderSimulcastProxy::Encode( + const VideoFrame& input_image, + const std::vector<VideoFrameType>* frame_types) { + return encoder_->Encode(input_image, frame_types); +} + +int EncoderSimulcastProxy::RegisterEncodeCompleteCallback( + EncodedImageCallback* callback) { + callback_ = callback; + return encoder_->RegisterEncodeCompleteCallback(callback); +} + +void EncoderSimulcastProxy::SetRates(const RateControlParameters& parameters) { + encoder_->SetRates(parameters); +} + +void EncoderSimulcastProxy::OnPacketLossRateUpdate(float packet_loss_rate) { + encoder_->OnPacketLossRateUpdate(packet_loss_rate); +} + +void EncoderSimulcastProxy::OnRttUpdate(int64_t rtt_ms) { + encoder_->OnRttUpdate(rtt_ms); +} + +void EncoderSimulcastProxy::OnLossNotification( + const LossNotification& loss_notification) { + encoder_->OnLossNotification(loss_notification); +} + +VideoEncoder::EncoderInfo EncoderSimulcastProxy::GetEncoderInfo() const { + return encoder_->GetEncoderInfo(); +} + +} // namespace webrtc diff --git a/media/engine/encoder_simulcast_proxy.h b/media/engine/encoder_simulcast_proxy.h new file mode 100644 index 0000000000..8e9e0ffb6e --- /dev/null +++ b/media/engine/encoder_simulcast_proxy.h @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + * + */ + +#ifndef MEDIA_ENGINE_ENCODER_SIMULCAST_PROXY_H_ +#define MEDIA_ENGINE_ENCODER_SIMULCAST_PROXY_H_ + +#include <stddef.h> +#include <stdint.h> + +#include <memory> +#include <vector> + +#include "api/video/video_bitrate_allocation.h" +#include "api/video/video_frame.h" +#include "api/video_codecs/sdp_video_format.h" +#include "api/video_codecs/video_codec.h" +#include "api/video_codecs/video_encoder.h" +#include "api/video_codecs/video_encoder_factory.h" +#include "modules/video_coding/include/video_codec_interface.h" +#include "rtc_base/system/rtc_export.h" + +namespace webrtc { + +// This class provides fallback to SimulcastEncoderAdapter if default VP8Encoder +// doesn't support simulcast for provided settings. +class RTC_EXPORT EncoderSimulcastProxy : public VideoEncoder { + public: + EncoderSimulcastProxy(VideoEncoderFactory* factory, + const SdpVideoFormat& format); + // Deprecated. Remove once all clients use constructor with both factory and + // SdpVideoFormat; + explicit EncoderSimulcastProxy(VideoEncoderFactory* factory); + + ~EncoderSimulcastProxy() override; + + // Implements VideoEncoder. + int Release() override; + void SetFecControllerOverride( + FecControllerOverride* fec_controller_override) override; + int InitEncode(const VideoCodec* codec_settings, + const VideoEncoder::Settings& settings) override; + int Encode(const VideoFrame& input_image, + const std::vector<VideoFrameType>* frame_types) override; + int RegisterEncodeCompleteCallback(EncodedImageCallback* callback) override; + void SetRates(const RateControlParameters& parameters) override; + void OnPacketLossRateUpdate(float packet_loss_rate) override; + void OnRttUpdate(int64_t rtt_ms) override; + void OnLossNotification(const LossNotification& loss_notification) override; + EncoderInfo GetEncoderInfo() const override; + + private: + VideoEncoderFactory* const factory_; + SdpVideoFormat video_format_; + std::unique_ptr<VideoEncoder> encoder_; + EncodedImageCallback* callback_; +}; + +} // namespace webrtc + +#endif // MEDIA_ENGINE_ENCODER_SIMULCAST_PROXY_H_ diff --git a/media/engine/encoder_simulcast_proxy_unittest.cc b/media/engine/encoder_simulcast_proxy_unittest.cc new file mode 100644 index 0000000000..ebbadb00a4 --- /dev/null +++ b/media/engine/encoder_simulcast_proxy_unittest.cc @@ -0,0 +1,215 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + * + */ + +#include "media/engine/encoder_simulcast_proxy.h" + +#include <memory> +#include <string> +#include <utility> + +#include "api/test/mock_video_encoder.h" +#include "api/test/mock_video_encoder_factory.h" +#include "api/video_codecs/video_encoder.h" +#include "api/video_codecs/vp8_temporal_layers.h" +#include "modules/video_coding/include/video_codec_interface.h" +#include "test/gmock.h" +#include "test/gtest.h" +#include "test/video_codec_settings.h" + +namespace webrtc { +namespace testing { +namespace { +const VideoEncoder::Capabilities kCapabilities(false); +const VideoEncoder::Settings kSettings(kCapabilities, 4, 1200); +} // namespace + +using ::testing::_; +using ::testing::ByMove; +using ::testing::NiceMock; +using ::testing::Return; + +TEST(EncoderSimulcastProxy, ChoosesCorrectImplementation) { + const std::string kImplementationName = "Fake"; + const std::string kSimulcastAdaptedImplementationName = + "SimulcastEncoderAdapter (Fake, Fake, Fake)"; + VideoCodec codec_settings; + webrtc::test::CodecSettings(kVideoCodecVP8, &codec_settings); + codec_settings.simulcastStream[0] = {test::kTestWidth, + test::kTestHeight, + test::kTestFrameRate, + 2, + 2000, + 1000, + 1000, + 56}; + codec_settings.simulcastStream[1] = {test::kTestWidth, + test::kTestHeight, + test::kTestFrameRate, + 2, + 3000, + 1000, + 1000, + 56}; + codec_settings.simulcastStream[2] = {test::kTestWidth, + test::kTestHeight, + test::kTestFrameRate, + 2, + 5000, + 1000, + 1000, + 56}; + codec_settings.numberOfSimulcastStreams = 3; + + auto mock_encoder = std::make_unique<NiceMock<MockVideoEncoder>>(); + NiceMock<MockVideoEncoderFactory> simulcast_factory; + + EXPECT_CALL(*mock_encoder, InitEncode(_, _)) + .WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); + VideoEncoder::EncoderInfo encoder_info; + encoder_info.implementation_name = kImplementationName; + EXPECT_CALL(*mock_encoder, GetEncoderInfo()) + .WillRepeatedly(Return(encoder_info)); + + EXPECT_CALL(simulcast_factory, CreateVideoEncoder) + .Times(1) + .WillOnce(Return(ByMove(std::move(mock_encoder)))); + + EncoderSimulcastProxy simulcast_enabled_proxy(&simulcast_factory, + SdpVideoFormat("VP8")); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + simulcast_enabled_proxy.InitEncode(&codec_settings, kSettings)); + EXPECT_EQ(kImplementationName, + simulcast_enabled_proxy.GetEncoderInfo().implementation_name); + + NiceMock<MockVideoEncoderFactory> nonsimulcast_factory; + + EXPECT_CALL(nonsimulcast_factory, CreateVideoEncoder) + .Times(4) + .WillOnce([&] { + auto mock_encoder = std::make_unique<NiceMock<MockVideoEncoder>>(); + EXPECT_CALL(*mock_encoder, InitEncode(_, _)) + .WillOnce(Return( + WEBRTC_VIDEO_CODEC_ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED)); + ON_CALL(*mock_encoder, GetEncoderInfo) + .WillByDefault(Return(encoder_info)); + return mock_encoder; + }) + .WillRepeatedly([&] { + auto mock_encoder = std::make_unique<NiceMock<MockVideoEncoder>>(); + EXPECT_CALL(*mock_encoder, InitEncode(_, _)) + .WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); + ON_CALL(*mock_encoder, GetEncoderInfo) + .WillByDefault(Return(encoder_info)); + return mock_encoder; + }); + + EncoderSimulcastProxy simulcast_disabled_proxy(&nonsimulcast_factory, + SdpVideoFormat("VP8")); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + simulcast_disabled_proxy.InitEncode(&codec_settings, kSettings)); + EXPECT_EQ(kSimulcastAdaptedImplementationName, + simulcast_disabled_proxy.GetEncoderInfo().implementation_name); + + // Cleanup. + simulcast_enabled_proxy.Release(); + simulcast_disabled_proxy.Release(); +} + +TEST(EncoderSimulcastProxy, ForwardsTrustedSetting) { + auto mock_encoder_owned = std::make_unique<NiceMock<MockVideoEncoder>>(); + auto* mock_encoder = mock_encoder_owned.get(); + NiceMock<MockVideoEncoderFactory> simulcast_factory; + + EXPECT_CALL(*mock_encoder, InitEncode(_, _)) + .WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); + + EXPECT_CALL(simulcast_factory, CreateVideoEncoder) + .Times(1) + .WillOnce(Return(ByMove(std::move(mock_encoder_owned)))); + + EncoderSimulcastProxy simulcast_enabled_proxy(&simulcast_factory, + SdpVideoFormat("VP8")); + VideoCodec codec_settings; + webrtc::test::CodecSettings(kVideoCodecVP8, &codec_settings); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + simulcast_enabled_proxy.InitEncode(&codec_settings, kSettings)); + + VideoEncoder::EncoderInfo info; + info.has_trusted_rate_controller = true; + EXPECT_CALL(*mock_encoder, GetEncoderInfo()).WillRepeatedly(Return(info)); + + EXPECT_TRUE( + simulcast_enabled_proxy.GetEncoderInfo().has_trusted_rate_controller); +} + +TEST(EncoderSimulcastProxy, ForwardsHardwareAccelerated) { + auto mock_encoder_owned = std::make_unique<NiceMock<MockVideoEncoder>>(); + NiceMock<MockVideoEncoder>* mock_encoder = mock_encoder_owned.get(); + NiceMock<MockVideoEncoderFactory> simulcast_factory; + + EXPECT_CALL(*mock_encoder, InitEncode(_, _)) + .WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); + + EXPECT_CALL(simulcast_factory, CreateVideoEncoder) + .Times(1) + .WillOnce(Return(ByMove(std::move(mock_encoder_owned)))); + + EncoderSimulcastProxy simulcast_enabled_proxy(&simulcast_factory, + SdpVideoFormat("VP8")); + VideoCodec codec_settings; + webrtc::test::CodecSettings(kVideoCodecVP8, &codec_settings); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + simulcast_enabled_proxy.InitEncode(&codec_settings, kSettings)); + + VideoEncoder::EncoderInfo info; + + info.is_hardware_accelerated = false; + EXPECT_CALL(*mock_encoder, GetEncoderInfo()).WillOnce(Return(info)); + EXPECT_FALSE( + simulcast_enabled_proxy.GetEncoderInfo().is_hardware_accelerated); + + info.is_hardware_accelerated = true; + EXPECT_CALL(*mock_encoder, GetEncoderInfo()).WillOnce(Return(info)); + EXPECT_TRUE(simulcast_enabled_proxy.GetEncoderInfo().is_hardware_accelerated); +} + +TEST(EncoderSimulcastProxy, ForwardsInternalSource) { + auto mock_encoder_owned = std::make_unique<NiceMock<MockVideoEncoder>>(); + NiceMock<MockVideoEncoder>* mock_encoder = mock_encoder_owned.get(); + NiceMock<MockVideoEncoderFactory> simulcast_factory; + + EXPECT_CALL(*mock_encoder, InitEncode(_, _)) + .WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); + + EXPECT_CALL(simulcast_factory, CreateVideoEncoder) + .Times(1) + .WillOnce(Return(ByMove(std::move(mock_encoder_owned)))); + + EncoderSimulcastProxy simulcast_enabled_proxy(&simulcast_factory, + SdpVideoFormat("VP8")); + VideoCodec codec_settings; + webrtc::test::CodecSettings(kVideoCodecVP8, &codec_settings); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + simulcast_enabled_proxy.InitEncode(&codec_settings, kSettings)); + + VideoEncoder::EncoderInfo info; + + info.has_internal_source = false; + EXPECT_CALL(*mock_encoder, GetEncoderInfo()).WillOnce(Return(info)); + EXPECT_FALSE(simulcast_enabled_proxy.GetEncoderInfo().has_internal_source); + + info.has_internal_source = true; + EXPECT_CALL(*mock_encoder, GetEncoderInfo()).WillOnce(Return(info)); + EXPECT_TRUE(simulcast_enabled_proxy.GetEncoderInfo().has_internal_source); +} + +} // namespace testing +} // namespace webrtc diff --git a/media/engine/fake_video_codec_factory.cc b/media/engine/fake_video_codec_factory.cc new file mode 100644 index 0000000000..ba7513be24 --- /dev/null +++ b/media/engine/fake_video_codec_factory.cc @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "media/engine/fake_video_codec_factory.h" + +#include <memory> + +#include "api/video_codecs/sdp_video_format.h" +#include "api/video_codecs/video_decoder.h" +#include "api/video_codecs/video_encoder.h" +#include "modules/include/module_common_types.h" +#include "modules/video_coding/include/video_codec_interface.h" +#include "modules/video_coding/include/video_error_codes.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "test/fake_decoder.h" +#include "test/fake_encoder.h" + +namespace { + +static const char kFakeCodecFactoryCodecName[] = "FakeCodec"; + +} // anonymous namespace + +namespace webrtc { + +FakeVideoEncoderFactory::FakeVideoEncoderFactory() = default; + +// static +std::unique_ptr<VideoEncoder> FakeVideoEncoderFactory::CreateVideoEncoder() { + return std::make_unique<test::FakeEncoder>(Clock::GetRealTimeClock()); +} + +std::vector<SdpVideoFormat> FakeVideoEncoderFactory::GetSupportedFormats() + const { + return std::vector<SdpVideoFormat>( + 1, SdpVideoFormat(kFakeCodecFactoryCodecName)); +} + +VideoEncoderFactory::CodecInfo FakeVideoEncoderFactory::QueryVideoEncoder( + const SdpVideoFormat& format) const { + return VideoEncoderFactory::CodecInfo{false, false}; +} + +std::unique_ptr<VideoEncoder> FakeVideoEncoderFactory::CreateVideoEncoder( + const SdpVideoFormat& format) { + return std::make_unique<test::FakeEncoder>(Clock::GetRealTimeClock()); +} + +FakeVideoDecoderFactory::FakeVideoDecoderFactory() = default; + +// static +std::unique_ptr<VideoDecoder> FakeVideoDecoderFactory::CreateVideoDecoder() { + return std::make_unique<test::FakeDecoder>(); +} + +std::vector<SdpVideoFormat> FakeVideoDecoderFactory::GetSupportedFormats() + const { + return std::vector<SdpVideoFormat>( + 1, SdpVideoFormat(kFakeCodecFactoryCodecName)); +} + +std::unique_ptr<VideoDecoder> FakeVideoDecoderFactory::CreateVideoDecoder( + const SdpVideoFormat& format) { + return std::make_unique<test::FakeDecoder>(); +} + +} // namespace webrtc diff --git a/media/engine/fake_video_codec_factory.h b/media/engine/fake_video_codec_factory.h new file mode 100644 index 0000000000..029a695337 --- /dev/null +++ b/media/engine/fake_video_codec_factory.h @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MEDIA_ENGINE_FAKE_VIDEO_CODEC_FACTORY_H_ +#define MEDIA_ENGINE_FAKE_VIDEO_CODEC_FACTORY_H_ + +#include <memory> +#include <vector> + +#include "api/video_codecs/video_decoder_factory.h" +#include "api/video_codecs/video_encoder_factory.h" +#include "rtc_base/system/rtc_export.h" + +namespace webrtc { + +// Provides a fake video encoder instance that produces frames large enough for +// the given bitrate constraints. +class RTC_EXPORT FakeVideoEncoderFactory : public VideoEncoderFactory { + public: + FakeVideoEncoderFactory(); + + static std::unique_ptr<VideoEncoder> CreateVideoEncoder(); + + // VideoEncoderFactory implementation + std::vector<SdpVideoFormat> GetSupportedFormats() const override; + VideoEncoderFactory::CodecInfo QueryVideoEncoder( + const SdpVideoFormat& format) const override; + std::unique_ptr<VideoEncoder> CreateVideoEncoder( + const SdpVideoFormat& format) override; +}; + +// Provides a fake video decoder instance that ignores the given bitstream and +// produces frames. +class RTC_EXPORT FakeVideoDecoderFactory : public VideoDecoderFactory { + public: + FakeVideoDecoderFactory(); + + static std::unique_ptr<VideoDecoder> CreateVideoDecoder(); + + // VideoDecoderFactory implementation + std::vector<SdpVideoFormat> GetSupportedFormats() const override; + std::unique_ptr<VideoDecoder> CreateVideoDecoder( + const SdpVideoFormat& format) override; +}; + +} // namespace webrtc + +#endif // MEDIA_ENGINE_FAKE_VIDEO_CODEC_FACTORY_H_ diff --git a/media/engine/fake_webrtc_call.cc b/media/engine/fake_webrtc_call.cc new file mode 100644 index 0000000000..78d4ba41e0 --- /dev/null +++ b/media/engine/fake_webrtc_call.cc @@ -0,0 +1,647 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "media/engine/fake_webrtc_call.h" + +#include <utility> + +#include "absl/algorithm/container.h" +#include "api/call/audio_sink.h" +#include "media/base/rtp_utils.h" +#include "rtc_base/checks.h" +#include "rtc_base/gunit.h" + +namespace cricket { +FakeAudioSendStream::FakeAudioSendStream( + int id, + const webrtc::AudioSendStream::Config& config) + : id_(id), config_(config) {} + +void FakeAudioSendStream::Reconfigure( + const webrtc::AudioSendStream::Config& config) { + config_ = config; +} + +const webrtc::AudioSendStream::Config& FakeAudioSendStream::GetConfig() const { + return config_; +} + +void FakeAudioSendStream::SetStats( + const webrtc::AudioSendStream::Stats& stats) { + stats_ = stats; +} + +FakeAudioSendStream::TelephoneEvent +FakeAudioSendStream::GetLatestTelephoneEvent() const { + return latest_telephone_event_; +} + +bool FakeAudioSendStream::SendTelephoneEvent(int payload_type, + int payload_frequency, + int event, + int duration_ms) { + latest_telephone_event_.payload_type = payload_type; + latest_telephone_event_.payload_frequency = payload_frequency; + latest_telephone_event_.event_code = event; + latest_telephone_event_.duration_ms = duration_ms; + return true; +} + +void FakeAudioSendStream::SetMuted(bool muted) { + muted_ = muted; +} + +webrtc::AudioSendStream::Stats FakeAudioSendStream::GetStats() const { + return stats_; +} + +webrtc::AudioSendStream::Stats FakeAudioSendStream::GetStats( + bool /*has_remote_tracks*/) const { + return stats_; +} + +FakeAudioReceiveStream::FakeAudioReceiveStream( + int id, + const webrtc::AudioReceiveStream::Config& config) + : id_(id), config_(config) {} + +const webrtc::AudioReceiveStream::Config& FakeAudioReceiveStream::GetConfig() + const { + return config_; +} + +void FakeAudioReceiveStream::SetStats( + const webrtc::AudioReceiveStream::Stats& stats) { + stats_ = stats; +} + +bool FakeAudioReceiveStream::VerifyLastPacket(const uint8_t* data, + size_t length) const { + return last_packet_ == rtc::Buffer(data, length); +} + +bool FakeAudioReceiveStream::DeliverRtp(const uint8_t* packet, + size_t length, + int64_t /* packet_time_us */) { + ++received_packets_; + last_packet_.SetData(packet, length); + return true; +} + +void FakeAudioReceiveStream::Reconfigure( + const webrtc::AudioReceiveStream::Config& config) { + config_ = config; +} + +webrtc::AudioReceiveStream::Stats FakeAudioReceiveStream::GetStats() const { + return stats_; +} + +void FakeAudioReceiveStream::SetSink(webrtc::AudioSinkInterface* sink) { + sink_ = sink; +} + +void FakeAudioReceiveStream::SetGain(float gain) { + gain_ = gain; +} + +FakeVideoSendStream::FakeVideoSendStream( + webrtc::VideoSendStream::Config config, + webrtc::VideoEncoderConfig encoder_config) + : sending_(false), + config_(std::move(config)), + codec_settings_set_(false), + resolution_scaling_enabled_(false), + framerate_scaling_enabled_(false), + source_(nullptr), + num_swapped_frames_(0) { + RTC_DCHECK(config.encoder_settings.encoder_factory != nullptr); + RTC_DCHECK(config.encoder_settings.bitrate_allocator_factory != nullptr); + ReconfigureVideoEncoder(std::move(encoder_config)); +} + +FakeVideoSendStream::~FakeVideoSendStream() { + if (source_) + source_->RemoveSink(this); +} + +const webrtc::VideoSendStream::Config& FakeVideoSendStream::GetConfig() const { + return config_; +} + +const webrtc::VideoEncoderConfig& FakeVideoSendStream::GetEncoderConfig() + const { + return encoder_config_; +} + +const std::vector<webrtc::VideoStream>& FakeVideoSendStream::GetVideoStreams() + const { + return video_streams_; +} + +bool FakeVideoSendStream::IsSending() const { + return sending_; +} + +bool FakeVideoSendStream::GetVp8Settings( + webrtc::VideoCodecVP8* settings) const { + if (!codec_settings_set_) { + return false; + } + + *settings = codec_specific_settings_.vp8; + return true; +} + +bool FakeVideoSendStream::GetVp9Settings( + webrtc::VideoCodecVP9* settings) const { + if (!codec_settings_set_) { + return false; + } + + *settings = codec_specific_settings_.vp9; + return true; +} + +bool FakeVideoSendStream::GetH264Settings( + webrtc::VideoCodecH264* settings) const { + if (!codec_settings_set_) { + return false; + } + + *settings = codec_specific_settings_.h264; + return true; +} + +int FakeVideoSendStream::GetNumberOfSwappedFrames() const { + return num_swapped_frames_; +} + +int FakeVideoSendStream::GetLastWidth() const { + return last_frame_->width(); +} + +int FakeVideoSendStream::GetLastHeight() const { + return last_frame_->height(); +} + +int64_t FakeVideoSendStream::GetLastTimestamp() const { + RTC_DCHECK(last_frame_->ntp_time_ms() == 0); + return last_frame_->render_time_ms(); +} + +void FakeVideoSendStream::OnFrame(const webrtc::VideoFrame& frame) { + ++num_swapped_frames_; + if (!last_frame_ || frame.width() != last_frame_->width() || + frame.height() != last_frame_->height() || + frame.rotation() != last_frame_->rotation()) { + video_streams_ = encoder_config_.video_stream_factory->CreateEncoderStreams( + frame.width(), frame.height(), encoder_config_); + } + last_frame_ = frame; +} + +void FakeVideoSendStream::SetStats( + const webrtc::VideoSendStream::Stats& stats) { + stats_ = stats; +} + +webrtc::VideoSendStream::Stats FakeVideoSendStream::GetStats() { + return stats_; +} + +void FakeVideoSendStream::ReconfigureVideoEncoder( + webrtc::VideoEncoderConfig config) { + int width, height; + if (last_frame_) { + width = last_frame_->width(); + height = last_frame_->height(); + } else { + width = height = 0; + } + video_streams_ = + config.video_stream_factory->CreateEncoderStreams(width, height, config); + if (config.encoder_specific_settings != NULL) { + const unsigned char num_temporal_layers = static_cast<unsigned char>( + video_streams_.back().num_temporal_layers.value_or(1)); + if (config_.rtp.payload_name == "VP8") { + config.encoder_specific_settings->FillVideoCodecVp8( + &codec_specific_settings_.vp8); + if (!video_streams_.empty()) { + codec_specific_settings_.vp8.numberOfTemporalLayers = + num_temporal_layers; + } + } else if (config_.rtp.payload_name == "VP9") { + config.encoder_specific_settings->FillVideoCodecVp9( + &codec_specific_settings_.vp9); + if (!video_streams_.empty()) { + codec_specific_settings_.vp9.numberOfTemporalLayers = + num_temporal_layers; + } + } else if (config_.rtp.payload_name == "H264") { + config.encoder_specific_settings->FillVideoCodecH264( + &codec_specific_settings_.h264); + codec_specific_settings_.h264.numberOfTemporalLayers = + num_temporal_layers; + } else { + ADD_FAILURE() << "Unsupported encoder payload: " + << config_.rtp.payload_name; + } + } + codec_settings_set_ = config.encoder_specific_settings != NULL; + encoder_config_ = std::move(config); + ++num_encoder_reconfigurations_; +} + +void FakeVideoSendStream::UpdateActiveSimulcastLayers( + const std::vector<bool> active_layers) { + sending_ = false; + for (const bool active_layer : active_layers) { + if (active_layer) { + sending_ = true; + break; + } + } +} + +void FakeVideoSendStream::Start() { + sending_ = true; +} + +void FakeVideoSendStream::Stop() { + sending_ = false; +} + +void FakeVideoSendStream::SetSource( + rtc::VideoSourceInterface<webrtc::VideoFrame>* source, + const webrtc::DegradationPreference& degradation_preference) { + if (source_) + source_->RemoveSink(this); + source_ = source; + switch (degradation_preference) { + case webrtc::DegradationPreference::MAINTAIN_FRAMERATE: + resolution_scaling_enabled_ = true; + framerate_scaling_enabled_ = false; + break; + case webrtc::DegradationPreference::MAINTAIN_RESOLUTION: + resolution_scaling_enabled_ = false; + framerate_scaling_enabled_ = true; + break; + case webrtc::DegradationPreference::BALANCED: + resolution_scaling_enabled_ = true; + framerate_scaling_enabled_ = true; + break; + case webrtc::DegradationPreference::DISABLED: + resolution_scaling_enabled_ = false; + framerate_scaling_enabled_ = false; + break; + } + if (source) + source->AddOrUpdateSink(this, resolution_scaling_enabled_ + ? sink_wants_ + : rtc::VideoSinkWants()); +} + +void FakeVideoSendStream::InjectVideoSinkWants( + const rtc::VideoSinkWants& wants) { + sink_wants_ = wants; + source_->AddOrUpdateSink(this, wants); +} + +FakeVideoReceiveStream::FakeVideoReceiveStream( + webrtc::VideoReceiveStream::Config config) + : config_(std::move(config)), + receiving_(false), + num_added_secondary_sinks_(0), + num_removed_secondary_sinks_(0) {} + +const webrtc::VideoReceiveStream::Config& FakeVideoReceiveStream::GetConfig() + const { + return config_; +} + +bool FakeVideoReceiveStream::IsReceiving() const { + return receiving_; +} + +void FakeVideoReceiveStream::InjectFrame(const webrtc::VideoFrame& frame) { + config_.renderer->OnFrame(frame); +} + +webrtc::VideoReceiveStream::Stats FakeVideoReceiveStream::GetStats() const { + return stats_; +} + +void FakeVideoReceiveStream::Start() { + receiving_ = true; +} + +void FakeVideoReceiveStream::Stop() { + receiving_ = false; +} + +void FakeVideoReceiveStream::SetStats( + const webrtc::VideoReceiveStream::Stats& stats) { + stats_ = stats; +} + +void FakeVideoReceiveStream::AddSecondarySink( + webrtc::RtpPacketSinkInterface* sink) { + ++num_added_secondary_sinks_; +} + +void FakeVideoReceiveStream::RemoveSecondarySink( + const webrtc::RtpPacketSinkInterface* sink) { + ++num_removed_secondary_sinks_; +} + +int FakeVideoReceiveStream::GetNumAddedSecondarySinks() const { + return num_added_secondary_sinks_; +} + +int FakeVideoReceiveStream::GetNumRemovedSecondarySinks() const { + return num_removed_secondary_sinks_; +} + +FakeFlexfecReceiveStream::FakeFlexfecReceiveStream( + const webrtc::FlexfecReceiveStream::Config& config) + : config_(config) {} + +const webrtc::FlexfecReceiveStream::Config& +FakeFlexfecReceiveStream::GetConfig() const { + return config_; +} + +// TODO(brandtr): Implement when the stats have been designed. +webrtc::FlexfecReceiveStream::Stats FakeFlexfecReceiveStream::GetStats() const { + return webrtc::FlexfecReceiveStream::Stats(); +} + +void FakeFlexfecReceiveStream::OnRtpPacket(const webrtc::RtpPacketReceived&) { + RTC_NOTREACHED() << "Not implemented."; +} + +FakeCall::FakeCall() + : audio_network_state_(webrtc::kNetworkUp), + video_network_state_(webrtc::kNetworkUp), + num_created_send_streams_(0), + num_created_receive_streams_(0) {} + +FakeCall::~FakeCall() { + EXPECT_EQ(0u, video_send_streams_.size()); + EXPECT_EQ(0u, audio_send_streams_.size()); + EXPECT_EQ(0u, video_receive_streams_.size()); + EXPECT_EQ(0u, audio_receive_streams_.size()); +} + +const std::vector<FakeVideoSendStream*>& FakeCall::GetVideoSendStreams() { + return video_send_streams_; +} + +const std::vector<FakeVideoReceiveStream*>& FakeCall::GetVideoReceiveStreams() { + return video_receive_streams_; +} + +const FakeVideoReceiveStream* FakeCall::GetVideoReceiveStream(uint32_t ssrc) { + for (const auto* p : GetVideoReceiveStreams()) { + if (p->GetConfig().rtp.remote_ssrc == ssrc) { + return p; + } + } + return nullptr; +} + +const std::vector<FakeAudioSendStream*>& FakeCall::GetAudioSendStreams() { + return audio_send_streams_; +} + +const FakeAudioSendStream* FakeCall::GetAudioSendStream(uint32_t ssrc) { + for (const auto* p : GetAudioSendStreams()) { + if (p->GetConfig().rtp.ssrc == ssrc) { + return p; + } + } + return nullptr; +} + +const std::vector<FakeAudioReceiveStream*>& FakeCall::GetAudioReceiveStreams() { + return audio_receive_streams_; +} + +const FakeAudioReceiveStream* FakeCall::GetAudioReceiveStream(uint32_t ssrc) { + for (const auto* p : GetAudioReceiveStreams()) { + if (p->GetConfig().rtp.remote_ssrc == ssrc) { + return p; + } + } + return nullptr; +} + +const std::vector<FakeFlexfecReceiveStream*>& +FakeCall::GetFlexfecReceiveStreams() { + return flexfec_receive_streams_; +} + +webrtc::NetworkState FakeCall::GetNetworkState(webrtc::MediaType media) const { + switch (media) { + case webrtc::MediaType::AUDIO: + return audio_network_state_; + case webrtc::MediaType::VIDEO: + return video_network_state_; + case webrtc::MediaType::DATA: + case webrtc::MediaType::ANY: + ADD_FAILURE() << "GetNetworkState called with unknown parameter."; + return webrtc::kNetworkDown; + } + // Even though all the values for the enum class are listed above,the compiler + // will emit a warning as the method may be called with a value outside of the + // valid enum range, unless this case is also handled. + ADD_FAILURE() << "GetNetworkState called with unknown parameter."; + return webrtc::kNetworkDown; +} + +webrtc::AudioSendStream* FakeCall::CreateAudioSendStream( + const webrtc::AudioSendStream::Config& config) { + FakeAudioSendStream* fake_stream = + new FakeAudioSendStream(next_stream_id_++, config); + audio_send_streams_.push_back(fake_stream); + ++num_created_send_streams_; + return fake_stream; +} + +void FakeCall::DestroyAudioSendStream(webrtc::AudioSendStream* send_stream) { + auto it = absl::c_find(audio_send_streams_, + static_cast<FakeAudioSendStream*>(send_stream)); + if (it == audio_send_streams_.end()) { + ADD_FAILURE() << "DestroyAudioSendStream called with unknown parameter."; + } else { + delete *it; + audio_send_streams_.erase(it); + } +} + +webrtc::AudioReceiveStream* FakeCall::CreateAudioReceiveStream( + const webrtc::AudioReceiveStream::Config& config) { + audio_receive_streams_.push_back( + new FakeAudioReceiveStream(next_stream_id_++, config)); + ++num_created_receive_streams_; + return audio_receive_streams_.back(); +} + +void FakeCall::DestroyAudioReceiveStream( + webrtc::AudioReceiveStream* receive_stream) { + auto it = absl::c_find(audio_receive_streams_, + static_cast<FakeAudioReceiveStream*>(receive_stream)); + if (it == audio_receive_streams_.end()) { + ADD_FAILURE() << "DestroyAudioReceiveStream called with unknown parameter."; + } else { + delete *it; + audio_receive_streams_.erase(it); + } +} + +webrtc::VideoSendStream* FakeCall::CreateVideoSendStream( + webrtc::VideoSendStream::Config config, + webrtc::VideoEncoderConfig encoder_config) { + FakeVideoSendStream* fake_stream = + new FakeVideoSendStream(std::move(config), std::move(encoder_config)); + video_send_streams_.push_back(fake_stream); + ++num_created_send_streams_; + return fake_stream; +} + +void FakeCall::DestroyVideoSendStream(webrtc::VideoSendStream* send_stream) { + auto it = absl::c_find(video_send_streams_, + static_cast<FakeVideoSendStream*>(send_stream)); + if (it == video_send_streams_.end()) { + ADD_FAILURE() << "DestroyVideoSendStream called with unknown parameter."; + } else { + delete *it; + video_send_streams_.erase(it); + } +} + +webrtc::VideoReceiveStream* FakeCall::CreateVideoReceiveStream( + webrtc::VideoReceiveStream::Config config) { + video_receive_streams_.push_back( + new FakeVideoReceiveStream(std::move(config))); + ++num_created_receive_streams_; + return video_receive_streams_.back(); +} + +void FakeCall::DestroyVideoReceiveStream( + webrtc::VideoReceiveStream* receive_stream) { + auto it = absl::c_find(video_receive_streams_, + static_cast<FakeVideoReceiveStream*>(receive_stream)); + if (it == video_receive_streams_.end()) { + ADD_FAILURE() << "DestroyVideoReceiveStream called with unknown parameter."; + } else { + delete *it; + video_receive_streams_.erase(it); + } +} + +webrtc::FlexfecReceiveStream* FakeCall::CreateFlexfecReceiveStream( + const webrtc::FlexfecReceiveStream::Config& config) { + FakeFlexfecReceiveStream* fake_stream = new FakeFlexfecReceiveStream(config); + flexfec_receive_streams_.push_back(fake_stream); + ++num_created_receive_streams_; + return fake_stream; +} + +void FakeCall::DestroyFlexfecReceiveStream( + webrtc::FlexfecReceiveStream* receive_stream) { + auto it = + absl::c_find(flexfec_receive_streams_, + static_cast<FakeFlexfecReceiveStream*>(receive_stream)); + if (it == flexfec_receive_streams_.end()) { + ADD_FAILURE() + << "DestroyFlexfecReceiveStream called with unknown parameter."; + } else { + delete *it; + flexfec_receive_streams_.erase(it); + } +} + +webrtc::PacketReceiver* FakeCall::Receiver() { + return this; +} + +FakeCall::DeliveryStatus FakeCall::DeliverPacket(webrtc::MediaType media_type, + rtc::CopyOnWriteBuffer packet, + int64_t packet_time_us) { + EXPECT_GE(packet.size(), 12u); + RTC_DCHECK(media_type == webrtc::MediaType::AUDIO || + media_type == webrtc::MediaType::VIDEO); + + uint32_t ssrc; + if (!GetRtpSsrc(packet.cdata(), packet.size(), &ssrc)) + return DELIVERY_PACKET_ERROR; + + if (media_type == webrtc::MediaType::VIDEO) { + for (auto receiver : video_receive_streams_) { + if (receiver->GetConfig().rtp.remote_ssrc == ssrc) + return DELIVERY_OK; + } + } + if (media_type == webrtc::MediaType::AUDIO) { + for (auto receiver : audio_receive_streams_) { + if (receiver->GetConfig().rtp.remote_ssrc == ssrc) { + receiver->DeliverRtp(packet.cdata(), packet.size(), packet_time_us); + return DELIVERY_OK; + } + } + } + return DELIVERY_UNKNOWN_SSRC; +} + +void FakeCall::SetStats(const webrtc::Call::Stats& stats) { + stats_ = stats; +} + +int FakeCall::GetNumCreatedSendStreams() const { + return num_created_send_streams_; +} + +int FakeCall::GetNumCreatedReceiveStreams() const { + return num_created_receive_streams_; +} + +webrtc::Call::Stats FakeCall::GetStats() const { + return stats_; +} + +void FakeCall::SignalChannelNetworkState(webrtc::MediaType media, + webrtc::NetworkState state) { + switch (media) { + case webrtc::MediaType::AUDIO: + audio_network_state_ = state; + break; + case webrtc::MediaType::VIDEO: + video_network_state_ = state; + break; + case webrtc::MediaType::DATA: + case webrtc::MediaType::ANY: + ADD_FAILURE() + << "SignalChannelNetworkState called with unknown parameter."; + } +} + +void FakeCall::OnAudioTransportOverheadChanged( + int transport_overhead_per_packet) {} + +void FakeCall::OnSentPacket(const rtc::SentPacket& sent_packet) { + last_sent_packet_ = sent_packet; + if (sent_packet.packet_id >= 0) { + last_sent_nonnegative_packet_id_ = sent_packet.packet_id; + } +} + +} // namespace cricket diff --git a/media/engine/fake_webrtc_call.h b/media/engine/fake_webrtc_call.h new file mode 100644 index 0000000000..4404dec5df --- /dev/null +++ b/media/engine/fake_webrtc_call.h @@ -0,0 +1,383 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// This file contains fake implementations, for use in unit tests, of the +// following classes: +// +// webrtc::Call +// webrtc::AudioSendStream +// webrtc::AudioReceiveStream +// webrtc::VideoSendStream +// webrtc::VideoReceiveStream + +#ifndef MEDIA_ENGINE_FAKE_WEBRTC_CALL_H_ +#define MEDIA_ENGINE_FAKE_WEBRTC_CALL_H_ + +#include <memory> +#include <string> +#include <vector> + +#include "api/video/video_frame.h" +#include "call/audio_receive_stream.h" +#include "call/audio_send_stream.h" +#include "call/call.h" +#include "call/flexfec_receive_stream.h" +#include "call/test/mock_rtp_transport_controller_send.h" +#include "call/video_receive_stream.h" +#include "call/video_send_stream.h" +#include "modules/rtp_rtcp/source/rtp_packet_received.h" +#include "rtc_base/buffer.h" + +namespace cricket { +class FakeAudioSendStream final : public webrtc::AudioSendStream { + public: + struct TelephoneEvent { + int payload_type = -1; + int payload_frequency = -1; + int event_code = 0; + int duration_ms = 0; + }; + + explicit FakeAudioSendStream(int id, + const webrtc::AudioSendStream::Config& config); + + int id() const { return id_; } + const webrtc::AudioSendStream::Config& GetConfig() const override; + void SetStats(const webrtc::AudioSendStream::Stats& stats); + TelephoneEvent GetLatestTelephoneEvent() const; + bool IsSending() const { return sending_; } + bool muted() const { return muted_; } + + private: + // webrtc::AudioSendStream implementation. + void Reconfigure(const webrtc::AudioSendStream::Config& config) override; + void Start() override { sending_ = true; } + void Stop() override { sending_ = false; } + void SendAudioData(std::unique_ptr<webrtc::AudioFrame> audio_frame) override { + } + bool SendTelephoneEvent(int payload_type, + int payload_frequency, + int event, + int duration_ms) override; + void SetMuted(bool muted) override; + webrtc::AudioSendStream::Stats GetStats() const override; + webrtc::AudioSendStream::Stats GetStats( + bool has_remote_tracks) const override; + + int id_ = -1; + TelephoneEvent latest_telephone_event_; + webrtc::AudioSendStream::Config config_; + webrtc::AudioSendStream::Stats stats_; + bool sending_ = false; + bool muted_ = false; +}; + +class FakeAudioReceiveStream final : public webrtc::AudioReceiveStream { + public: + explicit FakeAudioReceiveStream( + int id, + const webrtc::AudioReceiveStream::Config& config); + + int id() const { return id_; } + const webrtc::AudioReceiveStream::Config& GetConfig() const; + void SetStats(const webrtc::AudioReceiveStream::Stats& stats); + int received_packets() const { return received_packets_; } + bool VerifyLastPacket(const uint8_t* data, size_t length) const; + const webrtc::AudioSinkInterface* sink() const { return sink_; } + float gain() const { return gain_; } + bool DeliverRtp(const uint8_t* packet, size_t length, int64_t packet_time_us); + bool started() const { return started_; } + int base_mininum_playout_delay_ms() const { + return base_mininum_playout_delay_ms_; + } + + private: + // webrtc::AudioReceiveStream implementation. + void Reconfigure(const webrtc::AudioReceiveStream::Config& config) override; + void Start() override { started_ = true; } + void Stop() override { started_ = false; } + + webrtc::AudioReceiveStream::Stats GetStats() const override; + void SetSink(webrtc::AudioSinkInterface* sink) override; + void SetGain(float gain) override; + bool SetBaseMinimumPlayoutDelayMs(int delay_ms) override { + base_mininum_playout_delay_ms_ = delay_ms; + return true; + } + int GetBaseMinimumPlayoutDelayMs() const override { + return base_mininum_playout_delay_ms_; + } + std::vector<webrtc::RtpSource> GetSources() const override { + return std::vector<webrtc::RtpSource>(); + } + + int id_ = -1; + webrtc::AudioReceiveStream::Config config_; + webrtc::AudioReceiveStream::Stats stats_; + int received_packets_ = 0; + webrtc::AudioSinkInterface* sink_ = nullptr; + float gain_ = 1.0f; + rtc::Buffer last_packet_; + bool started_ = false; + int base_mininum_playout_delay_ms_ = 0; +}; + +class FakeVideoSendStream final + : public webrtc::VideoSendStream, + public rtc::VideoSinkInterface<webrtc::VideoFrame> { + public: + FakeVideoSendStream(webrtc::VideoSendStream::Config config, + webrtc::VideoEncoderConfig encoder_config); + ~FakeVideoSendStream() override; + const webrtc::VideoSendStream::Config& GetConfig() const; + const webrtc::VideoEncoderConfig& GetEncoderConfig() const; + const std::vector<webrtc::VideoStream>& GetVideoStreams() const; + + bool IsSending() const; + bool GetVp8Settings(webrtc::VideoCodecVP8* settings) const; + bool GetVp9Settings(webrtc::VideoCodecVP9* settings) const; + bool GetH264Settings(webrtc::VideoCodecH264* settings) const; + + int GetNumberOfSwappedFrames() const; + int GetLastWidth() const; + int GetLastHeight() const; + int64_t GetLastTimestamp() const; + void SetStats(const webrtc::VideoSendStream::Stats& stats); + int num_encoder_reconfigurations() const { + return num_encoder_reconfigurations_; + } + + bool resolution_scaling_enabled() const { + return resolution_scaling_enabled_; + } + bool framerate_scaling_enabled() const { return framerate_scaling_enabled_; } + void InjectVideoSinkWants(const rtc::VideoSinkWants& wants); + + rtc::VideoSourceInterface<webrtc::VideoFrame>* source() const { + return source_; + } + + private: + // rtc::VideoSinkInterface<VideoFrame> implementation. + void OnFrame(const webrtc::VideoFrame& frame) override; + + // webrtc::VideoSendStream implementation. + void UpdateActiveSimulcastLayers( + const std::vector<bool> active_layers) override; + void Start() override; + void Stop() override; + void SetSource( + rtc::VideoSourceInterface<webrtc::VideoFrame>* source, + const webrtc::DegradationPreference& degradation_preference) override; + webrtc::VideoSendStream::Stats GetStats() override; + void ReconfigureVideoEncoder(webrtc::VideoEncoderConfig config) override; + + bool sending_; + webrtc::VideoSendStream::Config config_; + webrtc::VideoEncoderConfig encoder_config_; + std::vector<webrtc::VideoStream> video_streams_; + rtc::VideoSinkWants sink_wants_; + + bool codec_settings_set_; + union CodecSpecificSettings { + webrtc::VideoCodecVP8 vp8; + webrtc::VideoCodecVP9 vp9; + webrtc::VideoCodecH264 h264; + } codec_specific_settings_; + bool resolution_scaling_enabled_; + bool framerate_scaling_enabled_; + rtc::VideoSourceInterface<webrtc::VideoFrame>* source_; + int num_swapped_frames_; + absl::optional<webrtc::VideoFrame> last_frame_; + webrtc::VideoSendStream::Stats stats_; + int num_encoder_reconfigurations_ = 0; +}; + +class FakeVideoReceiveStream final : public webrtc::VideoReceiveStream { + public: + explicit FakeVideoReceiveStream(webrtc::VideoReceiveStream::Config config); + + const webrtc::VideoReceiveStream::Config& GetConfig() const; + + bool IsReceiving() const; + + void InjectFrame(const webrtc::VideoFrame& frame); + + void SetStats(const webrtc::VideoReceiveStream::Stats& stats); + + void AddSecondarySink(webrtc::RtpPacketSinkInterface* sink) override; + void RemoveSecondarySink(const webrtc::RtpPacketSinkInterface* sink) override; + + int GetNumAddedSecondarySinks() const; + int GetNumRemovedSecondarySinks() const; + + std::vector<webrtc::RtpSource> GetSources() const override { + return std::vector<webrtc::RtpSource>(); + } + + int base_mininum_playout_delay_ms() const { + return base_mininum_playout_delay_ms_; + } + + void SetFrameDecryptor(rtc::scoped_refptr<webrtc::FrameDecryptorInterface> + frame_decryptor) override {} + + void SetDepacketizerToDecoderFrameTransformer( + rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer) + override {} + + RecordingState SetAndGetRecordingState(RecordingState state, + bool generate_key_frame) override { + return RecordingState(); + } + void GenerateKeyFrame() override {} + + private: + // webrtc::VideoReceiveStream implementation. + void Start() override; + void Stop() override; + + webrtc::VideoReceiveStream::Stats GetStats() const override; + + bool SetBaseMinimumPlayoutDelayMs(int delay_ms) override { + base_mininum_playout_delay_ms_ = delay_ms; + return true; + } + + int GetBaseMinimumPlayoutDelayMs() const override { + return base_mininum_playout_delay_ms_; + } + + webrtc::VideoReceiveStream::Config config_; + bool receiving_; + webrtc::VideoReceiveStream::Stats stats_; + + int base_mininum_playout_delay_ms_ = 0; + + int num_added_secondary_sinks_; + int num_removed_secondary_sinks_; +}; + +class FakeFlexfecReceiveStream final : public webrtc::FlexfecReceiveStream { + public: + explicit FakeFlexfecReceiveStream( + const webrtc::FlexfecReceiveStream::Config& config); + + const webrtc::FlexfecReceiveStream::Config& GetConfig() const override; + + private: + webrtc::FlexfecReceiveStream::Stats GetStats() const override; + + void OnRtpPacket(const webrtc::RtpPacketReceived& packet) override; + + webrtc::FlexfecReceiveStream::Config config_; +}; + +class FakeCall final : public webrtc::Call, public webrtc::PacketReceiver { + public: + FakeCall(); + ~FakeCall() override; + + webrtc::MockRtpTransportControllerSend* GetMockTransportControllerSend() { + return &transport_controller_send_; + } + + const std::vector<FakeVideoSendStream*>& GetVideoSendStreams(); + const std::vector<FakeVideoReceiveStream*>& GetVideoReceiveStreams(); + + const std::vector<FakeAudioSendStream*>& GetAudioSendStreams(); + const FakeAudioSendStream* GetAudioSendStream(uint32_t ssrc); + const std::vector<FakeAudioReceiveStream*>& GetAudioReceiveStreams(); + const FakeAudioReceiveStream* GetAudioReceiveStream(uint32_t ssrc); + const FakeVideoReceiveStream* GetVideoReceiveStream(uint32_t ssrc); + + const std::vector<FakeFlexfecReceiveStream*>& GetFlexfecReceiveStreams(); + + rtc::SentPacket last_sent_packet() const { return last_sent_packet_; } + + // This is useful if we care about the last media packet (with id populated) + // but not the last ICE packet (with -1 ID). + int last_sent_nonnegative_packet_id() const { + return last_sent_nonnegative_packet_id_; + } + + webrtc::NetworkState GetNetworkState(webrtc::MediaType media) const; + int GetNumCreatedSendStreams() const; + int GetNumCreatedReceiveStreams() const; + void SetStats(const webrtc::Call::Stats& stats); + + void SetClientBitratePreferences( + const webrtc::BitrateSettings& preferences) override {} + + private: + webrtc::AudioSendStream* CreateAudioSendStream( + const webrtc::AudioSendStream::Config& config) override; + void DestroyAudioSendStream(webrtc::AudioSendStream* send_stream) override; + + webrtc::AudioReceiveStream* CreateAudioReceiveStream( + const webrtc::AudioReceiveStream::Config& config) override; + void DestroyAudioReceiveStream( + webrtc::AudioReceiveStream* receive_stream) override; + + webrtc::VideoSendStream* CreateVideoSendStream( + webrtc::VideoSendStream::Config config, + webrtc::VideoEncoderConfig encoder_config) override; + void DestroyVideoSendStream(webrtc::VideoSendStream* send_stream) override; + + webrtc::VideoReceiveStream* CreateVideoReceiveStream( + webrtc::VideoReceiveStream::Config config) override; + void DestroyVideoReceiveStream( + webrtc::VideoReceiveStream* receive_stream) override; + + webrtc::FlexfecReceiveStream* CreateFlexfecReceiveStream( + const webrtc::FlexfecReceiveStream::Config& config) override; + void DestroyFlexfecReceiveStream( + webrtc::FlexfecReceiveStream* receive_stream) override; + + webrtc::PacketReceiver* Receiver() override; + + DeliveryStatus DeliverPacket(webrtc::MediaType media_type, + rtc::CopyOnWriteBuffer packet, + int64_t packet_time_us) override; + + webrtc::RtpTransportControllerSendInterface* GetTransportControllerSend() + override { + return &transport_controller_send_; + } + + webrtc::Call::Stats GetStats() const override; + + void SignalChannelNetworkState(webrtc::MediaType media, + webrtc::NetworkState state) override; + void OnAudioTransportOverheadChanged( + int transport_overhead_per_packet) override; + void OnSentPacket(const rtc::SentPacket& sent_packet) override; + + ::testing::NiceMock<webrtc::MockRtpTransportControllerSend> + transport_controller_send_; + + webrtc::NetworkState audio_network_state_; + webrtc::NetworkState video_network_state_; + rtc::SentPacket last_sent_packet_; + int last_sent_nonnegative_packet_id_ = -1; + int next_stream_id_ = 665; + webrtc::Call::Stats stats_; + std::vector<FakeVideoSendStream*> video_send_streams_; + std::vector<FakeAudioSendStream*> audio_send_streams_; + std::vector<FakeVideoReceiveStream*> video_receive_streams_; + std::vector<FakeAudioReceiveStream*> audio_receive_streams_; + std::vector<FakeFlexfecReceiveStream*> flexfec_receive_streams_; + + int num_created_send_streams_; + int num_created_receive_streams_; +}; + +} // namespace cricket +#endif // MEDIA_ENGINE_FAKE_WEBRTC_CALL_H_ diff --git a/media/engine/fake_webrtc_video_engine.cc b/media/engine/fake_webrtc_video_engine.cc new file mode 100644 index 0000000000..91f7e53956 --- /dev/null +++ b/media/engine/fake_webrtc_video_engine.cc @@ -0,0 +1,299 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "media/engine/fake_webrtc_video_engine.h" + +#include <algorithm> +#include <memory> + +#include "absl/strings/match.h" +#include "media/base/codec.h" +#include "media/base/media_constants.h" +#include "media/engine/simulcast_encoder_adapter.h" +#include "modules/video_coding/include/video_error_codes.h" +#include "rtc_base/time_utils.h" + +namespace cricket { + +namespace { + +static const int kEventTimeoutMs = 10000; + +bool IsFormatSupported( + const std::vector<webrtc::SdpVideoFormat>& supported_formats, + const webrtc::SdpVideoFormat& format) { + for (const webrtc::SdpVideoFormat& supported_format : supported_formats) { + if (IsSameCodec(format.name, format.parameters, supported_format.name, + supported_format.parameters)) { + return true; + } + } + return false; +} + +} // namespace + +// Decoder. +FakeWebRtcVideoDecoder::FakeWebRtcVideoDecoder( + FakeWebRtcVideoDecoderFactory* factory) + : num_frames_received_(0), factory_(factory) {} + +FakeWebRtcVideoDecoder::~FakeWebRtcVideoDecoder() { + if (factory_) { + factory_->DecoderDestroyed(this); + } +} + +int32_t FakeWebRtcVideoDecoder::InitDecode(const webrtc::VideoCodec*, int32_t) { + return WEBRTC_VIDEO_CODEC_OK; +} + +int32_t FakeWebRtcVideoDecoder::Decode(const webrtc::EncodedImage&, + bool, + int64_t) { + num_frames_received_++; + return WEBRTC_VIDEO_CODEC_OK; +} + +int32_t FakeWebRtcVideoDecoder::RegisterDecodeCompleteCallback( + webrtc::DecodedImageCallback*) { + return WEBRTC_VIDEO_CODEC_OK; +} + +int32_t FakeWebRtcVideoDecoder::Release() { + return WEBRTC_VIDEO_CODEC_OK; +} + +int FakeWebRtcVideoDecoder::GetNumFramesReceived() const { + return num_frames_received_; +} + +// Decoder factory. +FakeWebRtcVideoDecoderFactory::FakeWebRtcVideoDecoderFactory() + : num_created_decoders_(0) {} + +std::vector<webrtc::SdpVideoFormat> +FakeWebRtcVideoDecoderFactory::GetSupportedFormats() const { + std::vector<webrtc::SdpVideoFormat> formats; + + for (const webrtc::SdpVideoFormat& format : supported_codec_formats_) { + // Don't add same codec twice. + if (!IsFormatSupported(formats, format)) + formats.push_back(format); + } + + return formats; +} + +std::unique_ptr<webrtc::VideoDecoder> +FakeWebRtcVideoDecoderFactory::CreateVideoDecoder( + const webrtc::SdpVideoFormat& format) { + if (IsFormatSupported(supported_codec_formats_, format)) { + num_created_decoders_++; + std::unique_ptr<FakeWebRtcVideoDecoder> decoder = + std::make_unique<FakeWebRtcVideoDecoder>(this); + decoders_.push_back(decoder.get()); + return decoder; + } + + return nullptr; +} + +void FakeWebRtcVideoDecoderFactory::DecoderDestroyed( + FakeWebRtcVideoDecoder* decoder) { + decoders_.erase(std::remove(decoders_.begin(), decoders_.end(), decoder), + decoders_.end()); +} + +void FakeWebRtcVideoDecoderFactory::AddSupportedVideoCodecType( + const std::string& name) { + // This is to match the default H264 params of cricket::VideoCodec. + cricket::VideoCodec video_codec(name); + supported_codec_formats_.push_back( + webrtc::SdpVideoFormat(video_codec.name, video_codec.params)); +} + +int FakeWebRtcVideoDecoderFactory::GetNumCreatedDecoders() { + return num_created_decoders_; +} + +const std::vector<FakeWebRtcVideoDecoder*>& +FakeWebRtcVideoDecoderFactory::decoders() { + return decoders_; +} + +// Encoder. +FakeWebRtcVideoEncoder::FakeWebRtcVideoEncoder( + FakeWebRtcVideoEncoderFactory* factory) + : num_frames_encoded_(0), factory_(factory) {} + +FakeWebRtcVideoEncoder::~FakeWebRtcVideoEncoder() { + if (factory_) { + factory_->EncoderDestroyed(this); + } +} + +void FakeWebRtcVideoEncoder::SetFecControllerOverride( + webrtc::FecControllerOverride* fec_controller_override) { + // Ignored. +} + +int32_t FakeWebRtcVideoEncoder::InitEncode( + const webrtc::VideoCodec* codecSettings, + const VideoEncoder::Settings& settings) { + rtc::CritScope lock(&crit_); + codec_settings_ = *codecSettings; + init_encode_event_.Set(); + return WEBRTC_VIDEO_CODEC_OK; +} + +int32_t FakeWebRtcVideoEncoder::Encode( + const webrtc::VideoFrame& inputImage, + const std::vector<webrtc::VideoFrameType>* frame_types) { + rtc::CritScope lock(&crit_); + ++num_frames_encoded_; + init_encode_event_.Set(); + return WEBRTC_VIDEO_CODEC_OK; +} + +int32_t FakeWebRtcVideoEncoder::RegisterEncodeCompleteCallback( + webrtc::EncodedImageCallback* callback) { + return WEBRTC_VIDEO_CODEC_OK; +} + +int32_t FakeWebRtcVideoEncoder::Release() { + return WEBRTC_VIDEO_CODEC_OK; +} + +void FakeWebRtcVideoEncoder::SetRates(const RateControlParameters& parameters) { +} + +webrtc::VideoEncoder::EncoderInfo FakeWebRtcVideoEncoder::GetEncoderInfo() + const { + EncoderInfo info; + info.is_hardware_accelerated = true; + info.has_internal_source = false; + return info; +} + +bool FakeWebRtcVideoEncoder::WaitForInitEncode() { + return init_encode_event_.Wait(kEventTimeoutMs); +} + +webrtc::VideoCodec FakeWebRtcVideoEncoder::GetCodecSettings() { + rtc::CritScope lock(&crit_); + return codec_settings_; +} + +int FakeWebRtcVideoEncoder::GetNumEncodedFrames() { + rtc::CritScope lock(&crit_); + return num_frames_encoded_; +} + +// Video encoder factory. +FakeWebRtcVideoEncoderFactory::FakeWebRtcVideoEncoderFactory() + : num_created_encoders_(0), + encoders_have_internal_sources_(false), + vp8_factory_mode_(false) {} + +std::vector<webrtc::SdpVideoFormat> +FakeWebRtcVideoEncoderFactory::GetSupportedFormats() const { + std::vector<webrtc::SdpVideoFormat> formats; + + for (const webrtc::SdpVideoFormat& format : formats_) { + // Don't add same codec twice. + if (!IsFormatSupported(formats, format)) + formats.push_back(format); + } + + return formats; +} + +std::unique_ptr<webrtc::VideoEncoder> +FakeWebRtcVideoEncoderFactory::CreateVideoEncoder( + const webrtc::SdpVideoFormat& format) { + rtc::CritScope lock(&crit_); + std::unique_ptr<webrtc::VideoEncoder> encoder; + if (IsFormatSupported(formats_, format)) { + if (absl::EqualsIgnoreCase(format.name, kVp8CodecName) && + !vp8_factory_mode_) { + // The simulcast adapter will ask this factory for multiple VP8 + // encoders. Enter vp8_factory_mode so that we now create these encoders + // instead of more adapters. + vp8_factory_mode_ = true; + encoder = std::make_unique<webrtc::SimulcastEncoderAdapter>(this, format); + } else { + num_created_encoders_++; + created_video_encoder_event_.Set(); + encoder = std::make_unique<FakeWebRtcVideoEncoder>(this); + encoders_.push_back(static_cast<FakeWebRtcVideoEncoder*>(encoder.get())); + } + } + return encoder; +} + +webrtc::VideoEncoderFactory::CodecInfo +FakeWebRtcVideoEncoderFactory::QueryVideoEncoder( + const webrtc::SdpVideoFormat& format) const { + webrtc::VideoEncoderFactory::CodecInfo info; + info.has_internal_source = encoders_have_internal_sources_; + info.is_hardware_accelerated = true; + return info; +} + +bool FakeWebRtcVideoEncoderFactory::WaitForCreatedVideoEncoders( + int num_encoders) { + int64_t start_offset_ms = rtc::TimeMillis(); + int64_t wait_time = kEventTimeoutMs; + do { + if (GetNumCreatedEncoders() >= num_encoders) + return true; + wait_time = kEventTimeoutMs - (rtc::TimeMillis() - start_offset_ms); + } while (wait_time > 0 && created_video_encoder_event_.Wait(wait_time)); + return false; +} + +void FakeWebRtcVideoEncoderFactory::EncoderDestroyed( + FakeWebRtcVideoEncoder* encoder) { + rtc::CritScope lock(&crit_); + encoders_.erase(std::remove(encoders_.begin(), encoders_.end(), encoder), + encoders_.end()); +} + +void FakeWebRtcVideoEncoderFactory::set_encoders_have_internal_sources( + bool internal_source) { + encoders_have_internal_sources_ = internal_source; +} + +void FakeWebRtcVideoEncoderFactory::AddSupportedVideoCodec( + const webrtc::SdpVideoFormat& format) { + formats_.push_back(format); +} + +void FakeWebRtcVideoEncoderFactory::AddSupportedVideoCodecType( + const std::string& name) { + // This is to match the default H264 params of cricket::VideoCodec. + cricket::VideoCodec video_codec(name); + formats_.push_back( + webrtc::SdpVideoFormat(video_codec.name, video_codec.params)); +} + +int FakeWebRtcVideoEncoderFactory::GetNumCreatedEncoders() { + rtc::CritScope lock(&crit_); + return num_created_encoders_; +} + +const std::vector<FakeWebRtcVideoEncoder*> +FakeWebRtcVideoEncoderFactory::encoders() { + rtc::CritScope lock(&crit_); + return encoders_; +} + +} // namespace cricket diff --git a/media/engine/fake_webrtc_video_engine.h b/media/engine/fake_webrtc_video_engine.h new file mode 100644 index 0000000000..28dc4fe99b --- /dev/null +++ b/media/engine/fake_webrtc_video_engine.h @@ -0,0 +1,142 @@ +/* + * Copyright (c) 2010 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MEDIA_ENGINE_FAKE_WEBRTC_VIDEO_ENGINE_H_ +#define MEDIA_ENGINE_FAKE_WEBRTC_VIDEO_ENGINE_H_ + +#include <stddef.h> +#include <stdint.h> + +#include <memory> +#include <string> +#include <vector> + +#include "api/fec_controller_override.h" +#include "api/video/encoded_image.h" +#include "api/video/video_bitrate_allocation.h" +#include "api/video/video_frame.h" +#include "api/video_codecs/sdp_video_format.h" +#include "api/video_codecs/video_codec.h" +#include "api/video_codecs/video_decoder.h" +#include "api/video_codecs/video_decoder_factory.h" +#include "api/video_codecs/video_encoder.h" +#include "api/video_codecs/video_encoder_factory.h" +#include "modules/video_coding/include/video_codec_interface.h" +#include "rtc_base/critical_section.h" +#include "rtc_base/event.h" +#include "rtc_base/thread_annotations.h" + +namespace cricket { + +class FakeWebRtcVideoDecoderFactory; +class FakeWebRtcVideoEncoderFactory; + +// Fake class for mocking out webrtc::VideoDecoder +class FakeWebRtcVideoDecoder : public webrtc::VideoDecoder { + public: + explicit FakeWebRtcVideoDecoder(FakeWebRtcVideoDecoderFactory* factory); + ~FakeWebRtcVideoDecoder(); + + int32_t InitDecode(const webrtc::VideoCodec*, int32_t) override; + int32_t Decode(const webrtc::EncodedImage&, bool, int64_t) override; + int32_t RegisterDecodeCompleteCallback( + webrtc::DecodedImageCallback*) override; + int32_t Release() override; + + int GetNumFramesReceived() const; + + private: + int num_frames_received_; + FakeWebRtcVideoDecoderFactory* factory_; +}; + +// Fake class for mocking out webrtc::VideoDecoderFactory. +class FakeWebRtcVideoDecoderFactory : public webrtc::VideoDecoderFactory { + public: + FakeWebRtcVideoDecoderFactory(); + + std::vector<webrtc::SdpVideoFormat> GetSupportedFormats() const override; + std::unique_ptr<webrtc::VideoDecoder> CreateVideoDecoder( + const webrtc::SdpVideoFormat& format) override; + + void DecoderDestroyed(FakeWebRtcVideoDecoder* decoder); + void AddSupportedVideoCodecType(const std::string& name); + int GetNumCreatedDecoders(); + const std::vector<FakeWebRtcVideoDecoder*>& decoders(); + + private: + std::vector<webrtc::SdpVideoFormat> supported_codec_formats_; + std::vector<FakeWebRtcVideoDecoder*> decoders_; + int num_created_decoders_; +}; + +// Fake class for mocking out webrtc::VideoEnoder +class FakeWebRtcVideoEncoder : public webrtc::VideoEncoder { + public: + explicit FakeWebRtcVideoEncoder(FakeWebRtcVideoEncoderFactory* factory); + ~FakeWebRtcVideoEncoder(); + + void SetFecControllerOverride( + webrtc::FecControllerOverride* fec_controller_override) override; + int32_t InitEncode(const webrtc::VideoCodec* codecSettings, + const VideoEncoder::Settings& settings) override; + int32_t Encode( + const webrtc::VideoFrame& inputImage, + const std::vector<webrtc::VideoFrameType>* frame_types) override; + int32_t RegisterEncodeCompleteCallback( + webrtc::EncodedImageCallback* callback) override; + int32_t Release() override; + void SetRates(const RateControlParameters& parameters) override; + webrtc::VideoEncoder::EncoderInfo GetEncoderInfo() const override; + + bool WaitForInitEncode(); + webrtc::VideoCodec GetCodecSettings(); + int GetNumEncodedFrames(); + + private: + rtc::CriticalSection crit_; + rtc::Event init_encode_event_; + int num_frames_encoded_ RTC_GUARDED_BY(crit_); + webrtc::VideoCodec codec_settings_ RTC_GUARDED_BY(crit_); + FakeWebRtcVideoEncoderFactory* factory_; +}; + +// Fake class for mocking out webrtc::VideoEncoderFactory. +class FakeWebRtcVideoEncoderFactory : public webrtc::VideoEncoderFactory { + public: + FakeWebRtcVideoEncoderFactory(); + + std::vector<webrtc::SdpVideoFormat> GetSupportedFormats() const override; + std::unique_ptr<webrtc::VideoEncoder> CreateVideoEncoder( + const webrtc::SdpVideoFormat& format) override; + CodecInfo QueryVideoEncoder( + const webrtc::SdpVideoFormat& format) const override; + + bool WaitForCreatedVideoEncoders(int num_encoders); + void EncoderDestroyed(FakeWebRtcVideoEncoder* encoder); + void set_encoders_have_internal_sources(bool internal_source); + void AddSupportedVideoCodec(const webrtc::SdpVideoFormat& format); + void AddSupportedVideoCodecType(const std::string& name); + int GetNumCreatedEncoders(); + const std::vector<FakeWebRtcVideoEncoder*> encoders(); + + private: + rtc::CriticalSection crit_; + rtc::Event created_video_encoder_event_; + std::vector<webrtc::SdpVideoFormat> formats_; + std::vector<FakeWebRtcVideoEncoder*> encoders_ RTC_GUARDED_BY(crit_); + int num_created_encoders_ RTC_GUARDED_BY(crit_); + bool encoders_have_internal_sources_; + bool vp8_factory_mode_; +}; + +} // namespace cricket + +#endif // MEDIA_ENGINE_FAKE_WEBRTC_VIDEO_ENGINE_H_ diff --git a/media/engine/internal_decoder_factory.cc b/media/engine/internal_decoder_factory.cc new file mode 100644 index 0000000000..e68bb369b5 --- /dev/null +++ b/media/engine/internal_decoder_factory.cc @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "media/engine/internal_decoder_factory.h" + +#include "absl/strings/match.h" +#include "api/video_codecs/sdp_video_format.h" +#include "media/base/codec.h" +#include "media/base/media_constants.h" +#include "modules/video_coding/codecs/av1/libaom_av1_decoder.h" +#include "modules/video_coding/codecs/h264/include/h264.h" +#include "modules/video_coding/codecs/vp8/include/vp8.h" +#include "modules/video_coding/codecs/vp9/include/vp9.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" + +namespace webrtc { + +namespace { + +bool IsFormatSupported( + const std::vector<webrtc::SdpVideoFormat>& supported_formats, + const webrtc::SdpVideoFormat& format) { + for (const webrtc::SdpVideoFormat& supported_format : supported_formats) { + if (cricket::IsSameCodec(format.name, format.parameters, + supported_format.name, + supported_format.parameters)) { + return true; + } + } + return false; +} + +} // namespace + +std::vector<SdpVideoFormat> InternalDecoderFactory::GetSupportedFormats() + const { + std::vector<SdpVideoFormat> formats; + formats.push_back(SdpVideoFormat(cricket::kVp8CodecName)); + for (const SdpVideoFormat& format : SupportedVP9Codecs()) + formats.push_back(format); + for (const SdpVideoFormat& h264_format : SupportedH264Codecs()) + formats.push_back(h264_format); + if (kIsLibaomAv1DecoderSupported) + formats.push_back(SdpVideoFormat(cricket::kAv1CodecName)); + return formats; +} + +std::unique_ptr<VideoDecoder> InternalDecoderFactory::CreateVideoDecoder( + const SdpVideoFormat& format) { + if (!IsFormatSupported(GetSupportedFormats(), format)) { + RTC_LOG(LS_ERROR) << "Trying to create decoder for unsupported format"; + return nullptr; + } + + if (absl::EqualsIgnoreCase(format.name, cricket::kVp8CodecName)) + return VP8Decoder::Create(); + if (absl::EqualsIgnoreCase(format.name, cricket::kVp9CodecName)) + return VP9Decoder::Create(); + if (absl::EqualsIgnoreCase(format.name, cricket::kH264CodecName)) + return H264Decoder::Create(); + if (kIsLibaomAv1DecoderSupported && + absl::EqualsIgnoreCase(format.name, cricket::kAv1CodecName)) + return CreateLibaomAv1Decoder(); + + RTC_NOTREACHED(); + return nullptr; +} + +} // namespace webrtc diff --git a/media/engine/internal_decoder_factory.h b/media/engine/internal_decoder_factory.h new file mode 100644 index 0000000000..2a580dea0b --- /dev/null +++ b/media/engine/internal_decoder_factory.h @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MEDIA_ENGINE_INTERNAL_DECODER_FACTORY_H_ +#define MEDIA_ENGINE_INTERNAL_DECODER_FACTORY_H_ + +#include <memory> +#include <vector> + +#include "api/video_codecs/sdp_video_format.h" +#include "api/video_codecs/video_decoder.h" +#include "api/video_codecs/video_decoder_factory.h" +#include "rtc_base/system/rtc_export.h" + +namespace webrtc { + +class RTC_EXPORT InternalDecoderFactory : public VideoDecoderFactory { + public: + std::vector<SdpVideoFormat> GetSupportedFormats() const override; + std::unique_ptr<VideoDecoder> CreateVideoDecoder( + const SdpVideoFormat& format) override; +}; + +} // namespace webrtc + +#endif // MEDIA_ENGINE_INTERNAL_DECODER_FACTORY_H_ diff --git a/media/engine/internal_decoder_factory_unittest.cc b/media/engine/internal_decoder_factory_unittest.cc new file mode 100644 index 0000000000..705933d439 --- /dev/null +++ b/media/engine/internal_decoder_factory_unittest.cc @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "media/engine/internal_decoder_factory.h" + +#include "api/video_codecs/sdp_video_format.h" +#include "api/video_codecs/video_decoder.h" +#include "media/base/media_constants.h" +#include "modules/video_coding/codecs/av1/libaom_av1_decoder.h" +#include "test/gmock.h" +#include "test/gtest.h" + +namespace webrtc { + +using ::testing::Contains; +using ::testing::Field; +using ::testing::Not; + +TEST(InternalDecoderFactory, TestVP8) { + InternalDecoderFactory factory; + std::unique_ptr<VideoDecoder> decoder = + factory.CreateVideoDecoder(SdpVideoFormat(cricket::kVp8CodecName)); + EXPECT_TRUE(decoder); +} + +TEST(InternalDecoderFactory, Av1) { + InternalDecoderFactory factory; + if (kIsLibaomAv1DecoderSupported) { + EXPECT_THAT(factory.GetSupportedFormats(), + Contains(Field(&SdpVideoFormat::name, "AV1X"))); + EXPECT_TRUE(factory.CreateVideoDecoder(SdpVideoFormat("AV1X"))); + } else { + EXPECT_THAT(factory.GetSupportedFormats(), + Not(Contains(Field(&SdpVideoFormat::name, "AV1X")))); + } +} + +} // namespace webrtc diff --git a/media/engine/internal_encoder_factory.cc b/media/engine/internal_encoder_factory.cc new file mode 100644 index 0000000000..aabb810283 --- /dev/null +++ b/media/engine/internal_encoder_factory.cc @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "media/engine/internal_encoder_factory.h" + +#include <string> + +#include "absl/strings/match.h" +#include "api/video_codecs/sdp_video_format.h" +#include "media/base/codec.h" +#include "media/base/media_constants.h" +#include "modules/video_coding/codecs/av1/libaom_av1_encoder.h" +#include "modules/video_coding/codecs/h264/include/h264.h" +#include "modules/video_coding/codecs/vp8/include/vp8.h" +#include "modules/video_coding/codecs/vp9/include/vp9.h" +#include "rtc_base/logging.h" + +namespace webrtc { + +std::vector<SdpVideoFormat> InternalEncoderFactory::SupportedFormats() { + std::vector<SdpVideoFormat> supported_codecs; + supported_codecs.push_back(SdpVideoFormat(cricket::kVp8CodecName)); + for (const webrtc::SdpVideoFormat& format : webrtc::SupportedVP9Codecs()) + supported_codecs.push_back(format); + for (const webrtc::SdpVideoFormat& format : webrtc::SupportedH264Codecs()) + supported_codecs.push_back(format); + if (kIsLibaomAv1EncoderSupported) + supported_codecs.push_back(SdpVideoFormat(cricket::kAv1CodecName)); + return supported_codecs; +} + +std::vector<SdpVideoFormat> InternalEncoderFactory::GetSupportedFormats() + const { + return SupportedFormats(); +} + +VideoEncoderFactory::CodecInfo InternalEncoderFactory::QueryVideoEncoder( + const SdpVideoFormat& format) const { + CodecInfo info; + info.is_hardware_accelerated = false; + info.has_internal_source = false; + return info; +} + +std::unique_ptr<VideoEncoder> InternalEncoderFactory::CreateVideoEncoder( + const SdpVideoFormat& format) { + if (absl::EqualsIgnoreCase(format.name, cricket::kVp8CodecName)) + return VP8Encoder::Create(); + if (absl::EqualsIgnoreCase(format.name, cricket::kVp9CodecName)) + return VP9Encoder::Create(cricket::VideoCodec(format)); + if (absl::EqualsIgnoreCase(format.name, cricket::kH264CodecName)) + return H264Encoder::Create(cricket::VideoCodec(format)); + if (kIsLibaomAv1EncoderSupported && + absl::EqualsIgnoreCase(format.name, cricket::kAv1CodecName)) + return CreateLibaomAv1Encoder(); + RTC_LOG(LS_ERROR) << "Trying to created encoder of unsupported format " + << format.name; + return nullptr; +} + +} // namespace webrtc diff --git a/media/engine/internal_encoder_factory.h b/media/engine/internal_encoder_factory.h new file mode 100644 index 0000000000..c15d1790f3 --- /dev/null +++ b/media/engine/internal_encoder_factory.h @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MEDIA_ENGINE_INTERNAL_ENCODER_FACTORY_H_ +#define MEDIA_ENGINE_INTERNAL_ENCODER_FACTORY_H_ + +#include <memory> +#include <vector> + +#include "api/video_codecs/sdp_video_format.h" +#include "api/video_codecs/video_encoder.h" +#include "api/video_codecs/video_encoder_factory.h" +#include "rtc_base/system/rtc_export.h" + +namespace webrtc { + +class RTC_EXPORT InternalEncoderFactory : public VideoEncoderFactory { + public: + static std::vector<SdpVideoFormat> SupportedFormats(); + std::vector<SdpVideoFormat> GetSupportedFormats() const override; + + CodecInfo QueryVideoEncoder(const SdpVideoFormat& format) const override; + + std::unique_ptr<VideoEncoder> CreateVideoEncoder( + const SdpVideoFormat& format) override; +}; + +} // namespace webrtc + +#endif // MEDIA_ENGINE_INTERNAL_ENCODER_FACTORY_H_ diff --git a/media/engine/multiplex_codec_factory.cc b/media/engine/multiplex_codec_factory.cc new file mode 100644 index 0000000000..5a220d528a --- /dev/null +++ b/media/engine/multiplex_codec_factory.cc @@ -0,0 +1,121 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "media/engine/multiplex_codec_factory.h" + +#include <map> +#include <string> +#include <utility> + +#include "absl/strings/match.h" +#include "api/video_codecs/sdp_video_format.h" +#include "media/base/codec.h" +#include "media/base/media_constants.h" +#include "modules/video_coding/codecs/multiplex/include/multiplex_decoder_adapter.h" +#include "modules/video_coding/codecs/multiplex/include/multiplex_encoder_adapter.h" +#include "rtc_base/logging.h" + +namespace { + +bool IsMultiplexCodec(const cricket::VideoCodec& codec) { + return absl::EqualsIgnoreCase(codec.name.c_str(), + cricket::kMultiplexCodecName); +} + +} // anonymous namespace + +namespace webrtc { + +constexpr const char* kMultiplexAssociatedCodecName = cricket::kVp9CodecName; + +MultiplexEncoderFactory::MultiplexEncoderFactory( + std::unique_ptr<VideoEncoderFactory> factory, + bool supports_augmenting_data) + : factory_(std::move(factory)), + supports_augmenting_data_(supports_augmenting_data) {} + +std::vector<SdpVideoFormat> MultiplexEncoderFactory::GetSupportedFormats() + const { + std::vector<SdpVideoFormat> formats = factory_->GetSupportedFormats(); + for (const auto& format : formats) { + if (absl::EqualsIgnoreCase(format.name, kMultiplexAssociatedCodecName)) { + SdpVideoFormat multiplex_format = format; + multiplex_format.parameters[cricket::kCodecParamAssociatedCodecName] = + format.name; + multiplex_format.name = cricket::kMultiplexCodecName; + formats.push_back(multiplex_format); + break; + } + } + return formats; +} + +VideoEncoderFactory::CodecInfo MultiplexEncoderFactory::QueryVideoEncoder( + const SdpVideoFormat& format) const { + if (!IsMultiplexCodec(cricket::VideoCodec(format))) + return factory_->QueryVideoEncoder(format); + return factory_->QueryVideoEncoder( + SdpVideoFormat(kMultiplexAssociatedCodecName)); +} + +std::unique_ptr<VideoEncoder> MultiplexEncoderFactory::CreateVideoEncoder( + const SdpVideoFormat& format) { + if (!IsMultiplexCodec(cricket::VideoCodec(format))) + return factory_->CreateVideoEncoder(format); + const auto& it = + format.parameters.find(cricket::kCodecParamAssociatedCodecName); + if (it == format.parameters.end()) { + RTC_LOG(LS_ERROR) << "No assicated codec for multiplex."; + return nullptr; + } + SdpVideoFormat associated_format = format; + associated_format.name = it->second; + return std::unique_ptr<VideoEncoder>(new MultiplexEncoderAdapter( + factory_.get(), associated_format, supports_augmenting_data_)); +} + +MultiplexDecoderFactory::MultiplexDecoderFactory( + std::unique_ptr<VideoDecoderFactory> factory, + bool supports_augmenting_data) + : factory_(std::move(factory)), + supports_augmenting_data_(supports_augmenting_data) {} + +std::vector<SdpVideoFormat> MultiplexDecoderFactory::GetSupportedFormats() + const { + std::vector<SdpVideoFormat> formats = factory_->GetSupportedFormats(); + for (const auto& format : formats) { + if (absl::EqualsIgnoreCase(format.name, kMultiplexAssociatedCodecName)) { + SdpVideoFormat multiplex_format = format; + multiplex_format.parameters[cricket::kCodecParamAssociatedCodecName] = + format.name; + multiplex_format.name = cricket::kMultiplexCodecName; + formats.push_back(multiplex_format); + } + } + return formats; +} + +std::unique_ptr<VideoDecoder> MultiplexDecoderFactory::CreateVideoDecoder( + const SdpVideoFormat& format) { + if (!IsMultiplexCodec(cricket::VideoCodec(format))) + return factory_->CreateVideoDecoder(format); + const auto& it = + format.parameters.find(cricket::kCodecParamAssociatedCodecName); + if (it == format.parameters.end()) { + RTC_LOG(LS_ERROR) << "No assicated codec for multiplex."; + return nullptr; + } + SdpVideoFormat associated_format = format; + associated_format.name = it->second; + return std::unique_ptr<VideoDecoder>(new MultiplexDecoderAdapter( + factory_.get(), associated_format, supports_augmenting_data_)); +} + +} // namespace webrtc diff --git a/media/engine/multiplex_codec_factory.h b/media/engine/multiplex_codec_factory.h new file mode 100644 index 0000000000..b8a15e21f6 --- /dev/null +++ b/media/engine/multiplex_codec_factory.h @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MEDIA_ENGINE_MULTIPLEX_CODEC_FACTORY_H_ +#define MEDIA_ENGINE_MULTIPLEX_CODEC_FACTORY_H_ + +#include <memory> +#include <vector> + +#include "api/video_codecs/sdp_video_format.h" +#include "api/video_codecs/video_decoder.h" +#include "api/video_codecs/video_decoder_factory.h" +#include "api/video_codecs/video_encoder.h" +#include "api/video_codecs/video_encoder_factory.h" +#include "rtc_base/system/rtc_export.h" + +namespace webrtc { +// Multiplex codec is a completely modular/optional codec that allows users to +// send more than a frame's opaque content(RGB/YUV) over video channels. +// - Allows sending Alpha channel over the wire iff input is +// I420ABufferInterface. Users can expect to receive I420ABufferInterface as the +// decoded video frame buffer. I420A data is split into YUV/AXX portions, +// encoded/decoded seperately and bitstreams are concatanated. +// - Allows sending augmenting data over the wire attached to the frame. This +// attached data portion is not encoded in any way and sent as it is. Users can +// input AugmentedVideoFrameBuffer and can expect the same interface as the +// decoded video frame buffer. +// - Showcases an example of how to add a custom codec in webrtc video channel. +// How to use it end-to-end: +// - Wrap your existing VideoEncoderFactory implemention with +// MultiplexEncoderFactory and VideoDecoderFactory implemention with +// MultiplexDecoderFactory below. For actual coding, multiplex creates encoder +// and decoder instance(s) using these factories. +// - Use Multiplex*coderFactory classes in CreatePeerConnectionFactory() calls. +// - Select "multiplex" codec in SDP negotiation. +class RTC_EXPORT MultiplexEncoderFactory : public VideoEncoderFactory { + public: + // |supports_augmenting_data| defines if the encoder would support augmenting + // data. If set, the encoder expects to receive video frame buffers of type + // AugmentedVideoFrameBuffer. + MultiplexEncoderFactory(std::unique_ptr<VideoEncoderFactory> factory, + bool supports_augmenting_data = false); + + std::vector<SdpVideoFormat> GetSupportedFormats() const override; + CodecInfo QueryVideoEncoder(const SdpVideoFormat& format) const override; + std::unique_ptr<VideoEncoder> CreateVideoEncoder( + const SdpVideoFormat& format) override; + + private: + std::unique_ptr<VideoEncoderFactory> factory_; + const bool supports_augmenting_data_; +}; + +class RTC_EXPORT MultiplexDecoderFactory : public VideoDecoderFactory { + public: + // |supports_augmenting_data| defines if the decoder would support augmenting + // data. If set, the decoder is expected to output video frame buffers of type + // AugmentedVideoFrameBuffer. + MultiplexDecoderFactory(std::unique_ptr<VideoDecoderFactory> factory, + bool supports_augmenting_data = false); + + std::vector<SdpVideoFormat> GetSupportedFormats() const override; + std::unique_ptr<VideoDecoder> CreateVideoDecoder( + const SdpVideoFormat& format) override; + + private: + std::unique_ptr<VideoDecoderFactory> factory_; + const bool supports_augmenting_data_; +}; + +} // namespace webrtc + +#endif // MEDIA_ENGINE_MULTIPLEX_CODEC_FACTORY_H_ diff --git a/media/engine/multiplex_codec_factory_unittest.cc b/media/engine/multiplex_codec_factory_unittest.cc new file mode 100644 index 0000000000..1cde2f37d8 --- /dev/null +++ b/media/engine/multiplex_codec_factory_unittest.cc @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "media/engine/multiplex_codec_factory.h" + +#include <utility> + +#include "api/video_codecs/sdp_video_format.h" +#include "api/video_codecs/video_decoder.h" +#include "api/video_codecs/video_encoder.h" +#include "media/base/media_constants.h" +#include "media/engine/internal_decoder_factory.h" +#include "media/engine/internal_encoder_factory.h" +#include "test/gtest.h" + +namespace webrtc { + +TEST(MultiplexDecoderFactory, CreateVideoDecoder) { + std::unique_ptr<VideoDecoderFactory> internal_factory( + new InternalDecoderFactory()); + MultiplexDecoderFactory factory(std::move(internal_factory)); + std::unique_ptr<VideoDecoder> decoder = + factory.CreateVideoDecoder(SdpVideoFormat( + cricket::kMultiplexCodecName, + {{cricket::kCodecParamAssociatedCodecName, cricket::kVp9CodecName}})); + EXPECT_TRUE(decoder); +} + +TEST(MultiplexEncoderFactory, CreateVideoEncoder) { + std::unique_ptr<VideoEncoderFactory> internal_factory( + new InternalEncoderFactory()); + MultiplexEncoderFactory factory(std::move(internal_factory)); + std::unique_ptr<VideoEncoder> encoder = + factory.CreateVideoEncoder(SdpVideoFormat( + cricket::kMultiplexCodecName, + {{cricket::kCodecParamAssociatedCodecName, cricket::kVp9CodecName}})); + EXPECT_TRUE(encoder); +} + +} // namespace webrtc diff --git a/media/engine/null_webrtc_video_engine.h b/media/engine/null_webrtc_video_engine.h new file mode 100644 index 0000000000..a914af954b --- /dev/null +++ b/media/engine/null_webrtc_video_engine.h @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MEDIA_ENGINE_NULL_WEBRTC_VIDEO_ENGINE_H_ +#define MEDIA_ENGINE_NULL_WEBRTC_VIDEO_ENGINE_H_ + +#include <vector> + +#include "media/base/media_channel.h" +#include "media/base/media_engine.h" + +namespace webrtc { + +class Call; + +} // namespace webrtc + +namespace cricket { + +class VideoMediaChannel; + +// Video engine implementation that does nothing and can be used in +// CompositeMediaEngine. +class NullWebRtcVideoEngine : public VideoEngineInterface { + public: + std::vector<VideoCodec> send_codecs() const override { + return std::vector<VideoCodec>(); + } + + std::vector<VideoCodec> recv_codecs() const override { + return std::vector<VideoCodec>(); + } + + std::vector<webrtc::RtpHeaderExtensionCapability> GetRtpHeaderExtensions() + const override { + return {}; + } + + VideoMediaChannel* CreateMediaChannel( + webrtc::Call* call, + const MediaConfig& config, + const VideoOptions& options, + const webrtc::CryptoOptions& crypto_options, + webrtc::VideoBitrateAllocatorFactory* video_bitrate_allocator_factory) + override { + return nullptr; + } +}; + +} // namespace cricket + +#endif // MEDIA_ENGINE_NULL_WEBRTC_VIDEO_ENGINE_H_ diff --git a/media/engine/null_webrtc_video_engine_unittest.cc b/media/engine/null_webrtc_video_engine_unittest.cc new file mode 100644 index 0000000000..832bf8ad1a --- /dev/null +++ b/media/engine/null_webrtc_video_engine_unittest.cc @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "media/engine/null_webrtc_video_engine.h" + +#include <memory> +#include <utility> + +#include "api/task_queue/default_task_queue_factory.h" +#include "api/task_queue/task_queue_factory.h" +#include "media/engine/webrtc_voice_engine.h" +#include "modules/audio_device/include/mock_audio_device.h" +#include "modules/audio_processing/include/audio_processing.h" +#include "test/gtest.h" +#include "test/mock_audio_decoder_factory.h" +#include "test/mock_audio_encoder_factory.h" + +namespace cricket { + +// Simple test to check if NullWebRtcVideoEngine implements the methods +// required by CompositeMediaEngine. +TEST(NullWebRtcVideoEngineTest, CheckInterface) { + std::unique_ptr<webrtc::TaskQueueFactory> task_queue_factory = + webrtc::CreateDefaultTaskQueueFactory(); + rtc::scoped_refptr<webrtc::test::MockAudioDeviceModule> adm = + webrtc::test::MockAudioDeviceModule::CreateNice(); + auto audio_engine = std::make_unique<WebRtcVoiceEngine>( + task_queue_factory.get(), adm, + webrtc::MockAudioEncoderFactory::CreateUnusedFactory(), + webrtc::MockAudioDecoderFactory::CreateUnusedFactory(), nullptr, + webrtc::AudioProcessingBuilder().Create()); + + CompositeMediaEngine engine(std::move(audio_engine), + std::make_unique<NullWebRtcVideoEngine>()); + + EXPECT_TRUE(engine.Init()); +} + +} // namespace cricket diff --git a/media/engine/payload_type_mapper.cc b/media/engine/payload_type_mapper.cc new file mode 100644 index 0000000000..fcacd44883 --- /dev/null +++ b/media/engine/payload_type_mapper.cc @@ -0,0 +1,154 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "media/engine/payload_type_mapper.h" + +#include <utility> + +#include "absl/strings/ascii.h" +#include "api/audio_codecs/audio_format.h" +#include "media/base/media_constants.h" + +namespace cricket { + +webrtc::SdpAudioFormat AudioCodecToSdpAudioFormat(const AudioCodec& ac) { + return webrtc::SdpAudioFormat(ac.name, ac.clockrate, ac.channels, ac.params); +} + +PayloadTypeMapper::PayloadTypeMapper() + // RFC 3551 reserves payload type numbers in the range 96-127 exclusively + // for dynamic assignment. Once those are used up, it is recommended that + // payload types unassigned by the RFC are used for dynamic payload type + // mapping, before any static payload ids. At this point, we only support + // mapping within the exclusive range. + : next_unused_payload_type_(96), + max_payload_type_(127), + mappings_( + {// Static payload type assignments according to RFC 3551. + {{"PCMU", 8000, 1}, 0}, + {{"GSM", 8000, 1}, 3}, + {{"G723", 8000, 1}, 4}, + {{"DVI4", 8000, 1}, 5}, + {{"DVI4", 16000, 1}, 6}, + {{"LPC", 8000, 1}, 7}, + {{"PCMA", 8000, 1}, 8}, + {{"G722", 8000, 1}, 9}, + {{"L16", 44100, 2}, 10}, + {{"L16", 44100, 1}, 11}, + {{"QCELP", 8000, 1}, 12}, + {{"CN", 8000, 1}, 13}, + // RFC 4566 is a bit ambiguous on the contents of the "encoding + // parameters" field, which, for audio, encodes the number of + // channels. It is "optional and may be omitted if the number of + // channels is one". Does that necessarily imply that an omitted + // encoding parameter means one channel? Since RFC 3551 doesn't + // specify a value for this parameter for MPA, I've included both 0 + // and 1 here, to increase the chances it will be correctly used if + // someone implements an MPEG audio encoder/decoder. + {{"MPA", 90000, 0}, 14}, + {{"MPA", 90000, 1}, 14}, + {{"G728", 8000, 1}, 15}, + {{"DVI4", 11025, 1}, 16}, + {{"DVI4", 22050, 1}, 17}, + {{"G729", 8000, 1}, 18}, + + // Payload type assignments currently used by WebRTC. + // Includes data to reduce collisions (and thus reassignments) + {{kGoogleRtpDataCodecName, 0, 0}, kGoogleRtpDataCodecPlType}, + {{kIlbcCodecName, 8000, 1}, 102}, + {{kIsacCodecName, 16000, 1}, 103}, + {{kIsacCodecName, 32000, 1}, 104}, + {{kCnCodecName, 16000, 1}, 105}, + {{kCnCodecName, 32000, 1}, 106}, + {{kGoogleSctpDataCodecName, 0, 0}, kGoogleSctpDataCodecPlType}, + {{kOpusCodecName, + 48000, + 2, + {{"minptime", "10"}, {"useinbandfec", "1"}}}, + 111}, + // TODO(solenberg): Remove the hard coded 16k,32k,48k DTMF once we + // assign payload types dynamically for send side as well. + {{kDtmfCodecName, 48000, 1}, 110}, + {{kDtmfCodecName, 32000, 1}, 112}, + {{kDtmfCodecName, 16000, 1}, 113}, + {{kDtmfCodecName, 8000, 1}, 126}}) { + // TODO(ossu): Try to keep this as change-proof as possible until we're able + // to remove the payload type constants from everywhere in the code. + for (const auto& mapping : mappings_) { + used_payload_types_.insert(mapping.second); + } +} + +PayloadTypeMapper::~PayloadTypeMapper() = default; + +absl::optional<int> PayloadTypeMapper::GetMappingFor( + const webrtc::SdpAudioFormat& format) { + auto iter = mappings_.find(format); + if (iter != mappings_.end()) + return iter->second; + + for (; next_unused_payload_type_ <= max_payload_type_; + ++next_unused_payload_type_) { + int payload_type = next_unused_payload_type_; + if (used_payload_types_.find(payload_type) == used_payload_types_.end()) { + used_payload_types_.insert(payload_type); + mappings_[format] = payload_type; + ++next_unused_payload_type_; + return payload_type; + } + } + + return absl::nullopt; +} + +absl::optional<int> PayloadTypeMapper::FindMappingFor( + const webrtc::SdpAudioFormat& format) const { + auto iter = mappings_.find(format); + if (iter != mappings_.end()) + return iter->second; + + return absl::nullopt; +} + +absl::optional<AudioCodec> PayloadTypeMapper::ToAudioCodec( + const webrtc::SdpAudioFormat& format) { + // TODO(ossu): We can safely set bitrate to zero here, since that field is + // not presented in the SDP. It is used to ferry around some target bitrate + // values for certain codecs (ISAC and Opus) and in ways it really + // shouldn't. It should be removed once we no longer use CodecInsts in the + // ACM or NetEq. + auto opt_payload_type = GetMappingFor(format); + if (opt_payload_type) { + AudioCodec codec(*opt_payload_type, format.name, format.clockrate_hz, 0, + format.num_channels); + codec.params = format.parameters; + return std::move(codec); + } + + return absl::nullopt; +} + +bool PayloadTypeMapper::SdpAudioFormatOrdering::operator()( + const webrtc::SdpAudioFormat& a, + const webrtc::SdpAudioFormat& b) const { + if (a.clockrate_hz == b.clockrate_hz) { + if (a.num_channels == b.num_channels) { + int name_cmp = + absl::AsciiStrToLower(a.name).compare(absl::AsciiStrToLower(b.name)); + if (name_cmp == 0) + return a.parameters < b.parameters; + return name_cmp < 0; + } + return a.num_channels < b.num_channels; + } + return a.clockrate_hz < b.clockrate_hz; +} + +} // namespace cricket diff --git a/media/engine/payload_type_mapper.h b/media/engine/payload_type_mapper.h new file mode 100644 index 0000000000..d8ab4a4261 --- /dev/null +++ b/media/engine/payload_type_mapper.h @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MEDIA_ENGINE_PAYLOAD_TYPE_MAPPER_H_ +#define MEDIA_ENGINE_PAYLOAD_TYPE_MAPPER_H_ + +#include <map> +#include <set> + +#include "absl/types/optional.h" +#include "api/audio_codecs/audio_format.h" +#include "media/base/codec.h" + +namespace cricket { + +webrtc::SdpAudioFormat AudioCodecToSdpAudioFormat(const AudioCodec& ac); + +class PayloadTypeMapper { + public: + PayloadTypeMapper(); + ~PayloadTypeMapper(); + + // Finds the current payload type for |format| or assigns a new one, if no + // current mapping exists. Will return an empty value if it was unable to + // create a mapping, i.e. if all dynamic payload type ids have been used up. + absl::optional<int> GetMappingFor(const webrtc::SdpAudioFormat& format); + + // Finds the current payload type for |format|, if any. Returns an empty value + // if no payload type mapping exists for the format. + absl::optional<int> FindMappingFor( + const webrtc::SdpAudioFormat& format) const; + + // Like GetMappingFor, but fills in an AudioCodec structure with the necessary + // information instead. + absl::optional<AudioCodec> ToAudioCodec(const webrtc::SdpAudioFormat& format); + + private: + struct SdpAudioFormatOrdering { + bool operator()(const webrtc::SdpAudioFormat& a, + const webrtc::SdpAudioFormat& b) const; + }; + + int next_unused_payload_type_; + int max_payload_type_; + std::map<webrtc::SdpAudioFormat, int, SdpAudioFormatOrdering> mappings_; + std::set<int> used_payload_types_; +}; + +} // namespace cricket +#endif // MEDIA_ENGINE_PAYLOAD_TYPE_MAPPER_H_ diff --git a/media/engine/payload_type_mapper_unittest.cc b/media/engine/payload_type_mapper_unittest.cc new file mode 100644 index 0000000000..c8b2234c25 --- /dev/null +++ b/media/engine/payload_type_mapper_unittest.cc @@ -0,0 +1,144 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "media/engine/payload_type_mapper.h" + +#include <set> +#include <string> + +#include "absl/strings/string_view.h" +#include "media/base/media_constants.h" +#include "test/gtest.h" + +namespace cricket { + +class PayloadTypeMapperTest : public ::testing::Test { + protected: + PayloadTypeMapper mapper_; +}; + +TEST_F(PayloadTypeMapperTest, StaticPayloadTypes) { + EXPECT_EQ(0, mapper_.FindMappingFor({"pcmu", 8000, 1})); + EXPECT_EQ(3, mapper_.FindMappingFor({"gsm", 8000, 1})); + EXPECT_EQ(4, mapper_.FindMappingFor({"g723", 8000, 1})); + EXPECT_EQ(5, mapper_.FindMappingFor({"dvi4", 8000, 1})); + EXPECT_EQ(6, mapper_.FindMappingFor({"dvi4", 16000, 1})); + EXPECT_EQ(7, mapper_.FindMappingFor({"lpc", 8000, 1})); + EXPECT_EQ(8, mapper_.FindMappingFor({"pcma", 8000, 1})); + EXPECT_EQ(9, mapper_.FindMappingFor({"g722", 8000, 1})); + EXPECT_EQ(10, mapper_.FindMappingFor({"l16", 44100, 2})); + EXPECT_EQ(11, mapper_.FindMappingFor({"l16", 44100, 1})); + EXPECT_EQ(12, mapper_.FindMappingFor({"qcelp", 8000, 1})); + EXPECT_EQ(13, mapper_.FindMappingFor({"cn", 8000, 1})); + EXPECT_EQ(14, mapper_.FindMappingFor({"mpa", 90000, 0})); + EXPECT_EQ(14, mapper_.FindMappingFor({"mpa", 90000, 1})); + EXPECT_EQ(15, mapper_.FindMappingFor({"g728", 8000, 1})); + EXPECT_EQ(16, mapper_.FindMappingFor({"dvi4", 11025, 1})); + EXPECT_EQ(17, mapper_.FindMappingFor({"dvi4", 22050, 1})); + EXPECT_EQ(18, mapper_.FindMappingFor({"g729", 8000, 1})); +} + +TEST_F(PayloadTypeMapperTest, WebRTCPayloadTypes) { + // Tests that the payload mapper knows about the audio and data formats we've + // been using in WebRTC, with their hard coded values. + auto data_mapping = [this](const char* name) { + return mapper_.FindMappingFor({name, 0, 0}); + }; + EXPECT_EQ(kGoogleRtpDataCodecPlType, data_mapping(kGoogleRtpDataCodecName)); + EXPECT_EQ(kGoogleSctpDataCodecPlType, data_mapping(kGoogleSctpDataCodecName)); + + EXPECT_EQ(102, mapper_.FindMappingFor({kIlbcCodecName, 8000, 1})); + EXPECT_EQ(103, mapper_.FindMappingFor({kIsacCodecName, 16000, 1})); + EXPECT_EQ(104, mapper_.FindMappingFor({kIsacCodecName, 32000, 1})); + EXPECT_EQ(105, mapper_.FindMappingFor({kCnCodecName, 16000, 1})); + EXPECT_EQ(106, mapper_.FindMappingFor({kCnCodecName, 32000, 1})); + EXPECT_EQ(111, mapper_.FindMappingFor( + {kOpusCodecName, + 48000, + 2, + {{"minptime", "10"}, {"useinbandfec", "1"}}})); + // TODO(solenberg): Remove 16k, 32k, 48k DTMF checks once these payload types + // are dynamically assigned. + EXPECT_EQ(110, mapper_.FindMappingFor({kDtmfCodecName, 48000, 1})); + EXPECT_EQ(112, mapper_.FindMappingFor({kDtmfCodecName, 32000, 1})); + EXPECT_EQ(113, mapper_.FindMappingFor({kDtmfCodecName, 16000, 1})); + EXPECT_EQ(126, mapper_.FindMappingFor({kDtmfCodecName, 8000, 1})); +} + +TEST_F(PayloadTypeMapperTest, ValidDynamicPayloadTypes) { + // RFC 3551 says: + // "This profile reserves payload type numbers in the range 96-127 + // exclusively for dynamic assignment. Applications SHOULD first use + // values in this range for dynamic payload types. Those applications + // which need to define more than 32 dynamic payload types MAY bind + // codes below 96, in which case it is RECOMMENDED that unassigned + // payload type numbers be used first. However, the statically assigned + // payload types are default bindings and MAY be dynamically bound to + // new encodings if needed." + + // Tests that the payload mapper uses values in the dynamic payload type range + // (96 - 127) before any others and that the values returned are all valid. + bool has_been_below_96 = false; + std::set<int> used_payload_types; + for (int i = 0; i != 256; ++i) { + std::string format_name = "unknown_format_" + std::to_string(i); + webrtc::SdpAudioFormat format(format_name.c_str(), i * 100, (i % 2) + 1); + auto opt_payload_type = mapper_.GetMappingFor(format); + bool mapper_is_full = false; + + // There's a limited number of slots for payload types. We're fine with not + // being able to map them all. + if (opt_payload_type) { + int payload_type = *opt_payload_type; + EXPECT_FALSE(mapper_is_full) << "Mapping should not fail sporadically"; + EXPECT_EQ(used_payload_types.find(payload_type), used_payload_types.end()) + << "Payload types must not be reused"; + used_payload_types.insert(payload_type); + EXPECT_GE(payload_type, 0) << "Negative payload types are invalid"; + EXPECT_LE(payload_type, 127) << "Payload types above 127 are invalid"; + EXPECT_FALSE(payload_type >= 96 && has_been_below_96); + if (payload_type < 96) + has_been_below_96 = true; + + EXPECT_EQ(payload_type, mapper_.FindMappingFor(format)) + << "Mapping must be permanent after successful call to " + "GetMappingFor"; + EXPECT_EQ(payload_type, mapper_.GetMappingFor(format)) + << "Subsequent calls to GetMappingFor must return the same value"; + } else { + mapper_is_full = true; + } + } + + // Also, we must've been able to map at least one dynamic payload type. + EXPECT_FALSE(used_payload_types.empty()) + << "Mapper must support at least one user-defined payload type"; +} + +TEST_F(PayloadTypeMapperTest, ToAudioCodec) { + webrtc::SdpAudioFormat format("unknown_format", 4711, 17); + auto opt_payload_type = mapper_.GetMappingFor(format); + EXPECT_TRUE(opt_payload_type); + auto opt_audio_codec = mapper_.ToAudioCodec(format); + EXPECT_TRUE(opt_audio_codec); + + if (opt_payload_type && opt_audio_codec) { + int payload_type = *opt_payload_type; + const AudioCodec& codec = *opt_audio_codec; + + EXPECT_EQ(codec.id, payload_type); + EXPECT_EQ(codec.name, format.name); + EXPECT_EQ(codec.clockrate, format.clockrate_hz); + EXPECT_EQ(codec.channels, format.num_channels); + EXPECT_EQ(codec.params, format.parameters); + } +} + +} // namespace cricket diff --git a/media/engine/simulcast.cc b/media/engine/simulcast.cc new file mode 100644 index 0000000000..6e63ec6f7f --- /dev/null +++ b/media/engine/simulcast.cc @@ -0,0 +1,434 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "media/engine/simulcast.h" + +#include <stdint.h> +#include <stdio.h> + +#include <algorithm> +#include <string> + +#include "absl/types/optional.h" +#include "api/video/video_codec_constants.h" +#include "media/base/media_constants.h" +#include "modules/video_coding/utility/simulcast_rate_allocator.h" +#include "rtc_base/arraysize.h" +#include "rtc_base/checks.h" +#include "rtc_base/experiments/min_video_bitrate_experiment.h" +#include "rtc_base/experiments/normalize_simulcast_size_experiment.h" +#include "rtc_base/experiments/rate_control_settings.h" +#include "rtc_base/logging.h" +#include "system_wrappers/include/field_trial.h" + +namespace cricket { + +namespace { + +constexpr webrtc::DataRate Interpolate(const webrtc::DataRate& a, + const webrtc::DataRate& b, + float rate) { + return a * (1.0 - rate) + b * rate; +} + +constexpr char kUseLegacySimulcastLayerLimitFieldTrial[] = + "WebRTC-LegacySimulcastLayerLimit"; + +// Limits for legacy conference screensharing mode. Currently used for the +// lower of the two simulcast streams. +constexpr webrtc::DataRate kScreenshareDefaultTl0Bitrate = + webrtc::DataRate::KilobitsPerSec(200); +constexpr webrtc::DataRate kScreenshareDefaultTl1Bitrate = + webrtc::DataRate::KilobitsPerSec(1000); + +// Min/max bitrate for the higher one of the two simulcast stream used for +// screen content. +constexpr webrtc::DataRate kScreenshareHighStreamMinBitrate = + webrtc::DataRate::KilobitsPerSec(600); +constexpr webrtc::DataRate kScreenshareHighStreamMaxBitrate = + webrtc::DataRate::KilobitsPerSec(1250); + +} // namespace + +struct SimulcastFormat { + int width; + int height; + // The maximum number of simulcast layers can be used for + // resolutions at |widthxheigh| for legacy applications. + size_t max_layers; + // The maximum bitrate for encoding stream at |widthxheight|, when we are + // not sending the next higher spatial stream. + webrtc::DataRate max_bitrate; + // The target bitrate for encoding stream at |widthxheight|, when this layer + // is not the highest layer (i.e., when we are sending another higher spatial + // stream). + webrtc::DataRate target_bitrate; + // The minimum bitrate needed for encoding stream at |widthxheight|. + webrtc::DataRate min_bitrate; +}; + +// These tables describe from which resolution we can use how many +// simulcast layers at what bitrates (maximum, target, and minimum). +// Important!! Keep this table from high resolution to low resolution. +constexpr const SimulcastFormat kSimulcastFormats[] = { + {1920, 1080, 3, webrtc::DataRate::KilobitsPerSec(5000), + webrtc::DataRate::KilobitsPerSec(4000), + webrtc::DataRate::KilobitsPerSec(800)}, + {1280, 720, 3, webrtc::DataRate::KilobitsPerSec(2500), + webrtc::DataRate::KilobitsPerSec(2500), + webrtc::DataRate::KilobitsPerSec(600)}, + {960, 540, 3, webrtc::DataRate::KilobitsPerSec(1200), + webrtc::DataRate::KilobitsPerSec(1200), + webrtc::DataRate::KilobitsPerSec(350)}, + {640, 360, 2, webrtc::DataRate::KilobitsPerSec(700), + webrtc::DataRate::KilobitsPerSec(500), + webrtc::DataRate::KilobitsPerSec(150)}, + {480, 270, 2, webrtc::DataRate::KilobitsPerSec(450), + webrtc::DataRate::KilobitsPerSec(350), + webrtc::DataRate::KilobitsPerSec(150)}, + {320, 180, 1, webrtc::DataRate::KilobitsPerSec(200), + webrtc::DataRate::KilobitsPerSec(150), + webrtc::DataRate::KilobitsPerSec(30)}, + {0, 0, 1, webrtc::DataRate::KilobitsPerSec(200), + webrtc::DataRate::KilobitsPerSec(150), + webrtc::DataRate::KilobitsPerSec(30)}}; + +const int kMaxScreenshareSimulcastLayers = 2; + +// Multiway: Number of temporal layers for each simulcast stream. +int DefaultNumberOfTemporalLayers(int simulcast_id, bool screenshare) { + RTC_CHECK_GE(simulcast_id, 0); + RTC_CHECK_LT(simulcast_id, webrtc::kMaxSimulcastStreams); + + const int kDefaultNumTemporalLayers = 3; + const int kDefaultNumScreenshareTemporalLayers = 2; + int default_num_temporal_layers = screenshare + ? kDefaultNumScreenshareTemporalLayers + : kDefaultNumTemporalLayers; + + const std::string group_name = + screenshare ? webrtc::field_trial::FindFullName( + "WebRTC-VP8ScreenshareTemporalLayers") + : webrtc::field_trial::FindFullName( + "WebRTC-VP8ConferenceTemporalLayers"); + if (group_name.empty()) + return default_num_temporal_layers; + + int num_temporal_layers = default_num_temporal_layers; + if (sscanf(group_name.c_str(), "%d", &num_temporal_layers) == 1 && + num_temporal_layers > 0 && + num_temporal_layers <= webrtc::kMaxTemporalStreams) { + return num_temporal_layers; + } + + RTC_LOG(LS_WARNING) << "Attempt to set number of temporal layers to " + "incorrect value: " + << group_name; + + return default_num_temporal_layers; +} + +int FindSimulcastFormatIndex(int width, int height) { + RTC_DCHECK_GE(width, 0); + RTC_DCHECK_GE(height, 0); + for (uint32_t i = 0; i < arraysize(kSimulcastFormats); ++i) { + if (width * height >= + kSimulcastFormats[i].width * kSimulcastFormats[i].height) { + return i; + } + } + RTC_NOTREACHED(); + return -1; +} + +// Round size to nearest simulcast-friendly size. +// Simulcast stream width and height must both be dividable by +// |2 ^ (simulcast_layers - 1)|. +int NormalizeSimulcastSize(int size, size_t simulcast_layers) { + int base2_exponent = static_cast<int>(simulcast_layers) - 1; + const absl::optional<int> experimental_base2_exponent = + webrtc::NormalizeSimulcastSizeExperiment::GetBase2Exponent(); + if (experimental_base2_exponent && + (size > (1 << *experimental_base2_exponent))) { + base2_exponent = *experimental_base2_exponent; + } + return ((size >> base2_exponent) << base2_exponent); +} + +SimulcastFormat InterpolateSimulcastFormat(int width, int height) { + const int index = FindSimulcastFormatIndex(width, height); + if (index == 0) + return kSimulcastFormats[index]; + const int total_pixels_up = + kSimulcastFormats[index - 1].width * kSimulcastFormats[index - 1].height; + const int total_pixels_down = + kSimulcastFormats[index].width * kSimulcastFormats[index].height; + const int total_pixels = width * height; + const float rate = (total_pixels_up - total_pixels) / + static_cast<float>(total_pixels_up - total_pixels_down); + + size_t max_layers = kSimulcastFormats[index].max_layers; + webrtc::DataRate max_bitrate = + Interpolate(kSimulcastFormats[index - 1].max_bitrate, + kSimulcastFormats[index].max_bitrate, rate); + webrtc::DataRate target_bitrate = + Interpolate(kSimulcastFormats[index - 1].target_bitrate, + kSimulcastFormats[index].target_bitrate, rate); + webrtc::DataRate min_bitrate = + Interpolate(kSimulcastFormats[index - 1].min_bitrate, + kSimulcastFormats[index].min_bitrate, rate); + + return {width, height, max_layers, max_bitrate, target_bitrate, min_bitrate}; +} + +webrtc::DataRate FindSimulcastMaxBitrate(int width, int height) { + return InterpolateSimulcastFormat(width, height).max_bitrate; +} + +webrtc::DataRate FindSimulcastTargetBitrate(int width, int height) { + return InterpolateSimulcastFormat(width, height).target_bitrate; +} + +webrtc::DataRate FindSimulcastMinBitrate(int width, int height) { + return InterpolateSimulcastFormat(width, height).min_bitrate; +} + +void BoostMaxSimulcastLayer(webrtc::DataRate max_bitrate, + std::vector<webrtc::VideoStream>* layers) { + if (layers->empty()) + return; + + const webrtc::DataRate total_bitrate = GetTotalMaxBitrate(*layers); + + // We're still not using all available bits. + if (total_bitrate < max_bitrate) { + // Spend additional bits to boost the max layer. + const webrtc::DataRate bitrate_left = max_bitrate - total_bitrate; + layers->back().max_bitrate_bps += bitrate_left.bps(); + } +} + +webrtc::DataRate GetTotalMaxBitrate( + const std::vector<webrtc::VideoStream>& layers) { + if (layers.empty()) + return webrtc::DataRate::Zero(); + + int total_max_bitrate_bps = 0; + for (size_t s = 0; s < layers.size() - 1; ++s) { + total_max_bitrate_bps += layers[s].target_bitrate_bps; + } + total_max_bitrate_bps += layers.back().max_bitrate_bps; + return webrtc::DataRate::BitsPerSec(total_max_bitrate_bps); +} + +size_t LimitSimulcastLayerCount(int width, + int height, + size_t need_layers, + size_t layer_count) { + if (!webrtc::field_trial::IsDisabled( + kUseLegacySimulcastLayerLimitFieldTrial)) { + size_t adaptive_layer_count = std::max( + need_layers, + kSimulcastFormats[FindSimulcastFormatIndex(width, height)].max_layers); + if (layer_count > adaptive_layer_count) { + RTC_LOG(LS_WARNING) << "Reducing simulcast layer count from " + << layer_count << " to " << adaptive_layer_count; + layer_count = adaptive_layer_count; + } + } + return layer_count; +} + +std::vector<webrtc::VideoStream> GetSimulcastConfig( + size_t min_layers, + size_t max_layers, + int width, + int height, + double bitrate_priority, + int max_qp, + bool is_screenshare_with_conference_mode, + bool temporal_layers_supported) { + RTC_DCHECK_LE(min_layers, max_layers); + RTC_DCHECK(max_layers > 1 || is_screenshare_with_conference_mode); + + const bool base_heavy_tl3_rate_alloc = + webrtc::RateControlSettings::ParseFromFieldTrials() + .Vp8BaseHeavyTl3RateAllocation(); + if (is_screenshare_with_conference_mode) { + return GetScreenshareLayers(max_layers, width, height, bitrate_priority, + max_qp, temporal_layers_supported, + base_heavy_tl3_rate_alloc); + } else { + // Some applications rely on the old behavior limiting the simulcast layer + // count based on the resolution automatically, which they can get through + // the WebRTC-LegacySimulcastLayerLimit field trial until they update. + max_layers = + LimitSimulcastLayerCount(width, height, min_layers, max_layers); + + return GetNormalSimulcastLayers(max_layers, width, height, bitrate_priority, + max_qp, temporal_layers_supported, + base_heavy_tl3_rate_alloc); + } +} + +std::vector<webrtc::VideoStream> GetNormalSimulcastLayers( + size_t layer_count, + int width, + int height, + double bitrate_priority, + int max_qp, + bool temporal_layers_supported, + bool base_heavy_tl3_rate_alloc) { + std::vector<webrtc::VideoStream> layers(layer_count); + + // Format width and height has to be divisible by |2 ^ num_simulcast_layers - + // 1|. + width = NormalizeSimulcastSize(width, layer_count); + height = NormalizeSimulcastSize(height, layer_count); + // Add simulcast streams, from highest resolution (|s| = num_simulcast_layers + // -1) to lowest resolution at |s| = 0. + for (size_t s = layer_count - 1;; --s) { + layers[s].width = width; + layers[s].height = height; + // TODO(pbos): Fill actual temporal-layer bitrate thresholds. + layers[s].max_qp = max_qp; + layers[s].num_temporal_layers = + temporal_layers_supported ? DefaultNumberOfTemporalLayers(s, false) : 1; + layers[s].max_bitrate_bps = FindSimulcastMaxBitrate(width, height).bps(); + layers[s].target_bitrate_bps = + FindSimulcastTargetBitrate(width, height).bps(); + int num_temporal_layers = DefaultNumberOfTemporalLayers(s, false); + if (s == 0) { + // If alternative temporal rate allocation is selected, adjust the + // bitrate of the lowest simulcast stream so that absolute bitrate for + // the base temporal layer matches the bitrate for the base temporal + // layer with the default 3 simulcast streams. Otherwise we risk a + // higher threshold for receiving a feed at all. + float rate_factor = 1.0; + if (num_temporal_layers == 3) { + if (base_heavy_tl3_rate_alloc) { + // Base heavy allocation increases TL0 bitrate from 40% to 60%. + rate_factor = 0.4 / 0.6; + } + } else { + rate_factor = + webrtc::SimulcastRateAllocator::GetTemporalRateAllocation( + 3, 0, /*base_heavy_tl3_rate_alloc=*/false) / + webrtc::SimulcastRateAllocator::GetTemporalRateAllocation( + num_temporal_layers, 0, /*base_heavy_tl3_rate_alloc=*/false); + } + + layers[s].max_bitrate_bps = + static_cast<int>(layers[s].max_bitrate_bps * rate_factor); + layers[s].target_bitrate_bps = + static_cast<int>(layers[s].target_bitrate_bps * rate_factor); + } + layers[s].min_bitrate_bps = FindSimulcastMinBitrate(width, height).bps(); + layers[s].max_framerate = kDefaultVideoMaxFramerate; + + width /= 2; + height /= 2; + + if (s == 0) { + break; + } + } + // Currently the relative bitrate priority of the sender is controlled by + // the value of the lowest VideoStream. + // TODO(bugs.webrtc.org/8630): The web specification describes being able to + // control relative bitrate for each individual simulcast layer, but this + // is currently just implemented per rtp sender. + layers[0].bitrate_priority = bitrate_priority; + return layers; +} + +std::vector<webrtc::VideoStream> GetScreenshareLayers( + size_t max_layers, + int width, + int height, + double bitrate_priority, + int max_qp, + bool temporal_layers_supported, + bool base_heavy_tl3_rate_alloc) { + auto max_screenshare_layers = kMaxScreenshareSimulcastLayers; + size_t num_simulcast_layers = + std::min<int>(max_layers, max_screenshare_layers); + + std::vector<webrtc::VideoStream> layers(num_simulcast_layers); + // For legacy screenshare in conference mode, tl0 and tl1 bitrates are + // piggybacked on the VideoCodec struct as target and max bitrates, + // respectively. See eg. webrtc::LibvpxVp8Encoder::SetRates(). + layers[0].width = width; + layers[0].height = height; + layers[0].max_qp = max_qp; + layers[0].max_framerate = 5; + layers[0].min_bitrate_bps = webrtc::kDefaultMinVideoBitrateBps; + layers[0].target_bitrate_bps = kScreenshareDefaultTl0Bitrate.bps(); + layers[0].max_bitrate_bps = kScreenshareDefaultTl1Bitrate.bps(); + layers[0].num_temporal_layers = temporal_layers_supported ? 2 : 1; + + // With simulcast enabled, add another spatial layer. This one will have a + // more normal layout, with the regular 3 temporal layer pattern and no fps + // restrictions. The base simulcast layer will still use legacy setup. + if (num_simulcast_layers == kMaxScreenshareSimulcastLayers) { + // Add optional upper simulcast layer. + const int num_temporal_layers = DefaultNumberOfTemporalLayers(1, true); + int max_bitrate_bps; + bool using_boosted_bitrate = false; + if (!temporal_layers_supported) { + // Set the max bitrate to where the base layer would have been if temporal + // layers were enabled. + max_bitrate_bps = static_cast<int>( + kScreenshareHighStreamMaxBitrate.bps() * + webrtc::SimulcastRateAllocator::GetTemporalRateAllocation( + num_temporal_layers, 0, base_heavy_tl3_rate_alloc)); + } else if (DefaultNumberOfTemporalLayers(1, true) != 3 || + base_heavy_tl3_rate_alloc) { + // Experimental temporal layer mode used, use increased max bitrate. + max_bitrate_bps = kScreenshareHighStreamMaxBitrate.bps(); + using_boosted_bitrate = true; + } else { + // Keep current bitrates with default 3tl/8 frame settings. + // Lowest temporal layers of a 3 layer setup will have 40% of the total + // bitrate allocation for that simulcast layer. Make sure the gap between + // the target of the lower simulcast layer and first temporal layer of the + // higher one is at most 2x the bitrate, so that upswitching is not + // hampered by stalled bitrate estimates. + max_bitrate_bps = 2 * ((layers[0].target_bitrate_bps * 10) / 4); + } + + layers[1].width = width; + layers[1].height = height; + layers[1].max_qp = max_qp; + layers[1].max_framerate = kDefaultVideoMaxFramerate; + layers[1].num_temporal_layers = + temporal_layers_supported ? DefaultNumberOfTemporalLayers(1, true) : 1; + layers[1].min_bitrate_bps = using_boosted_bitrate + ? kScreenshareHighStreamMinBitrate.bps() + : layers[0].target_bitrate_bps * 2; + + // Cap max bitrate so it isn't overly high for the given resolution. + int resolution_limited_bitrate = + std::max<int>(FindSimulcastMaxBitrate(width, height).bps(), + layers[1].min_bitrate_bps); + max_bitrate_bps = + std::min<int>(max_bitrate_bps, resolution_limited_bitrate); + + layers[1].target_bitrate_bps = max_bitrate_bps; + layers[1].max_bitrate_bps = max_bitrate_bps; + } + + // The bitrate priority currently implemented on a per-sender level, so we + // just set it for the first simulcast layer. + layers[0].bitrate_priority = bitrate_priority; + return layers; +} + +} // namespace cricket diff --git a/media/engine/simulcast.h b/media/engine/simulcast.h new file mode 100644 index 0000000000..28b08560aa --- /dev/null +++ b/media/engine/simulcast.h @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MEDIA_ENGINE_SIMULCAST_H_ +#define MEDIA_ENGINE_SIMULCAST_H_ + +#include <stddef.h> + +#include <vector> + +#include "api/units/data_rate.h" +#include "api/video_codecs/video_encoder_config.h" + +namespace cricket { + +// Gets the total maximum bitrate for the |streams|. +webrtc::DataRate GetTotalMaxBitrate( + const std::vector<webrtc::VideoStream>& streams); + +// Adds any bitrate of |max_bitrate| that is above the total maximum bitrate for +// the |layers| to the highest quality layer. +void BoostMaxSimulcastLayer(webrtc::DataRate max_bitrate, + std::vector<webrtc::VideoStream>* layers); + +// Round size to nearest simulcast-friendly size +int NormalizeSimulcastSize(int size, size_t simulcast_layers); + +// Gets simulcast settings. +std::vector<webrtc::VideoStream> GetSimulcastConfig( + size_t min_layers, + size_t max_layers, + int width, + int height, + double bitrate_priority, + int max_qp, + bool is_screenshare_with_conference_mode, + bool temporal_layers_supported); + +// Gets the simulcast config layers for a non-screensharing case. +std::vector<webrtc::VideoStream> GetNormalSimulcastLayers( + size_t max_layers, + int width, + int height, + double bitrate_priority, + int max_qp, + bool temporal_layers_supported, + bool base_heavy_tl3_rate_alloc); + +// Gets simulcast config layers for screenshare settings. +std::vector<webrtc::VideoStream> GetScreenshareLayers( + size_t max_layers, + int width, + int height, + double bitrate_priority, + int max_qp, + bool temporal_layers_supported, + bool base_heavy_tl3_rate_alloc); + +} // namespace cricket + +#endif // MEDIA_ENGINE_SIMULCAST_H_ diff --git a/media/engine/simulcast_encoder_adapter.cc b/media/engine/simulcast_encoder_adapter.cc new file mode 100644 index 0000000000..863ccc756e --- /dev/null +++ b/media/engine/simulcast_encoder_adapter.cc @@ -0,0 +1,703 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "media/engine/simulcast_encoder_adapter.h" + +#include <stdio.h> +#include <string.h> + +#include <algorithm> +#include <cstdint> +#include <string> +#include <utility> + +#include "api/scoped_refptr.h" +#include "api/video/i420_buffer.h" +#include "api/video/video_codec_constants.h" +#include "api/video/video_frame_buffer.h" +#include "api/video/video_rotation.h" +#include "api/video_codecs/video_encoder.h" +#include "api/video_codecs/video_encoder_factory.h" +#include "api/video_codecs/video_encoder_software_fallback_wrapper.h" +#include "media/base/video_common.h" +#include "modules/video_coding/include/video_error_codes.h" +#include "modules/video_coding/utility/simulcast_rate_allocator.h" +#include "rtc_base/atomic_ops.h" +#include "rtc_base/checks.h" +#include "rtc_base/experiments/rate_control_settings.h" +#include "rtc_base/logging.h" +#include "system_wrappers/include/field_trial.h" + +namespace { + +const unsigned int kDefaultMinQp = 2; +const unsigned int kDefaultMaxQp = 56; +// Max qp for lowest spatial resolution when doing simulcast. +const unsigned int kLowestResMaxQp = 45; + +absl::optional<unsigned int> GetScreenshareBoostedQpValue() { + std::string experiment_group = + webrtc::field_trial::FindFullName("WebRTC-BoostedScreenshareQp"); + unsigned int qp; + if (sscanf(experiment_group.c_str(), "%u", &qp) != 1) + return absl::nullopt; + qp = std::min(qp, 63u); + qp = std::max(qp, 1u); + return qp; +} + +uint32_t SumStreamMaxBitrate(int streams, const webrtc::VideoCodec& codec) { + uint32_t bitrate_sum = 0; + for (int i = 0; i < streams; ++i) { + bitrate_sum += codec.simulcastStream[i].maxBitrate; + } + return bitrate_sum; +} + +int NumberOfStreams(const webrtc::VideoCodec& codec) { + int streams = + codec.numberOfSimulcastStreams < 1 ? 1 : codec.numberOfSimulcastStreams; + uint32_t simulcast_max_bitrate = SumStreamMaxBitrate(streams, codec); + if (simulcast_max_bitrate == 0) { + streams = 1; + } + return streams; +} + +int NumActiveStreams(const webrtc::VideoCodec& codec) { + int num_configured_streams = NumberOfStreams(codec); + int num_active_streams = 0; + for (int i = 0; i < num_configured_streams; ++i) { + if (codec.simulcastStream[i].active) { + ++num_active_streams; + } + } + return num_active_streams; +} + +int VerifyCodec(const webrtc::VideoCodec* inst) { + if (inst == nullptr) { + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + if (inst->maxFramerate < 1) { + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + // allow zero to represent an unspecified maxBitRate + if (inst->maxBitrate > 0 && inst->startBitrate > inst->maxBitrate) { + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + if (inst->width <= 1 || inst->height <= 1) { + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + if (inst->codecType == webrtc::kVideoCodecVP8 && + inst->VP8().automaticResizeOn && NumActiveStreams(*inst) > 1) { + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + return WEBRTC_VIDEO_CODEC_OK; +} + +bool StreamResolutionCompare(const webrtc::SimulcastStream& a, + const webrtc::SimulcastStream& b) { + return std::tie(a.height, a.width, a.maxBitrate, a.maxFramerate) < + std::tie(b.height, b.width, b.maxBitrate, b.maxFramerate); +} + +// An EncodedImageCallback implementation that forwards on calls to a +// SimulcastEncoderAdapter, but with the stream index it's registered with as +// the first parameter to Encoded. +class AdapterEncodedImageCallback : public webrtc::EncodedImageCallback { + public: + AdapterEncodedImageCallback(webrtc::SimulcastEncoderAdapter* adapter, + size_t stream_idx) + : adapter_(adapter), stream_idx_(stream_idx) {} + + EncodedImageCallback::Result OnEncodedImage( + const webrtc::EncodedImage& encoded_image, + const webrtc::CodecSpecificInfo* codec_specific_info, + const webrtc::RTPFragmentationHeader* fragmentation) override { + return adapter_->OnEncodedImage(stream_idx_, encoded_image, + codec_specific_info, fragmentation); + } + + private: + webrtc::SimulcastEncoderAdapter* const adapter_; + const size_t stream_idx_; +}; +} // namespace + +namespace webrtc { + +SimulcastEncoderAdapter::SimulcastEncoderAdapter(VideoEncoderFactory* factory, + const SdpVideoFormat& format) + : SimulcastEncoderAdapter(factory, nullptr, format) {} + +SimulcastEncoderAdapter::SimulcastEncoderAdapter( + VideoEncoderFactory* primary_factory, + VideoEncoderFactory* fallback_factory, + const SdpVideoFormat& format) + : inited_(0), + primary_encoder_factory_(primary_factory), + fallback_encoder_factory_(fallback_factory), + video_format_(format), + encoded_complete_callback_(nullptr), + experimental_boosted_screenshare_qp_(GetScreenshareBoostedQpValue()), + boost_base_layer_quality_(RateControlSettings::ParseFromFieldTrials() + .Vp8BoostBaseLayerQuality()), + prefer_temporal_support_on_base_layer_(field_trial::IsEnabled( + "WebRTC-Video-PreferTemporalSupportOnBaseLayer")) { + RTC_DCHECK(primary_factory); + + // The adapter is typically created on the worker thread, but operated on + // the encoder task queue. + encoder_queue_.Detach(); + + memset(&codec_, 0, sizeof(webrtc::VideoCodec)); +} + +SimulcastEncoderAdapter::~SimulcastEncoderAdapter() { + RTC_DCHECK(!Initialized()); + DestroyStoredEncoders(); +} + +void SimulcastEncoderAdapter::SetFecControllerOverride( + FecControllerOverride* fec_controller_override) { + // Ignored. +} + +int SimulcastEncoderAdapter::Release() { + RTC_DCHECK_RUN_ON(&encoder_queue_); + + while (!streaminfos_.empty()) { + std::unique_ptr<VideoEncoder> encoder = + std::move(streaminfos_.back().encoder); + // Even though it seems very unlikely, there are no guarantees that the + // encoder will not call back after being Release()'d. Therefore, we first + // disable the callbacks here. + encoder->RegisterEncodeCompleteCallback(nullptr); + encoder->Release(); + streaminfos_.pop_back(); // Deletes callback adapter. + stored_encoders_.push(std::move(encoder)); + } + + // It's legal to move the encoder to another queue now. + encoder_queue_.Detach(); + + rtc::AtomicOps::ReleaseStore(&inited_, 0); + + return WEBRTC_VIDEO_CODEC_OK; +} + +// TODO(eladalon): s/inst/codec_settings/g. +int SimulcastEncoderAdapter::InitEncode( + const VideoCodec* inst, + const VideoEncoder::Settings& settings) { + RTC_DCHECK_RUN_ON(&encoder_queue_); + + if (settings.number_of_cores < 1) { + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + + int ret = VerifyCodec(inst); + if (ret < 0) { + return ret; + } + + ret = Release(); + if (ret < 0) { + return ret; + } + + int number_of_streams = NumberOfStreams(*inst); + RTC_DCHECK_LE(number_of_streams, kMaxSimulcastStreams); + bool doing_simulcast_using_adapter = (number_of_streams > 1); + int num_active_streams = NumActiveStreams(*inst); + + codec_ = *inst; + SimulcastRateAllocator rate_allocator(codec_); + VideoBitrateAllocation allocation = + rate_allocator.Allocate(VideoBitrateAllocationParameters( + codec_.startBitrate * 1000, codec_.maxFramerate)); + std::vector<uint32_t> start_bitrates; + for (int i = 0; i < kMaxSimulcastStreams; ++i) { + uint32_t stream_bitrate = allocation.GetSpatialLayerSum(i) / 1000; + start_bitrates.push_back(stream_bitrate); + } + + // Create |number_of_streams| of encoder instances and init them. + const auto minmax = std::minmax_element( + std::begin(codec_.simulcastStream), + std::begin(codec_.simulcastStream) + number_of_streams, + StreamResolutionCompare); + const auto lowest_resolution_stream_index = + std::distance(std::begin(codec_.simulcastStream), minmax.first); + const auto highest_resolution_stream_index = + std::distance(std::begin(codec_.simulcastStream), minmax.second); + + RTC_DCHECK_LT(lowest_resolution_stream_index, number_of_streams); + RTC_DCHECK_LT(highest_resolution_stream_index, number_of_streams); + + const SdpVideoFormat format( + codec_.codecType == webrtc::kVideoCodecVP8 ? "VP8" : "H264", + video_format_.parameters); + + for (int i = 0; i < number_of_streams; ++i) { + // If an existing encoder instance exists, reuse it. + // TODO(brandtr): Set initial RTP state (e.g., picture_id/tl0_pic_idx) here, + // when we start storing that state outside the encoder wrappers. + std::unique_ptr<VideoEncoder> encoder; + if (!stored_encoders_.empty()) { + encoder = std::move(stored_encoders_.top()); + stored_encoders_.pop(); + } else { + encoder = primary_encoder_factory_->CreateVideoEncoder(format); + if (fallback_encoder_factory_ != nullptr) { + encoder = CreateVideoEncoderSoftwareFallbackWrapper( + fallback_encoder_factory_->CreateVideoEncoder(format), + std::move(encoder), + i == lowest_resolution_stream_index && + prefer_temporal_support_on_base_layer_); + } + } + + bool encoder_initialized = false; + if (doing_simulcast_using_adapter && i == 0 && + encoder->GetEncoderInfo().supports_simulcast) { + ret = encoder->InitEncode(&codec_, settings); + if (ret < 0) { + encoder->Release(); + } else { + doing_simulcast_using_adapter = false; + number_of_streams = 1; + encoder_initialized = true; + } + } + + VideoCodec stream_codec; + uint32_t start_bitrate_kbps = start_bitrates[i]; + const bool send_stream = doing_simulcast_using_adapter + ? start_bitrate_kbps > 0 + : num_active_streams > 0; + if (!doing_simulcast_using_adapter) { + stream_codec = codec_; + stream_codec.numberOfSimulcastStreams = + std::max<uint8_t>(1, stream_codec.numberOfSimulcastStreams); + } else { + // Cap start bitrate to the min bitrate in order to avoid strange codec + // behavior. Since sending will be false, this should not matter. + StreamResolution stream_resolution = + i == highest_resolution_stream_index + ? StreamResolution::HIGHEST + : i == lowest_resolution_stream_index ? StreamResolution::LOWEST + : StreamResolution::OTHER; + + start_bitrate_kbps = + std::max(codec_.simulcastStream[i].minBitrate, start_bitrate_kbps); + PopulateStreamCodec(codec_, i, start_bitrate_kbps, stream_resolution, + &stream_codec); + } + + // TODO(ronghuawu): Remove once this is handled in LibvpxVp8Encoder. + if (stream_codec.qpMax < kDefaultMinQp) { + stream_codec.qpMax = kDefaultMaxQp; + } + + if (!encoder_initialized) { + ret = encoder->InitEncode(&stream_codec, settings); + if (ret < 0) { + // Explicitly destroy the current encoder; because we haven't registered + // a StreamInfo for it yet, Release won't do anything about it. + encoder.reset(); + Release(); + return ret; + } + } + + if (!doing_simulcast_using_adapter) { + // Without simulcast, just pass through the encoder info from the one + // active encoder. + encoder->RegisterEncodeCompleteCallback(encoded_complete_callback_); + streaminfos_.emplace_back( + std::move(encoder), nullptr, + std::make_unique<FramerateController>(stream_codec.maxFramerate), + stream_codec.width, stream_codec.height, send_stream); + } else { + std::unique_ptr<EncodedImageCallback> callback( + new AdapterEncodedImageCallback(this, i)); + encoder->RegisterEncodeCompleteCallback(callback.get()); + streaminfos_.emplace_back( + std::move(encoder), std::move(callback), + std::make_unique<FramerateController>(stream_codec.maxFramerate), + stream_codec.width, stream_codec.height, send_stream); + } + } + + // To save memory, don't store encoders that we don't use. + DestroyStoredEncoders(); + + rtc::AtomicOps::ReleaseStore(&inited_, 1); + + return WEBRTC_VIDEO_CODEC_OK; +} + +int SimulcastEncoderAdapter::Encode( + const VideoFrame& input_image, + const std::vector<VideoFrameType>* frame_types) { + RTC_DCHECK_RUN_ON(&encoder_queue_); + + if (!Initialized()) { + return WEBRTC_VIDEO_CODEC_UNINITIALIZED; + } + if (encoded_complete_callback_ == nullptr) { + return WEBRTC_VIDEO_CODEC_UNINITIALIZED; + } + + // All active streams should generate a key frame if + // a key frame is requested by any stream. + bool send_key_frame = false; + if (frame_types) { + for (size_t i = 0; i < frame_types->size(); ++i) { + if (frame_types->at(i) == VideoFrameType::kVideoFrameKey) { + send_key_frame = true; + break; + } + } + } + for (size_t stream_idx = 0; stream_idx < streaminfos_.size(); ++stream_idx) { + if (streaminfos_[stream_idx].key_frame_request && + streaminfos_[stream_idx].send_stream) { + send_key_frame = true; + break; + } + } + + // Temporary thay may hold the result of texture to i420 buffer conversion. + rtc::scoped_refptr<I420BufferInterface> src_buffer; + int src_width = input_image.width(); + int src_height = input_image.height(); + for (size_t stream_idx = 0; stream_idx < streaminfos_.size(); ++stream_idx) { + // Don't encode frames in resolutions that we don't intend to send. + if (!streaminfos_[stream_idx].send_stream) { + continue; + } + + const uint32_t frame_timestamp_ms = + 1000 * input_image.timestamp() / 90000; // kVideoPayloadTypeFrequency; + + // If adapter is passed through and only one sw encoder does simulcast, + // frame types for all streams should be passed to the encoder unchanged. + // Otherwise a single per-encoder frame type is passed. + std::vector<VideoFrameType> stream_frame_types( + streaminfos_.size() == 1 ? NumberOfStreams(codec_) : 1); + if (send_key_frame) { + std::fill(stream_frame_types.begin(), stream_frame_types.end(), + VideoFrameType::kVideoFrameKey); + streaminfos_[stream_idx].key_frame_request = false; + } else { + if (streaminfos_[stream_idx].framerate_controller->DropFrame( + frame_timestamp_ms)) { + continue; + } + std::fill(stream_frame_types.begin(), stream_frame_types.end(), + VideoFrameType::kVideoFrameDelta); + } + streaminfos_[stream_idx].framerate_controller->AddFrame(frame_timestamp_ms); + + int dst_width = streaminfos_[stream_idx].width; + int dst_height = streaminfos_[stream_idx].height; + // If scaling isn't required, because the input resolution + // matches the destination or the input image is empty (e.g. + // a keyframe request for encoders with internal camera + // sources) or the source image has a native handle, pass the image on + // directly. Otherwise, we'll scale it to match what the encoder expects + // (below). + // For texture frames, the underlying encoder is expected to be able to + // correctly sample/scale the source texture. + // TODO(perkj): ensure that works going forward, and figure out how this + // affects webrtc:5683. + if ((dst_width == src_width && dst_height == src_height) || + (input_image.video_frame_buffer()->type() == + VideoFrameBuffer::Type::kNative && + streaminfos_[stream_idx] + .encoder->GetEncoderInfo() + .supports_native_handle)) { + int ret = streaminfos_[stream_idx].encoder->Encode(input_image, + &stream_frame_types); + if (ret != WEBRTC_VIDEO_CODEC_OK) { + return ret; + } + } else { + if (src_buffer == nullptr) { + src_buffer = input_image.video_frame_buffer()->ToI420(); + } + rtc::scoped_refptr<I420Buffer> dst_buffer = + I420Buffer::Create(dst_width, dst_height); + + dst_buffer->ScaleFrom(*src_buffer); + + // UpdateRect is not propagated to lower simulcast layers currently. + // TODO(ilnik): Consider scaling UpdateRect together with the buffer. + VideoFrame frame(input_image); + frame.set_video_frame_buffer(dst_buffer); + frame.set_rotation(webrtc::kVideoRotation_0); + frame.set_update_rect( + VideoFrame::UpdateRect{0, 0, frame.width(), frame.height()}); + int ret = + streaminfos_[stream_idx].encoder->Encode(frame, &stream_frame_types); + if (ret != WEBRTC_VIDEO_CODEC_OK) { + return ret; + } + } + } + + return WEBRTC_VIDEO_CODEC_OK; +} + +int SimulcastEncoderAdapter::RegisterEncodeCompleteCallback( + EncodedImageCallback* callback) { + RTC_DCHECK_RUN_ON(&encoder_queue_); + encoded_complete_callback_ = callback; + if (streaminfos_.size() == 1) { + streaminfos_[0].encoder->RegisterEncodeCompleteCallback(callback); + } + return WEBRTC_VIDEO_CODEC_OK; +} + +void SimulcastEncoderAdapter::SetRates( + const RateControlParameters& parameters) { + RTC_DCHECK_RUN_ON(&encoder_queue_); + + if (!Initialized()) { + RTC_LOG(LS_WARNING) << "SetRates while not initialized"; + return; + } + + if (parameters.framerate_fps < 1.0) { + RTC_LOG(LS_WARNING) << "Invalid framerate: " << parameters.framerate_fps; + return; + } + + codec_.maxFramerate = static_cast<uint32_t>(parameters.framerate_fps + 0.5); + + if (streaminfos_.size() == 1) { + // Not doing simulcast. + streaminfos_[0].encoder->SetRates(parameters); + return; + } + + for (size_t stream_idx = 0; stream_idx < streaminfos_.size(); ++stream_idx) { + uint32_t stream_bitrate_kbps = + parameters.bitrate.GetSpatialLayerSum(stream_idx) / 1000; + + // Need a key frame if we have not sent this stream before. + if (stream_bitrate_kbps > 0 && !streaminfos_[stream_idx].send_stream) { + streaminfos_[stream_idx].key_frame_request = true; + } + streaminfos_[stream_idx].send_stream = stream_bitrate_kbps > 0; + + // Slice the temporal layers out of the full allocation and pass it on to + // the encoder handling the current simulcast stream. + RateControlParameters stream_parameters = parameters; + stream_parameters.bitrate = VideoBitrateAllocation(); + for (int i = 0; i < kMaxTemporalStreams; ++i) { + if (parameters.bitrate.HasBitrate(stream_idx, i)) { + stream_parameters.bitrate.SetBitrate( + 0, i, parameters.bitrate.GetBitrate(stream_idx, i)); + } + } + + // Assign link allocation proportionally to spatial layer allocation. + if (!parameters.bandwidth_allocation.IsZero() && + parameters.bitrate.get_sum_bps() > 0) { + stream_parameters.bandwidth_allocation = + DataRate::BitsPerSec((parameters.bandwidth_allocation.bps() * + stream_parameters.bitrate.get_sum_bps()) / + parameters.bitrate.get_sum_bps()); + // Make sure we don't allocate bandwidth lower than target bitrate. + if (stream_parameters.bandwidth_allocation.bps() < + stream_parameters.bitrate.get_sum_bps()) { + stream_parameters.bandwidth_allocation = + DataRate::BitsPerSec(stream_parameters.bitrate.get_sum_bps()); + } + } + + stream_parameters.framerate_fps = std::min<double>( + parameters.framerate_fps, + streaminfos_[stream_idx].framerate_controller->GetTargetRate()); + + streaminfos_[stream_idx].encoder->SetRates(stream_parameters); + } +} + +void SimulcastEncoderAdapter::OnPacketLossRateUpdate(float packet_loss_rate) { + for (StreamInfo& info : streaminfos_) { + info.encoder->OnPacketLossRateUpdate(packet_loss_rate); + } +} + +void SimulcastEncoderAdapter::OnRttUpdate(int64_t rtt_ms) { + for (StreamInfo& info : streaminfos_) { + info.encoder->OnRttUpdate(rtt_ms); + } +} + +void SimulcastEncoderAdapter::OnLossNotification( + const LossNotification& loss_notification) { + for (StreamInfo& info : streaminfos_) { + info.encoder->OnLossNotification(loss_notification); + } +} + +// TODO(brandtr): Add task checker to this member function, when all encoder +// callbacks are coming in on the encoder queue. +EncodedImageCallback::Result SimulcastEncoderAdapter::OnEncodedImage( + size_t stream_idx, + const EncodedImage& encodedImage, + const CodecSpecificInfo* codecSpecificInfo, + const RTPFragmentationHeader* fragmentation) { + EncodedImage stream_image(encodedImage); + CodecSpecificInfo stream_codec_specific = *codecSpecificInfo; + + stream_image.SetSpatialIndex(stream_idx); + + return encoded_complete_callback_->OnEncodedImage( + stream_image, &stream_codec_specific, fragmentation); +} + +void SimulcastEncoderAdapter::PopulateStreamCodec( + const webrtc::VideoCodec& inst, + int stream_index, + uint32_t start_bitrate_kbps, + StreamResolution stream_resolution, + webrtc::VideoCodec* stream_codec) { + *stream_codec = inst; + + // Stream specific settings. + stream_codec->numberOfSimulcastStreams = 0; + stream_codec->width = inst.simulcastStream[stream_index].width; + stream_codec->height = inst.simulcastStream[stream_index].height; + stream_codec->maxBitrate = inst.simulcastStream[stream_index].maxBitrate; + stream_codec->minBitrate = inst.simulcastStream[stream_index].minBitrate; + stream_codec->maxFramerate = inst.simulcastStream[stream_index].maxFramerate; + stream_codec->qpMax = inst.simulcastStream[stream_index].qpMax; + stream_codec->active = inst.simulcastStream[stream_index].active; + // Settings that are based on stream/resolution. + if (stream_resolution == StreamResolution::LOWEST) { + // Settings for lowest spatial resolutions. + if (inst.mode == VideoCodecMode::kScreensharing) { + if (experimental_boosted_screenshare_qp_) { + stream_codec->qpMax = *experimental_boosted_screenshare_qp_; + } + } else if (boost_base_layer_quality_) { + stream_codec->qpMax = kLowestResMaxQp; + } + } + if (inst.codecType == webrtc::kVideoCodecVP8) { + stream_codec->VP8()->numberOfTemporalLayers = + inst.simulcastStream[stream_index].numberOfTemporalLayers; + if (stream_resolution != StreamResolution::HIGHEST) { + // For resolutions below CIF, set the codec |complexity| parameter to + // kComplexityHigher, which maps to cpu_used = -4. + int pixels_per_frame = stream_codec->width * stream_codec->height; + if (pixels_per_frame < 352 * 288) { + stream_codec->VP8()->complexity = + webrtc::VideoCodecComplexity::kComplexityHigher; + } + // Turn off denoising for all streams but the highest resolution. + stream_codec->VP8()->denoisingOn = false; + } + } else if (inst.codecType == webrtc::kVideoCodecH264) { + stream_codec->H264()->numberOfTemporalLayers = + inst.simulcastStream[stream_index].numberOfTemporalLayers; + } + // TODO(ronghuawu): what to do with targetBitrate. + + stream_codec->startBitrate = start_bitrate_kbps; +} + +bool SimulcastEncoderAdapter::Initialized() const { + return rtc::AtomicOps::AcquireLoad(&inited_) == 1; +} + +void SimulcastEncoderAdapter::DestroyStoredEncoders() { + while (!stored_encoders_.empty()) { + stored_encoders_.pop(); + } +} + +VideoEncoder::EncoderInfo SimulcastEncoderAdapter::GetEncoderInfo() const { + if (streaminfos_.size() == 1) { + // Not using simulcast adapting functionality, just pass through. + return streaminfos_[0].encoder->GetEncoderInfo(); + } + + VideoEncoder::EncoderInfo encoder_info; + encoder_info.implementation_name = "SimulcastEncoderAdapter"; + encoder_info.requested_resolution_alignment = 1; + encoder_info.supports_native_handle = true; + encoder_info.scaling_settings.thresholds = absl::nullopt; + if (streaminfos_.empty()) { + return encoder_info; + } + + encoder_info.scaling_settings = VideoEncoder::ScalingSettings::kOff; + int num_active_streams = NumActiveStreams(codec_); + + for (size_t i = 0; i < streaminfos_.size(); ++i) { + VideoEncoder::EncoderInfo encoder_impl_info = + streaminfos_[i].encoder->GetEncoderInfo(); + + if (i == 0) { + // Encoder name indicates names of all sub-encoders. + encoder_info.implementation_name += " ("; + encoder_info.implementation_name += encoder_impl_info.implementation_name; + + encoder_info.supports_native_handle = + encoder_impl_info.supports_native_handle; + encoder_info.has_trusted_rate_controller = + encoder_impl_info.has_trusted_rate_controller; + encoder_info.is_hardware_accelerated = + encoder_impl_info.is_hardware_accelerated; + encoder_info.has_internal_source = encoder_impl_info.has_internal_source; + } else { + encoder_info.implementation_name += ", "; + encoder_info.implementation_name += encoder_impl_info.implementation_name; + + // Native handle supported if any encoder supports it. + encoder_info.supports_native_handle |= + encoder_impl_info.supports_native_handle; + + // Trusted rate controller only if all encoders have it. + encoder_info.has_trusted_rate_controller &= + encoder_impl_info.has_trusted_rate_controller; + + // Uses hardware support if any of the encoders uses it. + // For example, if we are having issues with down-scaling due to + // pipelining delay in HW encoders we need higher encoder usage + // thresholds in CPU adaptation. + encoder_info.is_hardware_accelerated |= + encoder_impl_info.is_hardware_accelerated; + + // Has internal source only if all encoders have it. + encoder_info.has_internal_source &= encoder_impl_info.has_internal_source; + } + encoder_info.fps_allocation[i] = encoder_impl_info.fps_allocation[0]; + encoder_info.requested_resolution_alignment = cricket::LeastCommonMultiple( + encoder_info.requested_resolution_alignment, + encoder_impl_info.requested_resolution_alignment); + if (num_active_streams == 1 && codec_.simulcastStream[i].active) { + encoder_info.scaling_settings = encoder_impl_info.scaling_settings; + } + } + encoder_info.implementation_name += ")"; + + return encoder_info; +} + +} // namespace webrtc diff --git a/media/engine/simulcast_encoder_adapter.h b/media/engine/simulcast_encoder_adapter.h new file mode 100644 index 0000000000..a4cf863151 --- /dev/null +++ b/media/engine/simulcast_encoder_adapter.h @@ -0,0 +1,141 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + * + */ + +#ifndef MEDIA_ENGINE_SIMULCAST_ENCODER_ADAPTER_H_ +#define MEDIA_ENGINE_SIMULCAST_ENCODER_ADAPTER_H_ + +#include <memory> +#include <stack> +#include <string> +#include <utility> +#include <vector> + +#include "absl/types/optional.h" +#include "api/fec_controller_override.h" +#include "api/video_codecs/sdp_video_format.h" +#include "api/video_codecs/video_encoder.h" +#include "modules/video_coding/include/video_codec_interface.h" +#include "modules/video_coding/utility/framerate_controller.h" +#include "rtc_base/atomic_ops.h" +#include "rtc_base/synchronization/sequence_checker.h" +#include "rtc_base/system/rtc_export.h" + +namespace webrtc { + +class SimulcastRateAllocator; +class VideoEncoderFactory; + +// SimulcastEncoderAdapter implements simulcast support by creating multiple +// webrtc::VideoEncoder instances with the given VideoEncoderFactory. +// The object is created and destroyed on the worker thread, but all public +// interfaces should be called from the encoder task queue. +class RTC_EXPORT SimulcastEncoderAdapter : public VideoEncoder { + public: + // TODO(bugs.webrtc.org/11000): Remove when downstream usage is gone. + SimulcastEncoderAdapter(VideoEncoderFactory* primarty_factory, + const SdpVideoFormat& format); + // |primary_factory| produces the first-choice encoders to use. + // |fallback_factory|, if non-null, is used to create fallback encoder that + // will be used if InitEncode() fails for the primary encoder. + SimulcastEncoderAdapter(VideoEncoderFactory* primary_factory, + VideoEncoderFactory* fallback_factory, + const SdpVideoFormat& format); + ~SimulcastEncoderAdapter() override; + + // Implements VideoEncoder. + void SetFecControllerOverride( + FecControllerOverride* fec_controller_override) override; + int Release() override; + int InitEncode(const VideoCodec* codec_settings, + const VideoEncoder::Settings& settings) override; + int Encode(const VideoFrame& input_image, + const std::vector<VideoFrameType>* frame_types) override; + int RegisterEncodeCompleteCallback(EncodedImageCallback* callback) override; + void SetRates(const RateControlParameters& parameters) override; + void OnPacketLossRateUpdate(float packet_loss_rate) override; + void OnRttUpdate(int64_t rtt_ms) override; + void OnLossNotification(const LossNotification& loss_notification) override; + + // Eventual handler for the contained encoders' EncodedImageCallbacks, but + // called from an internal helper that also knows the correct stream + // index. + EncodedImageCallback::Result OnEncodedImage( + size_t stream_idx, + const EncodedImage& encoded_image, + const CodecSpecificInfo* codec_specific_info, + const RTPFragmentationHeader* fragmentation); + + EncoderInfo GetEncoderInfo() const override; + + private: + struct StreamInfo { + StreamInfo(std::unique_ptr<VideoEncoder> encoder, + std::unique_ptr<EncodedImageCallback> callback, + std::unique_ptr<FramerateController> framerate_controller, + uint16_t width, + uint16_t height, + bool send_stream) + : encoder(std::move(encoder)), + callback(std::move(callback)), + framerate_controller(std::move(framerate_controller)), + width(width), + height(height), + key_frame_request(false), + send_stream(send_stream) {} + std::unique_ptr<VideoEncoder> encoder; + std::unique_ptr<EncodedImageCallback> callback; + std::unique_ptr<FramerateController> framerate_controller; + uint16_t width; + uint16_t height; + bool key_frame_request; + bool send_stream; + }; + + enum class StreamResolution { + OTHER, + HIGHEST, + LOWEST, + }; + + // Populate the codec settings for each simulcast stream. + void PopulateStreamCodec(const webrtc::VideoCodec& inst, + int stream_index, + uint32_t start_bitrate_kbps, + StreamResolution stream_resolution, + webrtc::VideoCodec* stream_codec); + + bool Initialized() const; + + void DestroyStoredEncoders(); + + volatile int inited_; // Accessed atomically. + VideoEncoderFactory* const primary_encoder_factory_; + VideoEncoderFactory* const fallback_encoder_factory_; + const SdpVideoFormat video_format_; + VideoCodec codec_; + std::vector<StreamInfo> streaminfos_; + EncodedImageCallback* encoded_complete_callback_; + + // Used for checking the single-threaded access of the encoder interface. + SequenceChecker encoder_queue_; + + // Store encoders in between calls to Release and InitEncode, so they don't + // have to be recreated. Remaining encoders are destroyed by the destructor. + std::stack<std::unique_ptr<VideoEncoder>> stored_encoders_; + + const absl::optional<unsigned int> experimental_boosted_screenshare_qp_; + const bool boost_base_layer_quality_; + const bool prefer_temporal_support_on_base_layer_; +}; + +} // namespace webrtc + +#endif // MEDIA_ENGINE_SIMULCAST_ENCODER_ADAPTER_H_ diff --git a/media/engine/simulcast_encoder_adapter_unittest.cc b/media/engine/simulcast_encoder_adapter_unittest.cc new file mode 100644 index 0000000000..b467c49166 --- /dev/null +++ b/media/engine/simulcast_encoder_adapter_unittest.cc @@ -0,0 +1,1529 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "media/engine/simulcast_encoder_adapter.h" + +#include <array> +#include <memory> +#include <vector> + +#include "api/test/create_simulcast_test_fixture.h" +#include "api/test/simulcast_test_fixture.h" +#include "api/test/video/function_video_decoder_factory.h" +#include "api/test/video/function_video_encoder_factory.h" +#include "api/video_codecs/sdp_video_format.h" +#include "api/video_codecs/video_encoder.h" +#include "api/video_codecs/video_encoder_factory.h" +#include "common_video/include/video_frame_buffer.h" +#include "media/base/media_constants.h" +#include "media/engine/internal_encoder_factory.h" +#include "modules/video_coding/codecs/vp8/include/vp8.h" +#include "modules/video_coding/include/video_codec_interface.h" +#include "modules/video_coding/utility/simulcast_test_fixture_impl.h" +#include "rtc_base/checks.h" +#include "test/gmock.h" +#include "test/gtest.h" + +using ::testing::_; +using ::testing::Return; +using EncoderInfo = webrtc::VideoEncoder::EncoderInfo; +using FramerateFractions = + absl::InlinedVector<uint8_t, webrtc::kMaxTemporalStreams>; + +namespace webrtc { +namespace test { + +namespace { + +constexpr int kDefaultWidth = 1280; +constexpr int kDefaultHeight = 720; + +const VideoEncoder::Capabilities kCapabilities(false); +const VideoEncoder::Settings kSettings(kCapabilities, 1, 1200); + +std::unique_ptr<SimulcastTestFixture> CreateSpecificSimulcastTestFixture( + VideoEncoderFactory* internal_encoder_factory) { + std::unique_ptr<VideoEncoderFactory> encoder_factory = + std::make_unique<FunctionVideoEncoderFactory>( + [internal_encoder_factory]() { + return std::make_unique<SimulcastEncoderAdapter>( + internal_encoder_factory, + SdpVideoFormat(cricket::kVp8CodecName)); + }); + std::unique_ptr<VideoDecoderFactory> decoder_factory = + std::make_unique<FunctionVideoDecoderFactory>( + []() { return VP8Decoder::Create(); }); + return CreateSimulcastTestFixture(std::move(encoder_factory), + std::move(decoder_factory), + SdpVideoFormat(cricket::kVp8CodecName)); +} +} // namespace + +TEST(SimulcastEncoderAdapterSimulcastTest, TestKeyFrameRequestsOnAllStreams) { + InternalEncoderFactory internal_encoder_factory; + auto fixture = CreateSpecificSimulcastTestFixture(&internal_encoder_factory); + fixture->TestKeyFrameRequestsOnAllStreams(); +} + +TEST(SimulcastEncoderAdapterSimulcastTest, TestPaddingAllStreams) { + InternalEncoderFactory internal_encoder_factory; + auto fixture = CreateSpecificSimulcastTestFixture(&internal_encoder_factory); + fixture->TestPaddingAllStreams(); +} + +TEST(SimulcastEncoderAdapterSimulcastTest, TestPaddingTwoStreams) { + InternalEncoderFactory internal_encoder_factory; + auto fixture = CreateSpecificSimulcastTestFixture(&internal_encoder_factory); + fixture->TestPaddingTwoStreams(); +} + +TEST(SimulcastEncoderAdapterSimulcastTest, TestPaddingTwoStreamsOneMaxedOut) { + InternalEncoderFactory internal_encoder_factory; + auto fixture = CreateSpecificSimulcastTestFixture(&internal_encoder_factory); + fixture->TestPaddingTwoStreamsOneMaxedOut(); +} + +TEST(SimulcastEncoderAdapterSimulcastTest, TestPaddingOneStream) { + InternalEncoderFactory internal_encoder_factory; + auto fixture = CreateSpecificSimulcastTestFixture(&internal_encoder_factory); + fixture->TestPaddingOneStream(); +} + +TEST(SimulcastEncoderAdapterSimulcastTest, TestPaddingOneStreamTwoMaxedOut) { + InternalEncoderFactory internal_encoder_factory; + auto fixture = CreateSpecificSimulcastTestFixture(&internal_encoder_factory); + fixture->TestPaddingOneStreamTwoMaxedOut(); +} + +TEST(SimulcastEncoderAdapterSimulcastTest, TestSendAllStreams) { + InternalEncoderFactory internal_encoder_factory; + auto fixture = CreateSpecificSimulcastTestFixture(&internal_encoder_factory); + fixture->TestSendAllStreams(); +} + +TEST(SimulcastEncoderAdapterSimulcastTest, TestDisablingStreams) { + InternalEncoderFactory internal_encoder_factory; + auto fixture = CreateSpecificSimulcastTestFixture(&internal_encoder_factory); + fixture->TestDisablingStreams(); +} + +TEST(SimulcastEncoderAdapterSimulcastTest, TestActiveStreams) { + InternalEncoderFactory internal_encoder_factory; + auto fixture = CreateSpecificSimulcastTestFixture(&internal_encoder_factory); + fixture->TestActiveStreams(); +} + +TEST(SimulcastEncoderAdapterSimulcastTest, TestSwitchingToOneStream) { + InternalEncoderFactory internal_encoder_factory; + auto fixture = CreateSpecificSimulcastTestFixture(&internal_encoder_factory); + fixture->TestSwitchingToOneStream(); +} + +TEST(SimulcastEncoderAdapterSimulcastTest, TestSwitchingToOneOddStream) { + InternalEncoderFactory internal_encoder_factory; + auto fixture = CreateSpecificSimulcastTestFixture(&internal_encoder_factory); + fixture->TestSwitchingToOneOddStream(); +} + +TEST(SimulcastEncoderAdapterSimulcastTest, TestStrideEncodeDecode) { + InternalEncoderFactory internal_encoder_factory; + auto fixture = CreateSpecificSimulcastTestFixture(&internal_encoder_factory); + fixture->TestStrideEncodeDecode(); +} + +TEST(SimulcastEncoderAdapterSimulcastTest, + TestSpatioTemporalLayers333PatternEncoder) { + InternalEncoderFactory internal_encoder_factory; + auto fixture = CreateSpecificSimulcastTestFixture(&internal_encoder_factory); + fixture->TestSpatioTemporalLayers333PatternEncoder(); +} + +TEST(SimulcastEncoderAdapterSimulcastTest, + TestSpatioTemporalLayers321PatternEncoder) { + InternalEncoderFactory internal_encoder_factory; + auto fixture = CreateSpecificSimulcastTestFixture(&internal_encoder_factory); + fixture->TestSpatioTemporalLayers321PatternEncoder(); +} + +TEST(SimulcastEncoderAdapterSimulcastTest, TestDecodeWidthHeightSet) { + InternalEncoderFactory internal_encoder_factory; + auto fixture = CreateSpecificSimulcastTestFixture(&internal_encoder_factory); + fixture->TestDecodeWidthHeightSet(); +} + +class MockVideoEncoder; + +class MockVideoEncoderFactory : public VideoEncoderFactory { + public: + std::vector<SdpVideoFormat> GetSupportedFormats() const override; + + std::unique_ptr<VideoEncoder> CreateVideoEncoder( + const SdpVideoFormat& format) override; + + CodecInfo QueryVideoEncoder(const SdpVideoFormat& format) const override; + + const std::vector<MockVideoEncoder*>& encoders() const; + void SetEncoderNames(const std::vector<const char*>& encoder_names); + void set_init_encode_return_value(int32_t value); + void set_requested_resolution_alignments( + std::vector<int> requested_resolution_alignments) { + requested_resolution_alignments_ = requested_resolution_alignments; + } + void set_supports_simulcast(bool supports_simulcast) { + supports_simulcast_ = supports_simulcast; + } + + void DestroyVideoEncoder(VideoEncoder* encoder); + + private: + int32_t init_encode_return_value_ = 0; + std::vector<MockVideoEncoder*> encoders_; + std::vector<const char*> encoder_names_; + // Keep number of entries in sync with |kMaxSimulcastStreams|. + std::vector<int> requested_resolution_alignments_ = {1, 1, 1}; + bool supports_simulcast_ = false; +}; + +class MockVideoEncoder : public VideoEncoder { + public: + explicit MockVideoEncoder(MockVideoEncoderFactory* factory) + : factory_(factory), + scaling_settings_(VideoEncoder::ScalingSettings::kOff), + video_format_("unknown"), + callback_(nullptr) {} + + MOCK_METHOD1(SetFecControllerOverride, + void(FecControllerOverride* fec_controller_override)); + + // TODO(nisse): Valid overrides commented out, because the gmock + // methods don't use any override declarations, and we want to avoid + // warnings from -Winconsistent-missing-override. See + // http://crbug.com/428099. + int32_t InitEncode(const VideoCodec* codecSettings, + const VideoEncoder::Settings& settings) override { + codec_ = *codecSettings; + return init_encode_return_value_; + } + + MOCK_METHOD2( + Encode, + int32_t(const VideoFrame& inputImage, + const std::vector<VideoFrameType>* frame_types) /* override */); + + int32_t RegisterEncodeCompleteCallback( + EncodedImageCallback* callback) override { + callback_ = callback; + return 0; + } + + MOCK_METHOD0(Release, int32_t() /* override */); + + void SetRates(const RateControlParameters& parameters) { + last_set_rates_ = parameters; + } + + EncoderInfo GetEncoderInfo() const override { + EncoderInfo info; + info.supports_native_handle = supports_native_handle_; + info.implementation_name = implementation_name_; + info.scaling_settings = scaling_settings_; + info.requested_resolution_alignment = requested_resolution_alignment_; + info.has_trusted_rate_controller = has_trusted_rate_controller_; + info.is_hardware_accelerated = is_hardware_accelerated_; + info.has_internal_source = has_internal_source_; + info.fps_allocation[0] = fps_allocation_; + info.supports_simulcast = supports_simulcast_; + return info; + } + + virtual ~MockVideoEncoder() { factory_->DestroyVideoEncoder(this); } + + const VideoCodec& codec() const { return codec_; } + + void SendEncodedImage(int width, int height) { + // Sends a fake image of the given width/height. + EncodedImage image; + image._encodedWidth = width; + image._encodedHeight = height; + CodecSpecificInfo codec_specific_info; + codec_specific_info.codecType = webrtc::kVideoCodecVP8; + callback_->OnEncodedImage(image, &codec_specific_info, nullptr); + } + + void set_supports_native_handle(bool enabled) { + supports_native_handle_ = enabled; + } + + void set_implementation_name(const std::string& name) { + implementation_name_ = name; + } + + void set_init_encode_return_value(int32_t value) { + init_encode_return_value_ = value; + } + + void set_scaling_settings(const VideoEncoder::ScalingSettings& settings) { + scaling_settings_ = settings; + } + + void set_requested_resolution_alignment(int requested_resolution_alignment) { + requested_resolution_alignment_ = requested_resolution_alignment; + } + + void set_has_trusted_rate_controller(bool trusted) { + has_trusted_rate_controller_ = trusted; + } + + void set_is_hardware_accelerated(bool is_hardware_accelerated) { + is_hardware_accelerated_ = is_hardware_accelerated; + } + + void set_has_internal_source(bool has_internal_source) { + has_internal_source_ = has_internal_source; + } + + void set_fps_allocation(const FramerateFractions& fps_allocation) { + fps_allocation_ = fps_allocation; + } + + RateControlParameters last_set_rates() const { return last_set_rates_; } + + void set_supports_simulcast(bool supports_simulcast) { + supports_simulcast_ = supports_simulcast; + } + + void set_video_format(const SdpVideoFormat& video_format) { + video_format_ = video_format; + } + + bool supports_simulcast() const { return supports_simulcast_; } + + SdpVideoFormat video_format() const { return video_format_; } + + private: + MockVideoEncoderFactory* const factory_; + bool supports_native_handle_ = false; + std::string implementation_name_ = "unknown"; + VideoEncoder::ScalingSettings scaling_settings_; + int requested_resolution_alignment_ = 1; + bool has_trusted_rate_controller_ = false; + bool is_hardware_accelerated_ = false; + bool has_internal_source_ = false; + int32_t init_encode_return_value_ = 0; + VideoEncoder::RateControlParameters last_set_rates_; + FramerateFractions fps_allocation_; + bool supports_simulcast_ = false; + SdpVideoFormat video_format_; + + VideoCodec codec_; + EncodedImageCallback* callback_; +}; + +std::vector<SdpVideoFormat> MockVideoEncoderFactory::GetSupportedFormats() + const { + std::vector<SdpVideoFormat> formats = {SdpVideoFormat("VP8")}; + return formats; +} + +std::unique_ptr<VideoEncoder> MockVideoEncoderFactory::CreateVideoEncoder( + const SdpVideoFormat& format) { + std::unique_ptr<MockVideoEncoder> encoder( + new ::testing::NiceMock<MockVideoEncoder>(this)); + encoder->set_init_encode_return_value(init_encode_return_value_); + const char* encoder_name = encoder_names_.empty() + ? "codec_implementation_name" + : encoder_names_[encoders_.size()]; + encoder->set_implementation_name(encoder_name); + RTC_CHECK_LT(encoders_.size(), requested_resolution_alignments_.size()); + encoder->set_requested_resolution_alignment( + requested_resolution_alignments_[encoders_.size()]); + encoder->set_supports_simulcast(supports_simulcast_); + encoder->set_video_format(format); + encoders_.push_back(encoder.get()); + return encoder; +} + +void MockVideoEncoderFactory::DestroyVideoEncoder(VideoEncoder* encoder) { + for (size_t i = 0; i < encoders_.size(); ++i) { + if (encoders_[i] == encoder) { + encoders_.erase(encoders_.begin() + i); + break; + } + } +} + +VideoEncoderFactory::CodecInfo MockVideoEncoderFactory::QueryVideoEncoder( + const SdpVideoFormat& format) const { + return CodecInfo(); +} + +const std::vector<MockVideoEncoder*>& MockVideoEncoderFactory::encoders() + const { + return encoders_; +} +void MockVideoEncoderFactory::SetEncoderNames( + const std::vector<const char*>& encoder_names) { + encoder_names_ = encoder_names; +} +void MockVideoEncoderFactory::set_init_encode_return_value(int32_t value) { + init_encode_return_value_ = value; +} + +class TestSimulcastEncoderAdapterFakeHelper { + public: + explicit TestSimulcastEncoderAdapterFakeHelper( + bool use_fallback_factory, + const SdpVideoFormat& video_format) + : primary_factory_(new MockVideoEncoderFactory()), + fallback_factory_(use_fallback_factory ? new MockVideoEncoderFactory() + : nullptr), + video_format_(video_format) {} + + // Can only be called once as the SimulcastEncoderAdapter will take the + // ownership of |factory_|. + VideoEncoder* CreateMockEncoderAdapter() { + return new SimulcastEncoderAdapter(primary_factory_.get(), + fallback_factory_.get(), video_format_); + } + + MockVideoEncoderFactory* factory() { return primary_factory_.get(); } + MockVideoEncoderFactory* fallback_factory() { + return fallback_factory_.get(); + } + + private: + std::unique_ptr<MockVideoEncoderFactory> primary_factory_; + std::unique_ptr<MockVideoEncoderFactory> fallback_factory_; + SdpVideoFormat video_format_; +}; + +static const int kTestTemporalLayerProfile[3] = {3, 2, 1}; + +class TestSimulcastEncoderAdapterFake : public ::testing::Test, + public EncodedImageCallback { + public: + TestSimulcastEncoderAdapterFake() + : last_encoded_image_width_(-1), + last_encoded_image_height_(-1), + last_encoded_image_simulcast_index_(-1), + use_fallback_factory_(false) {} + + virtual ~TestSimulcastEncoderAdapterFake() { + if (adapter_) { + adapter_->Release(); + } + } + + void SetUp() override { + helper_ = std::make_unique<TestSimulcastEncoderAdapterFakeHelper>( + use_fallback_factory_, SdpVideoFormat("VP8", sdp_video_parameters_)); + adapter_.reset(helper_->CreateMockEncoderAdapter()); + last_encoded_image_width_ = -1; + last_encoded_image_height_ = -1; + last_encoded_image_simulcast_index_ = -1; + } + + Result OnEncodedImage(const EncodedImage& encoded_image, + const CodecSpecificInfo* codec_specific_info, + const RTPFragmentationHeader* fragmentation) override { + last_encoded_image_width_ = encoded_image._encodedWidth; + last_encoded_image_height_ = encoded_image._encodedHeight; + last_encoded_image_simulcast_index_ = + encoded_image.SpatialIndex().value_or(-1); + + return Result(Result::OK, encoded_image.Timestamp()); + } + + bool GetLastEncodedImageInfo(int* out_width, + int* out_height, + int* out_simulcast_index) { + if (last_encoded_image_width_ == -1) { + return false; + } + *out_width = last_encoded_image_width_; + *out_height = last_encoded_image_height_; + *out_simulcast_index = last_encoded_image_simulcast_index_; + return true; + } + + void SetupCodec() { + SimulcastTestFixtureImpl::DefaultSettings( + &codec_, static_cast<const int*>(kTestTemporalLayerProfile), + kVideoCodecVP8); + rate_allocator_.reset(new SimulcastRateAllocator(codec_)); + EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); + adapter_->RegisterEncodeCompleteCallback(this); + } + + void VerifyCodec(const VideoCodec& ref, int stream_index) { + const VideoCodec& target = + helper_->factory()->encoders()[stream_index]->codec(); + EXPECT_EQ(ref.codecType, target.codecType); + EXPECT_EQ(ref.plType, target.plType); + EXPECT_EQ(ref.width, target.width); + EXPECT_EQ(ref.height, target.height); + EXPECT_EQ(ref.startBitrate, target.startBitrate); + EXPECT_EQ(ref.maxBitrate, target.maxBitrate); + EXPECT_EQ(ref.minBitrate, target.minBitrate); + EXPECT_EQ(ref.maxFramerate, target.maxFramerate); + EXPECT_EQ(ref.VP8().complexity, target.VP8().complexity); + EXPECT_EQ(ref.VP8().numberOfTemporalLayers, + target.VP8().numberOfTemporalLayers); + EXPECT_EQ(ref.VP8().denoisingOn, target.VP8().denoisingOn); + EXPECT_EQ(ref.VP8().automaticResizeOn, target.VP8().automaticResizeOn); + EXPECT_EQ(ref.VP8().frameDroppingOn, target.VP8().frameDroppingOn); + EXPECT_EQ(ref.VP8().keyFrameInterval, target.VP8().keyFrameInterval); + EXPECT_EQ(ref.qpMax, target.qpMax); + EXPECT_EQ(0, target.numberOfSimulcastStreams); + EXPECT_EQ(ref.mode, target.mode); + + // No need to compare simulcastStream as numberOfSimulcastStreams should + // always be 0. + } + + void InitRefCodec(int stream_index, + VideoCodec* ref_codec, + bool reverse_layer_order = false) { + *ref_codec = codec_; + ref_codec->VP8()->numberOfTemporalLayers = + kTestTemporalLayerProfile[reverse_layer_order ? 2 - stream_index + : stream_index]; + ref_codec->width = codec_.simulcastStream[stream_index].width; + ref_codec->height = codec_.simulcastStream[stream_index].height; + ref_codec->maxBitrate = codec_.simulcastStream[stream_index].maxBitrate; + ref_codec->minBitrate = codec_.simulcastStream[stream_index].minBitrate; + ref_codec->qpMax = codec_.simulcastStream[stream_index].qpMax; + } + + void VerifyCodecSettings() { + EXPECT_EQ(3u, helper_->factory()->encoders().size()); + VideoCodec ref_codec; + + // stream 0, the lowest resolution stream. + InitRefCodec(0, &ref_codec); + ref_codec.qpMax = 45; + ref_codec.VP8()->complexity = + webrtc::VideoCodecComplexity::kComplexityHigher; + ref_codec.VP8()->denoisingOn = false; + ref_codec.startBitrate = 100; // Should equal to the target bitrate. + VerifyCodec(ref_codec, 0); + + // stream 1 + InitRefCodec(1, &ref_codec); + ref_codec.VP8()->denoisingOn = false; + // The start bitrate (300kbit) minus what we have for the lower layers + // (100kbit). + ref_codec.startBitrate = 200; + VerifyCodec(ref_codec, 1); + + // stream 2, the biggest resolution stream. + InitRefCodec(2, &ref_codec); + // We don't have enough bits to send this, so the adapter should have + // configured it to use the min bitrate for this layer (600kbit) but turn + // off sending. + ref_codec.startBitrate = 600; + VerifyCodec(ref_codec, 2); + } + + protected: + std::unique_ptr<TestSimulcastEncoderAdapterFakeHelper> helper_; + std::unique_ptr<VideoEncoder> adapter_; + VideoCodec codec_; + int last_encoded_image_width_; + int last_encoded_image_height_; + int last_encoded_image_simulcast_index_; + std::unique_ptr<SimulcastRateAllocator> rate_allocator_; + bool use_fallback_factory_; + SdpVideoFormat::Parameters sdp_video_parameters_; +}; + +TEST_F(TestSimulcastEncoderAdapterFake, InitEncode) { + SetupCodec(); + VerifyCodecSettings(); +} + +TEST_F(TestSimulcastEncoderAdapterFake, ReleaseWithoutInitEncode) { + EXPECT_EQ(0, adapter_->Release()); +} + +TEST_F(TestSimulcastEncoderAdapterFake, Reinit) { + SetupCodec(); + EXPECT_EQ(0, adapter_->Release()); + + EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); +} + +TEST_F(TestSimulcastEncoderAdapterFake, EncodedCallbackForDifferentEncoders) { + SetupCodec(); + + // Set bitrates so that we send all layers. + adapter_->SetRates(VideoEncoder::RateControlParameters( + rate_allocator_->Allocate(VideoBitrateAllocationParameters(1200, 30)), + 30.0)); + + // At this point, the simulcast encoder adapter should have 3 streams: HD, + // quarter HD, and quarter quarter HD. We're going to mostly ignore the exact + // resolutions, to test that the adapter forwards on the correct resolution + // and simulcast index values, going only off the encoder that generates the + // image. + std::vector<MockVideoEncoder*> encoders = helper_->factory()->encoders(); + ASSERT_EQ(3u, encoders.size()); + encoders[0]->SendEncodedImage(1152, 704); + int width; + int height; + int simulcast_index; + EXPECT_TRUE(GetLastEncodedImageInfo(&width, &height, &simulcast_index)); + EXPECT_EQ(1152, width); + EXPECT_EQ(704, height); + EXPECT_EQ(0, simulcast_index); + + encoders[1]->SendEncodedImage(300, 620); + EXPECT_TRUE(GetLastEncodedImageInfo(&width, &height, &simulcast_index)); + EXPECT_EQ(300, width); + EXPECT_EQ(620, height); + EXPECT_EQ(1, simulcast_index); + + encoders[2]->SendEncodedImage(120, 240); + EXPECT_TRUE(GetLastEncodedImageInfo(&width, &height, &simulcast_index)); + EXPECT_EQ(120, width); + EXPECT_EQ(240, height); + EXPECT_EQ(2, simulcast_index); +} + +// This test verifies that the underlying encoders are reused, when the adapter +// is reinited with different number of simulcast streams. It further checks +// that the allocated encoders are reused in the same order as before, starting +// with the lowest stream. +TEST_F(TestSimulcastEncoderAdapterFake, ReusesEncodersInOrder) { + // Set up common settings for three streams. + SimulcastTestFixtureImpl::DefaultSettings( + &codec_, static_cast<const int*>(kTestTemporalLayerProfile), + kVideoCodecVP8); + rate_allocator_.reset(new SimulcastRateAllocator(codec_)); + adapter_->RegisterEncodeCompleteCallback(this); + const uint32_t target_bitrate = + 1000 * (codec_.simulcastStream[0].targetBitrate + + codec_.simulcastStream[1].targetBitrate + + codec_.simulcastStream[2].minBitrate); + + // Input data. + rtc::scoped_refptr<VideoFrameBuffer> buffer(I420Buffer::Create(1280, 720)); + VideoFrame input_frame = VideoFrame::Builder() + .set_video_frame_buffer(buffer) + .set_timestamp_rtp(100) + .set_timestamp_ms(1000) + .set_rotation(kVideoRotation_180) + .build(); + std::vector<VideoFrameType> frame_types; + + // Encode with three streams. + EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); + VerifyCodecSettings(); + adapter_->SetRates(VideoEncoder::RateControlParameters( + rate_allocator_->Allocate( + VideoBitrateAllocationParameters(target_bitrate, 30)), + 30.0)); + + std::vector<MockVideoEncoder*> original_encoders = + helper_->factory()->encoders(); + ASSERT_EQ(3u, original_encoders.size()); + EXPECT_CALL(*original_encoders[0], Encode(_, _)) + .WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); + EXPECT_CALL(*original_encoders[1], Encode(_, _)) + .WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); + EXPECT_CALL(*original_encoders[2], Encode(_, _)) + .WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); + frame_types.resize(3, VideoFrameType::kVideoFrameKey); + EXPECT_EQ(0, adapter_->Encode(input_frame, &frame_types)); + EXPECT_CALL(*original_encoders[0], Release()) + .WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); + EXPECT_CALL(*original_encoders[1], Release()) + .WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); + EXPECT_CALL(*original_encoders[2], Release()) + .WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); + EXPECT_EQ(0, adapter_->Release()); + + // Encode with two streams. + codec_.width /= 2; + codec_.height /= 2; + codec_.numberOfSimulcastStreams = 2; + EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); + adapter_->SetRates(VideoEncoder::RateControlParameters( + rate_allocator_->Allocate( + VideoBitrateAllocationParameters(target_bitrate, 30)), + 30.0)); + std::vector<MockVideoEncoder*> new_encoders = helper_->factory()->encoders(); + ASSERT_EQ(2u, new_encoders.size()); + ASSERT_EQ(original_encoders[0], new_encoders[0]); + EXPECT_CALL(*original_encoders[0], Encode(_, _)) + .WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); + ASSERT_EQ(original_encoders[1], new_encoders[1]); + EXPECT_CALL(*original_encoders[1], Encode(_, _)) + .WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); + frame_types.resize(2, VideoFrameType::kVideoFrameKey); + EXPECT_EQ(0, adapter_->Encode(input_frame, &frame_types)); + EXPECT_CALL(*original_encoders[0], Release()) + .WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); + EXPECT_CALL(*original_encoders[1], Release()) + .WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); + EXPECT_EQ(0, adapter_->Release()); + + // Encode with single stream. + codec_.width /= 2; + codec_.height /= 2; + codec_.numberOfSimulcastStreams = 1; + EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); + adapter_->SetRates(VideoEncoder::RateControlParameters( + rate_allocator_->Allocate( + VideoBitrateAllocationParameters(target_bitrate, 30)), + 30.0)); + new_encoders = helper_->factory()->encoders(); + ASSERT_EQ(1u, new_encoders.size()); + ASSERT_EQ(original_encoders[0], new_encoders[0]); + EXPECT_CALL(*original_encoders[0], Encode(_, _)) + .WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); + frame_types.resize(1, VideoFrameType::kVideoFrameKey); + EXPECT_EQ(0, adapter_->Encode(input_frame, &frame_types)); + EXPECT_CALL(*original_encoders[0], Release()) + .WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); + EXPECT_EQ(0, adapter_->Release()); + + // Encode with three streams, again. + codec_.width *= 4; + codec_.height *= 4; + codec_.numberOfSimulcastStreams = 3; + EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); + adapter_->SetRates(VideoEncoder::RateControlParameters( + rate_allocator_->Allocate( + VideoBitrateAllocationParameters(target_bitrate, 30)), + 30.0)); + new_encoders = helper_->factory()->encoders(); + ASSERT_EQ(3u, new_encoders.size()); + // The first encoder is reused. + ASSERT_EQ(original_encoders[0], new_encoders[0]); + EXPECT_CALL(*original_encoders[0], Encode(_, _)) + .WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); + // The second and third encoders are new. + EXPECT_CALL(*new_encoders[1], Encode(_, _)) + .WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); + EXPECT_CALL(*new_encoders[2], Encode(_, _)) + .WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); + frame_types.resize(3, VideoFrameType::kVideoFrameKey); + EXPECT_EQ(0, adapter_->Encode(input_frame, &frame_types)); + EXPECT_CALL(*original_encoders[0], Release()) + .WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); + EXPECT_CALL(*new_encoders[1], Release()) + .WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); + EXPECT_CALL(*new_encoders[2], Release()) + .WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); + EXPECT_EQ(0, adapter_->Release()); +} + +TEST_F(TestSimulcastEncoderAdapterFake, DoesNotLeakEncoders) { + SetupCodec(); + VerifyCodecSettings(); + + EXPECT_EQ(3u, helper_->factory()->encoders().size()); + + // The adapter should destroy all encoders it has allocated. Since + // |helper_->factory()| is owned by |adapter_|, however, we need to rely on + // lsan to find leaks here. + EXPECT_EQ(0, adapter_->Release()); + adapter_.reset(); +} + +// This test verifies that an adapter reinit with the same codec settings as +// before does not change the underlying encoder codec settings. +TEST_F(TestSimulcastEncoderAdapterFake, ReinitDoesNotReorderEncoderSettings) { + SetupCodec(); + VerifyCodecSettings(); + + // Capture current codec settings. + std::vector<MockVideoEncoder*> encoders = helper_->factory()->encoders(); + ASSERT_EQ(3u, encoders.size()); + std::array<VideoCodec, 3> codecs_before; + for (int i = 0; i < 3; ++i) { + codecs_before[i] = encoders[i]->codec(); + } + + // Reinitialize and verify that the new codec settings are the same. + EXPECT_EQ(0, adapter_->Release()); + EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); + for (int i = 0; i < 3; ++i) { + const VideoCodec& codec_before = codecs_before[i]; + const VideoCodec& codec_after = encoders[i]->codec(); + + // webrtc::VideoCodec does not implement operator==. + EXPECT_EQ(codec_before.codecType, codec_after.codecType); + EXPECT_EQ(codec_before.plType, codec_after.plType); + EXPECT_EQ(codec_before.width, codec_after.width); + EXPECT_EQ(codec_before.height, codec_after.height); + EXPECT_EQ(codec_before.startBitrate, codec_after.startBitrate); + EXPECT_EQ(codec_before.maxBitrate, codec_after.maxBitrate); + EXPECT_EQ(codec_before.minBitrate, codec_after.minBitrate); + EXPECT_EQ(codec_before.maxFramerate, codec_after.maxFramerate); + EXPECT_EQ(codec_before.qpMax, codec_after.qpMax); + EXPECT_EQ(codec_before.numberOfSimulcastStreams, + codec_after.numberOfSimulcastStreams); + EXPECT_EQ(codec_before.mode, codec_after.mode); + EXPECT_EQ(codec_before.expect_encode_from_texture, + codec_after.expect_encode_from_texture); + } +} + +// This test is similar to the one above, except that it tests the simulcastIdx +// from the CodecSpecificInfo that is connected to an encoded frame. The +// PayloadRouter demuxes the incoming encoded frames on different RTP modules +// using the simulcastIdx, so it's important that there is no corresponding +// encoder reordering in between adapter reinits as this would lead to PictureID +// discontinuities. +TEST_F(TestSimulcastEncoderAdapterFake, ReinitDoesNotReorderFrameSimulcastIdx) { + SetupCodec(); + adapter_->SetRates(VideoEncoder::RateControlParameters( + rate_allocator_->Allocate(VideoBitrateAllocationParameters(1200, 30)), + 30.0)); + VerifyCodecSettings(); + + // Send frames on all streams. + std::vector<MockVideoEncoder*> encoders = helper_->factory()->encoders(); + ASSERT_EQ(3u, encoders.size()); + encoders[0]->SendEncodedImage(1152, 704); + int width; + int height; + int simulcast_index; + EXPECT_TRUE(GetLastEncodedImageInfo(&width, &height, &simulcast_index)); + EXPECT_EQ(0, simulcast_index); + + encoders[1]->SendEncodedImage(300, 620); + EXPECT_TRUE(GetLastEncodedImageInfo(&width, &height, &simulcast_index)); + EXPECT_EQ(1, simulcast_index); + + encoders[2]->SendEncodedImage(120, 240); + EXPECT_TRUE(GetLastEncodedImageInfo(&width, &height, &simulcast_index)); + EXPECT_EQ(2, simulcast_index); + + // Reinitialize. + EXPECT_EQ(0, adapter_->Release()); + EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); + adapter_->SetRates(VideoEncoder::RateControlParameters( + rate_allocator_->Allocate(VideoBitrateAllocationParameters(1200, 30)), + 30.0)); + + // Verify that the same encoder sends out frames on the same simulcast index. + encoders[0]->SendEncodedImage(1152, 704); + EXPECT_TRUE(GetLastEncodedImageInfo(&width, &height, &simulcast_index)); + EXPECT_EQ(0, simulcast_index); + + encoders[1]->SendEncodedImage(300, 620); + EXPECT_TRUE(GetLastEncodedImageInfo(&width, &height, &simulcast_index)); + EXPECT_EQ(1, simulcast_index); + + encoders[2]->SendEncodedImage(120, 240); + EXPECT_TRUE(GetLastEncodedImageInfo(&width, &height, &simulcast_index)); + EXPECT_EQ(2, simulcast_index); +} + +TEST_F(TestSimulcastEncoderAdapterFake, SupportsNativeHandleForSingleStreams) { + SimulcastTestFixtureImpl::DefaultSettings( + &codec_, static_cast<const int*>(kTestTemporalLayerProfile), + kVideoCodecVP8); + codec_.numberOfSimulcastStreams = 1; + EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); + adapter_->RegisterEncodeCompleteCallback(this); + ASSERT_EQ(1u, helper_->factory()->encoders().size()); + helper_->factory()->encoders()[0]->set_supports_native_handle(true); + EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); + EXPECT_TRUE(adapter_->GetEncoderInfo().supports_native_handle); + helper_->factory()->encoders()[0]->set_supports_native_handle(false); + EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); + EXPECT_FALSE(adapter_->GetEncoderInfo().supports_native_handle); +} + +TEST_F(TestSimulcastEncoderAdapterFake, SetRatesUnderMinBitrate) { + SimulcastTestFixtureImpl::DefaultSettings( + &codec_, static_cast<const int*>(kTestTemporalLayerProfile), + kVideoCodecVP8); + codec_.minBitrate = 50; + codec_.numberOfSimulcastStreams = 1; + EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); + rate_allocator_.reset(new SimulcastRateAllocator(codec_)); + + // Above min should be respected. + VideoBitrateAllocation target_bitrate = rate_allocator_->Allocate( + VideoBitrateAllocationParameters(codec_.minBitrate * 1000, 30)); + adapter_->SetRates(VideoEncoder::RateControlParameters(target_bitrate, 30.0)); + EXPECT_EQ(target_bitrate, + helper_->factory()->encoders()[0]->last_set_rates().bitrate); + + // Below min but non-zero should be replaced with the min bitrate. + VideoBitrateAllocation too_low_bitrate = rate_allocator_->Allocate( + VideoBitrateAllocationParameters((codec_.minBitrate - 1) * 1000, 30)); + adapter_->SetRates( + VideoEncoder::RateControlParameters(too_low_bitrate, 30.0)); + EXPECT_EQ(target_bitrate, + helper_->factory()->encoders()[0]->last_set_rates().bitrate); + + // Zero should be passed on as is, since it means "pause". + adapter_->SetRates( + VideoEncoder::RateControlParameters(VideoBitrateAllocation(), 30.0)); + EXPECT_EQ(VideoBitrateAllocation(), + helper_->factory()->encoders()[0]->last_set_rates().bitrate); +} + +TEST_F(TestSimulcastEncoderAdapterFake, SupportsImplementationName) { + EXPECT_EQ("SimulcastEncoderAdapter", + adapter_->GetEncoderInfo().implementation_name); + SimulcastTestFixtureImpl::DefaultSettings( + &codec_, static_cast<const int*>(kTestTemporalLayerProfile), + kVideoCodecVP8); + std::vector<const char*> encoder_names; + encoder_names.push_back("codec1"); + encoder_names.push_back("codec2"); + encoder_names.push_back("codec3"); + helper_->factory()->SetEncoderNames(encoder_names); + EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); + EXPECT_EQ("SimulcastEncoderAdapter (codec1, codec2, codec3)", + adapter_->GetEncoderInfo().implementation_name); + + // Single streams should not expose "SimulcastEncoderAdapter" in name. + EXPECT_EQ(0, adapter_->Release()); + codec_.numberOfSimulcastStreams = 1; + EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); + adapter_->RegisterEncodeCompleteCallback(this); + ASSERT_EQ(1u, helper_->factory()->encoders().size()); + EXPECT_EQ("codec1", adapter_->GetEncoderInfo().implementation_name); +} + +TEST_F(TestSimulcastEncoderAdapterFake, RuntimeEncoderInfoUpdate) { + SimulcastTestFixtureImpl::DefaultSettings( + &codec_, static_cast<const int*>(kTestTemporalLayerProfile), + kVideoCodecVP8); + std::vector<const char*> encoder_names; + encoder_names.push_back("codec1"); + encoder_names.push_back("codec2"); + encoder_names.push_back("codec3"); + helper_->factory()->SetEncoderNames(encoder_names); + EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); + EXPECT_EQ("SimulcastEncoderAdapter (codec1, codec2, codec3)", + adapter_->GetEncoderInfo().implementation_name); + + // Change name of first encoder to indicate it has done a fallback to another + // implementation. + helper_->factory()->encoders().front()->set_implementation_name("fallback1"); + EXPECT_EQ("SimulcastEncoderAdapter (fallback1, codec2, codec3)", + adapter_->GetEncoderInfo().implementation_name); +} + +TEST_F(TestSimulcastEncoderAdapterFake, + SupportsNativeHandleForMultipleStreams) { + SimulcastTestFixtureImpl::DefaultSettings( + &codec_, static_cast<const int*>(kTestTemporalLayerProfile), + kVideoCodecVP8); + codec_.numberOfSimulcastStreams = 3; + EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); + adapter_->RegisterEncodeCompleteCallback(this); + ASSERT_EQ(3u, helper_->factory()->encoders().size()); + for (MockVideoEncoder* encoder : helper_->factory()->encoders()) + encoder->set_supports_native_handle(true); + // As long as one encoder supports native handle, it's enabled. + helper_->factory()->encoders()[0]->set_supports_native_handle(false); + EXPECT_TRUE(adapter_->GetEncoderInfo().supports_native_handle); + // Once none do, then the adapter claims no support. + helper_->factory()->encoders()[1]->set_supports_native_handle(false); + helper_->factory()->encoders()[2]->set_supports_native_handle(false); + EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); + EXPECT_FALSE(adapter_->GetEncoderInfo().supports_native_handle); +} + +// TODO(nisse): Reuse definition in webrtc/test/fake_texture_handle.h. +class FakeNativeBufferI420 : public VideoFrameBuffer { + public: + FakeNativeBufferI420(int width, int height, bool allow_to_i420) + : width_(width), height_(height), allow_to_i420_(allow_to_i420) {} + + Type type() const override { return Type::kNative; } + int width() const override { return width_; } + int height() const override { return height_; } + + rtc::scoped_refptr<I420BufferInterface> ToI420() override { + if (allow_to_i420_) { + return I420Buffer::Create(width_, height_); + } else { + RTC_NOTREACHED(); + } + return nullptr; + } + + private: + const int width_; + const int height_; + const bool allow_to_i420_; +}; + +TEST_F(TestSimulcastEncoderAdapterFake, + NativeHandleForwardingForMultipleStreams) { + SimulcastTestFixtureImpl::DefaultSettings( + &codec_, static_cast<const int*>(kTestTemporalLayerProfile), + kVideoCodecVP8); + codec_.numberOfSimulcastStreams = 3; + // High start bitrate, so all streams are enabled. + codec_.startBitrate = 3000; + EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); + adapter_->RegisterEncodeCompleteCallback(this); + ASSERT_EQ(3u, helper_->factory()->encoders().size()); + for (MockVideoEncoder* encoder : helper_->factory()->encoders()) + encoder->set_supports_native_handle(true); + EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); + EXPECT_TRUE(adapter_->GetEncoderInfo().supports_native_handle); + + rtc::scoped_refptr<VideoFrameBuffer> buffer( + new rtc::RefCountedObject<FakeNativeBufferI420>(1280, 720, + /*allow_to_i420=*/false)); + VideoFrame input_frame = VideoFrame::Builder() + .set_video_frame_buffer(buffer) + .set_timestamp_rtp(100) + .set_timestamp_ms(1000) + .set_rotation(kVideoRotation_180) + .build(); + // Expect calls with the given video frame verbatim, since it's a texture + // frame and can't otherwise be modified/resized. + for (MockVideoEncoder* encoder : helper_->factory()->encoders()) + EXPECT_CALL(*encoder, Encode(::testing::Ref(input_frame), _)).Times(1); + std::vector<VideoFrameType> frame_types(3, VideoFrameType::kVideoFrameKey); + EXPECT_EQ(0, adapter_->Encode(input_frame, &frame_types)); +} + +TEST_F(TestSimulcastEncoderAdapterFake, NativeHandleForwardingOnlyIfSupported) { + SimulcastTestFixtureImpl::DefaultSettings( + &codec_, static_cast<const int*>(kTestTemporalLayerProfile), + kVideoCodecVP8); + codec_.numberOfSimulcastStreams = 3; + // High start bitrate, so all streams are enabled. + codec_.startBitrate = 3000; + EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); + adapter_->RegisterEncodeCompleteCallback(this); + ASSERT_EQ(3u, helper_->factory()->encoders().size()); + + // QVGA encoders has fallen back to software. + auto& encoders = helper_->factory()->encoders(); + encoders[0]->set_supports_native_handle(false); + encoders[1]->set_supports_native_handle(true); + encoders[2]->set_supports_native_handle(true); + + EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); + EXPECT_TRUE(adapter_->GetEncoderInfo().supports_native_handle); + + rtc::scoped_refptr<VideoFrameBuffer> buffer( + new rtc::RefCountedObject<FakeNativeBufferI420>(1280, 720, + /*allow_to_i420=*/true)); + VideoFrame input_frame = VideoFrame::Builder() + .set_video_frame_buffer(buffer) + .set_timestamp_rtp(100) + .set_timestamp_ms(1000) + .set_rotation(kVideoRotation_180) + .build(); + // Expect calls with the given video frame verbatim, since it's a texture + // frame and can't otherwise be modified/resized, but only on the two + // streams supporting it... + EXPECT_CALL(*encoders[1], Encode(::testing::Ref(input_frame), _)).Times(1); + EXPECT_CALL(*encoders[2], Encode(::testing::Ref(input_frame), _)).Times(1); + // ...the lowest one gets a software buffer. + EXPECT_CALL(*encoders[0], Encode) + .WillOnce([&](const VideoFrame& frame, + const std::vector<VideoFrameType>* frame_types) { + EXPECT_EQ(frame.video_frame_buffer()->type(), + VideoFrameBuffer::Type::kI420); + return 0; + }); + std::vector<VideoFrameType> frame_types(3, VideoFrameType::kVideoFrameKey); + EXPECT_EQ(0, adapter_->Encode(input_frame, &frame_types)); +} + +TEST_F(TestSimulcastEncoderAdapterFake, TestFailureReturnCodesFromEncodeCalls) { + SimulcastTestFixtureImpl::DefaultSettings( + &codec_, static_cast<const int*>(kTestTemporalLayerProfile), + kVideoCodecVP8); + codec_.numberOfSimulcastStreams = 3; + EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); + adapter_->RegisterEncodeCompleteCallback(this); + ASSERT_EQ(3u, helper_->factory()->encoders().size()); + // Tell the 2nd encoder to request software fallback. + EXPECT_CALL(*helper_->factory()->encoders()[1], Encode(_, _)) + .WillOnce(Return(WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE)); + + // Send a fake frame and assert the return is software fallback. + rtc::scoped_refptr<I420Buffer> input_buffer = + I420Buffer::Create(kDefaultWidth, kDefaultHeight); + input_buffer->InitializeData(); + VideoFrame input_frame = VideoFrame::Builder() + .set_video_frame_buffer(input_buffer) + .set_timestamp_rtp(0) + .set_timestamp_us(0) + .set_rotation(kVideoRotation_0) + .build(); + std::vector<VideoFrameType> frame_types(3, VideoFrameType::kVideoFrameKey); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE, + adapter_->Encode(input_frame, &frame_types)); +} + +TEST_F(TestSimulcastEncoderAdapterFake, TestInitFailureCleansUpEncoders) { + SimulcastTestFixtureImpl::DefaultSettings( + &codec_, static_cast<const int*>(kTestTemporalLayerProfile), + kVideoCodecVP8); + codec_.numberOfSimulcastStreams = 3; + helper_->factory()->set_init_encode_return_value( + WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE, + adapter_->InitEncode(&codec_, kSettings)); + EXPECT_TRUE(helper_->factory()->encoders().empty()); +} + +TEST_F(TestSimulcastEncoderAdapterFake, DoesNotAlterMaxQpForScreenshare) { + const int kHighMaxQp = 56; + const int kLowMaxQp = 46; + + SimulcastTestFixtureImpl::DefaultSettings( + &codec_, static_cast<const int*>(kTestTemporalLayerProfile), + kVideoCodecVP8); + codec_.numberOfSimulcastStreams = 3; + codec_.simulcastStream[0].qpMax = kHighMaxQp; + codec_.mode = VideoCodecMode::kScreensharing; + + EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); + EXPECT_EQ(3u, helper_->factory()->encoders().size()); + + // Just check the lowest stream, which is the one that where the adapter + // might alter the max qp setting. + VideoCodec ref_codec; + InitRefCodec(0, &ref_codec); + ref_codec.qpMax = kHighMaxQp; + ref_codec.VP8()->complexity = webrtc::VideoCodecComplexity::kComplexityHigher; + ref_codec.VP8()->denoisingOn = false; + ref_codec.startBitrate = 100; // Should equal to the target bitrate. + VerifyCodec(ref_codec, 0); + + // Change the max qp and try again. + codec_.simulcastStream[0].qpMax = kLowMaxQp; + EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); + EXPECT_EQ(3u, helper_->factory()->encoders().size()); + ref_codec.qpMax = kLowMaxQp; + VerifyCodec(ref_codec, 0); +} + +TEST_F(TestSimulcastEncoderAdapterFake, + DoesNotAlterMaxQpForScreenshareReversedLayer) { + const int kHighMaxQp = 56; + const int kLowMaxQp = 46; + + SimulcastTestFixtureImpl::DefaultSettings( + &codec_, static_cast<const int*>(kTestTemporalLayerProfile), + kVideoCodecVP8, true /* reverse_layer_order */); + codec_.numberOfSimulcastStreams = 3; + codec_.simulcastStream[2].qpMax = kHighMaxQp; + codec_.mode = VideoCodecMode::kScreensharing; + + EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); + EXPECT_EQ(3u, helper_->factory()->encoders().size()); + + // Just check the lowest stream, which is the one that where the adapter + // might alter the max qp setting. + VideoCodec ref_codec; + InitRefCodec(2, &ref_codec, true /* reverse_layer_order */); + ref_codec.qpMax = kHighMaxQp; + ref_codec.VP8()->complexity = webrtc::VideoCodecComplexity::kComplexityHigher; + ref_codec.VP8()->denoisingOn = false; + ref_codec.startBitrate = 100; // Should equal to the target bitrate. + VerifyCodec(ref_codec, 2); + + // Change the max qp and try again. + codec_.simulcastStream[2].qpMax = kLowMaxQp; + EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); + EXPECT_EQ(3u, helper_->factory()->encoders().size()); + ref_codec.qpMax = kLowMaxQp; + VerifyCodec(ref_codec, 2); +} + +TEST_F(TestSimulcastEncoderAdapterFake, ActivatesCorrectStreamsInInitEncode) { + // Set up common settings for three streams. + SimulcastTestFixtureImpl::DefaultSettings( + &codec_, static_cast<const int*>(kTestTemporalLayerProfile), + kVideoCodecVP8); + rate_allocator_.reset(new SimulcastRateAllocator(codec_)); + adapter_->RegisterEncodeCompleteCallback(this); + + // Only enough start bitrate for the lowest stream. + ASSERT_EQ(3u, codec_.numberOfSimulcastStreams); + codec_.startBitrate = codec_.simulcastStream[0].targetBitrate + + codec_.simulcastStream[1].minBitrate - 1; + + // Input data. + rtc::scoped_refptr<VideoFrameBuffer> buffer(I420Buffer::Create(1280, 720)); + VideoFrame input_frame = VideoFrame::Builder() + .set_video_frame_buffer(buffer) + .set_timestamp_rtp(100) + .set_timestamp_ms(1000) + .set_rotation(kVideoRotation_180) + .build(); + + // Encode with three streams. + EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); + std::vector<MockVideoEncoder*> original_encoders = + helper_->factory()->encoders(); + ASSERT_EQ(3u, original_encoders.size()); + // Only first encoder will be active and called. + EXPECT_CALL(*original_encoders[0], Encode(_, _)) + .WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); + EXPECT_CALL(*original_encoders[1], Encode(_, _)).Times(0); + EXPECT_CALL(*original_encoders[2], Encode(_, _)).Times(0); + + std::vector<VideoFrameType> frame_types; + frame_types.resize(3, VideoFrameType::kVideoFrameKey); + EXPECT_EQ(0, adapter_->Encode(input_frame, &frame_types)); +} + +TEST_F(TestSimulcastEncoderAdapterFake, TrustedRateControl) { + // Set up common settings for three streams. + SimulcastTestFixtureImpl::DefaultSettings( + &codec_, static_cast<const int*>(kTestTemporalLayerProfile), + kVideoCodecVP8); + rate_allocator_.reset(new SimulcastRateAllocator(codec_)); + adapter_->RegisterEncodeCompleteCallback(this); + + // Only enough start bitrate for the lowest stream. + ASSERT_EQ(3u, codec_.numberOfSimulcastStreams); + codec_.startBitrate = codec_.simulcastStream[0].targetBitrate + + codec_.simulcastStream[1].minBitrate - 1; + + // Input data. + rtc::scoped_refptr<VideoFrameBuffer> buffer(I420Buffer::Create(1280, 720)); + VideoFrame input_frame = VideoFrame::Builder() + .set_video_frame_buffer(buffer) + .set_timestamp_rtp(100) + .set_timestamp_ms(1000) + .set_rotation(kVideoRotation_180) + .build(); + + // No encoder trusted, so simulcast adapter should not be either. + EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); + EXPECT_FALSE(adapter_->GetEncoderInfo().has_trusted_rate_controller); + + // Encode with three streams. + std::vector<MockVideoEncoder*> original_encoders = + helper_->factory()->encoders(); + + // All encoders are trusted, so simulcast adapter should be too. + original_encoders[0]->set_has_trusted_rate_controller(true); + original_encoders[1]->set_has_trusted_rate_controller(true); + original_encoders[2]->set_has_trusted_rate_controller(true); + EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); + EXPECT_TRUE(adapter_->GetEncoderInfo().has_trusted_rate_controller); + + // One encoder not trusted, so simulcast adapter should not be either. + original_encoders[2]->set_has_trusted_rate_controller(false); + EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); + EXPECT_FALSE(adapter_->GetEncoderInfo().has_trusted_rate_controller); + + // No encoder trusted, so simulcast adapter should not be either. + original_encoders[0]->set_has_trusted_rate_controller(false); + original_encoders[1]->set_has_trusted_rate_controller(false); + EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); + EXPECT_FALSE(adapter_->GetEncoderInfo().has_trusted_rate_controller); +} + +TEST_F(TestSimulcastEncoderAdapterFake, ReportsHardwareAccelerated) { + SimulcastTestFixtureImpl::DefaultSettings( + &codec_, static_cast<const int*>(kTestTemporalLayerProfile), + kVideoCodecVP8); + codec_.numberOfSimulcastStreams = 3; + adapter_->RegisterEncodeCompleteCallback(this); + EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); + ASSERT_EQ(3u, helper_->factory()->encoders().size()); + + // None of the encoders uses HW support, so simulcast adapter reports false. + for (MockVideoEncoder* encoder : helper_->factory()->encoders()) { + encoder->set_is_hardware_accelerated(false); + } + EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); + EXPECT_FALSE(adapter_->GetEncoderInfo().is_hardware_accelerated); + + // One encoder uses HW support, so simulcast adapter reports true. + helper_->factory()->encoders()[2]->set_is_hardware_accelerated(true); + EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); + EXPECT_TRUE(adapter_->GetEncoderInfo().is_hardware_accelerated); +} + +TEST_F(TestSimulcastEncoderAdapterFake, + ReportsLeastCommonMultipleOfRequestedResolutionAlignments) { + SimulcastTestFixtureImpl::DefaultSettings( + &codec_, static_cast<const int*>(kTestTemporalLayerProfile), + kVideoCodecVP8); + codec_.numberOfSimulcastStreams = 3; + helper_->factory()->set_requested_resolution_alignments({2, 4, 7}); + EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); + + EXPECT_EQ(adapter_->GetEncoderInfo().requested_resolution_alignment, 28); +} + +TEST_F(TestSimulcastEncoderAdapterFake, ReportsInternalSource) { + SimulcastTestFixtureImpl::DefaultSettings( + &codec_, static_cast<const int*>(kTestTemporalLayerProfile), + kVideoCodecVP8); + codec_.numberOfSimulcastStreams = 3; + adapter_->RegisterEncodeCompleteCallback(this); + EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); + ASSERT_EQ(3u, helper_->factory()->encoders().size()); + + // All encoders have internal source, simulcast adapter reports true. + for (MockVideoEncoder* encoder : helper_->factory()->encoders()) { + encoder->set_has_internal_source(true); + } + EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); + EXPECT_TRUE(adapter_->GetEncoderInfo().has_internal_source); + + // One encoder does not have internal source, simulcast adapter reports false. + helper_->factory()->encoders()[2]->set_has_internal_source(false); + EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); + EXPECT_FALSE(adapter_->GetEncoderInfo().has_internal_source); +} + +TEST_F(TestSimulcastEncoderAdapterFake, ReportsFpsAllocation) { + SimulcastTestFixtureImpl::DefaultSettings( + &codec_, static_cast<const int*>(kTestTemporalLayerProfile), + kVideoCodecVP8); + codec_.numberOfSimulcastStreams = 3; + adapter_->RegisterEncodeCompleteCallback(this); + EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); + ASSERT_EQ(3u, helper_->factory()->encoders().size()); + + // Combination of three different supported mode: + // Simulcast stream 0 has undefined fps behavior. + // Simulcast stream 1 has three temporal layers. + // Simulcast stream 2 has 1 temporal layer. + FramerateFractions expected_fps_allocation[kMaxSpatialLayers]; + expected_fps_allocation[1].push_back(EncoderInfo::kMaxFramerateFraction / 4); + expected_fps_allocation[1].push_back(EncoderInfo::kMaxFramerateFraction / 2); + expected_fps_allocation[1].push_back(EncoderInfo::kMaxFramerateFraction); + expected_fps_allocation[2].push_back(EncoderInfo::kMaxFramerateFraction); + + // All encoders have internal source, simulcast adapter reports true. + for (size_t i = 0; i < codec_.numberOfSimulcastStreams; ++i) { + MockVideoEncoder* encoder = helper_->factory()->encoders()[i]; + encoder->set_fps_allocation(expected_fps_allocation[i]); + } + EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); + EXPECT_THAT(adapter_->GetEncoderInfo().fps_allocation, + ::testing::ElementsAreArray(expected_fps_allocation)); +} + +TEST_F(TestSimulcastEncoderAdapterFake, SetRateDistributesBandwithAllocation) { + SimulcastTestFixtureImpl::DefaultSettings( + &codec_, static_cast<const int*>(kTestTemporalLayerProfile), + kVideoCodecVP8); + codec_.numberOfSimulcastStreams = 3; + const DataRate target_bitrate = + DataRate::KilobitsPerSec(codec_.simulcastStream[0].targetBitrate + + codec_.simulcastStream[1].targetBitrate + + codec_.simulcastStream[2].minBitrate); + const DataRate bandwidth_allocation = + target_bitrate + DataRate::KilobitsPerSec(600); + + rate_allocator_.reset(new SimulcastRateAllocator(codec_)); + EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); + adapter_->RegisterEncodeCompleteCallback(this); + + // Set bitrates so that we send all layers. + adapter_->SetRates(VideoEncoder::RateControlParameters( + rate_allocator_->Allocate( + VideoBitrateAllocationParameters(target_bitrate.bps(), 30)), + 30.0, bandwidth_allocation)); + + std::vector<MockVideoEncoder*> encoders = helper_->factory()->encoders(); + + ASSERT_EQ(3u, encoders.size()); + + for (size_t i = 0; i < 3; ++i) { + const uint32_t layer_bitrate_bps = + (i < static_cast<size_t>(codec_.numberOfSimulcastStreams) - 1 + ? codec_.simulcastStream[i].targetBitrate + : codec_.simulcastStream[i].minBitrate) * + 1000; + EXPECT_EQ(layer_bitrate_bps, + encoders[i]->last_set_rates().bitrate.get_sum_bps()) + << i; + EXPECT_EQ( + (layer_bitrate_bps * bandwidth_allocation.bps()) / target_bitrate.bps(), + encoders[i]->last_set_rates().bandwidth_allocation.bps()) + << i; + } +} + +TEST_F(TestSimulcastEncoderAdapterFake, CanSetZeroBitrateWithHeadroom) { + SimulcastTestFixtureImpl::DefaultSettings( + &codec_, static_cast<const int*>(kTestTemporalLayerProfile), + kVideoCodecVP8); + codec_.numberOfSimulcastStreams = 3; + + rate_allocator_.reset(new SimulcastRateAllocator(codec_)); + EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); + adapter_->RegisterEncodeCompleteCallback(this); + + // Set allocated bitrate to 0, but keep (network) bandwidth allocation. + VideoEncoder::RateControlParameters rate_params; + rate_params.framerate_fps = 30; + rate_params.bandwidth_allocation = DataRate::KilobitsPerSec(600); + + adapter_->SetRates(rate_params); + + std::vector<MockVideoEncoder*> encoders = helper_->factory()->encoders(); + + ASSERT_EQ(3u, encoders.size()); + for (size_t i = 0; i < 3; ++i) { + EXPECT_EQ(0u, encoders[i]->last_set_rates().bitrate.get_sum_bps()); + } +} + +TEST_F(TestSimulcastEncoderAdapterFake, SupportsSimulcast) { + SimulcastTestFixtureImpl::DefaultSettings( + &codec_, static_cast<const int*>(kTestTemporalLayerProfile), + kVideoCodecVP8); + codec_.numberOfSimulcastStreams = 3; + + // Indicate that mock encoders internally support simulcast. + helper_->factory()->set_supports_simulcast(true); + adapter_->RegisterEncodeCompleteCallback(this); + EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); + + // Only one encoder should have been produced. + ASSERT_EQ(1u, helper_->factory()->encoders().size()); + + rtc::scoped_refptr<VideoFrameBuffer> buffer(I420Buffer::Create(1280, 720)); + VideoFrame input_frame = VideoFrame::Builder() + .set_video_frame_buffer(buffer) + .set_timestamp_rtp(100) + .set_timestamp_ms(1000) + .set_rotation(kVideoRotation_180) + .build(); + EXPECT_CALL(*helper_->factory()->encoders()[0], Encode) + .WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); + std::vector<VideoFrameType> frame_types(3, VideoFrameType::kVideoFrameKey); + EXPECT_EQ(0, adapter_->Encode(input_frame, &frame_types)); +} + +TEST_F(TestSimulcastEncoderAdapterFake, PassesSdpVideoFormatToEncoder) { + sdp_video_parameters_ = {{"test_param", "test_value"}}; + SetUp(); + SetupCodec(); + std::vector<MockVideoEncoder*> encoders = helper_->factory()->encoders(); + ASSERT_GT(encoders.size(), 0u); + EXPECT_EQ(encoders[0]->video_format(), + SdpVideoFormat("VP8", sdp_video_parameters_)); +} + +TEST_F(TestSimulcastEncoderAdapterFake, SupportsFallback) { + // Enable support for fallback encoder factory and re-setup. + use_fallback_factory_ = true; + SetUp(); + + SetupCodec(); + + // Make sure we have bitrate for all layers. + DataRate max_bitrate = DataRate::Zero(); + for (int i = 0; i < 3; ++i) { + max_bitrate += + DataRate::KilobitsPerSec(codec_.simulcastStream[i].maxBitrate); + } + const auto rate_settings = VideoEncoder::RateControlParameters( + rate_allocator_->Allocate( + VideoBitrateAllocationParameters(max_bitrate.bps(), 30)), + 30.0, max_bitrate); + adapter_->SetRates(rate_settings); + + std::vector<MockVideoEncoder*> primary_encoders = + helper_->factory()->encoders(); + std::vector<MockVideoEncoder*> fallback_encoders = + helper_->fallback_factory()->encoders(); + + ASSERT_EQ(3u, primary_encoders.size()); + ASSERT_EQ(3u, fallback_encoders.size()); + + // Create frame to test with. + rtc::scoped_refptr<VideoFrameBuffer> buffer(I420Buffer::Create(1280, 720)); + VideoFrame input_frame = VideoFrame::Builder() + .set_video_frame_buffer(buffer) + .set_timestamp_rtp(100) + .set_timestamp_ms(1000) + .set_rotation(kVideoRotation_180) + .build(); + std::vector<VideoFrameType> frame_types(3, VideoFrameType::kVideoFrameKey); + + // All primary encoders used. + for (auto codec : primary_encoders) { + EXPECT_CALL(*codec, Encode).WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); + } + EXPECT_EQ(0, adapter_->Encode(input_frame, &frame_types)); + + // Trigger fallback on first encoder. + primary_encoders[0]->set_init_encode_return_value( + WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE); + EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); + adapter_->SetRates(rate_settings); + EXPECT_CALL(*fallback_encoders[0], Encode) + .WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); + EXPECT_CALL(*primary_encoders[1], Encode) + .WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); + EXPECT_CALL(*primary_encoders[2], Encode) + .WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); + EXPECT_EQ(0, adapter_->Encode(input_frame, &frame_types)); + + // Trigger fallback on all encoder. + primary_encoders[1]->set_init_encode_return_value( + WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE); + primary_encoders[2]->set_init_encode_return_value( + WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE); + EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); + adapter_->SetRates(rate_settings); + EXPECT_CALL(*fallback_encoders[0], Encode) + .WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); + EXPECT_CALL(*fallback_encoders[1], Encode) + .WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); + EXPECT_CALL(*fallback_encoders[2], Encode) + .WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); + EXPECT_EQ(0, adapter_->Encode(input_frame, &frame_types)); + + // Return to primary encoders on all streams. + for (int i = 0; i < 3; ++i) { + primary_encoders[i]->set_init_encode_return_value(WEBRTC_VIDEO_CODEC_OK); + } + EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); + adapter_->SetRates(rate_settings); + for (auto codec : primary_encoders) { + EXPECT_CALL(*codec, Encode).WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); + } + EXPECT_EQ(0, adapter_->Encode(input_frame, &frame_types)); +} + +TEST_F(TestSimulcastEncoderAdapterFake, SupportsPerSimulcastLayerMaxFramerate) { + SimulcastTestFixtureImpl::DefaultSettings( + &codec_, static_cast<const int*>(kTestTemporalLayerProfile), + kVideoCodecVP8); + codec_.numberOfSimulcastStreams = 3; + codec_.simulcastStream[0].maxFramerate = 60; + codec_.simulcastStream[1].maxFramerate = 30; + codec_.simulcastStream[2].maxFramerate = 10; + + EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); + ASSERT_EQ(3u, helper_->factory()->encoders().size()); + EXPECT_EQ(60u, helper_->factory()->encoders()[0]->codec().maxFramerate); + EXPECT_EQ(30u, helper_->factory()->encoders()[1]->codec().maxFramerate); + EXPECT_EQ(10u, helper_->factory()->encoders()[2]->codec().maxFramerate); +} + +} // namespace test +} // namespace webrtc diff --git a/media/engine/simulcast_unittest.cc b/media/engine/simulcast_unittest.cc new file mode 100644 index 0000000000..0ce388a9ee --- /dev/null +++ b/media/engine/simulcast_unittest.cc @@ -0,0 +1,401 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "media/engine/simulcast.h" + +#include "media/base/media_constants.h" +#include "media/engine/constants.h" +#include "test/field_trial.h" +#include "test/gtest.h" + +namespace webrtc { +namespace { +constexpr int kQpMax = 55; +constexpr double kBitratePriority = 2.0; +constexpr bool kScreenshare = true; +constexpr int kDefaultTemporalLayers = 3; // Value from simulcast.cc. + +// Values from kSimulcastConfigs in simulcast.cc. +const std::vector<VideoStream> GetSimulcastBitrates720p() { + std::vector<VideoStream> streams(3); + streams[0].min_bitrate_bps = 30000; + streams[0].target_bitrate_bps = 150000; + streams[0].max_bitrate_bps = 200000; + streams[1].min_bitrate_bps = 150000; + streams[1].target_bitrate_bps = 500000; + streams[1].max_bitrate_bps = 700000; + streams[2].min_bitrate_bps = 600000; + streams[2].target_bitrate_bps = 2500000; + streams[2].max_bitrate_bps = 2500000; + return streams; +} +} // namespace + +TEST(SimulcastTest, TotalMaxBitrateIsZeroForNoStreams) { + std::vector<VideoStream> streams; + EXPECT_EQ(0, cricket::GetTotalMaxBitrate(streams).bps()); +} + +TEST(SimulcastTest, GetTotalMaxBitrateForSingleStream) { + std::vector<VideoStream> streams(1); + streams[0].max_bitrate_bps = 100000; + EXPECT_EQ(100000, cricket::GetTotalMaxBitrate(streams).bps()); +} + +TEST(SimulcastTest, GetTotalMaxBitrateForMultipleStreams) { + std::vector<VideoStream> streams(3); + streams[0].target_bitrate_bps = 100000; + streams[1].target_bitrate_bps = 200000; + streams[2].max_bitrate_bps = 400000; + EXPECT_EQ(700000, cricket::GetTotalMaxBitrate(streams).bps()); +} + +TEST(SimulcastTest, BandwidthAboveTotalMaxBitrateGivenToHighestStream) { + std::vector<VideoStream> streams(3); + streams[0].target_bitrate_bps = 100000; + streams[1].target_bitrate_bps = 200000; + streams[2].max_bitrate_bps = 400000; + + const webrtc::DataRate one_bps = webrtc::DataRate::BitsPerSec(1); + + // No bitrate above the total max to give to the highest stream. + const webrtc::DataRate max_total_bitrate = + cricket::GetTotalMaxBitrate(streams); + cricket::BoostMaxSimulcastLayer(max_total_bitrate, &streams); + EXPECT_EQ(400000, streams[2].max_bitrate_bps); + EXPECT_EQ(max_total_bitrate, cricket::GetTotalMaxBitrate(streams)); + + // The bitrate above the total max should be given to the highest stream. + cricket::BoostMaxSimulcastLayer(max_total_bitrate + one_bps, &streams); + EXPECT_EQ(400000 + 1, streams[2].max_bitrate_bps); + EXPECT_EQ(max_total_bitrate + one_bps, cricket::GetTotalMaxBitrate(streams)); +} + +TEST(SimulcastTest, GetConfig) { + const std::vector<VideoStream> kExpected = GetSimulcastBitrates720p(); + + const size_t kMinLayers = 1; + const size_t kMaxLayers = 3; + std::vector<VideoStream> streams = cricket::GetSimulcastConfig( + kMinLayers, kMaxLayers, 1280, 720, kBitratePriority, kQpMax, + !kScreenshare, true); + + EXPECT_EQ(kMaxLayers, streams.size()); + EXPECT_EQ(320u, streams[0].width); + EXPECT_EQ(180u, streams[0].height); + EXPECT_EQ(640u, streams[1].width); + EXPECT_EQ(360u, streams[1].height); + EXPECT_EQ(1280u, streams[2].width); + EXPECT_EQ(720u, streams[2].height); + + for (size_t i = 0; i < streams.size(); ++i) { + EXPECT_EQ(size_t{kDefaultTemporalLayers}, streams[i].num_temporal_layers); + EXPECT_EQ(cricket::kDefaultVideoMaxFramerate, streams[i].max_framerate); + EXPECT_EQ(kQpMax, streams[i].max_qp); + EXPECT_EQ(kExpected[i].min_bitrate_bps, streams[i].min_bitrate_bps); + EXPECT_EQ(kExpected[i].target_bitrate_bps, streams[i].target_bitrate_bps); + EXPECT_EQ(kExpected[i].max_bitrate_bps, streams[i].max_bitrate_bps); + EXPECT_TRUE(streams[i].active); + } + // Currently set on lowest stream. + EXPECT_EQ(kBitratePriority, streams[0].bitrate_priority); + EXPECT_FALSE(streams[1].bitrate_priority); + EXPECT_FALSE(streams[2].bitrate_priority); +} + +TEST(SimulcastTest, GetConfigWithBaseHeavyVP8TL3RateAllocation) { + test::ScopedFieldTrials field_trials( + "WebRTC-UseBaseHeavyVP8TL3RateAllocation/Enabled/"); + + const std::vector<VideoStream> kExpected = GetSimulcastBitrates720p(); + + const size_t kMinLayers = 1; + const size_t kMaxLayers = 3; + std::vector<VideoStream> streams = cricket::GetSimulcastConfig( + kMinLayers, kMaxLayers, 1280, 720, kBitratePriority, kQpMax, + !kScreenshare, true); + + EXPECT_EQ(kExpected[0].min_bitrate_bps, streams[0].min_bitrate_bps); + EXPECT_EQ(static_cast<int>(0.4 * kExpected[0].target_bitrate_bps / 0.6), + streams[0].target_bitrate_bps); + EXPECT_EQ(static_cast<int>(0.4 * kExpected[0].max_bitrate_bps / 0.6), + streams[0].max_bitrate_bps); + for (size_t i = 1; i < streams.size(); ++i) { + EXPECT_EQ(kExpected[i].min_bitrate_bps, streams[i].min_bitrate_bps); + EXPECT_EQ(kExpected[i].target_bitrate_bps, streams[i].target_bitrate_bps); + EXPECT_EQ(kExpected[i].max_bitrate_bps, streams[i].max_bitrate_bps); + } +} + +TEST(SimulcastTest, GetConfigWithLimitedMaxLayers) { + const size_t kMinLayers = 1; + const size_t kMaxLayers = 2; + std::vector<VideoStream> streams = cricket::GetSimulcastConfig( + kMinLayers, kMaxLayers, 1280, 720, kBitratePriority, kQpMax, + !kScreenshare, true); + + EXPECT_EQ(kMaxLayers, streams.size()); + EXPECT_EQ(640u, streams[0].width); + EXPECT_EQ(360u, streams[0].height); + EXPECT_EQ(1280u, streams[1].width); + EXPECT_EQ(720u, streams[1].height); +} + +TEST(SimulcastTest, GetConfigWithLimitedMaxLayersForResolution) { + test::ScopedFieldTrials field_trials( + "WebRTC-LegacySimulcastLayerLimit/Enabled/"); + const size_t kMinLayers = 1; + const size_t kMaxLayers = 3; + std::vector<VideoStream> streams = cricket::GetSimulcastConfig( + kMinLayers, kMaxLayers, 800, 600, kBitratePriority, kQpMax, !kScreenshare, + true); + + EXPECT_EQ(2u, streams.size()); + EXPECT_EQ(400u, streams[0].width); + EXPECT_EQ(300u, streams[0].height); + EXPECT_EQ(800u, streams[1].width); + EXPECT_EQ(600u, streams[1].height); +} + +TEST(SimulcastTest, GetConfigWithLowResolutionScreenshare) { + test::ScopedFieldTrials field_trials( + "WebRTC-LegacySimulcastLayerLimit/Enabled/"); + const size_t kMinLayers = 1; + const size_t kMaxLayers = 3; + std::vector<VideoStream> streams = + cricket::GetSimulcastConfig(kMinLayers, kMaxLayers, 100, 100, + kBitratePriority, kQpMax, kScreenshare, true); + + // Simulcast streams number is never decreased for screenshare, + // even for very low resolution. + EXPECT_GT(streams.size(), 1u); +} + +TEST(SimulcastTest, GetConfigWithNotLimitedMaxLayersForResolution) { + test::ScopedFieldTrials field_trials( + "WebRTC-LegacySimulcastLayerLimit/Disabled/"); + const size_t kMinLayers = 1; + const size_t kMaxLayers = 3; + std::vector<VideoStream> streams = cricket::GetSimulcastConfig( + kMinLayers, kMaxLayers, 800, 600, kBitratePriority, kQpMax, !kScreenshare, + true); + + EXPECT_EQ(kMaxLayers, streams.size()); + EXPECT_EQ(200u, streams[0].width); + EXPECT_EQ(150u, streams[0].height); + EXPECT_EQ(400u, streams[1].width); + EXPECT_EQ(300u, streams[1].height); + EXPECT_EQ(800u, streams[2].width); + EXPECT_EQ(600u, streams[2].height); +} + +TEST(SimulcastTest, GetConfigWithNormalizedResolution) { + const size_t kMinLayers = 1; + const size_t kMaxLayers = 2; + std::vector<VideoStream> streams = cricket::GetSimulcastConfig( + kMinLayers, kMaxLayers, 640 + 1, 360 + 1, kBitratePriority, kQpMax, + !kScreenshare, true); + + // Must be divisible by |2 ^ (num_layers - 1)|. + EXPECT_EQ(kMaxLayers, streams.size()); + EXPECT_EQ(320u, streams[0].width); + EXPECT_EQ(180u, streams[0].height); + EXPECT_EQ(640u, streams[1].width); + EXPECT_EQ(360u, streams[1].height); +} + +TEST(SimulcastTest, GetConfigWithNormalizedResolutionDivisibleBy4) { + test::ScopedFieldTrials field_trials( + "WebRTC-NormalizeSimulcastResolution/Enabled-2/"); + + const size_t kMinLayers = 1; + const size_t kMaxLayers = 2; + std::vector<VideoStream> streams = cricket::GetSimulcastConfig( + kMinLayers, kMaxLayers, 709, 501, kBitratePriority, kQpMax, !kScreenshare, + true); + + // Must be divisible by |2 ^ 2|. + EXPECT_EQ(kMaxLayers, streams.size()); + EXPECT_EQ(354u, streams[0].width); + EXPECT_EQ(250u, streams[0].height); + EXPECT_EQ(708u, streams[1].width); + EXPECT_EQ(500u, streams[1].height); +} + +TEST(SimulcastTest, GetConfigWithNormalizedResolutionDivisibleBy8) { + test::ScopedFieldTrials field_trials( + "WebRTC-NormalizeSimulcastResolution/Enabled-3/"); + + const size_t kMinLayers = 1; + const size_t kMaxLayers = 2; + std::vector<VideoStream> streams = cricket::GetSimulcastConfig( + kMinLayers, kMaxLayers, 709, 501, kBitratePriority, kQpMax, !kScreenshare, + true); + + // Must be divisible by |2 ^ 3|. + EXPECT_EQ(kMaxLayers, streams.size()); + EXPECT_EQ(352u, streams[0].width); + EXPECT_EQ(248u, streams[0].height); + EXPECT_EQ(704u, streams[1].width); + EXPECT_EQ(496u, streams[1].height); +} + +TEST(SimulcastTest, GetConfigForLegacyLayerLimit) { + test::ScopedFieldTrials field_trials( + "WebRTC-LegacySimulcastLayerLimit/Enabled/"); + + const size_t kMinLayers = 1; + const int kMaxLayers = 3; + std::vector<VideoStream> streams = cricket::GetSimulcastConfig( + kMinLayers, kMaxLayers, 320, 180, kBitratePriority, kQpMax, !kScreenshare, + true); + EXPECT_EQ(1u, streams.size()); + + streams = cricket::GetSimulcastConfig(kMinLayers, kMaxLayers, 640, 360, + kBitratePriority, kQpMax, !kScreenshare, + true); + EXPECT_EQ(2u, streams.size()); + + streams = cricket::GetSimulcastConfig(kMinLayers, kMaxLayers, 1920, 1080, + kBitratePriority, kQpMax, !kScreenshare, + true); + EXPECT_EQ(3u, streams.size()); +} + +TEST(SimulcastTest, GetConfigForLegacyLayerLimitWithRequiredHD) { + test::ScopedFieldTrials field_trials( + "WebRTC-LegacySimulcastLayerLimit/Enabled/"); + + const size_t kMinLayers = 3; // "HD" layer must be present! + const int kMaxLayers = 3; + std::vector<VideoStream> streams = cricket::GetSimulcastConfig( + kMinLayers, kMaxLayers, 320, 180, kBitratePriority, kQpMax, !kScreenshare, + true); + EXPECT_EQ(3u, streams.size()); + + streams = cricket::GetSimulcastConfig(kMinLayers, kMaxLayers, 640, 360, + kBitratePriority, kQpMax, !kScreenshare, + true); + EXPECT_EQ(3u, streams.size()); + + streams = cricket::GetSimulcastConfig(kMinLayers, kMaxLayers, 1920, 1080, + kBitratePriority, kQpMax, !kScreenshare, + true); + EXPECT_EQ(3u, streams.size()); +} + +TEST(SimulcastTest, GetConfigForScreenshareSimulcast) { + const size_t kMinLayers = 1; + const size_t kMaxLayers = 3; + std::vector<VideoStream> streams = + cricket::GetSimulcastConfig(kMinLayers, kMaxLayers, 1400, 800, + kBitratePriority, kQpMax, kScreenshare, true); + + EXPECT_GT(streams.size(), 1u); + for (size_t i = 0; i < streams.size(); ++i) { + EXPECT_EQ(1400u, streams[i].width) << "Screen content never scaled."; + EXPECT_EQ(800u, streams[i].height) << "Screen content never scaled."; + EXPECT_EQ(kQpMax, streams[i].max_qp); + EXPECT_TRUE(streams[i].active); + EXPECT_GT(streams[i].num_temporal_layers, size_t{1}); + EXPECT_GT(streams[i].max_framerate, 0); + EXPECT_GT(streams[i].min_bitrate_bps, 0); + EXPECT_GT(streams[i].target_bitrate_bps, streams[i].min_bitrate_bps); + EXPECT_GE(streams[i].max_bitrate_bps, streams[i].target_bitrate_bps); + } +} + +TEST(SimulcastTest, GetConfigForScreenshareSimulcastWithLimitedMaxLayers) { + const size_t kMinLayers = 1; + const size_t kMaxLayers = 1; + std::vector<VideoStream> streams = + cricket::GetSimulcastConfig(kMinLayers, kMaxLayers, 1400, 800, + kBitratePriority, kQpMax, kScreenshare, true); + + EXPECT_EQ(kMaxLayers, streams.size()); +} + +TEST(SimulcastTest, SimulcastScreenshareMaxBitrateAdjustedForResolution) { + constexpr int kScreenshareHighStreamMinBitrateBps = 600000; + constexpr int kScreenshareHighStreamMaxBitrateBps = 1250000; + constexpr int kMaxBitrate960_540 = 1200000; + + // Normal case, max bitrate not limited by resolution. + const size_t kMinLayers = 1; + const size_t kMaxLayers = 2; + std::vector<VideoStream> streams = + cricket::GetSimulcastConfig(kMinLayers, kMaxLayers, 1920, 1080, + kBitratePriority, kQpMax, kScreenshare, true); + EXPECT_EQ(kMaxLayers, streams.size()); + EXPECT_EQ(streams[1].max_bitrate_bps, kScreenshareHighStreamMaxBitrateBps); + EXPECT_EQ(streams[1].min_bitrate_bps, kScreenshareHighStreamMinBitrateBps); + EXPECT_GE(streams[1].max_bitrate_bps, streams[1].min_bitrate_bps); + + // At 960x540, the max bitrate is limited to 900kbps. + streams = + cricket::GetSimulcastConfig(kMinLayers, kMaxLayers, 960, 540, + kBitratePriority, kQpMax, kScreenshare, true); + EXPECT_EQ(kMaxLayers, streams.size()); + EXPECT_EQ(streams[1].max_bitrate_bps, kMaxBitrate960_540); + EXPECT_EQ(streams[1].min_bitrate_bps, kScreenshareHighStreamMinBitrateBps); + EXPECT_GE(streams[1].max_bitrate_bps, streams[1].min_bitrate_bps); + + // At 480x270, the max bitrate is limited to 450kbps. This is lower than + // the min bitrate, so use that as a lower bound. + streams = + cricket::GetSimulcastConfig(kMinLayers, kMaxLayers, 480, 270, + kBitratePriority, kQpMax, kScreenshare, true); + EXPECT_EQ(kMaxLayers, streams.size()); + EXPECT_EQ(streams[1].max_bitrate_bps, kScreenshareHighStreamMinBitrateBps); + EXPECT_EQ(streams[1].min_bitrate_bps, kScreenshareHighStreamMinBitrateBps); + EXPECT_GE(streams[1].max_bitrate_bps, streams[1].min_bitrate_bps); +} + +TEST(SimulcastTest, AveragesBitratesForNonStandardResolution) { + const size_t kMinLayers = 1; + const size_t kMaxLayers = 3; + std::vector<VideoStream> streams = cricket::GetSimulcastConfig( + kMinLayers, kMaxLayers, 900, 800, kBitratePriority, kQpMax, !kScreenshare, + true); + + EXPECT_EQ(kMaxLayers, streams.size()); + EXPECT_EQ(900u, streams[2].width); + EXPECT_EQ(800u, streams[2].height); + EXPECT_EQ(1850000, streams[2].max_bitrate_bps); + EXPECT_EQ(1850000, streams[2].target_bitrate_bps); + EXPECT_EQ(475000, streams[2].min_bitrate_bps); +} + +TEST(SimulcastTest, BitratesForCloseToStandardResolution) { + const size_t kMinLayers = 1; + const size_t kMaxLayers = 3; + // Resolution very close to 720p in number of pixels + const size_t kWidth = 1280; + const size_t kHeight = 716; + const std::vector<VideoStream> kExpectedNear = GetSimulcastBitrates720p(); + + std::vector<VideoStream> streams = cricket::GetSimulcastConfig( + kMinLayers, kMaxLayers, kWidth, kHeight, kBitratePriority, kQpMax, + !kScreenshare, true); + + EXPECT_EQ(kMaxLayers, streams.size()); + EXPECT_EQ(kWidth, streams[2].width); + EXPECT_EQ(kHeight, streams[2].height); + for (size_t i = 0; i < streams.size(); ++i) { + EXPECT_NEAR(kExpectedNear[i].max_bitrate_bps, streams[i].max_bitrate_bps, + 20000); + EXPECT_NEAR(kExpectedNear[i].target_bitrate_bps, + streams[i].target_bitrate_bps, 20000); + EXPECT_NEAR(kExpectedNear[i].min_bitrate_bps, streams[i].min_bitrate_bps, + 20000); + } +} + +} // namespace webrtc diff --git a/media/engine/unhandled_packets_buffer.cc b/media/engine/unhandled_packets_buffer.cc new file mode 100644 index 0000000000..ebc841e1fc --- /dev/null +++ b/media/engine/unhandled_packets_buffer.cc @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "media/engine/unhandled_packets_buffer.h" + +#include "absl/algorithm/container.h" +#include "rtc_base/logging.h" +#include "rtc_base/strings/string_builder.h" + +namespace cricket { + +UnhandledPacketsBuffer::UnhandledPacketsBuffer() { + buffer_.reserve(kMaxStashedPackets); +} + +UnhandledPacketsBuffer::~UnhandledPacketsBuffer() = default; + +// Store packet in buffer. +void UnhandledPacketsBuffer::AddPacket(uint32_t ssrc, + int64_t packet_time_us, + rtc::CopyOnWriteBuffer packet) { + if (buffer_.size() < kMaxStashedPackets) { + buffer_.push_back({ssrc, packet_time_us, packet}); + } else { + RTC_DCHECK_LT(insert_pos_, kMaxStashedPackets); + buffer_[insert_pos_] = {ssrc, packet_time_us, packet}; + } + insert_pos_ = (insert_pos_ + 1) % kMaxStashedPackets; +} + +// Backfill |consumer| with all stored packet related |ssrcs|. +void UnhandledPacketsBuffer::BackfillPackets( + rtc::ArrayView<const uint32_t> ssrcs, + std::function<void(uint32_t, int64_t, rtc::CopyOnWriteBuffer)> consumer) { + size_t start; + if (buffer_.size() < kMaxStashedPackets) { + start = 0; + } else { + start = insert_pos_; + } + + size_t count = 0; + std::vector<PacketWithMetadata> remaining; + remaining.reserve(kMaxStashedPackets); + for (size_t i = 0; i < buffer_.size(); ++i) { + const size_t pos = (i + start) % kMaxStashedPackets; + + // One or maybe 2 ssrcs is expected => loop array instead of more elaborate + // scheme. + const uint32_t ssrc = buffer_[pos].ssrc; + if (absl::c_linear_search(ssrcs, ssrc)) { + ++count; + consumer(ssrc, buffer_[pos].packet_time_us, buffer_[pos].packet); + } else { + remaining.push_back(buffer_[pos]); + } + } + + insert_pos_ = 0; // insert_pos is only used when buffer is full. + buffer_.swap(remaining); +} + +} // namespace cricket diff --git a/media/engine/unhandled_packets_buffer.h b/media/engine/unhandled_packets_buffer.h new file mode 100644 index 0000000000..ef03588165 --- /dev/null +++ b/media/engine/unhandled_packets_buffer.h @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MEDIA_ENGINE_UNHANDLED_PACKETS_BUFFER_H_ +#define MEDIA_ENGINE_UNHANDLED_PACKETS_BUFFER_H_ + +#include <stdint.h> + +#include <functional> +#include <tuple> +#include <utility> +#include <vector> + +#include "rtc_base/copy_on_write_buffer.h" + +namespace cricket { + +class UnhandledPacketsBuffer { + public: + // Visible for testing. + static constexpr size_t kMaxStashedPackets = 50; + + UnhandledPacketsBuffer(); + ~UnhandledPacketsBuffer(); + + // Store packet in buffer. + void AddPacket(uint32_t ssrc, + int64_t packet_time_us, + rtc::CopyOnWriteBuffer packet); + + // Feed all packets with |ssrcs| into |consumer|. + void BackfillPackets( + rtc::ArrayView<const uint32_t> ssrcs, + std::function<void(uint32_t, int64_t, rtc::CopyOnWriteBuffer)> consumer); + + private: + size_t insert_pos_ = 0; + struct PacketWithMetadata { + uint32_t ssrc; + int64_t packet_time_us; + rtc::CopyOnWriteBuffer packet; + }; + std::vector<PacketWithMetadata> buffer_; +}; + +} // namespace cricket + +#endif // MEDIA_ENGINE_UNHANDLED_PACKETS_BUFFER_H_ diff --git a/media/engine/unhandled_packets_buffer_unittest.cc b/media/engine/unhandled_packets_buffer_unittest.cc new file mode 100644 index 0000000000..11abd86850 --- /dev/null +++ b/media/engine/unhandled_packets_buffer_unittest.cc @@ -0,0 +1,171 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "media/engine/unhandled_packets_buffer.h" + +#include <memory> + +#include "absl/memory/memory.h" +#include "test/gmock.h" +#include "test/gtest.h" + +using ::testing::_; + +namespace { + +rtc::CopyOnWriteBuffer Create(int n) { + return rtc::CopyOnWriteBuffer(std::to_string(n)); +} + +constexpr int64_t kPacketTimeUs = 122; + +} // namespace + +namespace cricket { + +TEST(UnhandledPacketsBuffer, NoPackets) { + UnhandledPacketsBuffer buff; + buff.AddPacket(2, kPacketTimeUs, Create(3)); + + std::vector<uint32_t> ssrcs = {3}; + std::vector<rtc::CopyOnWriteBuffer> packets; + buff.BackfillPackets(ssrcs, [&packets](uint32_t ssrc, int64_t packet_time_us, + rtc::CopyOnWriteBuffer packet) { + packets.push_back(packet); + }); + EXPECT_EQ(0u, packets.size()); +} + +TEST(UnhandledPacketsBuffer, OnePacket) { + UnhandledPacketsBuffer buff; + buff.AddPacket(2, kPacketTimeUs, Create(3)); + + std::vector<uint32_t> ssrcs = {2}; + std::vector<rtc::CopyOnWriteBuffer> packets; + buff.BackfillPackets(ssrcs, [&packets](uint32_t ssrc, int64_t packet_time_us, + rtc::CopyOnWriteBuffer packet) { + packets.push_back(packet); + }); + ASSERT_EQ(1u, packets.size()); + EXPECT_EQ(Create(3), packets[0]); +} + +TEST(UnhandledPacketsBuffer, TwoPacketsTwoSsrcs) { + UnhandledPacketsBuffer buff; + buff.AddPacket(2, kPacketTimeUs, Create(3)); + buff.AddPacket(3, kPacketTimeUs, Create(4)); + + std::vector<uint32_t> ssrcs = {2, 3}; + std::vector<rtc::CopyOnWriteBuffer> packets; + buff.BackfillPackets(ssrcs, [&packets](uint32_t ssrc, int64_t packet_time_us, + rtc::CopyOnWriteBuffer packet) { + packets.push_back(packet); + }); + ASSERT_EQ(2u, packets.size()); + EXPECT_EQ(Create(3), packets[0]); + EXPECT_EQ(Create(4), packets[1]); +} + +TEST(UnhandledPacketsBuffer, TwoPacketsTwoSsrcsOneMatch) { + UnhandledPacketsBuffer buff; + buff.AddPacket(2, kPacketTimeUs, Create(3)); + buff.AddPacket(3, kPacketTimeUs, Create(4)); + + std::vector<uint32_t> ssrcs = {3}; + buff.BackfillPackets(ssrcs, [](uint32_t ssrc, int64_t packet_time_us, + rtc::CopyOnWriteBuffer packet) { + EXPECT_EQ(ssrc, 3u); + EXPECT_EQ(Create(4), packet); + }); + + std::vector<uint32_t> ssrcs_again = {2}; + buff.BackfillPackets(ssrcs, [](uint32_t ssrc, int64_t packet_time_us, + rtc::CopyOnWriteBuffer packet) { + EXPECT_EQ(ssrc, 2u); + EXPECT_EQ(Create(3), packet); + }); +} + +TEST(UnhandledPacketsBuffer, Full) { + const size_t cnt = UnhandledPacketsBuffer::kMaxStashedPackets; + UnhandledPacketsBuffer buff; + for (size_t i = 0; i < cnt; i++) { + buff.AddPacket(2, kPacketTimeUs, Create(i)); + } + + std::vector<uint32_t> ssrcs = {2}; + std::vector<rtc::CopyOnWriteBuffer> packets; + buff.BackfillPackets(ssrcs, [&packets](uint32_t ssrc, int64_t packet_time_us, + rtc::CopyOnWriteBuffer packet) { + packets.push_back(packet); + }); + ASSERT_EQ(cnt, packets.size()); + for (size_t i = 0; i < cnt; i++) { + EXPECT_EQ(Create(i), packets[i]); + } + + // Add a packet after backfill and check that it comes back. + buff.AddPacket(23, kPacketTimeUs, Create(1001)); + buff.BackfillPackets(ssrcs, [](uint32_t ssrc, int64_t packet_time_us, + rtc::CopyOnWriteBuffer packet) { + EXPECT_EQ(ssrc, 23u); + EXPECT_EQ(Create(1001), packet); + }); +} + +TEST(UnhandledPacketsBuffer, Wrap) { + UnhandledPacketsBuffer buff; + size_t cnt = UnhandledPacketsBuffer::kMaxStashedPackets + 10; + for (size_t i = 0; i < cnt; i++) { + buff.AddPacket(2, kPacketTimeUs, Create(i)); + } + + std::vector<uint32_t> ssrcs = {2}; + std::vector<rtc::CopyOnWriteBuffer> packets; + buff.BackfillPackets(ssrcs, [&packets](uint32_t ssrc, int64_t packet_time_us, + rtc::CopyOnWriteBuffer packet) { + packets.push_back(packet); + }); + for (size_t i = 0; i < packets.size(); i++) { + EXPECT_EQ(Create(i + 10), packets[i]); + } + + // Add a packet after backfill and check that it comes back. + buff.AddPacket(23, kPacketTimeUs, Create(1001)); + buff.BackfillPackets(ssrcs, [](uint32_t ssrc, int64_t packet_time_us, + rtc::CopyOnWriteBuffer packet) { + EXPECT_EQ(ssrc, 23u); + EXPECT_EQ(Create(1001), packet); + }); +} + +TEST(UnhandledPacketsBuffer, Interleaved) { + UnhandledPacketsBuffer buff; + buff.AddPacket(2, kPacketTimeUs, Create(2)); + buff.AddPacket(3, kPacketTimeUs, Create(3)); + + std::vector<uint32_t> ssrcs = {2}; + buff.BackfillPackets(ssrcs, [](uint32_t ssrc, int64_t packet_time_us, + rtc::CopyOnWriteBuffer packet) { + EXPECT_EQ(ssrc, 2u); + EXPECT_EQ(Create(2), packet); + }); + + buff.AddPacket(4, kPacketTimeUs, Create(4)); + + ssrcs = {3}; + buff.BackfillPackets(ssrcs, [](uint32_t ssrc, int64_t packet_time_us, + rtc::CopyOnWriteBuffer packet) { + EXPECT_EQ(ssrc, 3u); + EXPECT_EQ(Create(3), packet); + }); +} + +} // namespace cricket diff --git a/media/engine/webrtc_media_engine.cc b/media/engine/webrtc_media_engine.cc new file mode 100644 index 0000000000..b026b9d7c7 --- /dev/null +++ b/media/engine/webrtc_media_engine.cc @@ -0,0 +1,164 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "media/engine/webrtc_media_engine.h" + +#include <memory> +#include <utility> + +#include "absl/algorithm/container.h" +#include "media/engine/webrtc_voice_engine.h" +#include "system_wrappers/include/field_trial.h" + +#ifdef HAVE_WEBRTC_VIDEO +#include "media/engine/webrtc_video_engine.h" +#else +#include "media/engine/null_webrtc_video_engine.h" +#endif + +namespace cricket { + +std::unique_ptr<MediaEngineInterface> CreateMediaEngine( + MediaEngineDependencies dependencies) { + auto audio_engine = std::make_unique<WebRtcVoiceEngine>( + dependencies.task_queue_factory, std::move(dependencies.adm), + std::move(dependencies.audio_encoder_factory), + std::move(dependencies.audio_decoder_factory), + std::move(dependencies.audio_mixer), + std::move(dependencies.audio_processing)); +#ifdef HAVE_WEBRTC_VIDEO + auto video_engine = std::make_unique<WebRtcVideoEngine>( + std::move(dependencies.video_encoder_factory), + std::move(dependencies.video_decoder_factory)); +#else + auto video_engine = std::make_unique<NullWebRtcVideoEngine>(); +#endif + return std::make_unique<CompositeMediaEngine>(std::move(audio_engine), + std::move(video_engine)); +} + +namespace { +// Remove mutually exclusive extensions with lower priority. +void DiscardRedundantExtensions( + std::vector<webrtc::RtpExtension>* extensions, + rtc::ArrayView<const char* const> extensions_decreasing_prio) { + RTC_DCHECK(extensions); + bool found = false; + for (const char* uri : extensions_decreasing_prio) { + auto it = absl::c_find_if( + *extensions, + [uri](const webrtc::RtpExtension& rhs) { return rhs.uri == uri; }); + if (it != extensions->end()) { + if (found) { + extensions->erase(it); + } + found = true; + } + } +} +} // namespace + +bool ValidateRtpExtensions( + const std::vector<webrtc::RtpExtension>& extensions) { + bool id_used[1 + webrtc::RtpExtension::kMaxId] = {false}; + for (const auto& extension : extensions) { + if (extension.id < webrtc::RtpExtension::kMinId || + extension.id > webrtc::RtpExtension::kMaxId) { + RTC_LOG(LS_ERROR) << "Bad RTP extension ID: " << extension.ToString(); + return false; + } + if (id_used[extension.id]) { + RTC_LOG(LS_ERROR) << "Duplicate RTP extension ID: " + << extension.ToString(); + return false; + } + id_used[extension.id] = true; + } + return true; +} + +std::vector<webrtc::RtpExtension> FilterRtpExtensions( + const std::vector<webrtc::RtpExtension>& extensions, + bool (*supported)(absl::string_view), + bool filter_redundant_extensions) { + RTC_DCHECK(ValidateRtpExtensions(extensions)); + RTC_DCHECK(supported); + std::vector<webrtc::RtpExtension> result; + + // Ignore any extensions that we don't recognize. + for (const auto& extension : extensions) { + if (supported(extension.uri)) { + result.push_back(extension); + } else { + RTC_LOG(LS_WARNING) << "Unsupported RTP extension: " + << extension.ToString(); + } + } + + // Sort by name, ascending (prioritise encryption), so that we don't reset + // extensions if they were specified in a different order (also allows us + // to use std::unique below). + absl::c_sort(result, [](const webrtc::RtpExtension& rhs, + const webrtc::RtpExtension& lhs) { + return rhs.encrypt == lhs.encrypt ? rhs.uri < lhs.uri + : rhs.encrypt > lhs.encrypt; + }); + + // Remove unnecessary extensions (used on send side). + if (filter_redundant_extensions) { + auto it = std::unique( + result.begin(), result.end(), + [](const webrtc::RtpExtension& rhs, const webrtc::RtpExtension& lhs) { + return rhs.uri == lhs.uri && rhs.encrypt == lhs.encrypt; + }); + result.erase(it, result.end()); + + // Keep just the highest priority extension of any in the following lists. + if (webrtc::field_trial::IsEnabled("WebRTC-FilterAbsSendTimeExtension")) { + static const char* const kBweExtensionPriorities[] = { + webrtc::RtpExtension::kTransportSequenceNumberUri, + webrtc::RtpExtension::kAbsSendTimeUri, + webrtc::RtpExtension::kTimestampOffsetUri}; + DiscardRedundantExtensions(&result, kBweExtensionPriorities); + } else { + static const char* const kBweExtensionPriorities[] = { + webrtc::RtpExtension::kAbsSendTimeUri, + webrtc::RtpExtension::kTimestampOffsetUri}; + DiscardRedundantExtensions(&result, kBweExtensionPriorities); + } + } + return result; +} + +webrtc::BitrateConstraints GetBitrateConfigForCodec(const Codec& codec) { + webrtc::BitrateConstraints config; + int bitrate_kbps = 0; + if (codec.GetParam(kCodecParamMinBitrate, &bitrate_kbps) && + bitrate_kbps > 0) { + config.min_bitrate_bps = bitrate_kbps * 1000; + } else { + config.min_bitrate_bps = 0; + } + if (codec.GetParam(kCodecParamStartBitrate, &bitrate_kbps) && + bitrate_kbps > 0) { + config.start_bitrate_bps = bitrate_kbps * 1000; + } else { + // Do not reconfigure start bitrate unless it's specified and positive. + config.start_bitrate_bps = -1; + } + if (codec.GetParam(kCodecParamMaxBitrate, &bitrate_kbps) && + bitrate_kbps > 0) { + config.max_bitrate_bps = bitrate_kbps * 1000; + } else { + config.max_bitrate_bps = -1; + } + return config; +} +} // namespace cricket diff --git a/media/engine/webrtc_media_engine.h b/media/engine/webrtc_media_engine.h new file mode 100644 index 0000000000..dbb2a5fbb0 --- /dev/null +++ b/media/engine/webrtc_media_engine.h @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MEDIA_ENGINE_WEBRTC_MEDIA_ENGINE_H_ +#define MEDIA_ENGINE_WEBRTC_MEDIA_ENGINE_H_ + +#include <memory> +#include <string> +#include <vector> + +#include "api/audio/audio_mixer.h" +#include "api/audio_codecs/audio_decoder_factory.h" +#include "api/audio_codecs/audio_encoder_factory.h" +#include "api/rtp_parameters.h" +#include "api/task_queue/task_queue_factory.h" +#include "api/transport/bitrate_settings.h" +#include "api/video_codecs/video_decoder_factory.h" +#include "api/video_codecs/video_encoder_factory.h" +#include "media/base/codec.h" +#include "media/base/media_engine.h" +#include "modules/audio_device/include/audio_device.h" +#include "modules/audio_processing/include/audio_processing.h" +#include "rtc_base/system/rtc_export.h" + +namespace cricket { + +struct MediaEngineDependencies { + MediaEngineDependencies() = default; + MediaEngineDependencies(const MediaEngineDependencies&) = delete; + MediaEngineDependencies(MediaEngineDependencies&&) = default; + MediaEngineDependencies& operator=(const MediaEngineDependencies&) = delete; + MediaEngineDependencies& operator=(MediaEngineDependencies&&) = default; + ~MediaEngineDependencies() = default; + + webrtc::TaskQueueFactory* task_queue_factory = nullptr; + rtc::scoped_refptr<webrtc::AudioDeviceModule> adm; + rtc::scoped_refptr<webrtc::AudioEncoderFactory> audio_encoder_factory; + rtc::scoped_refptr<webrtc::AudioDecoderFactory> audio_decoder_factory; + rtc::scoped_refptr<webrtc::AudioMixer> audio_mixer; + rtc::scoped_refptr<webrtc::AudioProcessing> audio_processing; + + std::unique_ptr<webrtc::VideoEncoderFactory> video_encoder_factory; + std::unique_ptr<webrtc::VideoDecoderFactory> video_decoder_factory; +}; + +// CreateMediaEngine may be called on any thread, though the engine is +// only expected to be used on one thread, internally called the "worker +// thread". This is the thread Init must be called on. +RTC_EXPORT std::unique_ptr<MediaEngineInterface> CreateMediaEngine( + MediaEngineDependencies dependencies); + +// Verify that extension IDs are within 1-byte extension range and are not +// overlapping. +bool ValidateRtpExtensions(const std::vector<webrtc::RtpExtension>& extensions); + +// Discard any extensions not validated by the 'supported' predicate. Duplicate +// extensions are removed if 'filter_redundant_extensions' is set, and also any +// mutually exclusive extensions (see implementation for details) are removed. +std::vector<webrtc::RtpExtension> FilterRtpExtensions( + const std::vector<webrtc::RtpExtension>& extensions, + bool (*supported)(absl::string_view), + bool filter_redundant_extensions); + +webrtc::BitrateConstraints GetBitrateConfigForCodec(const Codec& codec); + +} // namespace cricket + +#endif // MEDIA_ENGINE_WEBRTC_MEDIA_ENGINE_H_ diff --git a/media/engine/webrtc_media_engine_defaults.cc b/media/engine/webrtc_media_engine_defaults.cc new file mode 100644 index 0000000000..1660873e8b --- /dev/null +++ b/media/engine/webrtc_media_engine_defaults.cc @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "media/engine/webrtc_media_engine_defaults.h" + +#include "api/audio_codecs/builtin_audio_decoder_factory.h" +#include "api/audio_codecs/builtin_audio_encoder_factory.h" +#include "api/task_queue/default_task_queue_factory.h" +#include "api/video/builtin_video_bitrate_allocator_factory.h" +#include "api/video_codecs/builtin_video_decoder_factory.h" +#include "api/video_codecs/builtin_video_encoder_factory.h" +#include "modules/audio_processing/include/audio_processing.h" +#include "rtc_base/checks.h" + +namespace webrtc { + +void SetMediaEngineDefaults(cricket::MediaEngineDependencies* deps) { + RTC_DCHECK(deps); + if (deps->task_queue_factory == nullptr) { + static TaskQueueFactory* const task_queue_factory = + CreateDefaultTaskQueueFactory().release(); + deps->task_queue_factory = task_queue_factory; + } + if (deps->audio_encoder_factory == nullptr) + deps->audio_encoder_factory = CreateBuiltinAudioEncoderFactory(); + if (deps->audio_decoder_factory == nullptr) + deps->audio_decoder_factory = CreateBuiltinAudioDecoderFactory(); + if (deps->audio_processing == nullptr) + deps->audio_processing = AudioProcessingBuilder().Create(); + + if (deps->video_encoder_factory == nullptr) + deps->video_encoder_factory = CreateBuiltinVideoEncoderFactory(); + if (deps->video_decoder_factory == nullptr) + deps->video_decoder_factory = CreateBuiltinVideoDecoderFactory(); +} + +} // namespace webrtc diff --git a/media/engine/webrtc_media_engine_defaults.h b/media/engine/webrtc_media_engine_defaults.h new file mode 100644 index 0000000000..16b1d462e3 --- /dev/null +++ b/media/engine/webrtc_media_engine_defaults.h @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MEDIA_ENGINE_WEBRTC_MEDIA_ENGINE_DEFAULTS_H_ +#define MEDIA_ENGINE_WEBRTC_MEDIA_ENGINE_DEFAULTS_H_ + +#include "media/engine/webrtc_media_engine.h" +#include "rtc_base/system/rtc_export.h" + +namespace webrtc { + +// Sets required but null dependencies with default factories. +RTC_EXPORT void SetMediaEngineDefaults(cricket::MediaEngineDependencies* deps); + +} // namespace webrtc + +#endif // MEDIA_ENGINE_WEBRTC_MEDIA_ENGINE_DEFAULTS_H_ diff --git a/media/engine/webrtc_media_engine_unittest.cc b/media/engine/webrtc_media_engine_unittest.cc new file mode 100644 index 0000000000..005a2d46be --- /dev/null +++ b/media/engine/webrtc_media_engine_unittest.cc @@ -0,0 +1,295 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "media/engine/webrtc_media_engine.h" + +#include <memory> +#include <utility> + +#include "media/engine/webrtc_media_engine_defaults.h" +#include "test/field_trial.h" +#include "test/gtest.h" + +using webrtc::RtpExtension; + +namespace cricket { +namespace { + +std::vector<RtpExtension> MakeUniqueExtensions() { + std::vector<RtpExtension> result; + char name[] = "a"; + for (int i = 0; i < 7; ++i) { + result.push_back(RtpExtension(name, 1 + i)); + name[0]++; + result.push_back(RtpExtension(name, 255 - i)); + name[0]++; + } + return result; +} + +std::vector<RtpExtension> MakeRedundantExtensions() { + std::vector<RtpExtension> result; + char name[] = "a"; + for (int i = 0; i < 7; ++i) { + result.push_back(RtpExtension(name, 1 + i)); + result.push_back(RtpExtension(name, 255 - i)); + name[0]++; + } + return result; +} + +bool SupportedExtensions1(absl::string_view name) { + return name == "c" || name == "i"; +} + +bool SupportedExtensions2(absl::string_view name) { + return name != "a" && name != "n"; +} + +bool IsSorted(const std::vector<webrtc::RtpExtension>& extensions) { + const std::string* last = nullptr; + for (const auto& extension : extensions) { + if (last && *last > extension.uri) { + return false; + } + last = &extension.uri; + } + return true; +} +} // namespace + +TEST(WebRtcMediaEngineTest, ValidateRtpExtensions_EmptyList) { + std::vector<RtpExtension> extensions; + EXPECT_TRUE(ValidateRtpExtensions(extensions)); +} + +TEST(WebRtcMediaEngineTest, ValidateRtpExtensions_AllGood) { + std::vector<RtpExtension> extensions = MakeUniqueExtensions(); + EXPECT_TRUE(ValidateRtpExtensions(extensions)); +} + +TEST(WebRtcMediaEngineTest, ValidateRtpExtensions_OutOfRangeId_Low) { + std::vector<RtpExtension> extensions = MakeUniqueExtensions(); + extensions.push_back(RtpExtension("foo", 0)); + EXPECT_FALSE(ValidateRtpExtensions(extensions)); +} + +TEST(WebRtcMediaEngineTest, ValidateRtpExtensions_OutOfRangeId_High) { + std::vector<RtpExtension> extensions = MakeUniqueExtensions(); + extensions.push_back(RtpExtension("foo", 256)); + EXPECT_FALSE(ValidateRtpExtensions(extensions)); +} + +TEST(WebRtcMediaEngineTest, ValidateRtpExtensions_OverlappingIds_StartOfSet) { + std::vector<RtpExtension> extensions = MakeUniqueExtensions(); + extensions.push_back(RtpExtension("foo", 1)); + EXPECT_FALSE(ValidateRtpExtensions(extensions)); +} + +TEST(WebRtcMediaEngineTest, ValidateRtpExtensions_OverlappingIds_EndOfSet) { + std::vector<RtpExtension> extensions = MakeUniqueExtensions(); + extensions.push_back(RtpExtension("foo", 255)); + EXPECT_FALSE(ValidateRtpExtensions(extensions)); +} + +TEST(WebRtcMediaEngineTest, FilterRtpExtensions_EmptyList) { + std::vector<RtpExtension> extensions; + std::vector<webrtc::RtpExtension> filtered = + FilterRtpExtensions(extensions, SupportedExtensions1, true); + EXPECT_EQ(0u, filtered.size()); +} + +TEST(WebRtcMediaEngineTest, FilterRtpExtensions_IncludeOnlySupported) { + std::vector<RtpExtension> extensions = MakeUniqueExtensions(); + std::vector<webrtc::RtpExtension> filtered = + FilterRtpExtensions(extensions, SupportedExtensions1, false); + EXPECT_EQ(2u, filtered.size()); + EXPECT_EQ("c", filtered[0].uri); + EXPECT_EQ("i", filtered[1].uri); +} + +TEST(WebRtcMediaEngineTest, FilterRtpExtensions_SortedByName_1) { + std::vector<RtpExtension> extensions = MakeUniqueExtensions(); + std::vector<webrtc::RtpExtension> filtered = + FilterRtpExtensions(extensions, SupportedExtensions2, false); + EXPECT_EQ(12u, filtered.size()); + EXPECT_TRUE(IsSorted(filtered)); +} + +TEST(WebRtcMediaEngineTest, FilterRtpExtensions_SortedByName_2) { + std::vector<RtpExtension> extensions = MakeUniqueExtensions(); + std::vector<webrtc::RtpExtension> filtered = + FilterRtpExtensions(extensions, SupportedExtensions2, true); + EXPECT_EQ(12u, filtered.size()); + EXPECT_TRUE(IsSorted(filtered)); +} + +TEST(WebRtcMediaEngineTest, FilterRtpExtensions_DontRemoveRedundant) { + std::vector<RtpExtension> extensions = MakeRedundantExtensions(); + std::vector<webrtc::RtpExtension> filtered = + FilterRtpExtensions(extensions, SupportedExtensions2, false); + EXPECT_EQ(12u, filtered.size()); + EXPECT_TRUE(IsSorted(filtered)); + EXPECT_EQ(filtered[0].uri, filtered[1].uri); +} + +TEST(WebRtcMediaEngineTest, FilterRtpExtensions_RemoveRedundant) { + std::vector<RtpExtension> extensions = MakeRedundantExtensions(); + std::vector<webrtc::RtpExtension> filtered = + FilterRtpExtensions(extensions, SupportedExtensions2, true); + EXPECT_EQ(6u, filtered.size()); + EXPECT_TRUE(IsSorted(filtered)); + EXPECT_NE(filtered[0].uri, filtered[1].uri); +} + +TEST(WebRtcMediaEngineTest, FilterRtpExtensions_RemoveRedundantEncrypted_1) { + std::vector<RtpExtension> extensions; + extensions.push_back(webrtc::RtpExtension("b", 1)); + extensions.push_back(webrtc::RtpExtension("b", 2, true)); + extensions.push_back(webrtc::RtpExtension("c", 3)); + extensions.push_back(webrtc::RtpExtension("b", 4)); + std::vector<webrtc::RtpExtension> filtered = + FilterRtpExtensions(extensions, SupportedExtensions2, true); + EXPECT_EQ(3u, filtered.size()); + EXPECT_TRUE(IsSorted(filtered)); + EXPECT_EQ(filtered[0].uri, filtered[1].uri); + EXPECT_NE(filtered[0].encrypt, filtered[1].encrypt); + EXPECT_NE(filtered[0].uri, filtered[2].uri); + EXPECT_NE(filtered[1].uri, filtered[2].uri); +} + +TEST(WebRtcMediaEngineTest, FilterRtpExtensions_RemoveRedundantEncrypted_2) { + std::vector<RtpExtension> extensions; + extensions.push_back(webrtc::RtpExtension("b", 1, true)); + extensions.push_back(webrtc::RtpExtension("b", 2)); + extensions.push_back(webrtc::RtpExtension("c", 3)); + extensions.push_back(webrtc::RtpExtension("b", 4)); + std::vector<webrtc::RtpExtension> filtered = + FilterRtpExtensions(extensions, SupportedExtensions2, true); + EXPECT_EQ(3u, filtered.size()); + EXPECT_TRUE(IsSorted(filtered)); + EXPECT_EQ(filtered[0].uri, filtered[1].uri); + EXPECT_NE(filtered[0].encrypt, filtered[1].encrypt); + EXPECT_NE(filtered[0].uri, filtered[2].uri); + EXPECT_NE(filtered[1].uri, filtered[2].uri); +} + +TEST(WebRtcMediaEngineTest, FilterRtpExtensions_RemoveRedundantBwe_1) { + webrtc::test::ScopedFieldTrials override_field_trials_( + "WebRTC-FilterAbsSendTimeExtension/Enabled/"); + std::vector<RtpExtension> extensions; + extensions.push_back( + RtpExtension(RtpExtension::kTransportSequenceNumberUri, 3)); + extensions.push_back(RtpExtension(RtpExtension::kTimestampOffsetUri, 9)); + extensions.push_back(RtpExtension(RtpExtension::kAbsSendTimeUri, 6)); + extensions.push_back( + RtpExtension(RtpExtension::kTransportSequenceNumberUri, 1)); + extensions.push_back(RtpExtension(RtpExtension::kTimestampOffsetUri, 14)); + std::vector<webrtc::RtpExtension> filtered = + FilterRtpExtensions(extensions, SupportedExtensions2, true); + EXPECT_EQ(1u, filtered.size()); + EXPECT_EQ(RtpExtension::kTransportSequenceNumberUri, filtered[0].uri); +} + +TEST(WebRtcMediaEngineTest, + FilterRtpExtensions_RemoveRedundantBwe_1_KeepAbsSendTime) { + std::vector<RtpExtension> extensions; + extensions.push_back( + RtpExtension(RtpExtension::kTransportSequenceNumberUri, 3)); + extensions.push_back(RtpExtension(RtpExtension::kTimestampOffsetUri, 9)); + extensions.push_back(RtpExtension(RtpExtension::kAbsSendTimeUri, 6)); + extensions.push_back( + RtpExtension(RtpExtension::kTransportSequenceNumberUri, 1)); + extensions.push_back(RtpExtension(RtpExtension::kTimestampOffsetUri, 14)); + std::vector<webrtc::RtpExtension> filtered = + FilterRtpExtensions(extensions, SupportedExtensions2, true); + EXPECT_EQ(2u, filtered.size()); + EXPECT_EQ(RtpExtension::kTransportSequenceNumberUri, filtered[0].uri); + EXPECT_EQ(RtpExtension::kAbsSendTimeUri, filtered[1].uri); +} + +TEST(WebRtcMediaEngineTest, FilterRtpExtensions_RemoveRedundantBweEncrypted_1) { + webrtc::test::ScopedFieldTrials override_field_trials_( + "WebRTC-FilterAbsSendTimeExtension/Enabled/"); + std::vector<RtpExtension> extensions; + extensions.push_back( + RtpExtension(RtpExtension::kTransportSequenceNumberUri, 3)); + extensions.push_back( + RtpExtension(RtpExtension::kTransportSequenceNumberUri, 4, true)); + extensions.push_back(RtpExtension(RtpExtension::kTimestampOffsetUri, 9)); + extensions.push_back(RtpExtension(RtpExtension::kAbsSendTimeUri, 6)); + extensions.push_back( + RtpExtension(RtpExtension::kTransportSequenceNumberUri, 1)); + extensions.push_back( + RtpExtension(RtpExtension::kTransportSequenceNumberUri, 2, true)); + extensions.push_back(RtpExtension(RtpExtension::kTimestampOffsetUri, 14)); + std::vector<webrtc::RtpExtension> filtered = + FilterRtpExtensions(extensions, SupportedExtensions2, true); + EXPECT_EQ(2u, filtered.size()); + EXPECT_EQ(RtpExtension::kTransportSequenceNumberUri, filtered[0].uri); + EXPECT_EQ(RtpExtension::kTransportSequenceNumberUri, filtered[1].uri); + EXPECT_NE(filtered[0].encrypt, filtered[1].encrypt); +} + +TEST(WebRtcMediaEngineTest, + FilterRtpExtensions_RemoveRedundantBweEncrypted_1_KeepAbsSendTime) { + std::vector<RtpExtension> extensions; + extensions.push_back( + RtpExtension(RtpExtension::kTransportSequenceNumberUri, 3)); + extensions.push_back( + RtpExtension(RtpExtension::kTransportSequenceNumberUri, 4, true)); + extensions.push_back(RtpExtension(RtpExtension::kTimestampOffsetUri, 9)); + extensions.push_back(RtpExtension(RtpExtension::kAbsSendTimeUri, 6)); + extensions.push_back( + RtpExtension(RtpExtension::kTransportSequenceNumberUri, 1)); + extensions.push_back( + RtpExtension(RtpExtension::kTransportSequenceNumberUri, 2, true)); + extensions.push_back(RtpExtension(RtpExtension::kTimestampOffsetUri, 14)); + std::vector<webrtc::RtpExtension> filtered = + FilterRtpExtensions(extensions, SupportedExtensions2, true); + EXPECT_EQ(3u, filtered.size()); + EXPECT_EQ(RtpExtension::kTransportSequenceNumberUri, filtered[0].uri); + EXPECT_EQ(RtpExtension::kTransportSequenceNumberUri, filtered[1].uri); + EXPECT_EQ(RtpExtension::kAbsSendTimeUri, filtered[2].uri); + EXPECT_NE(filtered[0].encrypt, filtered[1].encrypt); +} + +TEST(WebRtcMediaEngineTest, FilterRtpExtensions_RemoveRedundantBwe_2) { + std::vector<RtpExtension> extensions; + extensions.push_back(RtpExtension(RtpExtension::kTimestampOffsetUri, 1)); + extensions.push_back(RtpExtension(RtpExtension::kAbsSendTimeUri, 14)); + extensions.push_back(RtpExtension(RtpExtension::kTimestampOffsetUri, 7)); + std::vector<webrtc::RtpExtension> filtered = + FilterRtpExtensions(extensions, SupportedExtensions2, true); + EXPECT_EQ(1u, filtered.size()); + EXPECT_EQ(RtpExtension::kAbsSendTimeUri, filtered[0].uri); +} + +TEST(WebRtcMediaEngineTest, FilterRtpExtensions_RemoveRedundantBwe_3) { + std::vector<RtpExtension> extensions; + extensions.push_back(RtpExtension(RtpExtension::kTimestampOffsetUri, 2)); + extensions.push_back(RtpExtension(RtpExtension::kTimestampOffsetUri, 14)); + std::vector<webrtc::RtpExtension> filtered = + FilterRtpExtensions(extensions, SupportedExtensions2, true); + EXPECT_EQ(1u, filtered.size()); + EXPECT_EQ(RtpExtension::kTimestampOffsetUri, filtered[0].uri); +} + +TEST(WebRtcMediaEngineTest, Create) { + MediaEngineDependencies deps; + webrtc::SetMediaEngineDefaults(&deps); + + std::unique_ptr<MediaEngineInterface> engine = + CreateMediaEngine(std::move(deps)); + + EXPECT_TRUE(engine); +} + +} // namespace cricket diff --git a/media/engine/webrtc_video_engine.cc b/media/engine/webrtc_video_engine.cc new file mode 100644 index 0000000000..71a0939cb9 --- /dev/null +++ b/media/engine/webrtc_video_engine.cc @@ -0,0 +1,3600 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "media/engine/webrtc_video_engine.h" + +#include <stdio.h> + +#include <algorithm> +#include <set> +#include <string> +#include <utility> + +#include "absl/algorithm/container.h" +#include "absl/strings/match.h" +#include "api/media_stream_interface.h" +#include "api/transport/datagram_transport_interface.h" +#include "api/units/data_rate.h" +#include "api/video/video_codec_constants.h" +#include "api/video/video_codec_type.h" +#include "api/video_codecs/sdp_video_format.h" +#include "api/video_codecs/video_decoder_factory.h" +#include "api/video_codecs/video_encoder.h" +#include "api/video_codecs/video_encoder_factory.h" +#include "call/call.h" +#include "media/engine/simulcast.h" +#include "media/engine/webrtc_media_engine.h" +#include "media/engine/webrtc_voice_engine.h" +#include "rtc_base/copy_on_write_buffer.h" +#include "rtc_base/experiments/field_trial_parser.h" +#include "rtc_base/experiments/field_trial_units.h" +#include "rtc_base/experiments/min_video_bitrate_experiment.h" +#include "rtc_base/logging.h" +#include "rtc_base/numerics/safe_conversions.h" +#include "rtc_base/strings/string_builder.h" +#include "rtc_base/time_utils.h" +#include "rtc_base/trace_event.h" +#include "system_wrappers/include/field_trial.h" + +namespace cricket { + +namespace { + +const int kMinLayerSize = 16; + +const char* StreamTypeToString( + webrtc::VideoSendStream::StreamStats::StreamType type) { + switch (type) { + case webrtc::VideoSendStream::StreamStats::StreamType::kMedia: + return "kMedia"; + case webrtc::VideoSendStream::StreamStats::StreamType::kRtx: + return "kRtx"; + case webrtc::VideoSendStream::StreamStats::StreamType::kFlexfec: + return "kFlexfec"; + } + return nullptr; +} + +// If this field trial is enabled, we will enable sending FlexFEC and disable +// sending ULPFEC whenever the former has been negotiated in the SDPs. +bool IsFlexfecFieldTrialEnabled() { + return webrtc::field_trial::IsEnabled("WebRTC-FlexFEC-03"); +} + +// If this field trial is enabled, the "flexfec-03" codec will be advertised +// as being supported. This means that "flexfec-03" will appear in the default +// SDP offer, and we therefore need to be ready to receive FlexFEC packets from +// the remote. It also means that FlexFEC SSRCs will be generated by +// MediaSession and added as "a=ssrc:" and "a=ssrc-group:" lines in the local +// SDP. +bool IsFlexfecAdvertisedFieldTrialEnabled() { + return webrtc::field_trial::IsEnabled("WebRTC-FlexFEC-03-Advertised"); +} + +void AddDefaultFeedbackParams(VideoCodec* codec) { + // Don't add any feedback params for RED and ULPFEC. + if (codec->name == kRedCodecName || codec->name == kUlpfecCodecName) + return; + codec->AddFeedbackParam(FeedbackParam(kRtcpFbParamRemb, kParamValueEmpty)); + codec->AddFeedbackParam( + FeedbackParam(kRtcpFbParamTransportCc, kParamValueEmpty)); + // Don't add any more feedback params for FLEXFEC. + if (codec->name == kFlexfecCodecName) + return; + codec->AddFeedbackParam(FeedbackParam(kRtcpFbParamCcm, kRtcpFbCcmParamFir)); + codec->AddFeedbackParam(FeedbackParam(kRtcpFbParamNack, kParamValueEmpty)); + codec->AddFeedbackParam(FeedbackParam(kRtcpFbParamNack, kRtcpFbNackParamPli)); + if (codec->name == kVp8CodecName && + webrtc::field_trial::IsEnabled("WebRTC-RtcpLossNotification")) { + codec->AddFeedbackParam(FeedbackParam(kRtcpFbParamLntf, kParamValueEmpty)); + } +} + +// This function will assign dynamic payload types (in the range [96, 127]) to +// the input codecs, and also add ULPFEC, RED, FlexFEC, and associated RTX +// codecs for recognized codecs (VP8, VP9, H264, and RED). It will also add +// default feedback params to the codecs. +std::vector<VideoCodec> AssignPayloadTypesAndDefaultCodecs( + std::vector<webrtc::SdpVideoFormat> input_formats) { + if (input_formats.empty()) + return std::vector<VideoCodec>(); + static const int kFirstDynamicPayloadType = 96; + static const int kLastDynamicPayloadType = 127; + int payload_type = kFirstDynamicPayloadType; + + input_formats.push_back(webrtc::SdpVideoFormat(kRedCodecName)); + input_formats.push_back(webrtc::SdpVideoFormat(kUlpfecCodecName)); + + if (IsFlexfecAdvertisedFieldTrialEnabled()) { + webrtc::SdpVideoFormat flexfec_format(kFlexfecCodecName); + // This value is currently arbitrarily set to 10 seconds. (The unit + // is microseconds.) This parameter MUST be present in the SDP, but + // we never use the actual value anywhere in our code however. + // TODO(brandtr): Consider honouring this value in the sender and receiver. + flexfec_format.parameters = {{kFlexfecFmtpRepairWindow, "10000000"}}; + input_formats.push_back(flexfec_format); + } + + std::vector<VideoCodec> output_codecs; + for (const webrtc::SdpVideoFormat& format : input_formats) { + VideoCodec codec(format); + codec.id = payload_type; + AddDefaultFeedbackParams(&codec); + output_codecs.push_back(codec); + + // Increment payload type. + ++payload_type; + if (payload_type > kLastDynamicPayloadType) { + RTC_LOG(LS_ERROR) << "Out of dynamic payload types, skipping the rest."; + break; + } + + // Add associated RTX codec for non-FEC codecs. + if (!absl::EqualsIgnoreCase(codec.name, kUlpfecCodecName) && + !absl::EqualsIgnoreCase(codec.name, kFlexfecCodecName)) { + output_codecs.push_back( + VideoCodec::CreateRtxCodec(payload_type, codec.id)); + + // Increment payload type. + ++payload_type; + if (payload_type > kLastDynamicPayloadType) { + RTC_LOG(LS_ERROR) << "Out of dynamic payload types, skipping the rest."; + break; + } + } + } + return output_codecs; +} + +// is_decoder_factory is needed to keep track of the implict assumption that any +// H264 decoder also supports constrained base line profile. +// TODO(kron): Perhaps it better to move the implcit knowledge to the place +// where codecs are negotiated. +template <class T> +std::vector<VideoCodec> GetPayloadTypesAndDefaultCodecs( + const T* factory, + bool is_decoder_factory) { + if (!factory) { + return {}; + } + + std::vector<webrtc::SdpVideoFormat> supported_formats = + factory->GetSupportedFormats(); + if (is_decoder_factory) { + AddH264ConstrainedBaselineProfileToSupportedFormats(&supported_formats); + } + + return AssignPayloadTypesAndDefaultCodecs(std::move(supported_formats)); +} + +bool IsTemporalLayersSupported(const std::string& codec_name) { + return absl::EqualsIgnoreCase(codec_name, kVp8CodecName) || + absl::EqualsIgnoreCase(codec_name, kVp9CodecName); +} + +static std::string CodecVectorToString(const std::vector<VideoCodec>& codecs) { + rtc::StringBuilder out; + out << "{"; + for (size_t i = 0; i < codecs.size(); ++i) { + out << codecs[i].ToString(); + if (i != codecs.size() - 1) { + out << ", "; + } + } + out << "}"; + return out.Release(); +} + +static bool ValidateCodecFormats(const std::vector<VideoCodec>& codecs) { + bool has_video = false; + for (size_t i = 0; i < codecs.size(); ++i) { + if (!codecs[i].ValidateCodecFormat()) { + return false; + } + if (codecs[i].GetCodecType() == VideoCodec::CODEC_VIDEO) { + has_video = true; + } + } + if (!has_video) { + RTC_LOG(LS_ERROR) << "Setting codecs without a video codec is invalid: " + << CodecVectorToString(codecs); + return false; + } + return true; +} + +static bool ValidateStreamParams(const StreamParams& sp) { + if (sp.ssrcs.empty()) { + RTC_LOG(LS_ERROR) << "No SSRCs in stream parameters: " << sp.ToString(); + return false; + } + + std::vector<uint32_t> primary_ssrcs; + sp.GetPrimarySsrcs(&primary_ssrcs); + std::vector<uint32_t> rtx_ssrcs; + sp.GetFidSsrcs(primary_ssrcs, &rtx_ssrcs); + for (uint32_t rtx_ssrc : rtx_ssrcs) { + bool rtx_ssrc_present = false; + for (uint32_t sp_ssrc : sp.ssrcs) { + if (sp_ssrc == rtx_ssrc) { + rtx_ssrc_present = true; + break; + } + } + if (!rtx_ssrc_present) { + RTC_LOG(LS_ERROR) << "RTX SSRC '" << rtx_ssrc + << "' missing from StreamParams ssrcs: " + << sp.ToString(); + return false; + } + } + if (!rtx_ssrcs.empty() && primary_ssrcs.size() != rtx_ssrcs.size()) { + RTC_LOG(LS_ERROR) + << "RTX SSRCs exist, but don't cover all SSRCs (unsupported): " + << sp.ToString(); + return false; + } + + return true; +} + +// Returns true if the given codec is disallowed from doing simulcast. +bool IsCodecBlacklistedForSimulcast(const std::string& codec_name) { + return !webrtc::field_trial::IsDisabled("WebRTC-H264Simulcast") + ? absl::EqualsIgnoreCase(codec_name, kVp9CodecName) + : absl::EqualsIgnoreCase(codec_name, kH264CodecName) || + absl::EqualsIgnoreCase(codec_name, kVp9CodecName); +} + +// The selected thresholds for QVGA and VGA corresponded to a QP around 10. +// The change in QP declined above the selected bitrates. +static int GetMaxDefaultVideoBitrateKbps(int width, + int height, + bool is_screenshare) { + int max_bitrate; + if (width * height <= 320 * 240) { + max_bitrate = 600; + } else if (width * height <= 640 * 480) { + max_bitrate = 1700; + } else if (width * height <= 960 * 540) { + max_bitrate = 2000; + } else { + max_bitrate = 2500; + } + if (is_screenshare) + max_bitrate = std::max(max_bitrate, 1200); + return max_bitrate; +} + +bool GetVp9LayersFromFieldTrialGroup(size_t* num_spatial_layers, + size_t* num_temporal_layers) { + std::string group = webrtc::field_trial::FindFullName("WebRTC-SupportVP9SVC"); + if (group.empty()) + return false; + + if (sscanf(group.c_str(), "EnabledByFlag_%zuSL%zuTL", num_spatial_layers, + num_temporal_layers) != 2) { + return false; + } + if (*num_spatial_layers > webrtc::kMaxSpatialLayers || + *num_spatial_layers < 1) + return false; + + const size_t kMaxTemporalLayers = 3; + if (*num_temporal_layers > kMaxTemporalLayers || *num_temporal_layers < 1) + return false; + + return true; +} + +absl::optional<size_t> GetVp9SpatialLayersFromFieldTrial() { + size_t num_sl; + size_t num_tl; + if (GetVp9LayersFromFieldTrialGroup(&num_sl, &num_tl)) { + return num_sl; + } + return absl::nullopt; +} + +absl::optional<size_t> GetVp9TemporalLayersFromFieldTrial() { + size_t num_sl; + size_t num_tl; + if (GetVp9LayersFromFieldTrialGroup(&num_sl, &num_tl)) { + return num_tl; + } + return absl::nullopt; +} + +// Returns its smallest positive argument. If neither argument is positive, +// returns an arbitrary nonpositive value. +int MinPositive(int a, int b) { + if (a <= 0) { + return b; + } + if (b <= 0) { + return a; + } + return std::min(a, b); +} + +bool IsLayerActive(const webrtc::RtpEncodingParameters& layer) { + return layer.active && + (!layer.max_bitrate_bps || *layer.max_bitrate_bps > 0) && + (!layer.max_framerate || *layer.max_framerate > 0); +} + +size_t FindRequiredActiveLayers( + const webrtc::VideoEncoderConfig& encoder_config) { + // Need enough layers so that at least the first active one is present. + for (size_t i = 0; i < encoder_config.number_of_streams; ++i) { + if (encoder_config.simulcast_layers[i].active) { + return i + 1; + } + } + return 0; +} + +int NumActiveStreams(const webrtc::RtpParameters& rtp_parameters) { + int res = 0; + for (size_t i = 0; i < rtp_parameters.encodings.size(); ++i) { + if (rtp_parameters.encodings[i].active) { + ++res; + } + } + return res; +} + +std::map<uint32_t, webrtc::VideoSendStream::StreamStats> +MergeInfoAboutOutboundRtpSubstreams( + const std::map<uint32_t, webrtc::VideoSendStream::StreamStats>& + substreams) { + std::map<uint32_t, webrtc::VideoSendStream::StreamStats> rtp_substreams; + // Add substreams for all RTP media streams. + for (const auto& pair : substreams) { + uint32_t ssrc = pair.first; + const webrtc::VideoSendStream::StreamStats& substream = pair.second; + switch (substream.type) { + case webrtc::VideoSendStream::StreamStats::StreamType::kMedia: + break; + case webrtc::VideoSendStream::StreamStats::StreamType::kRtx: + case webrtc::VideoSendStream::StreamStats::StreamType::kFlexfec: + continue; + } + rtp_substreams.insert(std::make_pair(ssrc, substream)); + } + // Complement the kMedia substream stats with the associated kRtx and kFlexfec + // substream stats. + for (const auto& pair : substreams) { + switch (pair.second.type) { + case webrtc::VideoSendStream::StreamStats::StreamType::kMedia: + continue; + case webrtc::VideoSendStream::StreamStats::StreamType::kRtx: + case webrtc::VideoSendStream::StreamStats::StreamType::kFlexfec: + break; + } + // The associated substream is an RTX or FlexFEC substream that is + // referencing an RTP media substream. + const webrtc::VideoSendStream::StreamStats& associated_substream = + pair.second; + RTC_DCHECK(associated_substream.referenced_media_ssrc.has_value()); + uint32_t media_ssrc = associated_substream.referenced_media_ssrc.value(); + if (substreams.find(media_ssrc) == substreams.end()) { + RTC_LOG(LS_WARNING) << "Substream [ssrc: " << pair.first << ", type: " + << StreamTypeToString(associated_substream.type) + << "] is associated with a media ssrc (" << media_ssrc + << ") that does not have StreamStats. Ignoring its " + << "RTP stats."; + continue; + } + webrtc::VideoSendStream::StreamStats& rtp_substream = + rtp_substreams[media_ssrc]; + + // We only merge |rtp_stats|. All other metrics are not applicable for RTX + // and FlexFEC. + // TODO(hbos): kRtx and kFlexfec stats should use a separate struct to make + // it clear what is or is not applicable. + rtp_substream.rtp_stats.Add(associated_substream.rtp_stats); + } + return rtp_substreams; +} + +} // namespace + +// This constant is really an on/off, lower-level configurable NACK history +// duration hasn't been implemented. +static const int kNackHistoryMs = 1000; + +static const int kDefaultRtcpReceiverReportSsrc = 1; + +// Minimum time interval for logging stats. +static const int64_t kStatsLogIntervalMs = 10000; + +std::map<uint32_t, webrtc::VideoSendStream::StreamStats> +MergeInfoAboutOutboundRtpSubstreamsForTesting( + const std::map<uint32_t, webrtc::VideoSendStream::StreamStats>& + substreams) { + return MergeInfoAboutOutboundRtpSubstreams(substreams); +} + +rtc::scoped_refptr<webrtc::VideoEncoderConfig::EncoderSpecificSettings> +WebRtcVideoChannel::WebRtcVideoSendStream::ConfigureVideoEncoderSettings( + const VideoCodec& codec) { + RTC_DCHECK_RUN_ON(&thread_checker_); + bool is_screencast = parameters_.options.is_screencast.value_or(false); + // No automatic resizing when using simulcast or screencast. + bool automatic_resize = + !is_screencast && (parameters_.config.rtp.ssrcs.size() == 1 || + NumActiveStreams(rtp_parameters_) == 1); + bool frame_dropping = !is_screencast; + bool denoising; + bool codec_default_denoising = false; + if (is_screencast) { + denoising = false; + } else { + // Use codec default if video_noise_reduction is unset. + codec_default_denoising = !parameters_.options.video_noise_reduction; + denoising = parameters_.options.video_noise_reduction.value_or(false); + } + + if (absl::EqualsIgnoreCase(codec.name, kH264CodecName)) { + webrtc::VideoCodecH264 h264_settings = + webrtc::VideoEncoder::GetDefaultH264Settings(); + h264_settings.frameDroppingOn = frame_dropping; + return new rtc::RefCountedObject< + webrtc::VideoEncoderConfig::H264EncoderSpecificSettings>(h264_settings); + } + if (absl::EqualsIgnoreCase(codec.name, kVp8CodecName)) { + webrtc::VideoCodecVP8 vp8_settings = + webrtc::VideoEncoder::GetDefaultVp8Settings(); + vp8_settings.automaticResizeOn = automatic_resize; + // VP8 denoising is enabled by default. + vp8_settings.denoisingOn = codec_default_denoising ? true : denoising; + vp8_settings.frameDroppingOn = frame_dropping; + return new rtc::RefCountedObject< + webrtc::VideoEncoderConfig::Vp8EncoderSpecificSettings>(vp8_settings); + } + if (absl::EqualsIgnoreCase(codec.name, kVp9CodecName)) { + webrtc::VideoCodecVP9 vp9_settings = + webrtc::VideoEncoder::GetDefaultVp9Settings(); + const size_t default_num_spatial_layers = + parameters_.config.rtp.ssrcs.size(); + const size_t num_spatial_layers = + GetVp9SpatialLayersFromFieldTrial().value_or( + default_num_spatial_layers); + + const size_t default_num_temporal_layers = + num_spatial_layers > 1 ? kConferenceDefaultNumTemporalLayers : 1; + const size_t num_temporal_layers = + GetVp9TemporalLayersFromFieldTrial().value_or( + default_num_temporal_layers); + + vp9_settings.numberOfSpatialLayers = std::min<unsigned char>( + num_spatial_layers, kConferenceMaxNumSpatialLayers); + vp9_settings.numberOfTemporalLayers = std::min<unsigned char>( + num_temporal_layers, kConferenceMaxNumTemporalLayers); + + // VP9 denoising is disabled by default. + vp9_settings.denoisingOn = codec_default_denoising ? true : denoising; + vp9_settings.automaticResizeOn = automatic_resize; + // Ensure frame dropping is always enabled. + RTC_DCHECK(vp9_settings.frameDroppingOn); + if (!is_screencast) { + webrtc::FieldTrialFlag interlayer_pred_experiment_enabled = + webrtc::FieldTrialFlag("Enabled"); + webrtc::FieldTrialEnum<webrtc::InterLayerPredMode> inter_layer_pred_mode( + "inter_layer_pred_mode", webrtc::InterLayerPredMode::kOnKeyPic, + {{"off", webrtc::InterLayerPredMode::kOff}, + {"on", webrtc::InterLayerPredMode::kOn}, + {"onkeypic", webrtc::InterLayerPredMode::kOnKeyPic}}); + webrtc::ParseFieldTrial( + {&interlayer_pred_experiment_enabled, &inter_layer_pred_mode}, + webrtc::field_trial::FindFullName("WebRTC-Vp9InterLayerPred")); + if (interlayer_pred_experiment_enabled) { + vp9_settings.interLayerPred = inter_layer_pred_mode; + } else { + // Limit inter-layer prediction to key pictures by default. + vp9_settings.interLayerPred = webrtc::InterLayerPredMode::kOnKeyPic; + } + } else { + // Multiple spatial layers vp9 screenshare needs flexible mode. + vp9_settings.flexibleMode = vp9_settings.numberOfSpatialLayers > 1; + vp9_settings.interLayerPred = webrtc::InterLayerPredMode::kOn; + } + return new rtc::RefCountedObject< + webrtc::VideoEncoderConfig::Vp9EncoderSpecificSettings>(vp9_settings); + } + return nullptr; +} + +DefaultUnsignalledSsrcHandler::DefaultUnsignalledSsrcHandler() + : default_sink_(nullptr) {} + +UnsignalledSsrcHandler::Action DefaultUnsignalledSsrcHandler::OnUnsignalledSsrc( + WebRtcVideoChannel* channel, + uint32_t ssrc) { + absl::optional<uint32_t> default_recv_ssrc = + channel->GetDefaultReceiveStreamSsrc(); + + if (default_recv_ssrc) { + RTC_LOG(LS_INFO) << "Destroying old default receive stream for SSRC=" + << ssrc << "."; + channel->RemoveRecvStream(*default_recv_ssrc); + } + + StreamParams sp = channel->unsignaled_stream_params(); + sp.ssrcs.push_back(ssrc); + + RTC_LOG(LS_INFO) << "Creating default receive stream for SSRC=" << ssrc + << "."; + if (!channel->AddRecvStream(sp, /*default_stream=*/true)) { + RTC_LOG(LS_WARNING) << "Could not create default receive stream."; + } + + // SSRC 0 returns default_recv_base_minimum_delay_ms. + const int unsignaled_ssrc = 0; + int default_recv_base_minimum_delay_ms = + channel->GetBaseMinimumPlayoutDelayMs(unsignaled_ssrc).value_or(0); + // Set base minimum delay if it was set before for the default receive stream. + channel->SetBaseMinimumPlayoutDelayMs(ssrc, + default_recv_base_minimum_delay_ms); + channel->SetSink(ssrc, default_sink_); + return kDeliverPacket; +} + +rtc::VideoSinkInterface<webrtc::VideoFrame>* +DefaultUnsignalledSsrcHandler::GetDefaultSink() const { + return default_sink_; +} + +void DefaultUnsignalledSsrcHandler::SetDefaultSink( + WebRtcVideoChannel* channel, + rtc::VideoSinkInterface<webrtc::VideoFrame>* sink) { + default_sink_ = sink; + absl::optional<uint32_t> default_recv_ssrc = + channel->GetDefaultReceiveStreamSsrc(); + if (default_recv_ssrc) { + channel->SetSink(*default_recv_ssrc, default_sink_); + } +} + +WebRtcVideoEngine::WebRtcVideoEngine( + std::unique_ptr<webrtc::VideoEncoderFactory> video_encoder_factory, + std::unique_ptr<webrtc::VideoDecoderFactory> video_decoder_factory) + : decoder_factory_(std::move(video_decoder_factory)), + encoder_factory_(std::move(video_encoder_factory)) { + RTC_LOG(LS_INFO) << "WebRtcVideoEngine::WebRtcVideoEngine()"; +} + +WebRtcVideoEngine::~WebRtcVideoEngine() { + RTC_LOG(LS_INFO) << "WebRtcVideoEngine::~WebRtcVideoEngine"; +} + +VideoMediaChannel* WebRtcVideoEngine::CreateMediaChannel( + webrtc::Call* call, + const MediaConfig& config, + const VideoOptions& options, + const webrtc::CryptoOptions& crypto_options, + webrtc::VideoBitrateAllocatorFactory* video_bitrate_allocator_factory) { + RTC_LOG(LS_INFO) << "CreateMediaChannel. Options: " << options.ToString(); + return new WebRtcVideoChannel(call, config, options, crypto_options, + encoder_factory_.get(), decoder_factory_.get(), + video_bitrate_allocator_factory); +} +std::vector<VideoCodec> WebRtcVideoEngine::send_codecs() const { + return GetPayloadTypesAndDefaultCodecs(encoder_factory_.get(), + /*is_decoder_factory=*/false); +} + +std::vector<VideoCodec> WebRtcVideoEngine::recv_codecs() const { + return GetPayloadTypesAndDefaultCodecs(decoder_factory_.get(), + /*is_decoder_factory=*/true); +} + +std::vector<webrtc::RtpHeaderExtensionCapability> +WebRtcVideoEngine::GetRtpHeaderExtensions() const { + std::vector<webrtc::RtpHeaderExtensionCapability> result; + int id = 1; + for (const auto& uri : + {webrtc::RtpExtension::kTimestampOffsetUri, + webrtc::RtpExtension::kAbsSendTimeUri, + webrtc::RtpExtension::kVideoRotationUri, + webrtc::RtpExtension::kTransportSequenceNumberUri, + webrtc::RtpExtension::kPlayoutDelayUri, + webrtc::RtpExtension::kVideoContentTypeUri, + webrtc::RtpExtension::kVideoTimingUri, + webrtc::RtpExtension::kFrameMarkingUri, + webrtc::RtpExtension::kColorSpaceUri, webrtc::RtpExtension::kMidUri, + webrtc::RtpExtension::kRidUri, webrtc::RtpExtension::kRepairedRidUri}) { + result.emplace_back(uri, id++, webrtc::RtpTransceiverDirection::kSendRecv); + } + result.emplace_back( + webrtc::RtpExtension::kGenericFrameDescriptorUri00, id, + webrtc::field_trial::IsEnabled("WebRTC-GenericDescriptorAdvertised") + ? webrtc::RtpTransceiverDirection::kSendRecv + : webrtc::RtpTransceiverDirection::kStopped); + return result; +} + +WebRtcVideoChannel::WebRtcVideoChannel( + webrtc::Call* call, + const MediaConfig& config, + const VideoOptions& options, + const webrtc::CryptoOptions& crypto_options, + webrtc::VideoEncoderFactory* encoder_factory, + webrtc::VideoDecoderFactory* decoder_factory, + webrtc::VideoBitrateAllocatorFactory* bitrate_allocator_factory) + : VideoMediaChannel(config), + worker_thread_(rtc::Thread::Current()), + call_(call), + unsignalled_ssrc_handler_(&default_unsignalled_ssrc_handler_), + video_config_(config.video), + encoder_factory_(encoder_factory), + decoder_factory_(decoder_factory), + bitrate_allocator_factory_(bitrate_allocator_factory), + default_send_options_(options), + last_stats_log_ms_(-1), + discard_unknown_ssrc_packets_(webrtc::field_trial::IsEnabled( + "WebRTC-Video-DiscardPacketsWithUnknownSsrc")), + crypto_options_(crypto_options), + unknown_ssrc_packet_buffer_( + webrtc::field_trial::IsEnabled( + "WebRTC-Video-BufferPacketsWithUnknownSsrc") + ? new UnhandledPacketsBuffer() + : nullptr) { + RTC_DCHECK(thread_checker_.IsCurrent()); + + rtcp_receiver_report_ssrc_ = kDefaultRtcpReceiverReportSsrc; + sending_ = false; + recv_codecs_ = MapCodecs(GetPayloadTypesAndDefaultCodecs( + decoder_factory_, /*is_decoder_factory=*/true)); + recv_flexfec_payload_type_ = + recv_codecs_.empty() ? 0 : recv_codecs_.front().flexfec_payload_type; +} + +WebRtcVideoChannel::~WebRtcVideoChannel() { + for (auto& kv : send_streams_) + delete kv.second; + for (auto& kv : receive_streams_) + delete kv.second; +} + +std::vector<WebRtcVideoChannel::VideoCodecSettings> +WebRtcVideoChannel::SelectSendVideoCodecs( + const std::vector<VideoCodecSettings>& remote_mapped_codecs) const { + std::vector<webrtc::SdpVideoFormat> sdp_formats = + encoder_factory_ ? encoder_factory_->GetImplementations() + : std::vector<webrtc::SdpVideoFormat>(); + + // The returned vector holds the VideoCodecSettings in term of preference. + // They are orderd by receive codec preference first and local implementation + // preference second. + std::vector<VideoCodecSettings> encoders; + for (const VideoCodecSettings& remote_codec : remote_mapped_codecs) { + for (auto format_it = sdp_formats.begin(); + format_it != sdp_formats.end();) { + // For H264, we will limit the encode level to the remote offered level + // regardless if level asymmetry is allowed or not. This is strictly not + // following the spec in https://tools.ietf.org/html/rfc6184#section-8.2.2 + // since we should limit the encode level to the lower of local and remote + // level when level asymmetry is not allowed. + if (IsSameCodec(format_it->name, format_it->parameters, + remote_codec.codec.name, remote_codec.codec.params)) { + encoders.push_back(remote_codec); + + // To allow the VideoEncoderFactory to keep information about which + // implementation to instantitate when CreateEncoder is called the two + // parmeter sets are merged. + encoders.back().codec.params.insert(format_it->parameters.begin(), + format_it->parameters.end()); + + format_it = sdp_formats.erase(format_it); + } else { + ++format_it; + } + } + } + + return encoders; +} + +bool WebRtcVideoChannel::NonFlexfecReceiveCodecsHaveChanged( + std::vector<VideoCodecSettings> before, + std::vector<VideoCodecSettings> after) { + // The receive codec order doesn't matter, so we sort the codecs before + // comparing. This is necessary because currently the + // only way to change the send codec is to munge SDP, which causes + // the receive codec list to change order, which causes the streams + // to be recreates which causes a "blink" of black video. In order + // to support munging the SDP in this way without recreating receive + // streams, we ignore the order of the received codecs so that + // changing the order doesn't cause this "blink". + auto comparison = [](const VideoCodecSettings& codec1, + const VideoCodecSettings& codec2) { + return codec1.codec.id > codec2.codec.id; + }; + absl::c_sort(before, comparison); + absl::c_sort(after, comparison); + + // Changes in FlexFEC payload type are handled separately in + // WebRtcVideoChannel::GetChangedRecvParameters, so disregard FlexFEC in the + // comparison here. + return !absl::c_equal(before, after, + VideoCodecSettings::EqualsDisregardingFlexfec); +} + +bool WebRtcVideoChannel::GetChangedSendParameters( + const VideoSendParameters& params, + ChangedSendParameters* changed_params) const { + if (!ValidateCodecFormats(params.codecs) || + !ValidateRtpExtensions(params.extensions)) { + return false; + } + + std::vector<VideoCodecSettings> negotiated_codecs = + SelectSendVideoCodecs(MapCodecs(params.codecs)); + + // We should only fail here if send direction is enabled. + if (params.is_stream_active && negotiated_codecs.empty()) { + RTC_LOG(LS_ERROR) << "No video codecs supported."; + return false; + } + + // Never enable sending FlexFEC, unless we are in the experiment. + if (!IsFlexfecFieldTrialEnabled()) { + RTC_LOG(LS_INFO) << "WebRTC-FlexFEC-03 field trial is not enabled."; + for (VideoCodecSettings& codec : negotiated_codecs) + codec.flexfec_payload_type = -1; + } + + if (negotiated_codecs_ != negotiated_codecs) { + if (negotiated_codecs.empty()) { + changed_params->send_codec = absl::nullopt; + } else if (send_codec_ != negotiated_codecs.front()) { + changed_params->send_codec = negotiated_codecs.front(); + } + changed_params->negotiated_codecs = std::move(negotiated_codecs); + } + + // Handle RTP header extensions. + if (params.extmap_allow_mixed != ExtmapAllowMixed()) { + changed_params->extmap_allow_mixed = params.extmap_allow_mixed; + } + std::vector<webrtc::RtpExtension> filtered_extensions = FilterRtpExtensions( + params.extensions, webrtc::RtpExtension::IsSupportedForVideo, true); + if (!send_rtp_extensions_ || (*send_rtp_extensions_ != filtered_extensions)) { + changed_params->rtp_header_extensions = + absl::optional<std::vector<webrtc::RtpExtension>>(filtered_extensions); + } + + if (params.mid != send_params_.mid) { + changed_params->mid = params.mid; + } + + // Handle max bitrate. + if (params.max_bandwidth_bps != send_params_.max_bandwidth_bps && + params.max_bandwidth_bps >= -1) { + // 0 or -1 uncaps max bitrate. + // TODO(pbos): Reconsider how 0 should be treated. It is not mentioned as a + // special value and might very well be used for stopping sending. + changed_params->max_bandwidth_bps = + params.max_bandwidth_bps == 0 ? -1 : params.max_bandwidth_bps; + } + + // Handle conference mode. + if (params.conference_mode != send_params_.conference_mode) { + changed_params->conference_mode = params.conference_mode; + } + + // Handle RTCP mode. + if (params.rtcp.reduced_size != send_params_.rtcp.reduced_size) { + changed_params->rtcp_mode = params.rtcp.reduced_size + ? webrtc::RtcpMode::kReducedSize + : webrtc::RtcpMode::kCompound; + } + + return true; +} + +bool WebRtcVideoChannel::SetSendParameters(const VideoSendParameters& params) { + RTC_DCHECK_RUN_ON(&thread_checker_); + TRACE_EVENT0("webrtc", "WebRtcVideoChannel::SetSendParameters"); + RTC_LOG(LS_INFO) << "SetSendParameters: " << params.ToString(); + ChangedSendParameters changed_params; + if (!GetChangedSendParameters(params, &changed_params)) { + return false; + } + + if (changed_params.negotiated_codecs) { + for (const auto& send_codec : *changed_params.negotiated_codecs) + RTC_LOG(LS_INFO) << "Negotiated codec: " << send_codec.codec.ToString(); + } + + send_params_ = params; + return ApplyChangedParams(changed_params); +} + +void WebRtcVideoChannel::RequestEncoderFallback() { + invoker_.AsyncInvoke<void>( + RTC_FROM_HERE, worker_thread_, [this] { + RTC_DCHECK_RUN_ON(&thread_checker_); + if (negotiated_codecs_.size() <= 1) { + RTC_LOG(LS_WARNING) + << "Encoder failed but no fallback codec is available"; + return; + } + + ChangedSendParameters params; + params.negotiated_codecs = negotiated_codecs_; + params.negotiated_codecs->erase(params.negotiated_codecs->begin()); + params.send_codec = params.negotiated_codecs->front(); + ApplyChangedParams(params); + }); +} + +void WebRtcVideoChannel::RequestEncoderSwitch( + const EncoderSwitchRequestCallback::Config& conf) { + invoker_.AsyncInvoke<void>(RTC_FROM_HERE, worker_thread_, [this, conf] { + RTC_DCHECK_RUN_ON(&thread_checker_); + + if (!allow_codec_switching_) { + RTC_LOG(LS_INFO) << "Encoder switch requested but codec switching has" + " not been enabled yet."; + requested_encoder_switch_ = conf; + return; + } + + for (const VideoCodecSettings& codec_setting : negotiated_codecs_) { + if (codec_setting.codec.name == conf.codec_name) { + if (conf.param) { + auto it = codec_setting.codec.params.find(*conf.param); + + if (it == codec_setting.codec.params.end()) { + continue; + } + + if (conf.value && it->second != *conf.value) { + continue; + } + } + + if (send_codec_ == codec_setting) { + // Already using this codec, no switch required. + return; + } + + ChangedSendParameters params; + params.send_codec = codec_setting; + ApplyChangedParams(params); + return; + } + } + + RTC_LOG(LS_WARNING) << "Requested encoder with codec_name:" + << conf.codec_name + << ", param:" << conf.param.value_or("none") + << " and value:" << conf.value.value_or("none") + << "not found. No switch performed."; + }); +} + +void WebRtcVideoChannel::RequestEncoderSwitch( + const webrtc::SdpVideoFormat& format) { + invoker_.AsyncInvoke<void>(RTC_FROM_HERE, worker_thread_, [this, format] { + RTC_DCHECK_RUN_ON(&thread_checker_); + + for (const VideoCodecSettings& codec_setting : negotiated_codecs_) { + if (IsSameCodec(format.name, format.parameters, codec_setting.codec.name, + codec_setting.codec.params)) { + VideoCodecSettings new_codec_setting = codec_setting; + for (const auto& kv : format.parameters) { + new_codec_setting.codec.params[kv.first] = kv.second; + } + + if (send_codec_ == new_codec_setting) { + // Already using this codec, no switch required. + return; + } + + ChangedSendParameters params; + params.send_codec = new_codec_setting; + ApplyChangedParams(params); + return; + } + } + + RTC_LOG(LS_WARNING) << "Encoder switch failed: SdpVideoFormat " + << format.ToString() << " not negotiated."; + }); +} + +bool WebRtcVideoChannel::ApplyChangedParams( + const ChangedSendParameters& changed_params) { + RTC_DCHECK_RUN_ON(&thread_checker_); + if (changed_params.negotiated_codecs) + negotiated_codecs_ = *changed_params.negotiated_codecs; + + if (changed_params.send_codec) + send_codec_ = changed_params.send_codec; + + if (changed_params.extmap_allow_mixed) { + SetExtmapAllowMixed(*changed_params.extmap_allow_mixed); + } + if (changed_params.rtp_header_extensions) { + send_rtp_extensions_ = changed_params.rtp_header_extensions; + } + + if (changed_params.send_codec || changed_params.max_bandwidth_bps) { + if (send_params_.max_bandwidth_bps == -1) { + // Unset the global max bitrate (max_bitrate_bps) if max_bandwidth_bps is + // -1, which corresponds to no "b=AS" attribute in SDP. Note that the + // global max bitrate may be set below in GetBitrateConfigForCodec, from + // the codec max bitrate. + // TODO(pbos): This should be reconsidered (codec max bitrate should + // probably not affect global call max bitrate). + bitrate_config_.max_bitrate_bps = -1; + } + + if (send_codec_) { + // TODO(holmer): Changing the codec parameters shouldn't necessarily mean + // that we change the min/max of bandwidth estimation. Reevaluate this. + bitrate_config_ = GetBitrateConfigForCodec(send_codec_->codec); + if (!changed_params.send_codec) { + // If the codec isn't changing, set the start bitrate to -1 which means + // "unchanged" so that BWE isn't affected. + bitrate_config_.start_bitrate_bps = -1; + } + } + + if (send_params_.max_bandwidth_bps >= 0) { + // Note that max_bandwidth_bps intentionally takes priority over the + // bitrate config for the codec. This allows FEC to be applied above the + // codec target bitrate. + // TODO(pbos): Figure out whether b=AS means max bitrate for this + // WebRtcVideoChannel (in which case we're good), or per sender (SSRC), + // in which case this should not set a BitrateConstraints but rather + // reconfigure all senders. + bitrate_config_.max_bitrate_bps = send_params_.max_bandwidth_bps == 0 + ? -1 + : send_params_.max_bandwidth_bps; + } + + call_->GetTransportControllerSend()->SetSdpBitrateParameters( + bitrate_config_); + } + + for (auto& kv : send_streams_) { + kv.second->SetSendParameters(changed_params); + } + if (changed_params.send_codec || changed_params.rtcp_mode) { + // Update receive feedback parameters from new codec or RTCP mode. + RTC_LOG(LS_INFO) + << "SetFeedbackOptions on all the receive streams because the send " + "codec or RTCP mode has changed."; + for (auto& kv : receive_streams_) { + RTC_DCHECK(kv.second != nullptr); + kv.second->SetFeedbackParameters( + HasLntf(send_codec_->codec), HasNack(send_codec_->codec), + HasTransportCc(send_codec_->codec), + send_params_.rtcp.reduced_size ? webrtc::RtcpMode::kReducedSize + : webrtc::RtcpMode::kCompound); + } + } + return true; +} + +webrtc::RtpParameters WebRtcVideoChannel::GetRtpSendParameters( + uint32_t ssrc) const { + RTC_DCHECK_RUN_ON(&thread_checker_); + auto it = send_streams_.find(ssrc); + if (it == send_streams_.end()) { + RTC_LOG(LS_WARNING) << "Attempting to get RTP send parameters for stream " + "with ssrc " + << ssrc << " which doesn't exist."; + return webrtc::RtpParameters(); + } + + webrtc::RtpParameters rtp_params = it->second->GetRtpParameters(); + // Need to add the common list of codecs to the send stream-specific + // RTP parameters. + for (const VideoCodec& codec : send_params_.codecs) { + rtp_params.codecs.push_back(codec.ToCodecParameters()); + } + return rtp_params; +} + +webrtc::RTCError WebRtcVideoChannel::SetRtpSendParameters( + uint32_t ssrc, + const webrtc::RtpParameters& parameters) { + RTC_DCHECK_RUN_ON(&thread_checker_); + TRACE_EVENT0("webrtc", "WebRtcVideoChannel::SetRtpSendParameters"); + auto it = send_streams_.find(ssrc); + if (it == send_streams_.end()) { + RTC_LOG(LS_ERROR) << "Attempting to set RTP send parameters for stream " + "with ssrc " + << ssrc << " which doesn't exist."; + return webrtc::RTCError(webrtc::RTCErrorType::INTERNAL_ERROR); + } + + // TODO(deadbeef): Handle setting parameters with a list of codecs in a + // different order (which should change the send codec). + webrtc::RtpParameters current_parameters = GetRtpSendParameters(ssrc); + if (current_parameters.codecs != parameters.codecs) { + RTC_DLOG(LS_ERROR) << "Using SetParameters to change the set of codecs " + "is not currently supported."; + return webrtc::RTCError(webrtc::RTCErrorType::INTERNAL_ERROR); + } + + if (!parameters.encodings.empty()) { + // Note that these values come from: + // https://tools.ietf.org/html/draft-ietf-tsvwg-rtcweb-qos-16#section-5 + // TODO(deadbeef): Change values depending on whether we are sending a + // keyframe or non-keyframe. + rtc::DiffServCodePoint new_dscp = rtc::DSCP_DEFAULT; + switch (parameters.encodings[0].network_priority) { + case webrtc::Priority::kVeryLow: + new_dscp = rtc::DSCP_CS1; + break; + case webrtc::Priority::kLow: + new_dscp = rtc::DSCP_DEFAULT; + break; + case webrtc::Priority::kMedium: + new_dscp = rtc::DSCP_AF42; + break; + case webrtc::Priority::kHigh: + new_dscp = rtc::DSCP_AF41; + break; + } + SetPreferredDscp(new_dscp); + } + + return it->second->SetRtpParameters(parameters); +} + +webrtc::RtpParameters WebRtcVideoChannel::GetRtpReceiveParameters( + uint32_t ssrc) const { + RTC_DCHECK_RUN_ON(&thread_checker_); + webrtc::RtpParameters rtp_params; + auto it = receive_streams_.find(ssrc); + if (it == receive_streams_.end()) { + RTC_LOG(LS_WARNING) + << "Attempting to get RTP receive parameters for stream " + "with SSRC " + << ssrc << " which doesn't exist."; + return webrtc::RtpParameters(); + } + rtp_params = it->second->GetRtpParameters(); + + // Add codecs, which any stream is prepared to receive. + for (const VideoCodec& codec : recv_params_.codecs) { + rtp_params.codecs.push_back(codec.ToCodecParameters()); + } + + return rtp_params; +} + +webrtc::RtpParameters WebRtcVideoChannel::GetDefaultRtpReceiveParameters() + const { + RTC_DCHECK_RUN_ON(&thread_checker_); + webrtc::RtpParameters rtp_params; + if (!default_unsignalled_ssrc_handler_.GetDefaultSink()) { + RTC_LOG(LS_WARNING) << "Attempting to get RTP parameters for the default, " + "unsignaled video receive stream, but not yet " + "configured to receive such a stream."; + return rtp_params; + } + rtp_params.encodings.emplace_back(); + + // Add codecs, which any stream is prepared to receive. + for (const VideoCodec& codec : recv_params_.codecs) { + rtp_params.codecs.push_back(codec.ToCodecParameters()); + } + + return rtp_params; +} + +bool WebRtcVideoChannel::GetChangedRecvParameters( + const VideoRecvParameters& params, + ChangedRecvParameters* changed_params) const { + if (!ValidateCodecFormats(params.codecs) || + !ValidateRtpExtensions(params.extensions)) { + return false; + } + + // Handle receive codecs. + const std::vector<VideoCodecSettings> mapped_codecs = + MapCodecs(params.codecs); + if (mapped_codecs.empty()) { + RTC_LOG(LS_ERROR) << "SetRecvParameters called without any video codecs."; + return false; + } + + // Verify that every mapped codec is supported locally. + if (params.is_stream_active) { + const std::vector<VideoCodec> local_supported_codecs = + GetPayloadTypesAndDefaultCodecs(decoder_factory_, + /*is_decoder_factory=*/true); + for (const VideoCodecSettings& mapped_codec : mapped_codecs) { + if (!FindMatchingCodec(local_supported_codecs, mapped_codec.codec)) { + RTC_LOG(LS_ERROR) + << "SetRecvParameters called with unsupported video codec: " + << mapped_codec.codec.ToString(); + return false; + } + } + } + + if (NonFlexfecReceiveCodecsHaveChanged(recv_codecs_, mapped_codecs)) { + changed_params->codec_settings = + absl::optional<std::vector<VideoCodecSettings>>(mapped_codecs); + } + + // Handle RTP header extensions. + std::vector<webrtc::RtpExtension> filtered_extensions = FilterRtpExtensions( + params.extensions, webrtc::RtpExtension::IsSupportedForVideo, false); + if (filtered_extensions != recv_rtp_extensions_) { + changed_params->rtp_header_extensions = + absl::optional<std::vector<webrtc::RtpExtension>>(filtered_extensions); + } + + int flexfec_payload_type = mapped_codecs.front().flexfec_payload_type; + if (flexfec_payload_type != recv_flexfec_payload_type_) { + changed_params->flexfec_payload_type = flexfec_payload_type; + } + + return true; +} + +bool WebRtcVideoChannel::SetRecvParameters(const VideoRecvParameters& params) { + RTC_DCHECK_RUN_ON(&thread_checker_); + TRACE_EVENT0("webrtc", "WebRtcVideoChannel::SetRecvParameters"); + RTC_LOG(LS_INFO) << "SetRecvParameters: " << params.ToString(); + ChangedRecvParameters changed_params; + if (!GetChangedRecvParameters(params, &changed_params)) { + return false; + } + if (changed_params.flexfec_payload_type) { + RTC_LOG(LS_INFO) << "Changing FlexFEC payload type (recv) from " + << recv_flexfec_payload_type_ << " to " + << *changed_params.flexfec_payload_type; + recv_flexfec_payload_type_ = *changed_params.flexfec_payload_type; + } + if (changed_params.rtp_header_extensions) { + recv_rtp_extensions_ = *changed_params.rtp_header_extensions; + } + if (changed_params.codec_settings) { + RTC_LOG(LS_INFO) << "Changing recv codecs from " + << CodecSettingsVectorToString(recv_codecs_) << " to " + << CodecSettingsVectorToString( + *changed_params.codec_settings); + recv_codecs_ = *changed_params.codec_settings; + } + + for (auto& kv : receive_streams_) { + kv.second->SetRecvParameters(changed_params); + } + recv_params_ = params; + return true; +} + +std::string WebRtcVideoChannel::CodecSettingsVectorToString( + const std::vector<VideoCodecSettings>& codecs) { + rtc::StringBuilder out; + out << "{"; + for (size_t i = 0; i < codecs.size(); ++i) { + out << codecs[i].codec.ToString(); + if (i != codecs.size() - 1) { + out << ", "; + } + } + out << "}"; + return out.Release(); +} + +bool WebRtcVideoChannel::GetSendCodec(VideoCodec* codec) { + RTC_DCHECK_RUN_ON(&thread_checker_); + if (!send_codec_) { + RTC_LOG(LS_VERBOSE) << "GetSendCodec: No send codec set."; + return false; + } + *codec = send_codec_->codec; + return true; +} + +bool WebRtcVideoChannel::SetSend(bool send) { + RTC_DCHECK_RUN_ON(&thread_checker_); + TRACE_EVENT0("webrtc", "WebRtcVideoChannel::SetSend"); + RTC_LOG(LS_VERBOSE) << "SetSend: " << (send ? "true" : "false"); + if (send && !send_codec_) { + RTC_DLOG(LS_ERROR) << "SetSend(true) called before setting codec."; + return false; + } + for (const auto& kv : send_streams_) { + kv.second->SetSend(send); + } + sending_ = send; + return true; +} + +bool WebRtcVideoChannel::SetVideoSend( + uint32_t ssrc, + const VideoOptions* options, + rtc::VideoSourceInterface<webrtc::VideoFrame>* source) { + RTC_DCHECK_RUN_ON(&thread_checker_); + TRACE_EVENT0("webrtc", "SetVideoSend"); + RTC_DCHECK(ssrc != 0); + RTC_LOG(LS_INFO) << "SetVideoSend (ssrc= " << ssrc << ", options: " + << (options ? options->ToString() : "nullptr") + << ", source = " << (source ? "(source)" : "nullptr") << ")"; + + const auto& kv = send_streams_.find(ssrc); + if (kv == send_streams_.end()) { + // Allow unknown ssrc only if source is null. + RTC_CHECK(source == nullptr); + RTC_LOG(LS_ERROR) << "No sending stream on ssrc " << ssrc; + return false; + } + + return kv->second->SetVideoSend(options, source); +} + +bool WebRtcVideoChannel::ValidateSendSsrcAvailability( + const StreamParams& sp) const { + for (uint32_t ssrc : sp.ssrcs) { + if (send_ssrcs_.find(ssrc) != send_ssrcs_.end()) { + RTC_LOG(LS_ERROR) << "Send stream with SSRC '" << ssrc + << "' already exists."; + return false; + } + } + return true; +} + +bool WebRtcVideoChannel::ValidateReceiveSsrcAvailability( + const StreamParams& sp) const { + for (uint32_t ssrc : sp.ssrcs) { + if (receive_ssrcs_.find(ssrc) != receive_ssrcs_.end()) { + RTC_LOG(LS_ERROR) << "Receive stream with SSRC '" << ssrc + << "' already exists."; + return false; + } + } + return true; +} + +bool WebRtcVideoChannel::AddSendStream(const StreamParams& sp) { + RTC_DCHECK_RUN_ON(&thread_checker_); + RTC_LOG(LS_INFO) << "AddSendStream: " << sp.ToString(); + if (!ValidateStreamParams(sp)) + return false; + + if (!ValidateSendSsrcAvailability(sp)) + return false; + + for (uint32_t used_ssrc : sp.ssrcs) + send_ssrcs_.insert(used_ssrc); + + webrtc::VideoSendStream::Config config(this); + + for (const RidDescription& rid : sp.rids()) { + config.rtp.rids.push_back(rid.rid); + } + + config.suspend_below_min_bitrate = video_config_.suspend_below_min_bitrate; + config.periodic_alr_bandwidth_probing = + video_config_.periodic_alr_bandwidth_probing; + config.encoder_settings.experiment_cpu_load_estimator = + video_config_.experiment_cpu_load_estimator; + config.encoder_settings.encoder_factory = encoder_factory_; + config.encoder_settings.bitrate_allocator_factory = + bitrate_allocator_factory_; + config.encoder_settings.encoder_switch_request_callback = this; + config.crypto_options = crypto_options_; + config.rtp.extmap_allow_mixed = ExtmapAllowMixed(); + config.rtcp_report_interval_ms = video_config_.rtcp_report_interval_ms; + + // If sending through Datagram Transport, limit packet size to maximum + // packet size supported by datagram_transport. + if (media_transport_config().rtp_max_packet_size) { + config.rtp.max_packet_size = + media_transport_config().rtp_max_packet_size.value(); + } + + WebRtcVideoSendStream* stream = new WebRtcVideoSendStream( + call_, sp, std::move(config), default_send_options_, + video_config_.enable_cpu_adaptation, bitrate_config_.max_bitrate_bps, + send_codec_, send_rtp_extensions_, send_params_); + + uint32_t ssrc = sp.first_ssrc(); + RTC_DCHECK(ssrc != 0); + send_streams_[ssrc] = stream; + + if (rtcp_receiver_report_ssrc_ == kDefaultRtcpReceiverReportSsrc) { + rtcp_receiver_report_ssrc_ = ssrc; + RTC_LOG(LS_INFO) + << "SetLocalSsrc on all the receive streams because we added " + "a send stream."; + for (auto& kv : receive_streams_) + kv.second->SetLocalSsrc(ssrc); + } + if (sending_) { + stream->SetSend(true); + } + + return true; +} + +bool WebRtcVideoChannel::RemoveSendStream(uint32_t ssrc) { + RTC_DCHECK_RUN_ON(&thread_checker_); + RTC_LOG(LS_INFO) << "RemoveSendStream: " << ssrc; + + WebRtcVideoSendStream* removed_stream; + std::map<uint32_t, WebRtcVideoSendStream*>::iterator it = + send_streams_.find(ssrc); + if (it == send_streams_.end()) { + return false; + } + + for (uint32_t old_ssrc : it->second->GetSsrcs()) + send_ssrcs_.erase(old_ssrc); + + removed_stream = it->second; + send_streams_.erase(it); + + // Switch receiver report SSRCs, the one in use is no longer valid. + if (rtcp_receiver_report_ssrc_ == ssrc) { + rtcp_receiver_report_ssrc_ = send_streams_.empty() + ? kDefaultRtcpReceiverReportSsrc + : send_streams_.begin()->first; + RTC_LOG(LS_INFO) << "SetLocalSsrc on all the receive streams because the " + "previous local SSRC was removed."; + + for (auto& kv : receive_streams_) { + kv.second->SetLocalSsrc(rtcp_receiver_report_ssrc_); + } + } + + delete removed_stream; + + return true; +} + +void WebRtcVideoChannel::DeleteReceiveStream( + WebRtcVideoChannel::WebRtcVideoReceiveStream* stream) { + for (uint32_t old_ssrc : stream->GetSsrcs()) + receive_ssrcs_.erase(old_ssrc); + delete stream; +} + +bool WebRtcVideoChannel::AddRecvStream(const StreamParams& sp) { + return AddRecvStream(sp, false); +} + +bool WebRtcVideoChannel::AddRecvStream(const StreamParams& sp, + bool default_stream) { + RTC_DCHECK_RUN_ON(&thread_checker_); + + RTC_LOG(LS_INFO) << "AddRecvStream" + << (default_stream ? " (default stream)" : "") << ": " + << sp.ToString(); + if (!sp.has_ssrcs()) { + // This is a StreamParam with unsignaled SSRCs. Store it, so it can be used + // later when we know the SSRC on the first packet arrival. + unsignaled_stream_params_ = sp; + return true; + } + + if (!ValidateStreamParams(sp)) + return false; + + uint32_t ssrc = sp.first_ssrc(); + + // Remove running stream if this was a default stream. + const auto& prev_stream = receive_streams_.find(ssrc); + if (prev_stream != receive_streams_.end()) { + if (default_stream || !prev_stream->second->IsDefaultStream()) { + RTC_LOG(LS_ERROR) << "Receive stream for SSRC '" << ssrc + << "' already exists."; + return false; + } + DeleteReceiveStream(prev_stream->second); + receive_streams_.erase(prev_stream); + } + + if (!ValidateReceiveSsrcAvailability(sp)) + return false; + + for (uint32_t used_ssrc : sp.ssrcs) + receive_ssrcs_.insert(used_ssrc); + + webrtc::VideoReceiveStream::Config config(this); + webrtc::FlexfecReceiveStream::Config flexfec_config(this); + ConfigureReceiverRtp(&config, &flexfec_config, sp); + + config.crypto_options = crypto_options_; + config.enable_prerenderer_smoothing = + video_config_.enable_prerenderer_smoothing; + if (!sp.stream_ids().empty()) { + config.sync_group = sp.stream_ids()[0]; + } + + if (unsignaled_frame_transformer_ && !config.frame_transformer) + config.frame_transformer = unsignaled_frame_transformer_; + + receive_streams_[ssrc] = new WebRtcVideoReceiveStream( + this, call_, sp, std::move(config), decoder_factory_, default_stream, + recv_codecs_, flexfec_config); + + return true; +} + +void WebRtcVideoChannel::ConfigureReceiverRtp( + webrtc::VideoReceiveStream::Config* config, + webrtc::FlexfecReceiveStream::Config* flexfec_config, + const StreamParams& sp) const { + uint32_t ssrc = sp.first_ssrc(); + + config->rtp.remote_ssrc = ssrc; + config->rtp.local_ssrc = rtcp_receiver_report_ssrc_; + + // TODO(pbos): This protection is against setting the same local ssrc as + // remote which is not permitted by the lower-level API. RTCP requires a + // corresponding sender SSRC. Figure out what to do when we don't have + // (receive-only) or know a good local SSRC. + if (config->rtp.remote_ssrc == config->rtp.local_ssrc) { + if (config->rtp.local_ssrc != kDefaultRtcpReceiverReportSsrc) { + config->rtp.local_ssrc = kDefaultRtcpReceiverReportSsrc; + } else { + config->rtp.local_ssrc = kDefaultRtcpReceiverReportSsrc + 1; + } + } + + // Whether or not the receive stream sends reduced size RTCP is determined + // by the send params. + // TODO(deadbeef): Once we change "send_params" to "sender_params" and + // "recv_params" to "receiver_params", we should get this out of + // receiver_params_. + config->rtp.rtcp_mode = send_params_.rtcp.reduced_size + ? webrtc::RtcpMode::kReducedSize + : webrtc::RtcpMode::kCompound; + + config->rtp.transport_cc = + send_codec_ ? HasTransportCc(send_codec_->codec) : false; + + sp.GetFidSsrc(ssrc, &config->rtp.rtx_ssrc); + + config->rtp.extensions = recv_rtp_extensions_; + + // TODO(brandtr): Generalize when we add support for multistream protection. + flexfec_config->payload_type = recv_flexfec_payload_type_; + if (IsFlexfecAdvertisedFieldTrialEnabled() && + sp.GetFecFrSsrc(ssrc, &flexfec_config->remote_ssrc)) { + flexfec_config->protected_media_ssrcs = {ssrc}; + flexfec_config->local_ssrc = config->rtp.local_ssrc; + flexfec_config->rtcp_mode = config->rtp.rtcp_mode; + // TODO(brandtr): We should be spec-compliant and set |transport_cc| here + // based on the rtcp-fb for the FlexFEC codec, not the media codec. + flexfec_config->transport_cc = config->rtp.transport_cc; + flexfec_config->rtp_header_extensions = config->rtp.extensions; + } +} + +bool WebRtcVideoChannel::RemoveRecvStream(uint32_t ssrc) { + RTC_DCHECK_RUN_ON(&thread_checker_); + RTC_LOG(LS_INFO) << "RemoveRecvStream: " << ssrc; + + std::map<uint32_t, WebRtcVideoReceiveStream*>::iterator stream = + receive_streams_.find(ssrc); + if (stream == receive_streams_.end()) { + RTC_LOG(LS_ERROR) << "Stream not found for ssrc: " << ssrc; + return false; + } + DeleteReceiveStream(stream->second); + receive_streams_.erase(stream); + + return true; +} + +void WebRtcVideoChannel::ResetUnsignaledRecvStream() { + RTC_DCHECK_RUN_ON(&thread_checker_); + RTC_LOG(LS_INFO) << "ResetUnsignaledRecvStream."; + unsignaled_stream_params_ = StreamParams(); + + // Delete any created default streams. This is needed to avoid SSRC collisions + // in Call's RtpDemuxer, in the case that |this| has created a default video + // receiver, and then some other WebRtcVideoChannel gets the SSRC signaled + // in the corresponding Unified Plan "m=" section. + auto it = receive_streams_.begin(); + while (it != receive_streams_.end()) { + if (it->second->IsDefaultStream()) { + DeleteReceiveStream(it->second); + receive_streams_.erase(it++); + } else { + ++it; + } + } +} + +bool WebRtcVideoChannel::SetSink( + uint32_t ssrc, + rtc::VideoSinkInterface<webrtc::VideoFrame>* sink) { + RTC_DCHECK_RUN_ON(&thread_checker_); + RTC_LOG(LS_INFO) << "SetSink: ssrc:" << ssrc << " " + << (sink ? "(ptr)" : "nullptr"); + + std::map<uint32_t, WebRtcVideoReceiveStream*>::iterator it = + receive_streams_.find(ssrc); + if (it == receive_streams_.end()) { + return false; + } + + it->second->SetSink(sink); + return true; +} + +void WebRtcVideoChannel::SetDefaultSink( + rtc::VideoSinkInterface<webrtc::VideoFrame>* sink) { + RTC_DCHECK_RUN_ON(&thread_checker_); + RTC_LOG(LS_INFO) << "SetDefaultSink: " << (sink ? "(ptr)" : "nullptr"); + default_unsignalled_ssrc_handler_.SetDefaultSink(this, sink); +} + +bool WebRtcVideoChannel::GetStats(VideoMediaInfo* info) { + RTC_DCHECK_RUN_ON(&thread_checker_); + TRACE_EVENT0("webrtc", "WebRtcVideoChannel::GetStats"); + + // Log stats periodically. + bool log_stats = false; + int64_t now_ms = rtc::TimeMillis(); + if (last_stats_log_ms_ == -1 || + now_ms - last_stats_log_ms_ > kStatsLogIntervalMs) { + last_stats_log_ms_ = now_ms; + log_stats = true; + } + + info->Clear(); + FillSenderStats(info, log_stats); + FillReceiverStats(info, log_stats); + FillSendAndReceiveCodecStats(info); + // TODO(holmer): We should either have rtt available as a metric on + // VideoSend/ReceiveStreams, or we should remove rtt from VideoSenderInfo. + // TODO(nisse): Arrange to get correct RTT also when using MediaTransport. + webrtc::Call::Stats stats = call_->GetStats(); + if (stats.rtt_ms != -1) { + for (size_t i = 0; i < info->senders.size(); ++i) { + info->senders[i].rtt_ms = stats.rtt_ms; + } + for (size_t i = 0; i < info->aggregated_senders.size(); ++i) { + info->aggregated_senders[i].rtt_ms = stats.rtt_ms; + } + } + + if (log_stats) + RTC_LOG(LS_INFO) << stats.ToString(now_ms); + + return true; +} + +void WebRtcVideoChannel::FillSenderStats(VideoMediaInfo* video_media_info, + bool log_stats) { + for (std::map<uint32_t, WebRtcVideoSendStream*>::iterator it = + send_streams_.begin(); + it != send_streams_.end(); ++it) { + auto infos = it->second->GetPerLayerVideoSenderInfos(log_stats); + video_media_info->aggregated_senders.push_back( + it->second->GetAggregatedVideoSenderInfo(infos)); + for (auto&& info : infos) { + video_media_info->senders.push_back(info); + } + } +} + +void WebRtcVideoChannel::FillReceiverStats(VideoMediaInfo* video_media_info, + bool log_stats) { + for (std::map<uint32_t, WebRtcVideoReceiveStream*>::iterator it = + receive_streams_.begin(); + it != receive_streams_.end(); ++it) { + video_media_info->receivers.push_back( + it->second->GetVideoReceiverInfo(log_stats)); + } +} + +void WebRtcVideoChannel::FillBitrateInfo(BandwidthEstimationInfo* bwe_info) { + RTC_DCHECK_RUN_ON(&thread_checker_); + for (std::map<uint32_t, WebRtcVideoSendStream*>::iterator stream = + send_streams_.begin(); + stream != send_streams_.end(); ++stream) { + stream->second->FillBitrateInfo(bwe_info); + } +} + +void WebRtcVideoChannel::FillSendAndReceiveCodecStats( + VideoMediaInfo* video_media_info) { + for (const VideoCodec& codec : send_params_.codecs) { + webrtc::RtpCodecParameters codec_params = codec.ToCodecParameters(); + video_media_info->send_codecs.insert( + std::make_pair(codec_params.payload_type, std::move(codec_params))); + } + for (const VideoCodec& codec : recv_params_.codecs) { + webrtc::RtpCodecParameters codec_params = codec.ToCodecParameters(); + video_media_info->receive_codecs.insert( + std::make_pair(codec_params.payload_type, std::move(codec_params))); + } +} + +void WebRtcVideoChannel::OnPacketReceived(rtc::CopyOnWriteBuffer packet, + int64_t packet_time_us) { + RTC_DCHECK_RUN_ON(&thread_checker_); + const webrtc::PacketReceiver::DeliveryStatus delivery_result = + call_->Receiver()->DeliverPacket(webrtc::MediaType::VIDEO, packet, + packet_time_us); + switch (delivery_result) { + case webrtc::PacketReceiver::DELIVERY_OK: + return; + case webrtc::PacketReceiver::DELIVERY_PACKET_ERROR: + return; + case webrtc::PacketReceiver::DELIVERY_UNKNOWN_SSRC: + break; + } + + uint32_t ssrc = 0; + if (!GetRtpSsrc(packet.cdata(), packet.size(), &ssrc)) { + return; + } + + if (unknown_ssrc_packet_buffer_) { + unknown_ssrc_packet_buffer_->AddPacket(ssrc, packet_time_us, packet); + return; + } + + if (discard_unknown_ssrc_packets_) { + return; + } + + int payload_type = 0; + if (!GetRtpPayloadType(packet.cdata(), packet.size(), &payload_type)) { + return; + } + + // See if this payload_type is registered as one that usually gets its own + // SSRC (RTX) or at least is safe to drop either way (FEC). If it is, and + // it wasn't handled above by DeliverPacket, that means we don't know what + // stream it associates with, and we shouldn't ever create an implicit channel + // for these. + for (auto& codec : recv_codecs_) { + if (payload_type == codec.rtx_payload_type || + payload_type == codec.ulpfec.red_rtx_payload_type || + payload_type == codec.ulpfec.ulpfec_payload_type) { + return; + } + } + if (payload_type == recv_flexfec_payload_type_) { + return; + } + + switch (unsignalled_ssrc_handler_->OnUnsignalledSsrc(this, ssrc)) { + case UnsignalledSsrcHandler::kDropPacket: + return; + case UnsignalledSsrcHandler::kDeliverPacket: + break; + } + + if (call_->Receiver()->DeliverPacket(webrtc::MediaType::VIDEO, packet, + packet_time_us) != + webrtc::PacketReceiver::DELIVERY_OK) { + RTC_LOG(LS_WARNING) << "Failed to deliver RTP packet on re-delivery."; + return; + } +} + +void WebRtcVideoChannel::BackfillBufferedPackets( + rtc::ArrayView<const uint32_t> ssrcs) { + RTC_DCHECK_RUN_ON(&thread_checker_); + if (!unknown_ssrc_packet_buffer_) { + return; + } + + int delivery_ok_cnt = 0; + int delivery_unknown_ssrc_cnt = 0; + int delivery_packet_error_cnt = 0; + webrtc::PacketReceiver* receiver = this->call_->Receiver(); + unknown_ssrc_packet_buffer_->BackfillPackets( + ssrcs, [&](uint32_t ssrc, int64_t packet_time_us, + rtc::CopyOnWriteBuffer packet) { + switch (receiver->DeliverPacket(webrtc::MediaType::VIDEO, packet, + packet_time_us)) { + case webrtc::PacketReceiver::DELIVERY_OK: + delivery_ok_cnt++; + break; + case webrtc::PacketReceiver::DELIVERY_UNKNOWN_SSRC: + delivery_unknown_ssrc_cnt++; + break; + case webrtc::PacketReceiver::DELIVERY_PACKET_ERROR: + delivery_packet_error_cnt++; + break; + } + }); + rtc::StringBuilder out; + out << "[ "; + for (uint32_t ssrc : ssrcs) { + out << std::to_string(ssrc) << " "; + } + out << "]"; + auto level = rtc::LS_INFO; + if (delivery_unknown_ssrc_cnt > 0 || delivery_packet_error_cnt > 0) { + level = rtc::LS_ERROR; + } + int total = + delivery_ok_cnt + delivery_unknown_ssrc_cnt + delivery_packet_error_cnt; + RTC_LOG_V(level) << "Backfilled " << total + << " packets for ssrcs: " << out.Release() + << " ok: " << delivery_ok_cnt + << " error: " << delivery_packet_error_cnt + << " unknown: " << delivery_unknown_ssrc_cnt; +} + +void WebRtcVideoChannel::OnReadyToSend(bool ready) { + RTC_DCHECK_RUN_ON(&thread_checker_); + RTC_LOG(LS_VERBOSE) << "OnReadyToSend: " << (ready ? "Ready." : "Not ready."); + call_->SignalChannelNetworkState( + webrtc::MediaType::VIDEO, + ready ? webrtc::kNetworkUp : webrtc::kNetworkDown); +} + +void WebRtcVideoChannel::OnNetworkRouteChanged( + const std::string& transport_name, + const rtc::NetworkRoute& network_route) { + RTC_DCHECK_RUN_ON(&thread_checker_); + call_->GetTransportControllerSend()->OnNetworkRouteChanged(transport_name, + network_route); + call_->GetTransportControllerSend()->OnTransportOverheadChanged( + network_route.packet_overhead); +} + +void WebRtcVideoChannel::SetInterface( + NetworkInterface* iface, + const webrtc::MediaTransportConfig& media_transport_config) { + RTC_DCHECK_RUN_ON(&thread_checker_); + MediaChannel::SetInterface(iface, media_transport_config); + // Set the RTP recv/send buffer to a bigger size. + + // The group should be a positive integer with an explicit size, in + // which case that is used as UDP recevie buffer size. All other values shall + // result in the default value being used. + const std::string group_name = + webrtc::field_trial::FindFullName("WebRTC-IncreasedReceivebuffers"); + int recv_buffer_size = kVideoRtpRecvBufferSize; + if (!group_name.empty() && + (sscanf(group_name.c_str(), "%d", &recv_buffer_size) != 1 || + recv_buffer_size <= 0)) { + RTC_LOG(LS_WARNING) << "Invalid receive buffer size: " << group_name; + recv_buffer_size = kVideoRtpRecvBufferSize; + } + + MediaChannel::SetOption(NetworkInterface::ST_RTP, rtc::Socket::OPT_RCVBUF, + recv_buffer_size); + + // Speculative change to increase the outbound socket buffer size. + // In b/15152257, we are seeing a significant number of packets discarded + // due to lack of socket buffer space, although it's not yet clear what the + // ideal value should be. + MediaChannel::SetOption(NetworkInterface::ST_RTP, rtc::Socket::OPT_SNDBUF, + kVideoRtpSendBufferSize); +} + +void WebRtcVideoChannel::SetFrameDecryptor( + uint32_t ssrc, + rtc::scoped_refptr<webrtc::FrameDecryptorInterface> frame_decryptor) { + RTC_DCHECK_RUN_ON(&thread_checker_); + auto matching_stream = receive_streams_.find(ssrc); + if (matching_stream != receive_streams_.end()) { + matching_stream->second->SetFrameDecryptor(frame_decryptor); + } +} + +void WebRtcVideoChannel::SetFrameEncryptor( + uint32_t ssrc, + rtc::scoped_refptr<webrtc::FrameEncryptorInterface> frame_encryptor) { + RTC_DCHECK_RUN_ON(&thread_checker_); + auto matching_stream = send_streams_.find(ssrc); + if (matching_stream != send_streams_.end()) { + matching_stream->second->SetFrameEncryptor(frame_encryptor); + } else { + RTC_LOG(LS_ERROR) << "No stream found to attach frame encryptor"; + } +} + +void WebRtcVideoChannel::SetVideoCodecSwitchingEnabled(bool enabled) { + invoker_.AsyncInvoke<void>(RTC_FROM_HERE, worker_thread_, [this, enabled] { + RTC_DCHECK_RUN_ON(&thread_checker_); + allow_codec_switching_ = enabled; + if (allow_codec_switching_) { + RTC_LOG(LS_INFO) << "Encoder switching enabled."; + if (requested_encoder_switch_) { + RTC_LOG(LS_INFO) << "Executing cached video encoder switch request."; + RequestEncoderSwitch(*requested_encoder_switch_); + requested_encoder_switch_.reset(); + } + } + }); +} + +bool WebRtcVideoChannel::SetBaseMinimumPlayoutDelayMs(uint32_t ssrc, + int delay_ms) { + RTC_DCHECK_RUN_ON(&thread_checker_); + absl::optional<uint32_t> default_ssrc = GetDefaultReceiveStreamSsrc(); + + // SSRC of 0 represents the default receive stream. + if (ssrc == 0) { + default_recv_base_minimum_delay_ms_ = delay_ms; + } + + if (ssrc == 0 && !default_ssrc) { + return true; + } + + if (ssrc == 0 && default_ssrc) { + ssrc = default_ssrc.value(); + } + + auto stream = receive_streams_.find(ssrc); + if (stream != receive_streams_.end()) { + stream->second->SetBaseMinimumPlayoutDelayMs(delay_ms); + return true; + } else { + RTC_LOG(LS_ERROR) << "No stream found to set base minimum playout delay"; + return false; + } +} + +absl::optional<int> WebRtcVideoChannel::GetBaseMinimumPlayoutDelayMs( + uint32_t ssrc) const { + RTC_DCHECK_RUN_ON(&thread_checker_); + // SSRC of 0 represents the default receive stream. + if (ssrc == 0) { + return default_recv_base_minimum_delay_ms_; + } + + auto stream = receive_streams_.find(ssrc); + if (stream != receive_streams_.end()) { + return stream->second->GetBaseMinimumPlayoutDelayMs(); + } else { + RTC_LOG(LS_ERROR) << "No stream found to get base minimum playout delay"; + return absl::nullopt; + } +} + +absl::optional<uint32_t> WebRtcVideoChannel::GetDefaultReceiveStreamSsrc() { + RTC_DCHECK_RUN_ON(&thread_checker_); + absl::optional<uint32_t> ssrc; + for (auto it = receive_streams_.begin(); it != receive_streams_.end(); ++it) { + if (it->second->IsDefaultStream()) { + ssrc.emplace(it->first); + break; + } + } + return ssrc; +} + +std::vector<webrtc::RtpSource> WebRtcVideoChannel::GetSources( + uint32_t ssrc) const { + RTC_DCHECK_RUN_ON(&thread_checker_); + auto it = receive_streams_.find(ssrc); + if (it == receive_streams_.end()) { + // TODO(bugs.webrtc.org/9781): Investigate standard compliance + // with sources for streams that has been removed. + RTC_LOG(LS_ERROR) << "Attempting to get contributing sources for SSRC:" + << ssrc << " which doesn't exist."; + return {}; + } + return it->second->GetSources(); +} + +bool WebRtcVideoChannel::SendRtp(const uint8_t* data, + size_t len, + const webrtc::PacketOptions& options) { + rtc::CopyOnWriteBuffer packet(data, len, kMaxRtpPacketLen); + rtc::PacketOptions rtc_options; + rtc_options.packet_id = options.packet_id; + if (DscpEnabled()) { + rtc_options.dscp = PreferredDscp(); + } + rtc_options.info_signaled_after_sent.included_in_feedback = + options.included_in_feedback; + rtc_options.info_signaled_after_sent.included_in_allocation = + options.included_in_allocation; + return MediaChannel::SendPacket(&packet, rtc_options); +} + +bool WebRtcVideoChannel::SendRtcp(const uint8_t* data, size_t len) { + rtc::CopyOnWriteBuffer packet(data, len, kMaxRtpPacketLen); + rtc::PacketOptions rtc_options; + if (DscpEnabled()) { + rtc_options.dscp = PreferredDscp(); + } + + return MediaChannel::SendRtcp(&packet, rtc_options); +} + +WebRtcVideoChannel::WebRtcVideoSendStream::VideoSendStreamParameters:: + VideoSendStreamParameters( + webrtc::VideoSendStream::Config config, + const VideoOptions& options, + int max_bitrate_bps, + const absl::optional<VideoCodecSettings>& codec_settings) + : config(std::move(config)), + options(options), + max_bitrate_bps(max_bitrate_bps), + conference_mode(false), + codec_settings(codec_settings) {} + +WebRtcVideoChannel::WebRtcVideoSendStream::WebRtcVideoSendStream( + webrtc::Call* call, + const StreamParams& sp, + webrtc::VideoSendStream::Config config, + const VideoOptions& options, + bool enable_cpu_overuse_detection, + int max_bitrate_bps, + const absl::optional<VideoCodecSettings>& codec_settings, + const absl::optional<std::vector<webrtc::RtpExtension>>& rtp_extensions, + // TODO(deadbeef): Don't duplicate information between send_params, + // rtp_extensions, options, etc. + const VideoSendParameters& send_params) + : worker_thread_(rtc::Thread::Current()), + ssrcs_(sp.ssrcs), + ssrc_groups_(sp.ssrc_groups), + call_(call), + enable_cpu_overuse_detection_(enable_cpu_overuse_detection), + source_(nullptr), + stream_(nullptr), + encoder_sink_(nullptr), + parameters_(std::move(config), options, max_bitrate_bps, codec_settings), + rtp_parameters_(CreateRtpParametersWithEncodings(sp)), + sending_(false) { + // Maximum packet size may come in RtpConfig from external transport, for + // example from QuicTransportInterface implementation, so do not exceed + // given max_packet_size. + parameters_.config.rtp.max_packet_size = + std::min<size_t>(parameters_.config.rtp.max_packet_size, kVideoMtu); + parameters_.conference_mode = send_params.conference_mode; + + sp.GetPrimarySsrcs(¶meters_.config.rtp.ssrcs); + + // ValidateStreamParams should prevent this from happening. + RTC_CHECK(!parameters_.config.rtp.ssrcs.empty()); + rtp_parameters_.encodings[0].ssrc = parameters_.config.rtp.ssrcs[0]; + + // RTX. + sp.GetFidSsrcs(parameters_.config.rtp.ssrcs, + ¶meters_.config.rtp.rtx.ssrcs); + + // FlexFEC SSRCs. + // TODO(brandtr): This code needs to be generalized when we add support for + // multistream protection. + if (IsFlexfecFieldTrialEnabled()) { + uint32_t flexfec_ssrc; + bool flexfec_enabled = false; + for (uint32_t primary_ssrc : parameters_.config.rtp.ssrcs) { + if (sp.GetFecFrSsrc(primary_ssrc, &flexfec_ssrc)) { + if (flexfec_enabled) { + RTC_LOG(LS_INFO) + << "Multiple FlexFEC streams in local SDP, but " + "our implementation only supports a single FlexFEC " + "stream. Will not enable FlexFEC for proposed " + "stream with SSRC: " + << flexfec_ssrc << "."; + continue; + } + + flexfec_enabled = true; + parameters_.config.rtp.flexfec.ssrc = flexfec_ssrc; + parameters_.config.rtp.flexfec.protected_media_ssrcs = {primary_ssrc}; + } + } + } + + parameters_.config.rtp.c_name = sp.cname; + if (rtp_extensions) { + parameters_.config.rtp.extensions = *rtp_extensions; + rtp_parameters_.header_extensions = *rtp_extensions; + } + parameters_.config.rtp.rtcp_mode = send_params.rtcp.reduced_size + ? webrtc::RtcpMode::kReducedSize + : webrtc::RtcpMode::kCompound; + parameters_.config.rtp.mid = send_params.mid; + rtp_parameters_.rtcp.reduced_size = send_params.rtcp.reduced_size; + + if (codec_settings) { + SetCodec(*codec_settings); + } +} + +WebRtcVideoChannel::WebRtcVideoSendStream::~WebRtcVideoSendStream() { + if (stream_ != NULL) { + call_->DestroyVideoSendStream(stream_); + } +} + +bool WebRtcVideoChannel::WebRtcVideoSendStream::SetVideoSend( + const VideoOptions* options, + rtc::VideoSourceInterface<webrtc::VideoFrame>* source) { + TRACE_EVENT0("webrtc", "WebRtcVideoSendStream::SetVideoSend"); + RTC_DCHECK_RUN_ON(&thread_checker_); + + if (options) { + VideoOptions old_options = parameters_.options; + parameters_.options.SetAll(*options); + if (parameters_.options.is_screencast.value_or(false) != + old_options.is_screencast.value_or(false) && + parameters_.codec_settings) { + // If screen content settings change, we may need to recreate the codec + // instance so that the correct type is used. + + SetCodec(*parameters_.codec_settings); + // Mark screenshare parameter as being updated, then test for any other + // changes that may require codec reconfiguration. + old_options.is_screencast = options->is_screencast; + } + if (parameters_.options != old_options) { + ReconfigureEncoder(); + } + } + + if (source_ && stream_) { + stream_->SetSource(nullptr, webrtc::DegradationPreference::DISABLED); + } + // Switch to the new source. + source_ = source; + if (source && stream_) { + stream_->SetSource(this, GetDegradationPreference()); + } + return true; +} + +webrtc::DegradationPreference +WebRtcVideoChannel::WebRtcVideoSendStream::GetDegradationPreference() const { + // Do not adapt resolution for screen content as this will likely + // result in blurry and unreadable text. + // |this| acts like a VideoSource to make sure SinkWants are handled on the + // correct thread. + if (!enable_cpu_overuse_detection_) { + return webrtc::DegradationPreference::DISABLED; + } + + webrtc::DegradationPreference degradation_preference; + if (rtp_parameters_.degradation_preference.has_value()) { + degradation_preference = *rtp_parameters_.degradation_preference; + } else { + if (parameters_.options.content_hint == + webrtc::VideoTrackInterface::ContentHint::kFluid) { + degradation_preference = + webrtc::DegradationPreference::MAINTAIN_FRAMERATE; + } else if (parameters_.options.is_screencast.value_or(false) || + parameters_.options.content_hint == + webrtc::VideoTrackInterface::ContentHint::kDetailed || + parameters_.options.content_hint == + webrtc::VideoTrackInterface::ContentHint::kText) { + degradation_preference = + webrtc::DegradationPreference::MAINTAIN_RESOLUTION; + } else if (webrtc::field_trial::IsEnabled( + "WebRTC-Video-BalancedDegradation")) { + // Standard wants balanced by default, but it needs to be tuned first. + degradation_preference = webrtc::DegradationPreference::BALANCED; + } else { + // Keep MAINTAIN_FRAMERATE by default until BALANCED has been tuned for + // all codecs and launched. + degradation_preference = + webrtc::DegradationPreference::MAINTAIN_FRAMERATE; + } + } + + return degradation_preference; +} + +const std::vector<uint32_t>& +WebRtcVideoChannel::WebRtcVideoSendStream::GetSsrcs() const { + return ssrcs_; +} + +void WebRtcVideoChannel::WebRtcVideoSendStream::SetCodec( + const VideoCodecSettings& codec_settings) { + RTC_DCHECK_RUN_ON(&thread_checker_); + parameters_.encoder_config = CreateVideoEncoderConfig(codec_settings.codec); + RTC_DCHECK_GT(parameters_.encoder_config.number_of_streams, 0); + + parameters_.config.rtp.payload_name = codec_settings.codec.name; + parameters_.config.rtp.payload_type = codec_settings.codec.id; + parameters_.config.rtp.raw_payload = + codec_settings.codec.packetization == kPacketizationParamRaw; + parameters_.config.rtp.ulpfec = codec_settings.ulpfec; + parameters_.config.rtp.flexfec.payload_type = + codec_settings.flexfec_payload_type; + + // Set RTX payload type if RTX is enabled. + if (!parameters_.config.rtp.rtx.ssrcs.empty()) { + if (codec_settings.rtx_payload_type == -1) { + RTC_LOG(LS_WARNING) + << "RTX SSRCs configured but there's no configured RTX " + "payload type. Ignoring."; + parameters_.config.rtp.rtx.ssrcs.clear(); + } else { + parameters_.config.rtp.rtx.payload_type = codec_settings.rtx_payload_type; + } + } + + const bool has_lntf = HasLntf(codec_settings.codec); + parameters_.config.rtp.lntf.enabled = has_lntf; + parameters_.config.encoder_settings.capabilities.loss_notification = has_lntf; + + parameters_.config.rtp.nack.rtp_history_ms = + HasNack(codec_settings.codec) ? kNackHistoryMs : 0; + + parameters_.codec_settings = codec_settings; + + // TODO(nisse): Avoid recreation, it should be enough to call + // ReconfigureEncoder. + RTC_LOG(LS_INFO) << "RecreateWebRtcStream (send) because of SetCodec."; + RecreateWebRtcStream(); +} + +void WebRtcVideoChannel::WebRtcVideoSendStream::SetSendParameters( + const ChangedSendParameters& params) { + RTC_DCHECK_RUN_ON(&thread_checker_); + // |recreate_stream| means construction-time parameters have changed and the + // sending stream needs to be reset with the new config. + bool recreate_stream = false; + if (params.rtcp_mode) { + parameters_.config.rtp.rtcp_mode = *params.rtcp_mode; + rtp_parameters_.rtcp.reduced_size = + parameters_.config.rtp.rtcp_mode == webrtc::RtcpMode::kReducedSize; + recreate_stream = true; + } + if (params.extmap_allow_mixed) { + parameters_.config.rtp.extmap_allow_mixed = *params.extmap_allow_mixed; + recreate_stream = true; + } + if (params.rtp_header_extensions) { + parameters_.config.rtp.extensions = *params.rtp_header_extensions; + rtp_parameters_.header_extensions = *params.rtp_header_extensions; + recreate_stream = true; + } + if (params.mid) { + parameters_.config.rtp.mid = *params.mid; + recreate_stream = true; + } + if (params.max_bandwidth_bps) { + parameters_.max_bitrate_bps = *params.max_bandwidth_bps; + ReconfigureEncoder(); + } + if (params.conference_mode) { + parameters_.conference_mode = *params.conference_mode; + } + + // Set codecs and options. + if (params.send_codec) { + SetCodec(*params.send_codec); + recreate_stream = false; // SetCodec has already recreated the stream. + } else if (params.conference_mode && parameters_.codec_settings) { + SetCodec(*parameters_.codec_settings); + recreate_stream = false; // SetCodec has already recreated the stream. + } + if (recreate_stream) { + RTC_LOG(LS_INFO) + << "RecreateWebRtcStream (send) because of SetSendParameters"; + RecreateWebRtcStream(); + } +} + +webrtc::RTCError WebRtcVideoChannel::WebRtcVideoSendStream::SetRtpParameters( + const webrtc::RtpParameters& new_parameters) { + RTC_DCHECK_RUN_ON(&thread_checker_); + webrtc::RTCError error = CheckRtpParametersInvalidModificationAndValues( + rtp_parameters_, new_parameters); + if (!error.ok()) { + return error; + } + + bool new_param = false; + for (size_t i = 0; i < rtp_parameters_.encodings.size(); ++i) { + if ((new_parameters.encodings[i].min_bitrate_bps != + rtp_parameters_.encodings[i].min_bitrate_bps) || + (new_parameters.encodings[i].max_bitrate_bps != + rtp_parameters_.encodings[i].max_bitrate_bps) || + (new_parameters.encodings[i].max_framerate != + rtp_parameters_.encodings[i].max_framerate) || + (new_parameters.encodings[i].scale_resolution_down_by != + rtp_parameters_.encodings[i].scale_resolution_down_by) || + (new_parameters.encodings[i].num_temporal_layers != + rtp_parameters_.encodings[i].num_temporal_layers)) { + new_param = true; + break; + } + } + + bool new_degradation_preference = false; + if (new_parameters.degradation_preference != + rtp_parameters_.degradation_preference) { + new_degradation_preference = true; + } + + // TODO(bugs.webrtc.org/8807): The bitrate priority really doesn't require an + // entire encoder reconfiguration, it just needs to update the bitrate + // allocator. + bool reconfigure_encoder = + new_param || (new_parameters.encodings[0].bitrate_priority != + rtp_parameters_.encodings[0].bitrate_priority); + + // TODO(bugs.webrtc.org/8807): The active field as well should not require + // a full encoder reconfiguration, but it needs to update both the bitrate + // allocator and the video bitrate allocator. + bool new_send_state = false; + for (size_t i = 0; i < rtp_parameters_.encodings.size(); ++i) { + bool new_active = IsLayerActive(new_parameters.encodings[i]); + bool old_active = IsLayerActive(rtp_parameters_.encodings[i]); + if (new_active != old_active) { + new_send_state = true; + } + } + rtp_parameters_ = new_parameters; + // Codecs are currently handled at the WebRtcVideoChannel level. + rtp_parameters_.codecs.clear(); + if (reconfigure_encoder || new_send_state) { + ReconfigureEncoder(); + } + if (new_send_state) { + UpdateSendState(); + } + if (new_degradation_preference) { + if (source_ && stream_) { + stream_->SetSource(this, GetDegradationPreference()); + } + } + return webrtc::RTCError::OK(); +} + +webrtc::RtpParameters +WebRtcVideoChannel::WebRtcVideoSendStream::GetRtpParameters() const { + RTC_DCHECK_RUN_ON(&thread_checker_); + return rtp_parameters_; +} + +void WebRtcVideoChannel::WebRtcVideoSendStream::SetFrameEncryptor( + rtc::scoped_refptr<webrtc::FrameEncryptorInterface> frame_encryptor) { + RTC_DCHECK_RUN_ON(&thread_checker_); + parameters_.config.frame_encryptor = frame_encryptor; + if (stream_) { + RTC_LOG(LS_INFO) + << "RecreateWebRtcStream (send) because of SetFrameEncryptor, ssrc=" + << parameters_.config.rtp.ssrcs[0]; + RecreateWebRtcStream(); + } +} + +void WebRtcVideoChannel::WebRtcVideoSendStream::UpdateSendState() { + RTC_DCHECK_RUN_ON(&thread_checker_); + if (sending_) { + RTC_DCHECK(stream_ != nullptr); + size_t num_layers = rtp_parameters_.encodings.size(); + if (parameters_.encoder_config.number_of_streams == 1) { + // SVC is used. Only one simulcast layer is present. + num_layers = 1; + } + std::vector<bool> active_layers(num_layers); + for (size_t i = 0; i < num_layers; ++i) { + active_layers[i] = IsLayerActive(rtp_parameters_.encodings[i]); + } + if (parameters_.encoder_config.number_of_streams == 1 && + rtp_parameters_.encodings.size() > 1) { + // SVC is used. + // The only present simulcast layer should be active if any of the + // configured SVC layers is active. + active_layers[0] = + absl::c_any_of(rtp_parameters_.encodings, + [](const auto& encoding) { return encoding.active; }); + } + // This updates what simulcast layers are sending, and possibly starts + // or stops the VideoSendStream. + stream_->UpdateActiveSimulcastLayers(active_layers); + } else { + if (stream_ != nullptr) { + stream_->Stop(); + } + } +} + +webrtc::VideoEncoderConfig +WebRtcVideoChannel::WebRtcVideoSendStream::CreateVideoEncoderConfig( + const VideoCodec& codec) const { + RTC_DCHECK_RUN_ON(&thread_checker_); + webrtc::VideoEncoderConfig encoder_config; + encoder_config.codec_type = webrtc::PayloadStringToCodecType(codec.name); + encoder_config.video_format = + webrtc::SdpVideoFormat(codec.name, codec.params); + + bool is_screencast = parameters_.options.is_screencast.value_or(false); + if (is_screencast) { + encoder_config.min_transmit_bitrate_bps = + 1000 * parameters_.options.screencast_min_bitrate_kbps.value_or(0); + encoder_config.content_type = + webrtc::VideoEncoderConfig::ContentType::kScreen; + } else { + encoder_config.min_transmit_bitrate_bps = 0; + encoder_config.content_type = + webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo; + } + + // By default, the stream count for the codec configuration should match the + // number of negotiated ssrcs. But if the codec is blacklisted for simulcast + // or a screencast (and not in simulcast screenshare experiment), only + // configure a single stream. + encoder_config.number_of_streams = parameters_.config.rtp.ssrcs.size(); + if (IsCodecBlacklistedForSimulcast(codec.name)) { + encoder_config.number_of_streams = 1; + } + + // parameters_.max_bitrate comes from the max bitrate set at the SDP + // (m-section) level with the attribute "b=AS." Note that we override this + // value below if the RtpParameters max bitrate set with + // RtpSender::SetParameters has a lower value. + int stream_max_bitrate = parameters_.max_bitrate_bps; + // When simulcast is enabled (when there are multiple encodings), + // encodings[i].max_bitrate_bps will be enforced by + // encoder_config.simulcast_layers[i].max_bitrate_bps. Otherwise, it's + // enforced by stream_max_bitrate, taking the minimum of the two maximums + // (one coming from SDP, the other coming from RtpParameters). + if (rtp_parameters_.encodings[0].max_bitrate_bps && + rtp_parameters_.encodings.size() == 1) { + stream_max_bitrate = + MinPositive(*(rtp_parameters_.encodings[0].max_bitrate_bps), + parameters_.max_bitrate_bps); + } + + // The codec max bitrate comes from the "x-google-max-bitrate" parameter + // attribute set in the SDP for a specific codec. As done in + // WebRtcVideoChannel::SetSendParameters, this value does not override the + // stream max_bitrate set above. + int codec_max_bitrate_kbps; + if (codec.GetParam(kCodecParamMaxBitrate, &codec_max_bitrate_kbps) && + stream_max_bitrate == -1) { + stream_max_bitrate = codec_max_bitrate_kbps * 1000; + } + encoder_config.max_bitrate_bps = stream_max_bitrate; + + // The encoder config's default bitrate priority is set to 1.0, + // unless it is set through the sender's encoding parameters. + // The bitrate priority, which is used in the bitrate allocation, is done + // on a per sender basis, so we use the first encoding's value. + encoder_config.bitrate_priority = + rtp_parameters_.encodings[0].bitrate_priority; + + // Application-controlled state is held in the encoder_config's + // simulcast_layers. Currently this is used to control which simulcast layers + // are active and for configuring the min/max bitrate and max framerate. + // The encoder_config's simulcast_layers is also used for non-simulcast (when + // there is a single layer). + RTC_DCHECK_GE(rtp_parameters_.encodings.size(), + encoder_config.number_of_streams); + RTC_DCHECK_GT(encoder_config.number_of_streams, 0); + + // Copy all provided constraints. + encoder_config.simulcast_layers.resize(rtp_parameters_.encodings.size()); + for (size_t i = 0; i < encoder_config.simulcast_layers.size(); ++i) { + encoder_config.simulcast_layers[i].active = + rtp_parameters_.encodings[i].active; + if (rtp_parameters_.encodings[i].min_bitrate_bps) { + encoder_config.simulcast_layers[i].min_bitrate_bps = + *rtp_parameters_.encodings[i].min_bitrate_bps; + } + if (rtp_parameters_.encodings[i].max_bitrate_bps) { + encoder_config.simulcast_layers[i].max_bitrate_bps = + *rtp_parameters_.encodings[i].max_bitrate_bps; + } + if (rtp_parameters_.encodings[i].max_framerate) { + encoder_config.simulcast_layers[i].max_framerate = + *rtp_parameters_.encodings[i].max_framerate; + } + if (rtp_parameters_.encodings[i].scale_resolution_down_by) { + encoder_config.simulcast_layers[i].scale_resolution_down_by = + *rtp_parameters_.encodings[i].scale_resolution_down_by; + } + if (rtp_parameters_.encodings[i].num_temporal_layers) { + encoder_config.simulcast_layers[i].num_temporal_layers = + *rtp_parameters_.encodings[i].num_temporal_layers; + } + } + + int max_qp = kDefaultQpMax; + codec.GetParam(kCodecParamMaxQuantization, &max_qp); + encoder_config.video_stream_factory = + new rtc::RefCountedObject<EncoderStreamFactory>( + codec.name, max_qp, is_screencast, parameters_.conference_mode); + return encoder_config; +} + +void WebRtcVideoChannel::WebRtcVideoSendStream::ReconfigureEncoder() { + RTC_DCHECK_RUN_ON(&thread_checker_); + if (!stream_) { + // The webrtc::VideoSendStream |stream_| has not yet been created but other + // parameters has changed. + return; + } + + RTC_DCHECK_GT(parameters_.encoder_config.number_of_streams, 0); + + RTC_CHECK(parameters_.codec_settings); + VideoCodecSettings codec_settings = *parameters_.codec_settings; + + webrtc::VideoEncoderConfig encoder_config = + CreateVideoEncoderConfig(codec_settings.codec); + + encoder_config.encoder_specific_settings = + ConfigureVideoEncoderSettings(codec_settings.codec); + + stream_->ReconfigureVideoEncoder(encoder_config.Copy()); + + encoder_config.encoder_specific_settings = NULL; + + parameters_.encoder_config = std::move(encoder_config); +} + +void WebRtcVideoChannel::WebRtcVideoSendStream::SetSend(bool send) { + RTC_DCHECK_RUN_ON(&thread_checker_); + sending_ = send; + UpdateSendState(); +} + +void WebRtcVideoChannel::WebRtcVideoSendStream::RemoveSink( + rtc::VideoSinkInterface<webrtc::VideoFrame>* sink) { + RTC_DCHECK_RUN_ON(&thread_checker_); + RTC_DCHECK(encoder_sink_ == sink); + encoder_sink_ = nullptr; + source_->RemoveSink(sink); +} + +void WebRtcVideoChannel::WebRtcVideoSendStream::AddOrUpdateSink( + rtc::VideoSinkInterface<webrtc::VideoFrame>* sink, + const rtc::VideoSinkWants& wants) { + if (worker_thread_ == rtc::Thread::Current()) { + // AddOrUpdateSink is called on |worker_thread_| if this is the first + // registration of |sink|. + RTC_DCHECK_RUN_ON(&thread_checker_); + encoder_sink_ = sink; + source_->AddOrUpdateSink(encoder_sink_, wants); + } else { + // Subsequent calls to AddOrUpdateSink will happen on the encoder task + // queue. + invoker_.AsyncInvoke<void>( + RTC_FROM_HERE, worker_thread_, [this, sink, wants] { + RTC_DCHECK_RUN_ON(&thread_checker_); + // |sink| may be invalidated after this task was posted since + // RemoveSink is called on the worker thread. + bool encoder_sink_valid = (sink == encoder_sink_); + if (source_ && encoder_sink_valid) { + source_->AddOrUpdateSink(encoder_sink_, wants); + } + }); + } +} +std::vector<VideoSenderInfo> +WebRtcVideoChannel::WebRtcVideoSendStream::GetPerLayerVideoSenderInfos( + bool log_stats) { + RTC_DCHECK_RUN_ON(&thread_checker_); + VideoSenderInfo common_info; + if (parameters_.codec_settings) { + common_info.codec_name = parameters_.codec_settings->codec.name; + common_info.codec_payload_type = parameters_.codec_settings->codec.id; + } + std::vector<VideoSenderInfo> infos; + webrtc::VideoSendStream::Stats stats; + if (stream_ == nullptr) { + for (uint32_t ssrc : parameters_.config.rtp.ssrcs) { + common_info.add_ssrc(ssrc); + } + infos.push_back(common_info); + return infos; + } else { + stats = stream_->GetStats(); + if (log_stats) + RTC_LOG(LS_INFO) << stats.ToString(rtc::TimeMillis()); + + // Metrics that are in common for all substreams. + common_info.adapt_changes = stats.number_of_cpu_adapt_changes; + common_info.adapt_reason = + stats.cpu_limited_resolution ? ADAPTREASON_CPU : ADAPTREASON_NONE; + common_info.has_entered_low_resolution = stats.has_entered_low_resolution; + + // Get bandwidth limitation info from stream_->GetStats(). + // Input resolution (output from video_adapter) can be further scaled down + // or higher video layer(s) can be dropped due to bitrate constraints. + // Note, adapt_changes only include changes from the video_adapter. + if (stats.bw_limited_resolution) + common_info.adapt_reason |= ADAPTREASON_BANDWIDTH; + + common_info.quality_limitation_reason = stats.quality_limitation_reason; + common_info.quality_limitation_durations_ms = + stats.quality_limitation_durations_ms; + common_info.quality_limitation_resolution_changes = + stats.quality_limitation_resolution_changes; + common_info.encoder_implementation_name = stats.encoder_implementation_name; + common_info.ssrc_groups = ssrc_groups_; + common_info.framerate_input = stats.input_frame_rate; + common_info.avg_encode_ms = stats.avg_encode_time_ms; + common_info.encode_usage_percent = stats.encode_usage_percent; + common_info.nominal_bitrate = stats.media_bitrate_bps; + common_info.content_type = stats.content_type; + common_info.aggregated_framerate_sent = stats.encode_frame_rate; + common_info.aggregated_huge_frames_sent = stats.huge_frames_sent; + + // If we don't have any substreams, get the remaining metrics from |stats|. + // Otherwise, these values are obtained from |sub_stream| below. + if (stats.substreams.empty()) { + for (uint32_t ssrc : parameters_.config.rtp.ssrcs) { + common_info.add_ssrc(ssrc); + } + common_info.framerate_sent = stats.encode_frame_rate; + common_info.frames_encoded = stats.frames_encoded; + common_info.total_encode_time_ms = stats.total_encode_time_ms; + common_info.total_encoded_bytes_target = stats.total_encoded_bytes_target; + common_info.frames_sent = stats.frames_encoded; + common_info.huge_frames_sent = stats.huge_frames_sent; + infos.push_back(common_info); + return infos; + } + } + auto outbound_rtp_substreams = + MergeInfoAboutOutboundRtpSubstreams(stats.substreams); + for (const auto& pair : outbound_rtp_substreams) { + auto info = common_info; + info.add_ssrc(pair.first); + info.rid = parameters_.config.rtp.GetRidForSsrc(pair.first); + auto stream_stats = pair.second; + RTC_DCHECK_EQ(stream_stats.type, + webrtc::VideoSendStream::StreamStats::StreamType::kMedia); + info.payload_bytes_sent = stream_stats.rtp_stats.transmitted.payload_bytes; + info.header_and_padding_bytes_sent = + stream_stats.rtp_stats.transmitted.header_bytes + + stream_stats.rtp_stats.transmitted.padding_bytes; + info.packets_sent = stream_stats.rtp_stats.transmitted.packets; + info.total_packet_send_delay_ms += stream_stats.total_packet_send_delay_ms; + info.send_frame_width = stream_stats.width; + info.send_frame_height = stream_stats.height; + info.key_frames_encoded = stream_stats.frame_counts.key_frames; + info.framerate_sent = stream_stats.encode_frame_rate; + info.frames_encoded = stream_stats.frames_encoded; + info.frames_sent = stream_stats.frames_encoded; + info.retransmitted_bytes_sent = + stream_stats.rtp_stats.retransmitted.payload_bytes; + info.retransmitted_packets_sent = + stream_stats.rtp_stats.retransmitted.packets; + info.packets_lost = stream_stats.rtcp_stats.packets_lost; + info.firs_rcvd = stream_stats.rtcp_packet_type_counts.fir_packets; + info.nacks_rcvd = stream_stats.rtcp_packet_type_counts.nack_packets; + info.plis_rcvd = stream_stats.rtcp_packet_type_counts.pli_packets; + if (stream_stats.report_block_data.has_value()) { + info.report_block_datas.push_back(stream_stats.report_block_data.value()); + } + info.fraction_lost = + static_cast<float>(stream_stats.rtcp_stats.fraction_lost) / (1 << 8); + info.qp_sum = stream_stats.qp_sum; + info.total_encode_time_ms = stream_stats.total_encode_time_ms; + info.total_encoded_bytes_target = stream_stats.total_encoded_bytes_target; + info.huge_frames_sent = stream_stats.huge_frames_sent; + infos.push_back(info); + } + return infos; +} + +VideoSenderInfo +WebRtcVideoChannel::WebRtcVideoSendStream::GetAggregatedVideoSenderInfo( + const std::vector<VideoSenderInfo>& infos) const { + RTC_DCHECK_RUN_ON(&thread_checker_); + RTC_DCHECK(!infos.empty()); + if (infos.size() == 1) { + return infos[0]; + } + VideoSenderInfo info = infos[0]; + info.local_stats.clear(); + for (uint32_t ssrc : parameters_.config.rtp.ssrcs) { + info.add_ssrc(ssrc); + } + info.framerate_sent = info.aggregated_framerate_sent; + info.huge_frames_sent = info.aggregated_huge_frames_sent; + + for (size_t i = 1; i < infos.size(); i++) { + info.key_frames_encoded += infos[i].key_frames_encoded; + info.payload_bytes_sent += infos[i].payload_bytes_sent; + info.header_and_padding_bytes_sent += + infos[i].header_and_padding_bytes_sent; + info.packets_sent += infos[i].packets_sent; + info.total_packet_send_delay_ms += infos[i].total_packet_send_delay_ms; + info.retransmitted_bytes_sent += infos[i].retransmitted_bytes_sent; + info.retransmitted_packets_sent += infos[i].retransmitted_packets_sent; + info.packets_lost += infos[i].packets_lost; + if (infos[i].send_frame_width > info.send_frame_width) + info.send_frame_width = infos[i].send_frame_width; + if (infos[i].send_frame_height > info.send_frame_height) + info.send_frame_height = infos[i].send_frame_height; + info.firs_rcvd += infos[i].firs_rcvd; + info.nacks_rcvd += infos[i].nacks_rcvd; + info.plis_rcvd += infos[i].plis_rcvd; + if (infos[i].report_block_datas.size()) + info.report_block_datas.push_back(infos[i].report_block_datas[0]); + if (infos[i].qp_sum) { + if (!info.qp_sum) { + info.qp_sum = 0; + } + info.qp_sum = *info.qp_sum + *infos[i].qp_sum; + } + info.frames_encoded += infos[i].frames_encoded; + info.frames_sent += infos[i].frames_sent; + info.total_encode_time_ms += infos[i].total_encode_time_ms; + info.total_encoded_bytes_target += infos[i].total_encoded_bytes_target; + } + return info; +} + +void WebRtcVideoChannel::WebRtcVideoSendStream::FillBitrateInfo( + BandwidthEstimationInfo* bwe_info) { + RTC_DCHECK_RUN_ON(&thread_checker_); + if (stream_ == NULL) { + return; + } + webrtc::VideoSendStream::Stats stats = stream_->GetStats(); + for (std::map<uint32_t, webrtc::VideoSendStream::StreamStats>::iterator it = + stats.substreams.begin(); + it != stats.substreams.end(); ++it) { + bwe_info->transmit_bitrate += it->second.total_bitrate_bps; + bwe_info->retransmit_bitrate += it->second.retransmit_bitrate_bps; + } + bwe_info->target_enc_bitrate += stats.target_media_bitrate_bps; + bwe_info->actual_enc_bitrate += stats.media_bitrate_bps; +} + +void WebRtcVideoChannel::WebRtcVideoSendStream:: + SetEncoderToPacketizerFrameTransformer( + rtc::scoped_refptr<webrtc::FrameTransformerInterface> + frame_transformer) { + RTC_DCHECK_RUN_ON(&thread_checker_); + parameters_.config.frame_transformer = std::move(frame_transformer); + if (stream_) + RecreateWebRtcStream(); +} + +void WebRtcVideoChannel::WebRtcVideoSendStream::RecreateWebRtcStream() { + RTC_DCHECK_RUN_ON(&thread_checker_); + if (stream_ != NULL) { + call_->DestroyVideoSendStream(stream_); + } + + RTC_CHECK(parameters_.codec_settings); + RTC_DCHECK_EQ((parameters_.encoder_config.content_type == + webrtc::VideoEncoderConfig::ContentType::kScreen), + parameters_.options.is_screencast.value_or(false)) + << "encoder content type inconsistent with screencast option"; + parameters_.encoder_config.encoder_specific_settings = + ConfigureVideoEncoderSettings(parameters_.codec_settings->codec); + + webrtc::VideoSendStream::Config config = parameters_.config.Copy(); + if (!config.rtp.rtx.ssrcs.empty() && config.rtp.rtx.payload_type == -1) { + RTC_LOG(LS_WARNING) << "RTX SSRCs configured but there's no configured RTX " + "payload type the set codec. Ignoring RTX."; + config.rtp.rtx.ssrcs.clear(); + } + if (parameters_.encoder_config.number_of_streams == 1) { + // SVC is used instead of simulcast. Remove unnecessary SSRCs. + if (config.rtp.ssrcs.size() > 1) { + config.rtp.ssrcs.resize(1); + if (config.rtp.rtx.ssrcs.size() > 1) { + config.rtp.rtx.ssrcs.resize(1); + } + } + } + stream_ = call_->CreateVideoSendStream(std::move(config), + parameters_.encoder_config.Copy()); + + parameters_.encoder_config.encoder_specific_settings = NULL; + + if (source_) { + stream_->SetSource(this, GetDegradationPreference()); + } + + // Call stream_->Start() if necessary conditions are met. + UpdateSendState(); +} + +WebRtcVideoChannel::WebRtcVideoReceiveStream::WebRtcVideoReceiveStream( + WebRtcVideoChannel* channel, + webrtc::Call* call, + const StreamParams& sp, + webrtc::VideoReceiveStream::Config config, + webrtc::VideoDecoderFactory* decoder_factory, + bool default_stream, + const std::vector<VideoCodecSettings>& recv_codecs, + const webrtc::FlexfecReceiveStream::Config& flexfec_config) + : channel_(channel), + call_(call), + stream_params_(sp), + stream_(NULL), + default_stream_(default_stream), + config_(std::move(config)), + flexfec_config_(flexfec_config), + flexfec_stream_(nullptr), + decoder_factory_(decoder_factory), + sink_(NULL), + first_frame_timestamp_(-1), + estimated_remote_start_ntp_time_ms_(0) { + config_.renderer = this; + ConfigureCodecs(recv_codecs); + ConfigureFlexfecCodec(flexfec_config.payload_type); + MaybeRecreateWebRtcFlexfecStream(); + RecreateWebRtcVideoStream(); +} + +WebRtcVideoChannel::WebRtcVideoReceiveStream::~WebRtcVideoReceiveStream() { + if (flexfec_stream_) { + MaybeDissociateFlexfecFromVideo(); + call_->DestroyFlexfecReceiveStream(flexfec_stream_); + } + call_->DestroyVideoReceiveStream(stream_); +} + +const std::vector<uint32_t>& +WebRtcVideoChannel::WebRtcVideoReceiveStream::GetSsrcs() const { + return stream_params_.ssrcs; +} + +std::vector<webrtc::RtpSource> +WebRtcVideoChannel::WebRtcVideoReceiveStream::GetSources() { + RTC_DCHECK(stream_); + return stream_->GetSources(); +} + +webrtc::RtpParameters +WebRtcVideoChannel::WebRtcVideoReceiveStream::GetRtpParameters() const { + webrtc::RtpParameters rtp_parameters; + + std::vector<uint32_t> primary_ssrcs; + stream_params_.GetPrimarySsrcs(&primary_ssrcs); + for (uint32_t ssrc : primary_ssrcs) { + rtp_parameters.encodings.emplace_back(); + rtp_parameters.encodings.back().ssrc = ssrc; + } + + rtp_parameters.header_extensions = config_.rtp.extensions; + rtp_parameters.rtcp.reduced_size = + config_.rtp.rtcp_mode == webrtc::RtcpMode::kReducedSize; + + return rtp_parameters; +} + +void WebRtcVideoChannel::WebRtcVideoReceiveStream::ConfigureCodecs( + const std::vector<VideoCodecSettings>& recv_codecs) { + RTC_DCHECK(!recv_codecs.empty()); + config_.decoders.clear(); + config_.rtp.rtx_associated_payload_types.clear(); + config_.rtp.raw_payload_types.clear(); + for (const auto& recv_codec : recv_codecs) { + webrtc::SdpVideoFormat video_format(recv_codec.codec.name, + recv_codec.codec.params); + + webrtc::VideoReceiveStream::Decoder decoder; + decoder.decoder_factory = decoder_factory_; + decoder.video_format = video_format; + decoder.payload_type = recv_codec.codec.id; + decoder.video_format = + webrtc::SdpVideoFormat(recv_codec.codec.name, recv_codec.codec.params); + config_.decoders.push_back(decoder); + config_.rtp.rtx_associated_payload_types[recv_codec.rtx_payload_type] = + recv_codec.codec.id; + if (recv_codec.codec.packetization == kPacketizationParamRaw) { + config_.rtp.raw_payload_types.insert(recv_codec.codec.id); + } + } + + const auto& codec = recv_codecs.front(); + config_.rtp.ulpfec_payload_type = codec.ulpfec.ulpfec_payload_type; + config_.rtp.red_payload_type = codec.ulpfec.red_payload_type; + + config_.rtp.lntf.enabled = HasLntf(codec.codec); + config_.rtp.nack.rtp_history_ms = HasNack(codec.codec) ? kNackHistoryMs : 0; + config_.rtp.rtcp_xr.receiver_reference_time_report = HasRrtr(codec.codec); + if (codec.ulpfec.red_rtx_payload_type != -1) { + config_.rtp + .rtx_associated_payload_types[codec.ulpfec.red_rtx_payload_type] = + codec.ulpfec.red_payload_type; + } +} + +void WebRtcVideoChannel::WebRtcVideoReceiveStream::ConfigureFlexfecCodec( + int flexfec_payload_type) { + flexfec_config_.payload_type = flexfec_payload_type; +} + +void WebRtcVideoChannel::WebRtcVideoReceiveStream::SetLocalSsrc( + uint32_t local_ssrc) { + // TODO(pbos): Consider turning this sanity check into a RTC_DCHECK. You + // should not be able to create a sender with the same SSRC as a receiver, but + // right now this can't be done due to unittests depending on receiving what + // they are sending from the same MediaChannel. + if (local_ssrc == config_.rtp.local_ssrc) { + RTC_DLOG(LS_INFO) << "Ignoring call to SetLocalSsrc because parameters are " + "unchanged; local_ssrc=" + << local_ssrc; + return; + } + + config_.rtp.local_ssrc = local_ssrc; + flexfec_config_.local_ssrc = local_ssrc; + RTC_LOG(LS_INFO) + << "RecreateWebRtcStream (recv) because of SetLocalSsrc; local_ssrc=" + << local_ssrc; + MaybeRecreateWebRtcFlexfecStream(); + RecreateWebRtcVideoStream(); +} + +void WebRtcVideoChannel::WebRtcVideoReceiveStream::SetFeedbackParameters( + bool lntf_enabled, + bool nack_enabled, + bool transport_cc_enabled, + webrtc::RtcpMode rtcp_mode) { + int nack_history_ms = nack_enabled ? kNackHistoryMs : 0; + if (config_.rtp.lntf.enabled == lntf_enabled && + config_.rtp.nack.rtp_history_ms == nack_history_ms && + config_.rtp.transport_cc == transport_cc_enabled && + config_.rtp.rtcp_mode == rtcp_mode) { + RTC_LOG(LS_INFO) + << "Ignoring call to SetFeedbackParameters because parameters are " + "unchanged; lntf=" + << lntf_enabled << ", nack=" << nack_enabled + << ", transport_cc=" << transport_cc_enabled; + return; + } + config_.rtp.lntf.enabled = lntf_enabled; + config_.rtp.nack.rtp_history_ms = nack_history_ms; + config_.rtp.transport_cc = transport_cc_enabled; + config_.rtp.rtcp_mode = rtcp_mode; + // TODO(brandtr): We should be spec-compliant and set |transport_cc| here + // based on the rtcp-fb for the FlexFEC codec, not the media codec. + flexfec_config_.transport_cc = config_.rtp.transport_cc; + flexfec_config_.rtcp_mode = config_.rtp.rtcp_mode; + RTC_LOG(LS_INFO) + << "RecreateWebRtcStream (recv) because of SetFeedbackParameters; nack=" + << nack_enabled << ", transport_cc=" << transport_cc_enabled; + MaybeRecreateWebRtcFlexfecStream(); + RecreateWebRtcVideoStream(); +} + +void WebRtcVideoChannel::WebRtcVideoReceiveStream::SetRecvParameters( + const ChangedRecvParameters& params) { + bool video_needs_recreation = false; + bool flexfec_needs_recreation = false; + if (params.codec_settings) { + ConfigureCodecs(*params.codec_settings); + video_needs_recreation = true; + } + if (params.rtp_header_extensions) { + config_.rtp.extensions = *params.rtp_header_extensions; + flexfec_config_.rtp_header_extensions = *params.rtp_header_extensions; + video_needs_recreation = true; + flexfec_needs_recreation = true; + } + if (params.flexfec_payload_type) { + ConfigureFlexfecCodec(*params.flexfec_payload_type); + flexfec_needs_recreation = true; + } + if (flexfec_needs_recreation) { + RTC_LOG(LS_INFO) << "MaybeRecreateWebRtcFlexfecStream (recv) because of " + "SetRecvParameters"; + MaybeRecreateWebRtcFlexfecStream(); + } + if (video_needs_recreation) { + RTC_LOG(LS_INFO) + << "RecreateWebRtcVideoStream (recv) because of SetRecvParameters"; + RecreateWebRtcVideoStream(); + } +} + +void WebRtcVideoChannel::WebRtcVideoReceiveStream::RecreateWebRtcVideoStream() { + absl::optional<int> base_minimum_playout_delay_ms; + absl::optional<webrtc::VideoReceiveStream::RecordingState> recording_state; + if (stream_) { + base_minimum_playout_delay_ms = stream_->GetBaseMinimumPlayoutDelayMs(); + recording_state = stream_->SetAndGetRecordingState( + webrtc::VideoReceiveStream::RecordingState(), + /*generate_key_frame=*/false); + MaybeDissociateFlexfecFromVideo(); + call_->DestroyVideoReceiveStream(stream_); + stream_ = nullptr; + } + webrtc::VideoReceiveStream::Config config = config_.Copy(); + config.rtp.protected_by_flexfec = (flexfec_stream_ != nullptr); + config.stream_id = stream_params_.id; + stream_ = call_->CreateVideoReceiveStream(std::move(config)); + if (base_minimum_playout_delay_ms) { + stream_->SetBaseMinimumPlayoutDelayMs( + base_minimum_playout_delay_ms.value()); + } + if (recording_state) { + stream_->SetAndGetRecordingState(std::move(*recording_state), + /*generate_key_frame=*/false); + } + MaybeAssociateFlexfecWithVideo(); + stream_->Start(); + + if (webrtc::field_trial::IsEnabled( + "WebRTC-Video-BufferPacketsWithUnknownSsrc")) { + channel_->BackfillBufferedPackets(stream_params_.ssrcs); + } +} + +void WebRtcVideoChannel::WebRtcVideoReceiveStream:: + MaybeRecreateWebRtcFlexfecStream() { + if (flexfec_stream_) { + MaybeDissociateFlexfecFromVideo(); + call_->DestroyFlexfecReceiveStream(flexfec_stream_); + flexfec_stream_ = nullptr; + } + if (flexfec_config_.IsCompleteAndEnabled()) { + flexfec_stream_ = call_->CreateFlexfecReceiveStream(flexfec_config_); + MaybeAssociateFlexfecWithVideo(); + } +} + +void WebRtcVideoChannel::WebRtcVideoReceiveStream:: + MaybeAssociateFlexfecWithVideo() { + if (stream_ && flexfec_stream_) { + stream_->AddSecondarySink(flexfec_stream_); + } +} + +void WebRtcVideoChannel::WebRtcVideoReceiveStream:: + MaybeDissociateFlexfecFromVideo() { + if (stream_ && flexfec_stream_) { + stream_->RemoveSecondarySink(flexfec_stream_); + } +} + +void WebRtcVideoChannel::WebRtcVideoReceiveStream::OnFrame( + const webrtc::VideoFrame& frame) { + rtc::CritScope crit(&sink_lock_); + + int64_t time_now_ms = rtc::TimeMillis(); + if (first_frame_timestamp_ < 0) + first_frame_timestamp_ = time_now_ms; + int64_t elapsed_time_ms = time_now_ms - first_frame_timestamp_; + if (frame.ntp_time_ms() > 0) + estimated_remote_start_ntp_time_ms_ = frame.ntp_time_ms() - elapsed_time_ms; + + if (sink_ == NULL) { + RTC_LOG(LS_WARNING) << "VideoReceiveStream not connected to a VideoSink."; + return; + } + + sink_->OnFrame(frame); +} + +bool WebRtcVideoChannel::WebRtcVideoReceiveStream::IsDefaultStream() const { + return default_stream_; +} + +void WebRtcVideoChannel::WebRtcVideoReceiveStream::SetFrameDecryptor( + rtc::scoped_refptr<webrtc::FrameDecryptorInterface> frame_decryptor) { + config_.frame_decryptor = frame_decryptor; + if (stream_) { + RTC_LOG(LS_INFO) + << "Setting FrameDecryptor (recv) because of SetFrameDecryptor, " + "remote_ssrc=" + << config_.rtp.remote_ssrc; + stream_->SetFrameDecryptor(frame_decryptor); + } +} + +bool WebRtcVideoChannel::WebRtcVideoReceiveStream::SetBaseMinimumPlayoutDelayMs( + int delay_ms) { + return stream_ ? stream_->SetBaseMinimumPlayoutDelayMs(delay_ms) : false; +} + +int WebRtcVideoChannel::WebRtcVideoReceiveStream::GetBaseMinimumPlayoutDelayMs() + const { + return stream_ ? stream_->GetBaseMinimumPlayoutDelayMs() : 0; +} + +void WebRtcVideoChannel::WebRtcVideoReceiveStream::SetSink( + rtc::VideoSinkInterface<webrtc::VideoFrame>* sink) { + rtc::CritScope crit(&sink_lock_); + sink_ = sink; +} + +std::string +WebRtcVideoChannel::WebRtcVideoReceiveStream::GetCodecNameFromPayloadType( + int payload_type) { + for (const webrtc::VideoReceiveStream::Decoder& decoder : config_.decoders) { + if (decoder.payload_type == payload_type) { + return decoder.video_format.name; + } + } + return ""; +} + +VideoReceiverInfo +WebRtcVideoChannel::WebRtcVideoReceiveStream::GetVideoReceiverInfo( + bool log_stats) { + VideoReceiverInfo info; + info.ssrc_groups = stream_params_.ssrc_groups; + info.add_ssrc(config_.rtp.remote_ssrc); + webrtc::VideoReceiveStream::Stats stats = stream_->GetStats(); + info.decoder_implementation_name = stats.decoder_implementation_name; + if (stats.current_payload_type != -1) { + info.codec_payload_type = stats.current_payload_type; + } + info.payload_bytes_rcvd = stats.rtp_stats.packet_counter.payload_bytes; + info.header_and_padding_bytes_rcvd = + stats.rtp_stats.packet_counter.header_bytes + + stats.rtp_stats.packet_counter.padding_bytes; + info.packets_rcvd = stats.rtp_stats.packet_counter.packets; + info.packets_lost = stats.rtp_stats.packets_lost; + + info.framerate_rcvd = stats.network_frame_rate; + info.framerate_decoded = stats.decode_frame_rate; + info.framerate_output = stats.render_frame_rate; + info.frame_width = stats.width; + info.frame_height = stats.height; + + { + rtc::CritScope frame_cs(&sink_lock_); + info.capture_start_ntp_time_ms = estimated_remote_start_ntp_time_ms_; + } + + info.decode_ms = stats.decode_ms; + info.max_decode_ms = stats.max_decode_ms; + info.current_delay_ms = stats.current_delay_ms; + info.target_delay_ms = stats.target_delay_ms; + info.jitter_buffer_ms = stats.jitter_buffer_ms; + info.jitter_buffer_delay_seconds = stats.jitter_buffer_delay_seconds; + info.jitter_buffer_emitted_count = stats.jitter_buffer_emitted_count; + info.min_playout_delay_ms = stats.min_playout_delay_ms; + info.render_delay_ms = stats.render_delay_ms; + info.frames_received = + stats.frame_counts.key_frames + stats.frame_counts.delta_frames; + info.frames_dropped = stats.frames_dropped; + info.frames_decoded = stats.frames_decoded; + info.key_frames_decoded = stats.frame_counts.key_frames; + info.frames_rendered = stats.frames_rendered; + info.qp_sum = stats.qp_sum; + info.total_decode_time_ms = stats.total_decode_time_ms; + info.last_packet_received_timestamp_ms = + stats.rtp_stats.last_packet_received_timestamp_ms; + info.estimated_playout_ntp_timestamp_ms = + stats.estimated_playout_ntp_timestamp_ms; + info.first_frame_received_to_decoded_ms = + stats.first_frame_received_to_decoded_ms; + info.total_inter_frame_delay = stats.total_inter_frame_delay; + info.total_squared_inter_frame_delay = stats.total_squared_inter_frame_delay; + info.interframe_delay_max_ms = stats.interframe_delay_max_ms; + info.freeze_count = stats.freeze_count; + info.pause_count = stats.pause_count; + info.total_freezes_duration_ms = stats.total_freezes_duration_ms; + info.total_pauses_duration_ms = stats.total_pauses_duration_ms; + info.total_frames_duration_ms = stats.total_frames_duration_ms; + info.sum_squared_frame_durations = stats.sum_squared_frame_durations; + + info.content_type = stats.content_type; + + info.codec_name = GetCodecNameFromPayloadType(stats.current_payload_type); + + info.firs_sent = stats.rtcp_packet_type_counts.fir_packets; + info.plis_sent = stats.rtcp_packet_type_counts.pli_packets; + info.nacks_sent = stats.rtcp_packet_type_counts.nack_packets; + // TODO(bugs.webrtc.org/10662): Add stats for LNTF. + + info.timing_frame_info = stats.timing_frame_info; + + if (log_stats) + RTC_LOG(LS_INFO) << stats.ToString(rtc::TimeMillis()); + + return info; +} + +void WebRtcVideoChannel::WebRtcVideoReceiveStream:: + SetRecordableEncodedFrameCallback( + std::function<void(const webrtc::RecordableEncodedFrame&)> callback) { + if (stream_) { + stream_->SetAndGetRecordingState( + webrtc::VideoReceiveStream::RecordingState(std::move(callback)), + /*generate_key_frame=*/true); + } else { + RTC_LOG(LS_ERROR) << "Absent receive stream; ignoring setting encoded " + "frame sink"; + } +} + +void WebRtcVideoChannel::WebRtcVideoReceiveStream:: + ClearRecordableEncodedFrameCallback() { + if (stream_) { + stream_->SetAndGetRecordingState( + webrtc::VideoReceiveStream::RecordingState(), + /*generate_key_frame=*/false); + } else { + RTC_LOG(LS_ERROR) << "Absent receive stream; ignoring clearing encoded " + "frame sink"; + } +} + +void WebRtcVideoChannel::WebRtcVideoReceiveStream::GenerateKeyFrame() { + if (stream_) { + stream_->GenerateKeyFrame(); + } else { + RTC_LOG(LS_ERROR) + << "Absent receive stream; ignoring key frame generation request."; + } +} + +void WebRtcVideoChannel::WebRtcVideoReceiveStream:: + SetDepacketizerToDecoderFrameTransformer( + rtc::scoped_refptr<webrtc::FrameTransformerInterface> + frame_transformer) { + config_.frame_transformer = frame_transformer; + if (stream_) + stream_->SetDepacketizerToDecoderFrameTransformer(frame_transformer); +} + +WebRtcVideoChannel::VideoCodecSettings::VideoCodecSettings() + : flexfec_payload_type(-1), rtx_payload_type(-1) {} + +bool WebRtcVideoChannel::VideoCodecSettings::operator==( + const WebRtcVideoChannel::VideoCodecSettings& other) const { + return codec == other.codec && ulpfec == other.ulpfec && + flexfec_payload_type == other.flexfec_payload_type && + rtx_payload_type == other.rtx_payload_type; +} + +bool WebRtcVideoChannel::VideoCodecSettings::EqualsDisregardingFlexfec( + const WebRtcVideoChannel::VideoCodecSettings& a, + const WebRtcVideoChannel::VideoCodecSettings& b) { + return a.codec == b.codec && a.ulpfec == b.ulpfec && + a.rtx_payload_type == b.rtx_payload_type; +} + +bool WebRtcVideoChannel::VideoCodecSettings::operator!=( + const WebRtcVideoChannel::VideoCodecSettings& other) const { + return !(*this == other); +} + +std::vector<WebRtcVideoChannel::VideoCodecSettings> +WebRtcVideoChannel::MapCodecs(const std::vector<VideoCodec>& codecs) { + if (codecs.empty()) { + return {}; + } + + std::vector<VideoCodecSettings> video_codecs; + std::map<int, VideoCodec::CodecType> payload_codec_type; + // |rtx_mapping| maps video payload type to rtx payload type. + std::map<int, int> rtx_mapping; + + webrtc::UlpfecConfig ulpfec_config; + absl::optional<int> flexfec_payload_type; + + for (const VideoCodec& in_codec : codecs) { + const int payload_type = in_codec.id; + + if (payload_codec_type.find(payload_type) != payload_codec_type.end()) { + RTC_LOG(LS_ERROR) << "Payload type already registered: " + << in_codec.ToString(); + return {}; + } + payload_codec_type[payload_type] = in_codec.GetCodecType(); + + switch (in_codec.GetCodecType()) { + case VideoCodec::CODEC_RED: { + if (ulpfec_config.red_payload_type != -1) { + RTC_LOG(LS_ERROR) + << "Duplicate RED codec: ignoring PT=" << payload_type + << " in favor of PT=" << ulpfec_config.red_payload_type + << " which was specified first."; + break; + } + ulpfec_config.red_payload_type = payload_type; + break; + } + + case VideoCodec::CODEC_ULPFEC: { + if (ulpfec_config.ulpfec_payload_type != -1) { + RTC_LOG(LS_ERROR) + << "Duplicate ULPFEC codec: ignoring PT=" << payload_type + << " in favor of PT=" << ulpfec_config.ulpfec_payload_type + << " which was specified first."; + break; + } + ulpfec_config.ulpfec_payload_type = payload_type; + break; + } + + case VideoCodec::CODEC_FLEXFEC: { + if (flexfec_payload_type) { + RTC_LOG(LS_ERROR) + << "Duplicate FLEXFEC codec: ignoring PT=" << payload_type + << " in favor of PT=" << *flexfec_payload_type + << " which was specified first."; + break; + } + flexfec_payload_type = payload_type; + break; + } + + case VideoCodec::CODEC_RTX: { + int associated_payload_type; + if (!in_codec.GetParam(kCodecParamAssociatedPayloadType, + &associated_payload_type) || + !IsValidRtpPayloadType(associated_payload_type)) { + RTC_LOG(LS_ERROR) + << "RTX codec with invalid or no associated payload type: " + << in_codec.ToString(); + return {}; + } + rtx_mapping[associated_payload_type] = payload_type; + break; + } + + case VideoCodec::CODEC_VIDEO: { + video_codecs.emplace_back(); + video_codecs.back().codec = in_codec; + break; + } + } + } + + // One of these codecs should have been a video codec. Only having FEC + // parameters into this code is a logic error. + RTC_DCHECK(!video_codecs.empty()); + + for (const auto& entry : rtx_mapping) { + const int associated_payload_type = entry.first; + const int rtx_payload_type = entry.second; + auto it = payload_codec_type.find(associated_payload_type); + if (it == payload_codec_type.end()) { + RTC_LOG(LS_ERROR) << "RTX codec (PT=" << rtx_payload_type + << ") mapped to PT=" << associated_payload_type + << " which is not in the codec list."; + return {}; + } + const VideoCodec::CodecType associated_codec_type = it->second; + if (associated_codec_type != VideoCodec::CODEC_VIDEO && + associated_codec_type != VideoCodec::CODEC_RED) { + RTC_LOG(LS_ERROR) + << "RTX PT=" << rtx_payload_type + << " not mapped to regular video codec or RED codec (PT=" + << associated_payload_type << ")."; + return {}; + } + + if (associated_payload_type == ulpfec_config.red_payload_type) { + ulpfec_config.red_rtx_payload_type = rtx_payload_type; + } + } + + for (VideoCodecSettings& codec_settings : video_codecs) { + const int payload_type = codec_settings.codec.id; + codec_settings.ulpfec = ulpfec_config; + codec_settings.flexfec_payload_type = flexfec_payload_type.value_or(-1); + auto it = rtx_mapping.find(payload_type); + if (it != rtx_mapping.end()) { + const int rtx_payload_type = it->second; + codec_settings.rtx_payload_type = rtx_payload_type; + } + } + + return video_codecs; +} + +WebRtcVideoChannel::WebRtcVideoReceiveStream* +WebRtcVideoChannel::FindReceiveStream(uint32_t ssrc) { + if (ssrc == 0) { + absl::optional<uint32_t> default_ssrc = GetDefaultReceiveStreamSsrc(); + if (!default_ssrc) { + return nullptr; + } + ssrc = *default_ssrc; + } + auto it = receive_streams_.find(ssrc); + if (it != receive_streams_.end()) { + return it->second; + } + return nullptr; +} + +void WebRtcVideoChannel::SetRecordableEncodedFrameCallback( + uint32_t ssrc, + std::function<void(const webrtc::RecordableEncodedFrame&)> callback) { + RTC_DCHECK_RUN_ON(&thread_checker_); + WebRtcVideoReceiveStream* stream = FindReceiveStream(ssrc); + if (stream) { + stream->SetRecordableEncodedFrameCallback(std::move(callback)); + } else { + RTC_LOG(LS_ERROR) << "Absent receive stream; ignoring setting encoded " + "frame sink for ssrc " + << ssrc; + } +} + +void WebRtcVideoChannel::ClearRecordableEncodedFrameCallback(uint32_t ssrc) { + RTC_DCHECK_RUN_ON(&thread_checker_); + WebRtcVideoReceiveStream* stream = FindReceiveStream(ssrc); + if (stream) { + stream->ClearRecordableEncodedFrameCallback(); + } else { + RTC_LOG(LS_ERROR) << "Absent receive stream; ignoring clearing encoded " + "frame sink for ssrc " + << ssrc; + } +} + +void WebRtcVideoChannel::GenerateKeyFrame(uint32_t ssrc) { + RTC_DCHECK_RUN_ON(&thread_checker_); + WebRtcVideoReceiveStream* stream = FindReceiveStream(ssrc); + if (stream) { + stream->GenerateKeyFrame(); + } else { + RTC_LOG(LS_ERROR) + << "Absent receive stream; ignoring key frame generation for ssrc " + << ssrc; + } +} + +void WebRtcVideoChannel::SetEncoderToPacketizerFrameTransformer( + uint32_t ssrc, + rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer) { + RTC_DCHECK_RUN_ON(&thread_checker_); + auto matching_stream = send_streams_.find(ssrc); + if (matching_stream != send_streams_.end()) { + matching_stream->second->SetEncoderToPacketizerFrameTransformer( + std::move(frame_transformer)); + } +} + +void WebRtcVideoChannel::SetDepacketizerToDecoderFrameTransformer( + uint32_t ssrc, + rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer) { + RTC_DCHECK(frame_transformer); + RTC_DCHECK_RUN_ON(&thread_checker_); + if (ssrc == 0) { + // If the receiver is unsignaled, save the frame transformer and set it when + // the stream is associated with an ssrc. + unsignaled_frame_transformer_ = std::move(frame_transformer); + return; + } + + auto matching_stream = receive_streams_.find(ssrc); + if (matching_stream != receive_streams_.end()) { + matching_stream->second->SetDepacketizerToDecoderFrameTransformer( + std::move(frame_transformer)); + } +} + +// TODO(bugs.webrtc.org/8785): Consider removing max_qp as member of +// EncoderStreamFactory and instead set this value individually for each stream +// in the VideoEncoderConfig.simulcast_layers. +EncoderStreamFactory::EncoderStreamFactory(std::string codec_name, + int max_qp, + bool is_screenshare, + bool conference_mode) + + : codec_name_(codec_name), + max_qp_(max_qp), + is_screenshare_(is_screenshare), + conference_mode_(conference_mode) {} + +std::vector<webrtc::VideoStream> EncoderStreamFactory::CreateEncoderStreams( + int width, + int height, + const webrtc::VideoEncoderConfig& encoder_config) { + RTC_DCHECK_GT(encoder_config.number_of_streams, 0); + RTC_DCHECK_GE(encoder_config.simulcast_layers.size(), + encoder_config.number_of_streams); + + const absl::optional<webrtc::DataRate> experimental_min_bitrate = + GetExperimentalMinVideoBitrate(encoder_config.codec_type); + + if (encoder_config.number_of_streams > 1 || + ((absl::EqualsIgnoreCase(codec_name_, kVp8CodecName) || + absl::EqualsIgnoreCase(codec_name_, kH264CodecName)) && + is_screenshare_ && conference_mode_)) { + return CreateSimulcastOrConfereceModeScreenshareStreams( + width, height, encoder_config, experimental_min_bitrate); + } + + return CreateDefaultVideoStreams(width, height, encoder_config, + experimental_min_bitrate); +} + +std::vector<webrtc::VideoStream> +EncoderStreamFactory::CreateDefaultVideoStreams( + int width, + int height, + const webrtc::VideoEncoderConfig& encoder_config, + const absl::optional<webrtc::DataRate>& experimental_min_bitrate) const { + std::vector<webrtc::VideoStream> layers; + + // For unset max bitrates set default bitrate for non-simulcast. + int max_bitrate_bps = + (encoder_config.max_bitrate_bps > 0) + ? encoder_config.max_bitrate_bps + : GetMaxDefaultVideoBitrateKbps(width, height, is_screenshare_) * + 1000; + + int min_bitrate_bps = + experimental_min_bitrate + ? rtc::saturated_cast<int>(experimental_min_bitrate->bps()) + : webrtc::kDefaultMinVideoBitrateBps; + if (encoder_config.simulcast_layers[0].min_bitrate_bps > 0) { + // Use set min bitrate. + min_bitrate_bps = encoder_config.simulcast_layers[0].min_bitrate_bps; + // If only min bitrate is configured, make sure max is above min. + if (encoder_config.max_bitrate_bps <= 0) + max_bitrate_bps = std::max(min_bitrate_bps, max_bitrate_bps); + } + int max_framerate = (encoder_config.simulcast_layers[0].max_framerate > 0) + ? encoder_config.simulcast_layers[0].max_framerate + : kDefaultVideoMaxFramerate; + + webrtc::VideoStream layer; + layer.width = width; + layer.height = height; + layer.max_framerate = max_framerate; + + if (encoder_config.simulcast_layers[0].scale_resolution_down_by > 1.) { + layer.width = std::max<size_t>( + layer.width / + encoder_config.simulcast_layers[0].scale_resolution_down_by, + kMinLayerSize); + layer.height = std::max<size_t>( + layer.height / + encoder_config.simulcast_layers[0].scale_resolution_down_by, + kMinLayerSize); + } + + // In the case that the application sets a max bitrate that's lower than the + // min bitrate, we adjust it down (see bugs.webrtc.org/9141). + layer.min_bitrate_bps = std::min(min_bitrate_bps, max_bitrate_bps); + if (encoder_config.simulcast_layers[0].target_bitrate_bps <= 0) { + layer.target_bitrate_bps = max_bitrate_bps; + } else { + layer.target_bitrate_bps = + encoder_config.simulcast_layers[0].target_bitrate_bps; + } + layer.max_bitrate_bps = max_bitrate_bps; + layer.max_qp = max_qp_; + layer.bitrate_priority = encoder_config.bitrate_priority; + + if (absl::EqualsIgnoreCase(codec_name_, kVp9CodecName)) { + RTC_DCHECK(encoder_config.encoder_specific_settings); + // Use VP9 SVC layering from codec settings which might be initialized + // though field trial in ConfigureVideoEncoderSettings. + webrtc::VideoCodecVP9 vp9_settings; + encoder_config.encoder_specific_settings->FillVideoCodecVp9(&vp9_settings); + layer.num_temporal_layers = vp9_settings.numberOfTemporalLayers; + } + + if (IsTemporalLayersSupported(codec_name_)) { + // Use configured number of temporal layers if set. + if (encoder_config.simulcast_layers[0].num_temporal_layers) { + layer.num_temporal_layers = + *encoder_config.simulcast_layers[0].num_temporal_layers; + } + } + + layers.push_back(layer); + return layers; +} + +std::vector<webrtc::VideoStream> +EncoderStreamFactory::CreateSimulcastOrConfereceModeScreenshareStreams( + int width, + int height, + const webrtc::VideoEncoderConfig& encoder_config, + const absl::optional<webrtc::DataRate>& experimental_min_bitrate) const { + std::vector<webrtc::VideoStream> layers; + + const bool temporal_layers_supported = + absl::EqualsIgnoreCase(codec_name_, kVp8CodecName) || + absl::EqualsIgnoreCase(codec_name_, kH264CodecName); + // Use legacy simulcast screenshare if conference mode is explicitly enabled + // or use the regular simulcast configuration path which is generic. + layers = GetSimulcastConfig(FindRequiredActiveLayers(encoder_config), + encoder_config.number_of_streams, width, height, + encoder_config.bitrate_priority, max_qp_, + is_screenshare_ && conference_mode_, + temporal_layers_supported); + // Allow an experiment to override the minimum bitrate for the lowest + // spatial layer. The experiment's configuration has the lowest priority. + if (experimental_min_bitrate) { + layers[0].min_bitrate_bps = + rtc::saturated_cast<int>(experimental_min_bitrate->bps()); + } + // Update the active simulcast layers and configured bitrates. + bool is_highest_layer_max_bitrate_configured = false; + const bool has_scale_resolution_down_by = absl::c_any_of( + encoder_config.simulcast_layers, [](const webrtc::VideoStream& layer) { + return layer.scale_resolution_down_by != -1.; + }); + const int normalized_width = + NormalizeSimulcastSize(width, encoder_config.number_of_streams); + const int normalized_height = + NormalizeSimulcastSize(height, encoder_config.number_of_streams); + for (size_t i = 0; i < layers.size(); ++i) { + layers[i].active = encoder_config.simulcast_layers[i].active; + // Update with configured num temporal layers if supported by codec. + if (encoder_config.simulcast_layers[i].num_temporal_layers && + IsTemporalLayersSupported(codec_name_)) { + layers[i].num_temporal_layers = + *encoder_config.simulcast_layers[i].num_temporal_layers; + } + if (encoder_config.simulcast_layers[i].max_framerate > 0) { + layers[i].max_framerate = + encoder_config.simulcast_layers[i].max_framerate; + } + if (has_scale_resolution_down_by) { + const double scale_resolution_down_by = std::max( + encoder_config.simulcast_layers[i].scale_resolution_down_by, 1.0); + layers[i].width = std::max( + static_cast<int>(normalized_width / scale_resolution_down_by), + kMinLayerSize); + layers[i].height = std::max( + static_cast<int>(normalized_height / scale_resolution_down_by), + kMinLayerSize); + } + // Update simulcast bitrates with configured min and max bitrate. + if (encoder_config.simulcast_layers[i].min_bitrate_bps > 0) { + layers[i].min_bitrate_bps = + encoder_config.simulcast_layers[i].min_bitrate_bps; + } + if (encoder_config.simulcast_layers[i].max_bitrate_bps > 0) { + layers[i].max_bitrate_bps = + encoder_config.simulcast_layers[i].max_bitrate_bps; + } + if (encoder_config.simulcast_layers[i].target_bitrate_bps > 0) { + layers[i].target_bitrate_bps = + encoder_config.simulcast_layers[i].target_bitrate_bps; + } + if (encoder_config.simulcast_layers[i].min_bitrate_bps > 0 && + encoder_config.simulcast_layers[i].max_bitrate_bps > 0) { + // Min and max bitrate are configured. + // Set target to 3/4 of the max bitrate (or to max if below min). + if (encoder_config.simulcast_layers[i].target_bitrate_bps <= 0) + layers[i].target_bitrate_bps = layers[i].max_bitrate_bps * 3 / 4; + if (layers[i].target_bitrate_bps < layers[i].min_bitrate_bps) + layers[i].target_bitrate_bps = layers[i].max_bitrate_bps; + } else if (encoder_config.simulcast_layers[i].min_bitrate_bps > 0) { + // Only min bitrate is configured, make sure target/max are above min. + layers[i].target_bitrate_bps = + std::max(layers[i].target_bitrate_bps, layers[i].min_bitrate_bps); + layers[i].max_bitrate_bps = + std::max(layers[i].max_bitrate_bps, layers[i].min_bitrate_bps); + } else if (encoder_config.simulcast_layers[i].max_bitrate_bps > 0) { + // Only max bitrate is configured, make sure min/target are below max. + layers[i].min_bitrate_bps = + std::min(layers[i].min_bitrate_bps, layers[i].max_bitrate_bps); + layers[i].target_bitrate_bps = + std::min(layers[i].target_bitrate_bps, layers[i].max_bitrate_bps); + } + if (i == layers.size() - 1) { + is_highest_layer_max_bitrate_configured = + encoder_config.simulcast_layers[i].max_bitrate_bps > 0; + } + } + if (!is_screenshare_ && !is_highest_layer_max_bitrate_configured && + encoder_config.max_bitrate_bps > 0) { + // No application-configured maximum for the largest layer. + // If there is bitrate leftover, give it to the largest layer. + BoostMaxSimulcastLayer( + webrtc::DataRate::BitsPerSec(encoder_config.max_bitrate_bps), &layers); + } + return layers; +} + +} // namespace cricket diff --git a/media/engine/webrtc_video_engine.h b/media/engine/webrtc_video_engine.h new file mode 100644 index 0000000000..00d249541a --- /dev/null +++ b/media/engine/webrtc_video_engine.h @@ -0,0 +1,682 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MEDIA_ENGINE_WEBRTC_VIDEO_ENGINE_H_ +#define MEDIA_ENGINE_WEBRTC_VIDEO_ENGINE_H_ + +#include <map> +#include <memory> +#include <set> +#include <string> +#include <vector> + +#include "absl/types/optional.h" +#include "api/call/transport.h" +#include "api/video/video_bitrate_allocator_factory.h" +#include "api/video/video_frame.h" +#include "api/video/video_sink_interface.h" +#include "api/video/video_source_interface.h" +#include "api/video_codecs/sdp_video_format.h" +#include "call/call.h" +#include "call/flexfec_receive_stream.h" +#include "call/video_receive_stream.h" +#include "call/video_send_stream.h" +#include "media/base/media_engine.h" +#include "media/engine/constants.h" +#include "media/engine/unhandled_packets_buffer.h" +#include "rtc_base/async_invoker.h" +#include "rtc_base/critical_section.h" +#include "rtc_base/network_route.h" +#include "rtc_base/thread_annotations.h" +#include "rtc_base/thread_checker.h" + +namespace webrtc { +class VideoDecoderFactory; +class VideoEncoderFactory; +struct MediaConfig; +} // namespace webrtc + +namespace rtc { +class Thread; +} // namespace rtc + +namespace cricket { + +class WebRtcVideoChannel; + +// Public for testing. +// Inputs StreamStats for all types of substreams (kMedia, kRtx, kFlexfec) and +// merges any non-kMedia substream stats object into its referenced kMedia-type +// substream. The resulting substreams are all kMedia. This means, for example, +// that packet and byte counters of RTX and FlexFEC streams are accounted for in +// the relevant RTP media stream's stats. This makes the resulting StreamStats +// objects ready to be turned into "outbound-rtp" stats objects for GetStats() +// which does not create separate stream stats objects for complementary +// streams. +std::map<uint32_t, webrtc::VideoSendStream::StreamStats> +MergeInfoAboutOutboundRtpSubstreamsForTesting( + const std::map<uint32_t, webrtc::VideoSendStream::StreamStats>& substreams); + +class UnsignalledSsrcHandler { + public: + enum Action { + kDropPacket, + kDeliverPacket, + }; + virtual Action OnUnsignalledSsrc(WebRtcVideoChannel* channel, + uint32_t ssrc) = 0; + virtual ~UnsignalledSsrcHandler() = default; +}; + +// TODO(pbos): Remove, use external handlers only. +class DefaultUnsignalledSsrcHandler : public UnsignalledSsrcHandler { + public: + DefaultUnsignalledSsrcHandler(); + Action OnUnsignalledSsrc(WebRtcVideoChannel* channel, uint32_t ssrc) override; + + rtc::VideoSinkInterface<webrtc::VideoFrame>* GetDefaultSink() const; + void SetDefaultSink(WebRtcVideoChannel* channel, + rtc::VideoSinkInterface<webrtc::VideoFrame>* sink); + + virtual ~DefaultUnsignalledSsrcHandler() = default; + + private: + rtc::VideoSinkInterface<webrtc::VideoFrame>* default_sink_; +}; + +// WebRtcVideoEngine is used for the new native WebRTC Video API (webrtc:1667). +class WebRtcVideoEngine : public VideoEngineInterface { + public: + // These video codec factories represents all video codecs, i.e. both software + // and external hardware codecs. + WebRtcVideoEngine( + std::unique_ptr<webrtc::VideoEncoderFactory> video_encoder_factory, + std::unique_ptr<webrtc::VideoDecoderFactory> video_decoder_factory); + + ~WebRtcVideoEngine() override; + + VideoMediaChannel* CreateMediaChannel( + webrtc::Call* call, + const MediaConfig& config, + const VideoOptions& options, + const webrtc::CryptoOptions& crypto_options, + webrtc::VideoBitrateAllocatorFactory* video_bitrate_allocator_factory) + override; + + std::vector<VideoCodec> send_codecs() const override; + std::vector<VideoCodec> recv_codecs() const override; + std::vector<webrtc::RtpHeaderExtensionCapability> GetRtpHeaderExtensions() + const override; + + private: + const std::unique_ptr<webrtc::VideoDecoderFactory> decoder_factory_; + const std::unique_ptr<webrtc::VideoEncoderFactory> encoder_factory_; + const std::unique_ptr<webrtc::VideoBitrateAllocatorFactory> + bitrate_allocator_factory_; +}; + +class WebRtcVideoChannel : public VideoMediaChannel, + public webrtc::Transport, + public webrtc::EncoderSwitchRequestCallback { + public: + WebRtcVideoChannel( + webrtc::Call* call, + const MediaConfig& config, + const VideoOptions& options, + const webrtc::CryptoOptions& crypto_options, + webrtc::VideoEncoderFactory* encoder_factory, + webrtc::VideoDecoderFactory* decoder_factory, + webrtc::VideoBitrateAllocatorFactory* bitrate_allocator_factory); + ~WebRtcVideoChannel() override; + + // VideoMediaChannel implementation + bool SetSendParameters(const VideoSendParameters& params) override; + bool SetRecvParameters(const VideoRecvParameters& params) override; + webrtc::RtpParameters GetRtpSendParameters(uint32_t ssrc) const override; + webrtc::RTCError SetRtpSendParameters( + uint32_t ssrc, + const webrtc::RtpParameters& parameters) override; + webrtc::RtpParameters GetRtpReceiveParameters(uint32_t ssrc) const override; + webrtc::RtpParameters GetDefaultRtpReceiveParameters() const override; + bool GetSendCodec(VideoCodec* send_codec) override; + bool SetSend(bool send) override; + bool SetVideoSend( + uint32_t ssrc, + const VideoOptions* options, + rtc::VideoSourceInterface<webrtc::VideoFrame>* source) override; + bool AddSendStream(const StreamParams& sp) override; + bool RemoveSendStream(uint32_t ssrc) override; + bool AddRecvStream(const StreamParams& sp) override; + bool AddRecvStream(const StreamParams& sp, bool default_stream); + bool RemoveRecvStream(uint32_t ssrc) override; + void ResetUnsignaledRecvStream() override; + bool SetSink(uint32_t ssrc, + rtc::VideoSinkInterface<webrtc::VideoFrame>* sink) override; + void SetDefaultSink( + rtc::VideoSinkInterface<webrtc::VideoFrame>* sink) override; + void FillBitrateInfo(BandwidthEstimationInfo* bwe_info) override; + bool GetStats(VideoMediaInfo* info) override; + + void OnPacketReceived(rtc::CopyOnWriteBuffer packet, + int64_t packet_time_us) override; + void OnReadyToSend(bool ready) override; + void OnNetworkRouteChanged(const std::string& transport_name, + const rtc::NetworkRoute& network_route) override; + void SetInterface( + NetworkInterface* iface, + const webrtc::MediaTransportConfig& media_transport_config) override; + + // E2E Encrypted Video Frame API + // Set a frame decryptor to a particular ssrc that will intercept all + // incoming video frames and attempt to decrypt them before forwarding the + // result. + void SetFrameDecryptor(uint32_t ssrc, + rtc::scoped_refptr<webrtc::FrameDecryptorInterface> + frame_decryptor) override; + // Set a frame encryptor to a particular ssrc that will intercept all + // outgoing video frames and attempt to encrypt them and forward the result + // to the packetizer. + void SetFrameEncryptor(uint32_t ssrc, + rtc::scoped_refptr<webrtc::FrameEncryptorInterface> + frame_encryptor) override; + + void SetVideoCodecSwitchingEnabled(bool enabled) override; + + bool SetBaseMinimumPlayoutDelayMs(uint32_t ssrc, int delay_ms) override; + + absl::optional<int> GetBaseMinimumPlayoutDelayMs( + uint32_t ssrc) const override; + + // Implemented for VideoMediaChannelTest. + bool sending() const { + RTC_DCHECK_RUN_ON(&thread_checker_); + return sending_; + } + + absl::optional<uint32_t> GetDefaultReceiveStreamSsrc(); + + StreamParams unsignaled_stream_params() { + RTC_DCHECK_RUN_ON(&thread_checker_); + return unsignaled_stream_params_; + } + + // AdaptReason is used for expressing why a WebRtcVideoSendStream request + // a lower input frame size than the currently configured camera input frame + // size. There can be more than one reason OR:ed together. + enum AdaptReason { + ADAPTREASON_NONE = 0, + ADAPTREASON_CPU = 1, + ADAPTREASON_BANDWIDTH = 2, + }; + + static constexpr int kDefaultQpMax = 56; + + std::vector<webrtc::RtpSource> GetSources(uint32_t ssrc) const override; + + // Take the buffered packets for |ssrcs| and feed them into DeliverPacket. + // This method does nothing unless unknown_ssrc_packet_buffer_ is configured. + void BackfillBufferedPackets(rtc::ArrayView<const uint32_t> ssrcs); + + // Implements webrtc::EncoderSwitchRequestCallback. + void RequestEncoderFallback() override; + + // TODO(bugs.webrtc.org/11341) : Remove this version of RequestEncoderSwitch. + void RequestEncoderSwitch( + const EncoderSwitchRequestCallback::Config& conf) override; + void RequestEncoderSwitch(const webrtc::SdpVideoFormat& format) override; + + void SetRecordableEncodedFrameCallback( + uint32_t ssrc, + std::function<void(const webrtc::RecordableEncodedFrame&)> callback) + override; + void ClearRecordableEncodedFrameCallback(uint32_t ssrc) override; + void GenerateKeyFrame(uint32_t ssrc) override; + + void SetEncoderToPacketizerFrameTransformer( + uint32_t ssrc, + rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer) + override; + void SetDepacketizerToDecoderFrameTransformer( + uint32_t ssrc, + rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer) + override; + + private: + class WebRtcVideoReceiveStream; + + // Finds VideoReceiveStream corresponding to ssrc. Aware of unsignalled ssrc + // handling. + WebRtcVideoReceiveStream* FindReceiveStream(uint32_t ssrc) + RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_); + + struct VideoCodecSettings { + VideoCodecSettings(); + + // Checks if all members of |*this| are equal to the corresponding members + // of |other|. + bool operator==(const VideoCodecSettings& other) const; + bool operator!=(const VideoCodecSettings& other) const; + + // Checks if all members of |a|, except |flexfec_payload_type|, are equal + // to the corresponding members of |b|. + static bool EqualsDisregardingFlexfec(const VideoCodecSettings& a, + const VideoCodecSettings& b); + + VideoCodec codec; + webrtc::UlpfecConfig ulpfec; + int flexfec_payload_type; // -1 if absent. + int rtx_payload_type; // -1 if absent. + }; + + struct ChangedSendParameters { + // These optionals are unset if not changed. + absl::optional<VideoCodecSettings> send_codec; + absl::optional<std::vector<VideoCodecSettings>> negotiated_codecs; + absl::optional<std::vector<webrtc::RtpExtension>> rtp_header_extensions; + absl::optional<std::string> mid; + absl::optional<bool> extmap_allow_mixed; + absl::optional<int> max_bandwidth_bps; + absl::optional<bool> conference_mode; + absl::optional<webrtc::RtcpMode> rtcp_mode; + }; + + struct ChangedRecvParameters { + // These optionals are unset if not changed. + absl::optional<std::vector<VideoCodecSettings>> codec_settings; + absl::optional<std::vector<webrtc::RtpExtension>> rtp_header_extensions; + // Keep track of the FlexFEC payload type separately from |codec_settings|. + // This allows us to recreate the FlexfecReceiveStream separately from the + // VideoReceiveStream when the FlexFEC payload type is changed. + absl::optional<int> flexfec_payload_type; + }; + + bool GetChangedSendParameters(const VideoSendParameters& params, + ChangedSendParameters* changed_params) const + RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_); + bool ApplyChangedParams(const ChangedSendParameters& changed_params); + bool GetChangedRecvParameters(const VideoRecvParameters& params, + ChangedRecvParameters* changed_params) const + RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_); + + void ConfigureReceiverRtp( + webrtc::VideoReceiveStream::Config* config, + webrtc::FlexfecReceiveStream::Config* flexfec_config, + const StreamParams& sp) const + RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_); + bool ValidateSendSsrcAvailability(const StreamParams& sp) const + RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_); + bool ValidateReceiveSsrcAvailability(const StreamParams& sp) const + RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_); + void DeleteReceiveStream(WebRtcVideoReceiveStream* stream) + RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_); + + static std::string CodecSettingsVectorToString( + const std::vector<VideoCodecSettings>& codecs); + + // Wrapper for the sender part. + class WebRtcVideoSendStream + : public rtc::VideoSourceInterface<webrtc::VideoFrame> { + public: + WebRtcVideoSendStream( + webrtc::Call* call, + const StreamParams& sp, + webrtc::VideoSendStream::Config config, + const VideoOptions& options, + bool enable_cpu_overuse_detection, + int max_bitrate_bps, + const absl::optional<VideoCodecSettings>& codec_settings, + const absl::optional<std::vector<webrtc::RtpExtension>>& rtp_extensions, + const VideoSendParameters& send_params); + virtual ~WebRtcVideoSendStream(); + + void SetSendParameters(const ChangedSendParameters& send_params); + webrtc::RTCError SetRtpParameters(const webrtc::RtpParameters& parameters); + webrtc::RtpParameters GetRtpParameters() const; + + void SetFrameEncryptor( + rtc::scoped_refptr<webrtc::FrameEncryptorInterface> frame_encryptor); + + // Implements rtc::VideoSourceInterface<webrtc::VideoFrame>. + // WebRtcVideoSendStream acts as a source to the webrtc::VideoSendStream + // in |stream_|. This is done to proxy VideoSinkWants from the encoder to + // the worker thread. + void AddOrUpdateSink(rtc::VideoSinkInterface<webrtc::VideoFrame>* sink, + const rtc::VideoSinkWants& wants) override; + void RemoveSink(rtc::VideoSinkInterface<webrtc::VideoFrame>* sink) override; + + bool SetVideoSend(const VideoOptions* options, + rtc::VideoSourceInterface<webrtc::VideoFrame>* source); + + void SetSend(bool send); + + const std::vector<uint32_t>& GetSsrcs() const; + // Returns per ssrc VideoSenderInfos. Useful for simulcast scenario. + std::vector<VideoSenderInfo> GetPerLayerVideoSenderInfos(bool log_stats); + // Aggregates per ssrc VideoSenderInfos to single VideoSenderInfo for + // legacy reasons. Used in old GetStats API and track stats. + VideoSenderInfo GetAggregatedVideoSenderInfo( + const std::vector<VideoSenderInfo>& infos) const; + void FillBitrateInfo(BandwidthEstimationInfo* bwe_info); + + void SetEncoderToPacketizerFrameTransformer( + rtc::scoped_refptr<webrtc::FrameTransformerInterface> + frame_transformer); + + private: + // Parameters needed to reconstruct the underlying stream. + // webrtc::VideoSendStream doesn't support setting a lot of options on the + // fly, so when those need to be changed we tear down and reconstruct with + // similar parameters depending on which options changed etc. + struct VideoSendStreamParameters { + VideoSendStreamParameters( + webrtc::VideoSendStream::Config config, + const VideoOptions& options, + int max_bitrate_bps, + const absl::optional<VideoCodecSettings>& codec_settings); + webrtc::VideoSendStream::Config config; + VideoOptions options; + int max_bitrate_bps; + bool conference_mode; + absl::optional<VideoCodecSettings> codec_settings; + // Sent resolutions + bitrates etc. by the underlying VideoSendStream, + // typically changes when setting a new resolution or reconfiguring + // bitrates. + webrtc::VideoEncoderConfig encoder_config; + }; + + rtc::scoped_refptr<webrtc::VideoEncoderConfig::EncoderSpecificSettings> + ConfigureVideoEncoderSettings(const VideoCodec& codec); + void SetCodec(const VideoCodecSettings& codec); + void RecreateWebRtcStream(); + webrtc::VideoEncoderConfig CreateVideoEncoderConfig( + const VideoCodec& codec) const; + void ReconfigureEncoder(); + + // Calls Start or Stop according to whether or not |sending_| is true, + // and whether or not the encoding in |rtp_parameters_| is active. + void UpdateSendState(); + + webrtc::DegradationPreference GetDegradationPreference() const + RTC_EXCLUSIVE_LOCKS_REQUIRED(&thread_checker_); + + rtc::ThreadChecker thread_checker_; + rtc::Thread* worker_thread_; + const std::vector<uint32_t> ssrcs_ RTC_GUARDED_BY(&thread_checker_); + const std::vector<SsrcGroup> ssrc_groups_ RTC_GUARDED_BY(&thread_checker_); + webrtc::Call* const call_; + const bool enable_cpu_overuse_detection_; + rtc::VideoSourceInterface<webrtc::VideoFrame>* source_ + RTC_GUARDED_BY(&thread_checker_); + + webrtc::VideoSendStream* stream_ RTC_GUARDED_BY(&thread_checker_); + rtc::VideoSinkInterface<webrtc::VideoFrame>* encoder_sink_ + RTC_GUARDED_BY(&thread_checker_); + // Contains settings that are the same for all streams in the MediaChannel, + // such as codecs, header extensions, and the global bitrate limit for the + // entire channel. + VideoSendStreamParameters parameters_ RTC_GUARDED_BY(&thread_checker_); + // Contains settings that are unique for each stream, such as max_bitrate. + // Does *not* contain codecs, however. + // TODO(skvlad): Move ssrcs_ and ssrc_groups_ into rtp_parameters_. + // TODO(skvlad): Combine parameters_ and rtp_parameters_ once we have only + // one stream per MediaChannel. + webrtc::RtpParameters rtp_parameters_ RTC_GUARDED_BY(&thread_checker_); + + bool sending_ RTC_GUARDED_BY(&thread_checker_); + + // In order for the |invoker_| to protect other members from being + // destructed as they are used in asynchronous tasks it has to be destructed + // first. + rtc::AsyncInvoker invoker_; + }; + + // Wrapper for the receiver part, contains configs etc. that are needed to + // reconstruct the underlying VideoReceiveStream. + class WebRtcVideoReceiveStream + : public rtc::VideoSinkInterface<webrtc::VideoFrame> { + public: + WebRtcVideoReceiveStream( + WebRtcVideoChannel* channel, + webrtc::Call* call, + const StreamParams& sp, + webrtc::VideoReceiveStream::Config config, + webrtc::VideoDecoderFactory* decoder_factory, + bool default_stream, + const std::vector<VideoCodecSettings>& recv_codecs, + const webrtc::FlexfecReceiveStream::Config& flexfec_config); + ~WebRtcVideoReceiveStream(); + + const std::vector<uint32_t>& GetSsrcs() const; + + std::vector<webrtc::RtpSource> GetSources(); + + // Does not return codecs, they are filled by the owning WebRtcVideoChannel. + webrtc::RtpParameters GetRtpParameters() const; + + void SetLocalSsrc(uint32_t local_ssrc); + // TODO(deadbeef): Move these feedback parameters into the recv parameters. + void SetFeedbackParameters(bool lntf_enabled, + bool nack_enabled, + bool transport_cc_enabled, + webrtc::RtcpMode rtcp_mode); + void SetRecvParameters(const ChangedRecvParameters& recv_params); + + void OnFrame(const webrtc::VideoFrame& frame) override; + bool IsDefaultStream() const; + + void SetFrameDecryptor( + rtc::scoped_refptr<webrtc::FrameDecryptorInterface> frame_decryptor); + + bool SetBaseMinimumPlayoutDelayMs(int delay_ms); + + int GetBaseMinimumPlayoutDelayMs() const; + + void SetSink(rtc::VideoSinkInterface<webrtc::VideoFrame>* sink); + + VideoReceiverInfo GetVideoReceiverInfo(bool log_stats); + + void SetRecordableEncodedFrameCallback( + std::function<void(const webrtc::RecordableEncodedFrame&)> callback); + void ClearRecordableEncodedFrameCallback(); + void GenerateKeyFrame(); + + void SetDepacketizerToDecoderFrameTransformer( + rtc::scoped_refptr<webrtc::FrameTransformerInterface> + frame_transformer); + + private: + void RecreateWebRtcVideoStream(); + void MaybeRecreateWebRtcFlexfecStream(); + + void MaybeAssociateFlexfecWithVideo(); + void MaybeDissociateFlexfecFromVideo(); + + void ConfigureCodecs(const std::vector<VideoCodecSettings>& recv_codecs); + void ConfigureFlexfecCodec(int flexfec_payload_type); + + std::string GetCodecNameFromPayloadType(int payload_type); + + WebRtcVideoChannel* const channel_; + webrtc::Call* const call_; + const StreamParams stream_params_; + + // Both |stream_| and |flexfec_stream_| are managed by |this|. They are + // destroyed by calling call_->DestroyVideoReceiveStream and + // call_->DestroyFlexfecReceiveStream, respectively. + webrtc::VideoReceiveStream* stream_; + const bool default_stream_; + webrtc::VideoReceiveStream::Config config_; + webrtc::FlexfecReceiveStream::Config flexfec_config_; + webrtc::FlexfecReceiveStream* flexfec_stream_; + + webrtc::VideoDecoderFactory* const decoder_factory_; + + rtc::CriticalSection sink_lock_; + rtc::VideoSinkInterface<webrtc::VideoFrame>* sink_ + RTC_GUARDED_BY(sink_lock_); + // Expands remote RTP timestamps to int64_t to be able to estimate how long + // the stream has been running. + rtc::TimestampWrapAroundHandler timestamp_wraparound_handler_ + RTC_GUARDED_BY(sink_lock_); + int64_t first_frame_timestamp_ RTC_GUARDED_BY(sink_lock_); + // Start NTP time is estimated as current remote NTP time (estimated from + // RTCP) minus the elapsed time, as soon as remote NTP time is available. + int64_t estimated_remote_start_ntp_time_ms_ RTC_GUARDED_BY(sink_lock_); + }; + + void Construct(webrtc::Call* call, WebRtcVideoEngine* engine); + + bool SendRtp(const uint8_t* data, + size_t len, + const webrtc::PacketOptions& options) override; + bool SendRtcp(const uint8_t* data, size_t len) override; + + // Generate the list of codec parameters to pass down based on the negotiated + // "codecs". Note that VideoCodecSettings correspond to concrete codecs like + // VP8, VP9, H264 while VideoCodecs correspond also to "virtual" codecs like + // RTX, ULPFEC, FLEXFEC. + static std::vector<VideoCodecSettings> MapCodecs( + const std::vector<VideoCodec>& codecs); + // Get all codecs that are compatible with the receiver. + std::vector<VideoCodecSettings> SelectSendVideoCodecs( + const std::vector<VideoCodecSettings>& remote_mapped_codecs) const + RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_); + + static bool NonFlexfecReceiveCodecsHaveChanged( + std::vector<VideoCodecSettings> before, + std::vector<VideoCodecSettings> after); + + void FillSenderStats(VideoMediaInfo* info, bool log_stats) + RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_); + void FillReceiverStats(VideoMediaInfo* info, bool log_stats) + RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_); + void FillBandwidthEstimationStats(const webrtc::Call::Stats& stats, + VideoMediaInfo* info) + RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_); + void FillSendAndReceiveCodecStats(VideoMediaInfo* video_media_info) + RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_); + + rtc::Thread* worker_thread_; + rtc::ThreadChecker thread_checker_; + + uint32_t rtcp_receiver_report_ssrc_ RTC_GUARDED_BY(thread_checker_); + bool sending_ RTC_GUARDED_BY(thread_checker_); + webrtc::Call* const call_ RTC_GUARDED_BY(thread_checker_); + + DefaultUnsignalledSsrcHandler default_unsignalled_ssrc_handler_ + RTC_GUARDED_BY(thread_checker_); + UnsignalledSsrcHandler* const unsignalled_ssrc_handler_ + RTC_GUARDED_BY(thread_checker_); + + // Delay for unsignaled streams, which may be set before the stream exists. + int default_recv_base_minimum_delay_ms_ RTC_GUARDED_BY(thread_checker_) = 0; + + const MediaConfig::Video video_config_ RTC_GUARDED_BY(thread_checker_); + + // Using primary-ssrc (first ssrc) as key. + std::map<uint32_t, WebRtcVideoSendStream*> send_streams_ + RTC_GUARDED_BY(thread_checker_); + std::map<uint32_t, WebRtcVideoReceiveStream*> receive_streams_ + RTC_GUARDED_BY(thread_checker_); + std::set<uint32_t> send_ssrcs_ RTC_GUARDED_BY(thread_checker_); + std::set<uint32_t> receive_ssrcs_ RTC_GUARDED_BY(thread_checker_); + + absl::optional<VideoCodecSettings> send_codec_ + RTC_GUARDED_BY(thread_checker_); + std::vector<VideoCodecSettings> negotiated_codecs_ + RTC_GUARDED_BY(thread_checker_); + + absl::optional<std::vector<webrtc::RtpExtension>> send_rtp_extensions_ + RTC_GUARDED_BY(thread_checker_); + + webrtc::VideoEncoderFactory* const encoder_factory_ + RTC_GUARDED_BY(thread_checker_); + webrtc::VideoDecoderFactory* const decoder_factory_ + RTC_GUARDED_BY(thread_checker_); + webrtc::VideoBitrateAllocatorFactory* const bitrate_allocator_factory_ + RTC_GUARDED_BY(thread_checker_); + std::vector<VideoCodecSettings> recv_codecs_ RTC_GUARDED_BY(thread_checker_); + std::vector<webrtc::RtpExtension> recv_rtp_extensions_ + RTC_GUARDED_BY(thread_checker_); + // See reason for keeping track of the FlexFEC payload type separately in + // comment in WebRtcVideoChannel::ChangedRecvParameters. + int recv_flexfec_payload_type_ RTC_GUARDED_BY(thread_checker_); + webrtc::BitrateConstraints bitrate_config_ RTC_GUARDED_BY(thread_checker_); + // TODO(deadbeef): Don't duplicate information between + // send_params/recv_params, rtp_extensions, options, etc. + VideoSendParameters send_params_ RTC_GUARDED_BY(thread_checker_); + VideoOptions default_send_options_ RTC_GUARDED_BY(thread_checker_); + VideoRecvParameters recv_params_ RTC_GUARDED_BY(thread_checker_); + int64_t last_stats_log_ms_ RTC_GUARDED_BY(thread_checker_); + const bool discard_unknown_ssrc_packets_ RTC_GUARDED_BY(thread_checker_); + // This is a stream param that comes from the remote description, but wasn't + // signaled with any a=ssrc lines. It holds information that was signaled + // before the unsignaled receive stream is created when the first packet is + // received. + StreamParams unsignaled_stream_params_ RTC_GUARDED_BY(thread_checker_); + // Per peer connection crypto options that last for the lifetime of the peer + // connection. + const webrtc::CryptoOptions crypto_options_ RTC_GUARDED_BY(thread_checker_); + + // Optional frame transformer set on unsignaled streams. + rtc::scoped_refptr<webrtc::FrameTransformerInterface> + unsignaled_frame_transformer_ RTC_GUARDED_BY(thread_checker_); + + // Buffer for unhandled packets. + std::unique_ptr<UnhandledPacketsBuffer> unknown_ssrc_packet_buffer_ + RTC_GUARDED_BY(thread_checker_); + + bool allow_codec_switching_ = false; + absl::optional<EncoderSwitchRequestCallback::Config> + requested_encoder_switch_; + + // In order for the |invoker_| to protect other members from being destructed + // as they are used in asynchronous tasks it has to be destructed first. + rtc::AsyncInvoker invoker_; +}; + +class EncoderStreamFactory + : public webrtc::VideoEncoderConfig::VideoStreamFactoryInterface { + public: + EncoderStreamFactory(std::string codec_name, + int max_qp, + bool is_screenshare, + bool conference_mode); + + private: + std::vector<webrtc::VideoStream> CreateEncoderStreams( + int width, + int height, + const webrtc::VideoEncoderConfig& encoder_config) override; + + std::vector<webrtc::VideoStream> CreateDefaultVideoStreams( + int width, + int height, + const webrtc::VideoEncoderConfig& encoder_config, + const absl::optional<webrtc::DataRate>& experimental_min_bitrate) const; + + std::vector<webrtc::VideoStream> + CreateSimulcastOrConfereceModeScreenshareStreams( + int width, + int height, + const webrtc::VideoEncoderConfig& encoder_config, + const absl::optional<webrtc::DataRate>& experimental_min_bitrate) const; + + const std::string codec_name_; + const int max_qp_; + const bool is_screenshare_; + // Allows a screenshare specific configuration, which enables temporal + // layering and various settings. + const bool conference_mode_; +}; + +} // namespace cricket + +#endif // MEDIA_ENGINE_WEBRTC_VIDEO_ENGINE_H_ diff --git a/media/engine/webrtc_video_engine_unittest.cc b/media/engine/webrtc_video_engine_unittest.cc new file mode 100644 index 0000000000..ce36073449 --- /dev/null +++ b/media/engine/webrtc_video_engine_unittest.cc @@ -0,0 +1,8394 @@ +/* + * Copyright (c) 2004 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "media/engine/webrtc_video_engine.h" + +#include <map> +#include <memory> +#include <string> +#include <utility> +#include <vector> + +#include "absl/algorithm/container.h" +#include "absl/memory/memory.h" +#include "absl/strings/match.h" +#include "api/rtc_event_log/rtc_event_log.h" +#include "api/rtp_parameters.h" +#include "api/task_queue/default_task_queue_factory.h" +#include "api/test/mock_video_bitrate_allocator.h" +#include "api/test/mock_video_bitrate_allocator_factory.h" +#include "api/test/mock_video_decoder_factory.h" +#include "api/test/mock_video_encoder_factory.h" +#include "api/test/video/function_video_decoder_factory.h" +#include "api/transport/field_trial_based_config.h" +#include "api/transport/media/media_transport_config.h" +#include "api/units/time_delta.h" +#include "api/video/builtin_video_bitrate_allocator_factory.h" +#include "api/video/i420_buffer.h" +#include "api/video/video_bitrate_allocation.h" +#include "api/video_codecs/builtin_video_decoder_factory.h" +#include "api/video_codecs/builtin_video_encoder_factory.h" +#include "api/video_codecs/sdp_video_format.h" +#include "api/video_codecs/video_decoder_factory.h" +#include "api/video_codecs/video_encoder.h" +#include "api/video_codecs/video_encoder_factory.h" +#include "call/flexfec_receive_stream.h" +#include "common_video/h264/profile_level_id.h" +#include "media/base/fake_frame_source.h" +#include "media/base/fake_network_interface.h" +#include "media/base/fake_video_renderer.h" +#include "media/base/media_constants.h" +#include "media/base/rtp_utils.h" +#include "media/base/test_utils.h" +#include "media/engine/constants.h" +#include "media/engine/fake_webrtc_call.h" +#include "media/engine/fake_webrtc_video_engine.h" +#include "media/engine/simulcast.h" +#include "media/engine/webrtc_voice_engine.h" +#include "rtc_base/arraysize.h" +#include "rtc_base/experiments/min_video_bitrate_experiment.h" +#include "rtc_base/fake_clock.h" +#include "rtc_base/gunit.h" +#include "rtc_base/numerics/safe_conversions.h" +#include "rtc_base/time_utils.h" +#include "test/fake_decoder.h" +#include "test/field_trial.h" +#include "test/frame_forwarder.h" +#include "test/gmock.h" +#include "test/rtp_header_parser.h" + +using ::testing::_; +using ::testing::Contains; +using ::testing::Each; +using ::testing::ElementsAreArray; +using ::testing::Eq; +using ::testing::Field; +using ::testing::IsEmpty; +using ::testing::Pair; +using ::testing::Return; +using ::testing::SizeIs; +using ::testing::StrNe; +using ::testing::Values; +using webrtc::BitrateConstraints; +using webrtc::RtpExtension; + +namespace { +static const int kDefaultQpMax = 56; + +static const uint8_t kRedRtxPayloadType = 125; + +static const uint32_t kTimeout = 5000U; +static const uint32_t kSsrc = 1234u; +static const uint32_t kSsrcs4[] = {1, 2, 3, 4}; +static const int kVideoWidth = 640; +static const int kVideoHeight = 360; +static const int kFramerate = 30; + +static const uint32_t kSsrcs1[] = {1}; +static const uint32_t kSsrcs3[] = {1, 2, 3}; +static const uint32_t kRtxSsrcs1[] = {4}; +static const uint32_t kFlexfecSsrc = 5; +static const uint32_t kIncomingUnsignalledSsrc = 0xC0FFEE; + +constexpr uint32_t kRtpHeaderSize = 12; + +static const char kUnsupportedExtensionName[] = + "urn:ietf:params:rtp-hdrext:unsupported"; + +cricket::VideoCodec RemoveFeedbackParams(cricket::VideoCodec&& codec) { + codec.feedback_params = cricket::FeedbackParams(); + return std::move(codec); +} + +void VerifyCodecHasDefaultFeedbackParams(const cricket::VideoCodec& codec, + bool lntf_expected) { + EXPECT_EQ(lntf_expected, + codec.HasFeedbackParam(cricket::FeedbackParam( + cricket::kRtcpFbParamLntf, cricket::kParamValueEmpty))); + EXPECT_TRUE(codec.HasFeedbackParam(cricket::FeedbackParam( + cricket::kRtcpFbParamNack, cricket::kParamValueEmpty))); + EXPECT_TRUE(codec.HasFeedbackParam(cricket::FeedbackParam( + cricket::kRtcpFbParamNack, cricket::kRtcpFbNackParamPli))); + EXPECT_TRUE(codec.HasFeedbackParam(cricket::FeedbackParam( + cricket::kRtcpFbParamRemb, cricket::kParamValueEmpty))); + EXPECT_TRUE(codec.HasFeedbackParam(cricket::FeedbackParam( + cricket::kRtcpFbParamTransportCc, cricket::kParamValueEmpty))); + EXPECT_TRUE(codec.HasFeedbackParam(cricket::FeedbackParam( + cricket::kRtcpFbParamCcm, cricket::kRtcpFbCcmParamFir))); +} + +// Return true if any codec in |codecs| is an RTX codec with associated payload +// type |payload_type|. +bool HasRtxCodec(const std::vector<cricket::VideoCodec>& codecs, + int payload_type) { + for (const cricket::VideoCodec& codec : codecs) { + int associated_payload_type; + if (absl::EqualsIgnoreCase(codec.name.c_str(), "rtx") && + codec.GetParam(cricket::kCodecParamAssociatedPayloadType, + &associated_payload_type) && + associated_payload_type == payload_type) { + return true; + } + } + return false; +} + +// TODO(nisse): Duplicated in call.cc. +const int* FindKeyByValue(const std::map<int, int>& m, int v) { + for (const auto& kv : m) { + if (kv.second == v) + return &kv.first; + } + return nullptr; +} + +bool HasRtxReceiveAssociation(const webrtc::VideoReceiveStream::Config& config, + int payload_type) { + return FindKeyByValue(config.rtp.rtx_associated_payload_types, + payload_type) != nullptr; +} + +// Check that there's an Rtx payload type for each decoder. +bool VerifyRtxReceiveAssociations( + const webrtc::VideoReceiveStream::Config& config) { + for (const auto& decoder : config.decoders) { + if (!HasRtxReceiveAssociation(config, decoder.payload_type)) + return false; + } + return true; +} + +rtc::scoped_refptr<webrtc::VideoFrameBuffer> CreateBlackFrameBuffer( + int width, + int height) { + rtc::scoped_refptr<webrtc::I420Buffer> buffer = + webrtc::I420Buffer::Create(width, height); + webrtc::I420Buffer::SetBlack(buffer); + return buffer; +} + +void VerifySendStreamHasRtxTypes(const webrtc::VideoSendStream::Config& config, + const std::map<int, int>& rtx_types) { + std::map<int, int>::const_iterator it; + it = rtx_types.find(config.rtp.payload_type); + EXPECT_TRUE(it != rtx_types.end() && + it->second == config.rtp.rtx.payload_type); + + if (config.rtp.ulpfec.red_rtx_payload_type != -1) { + it = rtx_types.find(config.rtp.ulpfec.red_payload_type); + EXPECT_TRUE(it != rtx_types.end() && + it->second == config.rtp.ulpfec.red_rtx_payload_type); + } +} + +cricket::MediaConfig GetMediaConfig() { + cricket::MediaConfig media_config; + media_config.video.enable_cpu_adaptation = false; + return media_config; +} + +// Values from GetMaxDefaultVideoBitrateKbps in webrtcvideoengine.cc. +int GetMaxDefaultBitrateBps(size_t width, size_t height) { + if (width * height <= 320 * 240) { + return 600000; + } else if (width * height <= 640 * 480) { + return 1700000; + } else if (width * height <= 960 * 540) { + return 2000000; + } else { + return 2500000; + } +} + +class MockVideoSource : public rtc::VideoSourceInterface<webrtc::VideoFrame> { + public: + MOCK_METHOD2(AddOrUpdateSink, + void(rtc::VideoSinkInterface<webrtc::VideoFrame>* sink, + const rtc::VideoSinkWants& wants)); + MOCK_METHOD1(RemoveSink, + void(rtc::VideoSinkInterface<webrtc::VideoFrame>* sink)); +}; + +} // namespace + +#define EXPECT_FRAME_WAIT(c, w, h, t) \ + EXPECT_EQ_WAIT((c), renderer_.num_rendered_frames(), (t)); \ + EXPECT_EQ((w), renderer_.width()); \ + EXPECT_EQ((h), renderer_.height()); \ + EXPECT_EQ(0, renderer_.errors()); + +#define EXPECT_FRAME_ON_RENDERER_WAIT(r, c, w, h, t) \ + EXPECT_EQ_WAIT((c), (r).num_rendered_frames(), (t)); \ + EXPECT_EQ((w), (r).width()); \ + EXPECT_EQ((h), (r).height()); \ + EXPECT_EQ(0, (r).errors()); + +namespace cricket { +class WebRtcVideoEngineTest : public ::testing::Test { + public: + WebRtcVideoEngineTest() : WebRtcVideoEngineTest("") {} + explicit WebRtcVideoEngineTest(const std::string& field_trials) + : override_field_trials_( + field_trials.empty() + ? nullptr + : std::make_unique<webrtc::test::ScopedFieldTrials>( + field_trials)), + task_queue_factory_(webrtc::CreateDefaultTaskQueueFactory()), + call_(webrtc::Call::Create([&] { + webrtc::Call::Config call_config(&event_log_); + call_config.task_queue_factory = task_queue_factory_.get(); + call_config.trials = &field_trials_; + return call_config; + }())), + encoder_factory_(new cricket::FakeWebRtcVideoEncoderFactory), + decoder_factory_(new cricket::FakeWebRtcVideoDecoderFactory), + video_bitrate_allocator_factory_( + webrtc::CreateBuiltinVideoBitrateAllocatorFactory()), + engine_(std::unique_ptr<cricket::FakeWebRtcVideoEncoderFactory>( + encoder_factory_), + std::unique_ptr<cricket::FakeWebRtcVideoDecoderFactory>( + decoder_factory_)) { + // Ensure fake clock doesn't return 0, which will cause some initializations + // fail inside RTP senders. + fake_clock_.AdvanceTime(webrtc::TimeDelta::Micros(1)); + } + + protected: + void AssignDefaultAptRtxTypes(); + void AssignDefaultCodec(); + + // Find the index of the codec in the engine with the given name. The codec + // must be present. + size_t GetEngineCodecIndex(const std::string& name) const; + + // Find the codec in the engine with the given name. The codec must be + // present. + cricket::VideoCodec GetEngineCodec(const std::string& name) const; + void AddSupportedVideoCodecType(const std::string& name); + VideoMediaChannel* SetSendParamsWithAllSupportedCodecs(); + + VideoMediaChannel* SetRecvParamsWithSupportedCodecs( + const std::vector<VideoCodec>& codecs); + + void ExpectRtpCapabilitySupport(const char* uri, bool supported) const; + + // Has to be the first one, so it is initialized before the call or there is a + // race condition in the clock access. + rtc::ScopedFakeClock fake_clock_; + std::unique_ptr<webrtc::test::ScopedFieldTrials> override_field_trials_; + webrtc::FieldTrialBasedConfig field_trials_; + webrtc::RtcEventLogNull event_log_; + std::unique_ptr<webrtc::TaskQueueFactory> task_queue_factory_; + // Used in WebRtcVideoEngineVoiceTest, but defined here so it's properly + // initialized when the constructor is called. + std::unique_ptr<webrtc::Call> call_; + cricket::FakeWebRtcVideoEncoderFactory* encoder_factory_; + cricket::FakeWebRtcVideoDecoderFactory* decoder_factory_; + std::unique_ptr<webrtc::VideoBitrateAllocatorFactory> + video_bitrate_allocator_factory_; + WebRtcVideoEngine engine_; + VideoCodec default_codec_; + std::map<int, int> default_apt_rtx_types_; +}; + +TEST_F(WebRtcVideoEngineTest, DefaultRtxCodecHasAssociatedPayloadTypeSet) { + encoder_factory_->AddSupportedVideoCodecType("VP8"); + AssignDefaultCodec(); + + std::vector<VideoCodec> engine_codecs = engine_.send_codecs(); + for (size_t i = 0; i < engine_codecs.size(); ++i) { + if (engine_codecs[i].name != kRtxCodecName) + continue; + int associated_payload_type; + EXPECT_TRUE(engine_codecs[i].GetParam(kCodecParamAssociatedPayloadType, + &associated_payload_type)); + EXPECT_EQ(default_codec_.id, associated_payload_type); + return; + } + FAIL() << "No RTX codec found among default codecs."; +} + +TEST_F(WebRtcVideoEngineTest, SupportsTimestampOffsetHeaderExtension) { + ExpectRtpCapabilitySupport(RtpExtension::kTimestampOffsetUri, true); +} + +TEST_F(WebRtcVideoEngineTest, SupportsAbsoluteSenderTimeHeaderExtension) { + ExpectRtpCapabilitySupport(RtpExtension::kAbsSendTimeUri, true); +} + +TEST_F(WebRtcVideoEngineTest, SupportsTransportSequenceNumberHeaderExtension) { + ExpectRtpCapabilitySupport(RtpExtension::kTransportSequenceNumberUri, true); +} + +TEST_F(WebRtcVideoEngineTest, SupportsVideoRotationHeaderExtension) { + ExpectRtpCapabilitySupport(RtpExtension::kVideoRotationUri, true); +} + +TEST_F(WebRtcVideoEngineTest, SupportsPlayoutDelayHeaderExtension) { + ExpectRtpCapabilitySupport(RtpExtension::kPlayoutDelayUri, true); +} + +TEST_F(WebRtcVideoEngineTest, SupportsVideoContentTypeHeaderExtension) { + ExpectRtpCapabilitySupport(RtpExtension::kVideoContentTypeUri, true); +} + +TEST_F(WebRtcVideoEngineTest, SupportsVideoTimingHeaderExtension) { + ExpectRtpCapabilitySupport(RtpExtension::kVideoTimingUri, true); +} + +TEST_F(WebRtcVideoEngineTest, SupportsFrameMarkingHeaderExtension) { + ExpectRtpCapabilitySupport(RtpExtension::kFrameMarkingUri, true); +} + +TEST_F(WebRtcVideoEngineTest, SupportsColorSpaceHeaderExtension) { + ExpectRtpCapabilitySupport(RtpExtension::kColorSpaceUri, true); +} + +TEST_F(WebRtcVideoEngineTest, AdvertiseGenericDescriptor00) { + ExpectRtpCapabilitySupport(RtpExtension::kGenericFrameDescriptorUri00, false); +} + +class WebRtcVideoEngineTestWithGenericDescriptor + : public WebRtcVideoEngineTest { + public: + WebRtcVideoEngineTestWithGenericDescriptor() + : WebRtcVideoEngineTest("WebRTC-GenericDescriptorAdvertised/Enabled/") {} +}; + +TEST_F(WebRtcVideoEngineTestWithGenericDescriptor, + AdvertiseGenericDescriptor00) { + ExpectRtpCapabilitySupport(RtpExtension::kGenericFrameDescriptorUri00, true); +} + +TEST_F(WebRtcVideoEngineTest, CVOSetHeaderExtensionBeforeCapturer) { + // Allocate the source first to prevent early destruction before channel's + // dtor is called. + ::testing::NiceMock<MockVideoSource> video_source; + + AddSupportedVideoCodecType("VP8"); + + std::unique_ptr<VideoMediaChannel> channel( + SetSendParamsWithAllSupportedCodecs()); + EXPECT_TRUE(channel->AddSendStream(StreamParams::CreateLegacy(kSsrc))); + + // Add CVO extension. + const int id = 1; + cricket::VideoSendParameters parameters; + parameters.codecs.push_back(GetEngineCodec("VP8")); + parameters.extensions.push_back( + RtpExtension(RtpExtension::kVideoRotationUri, id)); + EXPECT_TRUE(channel->SetSendParameters(parameters)); + + EXPECT_CALL( + video_source, + AddOrUpdateSink(_, Field(&rtc::VideoSinkWants::rotation_applied, false))); + // Set capturer. + EXPECT_TRUE(channel->SetVideoSend(kSsrc, nullptr, &video_source)); + + // Verify capturer has turned off applying rotation. + ::testing::Mock::VerifyAndClear(&video_source); + + // Verify removing header extension turns on applying rotation. + parameters.extensions.clear(); + EXPECT_CALL( + video_source, + AddOrUpdateSink(_, Field(&rtc::VideoSinkWants::rotation_applied, true))); + + EXPECT_TRUE(channel->SetSendParameters(parameters)); +} + +TEST_F(WebRtcVideoEngineTest, CVOSetHeaderExtensionBeforeAddSendStream) { + // Allocate the source first to prevent early destruction before channel's + // dtor is called. + ::testing::NiceMock<MockVideoSource> video_source; + + AddSupportedVideoCodecType("VP8"); + + std::unique_ptr<VideoMediaChannel> channel( + SetSendParamsWithAllSupportedCodecs()); + // Add CVO extension. + const int id = 1; + cricket::VideoSendParameters parameters; + parameters.codecs.push_back(GetEngineCodec("VP8")); + parameters.extensions.push_back( + RtpExtension(RtpExtension::kVideoRotationUri, id)); + EXPECT_TRUE(channel->SetSendParameters(parameters)); + EXPECT_TRUE(channel->AddSendStream(StreamParams::CreateLegacy(kSsrc))); + + // Set source. + EXPECT_CALL( + video_source, + AddOrUpdateSink(_, Field(&rtc::VideoSinkWants::rotation_applied, false))); + EXPECT_TRUE(channel->SetVideoSend(kSsrc, nullptr, &video_source)); +} + +TEST_F(WebRtcVideoEngineTest, CVOSetHeaderExtensionAfterCapturer) { + ::testing::NiceMock<MockVideoSource> video_source; + + AddSupportedVideoCodecType("VP8"); + AddSupportedVideoCodecType("VP9"); + + std::unique_ptr<VideoMediaChannel> channel( + SetSendParamsWithAllSupportedCodecs()); + EXPECT_TRUE(channel->AddSendStream(StreamParams::CreateLegacy(kSsrc))); + + // Set capturer. + EXPECT_CALL( + video_source, + AddOrUpdateSink(_, Field(&rtc::VideoSinkWants::rotation_applied, true))); + EXPECT_TRUE(channel->SetVideoSend(kSsrc, nullptr, &video_source)); + + // Verify capturer has turned on applying rotation. + ::testing::Mock::VerifyAndClear(&video_source); + + // Add CVO extension. + const int id = 1; + cricket::VideoSendParameters parameters; + parameters.codecs.push_back(GetEngineCodec("VP8")); + parameters.codecs.push_back(GetEngineCodec("VP9")); + parameters.extensions.push_back( + RtpExtension(RtpExtension::kVideoRotationUri, id)); + // Also remove the first codec to trigger a codec change as well. + parameters.codecs.erase(parameters.codecs.begin()); + EXPECT_CALL( + video_source, + AddOrUpdateSink(_, Field(&rtc::VideoSinkWants::rotation_applied, false))); + EXPECT_TRUE(channel->SetSendParameters(parameters)); + + // Verify capturer has turned off applying rotation. + ::testing::Mock::VerifyAndClear(&video_source); + + // Verify removing header extension turns on applying rotation. + parameters.extensions.clear(); + EXPECT_CALL( + video_source, + AddOrUpdateSink(_, Field(&rtc::VideoSinkWants::rotation_applied, true))); + EXPECT_TRUE(channel->SetSendParameters(parameters)); +} + +TEST_F(WebRtcVideoEngineTest, SetSendFailsBeforeSettingCodecs) { + AddSupportedVideoCodecType("VP8"); + + std::unique_ptr<VideoMediaChannel> channel(engine_.CreateMediaChannel( + call_.get(), GetMediaConfig(), VideoOptions(), webrtc::CryptoOptions(), + video_bitrate_allocator_factory_.get())); + + EXPECT_TRUE(channel->AddSendStream(StreamParams::CreateLegacy(123))); + + EXPECT_FALSE(channel->SetSend(true)) + << "Channel should not start without codecs."; + EXPECT_TRUE(channel->SetSend(false)) + << "Channel should be stoppable even without set codecs."; +} + +TEST_F(WebRtcVideoEngineTest, GetStatsWithoutSendCodecsSetDoesNotCrash) { + AddSupportedVideoCodecType("VP8"); + + std::unique_ptr<VideoMediaChannel> channel(engine_.CreateMediaChannel( + call_.get(), GetMediaConfig(), VideoOptions(), webrtc::CryptoOptions(), + video_bitrate_allocator_factory_.get())); + EXPECT_TRUE(channel->AddSendStream(StreamParams::CreateLegacy(123))); + VideoMediaInfo info; + channel->GetStats(&info); +} + +TEST_F(WebRtcVideoEngineTest, UseFactoryForVp8WhenSupported) { + AddSupportedVideoCodecType("VP8"); + + std::unique_ptr<VideoMediaChannel> channel( + SetSendParamsWithAllSupportedCodecs()); + channel->OnReadyToSend(true); + + EXPECT_TRUE( + channel->AddSendStream(cricket::StreamParams::CreateLegacy(kSsrc))); + EXPECT_EQ(0, encoder_factory_->GetNumCreatedEncoders()); + EXPECT_TRUE(channel->SetSend(true)); + webrtc::test::FrameForwarder frame_forwarder; + cricket::FakeFrameSource frame_source(1280, 720, + rtc::kNumMicrosecsPerSec / 30); + EXPECT_TRUE(channel->SetVideoSend(kSsrc, nullptr, &frame_forwarder)); + frame_forwarder.IncomingCapturedFrame(frame_source.GetFrame()); + // Sending one frame will have allocate the encoder. + ASSERT_TRUE(encoder_factory_->WaitForCreatedVideoEncoders(1)); + EXPECT_TRUE_WAIT(encoder_factory_->encoders()[0]->GetNumEncodedFrames() > 0, + kTimeout); + + int num_created_encoders = encoder_factory_->GetNumCreatedEncoders(); + EXPECT_EQ(num_created_encoders, 1); + + // Setting codecs of the same type should not reallocate any encoders + // (expecting a no-op). + cricket::VideoSendParameters parameters; + parameters.codecs.push_back(GetEngineCodec("VP8")); + EXPECT_TRUE(channel->SetSendParameters(parameters)); + EXPECT_EQ(num_created_encoders, encoder_factory_->GetNumCreatedEncoders()); + + // Remove stream previously added to free the external encoder instance. + EXPECT_TRUE(channel->RemoveSendStream(kSsrc)); + EXPECT_EQ(0u, encoder_factory_->encoders().size()); +} + +// Test that when an encoder factory supports H264, we add an RTX +// codec for it. +// TODO(deadbeef): This test should be updated if/when we start +// adding RTX codecs for unrecognized codec names. +TEST_F(WebRtcVideoEngineTest, RtxCodecAddedForH264Codec) { + using webrtc::H264::kLevel1; + using webrtc::H264::ProfileLevelId; + using webrtc::H264::ProfileLevelIdToString; + webrtc::SdpVideoFormat h264_constrained_baseline("H264"); + h264_constrained_baseline.parameters[kH264FmtpProfileLevelId] = + *ProfileLevelIdToString( + ProfileLevelId(webrtc::H264::kProfileConstrainedBaseline, kLevel1)); + webrtc::SdpVideoFormat h264_constrained_high("H264"); + h264_constrained_high.parameters[kH264FmtpProfileLevelId] = + *ProfileLevelIdToString( + ProfileLevelId(webrtc::H264::kProfileConstrainedHigh, kLevel1)); + webrtc::SdpVideoFormat h264_high("H264"); + h264_high.parameters[kH264FmtpProfileLevelId] = *ProfileLevelIdToString( + ProfileLevelId(webrtc::H264::kProfileHigh, kLevel1)); + + encoder_factory_->AddSupportedVideoCodec(h264_constrained_baseline); + encoder_factory_->AddSupportedVideoCodec(h264_constrained_high); + encoder_factory_->AddSupportedVideoCodec(h264_high); + + // First figure out what payload types the test codecs got assigned. + const std::vector<cricket::VideoCodec> codecs = engine_.send_codecs(); + // Now search for RTX codecs for them. Expect that they all have associated + // RTX codecs. + EXPECT_TRUE(HasRtxCodec( + codecs, + FindMatchingCodec(codecs, cricket::VideoCodec(h264_constrained_baseline)) + ->id)); + EXPECT_TRUE(HasRtxCodec( + codecs, + FindMatchingCodec(codecs, cricket::VideoCodec(h264_constrained_high)) + ->id)); + EXPECT_TRUE(HasRtxCodec( + codecs, FindMatchingCodec(codecs, cricket::VideoCodec(h264_high))->id)); +} + +#if defined(RTC_ENABLE_VP9) +TEST_F(WebRtcVideoEngineTest, CanConstructDecoderForVp9EncoderFactory) { + AddSupportedVideoCodecType("VP9"); + + std::unique_ptr<VideoMediaChannel> channel( + SetSendParamsWithAllSupportedCodecs()); + + EXPECT_TRUE( + channel->AddRecvStream(cricket::StreamParams::CreateLegacy(kSsrc))); +} +#endif // defined(RTC_ENABLE_VP9) + +TEST_F(WebRtcVideoEngineTest, PropagatesInputFrameTimestamp) { + AddSupportedVideoCodecType("VP8"); + FakeCall* fake_call = new FakeCall(); + call_.reset(fake_call); + std::unique_ptr<VideoMediaChannel> channel( + SetSendParamsWithAllSupportedCodecs()); + + EXPECT_TRUE( + channel->AddSendStream(cricket::StreamParams::CreateLegacy(kSsrc))); + + webrtc::test::FrameForwarder frame_forwarder; + cricket::FakeFrameSource frame_source(1280, 720, + rtc::kNumMicrosecsPerSec / 60); + EXPECT_TRUE(channel->SetVideoSend(kSsrc, nullptr, &frame_forwarder)); + channel->SetSend(true); + + FakeVideoSendStream* stream = fake_call->GetVideoSendStreams()[0]; + + frame_forwarder.IncomingCapturedFrame(frame_source.GetFrame()); + int64_t last_timestamp = stream->GetLastTimestamp(); + for (int i = 0; i < 10; i++) { + frame_forwarder.IncomingCapturedFrame(frame_source.GetFrame()); + int64_t timestamp = stream->GetLastTimestamp(); + int64_t interval = timestamp - last_timestamp; + + // Precision changes from nanosecond to millisecond. + // Allow error to be no more than 1. + EXPECT_NEAR(cricket::VideoFormat::FpsToInterval(60) / 1E6, interval, 1); + + last_timestamp = timestamp; + } + + frame_forwarder.IncomingCapturedFrame( + frame_source.GetFrame(1280, 720, webrtc::VideoRotation::kVideoRotation_0, + rtc::kNumMicrosecsPerSec / 30)); + last_timestamp = stream->GetLastTimestamp(); + for (int i = 0; i < 10; i++) { + frame_forwarder.IncomingCapturedFrame(frame_source.GetFrame( + 1280, 720, webrtc::VideoRotation::kVideoRotation_0, + rtc::kNumMicrosecsPerSec / 30)); + int64_t timestamp = stream->GetLastTimestamp(); + int64_t interval = timestamp - last_timestamp; + + // Precision changes from nanosecond to millisecond. + // Allow error to be no more than 1. + EXPECT_NEAR(cricket::VideoFormat::FpsToInterval(30) / 1E6, interval, 1); + + last_timestamp = timestamp; + } + + // Remove stream previously added to free the external encoder instance. + EXPECT_TRUE(channel->RemoveSendStream(kSsrc)); +} + +void WebRtcVideoEngineTest::AssignDefaultAptRtxTypes() { + std::vector<VideoCodec> engine_codecs = engine_.send_codecs(); + RTC_DCHECK(!engine_codecs.empty()); + for (const cricket::VideoCodec& codec : engine_codecs) { + if (codec.name == "rtx") { + int associated_payload_type; + if (codec.GetParam(kCodecParamAssociatedPayloadType, + &associated_payload_type)) { + default_apt_rtx_types_[associated_payload_type] = codec.id; + } + } + } +} + +void WebRtcVideoEngineTest::AssignDefaultCodec() { + std::vector<VideoCodec> engine_codecs = engine_.send_codecs(); + RTC_DCHECK(!engine_codecs.empty()); + bool codec_set = false; + for (const cricket::VideoCodec& codec : engine_codecs) { + if (!codec_set && codec.name != "rtx" && codec.name != "red" && + codec.name != "ulpfec") { + default_codec_ = codec; + codec_set = true; + } + } + + RTC_DCHECK(codec_set); +} + +size_t WebRtcVideoEngineTest::GetEngineCodecIndex( + const std::string& name) const { + const std::vector<cricket::VideoCodec> codecs = engine_.send_codecs(); + for (size_t i = 0; i < codecs.size(); ++i) { + const cricket::VideoCodec engine_codec = codecs[i]; + if (!absl::EqualsIgnoreCase(name, engine_codec.name)) + continue; + // The tests only use H264 Constrained Baseline. Make sure we don't return + // an internal H264 codec from the engine with a different H264 profile. + if (absl::EqualsIgnoreCase(name.c_str(), kH264CodecName)) { + const absl::optional<webrtc::H264::ProfileLevelId> profile_level_id = + webrtc::H264::ParseSdpProfileLevelId(engine_codec.params); + if (profile_level_id->profile != + webrtc::H264::kProfileConstrainedBaseline) { + continue; + } + } + return i; + } + // This point should never be reached. + ADD_FAILURE() << "Unrecognized codec name: " << name; + return -1; +} + +cricket::VideoCodec WebRtcVideoEngineTest::GetEngineCodec( + const std::string& name) const { + return engine_.send_codecs()[GetEngineCodecIndex(name)]; +} + +void WebRtcVideoEngineTest::AddSupportedVideoCodecType( + const std::string& name) { + encoder_factory_->AddSupportedVideoCodecType(name); + decoder_factory_->AddSupportedVideoCodecType(name); +} + +VideoMediaChannel* +WebRtcVideoEngineTest::SetSendParamsWithAllSupportedCodecs() { + VideoMediaChannel* channel = engine_.CreateMediaChannel( + call_.get(), GetMediaConfig(), VideoOptions(), webrtc::CryptoOptions(), + video_bitrate_allocator_factory_.get()); + cricket::VideoSendParameters parameters; + // We need to look up the codec in the engine to get the correct payload type. + for (const webrtc::SdpVideoFormat& format : + encoder_factory_->GetSupportedFormats()) { + cricket::VideoCodec engine_codec = GetEngineCodec(format.name); + if (!absl::c_linear_search(parameters.codecs, engine_codec)) { + parameters.codecs.push_back(engine_codec); + } + } + + EXPECT_TRUE(channel->SetSendParameters(parameters)); + + return channel; +} + +VideoMediaChannel* WebRtcVideoEngineTest::SetRecvParamsWithSupportedCodecs( + const std::vector<VideoCodec>& codecs) { + VideoMediaChannel* channel = engine_.CreateMediaChannel( + call_.get(), GetMediaConfig(), VideoOptions(), webrtc::CryptoOptions(), + video_bitrate_allocator_factory_.get()); + cricket::VideoRecvParameters parameters; + parameters.codecs = codecs; + EXPECT_TRUE(channel->SetRecvParameters(parameters)); + + return channel; +} + +void WebRtcVideoEngineTest::ExpectRtpCapabilitySupport(const char* uri, + bool supported) const { + const std::vector<webrtc::RtpExtension> header_extensions = + GetDefaultEnabledRtpHeaderExtensions(engine_); + if (supported) { + EXPECT_THAT(header_extensions, Contains(Field(&RtpExtension::uri, uri))); + } else { + EXPECT_THAT(header_extensions, Each(Field(&RtpExtension::uri, StrNe(uri)))); + } +} + +TEST_F(WebRtcVideoEngineTest, UsesSimulcastAdapterForVp8Factories) { + AddSupportedVideoCodecType("VP8"); + + std::unique_ptr<VideoMediaChannel> channel( + SetSendParamsWithAllSupportedCodecs()); + + std::vector<uint32_t> ssrcs = MAKE_VECTOR(kSsrcs3); + + EXPECT_TRUE(channel->AddSendStream(CreateSimStreamParams("cname", ssrcs))); + EXPECT_TRUE(channel->SetSend(true)); + + webrtc::test::FrameForwarder frame_forwarder; + cricket::FakeFrameSource frame_source(1280, 720, + rtc::kNumMicrosecsPerSec / 60); + EXPECT_TRUE(channel->SetVideoSend(ssrcs.front(), nullptr, &frame_forwarder)); + frame_forwarder.IncomingCapturedFrame(frame_source.GetFrame()); + + ASSERT_TRUE(encoder_factory_->WaitForCreatedVideoEncoders(2)); + + // Verify that encoders are configured for simulcast through adapter + // (increasing resolution and only configured to send one stream each). + int prev_width = -1; + for (size_t i = 0; i < encoder_factory_->encoders().size(); ++i) { + ASSERT_TRUE(encoder_factory_->encoders()[i]->WaitForInitEncode()); + webrtc::VideoCodec codec_settings = + encoder_factory_->encoders()[i]->GetCodecSettings(); + EXPECT_EQ(0, codec_settings.numberOfSimulcastStreams); + EXPECT_GT(codec_settings.width, prev_width); + prev_width = codec_settings.width; + } + + EXPECT_TRUE(channel->SetVideoSend(ssrcs.front(), nullptr, nullptr)); + + channel.reset(); + ASSERT_EQ(0u, encoder_factory_->encoders().size()); +} + +TEST_F(WebRtcVideoEngineTest, ChannelWithH264CanChangeToVp8) { + AddSupportedVideoCodecType("VP8"); + AddSupportedVideoCodecType("H264"); + + // Frame source. + webrtc::test::FrameForwarder frame_forwarder; + cricket::FakeFrameSource frame_source(1280, 720, + rtc::kNumMicrosecsPerSec / 30); + + std::unique_ptr<VideoMediaChannel> channel(engine_.CreateMediaChannel( + call_.get(), GetMediaConfig(), VideoOptions(), webrtc::CryptoOptions(), + video_bitrate_allocator_factory_.get())); + cricket::VideoSendParameters parameters; + parameters.codecs.push_back(GetEngineCodec("H264")); + EXPECT_TRUE(channel->SetSendParameters(parameters)); + + EXPECT_TRUE( + channel->AddSendStream(cricket::StreamParams::CreateLegacy(kSsrc))); + EXPECT_TRUE(channel->SetVideoSend(kSsrc, nullptr, &frame_forwarder)); + // Sending one frame will have allocate the encoder. + frame_forwarder.IncomingCapturedFrame(frame_source.GetFrame()); + + ASSERT_EQ_WAIT(1u, encoder_factory_->encoders().size(), kTimeout); + + cricket::VideoSendParameters new_parameters; + new_parameters.codecs.push_back(GetEngineCodec("VP8")); + EXPECT_TRUE(channel->SetSendParameters(new_parameters)); + + // Sending one frame will switch encoder. + frame_forwarder.IncomingCapturedFrame(frame_source.GetFrame()); + + EXPECT_EQ_WAIT(1u, encoder_factory_->encoders().size(), kTimeout); +} + +TEST_F(WebRtcVideoEngineTest, + UsesSimulcastAdapterForVp8WithCombinedVP8AndH264Factory) { + AddSupportedVideoCodecType("VP8"); + AddSupportedVideoCodecType("H264"); + + std::unique_ptr<VideoMediaChannel> channel(engine_.CreateMediaChannel( + call_.get(), GetMediaConfig(), VideoOptions(), webrtc::CryptoOptions(), + video_bitrate_allocator_factory_.get())); + cricket::VideoSendParameters parameters; + parameters.codecs.push_back(GetEngineCodec("VP8")); + EXPECT_TRUE(channel->SetSendParameters(parameters)); + + std::vector<uint32_t> ssrcs = MAKE_VECTOR(kSsrcs3); + + EXPECT_TRUE(channel->AddSendStream(CreateSimStreamParams("cname", ssrcs))); + EXPECT_TRUE(channel->SetSend(true)); + + // Send a fake frame, or else the media engine will configure the simulcast + // encoder adapter at a low-enough size that it'll only create a single + // encoder layer. + webrtc::test::FrameForwarder frame_forwarder; + cricket::FakeFrameSource frame_source(1280, 720, + rtc::kNumMicrosecsPerSec / 30); + EXPECT_TRUE(channel->SetVideoSend(ssrcs.front(), nullptr, &frame_forwarder)); + frame_forwarder.IncomingCapturedFrame(frame_source.GetFrame()); + + ASSERT_TRUE(encoder_factory_->WaitForCreatedVideoEncoders(2)); + ASSERT_TRUE(encoder_factory_->encoders()[0]->WaitForInitEncode()); + EXPECT_EQ(webrtc::kVideoCodecVP8, + encoder_factory_->encoders()[0]->GetCodecSettings().codecType); + + channel.reset(); + // Make sure DestroyVideoEncoder was called on the factory. + EXPECT_EQ(0u, encoder_factory_->encoders().size()); +} + +TEST_F(WebRtcVideoEngineTest, + DestroysNonSimulcastEncoderFromCombinedVP8AndH264Factory) { + AddSupportedVideoCodecType("VP8"); + AddSupportedVideoCodecType("H264"); + + std::unique_ptr<VideoMediaChannel> channel(engine_.CreateMediaChannel( + call_.get(), GetMediaConfig(), VideoOptions(), webrtc::CryptoOptions(), + video_bitrate_allocator_factory_.get())); + cricket::VideoSendParameters parameters; + parameters.codecs.push_back(GetEngineCodec("H264")); + EXPECT_TRUE(channel->SetSendParameters(parameters)); + + EXPECT_TRUE( + channel->AddSendStream(cricket::StreamParams::CreateLegacy(kSsrc))); + + // Send a frame of 720p. This should trigger a "real" encoder initialization. + webrtc::test::FrameForwarder frame_forwarder; + cricket::FakeFrameSource frame_source(1280, 720, + rtc::kNumMicrosecsPerSec / 30); + EXPECT_TRUE(channel->SetVideoSend(kSsrc, nullptr, &frame_forwarder)); + frame_forwarder.IncomingCapturedFrame(frame_source.GetFrame()); + ASSERT_TRUE(encoder_factory_->WaitForCreatedVideoEncoders(1)); + ASSERT_EQ(1u, encoder_factory_->encoders().size()); + ASSERT_TRUE(encoder_factory_->encoders()[0]->WaitForInitEncode()); + EXPECT_EQ(webrtc::kVideoCodecH264, + encoder_factory_->encoders()[0]->GetCodecSettings().codecType); + + channel.reset(); + // Make sure DestroyVideoEncoder was called on the factory. + ASSERT_EQ(0u, encoder_factory_->encoders().size()); +} + +TEST_F(WebRtcVideoEngineTest, SimulcastEnabledForH264BehindFieldTrial) { + RTC_DCHECK(!override_field_trials_); + override_field_trials_ = std::make_unique<webrtc::test::ScopedFieldTrials>( + "WebRTC-H264Simulcast/Enabled/"); + AddSupportedVideoCodecType("H264"); + + std::unique_ptr<VideoMediaChannel> channel(engine_.CreateMediaChannel( + call_.get(), GetMediaConfig(), VideoOptions(), webrtc::CryptoOptions(), + video_bitrate_allocator_factory_.get())); + cricket::VideoSendParameters parameters; + parameters.codecs.push_back(GetEngineCodec("H264")); + EXPECT_TRUE(channel->SetSendParameters(parameters)); + + const std::vector<uint32_t> ssrcs = MAKE_VECTOR(kSsrcs3); + EXPECT_TRUE( + channel->AddSendStream(cricket::CreateSimStreamParams("cname", ssrcs))); + + // Send a frame of 720p. This should trigger a "real" encoder initialization. + webrtc::test::FrameForwarder frame_forwarder; + cricket::FakeFrameSource frame_source(1280, 720, + rtc::kNumMicrosecsPerSec / 30); + EXPECT_TRUE(channel->SetVideoSend(ssrcs[0], nullptr, &frame_forwarder)); + frame_forwarder.IncomingCapturedFrame(frame_source.GetFrame()); + + ASSERT_TRUE(encoder_factory_->WaitForCreatedVideoEncoders(1)); + ASSERT_EQ(1u, encoder_factory_->encoders().size()); + FakeWebRtcVideoEncoder* encoder = encoder_factory_->encoders()[0]; + ASSERT_TRUE(encoder_factory_->encoders()[0]->WaitForInitEncode()); + EXPECT_EQ(webrtc::kVideoCodecH264, encoder->GetCodecSettings().codecType); + EXPECT_LT(1u, encoder->GetCodecSettings().numberOfSimulcastStreams); + EXPECT_TRUE(channel->SetVideoSend(ssrcs[0], nullptr, nullptr)); +} + +// Test that the FlexFEC field trial properly alters the output of +// WebRtcVideoEngine::codecs(), for an existing |engine_| object. +// +// TODO(brandtr): Remove this test, when the FlexFEC field trial is gone. +TEST_F(WebRtcVideoEngineTest, + Flexfec03SupportedAsInternalCodecBehindFieldTrial) { + encoder_factory_->AddSupportedVideoCodecType("VP8"); + + auto flexfec = Field("name", &VideoCodec::name, "flexfec-03"); + + // FlexFEC is not active without field trial. + EXPECT_THAT(engine_.send_codecs(), Not(Contains(flexfec))); + + // FlexFEC is active with field trial. + RTC_DCHECK(!override_field_trials_); + override_field_trials_ = std::make_unique<webrtc::test::ScopedFieldTrials>( + "WebRTC-FlexFEC-03-Advertised/Enabled/"); + EXPECT_THAT(engine_.send_codecs(), Contains(flexfec)); +} + +// Test that codecs are added in the order they are reported from the factory. +TEST_F(WebRtcVideoEngineTest, ReportSupportedCodecs) { + encoder_factory_->AddSupportedVideoCodecType("VP8"); + const char* kFakeCodecName = "FakeCodec"; + encoder_factory_->AddSupportedVideoCodecType(kFakeCodecName); + + // The last reported codec should appear after the first codec in the vector. + const size_t vp8_index = GetEngineCodecIndex("VP8"); + const size_t fake_codec_index = GetEngineCodecIndex(kFakeCodecName); + EXPECT_LT(vp8_index, fake_codec_index); +} + +// Test that a codec that was added after the engine was initialized +// does show up in the codec list after it was added. +TEST_F(WebRtcVideoEngineTest, ReportSupportedAddedCodec) { + const char* kFakeExternalCodecName1 = "FakeExternalCodec1"; + const char* kFakeExternalCodecName2 = "FakeExternalCodec2"; + + // Set up external encoder factory with first codec, and initialize engine. + encoder_factory_->AddSupportedVideoCodecType(kFakeExternalCodecName1); + + std::vector<cricket::VideoCodec> codecs_before(engine_.send_codecs()); + + // Add second codec. + encoder_factory_->AddSupportedVideoCodecType(kFakeExternalCodecName2); + std::vector<cricket::VideoCodec> codecs_after(engine_.send_codecs()); + // The codec itself and RTX should have been added. + EXPECT_EQ(codecs_before.size() + 2, codecs_after.size()); + + // Check that both fake codecs are present and that the second fake codec + // appears after the first fake codec. + const size_t fake_codec_index1 = GetEngineCodecIndex(kFakeExternalCodecName1); + const size_t fake_codec_index2 = GetEngineCodecIndex(kFakeExternalCodecName2); + EXPECT_LT(fake_codec_index1, fake_codec_index2); +} + +TEST_F(WebRtcVideoEngineTest, ReportRtxForExternalCodec) { + const char* kFakeCodecName = "FakeCodec"; + encoder_factory_->AddSupportedVideoCodecType(kFakeCodecName); + + const size_t fake_codec_index = GetEngineCodecIndex(kFakeCodecName); + EXPECT_EQ("rtx", engine_.send_codecs().at(fake_codec_index + 1).name); +} + +TEST_F(WebRtcVideoEngineTest, RegisterDecodersIfSupported) { + AddSupportedVideoCodecType("VP8"); + cricket::VideoRecvParameters parameters; + parameters.codecs.push_back(GetEngineCodec("VP8")); + + std::unique_ptr<VideoMediaChannel> channel( + SetRecvParamsWithSupportedCodecs(parameters.codecs)); + + EXPECT_TRUE( + channel->AddRecvStream(cricket::StreamParams::CreateLegacy(kSsrc))); + ASSERT_EQ(1u, decoder_factory_->decoders().size()); + + // Setting codecs of the same type should not reallocate the decoder. + EXPECT_TRUE(channel->SetRecvParameters(parameters)); + EXPECT_EQ(1, decoder_factory_->GetNumCreatedDecoders()); + + // Remove stream previously added to free the external decoder instance. + EXPECT_TRUE(channel->RemoveRecvStream(kSsrc)); + EXPECT_EQ(0u, decoder_factory_->decoders().size()); +} + +// Verifies that we can set up decoders. +TEST_F(WebRtcVideoEngineTest, RegisterH264DecoderIfSupported) { + // TODO(pbos): Do not assume that encoder/decoder support is symmetric. We + // can't even query the WebRtcVideoDecoderFactory for supported codecs. + // For now we add a FakeWebRtcVideoEncoderFactory to add H264 to supported + // codecs. + AddSupportedVideoCodecType("H264"); + std::vector<cricket::VideoCodec> codecs; + codecs.push_back(GetEngineCodec("H264")); + + std::unique_ptr<VideoMediaChannel> channel( + SetRecvParamsWithSupportedCodecs(codecs)); + + EXPECT_TRUE( + channel->AddRecvStream(cricket::StreamParams::CreateLegacy(kSsrc))); + ASSERT_EQ(1u, decoder_factory_->decoders().size()); +} + +// Tests when GetSources is called with non-existing ssrc, it will return an +// empty list of RtpSource without crashing. +TEST_F(WebRtcVideoEngineTest, GetSourcesWithNonExistingSsrc) { + // Setup an recv stream with |kSsrc|. + AddSupportedVideoCodecType("VP8"); + cricket::VideoRecvParameters parameters; + parameters.codecs.push_back(GetEngineCodec("VP8")); + std::unique_ptr<VideoMediaChannel> channel( + SetRecvParamsWithSupportedCodecs(parameters.codecs)); + + EXPECT_TRUE( + channel->AddRecvStream(cricket::StreamParams::CreateLegacy(kSsrc))); + + // Call GetSources with |kSsrc + 1| which doesn't exist. + std::vector<webrtc::RtpSource> sources = channel->GetSources(kSsrc + 1); + EXPECT_EQ(0u, sources.size()); +} + +TEST(WebRtcVideoEngineNewVideoCodecFactoryTest, NullFactories) { + std::unique_ptr<webrtc::VideoEncoderFactory> encoder_factory; + std::unique_ptr<webrtc::VideoDecoderFactory> decoder_factory; + WebRtcVideoEngine engine(std::move(encoder_factory), + std::move(decoder_factory)); + EXPECT_EQ(0u, engine.send_codecs().size()); + EXPECT_EQ(0u, engine.recv_codecs().size()); +} + +TEST(WebRtcVideoEngineNewVideoCodecFactoryTest, EmptyFactories) { + // |engine| take ownership of the factories. + webrtc::MockVideoEncoderFactory* encoder_factory = + new webrtc::MockVideoEncoderFactory(); + webrtc::MockVideoDecoderFactory* decoder_factory = + new webrtc::MockVideoDecoderFactory(); + WebRtcVideoEngine engine( + (std::unique_ptr<webrtc::VideoEncoderFactory>(encoder_factory)), + (std::unique_ptr<webrtc::VideoDecoderFactory>(decoder_factory))); + // TODO(kron): Change to Times(1) once send and receive codecs are changed + // to be treated independently. + EXPECT_CALL(*encoder_factory, GetSupportedFormats()).Times(1); + EXPECT_EQ(0u, engine.send_codecs().size()); + EXPECT_EQ(0u, engine.recv_codecs().size()); + EXPECT_CALL(*encoder_factory, Die()); + EXPECT_CALL(*decoder_factory, Die()); +} + +// Test full behavior in the video engine when video codec factories of the new +// type are injected supporting the single codec Vp8. Check the returned codecs +// from the engine and that we will create a Vp8 encoder and decoder using the +// new factories. +TEST(WebRtcVideoEngineNewVideoCodecFactoryTest, Vp8) { + // |engine| take ownership of the factories. + webrtc::MockVideoEncoderFactory* encoder_factory = + new webrtc::MockVideoEncoderFactory(); + webrtc::MockVideoDecoderFactory* decoder_factory = + new webrtc::MockVideoDecoderFactory(); + std::unique_ptr<webrtc::MockVideoBitrateAllocatorFactory> + rate_allocator_factory = + std::make_unique<webrtc::MockVideoBitrateAllocatorFactory>(); + EXPECT_CALL(*rate_allocator_factory, + CreateVideoBitrateAllocator(Field(&webrtc::VideoCodec::codecType, + webrtc::kVideoCodecVP8))) + .WillOnce( + [] { return std::make_unique<webrtc::MockVideoBitrateAllocator>(); }); + WebRtcVideoEngine engine( + (std::unique_ptr<webrtc::VideoEncoderFactory>(encoder_factory)), + (std::unique_ptr<webrtc::VideoDecoderFactory>(decoder_factory))); + const webrtc::SdpVideoFormat vp8_format("VP8"); + const std::vector<webrtc::SdpVideoFormat> supported_formats = {vp8_format}; + EXPECT_CALL(*encoder_factory, GetSupportedFormats()) + .WillRepeatedly(Return(supported_formats)); + EXPECT_CALL(*decoder_factory, GetSupportedFormats()) + .WillRepeatedly(Return(supported_formats)); + + // Verify the codecs from the engine. + const std::vector<VideoCodec> engine_codecs = engine.send_codecs(); + // Verify default codecs has been added correctly. + EXPECT_EQ(5u, engine_codecs.size()); + EXPECT_EQ("VP8", engine_codecs.at(0).name); + + // RTX codec for VP8. + EXPECT_EQ("rtx", engine_codecs.at(1).name); + int vp8_associated_payload; + EXPECT_TRUE(engine_codecs.at(1).GetParam(kCodecParamAssociatedPayloadType, + &vp8_associated_payload)); + EXPECT_EQ(vp8_associated_payload, engine_codecs.at(0).id); + + EXPECT_EQ(kRedCodecName, engine_codecs.at(2).name); + + // RTX codec for RED. + EXPECT_EQ("rtx", engine_codecs.at(3).name); + int red_associated_payload; + EXPECT_TRUE(engine_codecs.at(3).GetParam(kCodecParamAssociatedPayloadType, + &red_associated_payload)); + EXPECT_EQ(red_associated_payload, engine_codecs.at(2).id); + + EXPECT_EQ(kUlpfecCodecName, engine_codecs.at(4).name); + + int associated_payload_type; + EXPECT_TRUE(engine_codecs.at(1).GetParam( + cricket::kCodecParamAssociatedPayloadType, &associated_payload_type)); + EXPECT_EQ(engine_codecs.at(0).id, associated_payload_type); + // Verify default parameters has been added to the VP8 codec. + VerifyCodecHasDefaultFeedbackParams(engine_codecs.at(0), + /*lntf_expected=*/false); + + // Mock encoder creation. |engine| take ownership of the encoder. + webrtc::VideoEncoderFactory::CodecInfo codec_info; + codec_info.is_hardware_accelerated = false; + codec_info.has_internal_source = false; + const webrtc::SdpVideoFormat format("VP8"); + EXPECT_CALL(*encoder_factory, QueryVideoEncoder(format)) + .WillRepeatedly(Return(codec_info)); + rtc::Event encoder_created; + EXPECT_CALL(*encoder_factory, CreateVideoEncoder(format)).WillOnce([&] { + encoder_created.Set(); + return std::make_unique<FakeWebRtcVideoEncoder>(nullptr); + }); + + // Mock decoder creation. |engine| take ownership of the decoder. + EXPECT_CALL(*decoder_factory, CreateVideoDecoder(format)).WillOnce([] { + return std::make_unique<FakeWebRtcVideoDecoder>(nullptr); + }); + + // Create a call. + webrtc::RtcEventLogNull event_log; + auto task_queue_factory = webrtc::CreateDefaultTaskQueueFactory(); + webrtc::Call::Config call_config(&event_log); + webrtc::FieldTrialBasedConfig field_trials; + call_config.trials = &field_trials; + call_config.task_queue_factory = task_queue_factory.get(); + const auto call = absl::WrapUnique(webrtc::Call::Create(call_config)); + + // Create send channel. + const int send_ssrc = 123; + std::unique_ptr<VideoMediaChannel> send_channel(engine.CreateMediaChannel( + call.get(), GetMediaConfig(), VideoOptions(), webrtc::CryptoOptions(), + rate_allocator_factory.get())); + cricket::VideoSendParameters send_parameters; + send_parameters.codecs.push_back(engine_codecs.at(0)); + EXPECT_TRUE(send_channel->SetSendParameters(send_parameters)); + send_channel->OnReadyToSend(true); + EXPECT_TRUE( + send_channel->AddSendStream(StreamParams::CreateLegacy(send_ssrc))); + EXPECT_TRUE(send_channel->SetSend(true)); + + // Set capturer. + webrtc::test::FrameForwarder frame_forwarder; + cricket::FakeFrameSource frame_source(1280, 720, + rtc::kNumMicrosecsPerSec / 30); + EXPECT_TRUE(send_channel->SetVideoSend(send_ssrc, nullptr, &frame_forwarder)); + // Sending one frame will allocate the encoder. + frame_forwarder.IncomingCapturedFrame(frame_source.GetFrame()); + encoder_created.Wait(kTimeout); + + // Create recv channel. + const int recv_ssrc = 321; + std::unique_ptr<VideoMediaChannel> recv_channel(engine.CreateMediaChannel( + call.get(), GetMediaConfig(), VideoOptions(), webrtc::CryptoOptions(), + rate_allocator_factory.get())); + cricket::VideoRecvParameters recv_parameters; + recv_parameters.codecs.push_back(engine_codecs.at(0)); + EXPECT_TRUE(recv_channel->SetRecvParameters(recv_parameters)); + EXPECT_TRUE(recv_channel->AddRecvStream( + cricket::StreamParams::CreateLegacy(recv_ssrc))); + + // Remove streams previously added to free the encoder and decoder instance. + EXPECT_CALL(*encoder_factory, Die()); + EXPECT_CALL(*decoder_factory, Die()); + EXPECT_CALL(*rate_allocator_factory, Die()); + EXPECT_TRUE(send_channel->RemoveSendStream(send_ssrc)); + EXPECT_TRUE(recv_channel->RemoveRecvStream(recv_ssrc)); +} + +// Test behavior when decoder factory fails to create a decoder (returns null). +TEST(WebRtcVideoEngineNewVideoCodecFactoryTest, NullDecoder) { + // |engine| take ownership of the factories. + webrtc::MockVideoEncoderFactory* encoder_factory = + new webrtc::MockVideoEncoderFactory(); + webrtc::MockVideoDecoderFactory* decoder_factory = + new webrtc::MockVideoDecoderFactory(); + std::unique_ptr<webrtc::MockVideoBitrateAllocatorFactory> + rate_allocator_factory = + std::make_unique<webrtc::MockVideoBitrateAllocatorFactory>(); + WebRtcVideoEngine engine( + (std::unique_ptr<webrtc::VideoEncoderFactory>(encoder_factory)), + (std::unique_ptr<webrtc::VideoDecoderFactory>(decoder_factory))); + const webrtc::SdpVideoFormat vp8_format("VP8"); + const std::vector<webrtc::SdpVideoFormat> supported_formats = {vp8_format}; + EXPECT_CALL(*encoder_factory, GetSupportedFormats()) + .WillRepeatedly(Return(supported_formats)); + + // Decoder creation fails. + EXPECT_CALL(*decoder_factory, CreateVideoDecoder).WillOnce([] { + return nullptr; + }); + + // Create a call. + webrtc::RtcEventLogNull event_log; + auto task_queue_factory = webrtc::CreateDefaultTaskQueueFactory(); + webrtc::Call::Config call_config(&event_log); + webrtc::FieldTrialBasedConfig field_trials; + call_config.trials = &field_trials; + call_config.task_queue_factory = task_queue_factory.get(); + const auto call = absl::WrapUnique(webrtc::Call::Create(call_config)); + + // Create recv channel. + EXPECT_CALL(*decoder_factory, GetSupportedFormats()) + .WillRepeatedly(::testing::Return(supported_formats)); + const int recv_ssrc = 321; + std::unique_ptr<VideoMediaChannel> recv_channel(engine.CreateMediaChannel( + call.get(), GetMediaConfig(), VideoOptions(), webrtc::CryptoOptions(), + rate_allocator_factory.get())); + cricket::VideoRecvParameters recv_parameters; + recv_parameters.codecs.push_back(engine.recv_codecs().front()); + EXPECT_TRUE(recv_channel->SetRecvParameters(recv_parameters)); + EXPECT_TRUE(recv_channel->AddRecvStream( + cricket::StreamParams::CreateLegacy(recv_ssrc))); + + // Remove streams previously added to free the encoder and decoder instance. + EXPECT_TRUE(recv_channel->RemoveRecvStream(recv_ssrc)); +} + +TEST_F(WebRtcVideoEngineTest, DISABLED_RecreatesEncoderOnContentTypeChange) { + encoder_factory_->AddSupportedVideoCodecType("VP8"); + std::unique_ptr<FakeCall> fake_call(new FakeCall()); + std::unique_ptr<VideoMediaChannel> channel( + SetSendParamsWithAllSupportedCodecs()); + ASSERT_TRUE( + channel->AddSendStream(cricket::StreamParams::CreateLegacy(kSsrc))); + cricket::VideoCodec codec = GetEngineCodec("VP8"); + cricket::VideoSendParameters parameters; + parameters.codecs.push_back(codec); + channel->OnReadyToSend(true); + channel->SetSend(true); + ASSERT_TRUE(channel->SetSendParameters(parameters)); + + webrtc::test::FrameForwarder frame_forwarder; + cricket::FakeFrameSource frame_source(1280, 720, + rtc::kNumMicrosecsPerSec / 30); + VideoOptions options; + EXPECT_TRUE(channel->SetVideoSend(kSsrc, &options, &frame_forwarder)); + + frame_forwarder.IncomingCapturedFrame(frame_source.GetFrame()); + ASSERT_TRUE(encoder_factory_->WaitForCreatedVideoEncoders(1)); + EXPECT_EQ(webrtc::VideoCodecMode::kRealtimeVideo, + encoder_factory_->encoders().back()->GetCodecSettings().mode); + + EXPECT_TRUE(channel->SetVideoSend(kSsrc, &options, &frame_forwarder)); + frame_forwarder.IncomingCapturedFrame(frame_source.GetFrame()); + // No change in content type, keep current encoder. + EXPECT_EQ(1, encoder_factory_->GetNumCreatedEncoders()); + + options.is_screencast.emplace(true); + EXPECT_TRUE(channel->SetVideoSend(kSsrc, &options, &frame_forwarder)); + frame_forwarder.IncomingCapturedFrame(frame_source.GetFrame()); + // Change to screen content, recreate encoder. For the simulcast encoder + // adapter case, this will result in two calls since InitEncode triggers a + // a new instance. + ASSERT_TRUE(encoder_factory_->WaitForCreatedVideoEncoders(2)); + EXPECT_EQ(webrtc::VideoCodecMode::kScreensharing, + encoder_factory_->encoders().back()->GetCodecSettings().mode); + + EXPECT_TRUE(channel->SetVideoSend(kSsrc, &options, &frame_forwarder)); + frame_forwarder.IncomingCapturedFrame(frame_source.GetFrame()); + // Still screen content, no need to update encoder. + EXPECT_EQ(2, encoder_factory_->GetNumCreatedEncoders()); + + options.is_screencast.emplace(false); + options.video_noise_reduction.emplace(false); + EXPECT_TRUE(channel->SetVideoSend(kSsrc, &options, &frame_forwarder)); + // Change back to regular video content, update encoder. Also change + // a non |is_screencast| option just to verify it doesn't affect recreation. + frame_forwarder.IncomingCapturedFrame(frame_source.GetFrame()); + ASSERT_TRUE(encoder_factory_->WaitForCreatedVideoEncoders(3)); + EXPECT_EQ(webrtc::VideoCodecMode::kRealtimeVideo, + encoder_factory_->encoders().back()->GetCodecSettings().mode); + + // Remove stream previously added to free the external encoder instance. + EXPECT_TRUE(channel->RemoveSendStream(kSsrc)); + EXPECT_EQ(0u, encoder_factory_->encoders().size()); +} + +class WebRtcVideoChannelEncodedFrameCallbackTest : public ::testing::Test { + protected: + webrtc::Call::Config GetCallConfig( + webrtc::RtcEventLogNull* event_log, + webrtc::TaskQueueFactory* task_queue_factory) { + webrtc::Call::Config call_config(event_log); + call_config.task_queue_factory = task_queue_factory; + call_config.trials = &field_trials_; + return call_config; + } + + WebRtcVideoChannelEncodedFrameCallbackTest() + : task_queue_factory_(webrtc::CreateDefaultTaskQueueFactory()), + call_(absl::WrapUnique(webrtc::Call::Create( + GetCallConfig(&event_log_, task_queue_factory_.get())))), + video_bitrate_allocator_factory_( + webrtc::CreateBuiltinVideoBitrateAllocatorFactory()), + engine_( + webrtc::CreateBuiltinVideoEncoderFactory(), + std::make_unique<webrtc::test::FunctionVideoDecoderFactory>( + []() { return std::make_unique<webrtc::test::FakeDecoder>(); }, + kSdpVideoFormats)), + channel_(absl::WrapUnique(static_cast<cricket::WebRtcVideoChannel*>( + engine_.CreateMediaChannel( + call_.get(), + cricket::MediaConfig(), + cricket::VideoOptions(), + webrtc::CryptoOptions(), + video_bitrate_allocator_factory_.get())))) { + network_interface_.SetDestination(channel_.get()); + channel_->SetInterface(&network_interface_, webrtc::MediaTransportConfig()); + cricket::VideoRecvParameters parameters; + parameters.codecs = engine_.recv_codecs(); + channel_->SetRecvParameters(parameters); + } + + void DeliverKeyFrame(uint32_t ssrc) { + webrtc::RtpPacket packet; + packet.SetMarker(true); + packet.SetPayloadType(96); // VP8 + packet.SetSsrc(ssrc); + + // VP8 Keyframe + 1 byte payload + uint8_t* buf_ptr = packet.AllocatePayload(11); + memset(buf_ptr, 0, 11); // Pass MSAN (don't care about bytes 1-9) + buf_ptr[0] = 0x10; // Partition ID 0 + beginning of partition. + call_->Receiver()->DeliverPacket(webrtc::MediaType::VIDEO, packet.Buffer(), + /*packet_time_us=*/0); + } + + void DeliverKeyFrameAndWait(uint32_t ssrc) { + DeliverKeyFrame(ssrc); + EXPECT_EQ_WAIT(1, renderer_.num_rendered_frames(), kTimeout); + EXPECT_EQ(0, renderer_.errors()); + } + + static const std::vector<webrtc::SdpVideoFormat> kSdpVideoFormats; + webrtc::FieldTrialBasedConfig field_trials_; + webrtc::RtcEventLogNull event_log_; + std::unique_ptr<webrtc::TaskQueueFactory> task_queue_factory_; + std::unique_ptr<webrtc::Call> call_; + std::unique_ptr<webrtc::VideoBitrateAllocatorFactory> + video_bitrate_allocator_factory_; + WebRtcVideoEngine engine_; + std::unique_ptr<WebRtcVideoChannel> channel_; + cricket::FakeNetworkInterface network_interface_; + cricket::FakeVideoRenderer renderer_; +}; + +const std::vector<webrtc::SdpVideoFormat> + WebRtcVideoChannelEncodedFrameCallbackTest::kSdpVideoFormats = { + webrtc::SdpVideoFormat("VP8")}; + +TEST_F(WebRtcVideoChannelEncodedFrameCallbackTest, + SetEncodedFrameBufferFunction_DefaultStream) { + testing::MockFunction<void(const webrtc::RecordableEncodedFrame&)> callback; + EXPECT_CALL(callback, Call); + EXPECT_TRUE(channel_->AddRecvStream( + cricket::StreamParams::CreateLegacy(kSsrc), /*is_default_stream=*/true)); + channel_->SetRecordableEncodedFrameCallback(/*ssrc=*/0, + callback.AsStdFunction()); + EXPECT_TRUE(channel_->SetSink(kSsrc, &renderer_)); + DeliverKeyFrame(kSsrc); + EXPECT_EQ_WAIT(1, renderer_.num_rendered_frames(), kTimeout); + EXPECT_EQ(0, renderer_.errors()); + channel_->RemoveRecvStream(kSsrc); +} + +TEST_F(WebRtcVideoChannelEncodedFrameCallbackTest, + SetEncodedFrameBufferFunction_MatchSsrcWithDefaultStream) { + testing::MockFunction<void(const webrtc::RecordableEncodedFrame&)> callback; + EXPECT_CALL(callback, Call); + EXPECT_TRUE(channel_->AddRecvStream( + cricket::StreamParams::CreateLegacy(kSsrc), /*is_default_stream=*/true)); + EXPECT_TRUE(channel_->SetSink(kSsrc, &renderer_)); + channel_->SetRecordableEncodedFrameCallback(kSsrc, callback.AsStdFunction()); + DeliverKeyFrame(kSsrc); + EXPECT_EQ_WAIT(1, renderer_.num_rendered_frames(), kTimeout); + EXPECT_EQ(0, renderer_.errors()); + channel_->RemoveRecvStream(kSsrc); +} + +TEST_F(WebRtcVideoChannelEncodedFrameCallbackTest, + SetEncodedFrameBufferFunction_MatchSsrc) { + testing::MockFunction<void(const webrtc::RecordableEncodedFrame&)> callback; + EXPECT_CALL(callback, Call); + EXPECT_TRUE(channel_->AddRecvStream( + cricket::StreamParams::CreateLegacy(kSsrc), /*is_default_stream=*/false)); + EXPECT_TRUE(channel_->SetSink(kSsrc, &renderer_)); + channel_->SetRecordableEncodedFrameCallback(kSsrc, callback.AsStdFunction()); + DeliverKeyFrame(kSsrc); + EXPECT_EQ_WAIT(1, renderer_.num_rendered_frames(), kTimeout); + EXPECT_EQ(0, renderer_.errors()); + channel_->RemoveRecvStream(kSsrc); +} + +TEST_F(WebRtcVideoChannelEncodedFrameCallbackTest, + SetEncodedFrameBufferFunction_MismatchSsrc) { + testing::StrictMock< + testing::MockFunction<void(const webrtc::RecordableEncodedFrame&)>> + callback; + EXPECT_TRUE( + channel_->AddRecvStream(cricket::StreamParams::CreateLegacy(kSsrc + 1), + /*is_default_stream=*/false)); + EXPECT_TRUE(channel_->SetSink(kSsrc + 1, &renderer_)); + channel_->SetRecordableEncodedFrameCallback(kSsrc, callback.AsStdFunction()); + DeliverKeyFrame(kSsrc); // Expected to not cause function to fire. + DeliverKeyFrameAndWait(kSsrc + 1); + channel_->RemoveRecvStream(kSsrc + 1); +} + +TEST_F(WebRtcVideoChannelEncodedFrameCallbackTest, + SetEncodedFrameBufferFunction_MismatchSsrcWithDefaultStream) { + testing::StrictMock< + testing::MockFunction<void(const webrtc::RecordableEncodedFrame&)>> + callback; + EXPECT_TRUE( + channel_->AddRecvStream(cricket::StreamParams::CreateLegacy(kSsrc + 1), + /*is_default_stream=*/true)); + EXPECT_TRUE(channel_->SetSink(kSsrc + 1, &renderer_)); + channel_->SetRecordableEncodedFrameCallback(kSsrc, callback.AsStdFunction()); + DeliverKeyFrame(kSsrc); // Expected to not cause function to fire. + DeliverKeyFrameAndWait(kSsrc + 1); + channel_->RemoveRecvStream(kSsrc + 1); +} + +class WebRtcVideoChannelBaseTest : public ::testing::Test { + protected: + WebRtcVideoChannelBaseTest() + : task_queue_factory_(webrtc::CreateDefaultTaskQueueFactory()), + video_bitrate_allocator_factory_( + webrtc::CreateBuiltinVideoBitrateAllocatorFactory()), + engine_(webrtc::CreateBuiltinVideoEncoderFactory(), + webrtc::CreateBuiltinVideoDecoderFactory()) {} + + virtual void SetUp() { + // One testcase calls SetUp in a loop, only create call_ once. + if (!call_) { + webrtc::Call::Config call_config(&event_log_); + call_config.task_queue_factory = task_queue_factory_.get(); + call_config.trials = &field_trials_; + call_.reset(webrtc::Call::Create(call_config)); + } + cricket::MediaConfig media_config; + // Disabling cpu overuse detection actually disables quality scaling too; it + // implies DegradationPreference kMaintainResolution. Automatic scaling + // needs to be disabled, otherwise, tests which check the size of received + // frames become flaky. + media_config.video.enable_cpu_adaptation = false; + channel_.reset( + static_cast<cricket::WebRtcVideoChannel*>(engine_.CreateMediaChannel( + call_.get(), media_config, cricket::VideoOptions(), + webrtc::CryptoOptions(), video_bitrate_allocator_factory_.get()))); + channel_->OnReadyToSend(true); + EXPECT_TRUE(channel_.get() != NULL); + network_interface_.SetDestination(channel_.get()); + channel_->SetInterface(&network_interface_, webrtc::MediaTransportConfig()); + cricket::VideoRecvParameters parameters; + parameters.codecs = engine_.send_codecs(); + channel_->SetRecvParameters(parameters); + EXPECT_TRUE(channel_->AddSendStream(DefaultSendStreamParams())); + frame_forwarder_ = std::make_unique<webrtc::test::FrameForwarder>(); + frame_source_ = std::make_unique<cricket::FakeFrameSource>( + 640, 480, rtc::kNumMicrosecsPerSec / kFramerate); + EXPECT_TRUE(channel_->SetVideoSend(kSsrc, nullptr, frame_forwarder_.get())); + } + + // Utility method to setup an additional stream to send and receive video. + // Used to test send and recv between two streams. + void SetUpSecondStream() { + SetUpSecondStreamWithNoRecv(); + // Setup recv for second stream. + EXPECT_TRUE(channel_->AddRecvStream( + cricket::StreamParams::CreateLegacy(kSsrc + 2))); + // Make the second renderer available for use by a new stream. + EXPECT_TRUE(channel_->SetSink(kSsrc + 2, &renderer2_)); + } + // Setup an additional stream just to send video. Defer add recv stream. + // This is required if you want to test unsignalled recv of video rtp packets. + void SetUpSecondStreamWithNoRecv() { + // SetUp() already added kSsrc make sure duplicate SSRCs cant be added. + EXPECT_TRUE( + channel_->AddRecvStream(cricket::StreamParams::CreateLegacy(kSsrc))); + EXPECT_TRUE(channel_->SetSink(kSsrc, &renderer_)); + EXPECT_FALSE( + channel_->AddSendStream(cricket::StreamParams::CreateLegacy(kSsrc))); + EXPECT_TRUE(channel_->AddSendStream( + cricket::StreamParams::CreateLegacy(kSsrc + 2))); + // We dont add recv for the second stream. + + // Setup the receive and renderer for second stream after send. + frame_forwarder_2_ = std::make_unique<webrtc::test::FrameForwarder>(); + EXPECT_TRUE( + channel_->SetVideoSend(kSsrc + 2, nullptr, frame_forwarder_2_.get())); + } + virtual void TearDown() { channel_.reset(); } + bool SetDefaultCodec() { return SetOneCodec(DefaultCodec()); } + + bool SetOneCodec(const cricket::VideoCodec& codec) { + frame_source_ = std::make_unique<cricket::FakeFrameSource>( + kVideoWidth, kVideoHeight, rtc::kNumMicrosecsPerSec / kFramerate); + + bool sending = channel_->sending(); + bool success = SetSend(false); + if (success) { + cricket::VideoSendParameters parameters; + parameters.codecs.push_back(codec); + success = channel_->SetSendParameters(parameters); + } + if (success) { + success = SetSend(sending); + } + return success; + } + bool SetSend(bool send) { return channel_->SetSend(send); } + void SendFrame() { + if (frame_forwarder_2_) { + frame_forwarder_2_->IncomingCapturedFrame(frame_source_->GetFrame()); + } + frame_forwarder_->IncomingCapturedFrame(frame_source_->GetFrame()); + } + bool WaitAndSendFrame(int wait_ms) { + bool ret = rtc::Thread::Current()->ProcessMessages(wait_ms); + SendFrame(); + return ret; + } + int NumRtpBytes() { return network_interface_.NumRtpBytes(); } + int NumRtpBytes(uint32_t ssrc) { + return network_interface_.NumRtpBytes(ssrc); + } + int NumRtpPackets() { return network_interface_.NumRtpPackets(); } + int NumRtpPackets(uint32_t ssrc) { + return network_interface_.NumRtpPackets(ssrc); + } + int NumSentSsrcs() { return network_interface_.NumSentSsrcs(); } + const rtc::CopyOnWriteBuffer* GetRtpPacket(int index) { + return network_interface_.GetRtpPacket(index); + } + static int GetPayloadType(const rtc::CopyOnWriteBuffer* p) { + webrtc::RTPHeader header; + EXPECT_TRUE(ParseRtpPacket(p, &header)); + return header.payloadType; + } + + static bool ParseRtpPacket(const rtc::CopyOnWriteBuffer* p, + webrtc::RTPHeader* header) { + std::unique_ptr<webrtc::RtpHeaderParser> parser( + webrtc::RtpHeaderParser::CreateForTest()); + return parser->Parse(p->cdata(), p->size(), header); + } + + // Tests that we can send and receive frames. + void SendAndReceive(const cricket::VideoCodec& codec) { + EXPECT_TRUE(SetOneCodec(codec)); + EXPECT_TRUE(SetSend(true)); + channel_->SetDefaultSink(&renderer_); + EXPECT_EQ(0, renderer_.num_rendered_frames()); + SendFrame(); + EXPECT_FRAME_WAIT(1, kVideoWidth, kVideoHeight, kTimeout); + std::unique_ptr<const rtc::CopyOnWriteBuffer> p(GetRtpPacket(0)); + EXPECT_EQ(codec.id, GetPayloadType(p.get())); + } + + void SendReceiveManyAndGetStats(const cricket::VideoCodec& codec, + int duration_sec, + int fps) { + EXPECT_TRUE(SetOneCodec(codec)); + EXPECT_TRUE(SetSend(true)); + channel_->SetDefaultSink(&renderer_); + EXPECT_EQ(0, renderer_.num_rendered_frames()); + for (int i = 0; i < duration_sec; ++i) { + for (int frame = 1; frame <= fps; ++frame) { + EXPECT_TRUE(WaitAndSendFrame(1000 / fps)); + EXPECT_FRAME_WAIT(frame + i * fps, kVideoWidth, kVideoHeight, kTimeout); + } + } + std::unique_ptr<const rtc::CopyOnWriteBuffer> p(GetRtpPacket(0)); + EXPECT_EQ(codec.id, GetPayloadType(p.get())); + } + + cricket::VideoSenderInfo GetSenderStats(size_t i) { + cricket::VideoMediaInfo info; + EXPECT_TRUE(channel_->GetStats(&info)); + return info.senders[i]; + } + + cricket::VideoReceiverInfo GetReceiverStats(size_t i) { + cricket::VideoMediaInfo info; + EXPECT_TRUE(channel_->GetStats(&info)); + return info.receivers[i]; + } + + // Two streams one channel tests. + + // Tests that we can send and receive frames. + void TwoStreamsSendAndReceive(const cricket::VideoCodec& codec) { + SetUpSecondStream(); + // Test sending and receiving on first stream. + SendAndReceive(codec); + // Test sending and receiving on second stream. + EXPECT_EQ_WAIT(1, renderer2_.num_rendered_frames(), kTimeout); + EXPECT_GT(NumRtpPackets(), 0); + EXPECT_EQ(1, renderer2_.num_rendered_frames()); + } + + cricket::VideoCodec GetEngineCodec(const std::string& name) { + for (const cricket::VideoCodec& engine_codec : engine_.send_codecs()) { + if (absl::EqualsIgnoreCase(name, engine_codec.name)) + return engine_codec; + } + // This point should never be reached. + ADD_FAILURE() << "Unrecognized codec name: " << name; + return cricket::VideoCodec(); + } + + cricket::VideoCodec DefaultCodec() { return GetEngineCodec("VP8"); } + + cricket::StreamParams DefaultSendStreamParams() { + return cricket::StreamParams::CreateLegacy(kSsrc); + } + + webrtc::RtcEventLogNull event_log_; + webrtc::FieldTrialBasedConfig field_trials_; + std::unique_ptr<webrtc::TaskQueueFactory> task_queue_factory_; + std::unique_ptr<webrtc::Call> call_; + std::unique_ptr<webrtc::VideoBitrateAllocatorFactory> + video_bitrate_allocator_factory_; + WebRtcVideoEngine engine_; + + std::unique_ptr<cricket::FakeFrameSource> frame_source_; + std::unique_ptr<webrtc::test::FrameForwarder> frame_forwarder_; + std::unique_ptr<webrtc::test::FrameForwarder> frame_forwarder_2_; + + std::unique_ptr<WebRtcVideoChannel> channel_; + cricket::FakeNetworkInterface network_interface_; + cricket::FakeVideoRenderer renderer_; + + // Used by test cases where 2 streams are run on the same channel. + cricket::FakeVideoRenderer renderer2_; +}; + +// Test that SetSend works. +TEST_F(WebRtcVideoChannelBaseTest, SetSend) { + EXPECT_FALSE(channel_->sending()); + EXPECT_TRUE(channel_->SetVideoSend(kSsrc, nullptr, frame_forwarder_.get())); + EXPECT_TRUE(SetOneCodec(DefaultCodec())); + EXPECT_FALSE(channel_->sending()); + EXPECT_TRUE(SetSend(true)); + EXPECT_TRUE(channel_->sending()); + SendFrame(); + EXPECT_TRUE_WAIT(NumRtpPackets() > 0, kTimeout); + EXPECT_TRUE(SetSend(false)); + EXPECT_FALSE(channel_->sending()); +} + +// Test that SetSend fails without codecs being set. +TEST_F(WebRtcVideoChannelBaseTest, SetSendWithoutCodecs) { + EXPECT_FALSE(channel_->sending()); + EXPECT_FALSE(SetSend(true)); + EXPECT_FALSE(channel_->sending()); +} + +// Test that we properly set the send and recv buffer sizes by the time +// SetSend is called. +TEST_F(WebRtcVideoChannelBaseTest, SetSendSetsTransportBufferSizes) { + EXPECT_TRUE(SetOneCodec(DefaultCodec())); + EXPECT_TRUE(SetSend(true)); + EXPECT_EQ(64 * 1024, network_interface_.sendbuf_size()); + EXPECT_EQ(256 * 1024, network_interface_.recvbuf_size()); +} + +// Test that we properly set the send and recv buffer sizes when overriding +// via field trials. +TEST_F(WebRtcVideoChannelBaseTest, OverridesRecvBufferSize) { + // Set field trial to override the default recv buffer size, and then re-run + // setup where the interface is created and configured. + const int kCustomRecvBufferSize = 123456; + webrtc::test::ScopedFieldTrials field_trial( + "WebRTC-IncreasedReceivebuffers/123456/"); + SetUp(); + + EXPECT_TRUE(SetOneCodec(DefaultCodec())); + EXPECT_TRUE(SetSend(true)); + EXPECT_EQ(64 * 1024, network_interface_.sendbuf_size()); + EXPECT_EQ(kCustomRecvBufferSize, network_interface_.recvbuf_size()); +} + +// Test that we properly set the send and recv buffer sizes when overriding +// via field trials with suffix. +TEST_F(WebRtcVideoChannelBaseTest, OverridesRecvBufferSizeWithSuffix) { + // Set field trial to override the default recv buffer size, and then re-run + // setup where the interface is created and configured. + const int kCustomRecvBufferSize = 123456; + webrtc::test::ScopedFieldTrials field_trial( + "WebRTC-IncreasedReceivebuffers/123456_Dogfood/"); + SetUp(); + + EXPECT_TRUE(SetOneCodec(DefaultCodec())); + EXPECT_TRUE(SetSend(true)); + EXPECT_EQ(64 * 1024, network_interface_.sendbuf_size()); + EXPECT_EQ(kCustomRecvBufferSize, network_interface_.recvbuf_size()); +} + +// Test that we properly set the send and recv buffer sizes when overriding +// via field trials that don't make any sense. +TEST_F(WebRtcVideoChannelBaseTest, InvalidRecvBufferSize) { + // Set bogus field trial values to override the default recv buffer size, and + // then re-run setup where the interface is created and configured. The + // default value should still be used. + + for (std::string group : {" ", "NotANumber", "-1", "0"}) { + std::string field_trial_string = "WebRTC-IncreasedReceivebuffers/"; + field_trial_string += group; + field_trial_string += "/"; + webrtc::test::ScopedFieldTrials field_trial(field_trial_string); + SetUp(); + + EXPECT_TRUE(SetOneCodec(DefaultCodec())); + EXPECT_TRUE(SetSend(true)); + EXPECT_EQ(64 * 1024, network_interface_.sendbuf_size()); + EXPECT_EQ(256 * 1024, network_interface_.recvbuf_size()); + } +} + +// Test that stats work properly for a 1-1 call. +TEST_F(WebRtcVideoChannelBaseTest, GetStats) { + SetUp(); + + const int kDurationSec = 3; + const int kFps = 10; + SendReceiveManyAndGetStats(DefaultCodec(), kDurationSec, kFps); + + cricket::VideoMediaInfo info; + EXPECT_TRUE(channel_->GetStats(&info)); + + ASSERT_EQ(1U, info.senders.size()); + // TODO(whyuan): bytes_sent and bytes_rcvd are different. Are both payload? + // For webrtc, bytes_sent does not include the RTP header length. + EXPECT_EQ(info.senders[0].payload_bytes_sent, + NumRtpBytes() - kRtpHeaderSize * NumRtpPackets()); + EXPECT_EQ(NumRtpPackets(), info.senders[0].packets_sent); + EXPECT_EQ(0.0, info.senders[0].fraction_lost); + ASSERT_TRUE(info.senders[0].codec_payload_type); + EXPECT_EQ(DefaultCodec().id, *info.senders[0].codec_payload_type); + EXPECT_EQ(0, info.senders[0].firs_rcvd); + EXPECT_EQ(0, info.senders[0].plis_rcvd); + EXPECT_EQ(0, info.senders[0].nacks_rcvd); + EXPECT_EQ(kVideoWidth, info.senders[0].send_frame_width); + EXPECT_EQ(kVideoHeight, info.senders[0].send_frame_height); + EXPECT_GT(info.senders[0].framerate_input, 0); + EXPECT_GT(info.senders[0].framerate_sent, 0); + + EXPECT_EQ(1U, info.send_codecs.count(DefaultCodec().id)); + EXPECT_EQ(DefaultCodec().ToCodecParameters(), + info.send_codecs[DefaultCodec().id]); + + ASSERT_EQ(1U, info.receivers.size()); + EXPECT_EQ(1U, info.senders[0].ssrcs().size()); + EXPECT_EQ(1U, info.receivers[0].ssrcs().size()); + EXPECT_EQ(info.senders[0].ssrcs()[0], info.receivers[0].ssrcs()[0]); + ASSERT_TRUE(info.receivers[0].codec_payload_type); + EXPECT_EQ(DefaultCodec().id, *info.receivers[0].codec_payload_type); + EXPECT_EQ(NumRtpBytes() - kRtpHeaderSize * NumRtpPackets(), + info.receivers[0].payload_bytes_rcvd); + EXPECT_EQ(NumRtpPackets(), info.receivers[0].packets_rcvd); + EXPECT_EQ(0, info.receivers[0].packets_lost); + // TODO(asapersson): Not set for webrtc. Handle missing stats. + // EXPECT_EQ(0, info.receivers[0].packets_concealed); + EXPECT_EQ(0, info.receivers[0].firs_sent); + EXPECT_EQ(0, info.receivers[0].plis_sent); + EXPECT_EQ(0, info.receivers[0].nacks_sent); + EXPECT_EQ(kVideoWidth, info.receivers[0].frame_width); + EXPECT_EQ(kVideoHeight, info.receivers[0].frame_height); + EXPECT_GT(info.receivers[0].framerate_rcvd, 0); + EXPECT_GT(info.receivers[0].framerate_decoded, 0); + EXPECT_GT(info.receivers[0].framerate_output, 0); + + EXPECT_EQ(1U, info.receive_codecs.count(DefaultCodec().id)); + EXPECT_EQ(DefaultCodec().ToCodecParameters(), + info.receive_codecs[DefaultCodec().id]); +} + +// Test that stats work properly for a conf call with multiple recv streams. +TEST_F(WebRtcVideoChannelBaseTest, GetStatsMultipleRecvStreams) { + SetUp(); + + cricket::FakeVideoRenderer renderer1, renderer2; + EXPECT_TRUE(SetOneCodec(DefaultCodec())); + cricket::VideoSendParameters parameters; + parameters.codecs.push_back(DefaultCodec()); + parameters.conference_mode = true; + EXPECT_TRUE(channel_->SetSendParameters(parameters)); + EXPECT_TRUE(SetSend(true)); + EXPECT_TRUE(channel_->AddRecvStream(cricket::StreamParams::CreateLegacy(1))); + EXPECT_TRUE(channel_->AddRecvStream(cricket::StreamParams::CreateLegacy(2))); + EXPECT_TRUE(channel_->SetSink(1, &renderer1)); + EXPECT_TRUE(channel_->SetSink(2, &renderer2)); + EXPECT_EQ(0, renderer1.num_rendered_frames()); + EXPECT_EQ(0, renderer2.num_rendered_frames()); + std::vector<uint32_t> ssrcs; + ssrcs.push_back(1); + ssrcs.push_back(2); + network_interface_.SetConferenceMode(true, ssrcs); + SendFrame(); + EXPECT_FRAME_ON_RENDERER_WAIT(renderer1, 1, kVideoWidth, kVideoHeight, + kTimeout); + EXPECT_FRAME_ON_RENDERER_WAIT(renderer2, 1, kVideoWidth, kVideoHeight, + kTimeout); + + EXPECT_TRUE(channel_->SetSend(false)); + + cricket::VideoMediaInfo info; + EXPECT_TRUE(channel_->GetStats(&info)); + ASSERT_EQ(1U, info.senders.size()); + // TODO(whyuan): bytes_sent and bytes_rcvd are different. Are both payload? + // For webrtc, bytes_sent does not include the RTP header length. + EXPECT_EQ_WAIT(NumRtpBytes() - kRtpHeaderSize * NumRtpPackets(), + GetSenderStats(0).payload_bytes_sent, kTimeout); + EXPECT_EQ_WAIT(NumRtpPackets(), GetSenderStats(0).packets_sent, kTimeout); + EXPECT_EQ(kVideoWidth, GetSenderStats(0).send_frame_width); + EXPECT_EQ(kVideoHeight, GetSenderStats(0).send_frame_height); + + ASSERT_EQ(2U, info.receivers.size()); + for (size_t i = 0; i < info.receivers.size(); ++i) { + EXPECT_EQ(1U, GetReceiverStats(i).ssrcs().size()); + EXPECT_EQ(i + 1, GetReceiverStats(i).ssrcs()[0]); + EXPECT_EQ_WAIT(NumRtpBytes() - kRtpHeaderSize * NumRtpPackets(), + GetReceiverStats(i).payload_bytes_rcvd, kTimeout); + EXPECT_EQ_WAIT(NumRtpPackets(), GetReceiverStats(i).packets_rcvd, kTimeout); + EXPECT_EQ_WAIT(kVideoWidth, GetReceiverStats(i).frame_width, kTimeout); + EXPECT_EQ_WAIT(kVideoHeight, GetReceiverStats(i).frame_height, kTimeout); + } +} + +// Test that stats work properly for a conf call with multiple send streams. +TEST_F(WebRtcVideoChannelBaseTest, GetStatsMultipleSendStreams) { + // Normal setup; note that we set the SSRC explicitly to ensure that + // it will come first in the senders map. + EXPECT_TRUE(SetOneCodec(DefaultCodec())); + cricket::VideoSendParameters parameters; + parameters.codecs.push_back(DefaultCodec()); + parameters.conference_mode = true; + EXPECT_TRUE(channel_->SetSendParameters(parameters)); + EXPECT_TRUE( + channel_->AddRecvStream(cricket::StreamParams::CreateLegacy(kSsrc))); + EXPECT_TRUE(channel_->SetSink(kSsrc, &renderer_)); + EXPECT_TRUE(SetSend(true)); + SendFrame(); + EXPECT_TRUE_WAIT(NumRtpPackets() > 0, kTimeout); + EXPECT_FRAME_WAIT(1, kVideoWidth, kVideoHeight, kTimeout); + + // Add an additional capturer, and hook up a renderer to receive it. + cricket::FakeVideoRenderer renderer2; + webrtc::test::FrameForwarder frame_forwarder; + const int kTestWidth = 160; + const int kTestHeight = 120; + cricket::FakeFrameSource frame_source(kTestWidth, kTestHeight, + rtc::kNumMicrosecsPerSec / 5); + EXPECT_TRUE( + channel_->AddSendStream(cricket::StreamParams::CreateLegacy(5678))); + EXPECT_TRUE(channel_->SetVideoSend(5678, nullptr, &frame_forwarder)); + EXPECT_TRUE( + channel_->AddRecvStream(cricket::StreamParams::CreateLegacy(5678))); + EXPECT_TRUE(channel_->SetSink(5678, &renderer2)); + frame_forwarder.IncomingCapturedFrame(frame_source.GetFrame()); + EXPECT_FRAME_ON_RENDERER_WAIT(renderer2, 1, kTestWidth, kTestHeight, + kTimeout); + + // Get stats, and make sure they are correct for two senders. We wait until + // the number of expected packets have been sent to avoid races where we + // check stats before it has been updated. + cricket::VideoMediaInfo info; + for (uint32_t i = 0; i < kTimeout; ++i) { + rtc::Thread::Current()->ProcessMessages(1); + EXPECT_TRUE(channel_->GetStats(&info)); + ASSERT_EQ(2U, info.senders.size()); + if (info.senders[0].packets_sent + info.senders[1].packets_sent == + NumRtpPackets()) { + // Stats have been updated for both sent frames, expectations can be + // checked now. + break; + } + } + EXPECT_EQ(NumRtpPackets(), + info.senders[0].packets_sent + info.senders[1].packets_sent) + << "Timed out while waiting for packet counts for all sent packets."; + EXPECT_EQ(1U, info.senders[0].ssrcs().size()); + EXPECT_EQ(1234U, info.senders[0].ssrcs()[0]); + EXPECT_EQ(kVideoWidth, info.senders[0].send_frame_width); + EXPECT_EQ(kVideoHeight, info.senders[0].send_frame_height); + EXPECT_EQ(1U, info.senders[1].ssrcs().size()); + EXPECT_EQ(5678U, info.senders[1].ssrcs()[0]); + EXPECT_EQ(kTestWidth, info.senders[1].send_frame_width); + EXPECT_EQ(kTestHeight, info.senders[1].send_frame_height); + // The capturer must be unregistered here as it runs out of it's scope next. + channel_->SetVideoSend(5678, nullptr, nullptr); +} + +// Test that we can set the bandwidth. +TEST_F(WebRtcVideoChannelBaseTest, SetSendBandwidth) { + cricket::VideoSendParameters parameters; + parameters.codecs.push_back(DefaultCodec()); + parameters.max_bandwidth_bps = -1; // <= 0 means unlimited. + EXPECT_TRUE(channel_->SetSendParameters(parameters)); + parameters.max_bandwidth_bps = 128 * 1024; + EXPECT_TRUE(channel_->SetSendParameters(parameters)); +} + +// Test that we can set the SSRC for the default send source. +TEST_F(WebRtcVideoChannelBaseTest, SetSendSsrc) { + EXPECT_TRUE(SetDefaultCodec()); + EXPECT_TRUE(SetSend(true)); + SendFrame(); + EXPECT_TRUE_WAIT(NumRtpPackets() > 0, kTimeout); + webrtc::RTPHeader header; + std::unique_ptr<const rtc::CopyOnWriteBuffer> p(GetRtpPacket(0)); + EXPECT_TRUE(ParseRtpPacket(p.get(), &header)); + EXPECT_EQ(kSsrc, header.ssrc); + + // Packets are being paced out, so these can mismatch between the first and + // second call to NumRtpPackets until pending packets are paced out. + EXPECT_EQ_WAIT(NumRtpPackets(), NumRtpPackets(header.ssrc), kTimeout); + EXPECT_EQ_WAIT(NumRtpBytes(), NumRtpBytes(header.ssrc), kTimeout); + EXPECT_EQ(1, NumSentSsrcs()); + EXPECT_EQ(0, NumRtpPackets(kSsrc - 1)); + EXPECT_EQ(0, NumRtpBytes(kSsrc - 1)); +} + +// Test that we can set the SSRC even after codecs are set. +TEST_F(WebRtcVideoChannelBaseTest, SetSendSsrcAfterSetCodecs) { + // Remove stream added in Setup. + EXPECT_TRUE(channel_->RemoveSendStream(kSsrc)); + EXPECT_TRUE(SetDefaultCodec()); + EXPECT_TRUE( + channel_->AddSendStream(cricket::StreamParams::CreateLegacy(999))); + EXPECT_TRUE(channel_->SetVideoSend(999u, nullptr, frame_forwarder_.get())); + EXPECT_TRUE(SetSend(true)); + EXPECT_TRUE(WaitAndSendFrame(0)); + EXPECT_TRUE_WAIT(NumRtpPackets() > 0, kTimeout); + webrtc::RTPHeader header; + std::unique_ptr<const rtc::CopyOnWriteBuffer> p(GetRtpPacket(0)); + EXPECT_TRUE(ParseRtpPacket(p.get(), &header)); + EXPECT_EQ(999u, header.ssrc); + // Packets are being paced out, so these can mismatch between the first and + // second call to NumRtpPackets until pending packets are paced out. + EXPECT_EQ_WAIT(NumRtpPackets(), NumRtpPackets(header.ssrc), kTimeout); + EXPECT_EQ_WAIT(NumRtpBytes(), NumRtpBytes(header.ssrc), kTimeout); + EXPECT_EQ(1, NumSentSsrcs()); + EXPECT_EQ(0, NumRtpPackets(kSsrc)); + EXPECT_EQ(0, NumRtpBytes(kSsrc)); +} + +// Test that we can set the default video renderer before and after +// media is received. +TEST_F(WebRtcVideoChannelBaseTest, SetSink) { + uint8_t data1[] = {0x80, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; + + rtc::CopyOnWriteBuffer packet1(data1, sizeof(data1)); + rtc::SetBE32(packet1.data() + 8, kSsrc); + channel_->SetDefaultSink(NULL); + EXPECT_TRUE(SetDefaultCodec()); + EXPECT_TRUE(SetSend(true)); + EXPECT_EQ(0, renderer_.num_rendered_frames()); + channel_->OnPacketReceived(packet1, /* packet_time_us */ -1); + channel_->SetDefaultSink(&renderer_); + SendFrame(); + EXPECT_FRAME_WAIT(1, kVideoWidth, kVideoHeight, kTimeout); +} + +// Tests setting up and configuring a send stream. +TEST_F(WebRtcVideoChannelBaseTest, AddRemoveSendStreams) { + EXPECT_TRUE(SetOneCodec(DefaultCodec())); + EXPECT_TRUE(SetSend(true)); + channel_->SetDefaultSink(&renderer_); + SendFrame(); + EXPECT_FRAME_WAIT(1, kVideoWidth, kVideoHeight, kTimeout); + EXPECT_GT(NumRtpPackets(), 0); + webrtc::RTPHeader header; + size_t last_packet = NumRtpPackets() - 1; + std::unique_ptr<const rtc::CopyOnWriteBuffer> p( + GetRtpPacket(static_cast<int>(last_packet))); + EXPECT_TRUE(ParseRtpPacket(p.get(), &header)); + EXPECT_EQ(kSsrc, header.ssrc); + + // Remove the send stream that was added during Setup. + EXPECT_TRUE(channel_->RemoveSendStream(kSsrc)); + int rtp_packets = NumRtpPackets(); + + EXPECT_TRUE( + channel_->AddSendStream(cricket::StreamParams::CreateLegacy(789u))); + EXPECT_TRUE(channel_->SetVideoSend(789u, nullptr, frame_forwarder_.get())); + EXPECT_EQ(rtp_packets, NumRtpPackets()); + // Wait 30ms to guarantee the engine does not drop the frame. + EXPECT_TRUE(WaitAndSendFrame(30)); + EXPECT_TRUE_WAIT(NumRtpPackets() > rtp_packets, kTimeout); + + last_packet = NumRtpPackets() - 1; + p.reset(GetRtpPacket(static_cast<int>(last_packet))); + EXPECT_TRUE(ParseRtpPacket(p.get(), &header)); + EXPECT_EQ(789u, header.ssrc); +} + +// Tests the behavior of incoming streams in a conference scenario. +TEST_F(WebRtcVideoChannelBaseTest, SimulateConference) { + cricket::FakeVideoRenderer renderer1, renderer2; + EXPECT_TRUE(SetDefaultCodec()); + cricket::VideoSendParameters parameters; + parameters.codecs.push_back(DefaultCodec()); + parameters.conference_mode = true; + EXPECT_TRUE(channel_->SetSendParameters(parameters)); + EXPECT_TRUE(SetSend(true)); + EXPECT_TRUE(channel_->AddRecvStream(cricket::StreamParams::CreateLegacy(1))); + EXPECT_TRUE(channel_->AddRecvStream(cricket::StreamParams::CreateLegacy(2))); + EXPECT_TRUE(channel_->SetSink(1, &renderer1)); + EXPECT_TRUE(channel_->SetSink(2, &renderer2)); + EXPECT_EQ(0, renderer1.num_rendered_frames()); + EXPECT_EQ(0, renderer2.num_rendered_frames()); + std::vector<uint32_t> ssrcs; + ssrcs.push_back(1); + ssrcs.push_back(2); + network_interface_.SetConferenceMode(true, ssrcs); + SendFrame(); + EXPECT_FRAME_ON_RENDERER_WAIT(renderer1, 1, kVideoWidth, kVideoHeight, + kTimeout); + EXPECT_FRAME_ON_RENDERER_WAIT(renderer2, 1, kVideoWidth, kVideoHeight, + kTimeout); + + std::unique_ptr<const rtc::CopyOnWriteBuffer> p(GetRtpPacket(0)); + EXPECT_EQ(DefaultCodec().id, GetPayloadType(p.get())); + EXPECT_EQ(kVideoWidth, renderer1.width()); + EXPECT_EQ(kVideoHeight, renderer1.height()); + EXPECT_EQ(kVideoWidth, renderer2.width()); + EXPECT_EQ(kVideoHeight, renderer2.height()); + EXPECT_TRUE(channel_->RemoveRecvStream(2)); + EXPECT_TRUE(channel_->RemoveRecvStream(1)); +} + +// Tests that we can add and remove capturers and frames are sent out properly +TEST_F(WebRtcVideoChannelBaseTest, DISABLED_AddRemoveCapturer) { + using cricket::FOURCC_I420; + using cricket::VideoCodec; + using cricket::VideoFormat; + using cricket::VideoOptions; + + VideoCodec codec = DefaultCodec(); + const int time_between_send_ms = VideoFormat::FpsToInterval(kFramerate); + EXPECT_TRUE(SetOneCodec(codec)); + EXPECT_TRUE(SetSend(true)); + channel_->SetDefaultSink(&renderer_); + EXPECT_EQ(0, renderer_.num_rendered_frames()); + SendFrame(); + EXPECT_FRAME_WAIT(1, kVideoWidth, kVideoHeight, kTimeout); + + webrtc::test::FrameForwarder frame_forwarder; + cricket::FakeFrameSource frame_source(480, 360, rtc::kNumMicrosecsPerSec / 30, + rtc::kNumMicrosecsPerSec / 30); + + // TODO(nisse): This testcase fails if we don't configure + // screencast. It's unclear why, I see nothing obvious in this + // test which is related to screencast logic. + VideoOptions video_options; + video_options.is_screencast = true; + channel_->SetVideoSend(kSsrc, &video_options, nullptr); + + int captured_frames = 1; + for (int iterations = 0; iterations < 2; ++iterations) { + EXPECT_TRUE(channel_->SetVideoSend(kSsrc, nullptr, &frame_forwarder)); + rtc::Thread::Current()->ProcessMessages(time_between_send_ms); + frame_forwarder.IncomingCapturedFrame(frame_source.GetFrame()); + + ++captured_frames; + // Wait until frame of right size is captured. + EXPECT_TRUE_WAIT(renderer_.num_rendered_frames() >= captured_frames && + 480 == renderer_.width() && + 360 == renderer_.height() && !renderer_.black_frame(), + kTimeout); + EXPECT_GE(renderer_.num_rendered_frames(), captured_frames); + EXPECT_EQ(480, renderer_.width()); + EXPECT_EQ(360, renderer_.height()); + captured_frames = renderer_.num_rendered_frames() + 1; + EXPECT_FALSE(renderer_.black_frame()); + EXPECT_TRUE(channel_->SetVideoSend(kSsrc, nullptr, nullptr)); + // Make sure a black frame is generated within the specified timeout. + // The black frame should be the resolution of the previous frame to + // prevent expensive encoder reconfigurations. + EXPECT_TRUE_WAIT(renderer_.num_rendered_frames() >= captured_frames && + 480 == renderer_.width() && + 360 == renderer_.height() && renderer_.black_frame(), + kTimeout); + EXPECT_GE(renderer_.num_rendered_frames(), captured_frames); + EXPECT_EQ(480, renderer_.width()); + EXPECT_EQ(360, renderer_.height()); + EXPECT_TRUE(renderer_.black_frame()); + + // The black frame has the same timestamp as the next frame since it's + // timestamp is set to the last frame's timestamp + interval. WebRTC will + // not render a frame with the same timestamp so capture another frame + // with the frame capturer to increment the next frame's timestamp. + frame_forwarder.IncomingCapturedFrame(frame_source.GetFrame()); + } +} + +// Tests that if SetVideoSend is called with a NULL capturer after the +// capturer was already removed, the application doesn't crash (and no black +// frame is sent). +TEST_F(WebRtcVideoChannelBaseTest, RemoveCapturerWithoutAdd) { + EXPECT_TRUE(SetOneCodec(DefaultCodec())); + EXPECT_TRUE(SetSend(true)); + channel_->SetDefaultSink(&renderer_); + EXPECT_EQ(0, renderer_.num_rendered_frames()); + SendFrame(); + EXPECT_FRAME_WAIT(1, kVideoWidth, kVideoHeight, kTimeout); + // Wait for one frame so they don't get dropped because we send frames too + // tightly. + rtc::Thread::Current()->ProcessMessages(30); + // Remove the capturer. + EXPECT_TRUE(channel_->SetVideoSend(kSsrc, nullptr, nullptr)); + + // No capturer was added, so this SetVideoSend shouldn't do anything. + EXPECT_TRUE(channel_->SetVideoSend(kSsrc, nullptr, nullptr)); + rtc::Thread::Current()->ProcessMessages(300); + // Verify no more frames were sent. + EXPECT_EQ(1, renderer_.num_rendered_frames()); +} + +// Tests that we can add and remove capturer as unique sources. +TEST_F(WebRtcVideoChannelBaseTest, AddRemoveCapturerMultipleSources) { + // WebRTC implementation will drop frames if pushed to quickly. Wait the + // interval time to avoid that. + // WebRTC implementation will drop frames if pushed to quickly. Wait the + // interval time to avoid that. + // Set up the stream associated with the engine. + EXPECT_TRUE( + channel_->AddRecvStream(cricket::StreamParams::CreateLegacy(kSsrc))); + EXPECT_TRUE(channel_->SetSink(kSsrc, &renderer_)); + cricket::VideoFormat capture_format( + kVideoWidth, kVideoHeight, + cricket::VideoFormat::FpsToInterval(kFramerate), cricket::FOURCC_I420); + // Set up additional stream 1. + cricket::FakeVideoRenderer renderer1; + EXPECT_FALSE(channel_->SetSink(1, &renderer1)); + EXPECT_TRUE(channel_->AddRecvStream(cricket::StreamParams::CreateLegacy(1))); + EXPECT_TRUE(channel_->SetSink(1, &renderer1)); + EXPECT_TRUE(channel_->AddSendStream(cricket::StreamParams::CreateLegacy(1))); + + webrtc::test::FrameForwarder frame_forwarder1; + cricket::FakeFrameSource frame_source(kVideoWidth, kVideoHeight, + rtc::kNumMicrosecsPerSec / kFramerate); + + // Set up additional stream 2. + cricket::FakeVideoRenderer renderer2; + EXPECT_FALSE(channel_->SetSink(2, &renderer2)); + EXPECT_TRUE(channel_->AddRecvStream(cricket::StreamParams::CreateLegacy(2))); + EXPECT_TRUE(channel_->SetSink(2, &renderer2)); + EXPECT_TRUE(channel_->AddSendStream(cricket::StreamParams::CreateLegacy(2))); + webrtc::test::FrameForwarder frame_forwarder2; + + // State for all the streams. + EXPECT_TRUE(SetOneCodec(DefaultCodec())); + // A limitation in the lmi implementation requires that SetVideoSend() is + // called after SetOneCodec(). + // TODO(hellner): this seems like an unnecessary constraint, fix it. + EXPECT_TRUE(channel_->SetVideoSend(1, nullptr, &frame_forwarder1)); + EXPECT_TRUE(channel_->SetVideoSend(2, nullptr, &frame_forwarder2)); + EXPECT_TRUE(SetSend(true)); + // Test capturer associated with engine. + const int kTestWidth = 160; + const int kTestHeight = 120; + frame_forwarder1.IncomingCapturedFrame(frame_source.GetFrame( + kTestWidth, kTestHeight, webrtc::VideoRotation::kVideoRotation_0, + rtc::kNumMicrosecsPerSec / kFramerate)); + EXPECT_FRAME_ON_RENDERER_WAIT(renderer1, 1, kTestWidth, kTestHeight, + kTimeout); + // Capture a frame with additional capturer2, frames should be received + frame_forwarder2.IncomingCapturedFrame(frame_source.GetFrame( + kTestWidth, kTestHeight, webrtc::VideoRotation::kVideoRotation_0, + rtc::kNumMicrosecsPerSec / kFramerate)); + EXPECT_FRAME_ON_RENDERER_WAIT(renderer2, 1, kTestWidth, kTestHeight, + kTimeout); + // Successfully remove the capturer. + EXPECT_TRUE(channel_->SetVideoSend(kSsrc, nullptr, nullptr)); + // The capturers must be unregistered here as it runs out of it's scope + // next. + EXPECT_TRUE(channel_->SetVideoSend(1, nullptr, nullptr)); + EXPECT_TRUE(channel_->SetVideoSend(2, nullptr, nullptr)); +} + +// Tests empty StreamParams is rejected. +TEST_F(WebRtcVideoChannelBaseTest, RejectEmptyStreamParams) { + // Remove the send stream that was added during Setup. + EXPECT_TRUE(channel_->RemoveSendStream(kSsrc)); + + cricket::StreamParams empty; + EXPECT_FALSE(channel_->AddSendStream(empty)); + EXPECT_TRUE( + channel_->AddSendStream(cricket::StreamParams::CreateLegacy(789u))); +} + +// Test that multiple send streams can be created and deleted properly. +TEST_F(WebRtcVideoChannelBaseTest, MultipleSendStreams) { + // Remove stream added in Setup. I.e. remove stream corresponding to default + // channel. + EXPECT_TRUE(channel_->RemoveSendStream(kSsrc)); + const unsigned int kSsrcsSize = sizeof(kSsrcs4) / sizeof(kSsrcs4[0]); + for (unsigned int i = 0; i < kSsrcsSize; ++i) { + EXPECT_TRUE(channel_->AddSendStream( + cricket::StreamParams::CreateLegacy(kSsrcs4[i]))); + } + // Delete one of the non default channel streams, let the destructor delete + // the remaining ones. + EXPECT_TRUE(channel_->RemoveSendStream(kSsrcs4[kSsrcsSize - 1])); + // Stream should already be deleted. + EXPECT_FALSE(channel_->RemoveSendStream(kSsrcs4[kSsrcsSize - 1])); +} + +TEST_F(WebRtcVideoChannelBaseTest, SendAndReceiveVp8Vga) { + SendAndReceive(GetEngineCodec("VP8")); +} + +TEST_F(WebRtcVideoChannelBaseTest, SendAndReceiveVp8Qvga) { + SendAndReceive(GetEngineCodec("VP8")); +} + +TEST_F(WebRtcVideoChannelBaseTest, SendAndReceiveVp8SvcQqvga) { + SendAndReceive(GetEngineCodec("VP8")); +} + +TEST_F(WebRtcVideoChannelBaseTest, TwoStreamsSendAndReceive) { + // Set a high bitrate to not be downscaled by VP8 due to low initial start + // bitrates. This currently happens at <250k, and two streams sharing 300k + // initially will use QVGA instead of VGA. + // TODO(pbos): Set up the quality scaler so that both senders reliably start + // at QVGA, then verify that instead. + cricket::VideoCodec codec = GetEngineCodec("VP8"); + codec.params[kCodecParamStartBitrate] = "1000000"; + TwoStreamsSendAndReceive(codec); +} + +TEST_F(WebRtcVideoChannelBaseTest, RequestEncoderFallback) { + cricket::VideoSendParameters parameters; + parameters.codecs.push_back(GetEngineCodec("VP9")); + parameters.codecs.push_back(GetEngineCodec("VP8")); + EXPECT_TRUE(channel_->SetSendParameters(parameters)); + + VideoCodec codec; + ASSERT_TRUE(channel_->GetSendCodec(&codec)); + EXPECT_EQ("VP9", codec.name); + + // RequestEncoderFallback will post a task to the worker thread (which is also + // the current thread), hence the ProcessMessages call. + channel_->RequestEncoderFallback(); + rtc::Thread::Current()->ProcessMessages(30); + ASSERT_TRUE(channel_->GetSendCodec(&codec)); + EXPECT_EQ("VP8", codec.name); + + // No other codec to fall back to, keep using VP8. + channel_->RequestEncoderFallback(); + rtc::Thread::Current()->ProcessMessages(30); + ASSERT_TRUE(channel_->GetSendCodec(&codec)); + EXPECT_EQ("VP8", codec.name); +} + +TEST_F(WebRtcVideoChannelBaseTest, RequestEncoderSwitchWithConfig) { + const std::string kParam = "the-param"; + const std::string kPing = "ping"; + const std::string kPong = "pong"; + + cricket::VideoSendParameters parameters; + VideoCodec vp9 = GetEngineCodec("VP9"); + vp9.params[kParam] = kPong; + parameters.codecs.push_back(vp9); + + VideoCodec vp8 = GetEngineCodec("VP8"); + vp8.params[kParam] = kPing; + parameters.codecs.push_back(vp8); + + EXPECT_TRUE(channel_->SetSendParameters(parameters)); + channel_->SetVideoCodecSwitchingEnabled(true); + + VideoCodec codec; + ASSERT_TRUE(channel_->GetSendCodec(&codec)); + EXPECT_THAT(codec.name, Eq("VP9")); + + // RequestEncoderSwitch will post a task to the worker thread (which is also + // the current thread), hence the ProcessMessages call. + webrtc::EncoderSwitchRequestCallback::Config conf1{"VP8", kParam, kPing}; + channel_->RequestEncoderSwitch(conf1); + rtc::Thread::Current()->ProcessMessages(30); + ASSERT_TRUE(channel_->GetSendCodec(&codec)); + EXPECT_THAT(codec.name, Eq("VP8")); + EXPECT_THAT(codec.params, Contains(Pair(kParam, kPing))); + + webrtc::EncoderSwitchRequestCallback::Config conf2{"VP9", kParam, kPong}; + channel_->RequestEncoderSwitch(conf2); + rtc::Thread::Current()->ProcessMessages(30); + ASSERT_TRUE(channel_->GetSendCodec(&codec)); + EXPECT_THAT(codec.name, Eq("VP9")); + EXPECT_THAT(codec.params, Contains(Pair(kParam, kPong))); +} + +TEST_F(WebRtcVideoChannelBaseTest, RequestEncoderSwitchIncorrectParam) { + const std::string kParam = "the-param"; + const std::string kPing = "ping"; + const std::string kPong = "pong"; + + cricket::VideoSendParameters parameters; + VideoCodec vp9 = GetEngineCodec("VP9"); + vp9.params[kParam] = kPong; + parameters.codecs.push_back(vp9); + + VideoCodec vp8 = GetEngineCodec("VP8"); + vp8.params[kParam] = kPing; + parameters.codecs.push_back(vp8); + + EXPECT_TRUE(channel_->SetSendParameters(parameters)); + channel_->SetVideoCodecSwitchingEnabled(true); + + VideoCodec codec; + ASSERT_TRUE(channel_->GetSendCodec(&codec)); + EXPECT_THAT(codec.name, Eq("VP9")); + + // RequestEncoderSwitch will post a task to the worker thread (which is also + // the current thread), hence the ProcessMessages call. + webrtc::EncoderSwitchRequestCallback::Config conf1{"VP8", kParam, kPing}; + channel_->RequestEncoderSwitch(conf1); + rtc::Thread::Current()->ProcessMessages(30); + ASSERT_TRUE(channel_->GetSendCodec(&codec)); + EXPECT_THAT(codec.name, Eq("VP8")); + EXPECT_THAT(codec.params, Contains(Pair(kParam, kPing))); + + // Incorrect conf2.value, expect no codec switch. + webrtc::EncoderSwitchRequestCallback::Config conf2{"VP9", kParam, kPing}; + channel_->RequestEncoderSwitch(conf2); + rtc::Thread::Current()->ProcessMessages(30); + ASSERT_TRUE(channel_->GetSendCodec(&codec)); + EXPECT_THAT(codec.name, Eq("VP8")); + EXPECT_THAT(codec.params, Contains(Pair(kParam, kPing))); +} + +TEST_F(WebRtcVideoChannelBaseTest, + RequestEncoderSwitchWithConfigBeforeEnabling) { + const std::string kParam = "the-param"; + const std::string kPing = "ping"; + const std::string kPong = "pong"; + + cricket::VideoSendParameters parameters; + VideoCodec vp9 = GetEngineCodec("VP9"); + vp9.params[kParam] = kPong; + parameters.codecs.push_back(vp9); + + VideoCodec vp8 = GetEngineCodec("VP8"); + vp8.params[kParam] = kPing; + parameters.codecs.push_back(vp8); + + EXPECT_TRUE(channel_->SetSendParameters(parameters)); + + VideoCodec codec; + ASSERT_TRUE(channel_->GetSendCodec(&codec)); + EXPECT_THAT(codec.name, Eq("VP9")); + + webrtc::EncoderSwitchRequestCallback::Config conf{"VP8", kParam, kPing}; + channel_->RequestEncoderSwitch(conf); + + // Enable codec switching after it has been requested. + channel_->SetVideoCodecSwitchingEnabled(true); + + // RequestEncoderSwitch will post a task to the worker thread (which is also + // the current thread), hence the ProcessMessages call. + rtc::Thread::Current()->ProcessMessages(30); + ASSERT_TRUE(channel_->GetSendCodec(&codec)); + EXPECT_THAT(codec.name, Eq("VP8")); + EXPECT_THAT(codec.params, Contains(Pair(kParam, kPing))); +} + +class WebRtcVideoChannelTest : public WebRtcVideoEngineTest { + public: + WebRtcVideoChannelTest() : WebRtcVideoChannelTest("") {} + explicit WebRtcVideoChannelTest(const char* field_trials) + : WebRtcVideoEngineTest(field_trials), + frame_source_(1280, 720, rtc::kNumMicrosecsPerSec / 30), + last_ssrc_(0) {} + void SetUp() override { + AddSupportedVideoCodecType("VP8"); + AddSupportedVideoCodecType("VP9"); +#if defined(WEBRTC_USE_H264) + AddSupportedVideoCodecType("H264"); +#endif + + fake_call_.reset(new FakeCall()); + channel_.reset(engine_.CreateMediaChannel( + fake_call_.get(), GetMediaConfig(), VideoOptions(), + webrtc::CryptoOptions(), video_bitrate_allocator_factory_.get())); + channel_->OnReadyToSend(true); + last_ssrc_ = 123; + send_parameters_.codecs = engine_.send_codecs(); + recv_parameters_.codecs = engine_.recv_codecs(); + ASSERT_TRUE(channel_->SetSendParameters(send_parameters_)); + } + + cricket::VideoCodec GetEngineCodec(const std::string& name) { + for (const cricket::VideoCodec& engine_codec : engine_.send_codecs()) { + if (absl::EqualsIgnoreCase(name, engine_codec.name)) + return engine_codec; + } + // This point should never be reached. + ADD_FAILURE() << "Unrecognized codec name: " << name; + return cricket::VideoCodec(); + } + + cricket::VideoCodec DefaultCodec() { return GetEngineCodec("VP8"); } + + protected: + FakeVideoSendStream* AddSendStream() { + return AddSendStream(StreamParams::CreateLegacy(++last_ssrc_)); + } + + FakeVideoSendStream* AddSendStream(const StreamParams& sp) { + size_t num_streams = fake_call_->GetVideoSendStreams().size(); + EXPECT_TRUE(channel_->AddSendStream(sp)); + std::vector<FakeVideoSendStream*> streams = + fake_call_->GetVideoSendStreams(); + EXPECT_EQ(num_streams + 1, streams.size()); + return streams[streams.size() - 1]; + } + + std::vector<FakeVideoSendStream*> GetFakeSendStreams() { + return fake_call_->GetVideoSendStreams(); + } + + FakeVideoReceiveStream* AddRecvStream() { + return AddRecvStream(StreamParams::CreateLegacy(++last_ssrc_)); + } + + FakeVideoReceiveStream* AddRecvStream(const StreamParams& sp) { + size_t num_streams = fake_call_->GetVideoReceiveStreams().size(); + EXPECT_TRUE(channel_->AddRecvStream(sp)); + std::vector<FakeVideoReceiveStream*> streams = + fake_call_->GetVideoReceiveStreams(); + EXPECT_EQ(num_streams + 1, streams.size()); + return streams[streams.size() - 1]; + } + + void SetSendCodecsShouldWorkForBitrates(const char* min_bitrate_kbps, + int expected_min_bitrate_bps, + const char* start_bitrate_kbps, + int expected_start_bitrate_bps, + const char* max_bitrate_kbps, + int expected_max_bitrate_bps) { + ExpectSetBitrateParameters(expected_min_bitrate_bps, + expected_start_bitrate_bps, + expected_max_bitrate_bps); + auto& codecs = send_parameters_.codecs; + codecs.clear(); + codecs.push_back(GetEngineCodec("VP8")); + codecs[0].params[kCodecParamMinBitrate] = min_bitrate_kbps; + codecs[0].params[kCodecParamStartBitrate] = start_bitrate_kbps; + codecs[0].params[kCodecParamMaxBitrate] = max_bitrate_kbps; + EXPECT_TRUE(channel_->SetSendParameters(send_parameters_)); + } + + void ExpectSetBitrateParameters(int min_bitrate_bps, + int start_bitrate_bps, + int max_bitrate_bps) { + EXPECT_CALL( + *fake_call_->GetMockTransportControllerSend(), + SetSdpBitrateParameters(AllOf( + Field(&BitrateConstraints::min_bitrate_bps, min_bitrate_bps), + Field(&BitrateConstraints::start_bitrate_bps, start_bitrate_bps), + Field(&BitrateConstraints::max_bitrate_bps, max_bitrate_bps)))); + } + + void ExpectSetMaxBitrate(int max_bitrate_bps) { + EXPECT_CALL(*fake_call_->GetMockTransportControllerSend(), + SetSdpBitrateParameters(Field( + &BitrateConstraints::max_bitrate_bps, max_bitrate_bps))); + } + + void TestExtmapAllowMixedCaller(bool extmap_allow_mixed) { + // For a caller, the answer will be applied in set remote description + // where SetSendParameters() is called. + EXPECT_TRUE( + channel_->AddSendStream(cricket::StreamParams::CreateLegacy(kSsrc))); + send_parameters_.extmap_allow_mixed = extmap_allow_mixed; + EXPECT_TRUE(channel_->SetSendParameters(send_parameters_)); + const webrtc::VideoSendStream::Config& config = + fake_call_->GetVideoSendStreams()[0]->GetConfig(); + EXPECT_EQ(extmap_allow_mixed, config.rtp.extmap_allow_mixed); + } + + void TestExtmapAllowMixedCallee(bool extmap_allow_mixed) { + // For a callee, the answer will be applied in set local description + // where SetExtmapAllowMixed() and AddSendStream() are called. + channel_->SetExtmapAllowMixed(extmap_allow_mixed); + EXPECT_TRUE( + channel_->AddSendStream(cricket::StreamParams::CreateLegacy(kSsrc))); + const webrtc::VideoSendStream::Config& config = + fake_call_->GetVideoSendStreams()[0]->GetConfig(); + EXPECT_EQ(extmap_allow_mixed, config.rtp.extmap_allow_mixed); + } + + void TestSetSendRtpHeaderExtensions(const std::string& ext_uri) { + // Enable extension. + const int id = 1; + cricket::VideoSendParameters parameters = send_parameters_; + parameters.extensions.push_back(RtpExtension(ext_uri, id)); + EXPECT_TRUE(channel_->SetSendParameters(parameters)); + FakeVideoSendStream* send_stream = + AddSendStream(cricket::StreamParams::CreateLegacy(123)); + + // Verify the send extension id. + ASSERT_EQ(1u, send_stream->GetConfig().rtp.extensions.size()); + EXPECT_EQ(id, send_stream->GetConfig().rtp.extensions[0].id); + EXPECT_EQ(ext_uri, send_stream->GetConfig().rtp.extensions[0].uri); + // Verify call with same set of extensions returns true. + EXPECT_TRUE(channel_->SetSendParameters(parameters)); + // Verify that SetSendRtpHeaderExtensions doesn't implicitly add them for + // receivers. + EXPECT_TRUE(AddRecvStream(cricket::StreamParams::CreateLegacy(123)) + ->GetConfig() + .rtp.extensions.empty()); + + // Verify that existing RTP header extensions can be removed. + EXPECT_TRUE(channel_->SetSendParameters(send_parameters_)); + ASSERT_EQ(1u, fake_call_->GetVideoSendStreams().size()); + send_stream = fake_call_->GetVideoSendStreams()[0]; + EXPECT_TRUE(send_stream->GetConfig().rtp.extensions.empty()); + + // Verify that adding receive RTP header extensions adds them for existing + // streams. + EXPECT_TRUE(channel_->SetSendParameters(parameters)); + send_stream = fake_call_->GetVideoSendStreams()[0]; + ASSERT_EQ(1u, send_stream->GetConfig().rtp.extensions.size()); + EXPECT_EQ(id, send_stream->GetConfig().rtp.extensions[0].id); + EXPECT_EQ(ext_uri, send_stream->GetConfig().rtp.extensions[0].uri); + } + + void TestSetRecvRtpHeaderExtensions(const std::string& ext_uri) { + // Enable extension. + const int id = 1; + cricket::VideoRecvParameters parameters = recv_parameters_; + parameters.extensions.push_back(RtpExtension(ext_uri, id)); + EXPECT_TRUE(channel_->SetRecvParameters(parameters)); + + FakeVideoReceiveStream* recv_stream = + AddRecvStream(cricket::StreamParams::CreateLegacy(123)); + + // Verify the recv extension id. + ASSERT_EQ(1u, recv_stream->GetConfig().rtp.extensions.size()); + EXPECT_EQ(id, recv_stream->GetConfig().rtp.extensions[0].id); + EXPECT_EQ(ext_uri, recv_stream->GetConfig().rtp.extensions[0].uri); + // Verify call with same set of extensions returns true. + EXPECT_TRUE(channel_->SetRecvParameters(parameters)); + + // Verify that SetRecvRtpHeaderExtensions doesn't implicitly add them for + // senders. + EXPECT_TRUE(AddSendStream(cricket::StreamParams::CreateLegacy(123)) + ->GetConfig() + .rtp.extensions.empty()); + + // Verify that existing RTP header extensions can be removed. + EXPECT_TRUE(channel_->SetRecvParameters(recv_parameters_)); + ASSERT_EQ(1u, fake_call_->GetVideoReceiveStreams().size()); + recv_stream = fake_call_->GetVideoReceiveStreams()[0]; + EXPECT_TRUE(recv_stream->GetConfig().rtp.extensions.empty()); + + // Verify that adding receive RTP header extensions adds them for existing + // streams. + EXPECT_TRUE(channel_->SetRecvParameters(parameters)); + recv_stream = fake_call_->GetVideoReceiveStreams()[0]; + ASSERT_EQ(1u, recv_stream->GetConfig().rtp.extensions.size()); + EXPECT_EQ(id, recv_stream->GetConfig().rtp.extensions[0].id); + EXPECT_EQ(ext_uri, recv_stream->GetConfig().rtp.extensions[0].uri); + } + + void TestLossNotificationState(bool expect_lntf_enabled) { + AssignDefaultCodec(); + VerifyCodecHasDefaultFeedbackParams(default_codec_, expect_lntf_enabled); + + cricket::VideoSendParameters parameters; + parameters.codecs = engine_.send_codecs(); + EXPECT_TRUE(channel_->SetSendParameters(parameters)); + EXPECT_TRUE(channel_->SetSend(true)); + + // Send side. + FakeVideoSendStream* send_stream = + AddSendStream(cricket::StreamParams::CreateLegacy(1)); + EXPECT_EQ(send_stream->GetConfig().rtp.lntf.enabled, expect_lntf_enabled); + + // Receiver side. + FakeVideoReceiveStream* recv_stream = + AddRecvStream(cricket::StreamParams::CreateLegacy(1)); + EXPECT_EQ(recv_stream->GetConfig().rtp.lntf.enabled, expect_lntf_enabled); + } + + void TestExtensionFilter(const std::vector<std::string>& extensions, + const std::string& expected_extension) { + cricket::VideoSendParameters parameters = send_parameters_; + int expected_id = -1; + int id = 1; + for (const std::string& extension : extensions) { + if (extension == expected_extension) + expected_id = id; + parameters.extensions.push_back(RtpExtension(extension, id++)); + } + EXPECT_TRUE(channel_->SetSendParameters(parameters)); + FakeVideoSendStream* send_stream = + AddSendStream(cricket::StreamParams::CreateLegacy(123)); + + // Verify that only one of them has been set, and that it is the one with + // highest priority (transport sequence number). + ASSERT_EQ(1u, send_stream->GetConfig().rtp.extensions.size()); + EXPECT_EQ(expected_id, send_stream->GetConfig().rtp.extensions[0].id); + EXPECT_EQ(expected_extension, + send_stream->GetConfig().rtp.extensions[0].uri); + } + + void TestDegradationPreference(bool resolution_scaling_enabled, + bool fps_scaling_enabled); + + void TestCpuAdaptation(bool enable_overuse, bool is_screenshare); + void TestReceiverLocalSsrcConfiguration(bool receiver_first); + void TestReceiveUnsignaledSsrcPacket(uint8_t payload_type, + bool expect_created_receive_stream); + + FakeVideoSendStream* SetDenoisingOption( + uint32_t ssrc, + webrtc::test::FrameForwarder* frame_forwarder, + bool enabled) { + cricket::VideoOptions options; + options.video_noise_reduction = enabled; + EXPECT_TRUE(channel_->SetVideoSend(ssrc, &options, frame_forwarder)); + // Options only take effect on the next frame. + frame_forwarder->IncomingCapturedFrame(frame_source_.GetFrame()); + + return fake_call_->GetVideoSendStreams().back(); + } + + FakeVideoSendStream* SetUpSimulcast(bool enabled, bool with_rtx) { + const int kRtxSsrcOffset = 0xDEADBEEF; + last_ssrc_ += 3; + std::vector<uint32_t> ssrcs; + std::vector<uint32_t> rtx_ssrcs; + uint32_t num_streams = enabled ? 3 : 1; + for (uint32_t i = 0; i < num_streams; ++i) { + uint32_t ssrc = last_ssrc_ + i; + ssrcs.push_back(ssrc); + if (with_rtx) { + rtx_ssrcs.push_back(ssrc + kRtxSsrcOffset); + } + } + if (with_rtx) { + return AddSendStream( + cricket::CreateSimWithRtxStreamParams("cname", ssrcs, rtx_ssrcs)); + } + return AddSendStream(CreateSimStreamParams("cname", ssrcs)); + } + + int GetMaxEncoderBitrate() { + std::vector<FakeVideoSendStream*> streams = + fake_call_->GetVideoSendStreams(); + EXPECT_EQ(1u, streams.size()); + FakeVideoSendStream* stream = streams[streams.size() - 1]; + EXPECT_EQ(1u, stream->GetEncoderConfig().number_of_streams); + return stream->GetVideoStreams()[0].max_bitrate_bps; + } + + void SetAndExpectMaxBitrate(int global_max, + int stream_max, + int expected_encoder_bitrate) { + VideoSendParameters limited_send_params = send_parameters_; + limited_send_params.max_bandwidth_bps = global_max; + EXPECT_TRUE(channel_->SetSendParameters(limited_send_params)); + webrtc::RtpParameters parameters = + channel_->GetRtpSendParameters(last_ssrc_); + EXPECT_EQ(1UL, parameters.encodings.size()); + parameters.encodings[0].max_bitrate_bps = stream_max; + EXPECT_TRUE(channel_->SetRtpSendParameters(last_ssrc_, parameters).ok()); + // Read back the parameteres and verify they have the correct value + parameters = channel_->GetRtpSendParameters(last_ssrc_); + EXPECT_EQ(1UL, parameters.encodings.size()); + EXPECT_EQ(stream_max, parameters.encodings[0].max_bitrate_bps); + // Verify that the new value propagated down to the encoder + EXPECT_EQ(expected_encoder_bitrate, GetMaxEncoderBitrate()); + } + + // Values from kSimulcastConfigs in simulcast.cc. + const std::vector<webrtc::VideoStream> GetSimulcastBitrates720p() const { + std::vector<webrtc::VideoStream> layers(3); + layers[0].min_bitrate_bps = 30000; + layers[0].target_bitrate_bps = 150000; + layers[0].max_bitrate_bps = 200000; + layers[1].min_bitrate_bps = 150000; + layers[1].target_bitrate_bps = 500000; + layers[1].max_bitrate_bps = 700000; + layers[2].min_bitrate_bps = 600000; + layers[2].target_bitrate_bps = 2500000; + layers[2].max_bitrate_bps = 2500000; + return layers; + } + + cricket::FakeFrameSource frame_source_; + std::unique_ptr<FakeCall> fake_call_; + std::unique_ptr<VideoMediaChannel> channel_; + cricket::VideoSendParameters send_parameters_; + cricket::VideoRecvParameters recv_parameters_; + uint32_t last_ssrc_; +}; + +TEST_F(WebRtcVideoChannelTest, SetsSyncGroupFromSyncLabel) { + const uint32_t kVideoSsrc = 123; + const std::string kSyncLabel = "AvSyncLabel"; + + cricket::StreamParams sp = cricket::StreamParams::CreateLegacy(kVideoSsrc); + sp.set_stream_ids({kSyncLabel}); + EXPECT_TRUE(channel_->AddRecvStream(sp)); + + EXPECT_EQ(1u, fake_call_->GetVideoReceiveStreams().size()); + EXPECT_EQ(kSyncLabel, + fake_call_->GetVideoReceiveStreams()[0]->GetConfig().sync_group) + << "SyncGroup should be set based on sync_label"; +} + +TEST_F(WebRtcVideoChannelTest, RecvStreamWithSimAndRtx) { + cricket::VideoSendParameters parameters; + parameters.codecs = engine_.send_codecs(); + EXPECT_TRUE(channel_->SetSendParameters(parameters)); + EXPECT_TRUE(channel_->SetSend(true)); + parameters.conference_mode = true; + EXPECT_TRUE(channel_->SetSendParameters(parameters)); + + // Send side. + const std::vector<uint32_t> ssrcs = MAKE_VECTOR(kSsrcs1); + const std::vector<uint32_t> rtx_ssrcs = MAKE_VECTOR(kRtxSsrcs1); + FakeVideoSendStream* send_stream = AddSendStream( + cricket::CreateSimWithRtxStreamParams("cname", ssrcs, rtx_ssrcs)); + + ASSERT_EQ(rtx_ssrcs.size(), send_stream->GetConfig().rtp.rtx.ssrcs.size()); + for (size_t i = 0; i < rtx_ssrcs.size(); ++i) + EXPECT_EQ(rtx_ssrcs[i], send_stream->GetConfig().rtp.rtx.ssrcs[i]); + + // Receiver side. + FakeVideoReceiveStream* recv_stream = AddRecvStream( + cricket::CreateSimWithRtxStreamParams("cname", ssrcs, rtx_ssrcs)); + EXPECT_FALSE( + recv_stream->GetConfig().rtp.rtx_associated_payload_types.empty()); + EXPECT_TRUE(VerifyRtxReceiveAssociations(recv_stream->GetConfig())) + << "RTX should be mapped for all decoders/payload types."; + EXPECT_TRUE(HasRtxReceiveAssociation(recv_stream->GetConfig(), + GetEngineCodec("red").id)) + << "RTX should be mapped for the RED payload type"; + + EXPECT_EQ(rtx_ssrcs[0], recv_stream->GetConfig().rtp.rtx_ssrc); +} + +TEST_F(WebRtcVideoChannelTest, RecvStreamWithRtx) { + // Setup one channel with an associated RTX stream. + cricket::StreamParams params = + cricket::StreamParams::CreateLegacy(kSsrcs1[0]); + params.AddFidSsrc(kSsrcs1[0], kRtxSsrcs1[0]); + FakeVideoReceiveStream* recv_stream = AddRecvStream(params); + EXPECT_EQ(kRtxSsrcs1[0], recv_stream->GetConfig().rtp.rtx_ssrc); + + EXPECT_TRUE(VerifyRtxReceiveAssociations(recv_stream->GetConfig())) + << "RTX should be mapped for all decoders/payload types."; + EXPECT_TRUE(HasRtxReceiveAssociation(recv_stream->GetConfig(), + GetEngineCodec("red").id)) + << "RTX should be mapped for the RED payload type"; +} + +TEST_F(WebRtcVideoChannelTest, RecvStreamNoRtx) { + // Setup one channel without an associated RTX stream. + cricket::StreamParams params = + cricket::StreamParams::CreateLegacy(kSsrcs1[0]); + FakeVideoReceiveStream* recv_stream = AddRecvStream(params); + ASSERT_EQ(0U, recv_stream->GetConfig().rtp.rtx_ssrc); +} + +// Test propagation of extmap allow mixed setting. +TEST_F(WebRtcVideoChannelTest, SetExtmapAllowMixedAsCaller) { + TestExtmapAllowMixedCaller(/*extmap_allow_mixed=*/true); +} +TEST_F(WebRtcVideoChannelTest, SetExtmapAllowMixedDisabledAsCaller) { + TestExtmapAllowMixedCaller(/*extmap_allow_mixed=*/false); +} +TEST_F(WebRtcVideoChannelTest, SetExtmapAllowMixedAsCallee) { + TestExtmapAllowMixedCallee(/*extmap_allow_mixed=*/true); +} +TEST_F(WebRtcVideoChannelTest, SetExtmapAllowMixedDisabledAsCallee) { + TestExtmapAllowMixedCallee(/*extmap_allow_mixed=*/false); +} + +TEST_F(WebRtcVideoChannelTest, NoHeaderExtesionsByDefault) { + FakeVideoSendStream* send_stream = + AddSendStream(cricket::StreamParams::CreateLegacy(kSsrcs1[0])); + ASSERT_TRUE(send_stream->GetConfig().rtp.extensions.empty()); + + FakeVideoReceiveStream* recv_stream = + AddRecvStream(cricket::StreamParams::CreateLegacy(kSsrcs1[0])); + ASSERT_TRUE(recv_stream->GetConfig().rtp.extensions.empty()); +} + +// Test support for RTP timestamp offset header extension. +TEST_F(WebRtcVideoChannelTest, SendRtpTimestampOffsetHeaderExtensions) { + TestSetSendRtpHeaderExtensions(RtpExtension::kTimestampOffsetUri); +} + +TEST_F(WebRtcVideoChannelTest, RecvRtpTimestampOffsetHeaderExtensions) { + TestSetRecvRtpHeaderExtensions(RtpExtension::kTimestampOffsetUri); +} + +// Test support for absolute send time header extension. +TEST_F(WebRtcVideoChannelTest, SendAbsoluteSendTimeHeaderExtensions) { + TestSetSendRtpHeaderExtensions(RtpExtension::kAbsSendTimeUri); +} + +TEST_F(WebRtcVideoChannelTest, RecvAbsoluteSendTimeHeaderExtensions) { + TestSetRecvRtpHeaderExtensions(RtpExtension::kAbsSendTimeUri); +} + +TEST_F(WebRtcVideoChannelTest, FiltersExtensionsPicksTransportSeqNum) { + webrtc::test::ScopedFieldTrials override_field_trials_( + "WebRTC-FilterAbsSendTimeExtension/Enabled/"); + // Enable three redundant extensions. + std::vector<std::string> extensions; + extensions.push_back(RtpExtension::kAbsSendTimeUri); + extensions.push_back(RtpExtension::kTimestampOffsetUri); + extensions.push_back(RtpExtension::kTransportSequenceNumberUri); + TestExtensionFilter(extensions, RtpExtension::kTransportSequenceNumberUri); +} + +TEST_F(WebRtcVideoChannelTest, FiltersExtensionsPicksAbsSendTime) { + // Enable two redundant extensions. + std::vector<std::string> extensions; + extensions.push_back(RtpExtension::kAbsSendTimeUri); + extensions.push_back(RtpExtension::kTimestampOffsetUri); + TestExtensionFilter(extensions, RtpExtension::kAbsSendTimeUri); +} + +// Test support for transport sequence number header extension. +TEST_F(WebRtcVideoChannelTest, SendTransportSequenceNumberHeaderExtensions) { + TestSetSendRtpHeaderExtensions(RtpExtension::kTransportSequenceNumberUri); +} +TEST_F(WebRtcVideoChannelTest, RecvTransportSequenceNumberHeaderExtensions) { + TestSetRecvRtpHeaderExtensions(RtpExtension::kTransportSequenceNumberUri); +} + +// Test support for video rotation header extension. +TEST_F(WebRtcVideoChannelTest, SendVideoRotationHeaderExtensions) { + TestSetSendRtpHeaderExtensions(RtpExtension::kVideoRotationUri); +} +TEST_F(WebRtcVideoChannelTest, RecvVideoRotationHeaderExtensions) { + TestSetRecvRtpHeaderExtensions(RtpExtension::kVideoRotationUri); +} + +TEST_F(WebRtcVideoChannelTest, IdenticalSendExtensionsDoesntRecreateStream) { + const int kAbsSendTimeId = 1; + const int kVideoRotationId = 2; + send_parameters_.extensions.push_back( + RtpExtension(RtpExtension::kAbsSendTimeUri, kAbsSendTimeId)); + send_parameters_.extensions.push_back( + RtpExtension(RtpExtension::kVideoRotationUri, kVideoRotationId)); + + EXPECT_TRUE(channel_->SetSendParameters(send_parameters_)); + FakeVideoSendStream* send_stream = + AddSendStream(cricket::StreamParams::CreateLegacy(123)); + + EXPECT_EQ(1, fake_call_->GetNumCreatedSendStreams()); + ASSERT_EQ(2u, send_stream->GetConfig().rtp.extensions.size()); + + // Setting the same extensions (even if in different order) shouldn't + // reallocate the stream. + absl::c_reverse(send_parameters_.extensions); + EXPECT_TRUE(channel_->SetSendParameters(send_parameters_)); + + EXPECT_EQ(1, fake_call_->GetNumCreatedSendStreams()); + + // Setting different extensions should recreate the stream. + send_parameters_.extensions.resize(1); + EXPECT_TRUE(channel_->SetSendParameters(send_parameters_)); + + EXPECT_EQ(2, fake_call_->GetNumCreatedSendStreams()); +} + +TEST_F(WebRtcVideoChannelTest, IdenticalRecvExtensionsDoesntRecreateStream) { + const int kTOffsetId = 1; + const int kAbsSendTimeId = 2; + const int kVideoRotationId = 3; + recv_parameters_.extensions.push_back( + RtpExtension(RtpExtension::kAbsSendTimeUri, kAbsSendTimeId)); + recv_parameters_.extensions.push_back( + RtpExtension(RtpExtension::kTimestampOffsetUri, kTOffsetId)); + recv_parameters_.extensions.push_back( + RtpExtension(RtpExtension::kVideoRotationUri, kVideoRotationId)); + + EXPECT_TRUE(channel_->SetRecvParameters(recv_parameters_)); + FakeVideoReceiveStream* recv_stream = + AddRecvStream(cricket::StreamParams::CreateLegacy(123)); + + EXPECT_EQ(1, fake_call_->GetNumCreatedReceiveStreams()); + ASSERT_EQ(3u, recv_stream->GetConfig().rtp.extensions.size()); + + // Setting the same extensions (even if in different order) shouldn't + // reallocate the stream. + absl::c_reverse(recv_parameters_.extensions); + EXPECT_TRUE(channel_->SetRecvParameters(recv_parameters_)); + + EXPECT_EQ(1, fake_call_->GetNumCreatedReceiveStreams()); + + // Setting different extensions should recreate the stream. + recv_parameters_.extensions.resize(1); + EXPECT_TRUE(channel_->SetRecvParameters(recv_parameters_)); + + EXPECT_EQ(2, fake_call_->GetNumCreatedReceiveStreams()); +} + +TEST_F(WebRtcVideoChannelTest, + SetSendRtpHeaderExtensionsExcludeUnsupportedExtensions) { + const int kUnsupportedId = 1; + const int kTOffsetId = 2; + + send_parameters_.extensions.push_back( + RtpExtension(kUnsupportedExtensionName, kUnsupportedId)); + send_parameters_.extensions.push_back( + RtpExtension(RtpExtension::kTimestampOffsetUri, kTOffsetId)); + EXPECT_TRUE(channel_->SetSendParameters(send_parameters_)); + FakeVideoSendStream* send_stream = + AddSendStream(cricket::StreamParams::CreateLegacy(123)); + + // Only timestamp offset extension is set to send stream, + // unsupported rtp extension is ignored. + ASSERT_EQ(1u, send_stream->GetConfig().rtp.extensions.size()); + EXPECT_STREQ(RtpExtension::kTimestampOffsetUri, + send_stream->GetConfig().rtp.extensions[0].uri.c_str()); +} + +TEST_F(WebRtcVideoChannelTest, + SetRecvRtpHeaderExtensionsExcludeUnsupportedExtensions) { + const int kUnsupportedId = 1; + const int kTOffsetId = 2; + + recv_parameters_.extensions.push_back( + RtpExtension(kUnsupportedExtensionName, kUnsupportedId)); + recv_parameters_.extensions.push_back( + RtpExtension(RtpExtension::kTimestampOffsetUri, kTOffsetId)); + EXPECT_TRUE(channel_->SetRecvParameters(recv_parameters_)); + FakeVideoReceiveStream* recv_stream = + AddRecvStream(cricket::StreamParams::CreateLegacy(123)); + + // Only timestamp offset extension is set to receive stream, + // unsupported rtp extension is ignored. + ASSERT_EQ(1u, recv_stream->GetConfig().rtp.extensions.size()); + EXPECT_STREQ(RtpExtension::kTimestampOffsetUri, + recv_stream->GetConfig().rtp.extensions[0].uri.c_str()); +} + +TEST_F(WebRtcVideoChannelTest, SetSendRtpHeaderExtensionsRejectsIncorrectIds) { + const int kIncorrectIds[] = {-2, -1, 0, 15, 16}; + for (size_t i = 0; i < arraysize(kIncorrectIds); ++i) { + send_parameters_.extensions.push_back( + RtpExtension(RtpExtension::kTimestampOffsetUri, kIncorrectIds[i])); + EXPECT_FALSE(channel_->SetSendParameters(send_parameters_)) + << "Bad extension id '" << kIncorrectIds[i] << "' accepted."; + } +} + +TEST_F(WebRtcVideoChannelTest, SetRecvRtpHeaderExtensionsRejectsIncorrectIds) { + const int kIncorrectIds[] = {-2, -1, 0, 15, 16}; + for (size_t i = 0; i < arraysize(kIncorrectIds); ++i) { + recv_parameters_.extensions.push_back( + RtpExtension(RtpExtension::kTimestampOffsetUri, kIncorrectIds[i])); + EXPECT_FALSE(channel_->SetRecvParameters(recv_parameters_)) + << "Bad extension id '" << kIncorrectIds[i] << "' accepted."; + } +} + +TEST_F(WebRtcVideoChannelTest, SetSendRtpHeaderExtensionsRejectsDuplicateIds) { + const int id = 1; + send_parameters_.extensions.push_back( + RtpExtension(RtpExtension::kTimestampOffsetUri, id)); + send_parameters_.extensions.push_back( + RtpExtension(RtpExtension::kAbsSendTimeUri, id)); + EXPECT_FALSE(channel_->SetSendParameters(send_parameters_)); + + // Duplicate entries are also not supported. + send_parameters_.extensions.clear(); + send_parameters_.extensions.push_back( + RtpExtension(RtpExtension::kTimestampOffsetUri, id)); + send_parameters_.extensions.push_back(send_parameters_.extensions.back()); + EXPECT_FALSE(channel_->SetSendParameters(send_parameters_)); +} + +TEST_F(WebRtcVideoChannelTest, SetRecvRtpHeaderExtensionsRejectsDuplicateIds) { + const int id = 1; + recv_parameters_.extensions.push_back( + RtpExtension(RtpExtension::kTimestampOffsetUri, id)); + recv_parameters_.extensions.push_back( + RtpExtension(RtpExtension::kAbsSendTimeUri, id)); + EXPECT_FALSE(channel_->SetRecvParameters(recv_parameters_)); + + // Duplicate entries are also not supported. + recv_parameters_.extensions.clear(); + recv_parameters_.extensions.push_back( + RtpExtension(RtpExtension::kTimestampOffsetUri, id)); + recv_parameters_.extensions.push_back(recv_parameters_.extensions.back()); + EXPECT_FALSE(channel_->SetRecvParameters(recv_parameters_)); +} + +TEST_F(WebRtcVideoChannelTest, AddRecvStreamOnlyUsesOneReceiveStream) { + EXPECT_TRUE(channel_->AddRecvStream(cricket::StreamParams::CreateLegacy(1))); + EXPECT_EQ(1u, fake_call_->GetVideoReceiveStreams().size()); +} + +TEST_F(WebRtcVideoChannelTest, RtcpIsCompoundByDefault) { + FakeVideoReceiveStream* stream = AddRecvStream(); + EXPECT_EQ(webrtc::RtcpMode::kCompound, stream->GetConfig().rtp.rtcp_mode); +} + +TEST_F(WebRtcVideoChannelTest, TransportCcIsEnabledByDefault) { + FakeVideoReceiveStream* stream = AddRecvStream(); + EXPECT_TRUE(stream->GetConfig().rtp.transport_cc); +} + +TEST_F(WebRtcVideoChannelTest, TransportCcCanBeEnabledAndDisabled) { + FakeVideoReceiveStream* stream = AddRecvStream(); + EXPECT_TRUE(stream->GetConfig().rtp.transport_cc); + + // Verify that transport cc feedback is turned off when send(!) codecs without + // transport cc feedback are set. + cricket::VideoSendParameters parameters; + parameters.codecs.push_back(RemoveFeedbackParams(GetEngineCodec("VP8"))); + EXPECT_TRUE(parameters.codecs[0].feedback_params.params().empty()); + EXPECT_TRUE(channel_->SetSendParameters(parameters)); + stream = fake_call_->GetVideoReceiveStreams()[0]; + EXPECT_FALSE(stream->GetConfig().rtp.transport_cc); + + // Verify that transport cc feedback is turned on when setting default codecs + // since the default codecs have transport cc feedback enabled. + parameters.codecs = engine_.send_codecs(); + EXPECT_TRUE(channel_->SetSendParameters(parameters)); + stream = fake_call_->GetVideoReceiveStreams()[0]; + EXPECT_TRUE(stream->GetConfig().rtp.transport_cc); +} + +TEST_F(WebRtcVideoChannelTest, LossNotificationIsDisabledByDefault) { + TestLossNotificationState(false); +} + +TEST_F(WebRtcVideoChannelTest, LossNotificationIsEnabledByFieldTrial) { + RTC_DCHECK(!override_field_trials_); + override_field_trials_ = std::make_unique<webrtc::test::ScopedFieldTrials>( + "WebRTC-RtcpLossNotification/Enabled/"); + SetUp(); + TestLossNotificationState(true); +} + +TEST_F(WebRtcVideoChannelTest, LossNotificationCanBeEnabledAndDisabled) { + RTC_DCHECK(!override_field_trials_); + override_field_trials_ = std::make_unique<webrtc::test::ScopedFieldTrials>( + "WebRTC-RtcpLossNotification/Enabled/"); + SetUp(); + + AssignDefaultCodec(); + VerifyCodecHasDefaultFeedbackParams(default_codec_, true); + + { + cricket::VideoSendParameters parameters; + parameters.codecs = engine_.send_codecs(); + EXPECT_TRUE(channel_->SetSendParameters(parameters)); + EXPECT_TRUE(channel_->SetSend(true)); + } + + // Start with LNTF enabled. + FakeVideoSendStream* send_stream = + AddSendStream(cricket::StreamParams::CreateLegacy(1)); + ASSERT_TRUE(send_stream->GetConfig().rtp.lntf.enabled); + FakeVideoReceiveStream* recv_stream = + AddRecvStream(cricket::StreamParams::CreateLegacy(1)); + ASSERT_TRUE(recv_stream->GetConfig().rtp.lntf.enabled); + + // Verify that LNTF is turned off when send(!) codecs without LNTF are set. + cricket::VideoSendParameters parameters; + parameters.codecs.push_back(RemoveFeedbackParams(GetEngineCodec("VP8"))); + EXPECT_TRUE(parameters.codecs[0].feedback_params.params().empty()); + EXPECT_TRUE(channel_->SetSendParameters(parameters)); + recv_stream = fake_call_->GetVideoReceiveStreams()[0]; + EXPECT_FALSE(recv_stream->GetConfig().rtp.lntf.enabled); + send_stream = fake_call_->GetVideoSendStreams()[0]; + EXPECT_FALSE(send_stream->GetConfig().rtp.lntf.enabled); + + // Setting the default codecs again, including VP8, turns LNTF back on. + parameters.codecs = engine_.send_codecs(); + EXPECT_TRUE(channel_->SetSendParameters(parameters)); + recv_stream = fake_call_->GetVideoReceiveStreams()[0]; + EXPECT_TRUE(recv_stream->GetConfig().rtp.lntf.enabled); + send_stream = fake_call_->GetVideoSendStreams()[0]; + EXPECT_TRUE(send_stream->GetConfig().rtp.lntf.enabled); +} + +TEST_F(WebRtcVideoChannelTest, NackIsEnabledByDefault) { + AssignDefaultCodec(); + VerifyCodecHasDefaultFeedbackParams(default_codec_, false); + + cricket::VideoSendParameters parameters; + parameters.codecs = engine_.send_codecs(); + EXPECT_TRUE(channel_->SetSendParameters(parameters)); + EXPECT_TRUE(channel_->SetSend(true)); + + // Send side. + FakeVideoSendStream* send_stream = + AddSendStream(cricket::StreamParams::CreateLegacy(1)); + EXPECT_GT(send_stream->GetConfig().rtp.nack.rtp_history_ms, 0); + + // Receiver side. + FakeVideoReceiveStream* recv_stream = + AddRecvStream(cricket::StreamParams::CreateLegacy(1)); + EXPECT_GT(recv_stream->GetConfig().rtp.nack.rtp_history_ms, 0); + + // Nack history size should match between sender and receiver. + EXPECT_EQ(send_stream->GetConfig().rtp.nack.rtp_history_ms, + recv_stream->GetConfig().rtp.nack.rtp_history_ms); +} + +TEST_F(WebRtcVideoChannelTest, NackCanBeEnabledAndDisabled) { + FakeVideoSendStream* send_stream = AddSendStream(); + FakeVideoReceiveStream* recv_stream = AddRecvStream(); + + EXPECT_GT(recv_stream->GetConfig().rtp.nack.rtp_history_ms, 0); + EXPECT_GT(send_stream->GetConfig().rtp.nack.rtp_history_ms, 0); + + // Verify that NACK is turned off when send(!) codecs without NACK are set. + cricket::VideoSendParameters parameters; + parameters.codecs.push_back(RemoveFeedbackParams(GetEngineCodec("VP8"))); + EXPECT_TRUE(parameters.codecs[0].feedback_params.params().empty()); + EXPECT_TRUE(channel_->SetSendParameters(parameters)); + recv_stream = fake_call_->GetVideoReceiveStreams()[0]; + EXPECT_EQ(0, recv_stream->GetConfig().rtp.nack.rtp_history_ms); + send_stream = fake_call_->GetVideoSendStreams()[0]; + EXPECT_EQ(0, send_stream->GetConfig().rtp.nack.rtp_history_ms); + + // Verify that NACK is turned on when setting default codecs since the + // default codecs have NACK enabled. + parameters.codecs = engine_.send_codecs(); + EXPECT_TRUE(channel_->SetSendParameters(parameters)); + recv_stream = fake_call_->GetVideoReceiveStreams()[0]; + EXPECT_GT(recv_stream->GetConfig().rtp.nack.rtp_history_ms, 0); + send_stream = fake_call_->GetVideoSendStreams()[0]; + EXPECT_GT(send_stream->GetConfig().rtp.nack.rtp_history_ms, 0); +} + +// This test verifies that new frame sizes reconfigures encoders even though not +// (yet) sending. The purpose of this is to permit encoding as quickly as +// possible once we start sending. Likely the frames being input are from the +// same source that will be sent later, which just means that we're ready +// earlier. +TEST_F(WebRtcVideoChannelTest, ReconfiguresEncodersWhenNotSending) { + cricket::VideoSendParameters parameters; + parameters.codecs.push_back(GetEngineCodec("VP8")); + ASSERT_TRUE(channel_->SetSendParameters(parameters)); + channel_->SetSend(false); + + FakeVideoSendStream* stream = AddSendStream(); + + // No frames entered. + std::vector<webrtc::VideoStream> streams = stream->GetVideoStreams(); + EXPECT_EQ(0u, streams[0].width); + EXPECT_EQ(0u, streams[0].height); + + webrtc::test::FrameForwarder frame_forwarder; + cricket::FakeFrameSource frame_source(1280, 720, + rtc::kNumMicrosecsPerSec / 30); + + EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, nullptr, &frame_forwarder)); + frame_forwarder.IncomingCapturedFrame(frame_source.GetFrame()); + + // Frame entered, should be reconfigured to new dimensions. + streams = stream->GetVideoStreams(); + EXPECT_EQ(rtc::checked_cast<size_t>(1280), streams[0].width); + EXPECT_EQ(rtc::checked_cast<size_t>(720), streams[0].height); + + EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, nullptr, nullptr)); +} + +TEST_F(WebRtcVideoChannelTest, UsesCorrectSettingsForScreencast) { + static const int kScreenshareMinBitrateKbps = 800; + cricket::VideoCodec codec = GetEngineCodec("VP8"); + cricket::VideoSendParameters parameters; + parameters.codecs.push_back(codec); + EXPECT_TRUE(channel_->SetSendParameters(parameters)); + AddSendStream(); + + webrtc::test::FrameForwarder frame_forwarder; + cricket::FakeFrameSource frame_source(1280, 720, + rtc::kNumMicrosecsPerSec / 30); + VideoOptions min_bitrate_options; + min_bitrate_options.screencast_min_bitrate_kbps = kScreenshareMinBitrateKbps; + EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, &min_bitrate_options, + &frame_forwarder)); + + EXPECT_TRUE(channel_->SetSend(true)); + + frame_forwarder.IncomingCapturedFrame(frame_source.GetFrame()); + ASSERT_EQ(1u, fake_call_->GetVideoSendStreams().size()); + FakeVideoSendStream* send_stream = fake_call_->GetVideoSendStreams().front(); + + EXPECT_EQ(1, send_stream->GetNumberOfSwappedFrames()); + + // Verify non-screencast settings. + webrtc::VideoEncoderConfig encoder_config = + send_stream->GetEncoderConfig().Copy(); + EXPECT_EQ(webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo, + encoder_config.content_type); + std::vector<webrtc::VideoStream> streams = send_stream->GetVideoStreams(); + EXPECT_EQ(rtc::checked_cast<size_t>(1280), streams.front().width); + EXPECT_EQ(rtc::checked_cast<size_t>(720), streams.front().height); + EXPECT_EQ(0, encoder_config.min_transmit_bitrate_bps) + << "Non-screenshare shouldn't use min-transmit bitrate."; + + EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, nullptr, nullptr)); + EXPECT_EQ(1, send_stream->GetNumberOfSwappedFrames()); + VideoOptions screencast_options; + screencast_options.is_screencast = true; + EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, &screencast_options, + &frame_forwarder)); + frame_forwarder.IncomingCapturedFrame(frame_source.GetFrame()); + // Send stream recreated after option change. + ASSERT_EQ(2, fake_call_->GetNumCreatedSendStreams()); + send_stream = fake_call_->GetVideoSendStreams().front(); + EXPECT_EQ(1, send_stream->GetNumberOfSwappedFrames()); + + // Verify screencast settings. + encoder_config = send_stream->GetEncoderConfig().Copy(); + EXPECT_EQ(webrtc::VideoEncoderConfig::ContentType::kScreen, + encoder_config.content_type); + EXPECT_EQ(kScreenshareMinBitrateKbps * 1000, + encoder_config.min_transmit_bitrate_bps); + + streams = send_stream->GetVideoStreams(); + EXPECT_EQ(rtc::checked_cast<size_t>(1280), streams.front().width); + EXPECT_EQ(rtc::checked_cast<size_t>(720), streams.front().height); + EXPECT_FALSE(streams[0].num_temporal_layers.has_value()); + EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, nullptr, nullptr)); +} + +TEST_F(WebRtcVideoChannelTest, + ConferenceModeScreencastConfiguresTemporalLayer) { + static const int kConferenceScreencastTemporalBitrateBps = 200 * 1000; + send_parameters_.conference_mode = true; + channel_->SetSendParameters(send_parameters_); + + AddSendStream(); + VideoOptions options; + options.is_screencast = true; + webrtc::test::FrameForwarder frame_forwarder; + cricket::FakeFrameSource frame_source(1280, 720, + rtc::kNumMicrosecsPerSec / 30); + EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, &options, &frame_forwarder)); + EXPECT_TRUE(channel_->SetSend(true)); + + frame_forwarder.IncomingCapturedFrame(frame_source.GetFrame()); + ASSERT_EQ(1u, fake_call_->GetVideoSendStreams().size()); + FakeVideoSendStream* send_stream = fake_call_->GetVideoSendStreams().front(); + + webrtc::VideoEncoderConfig encoder_config = + send_stream->GetEncoderConfig().Copy(); + + // Verify screencast settings. + encoder_config = send_stream->GetEncoderConfig().Copy(); + EXPECT_EQ(webrtc::VideoEncoderConfig::ContentType::kScreen, + encoder_config.content_type); + + std::vector<webrtc::VideoStream> streams = send_stream->GetVideoStreams(); + ASSERT_EQ(1u, streams.size()); + ASSERT_EQ(2u, streams[0].num_temporal_layers); + EXPECT_EQ(kConferenceScreencastTemporalBitrateBps, + streams[0].target_bitrate_bps); + + EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, nullptr, nullptr)); +} + +TEST_F(WebRtcVideoChannelTest, SuspendBelowMinBitrateDisabledByDefault) { + FakeVideoSendStream* stream = AddSendStream(); + EXPECT_FALSE(stream->GetConfig().suspend_below_min_bitrate); +} + +TEST_F(WebRtcVideoChannelTest, SetMediaConfigSuspendBelowMinBitrate) { + MediaConfig media_config = GetMediaConfig(); + media_config.video.suspend_below_min_bitrate = true; + + channel_.reset(engine_.CreateMediaChannel( + fake_call_.get(), media_config, VideoOptions(), webrtc::CryptoOptions(), + video_bitrate_allocator_factory_.get())); + channel_->OnReadyToSend(true); + + channel_->SetSendParameters(send_parameters_); + + FakeVideoSendStream* stream = AddSendStream(); + EXPECT_TRUE(stream->GetConfig().suspend_below_min_bitrate); + + media_config.video.suspend_below_min_bitrate = false; + channel_.reset(engine_.CreateMediaChannel( + fake_call_.get(), media_config, VideoOptions(), webrtc::CryptoOptions(), + video_bitrate_allocator_factory_.get())); + channel_->OnReadyToSend(true); + + channel_->SetSendParameters(send_parameters_); + + stream = AddSendStream(); + EXPECT_FALSE(stream->GetConfig().suspend_below_min_bitrate); +} + +TEST_F(WebRtcVideoChannelTest, Vp8DenoisingEnabledByDefault) { + FakeVideoSendStream* stream = AddSendStream(); + webrtc::VideoCodecVP8 vp8_settings; + ASSERT_TRUE(stream->GetVp8Settings(&vp8_settings)) << "No VP8 config set."; + EXPECT_TRUE(vp8_settings.denoisingOn); +} + +TEST_F(WebRtcVideoChannelTest, VerifyVp8SpecificSettings) { + cricket::VideoSendParameters parameters; + parameters.codecs.push_back(GetEngineCodec("VP8")); + ASSERT_TRUE(channel_->SetSendParameters(parameters)); + + // Single-stream settings should apply with RTX as well (verifies that we + // check number of regular SSRCs and not StreamParams::ssrcs which contains + // both RTX and regular SSRCs). + FakeVideoSendStream* stream = SetUpSimulcast(false, true); + + webrtc::test::FrameForwarder frame_forwarder; + EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, nullptr, &frame_forwarder)); + channel_->SetSend(true); + + frame_forwarder.IncomingCapturedFrame(frame_source_.GetFrame()); + + webrtc::VideoCodecVP8 vp8_settings; + ASSERT_TRUE(stream->GetVp8Settings(&vp8_settings)) << "No VP8 config set."; + EXPECT_TRUE(vp8_settings.denoisingOn) + << "VP8 denoising should be on by default."; + + stream = SetDenoisingOption(last_ssrc_, &frame_forwarder, false); + + ASSERT_TRUE(stream->GetVp8Settings(&vp8_settings)) << "No VP8 config set."; + EXPECT_FALSE(vp8_settings.denoisingOn); + EXPECT_TRUE(vp8_settings.automaticResizeOn); + EXPECT_TRUE(vp8_settings.frameDroppingOn); + + stream = SetDenoisingOption(last_ssrc_, &frame_forwarder, true); + + ASSERT_TRUE(stream->GetVp8Settings(&vp8_settings)) << "No VP8 config set."; + EXPECT_TRUE(vp8_settings.denoisingOn); + EXPECT_TRUE(vp8_settings.automaticResizeOn); + EXPECT_TRUE(vp8_settings.frameDroppingOn); + + EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, nullptr, nullptr)); + stream = SetUpSimulcast(true, false); + EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, nullptr, &frame_forwarder)); + channel_->SetSend(true); + frame_forwarder.IncomingCapturedFrame(frame_source_.GetFrame()); + + EXPECT_EQ(3u, stream->GetVideoStreams().size()); + ASSERT_TRUE(stream->GetVp8Settings(&vp8_settings)) << "No VP8 config set."; + // Autmatic resize off when using simulcast. + EXPECT_FALSE(vp8_settings.automaticResizeOn); + EXPECT_TRUE(vp8_settings.frameDroppingOn); + + // In screen-share mode, denoising is forced off. + VideoOptions options; + options.is_screencast = true; + EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, &options, &frame_forwarder)); + + stream = SetDenoisingOption(last_ssrc_, &frame_forwarder, false); + + EXPECT_EQ(3u, stream->GetVideoStreams().size()); + ASSERT_TRUE(stream->GetVp8Settings(&vp8_settings)) << "No VP8 config set."; + EXPECT_FALSE(vp8_settings.denoisingOn); + // Resizing and frame dropping always off for screen sharing. + EXPECT_FALSE(vp8_settings.automaticResizeOn); + EXPECT_FALSE(vp8_settings.frameDroppingOn); + + stream = SetDenoisingOption(last_ssrc_, &frame_forwarder, true); + + ASSERT_TRUE(stream->GetVp8Settings(&vp8_settings)) << "No VP8 config set."; + EXPECT_FALSE(vp8_settings.denoisingOn); + EXPECT_FALSE(vp8_settings.automaticResizeOn); + EXPECT_FALSE(vp8_settings.frameDroppingOn); + + EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, nullptr, nullptr)); +} + +// Test that setting the same options doesn't result in the encoder being +// reconfigured. +TEST_F(WebRtcVideoChannelTest, SetIdenticalOptionsDoesntReconfigureEncoder) { + VideoOptions options; + webrtc::test::FrameForwarder frame_forwarder; + + AddSendStream(); + cricket::VideoSendParameters parameters; + parameters.codecs.push_back(GetEngineCodec("VP8")); + ASSERT_TRUE(channel_->SetSendParameters(parameters)); + FakeVideoSendStream* send_stream = fake_call_->GetVideoSendStreams().front(); + + EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, &options, &frame_forwarder)); + EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, &options, &frame_forwarder)); + frame_forwarder.IncomingCapturedFrame(frame_source_.GetFrame()); + // Expect 1 reconfigurations at this point from the initial configuration. + EXPECT_EQ(1, send_stream->num_encoder_reconfigurations()); + + // Set the options one more time and expect no additional reconfigurations. + EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, &options, &frame_forwarder)); + EXPECT_EQ(1, send_stream->num_encoder_reconfigurations()); + + // Change |options| and expect 2 reconfigurations. + options.video_noise_reduction = true; + EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, &options, &frame_forwarder)); + EXPECT_EQ(2, send_stream->num_encoder_reconfigurations()); + + EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, nullptr, nullptr)); +} + +class Vp9SettingsTest : public WebRtcVideoChannelTest { + public: + Vp9SettingsTest() : Vp9SettingsTest("") {} + explicit Vp9SettingsTest(const char* field_trials) + : WebRtcVideoChannelTest(field_trials) { + encoder_factory_->AddSupportedVideoCodecType("VP9"); + } + virtual ~Vp9SettingsTest() {} + + protected: + void TearDown() override { + // Remove references to encoder_factory_ since this will be destroyed + // before channel_ and engine_. + ASSERT_TRUE(channel_->SetSendParameters(send_parameters_)); + } +}; + +TEST_F(Vp9SettingsTest, VerifyVp9SpecificSettings) { + cricket::VideoSendParameters parameters; + parameters.codecs.push_back(GetEngineCodec("VP9")); + ASSERT_TRUE(channel_->SetSendParameters(parameters)); + + FakeVideoSendStream* stream = SetUpSimulcast(false, false); + + webrtc::test::FrameForwarder frame_forwarder; + EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, nullptr, &frame_forwarder)); + channel_->SetSend(true); + + frame_forwarder.IncomingCapturedFrame(frame_source_.GetFrame()); + + webrtc::VideoCodecVP9 vp9_settings; + ASSERT_TRUE(stream->GetVp9Settings(&vp9_settings)) << "No VP9 config set."; + EXPECT_TRUE(vp9_settings.denoisingOn) + << "VP9 denoising should be on by default."; + + stream = SetDenoisingOption(last_ssrc_, &frame_forwarder, false); + + ASSERT_TRUE(stream->GetVp9Settings(&vp9_settings)) << "No VP9 config set."; + EXPECT_FALSE(vp9_settings.denoisingOn); + // Frame dropping always on for real time video. + EXPECT_TRUE(vp9_settings.frameDroppingOn); + + stream = SetDenoisingOption(last_ssrc_, &frame_forwarder, true); + + ASSERT_TRUE(stream->GetVp9Settings(&vp9_settings)) << "No VP9 config set."; + EXPECT_TRUE(vp9_settings.denoisingOn); + EXPECT_TRUE(vp9_settings.frameDroppingOn); + + // In screen-share mode, denoising is forced off. + VideoOptions options; + options.is_screencast = true; + EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, &options, &frame_forwarder)); + + stream = SetDenoisingOption(last_ssrc_, &frame_forwarder, false); + + ASSERT_TRUE(stream->GetVp9Settings(&vp9_settings)) << "No VP9 config set."; + EXPECT_FALSE(vp9_settings.denoisingOn); + // Frame dropping always on for screen sharing. + EXPECT_TRUE(vp9_settings.frameDroppingOn); + + stream = SetDenoisingOption(last_ssrc_, &frame_forwarder, false); + + ASSERT_TRUE(stream->GetVp9Settings(&vp9_settings)) << "No VP9 config set."; + EXPECT_FALSE(vp9_settings.denoisingOn); + EXPECT_TRUE(vp9_settings.frameDroppingOn); + + EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, nullptr, nullptr)); +} + +TEST_F(Vp9SettingsTest, MultipleSsrcsEnablesSvc) { + cricket::VideoSendParameters parameters; + parameters.codecs.push_back(GetEngineCodec("VP9")); + ASSERT_TRUE(channel_->SetSendParameters(parameters)); + + std::vector<uint32_t> ssrcs = MAKE_VECTOR(kSsrcs3); + + FakeVideoSendStream* stream = + AddSendStream(CreateSimStreamParams("cname", ssrcs)); + + webrtc::VideoSendStream::Config config = stream->GetConfig().Copy(); + + webrtc::test::FrameForwarder frame_forwarder; + EXPECT_TRUE(channel_->SetVideoSend(ssrcs[0], nullptr, &frame_forwarder)); + channel_->SetSend(true); + + frame_forwarder.IncomingCapturedFrame(frame_source_.GetFrame()); + + webrtc::VideoCodecVP9 vp9_settings; + ASSERT_TRUE(stream->GetVp9Settings(&vp9_settings)) << "No VP9 config set."; + + const size_t kNumSpatialLayers = ssrcs.size(); + const size_t kNumTemporalLayers = 3; + EXPECT_EQ(vp9_settings.numberOfSpatialLayers, kNumSpatialLayers); + EXPECT_EQ(vp9_settings.numberOfTemporalLayers, kNumTemporalLayers); + + EXPECT_TRUE(channel_->SetVideoSend(ssrcs[0], nullptr, nullptr)); +} + +TEST_F(Vp9SettingsTest, SvcModeCreatesSingleRtpStream) { + cricket::VideoSendParameters parameters; + parameters.codecs.push_back(GetEngineCodec("VP9")); + ASSERT_TRUE(channel_->SetSendParameters(parameters)); + + std::vector<uint32_t> ssrcs = MAKE_VECTOR(kSsrcs3); + + FakeVideoSendStream* stream = + AddSendStream(CreateSimStreamParams("cname", ssrcs)); + + webrtc::VideoSendStream::Config config = stream->GetConfig().Copy(); + + // Despite 3 ssrcs provided, single layer is used. + EXPECT_EQ(1u, config.rtp.ssrcs.size()); + + webrtc::test::FrameForwarder frame_forwarder; + EXPECT_TRUE(channel_->SetVideoSend(ssrcs[0], nullptr, &frame_forwarder)); + channel_->SetSend(true); + + frame_forwarder.IncomingCapturedFrame(frame_source_.GetFrame()); + + webrtc::VideoCodecVP9 vp9_settings; + ASSERT_TRUE(stream->GetVp9Settings(&vp9_settings)) << "No VP9 config set."; + + const size_t kNumSpatialLayers = ssrcs.size(); + EXPECT_EQ(vp9_settings.numberOfSpatialLayers, kNumSpatialLayers); + + EXPECT_TRUE(channel_->SetVideoSend(ssrcs[0], nullptr, nullptr)); +} + +TEST_F(Vp9SettingsTest, AllEncodingParametersCopied) { + cricket::VideoSendParameters send_parameters; + send_parameters.codecs.push_back(GetEngineCodec("VP9")); + ASSERT_TRUE(channel_->SetSendParameters(send_parameters)); + + const size_t kNumSpatialLayers = 3; + std::vector<uint32_t> ssrcs = MAKE_VECTOR(kSsrcs3); + + FakeVideoSendStream* stream = + AddSendStream(CreateSimStreamParams("cname", ssrcs)); + + webrtc::RtpParameters parameters = channel_->GetRtpSendParameters(ssrcs[0]); + ASSERT_EQ(kNumSpatialLayers, parameters.encodings.size()); + ASSERT_TRUE(parameters.encodings[0].active); + ASSERT_TRUE(parameters.encodings[1].active); + ASSERT_TRUE(parameters.encodings[2].active); + // Invert value to verify copying. + parameters.encodings[1].active = false; + EXPECT_TRUE(channel_->SetRtpSendParameters(ssrcs[0], parameters).ok()); + + webrtc::VideoEncoderConfig encoder_config = stream->GetEncoderConfig().Copy(); + + // number_of_streams should be 1 since all spatial layers are sent on the + // same SSRC. But encoding parameters of all layers is supposed to be copied + // and stored in simulcast_layers[]. + EXPECT_EQ(1u, encoder_config.number_of_streams); + EXPECT_EQ(encoder_config.simulcast_layers.size(), kNumSpatialLayers); + EXPECT_TRUE(encoder_config.simulcast_layers[0].active); + EXPECT_FALSE(encoder_config.simulcast_layers[1].active); + EXPECT_TRUE(encoder_config.simulcast_layers[2].active); +} + +class Vp9SettingsTestWithFieldTrial + : public Vp9SettingsTest, + public ::testing::WithParamInterface< + ::testing::tuple<const char*, int, int, webrtc::InterLayerPredMode>> { + protected: + Vp9SettingsTestWithFieldTrial() + : Vp9SettingsTest(::testing::get<0>(GetParam())), + num_spatial_layers_(::testing::get<1>(GetParam())), + num_temporal_layers_(::testing::get<2>(GetParam())), + inter_layer_pred_mode_(::testing::get<3>(GetParam())) {} + + void VerifySettings(int num_spatial_layers, + int num_temporal_layers, + webrtc::InterLayerPredMode interLayerPred) { + cricket::VideoSendParameters parameters; + parameters.codecs.push_back(GetEngineCodec("VP9")); + ASSERT_TRUE(channel_->SetSendParameters(parameters)); + + FakeVideoSendStream* stream = SetUpSimulcast(false, false); + + webrtc::test::FrameForwarder frame_forwarder; + EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, nullptr, &frame_forwarder)); + channel_->SetSend(true); + + frame_forwarder.IncomingCapturedFrame(frame_source_.GetFrame()); + + webrtc::VideoCodecVP9 vp9_settings; + ASSERT_TRUE(stream->GetVp9Settings(&vp9_settings)) << "No VP9 config set."; + EXPECT_EQ(num_spatial_layers, vp9_settings.numberOfSpatialLayers); + EXPECT_EQ(num_temporal_layers, vp9_settings.numberOfTemporalLayers); + EXPECT_EQ(inter_layer_pred_mode_, vp9_settings.interLayerPred); + + EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, nullptr, nullptr)); + } + + const uint8_t num_spatial_layers_; + const uint8_t num_temporal_layers_; + const webrtc::InterLayerPredMode inter_layer_pred_mode_; +}; + +TEST_P(Vp9SettingsTestWithFieldTrial, VerifyCodecSettings) { + VerifySettings(num_spatial_layers_, num_temporal_layers_, + inter_layer_pred_mode_); +} + +INSTANTIATE_TEST_SUITE_P( + All, + Vp9SettingsTestWithFieldTrial, + Values( + std::make_tuple("", 1, 1, webrtc::InterLayerPredMode::kOnKeyPic), + std::make_tuple("WebRTC-SupportVP9SVC/Default/", + 1, + 1, + webrtc::InterLayerPredMode::kOnKeyPic), + std::make_tuple("WebRTC-SupportVP9SVC/EnabledByFlag_2SL3TL/", + 2, + 3, + webrtc::InterLayerPredMode::kOnKeyPic), + std::make_tuple("WebRTC-Vp9InterLayerPred/Default/", + 1, + 1, + webrtc::InterLayerPredMode::kOnKeyPic), + std::make_tuple("WebRTC-Vp9InterLayerPred/Disabled/", + 1, + 1, + webrtc::InterLayerPredMode::kOnKeyPic), + std::make_tuple( + "WebRTC-Vp9InterLayerPred/Enabled,inter_layer_pred_mode:off/", + 1, + 1, + webrtc::InterLayerPredMode::kOff), + std::make_tuple( + "WebRTC-Vp9InterLayerPred/Enabled,inter_layer_pred_mode:on/", + 1, + 1, + webrtc::InterLayerPredMode::kOn), + std::make_tuple( + "WebRTC-Vp9InterLayerPred/Enabled,inter_layer_pred_mode:onkeypic/", + 1, + 1, + webrtc::InterLayerPredMode::kOnKeyPic))); + +TEST_F(WebRtcVideoChannelTest, VerifyMinBitrate) { + std::vector<webrtc::VideoStream> streams = AddSendStream()->GetVideoStreams(); + ASSERT_EQ(1u, streams.size()); + EXPECT_EQ(webrtc::kDefaultMinVideoBitrateBps, streams[0].min_bitrate_bps); +} + +TEST_F(WebRtcVideoChannelTest, VerifyMinBitrateWithForcedFallbackFieldTrial) { + RTC_DCHECK(!override_field_trials_); + override_field_trials_ = std::make_unique<webrtc::test::ScopedFieldTrials>( + "WebRTC-VP8-Forced-Fallback-Encoder-v2/Enabled-1,2,34567/"); + std::vector<webrtc::VideoStream> streams = AddSendStream()->GetVideoStreams(); + ASSERT_EQ(1u, streams.size()); + EXPECT_EQ(34567, streams[0].min_bitrate_bps); +} + +TEST_F(WebRtcVideoChannelTest, + BalancedDegradationPreferenceNotSupportedWithoutFieldtrial) { + RTC_DCHECK(!override_field_trials_); + override_field_trials_ = std::make_unique<webrtc::test::ScopedFieldTrials>( + "WebRTC-Video-BalancedDegradation/Disabled/"); + const bool kResolutionScalingEnabled = true; + const bool kFpsScalingEnabled = false; + TestDegradationPreference(kResolutionScalingEnabled, kFpsScalingEnabled); +} + +TEST_F(WebRtcVideoChannelTest, + BalancedDegradationPreferenceSupportedBehindFieldtrial) { + RTC_DCHECK(!override_field_trials_); + override_field_trials_ = std::make_unique<webrtc::test::ScopedFieldTrials>( + "WebRTC-Video-BalancedDegradation/Enabled/"); + const bool kResolutionScalingEnabled = true; + const bool kFpsScalingEnabled = true; + TestDegradationPreference(kResolutionScalingEnabled, kFpsScalingEnabled); +} + +TEST_F(WebRtcVideoChannelTest, AdaptsOnOveruse) { + TestCpuAdaptation(true, false); +} + +TEST_F(WebRtcVideoChannelTest, DoesNotAdaptOnOveruseWhenDisabled) { + TestCpuAdaptation(false, false); +} + +TEST_F(WebRtcVideoChannelTest, DoesNotAdaptWhenScreeensharing) { + TestCpuAdaptation(false, true); +} + +TEST_F(WebRtcVideoChannelTest, DoesNotAdaptOnOveruseWhenScreensharing) { + TestCpuAdaptation(true, true); +} + +TEST_F(WebRtcVideoChannelTest, PreviousAdaptationDoesNotApplyToScreenshare) { + cricket::VideoCodec codec = GetEngineCodec("VP8"); + cricket::VideoSendParameters parameters; + parameters.codecs.push_back(codec); + + MediaConfig media_config = GetMediaConfig(); + media_config.video.enable_cpu_adaptation = true; + channel_.reset(engine_.CreateMediaChannel( + fake_call_.get(), media_config, VideoOptions(), webrtc::CryptoOptions(), + video_bitrate_allocator_factory_.get())); + channel_->OnReadyToSend(true); + ASSERT_TRUE(channel_->SetSendParameters(parameters)); + + AddSendStream(); + webrtc::test::FrameForwarder frame_forwarder; + + ASSERT_TRUE(channel_->SetSend(true)); + cricket::VideoOptions camera_options; + camera_options.is_screencast = false; + channel_->SetVideoSend(last_ssrc_, &camera_options, &frame_forwarder); + + ASSERT_EQ(1u, fake_call_->GetVideoSendStreams().size()); + FakeVideoSendStream* send_stream = fake_call_->GetVideoSendStreams().front(); + + EXPECT_TRUE(send_stream->resolution_scaling_enabled()); + // Dont' expect anything on framerate_scaling_enabled, since the default is + // transitioning from MAINTAIN_FRAMERATE to BALANCED. + + // Switch to screen share. Expect no resolution scaling. + cricket::VideoOptions screenshare_options; + screenshare_options.is_screencast = true; + channel_->SetVideoSend(last_ssrc_, &screenshare_options, &frame_forwarder); + ASSERT_EQ(2, fake_call_->GetNumCreatedSendStreams()); + send_stream = fake_call_->GetVideoSendStreams().front(); + EXPECT_FALSE(send_stream->resolution_scaling_enabled()); + + // Switch back to the normal capturer. Expect resolution scaling to be + // reenabled. + channel_->SetVideoSend(last_ssrc_, &camera_options, &frame_forwarder); + send_stream = fake_call_->GetVideoSendStreams().front(); + ASSERT_EQ(3, fake_call_->GetNumCreatedSendStreams()); + send_stream = fake_call_->GetVideoSendStreams().front(); + EXPECT_TRUE(send_stream->resolution_scaling_enabled()); + + EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, nullptr, nullptr)); +} + +// TODO(asapersson): Remove this test when the balanced field trial is removed. +void WebRtcVideoChannelTest::TestDegradationPreference( + bool resolution_scaling_enabled, + bool fps_scaling_enabled) { + cricket::VideoCodec codec = GetEngineCodec("VP8"); + cricket::VideoSendParameters parameters; + parameters.codecs.push_back(codec); + + MediaConfig media_config = GetMediaConfig(); + media_config.video.enable_cpu_adaptation = true; + channel_.reset(engine_.CreateMediaChannel( + fake_call_.get(), media_config, VideoOptions(), webrtc::CryptoOptions(), + video_bitrate_allocator_factory_.get())); + channel_->OnReadyToSend(true); + + EXPECT_TRUE(channel_->SetSendParameters(parameters)); + + AddSendStream(); + + webrtc::test::FrameForwarder frame_forwarder; + VideoOptions options; + EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, &options, &frame_forwarder)); + + EXPECT_TRUE(channel_->SetSend(true)); + + FakeVideoSendStream* send_stream = fake_call_->GetVideoSendStreams().front(); + EXPECT_EQ(resolution_scaling_enabled, + send_stream->resolution_scaling_enabled()); + EXPECT_EQ(fps_scaling_enabled, send_stream->framerate_scaling_enabled()); + + EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, nullptr, nullptr)); +} + +void WebRtcVideoChannelTest::TestCpuAdaptation(bool enable_overuse, + bool is_screenshare) { + cricket::VideoCodec codec = GetEngineCodec("VP8"); + cricket::VideoSendParameters parameters; + parameters.codecs.push_back(codec); + + MediaConfig media_config = GetMediaConfig(); + if (enable_overuse) { + media_config.video.enable_cpu_adaptation = true; + } + channel_.reset(engine_.CreateMediaChannel( + fake_call_.get(), media_config, VideoOptions(), webrtc::CryptoOptions(), + video_bitrate_allocator_factory_.get())); + channel_->OnReadyToSend(true); + + EXPECT_TRUE(channel_->SetSendParameters(parameters)); + + AddSendStream(); + + webrtc::test::FrameForwarder frame_forwarder; + VideoOptions options; + options.is_screencast = is_screenshare; + EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, &options, &frame_forwarder)); + + EXPECT_TRUE(channel_->SetSend(true)); + + FakeVideoSendStream* send_stream = fake_call_->GetVideoSendStreams().front(); + + if (!enable_overuse) { + EXPECT_FALSE(send_stream->resolution_scaling_enabled()); + EXPECT_FALSE(send_stream->framerate_scaling_enabled()); + } else if (is_screenshare) { + EXPECT_FALSE(send_stream->resolution_scaling_enabled()); + EXPECT_TRUE(send_stream->framerate_scaling_enabled()); + } else { + EXPECT_TRUE(send_stream->resolution_scaling_enabled()); + EXPECT_FALSE(send_stream->framerate_scaling_enabled()); + } + EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, nullptr, nullptr)); +} + +TEST_F(WebRtcVideoChannelTest, EstimatesNtpStartTimeCorrectly) { + // Start at last timestamp to verify that wraparounds are estimated correctly. + static const uint32_t kInitialTimestamp = 0xFFFFFFFFu; + static const int64_t kInitialNtpTimeMs = 1247891230; + static const int kFrameOffsetMs = 20; + EXPECT_TRUE(channel_->SetRecvParameters(recv_parameters_)); + + FakeVideoReceiveStream* stream = AddRecvStream(); + cricket::FakeVideoRenderer renderer; + EXPECT_TRUE(channel_->SetSink(last_ssrc_, &renderer)); + + webrtc::VideoFrame video_frame = + webrtc::VideoFrame::Builder() + .set_video_frame_buffer(CreateBlackFrameBuffer(4, 4)) + .set_timestamp_rtp(kInitialTimestamp) + .set_timestamp_us(0) + .set_rotation(webrtc::kVideoRotation_0) + .build(); + // Initial NTP time is not available on the first frame, but should still be + // able to be estimated. + stream->InjectFrame(video_frame); + + EXPECT_EQ(1, renderer.num_rendered_frames()); + + // This timestamp is kInitialTimestamp (-1) + kFrameOffsetMs * 90, which + // triggers a constant-overflow warning, hence we're calculating it explicitly + // here. + fake_clock_.AdvanceTime(webrtc::TimeDelta::Millis(kFrameOffsetMs)); + video_frame.set_timestamp(kFrameOffsetMs * 90 - 1); + video_frame.set_ntp_time_ms(kInitialNtpTimeMs + kFrameOffsetMs); + stream->InjectFrame(video_frame); + + EXPECT_EQ(2, renderer.num_rendered_frames()); + + // Verify that NTP time has been correctly deduced. + cricket::VideoMediaInfo info; + ASSERT_TRUE(channel_->GetStats(&info)); + ASSERT_EQ(1u, info.receivers.size()); + EXPECT_EQ(kInitialNtpTimeMs, info.receivers[0].capture_start_ntp_time_ms); +} + +TEST_F(WebRtcVideoChannelTest, SetDefaultSendCodecs) { + AssignDefaultAptRtxTypes(); + ASSERT_TRUE(channel_->SetSendParameters(send_parameters_)); + + VideoCodec codec; + EXPECT_TRUE(channel_->GetSendCodec(&codec)); + EXPECT_TRUE(codec.Matches(engine_.send_codecs()[0])); + + // Using a RTX setup to verify that the default RTX payload type is good. + const std::vector<uint32_t> ssrcs = MAKE_VECTOR(kSsrcs1); + const std::vector<uint32_t> rtx_ssrcs = MAKE_VECTOR(kRtxSsrcs1); + FakeVideoSendStream* stream = AddSendStream( + cricket::CreateSimWithRtxStreamParams("cname", ssrcs, rtx_ssrcs)); + webrtc::VideoSendStream::Config config = stream->GetConfig().Copy(); + + // Make sure NACK and FEC are enabled on the correct payload types. + EXPECT_EQ(1000, config.rtp.nack.rtp_history_ms); + EXPECT_EQ(GetEngineCodec("ulpfec").id, config.rtp.ulpfec.ulpfec_payload_type); + EXPECT_EQ(GetEngineCodec("red").id, config.rtp.ulpfec.red_payload_type); + + EXPECT_EQ(1u, config.rtp.rtx.ssrcs.size()); + EXPECT_EQ(kRtxSsrcs1[0], config.rtp.rtx.ssrcs[0]); + VerifySendStreamHasRtxTypes(config, default_apt_rtx_types_); + // TODO(juberti): Check RTCP, PLI, TMMBR. +} + +TEST_F(WebRtcVideoChannelTest, SetSendCodecsWithoutPacketization) { + cricket::VideoSendParameters parameters; + parameters.codecs.push_back(GetEngineCodec("VP8")); + EXPECT_TRUE(channel_->SetSendParameters(parameters)); + + FakeVideoSendStream* stream = AddSendStream(); + const webrtc::VideoSendStream::Config config = stream->GetConfig().Copy(); + EXPECT_FALSE(config.rtp.raw_payload); +} + +TEST_F(WebRtcVideoChannelTest, SetSendCodecsWithPacketization) { + cricket::VideoSendParameters parameters; + parameters.codecs.push_back(GetEngineCodec("VP8")); + parameters.codecs.back().packetization = kPacketizationParamRaw; + EXPECT_TRUE(channel_->SetSendParameters(parameters)); + + FakeVideoSendStream* stream = AddSendStream(); + const webrtc::VideoSendStream::Config config = stream->GetConfig().Copy(); + EXPECT_TRUE(config.rtp.raw_payload); +} + +// The following four tests ensures that FlexFEC is not activated by default +// when the field trials are not enabled. +// TODO(brandtr): Remove or update these tests when FlexFEC _is_ enabled by +// default. +TEST_F(WebRtcVideoChannelTest, FlexfecSendCodecWithoutSsrcNotExposedByDefault) { + FakeVideoSendStream* stream = AddSendStream(); + webrtc::VideoSendStream::Config config = stream->GetConfig().Copy(); + + EXPECT_EQ(-1, config.rtp.flexfec.payload_type); + EXPECT_EQ(0U, config.rtp.flexfec.ssrc); + EXPECT_TRUE(config.rtp.flexfec.protected_media_ssrcs.empty()); +} + +TEST_F(WebRtcVideoChannelTest, FlexfecSendCodecWithSsrcNotExposedByDefault) { + FakeVideoSendStream* stream = AddSendStream( + CreatePrimaryWithFecFrStreamParams("cname", kSsrcs1[0], kFlexfecSsrc)); + webrtc::VideoSendStream::Config config = stream->GetConfig().Copy(); + + EXPECT_EQ(-1, config.rtp.flexfec.payload_type); + EXPECT_EQ(0U, config.rtp.flexfec.ssrc); + EXPECT_TRUE(config.rtp.flexfec.protected_media_ssrcs.empty()); +} + +TEST_F(WebRtcVideoChannelTest, FlexfecRecvCodecWithoutSsrcNotExposedByDefault) { + AddRecvStream(); + + const std::vector<FakeFlexfecReceiveStream*>& streams = + fake_call_->GetFlexfecReceiveStreams(); + EXPECT_TRUE(streams.empty()); +} + +TEST_F(WebRtcVideoChannelTest, FlexfecRecvCodecWithSsrcNotExposedByDefault) { + AddRecvStream( + CreatePrimaryWithFecFrStreamParams("cname", kSsrcs1[0], kFlexfecSsrc)); + + const std::vector<FakeFlexfecReceiveStream*>& streams = + fake_call_->GetFlexfecReceiveStreams(); + EXPECT_TRUE(streams.empty()); +} + +// TODO(brandtr): When FlexFEC is no longer behind a field trial, merge all +// tests that use this test fixture into the corresponding "non-field trial" +// tests. +class WebRtcVideoChannelFlexfecRecvTest : public WebRtcVideoChannelTest { + public: + WebRtcVideoChannelFlexfecRecvTest() + : WebRtcVideoChannelTest("WebRTC-FlexFEC-03-Advertised/Enabled/") {} +}; + +TEST_F(WebRtcVideoChannelFlexfecRecvTest, + DefaultFlexfecCodecHasTransportCcAndRembFeedbackParam) { + EXPECT_TRUE(cricket::HasTransportCc(GetEngineCodec("flexfec-03"))); + EXPECT_TRUE(cricket::HasRemb(GetEngineCodec("flexfec-03"))); +} + +TEST_F(WebRtcVideoChannelFlexfecRecvTest, SetDefaultRecvCodecsWithoutSsrc) { + AddRecvStream(); + + const std::vector<FakeFlexfecReceiveStream*>& streams = + fake_call_->GetFlexfecReceiveStreams(); + EXPECT_TRUE(streams.empty()); + + const std::vector<FakeVideoReceiveStream*>& video_streams = + fake_call_->GetVideoReceiveStreams(); + ASSERT_EQ(1U, video_streams.size()); + const FakeVideoReceiveStream& video_stream = *video_streams.front(); + EXPECT_EQ(0, video_stream.GetNumAddedSecondarySinks()); + EXPECT_EQ(0, video_stream.GetNumRemovedSecondarySinks()); +} + +TEST_F(WebRtcVideoChannelFlexfecRecvTest, SetDefaultRecvCodecsWithSsrc) { + AddRecvStream( + CreatePrimaryWithFecFrStreamParams("cname", kSsrcs1[0], kFlexfecSsrc)); + + const std::vector<FakeFlexfecReceiveStream*>& streams = + fake_call_->GetFlexfecReceiveStreams(); + ASSERT_EQ(1U, streams.size()); + const FakeFlexfecReceiveStream* stream = streams.front(); + const webrtc::FlexfecReceiveStream::Config& config = stream->GetConfig(); + EXPECT_EQ(GetEngineCodec("flexfec-03").id, config.payload_type); + EXPECT_EQ(kFlexfecSsrc, config.remote_ssrc); + ASSERT_EQ(1U, config.protected_media_ssrcs.size()); + EXPECT_EQ(kSsrcs1[0], config.protected_media_ssrcs[0]); + + const std::vector<FakeVideoReceiveStream*>& video_streams = + fake_call_->GetVideoReceiveStreams(); + ASSERT_EQ(1U, video_streams.size()); + const FakeVideoReceiveStream& video_stream = *video_streams.front(); + EXPECT_EQ(1, video_stream.GetNumAddedSecondarySinks()); + const webrtc::VideoReceiveStream::Config& video_config = + video_stream.GetConfig(); + EXPECT_TRUE(video_config.rtp.protected_by_flexfec); +} + +TEST_F(WebRtcVideoChannelFlexfecRecvTest, + EnablingFlexfecDoesNotRecreateVideoReceiveStream) { + cricket::VideoRecvParameters recv_parameters; + recv_parameters.codecs.push_back(GetEngineCodec("VP8")); + ASSERT_TRUE(channel_->SetRecvParameters(recv_parameters)); + + AddRecvStream( + CreatePrimaryWithFecFrStreamParams("cname", kSsrcs1[0], kFlexfecSsrc)); + EXPECT_EQ(1, fake_call_->GetNumCreatedReceiveStreams()); + const std::vector<FakeVideoReceiveStream*>& video_streams = + fake_call_->GetVideoReceiveStreams(); + ASSERT_EQ(1U, video_streams.size()); + const FakeVideoReceiveStream& video_stream = *video_streams.front(); + EXPECT_EQ(0, video_stream.GetNumAddedSecondarySinks()); + EXPECT_EQ(0, video_stream.GetNumRemovedSecondarySinks()); + + // Enable FlexFEC. + recv_parameters.codecs.push_back(GetEngineCodec("flexfec-03")); + ASSERT_TRUE(channel_->SetRecvParameters(recv_parameters)); + EXPECT_EQ(2, fake_call_->GetNumCreatedReceiveStreams()) + << "Enabling FlexFEC should create FlexfecReceiveStream."; + EXPECT_EQ(1U, fake_call_->GetVideoReceiveStreams().size()) + << "Enabling FlexFEC should not create VideoReceiveStream."; + EXPECT_EQ(1U, fake_call_->GetFlexfecReceiveStreams().size()) + << "Enabling FlexFEC should create a single FlexfecReceiveStream."; + EXPECT_EQ(1, video_stream.GetNumAddedSecondarySinks()); + EXPECT_EQ(0, video_stream.GetNumRemovedSecondarySinks()); +} + +TEST_F(WebRtcVideoChannelFlexfecRecvTest, + DisablingFlexfecDoesNotRecreateVideoReceiveStream) { + cricket::VideoRecvParameters recv_parameters; + recv_parameters.codecs.push_back(GetEngineCodec("VP8")); + recv_parameters.codecs.push_back(GetEngineCodec("flexfec-03")); + ASSERT_TRUE(channel_->SetRecvParameters(recv_parameters)); + + AddRecvStream( + CreatePrimaryWithFecFrStreamParams("cname", kSsrcs1[0], kFlexfecSsrc)); + EXPECT_EQ(2, fake_call_->GetNumCreatedReceiveStreams()); + EXPECT_EQ(1U, fake_call_->GetFlexfecReceiveStreams().size()); + const std::vector<FakeVideoReceiveStream*>& video_streams = + fake_call_->GetVideoReceiveStreams(); + ASSERT_EQ(1U, video_streams.size()); + const FakeVideoReceiveStream& video_stream = *video_streams.front(); + EXPECT_EQ(1, video_stream.GetNumAddedSecondarySinks()); + EXPECT_EQ(0, video_stream.GetNumRemovedSecondarySinks()); + + // Disable FlexFEC. + recv_parameters.codecs.clear(); + recv_parameters.codecs.push_back(GetEngineCodec("VP8")); + ASSERT_TRUE(channel_->SetRecvParameters(recv_parameters)); + EXPECT_EQ(2, fake_call_->GetNumCreatedReceiveStreams()) + << "Disabling FlexFEC should not recreate VideoReceiveStream."; + EXPECT_EQ(1U, fake_call_->GetVideoReceiveStreams().size()) + << "Disabling FlexFEC should not destroy VideoReceiveStream."; + EXPECT_TRUE(fake_call_->GetFlexfecReceiveStreams().empty()) + << "Disabling FlexFEC should destroy FlexfecReceiveStream."; + EXPECT_EQ(1, video_stream.GetNumAddedSecondarySinks()); + EXPECT_EQ(1, video_stream.GetNumRemovedSecondarySinks()); +} + +TEST_F(WebRtcVideoChannelFlexfecRecvTest, DuplicateFlexfecCodecIsDropped) { + constexpr int kUnusedPayloadType1 = 127; + + cricket::VideoRecvParameters recv_parameters; + recv_parameters.codecs.push_back(GetEngineCodec("VP8")); + recv_parameters.codecs.push_back(GetEngineCodec("flexfec-03")); + cricket::VideoCodec duplicate = GetEngineCodec("flexfec-03"); + duplicate.id = kUnusedPayloadType1; + recv_parameters.codecs.push_back(duplicate); + ASSERT_TRUE(channel_->SetRecvParameters(recv_parameters)); + + AddRecvStream( + CreatePrimaryWithFecFrStreamParams("cname", kSsrcs1[0], kFlexfecSsrc)); + + const std::vector<FakeFlexfecReceiveStream*>& streams = + fake_call_->GetFlexfecReceiveStreams(); + ASSERT_EQ(1U, streams.size()); + const FakeFlexfecReceiveStream* stream = streams.front(); + const webrtc::FlexfecReceiveStream::Config& config = stream->GetConfig(); + EXPECT_EQ(GetEngineCodec("flexfec-03").id, config.payload_type); +} + +// TODO(brandtr): When FlexFEC is no longer behind a field trial, merge all +// tests that use this test fixture into the corresponding "non-field trial" +// tests. +class WebRtcVideoChannelFlexfecSendRecvTest : public WebRtcVideoChannelTest { + public: + WebRtcVideoChannelFlexfecSendRecvTest() + : WebRtcVideoChannelTest( + "WebRTC-FlexFEC-03-Advertised/Enabled/WebRTC-FlexFEC-03/Enabled/") { + } +}; + +TEST_F(WebRtcVideoChannelFlexfecSendRecvTest, SetDefaultSendCodecsWithoutSsrc) { + FakeVideoSendStream* stream = AddSendStream(); + webrtc::VideoSendStream::Config config = stream->GetConfig().Copy(); + + EXPECT_EQ(GetEngineCodec("flexfec-03").id, config.rtp.flexfec.payload_type); + EXPECT_EQ(0U, config.rtp.flexfec.ssrc); + EXPECT_TRUE(config.rtp.flexfec.protected_media_ssrcs.empty()); +} + +TEST_F(WebRtcVideoChannelFlexfecSendRecvTest, SetDefaultSendCodecsWithSsrc) { + FakeVideoSendStream* stream = AddSendStream( + CreatePrimaryWithFecFrStreamParams("cname", kSsrcs1[0], kFlexfecSsrc)); + webrtc::VideoSendStream::Config config = stream->GetConfig().Copy(); + + EXPECT_EQ(GetEngineCodec("flexfec-03").id, config.rtp.flexfec.payload_type); + EXPECT_EQ(kFlexfecSsrc, config.rtp.flexfec.ssrc); + ASSERT_EQ(1U, config.rtp.flexfec.protected_media_ssrcs.size()); + EXPECT_EQ(kSsrcs1[0], config.rtp.flexfec.protected_media_ssrcs[0]); +} + +TEST_F(WebRtcVideoChannelTest, SetSendCodecsWithoutFec) { + cricket::VideoSendParameters parameters; + parameters.codecs.push_back(GetEngineCodec("VP8")); + ASSERT_TRUE(channel_->SetSendParameters(parameters)); + + FakeVideoSendStream* stream = AddSendStream(); + webrtc::VideoSendStream::Config config = stream->GetConfig().Copy(); + + EXPECT_EQ(-1, config.rtp.ulpfec.ulpfec_payload_type); + EXPECT_EQ(-1, config.rtp.ulpfec.red_payload_type); +} + +TEST_F(WebRtcVideoChannelFlexfecSendRecvTest, SetSendCodecsWithoutFec) { + cricket::VideoSendParameters parameters; + parameters.codecs.push_back(GetEngineCodec("VP8")); + ASSERT_TRUE(channel_->SetSendParameters(parameters)); + + FakeVideoSendStream* stream = AddSendStream(); + webrtc::VideoSendStream::Config config = stream->GetConfig().Copy(); + + EXPECT_EQ(-1, config.rtp.flexfec.payload_type); +} + +TEST_F(WebRtcVideoChannelFlexfecRecvTest, SetRecvCodecsWithFec) { + AddRecvStream( + CreatePrimaryWithFecFrStreamParams("cname", kSsrcs1[0], kFlexfecSsrc)); + + cricket::VideoRecvParameters recv_parameters; + recv_parameters.codecs.push_back(GetEngineCodec("VP8")); + recv_parameters.codecs.push_back(GetEngineCodec("flexfec-03")); + ASSERT_TRUE(channel_->SetRecvParameters(recv_parameters)); + + const std::vector<FakeFlexfecReceiveStream*>& flexfec_streams = + fake_call_->GetFlexfecReceiveStreams(); + ASSERT_EQ(1U, flexfec_streams.size()); + const FakeFlexfecReceiveStream* flexfec_stream = flexfec_streams.front(); + const webrtc::FlexfecReceiveStream::Config& flexfec_stream_config = + flexfec_stream->GetConfig(); + EXPECT_EQ(GetEngineCodec("flexfec-03").id, + flexfec_stream_config.payload_type); + EXPECT_EQ(kFlexfecSsrc, flexfec_stream_config.remote_ssrc); + ASSERT_EQ(1U, flexfec_stream_config.protected_media_ssrcs.size()); + EXPECT_EQ(kSsrcs1[0], flexfec_stream_config.protected_media_ssrcs[0]); + const std::vector<FakeVideoReceiveStream*>& video_streams = + fake_call_->GetVideoReceiveStreams(); + const FakeVideoReceiveStream* video_stream = video_streams.front(); + const webrtc::VideoReceiveStream::Config& video_stream_config = + video_stream->GetConfig(); + EXPECT_EQ(video_stream_config.rtp.local_ssrc, + flexfec_stream_config.local_ssrc); + EXPECT_EQ(video_stream_config.rtp.rtcp_mode, flexfec_stream_config.rtcp_mode); + EXPECT_EQ(video_stream_config.rtcp_send_transport, + flexfec_stream_config.rtcp_send_transport); + // TODO(brandtr): Update this EXPECT when we set |transport_cc| in a + // spec-compliant way. + EXPECT_EQ(video_stream_config.rtp.transport_cc, + flexfec_stream_config.transport_cc); + EXPECT_EQ(video_stream_config.rtp.rtcp_mode, flexfec_stream_config.rtcp_mode); + EXPECT_EQ(video_stream_config.rtp.extensions, + flexfec_stream_config.rtp_header_extensions); +} + +// We should not send FlexFEC, even if we advertise it, unless the right +// field trial is set. +// TODO(brandtr): Remove when FlexFEC is enabled by default. +TEST_F(WebRtcVideoChannelFlexfecRecvTest, + SetSendCodecsWithoutSsrcWithFecDoesNotEnableFec) { + cricket::VideoSendParameters parameters; + parameters.codecs.push_back(GetEngineCodec("VP8")); + parameters.codecs.push_back(GetEngineCodec("flexfec-03")); + ASSERT_TRUE(channel_->SetSendParameters(parameters)); + + FakeVideoSendStream* stream = AddSendStream(); + webrtc::VideoSendStream::Config config = stream->GetConfig().Copy(); + + EXPECT_EQ(-1, config.rtp.flexfec.payload_type); + EXPECT_EQ(0u, config.rtp.flexfec.ssrc); + EXPECT_TRUE(config.rtp.flexfec.protected_media_ssrcs.empty()); +} + +TEST_F(WebRtcVideoChannelFlexfecRecvTest, + SetSendCodecsWithSsrcWithFecDoesNotEnableFec) { + cricket::VideoSendParameters parameters; + parameters.codecs.push_back(GetEngineCodec("VP8")); + parameters.codecs.push_back(GetEngineCodec("flexfec-03")); + ASSERT_TRUE(channel_->SetSendParameters(parameters)); + + FakeVideoSendStream* stream = AddSendStream( + CreatePrimaryWithFecFrStreamParams("cname", kSsrcs1[0], kFlexfecSsrc)); + webrtc::VideoSendStream::Config config = stream->GetConfig().Copy(); + + EXPECT_EQ(-1, config.rtp.flexfec.payload_type); + EXPECT_EQ(0u, config.rtp.flexfec.ssrc); + EXPECT_TRUE(config.rtp.flexfec.protected_media_ssrcs.empty()); +} + +TEST_F(WebRtcVideoChannelTest, + SetSendCodecRejectsRtxWithoutAssociatedPayloadType) { + const int kUnusedPayloadType = 127; + EXPECT_FALSE(FindCodecById(engine_.send_codecs(), kUnusedPayloadType)); + + cricket::VideoSendParameters parameters; + cricket::VideoCodec rtx_codec(kUnusedPayloadType, "rtx"); + parameters.codecs.push_back(rtx_codec); + EXPECT_FALSE(channel_->SetSendParameters(parameters)) + << "RTX codec without associated payload type should be rejected."; +} + +TEST_F(WebRtcVideoChannelTest, + SetSendCodecRejectsRtxWithoutMatchingVideoCodec) { + const int kUnusedPayloadType1 = 126; + const int kUnusedPayloadType2 = 127; + EXPECT_FALSE(FindCodecById(engine_.send_codecs(), kUnusedPayloadType1)); + EXPECT_FALSE(FindCodecById(engine_.send_codecs(), kUnusedPayloadType2)); + { + cricket::VideoCodec rtx_codec = cricket::VideoCodec::CreateRtxCodec( + kUnusedPayloadType1, GetEngineCodec("VP8").id); + cricket::VideoSendParameters parameters; + parameters.codecs.push_back(GetEngineCodec("VP8")); + parameters.codecs.push_back(rtx_codec); + ASSERT_TRUE(channel_->SetSendParameters(parameters)); + } + { + cricket::VideoCodec rtx_codec = cricket::VideoCodec::CreateRtxCodec( + kUnusedPayloadType1, kUnusedPayloadType2); + cricket::VideoSendParameters parameters; + parameters.codecs.push_back(GetEngineCodec("VP8")); + parameters.codecs.push_back(rtx_codec); + EXPECT_FALSE(channel_->SetSendParameters(parameters)) + << "RTX without matching video codec should be rejected."; + } +} + +TEST_F(WebRtcVideoChannelTest, SetSendCodecsWithChangedRtxPayloadType) { + const int kUnusedPayloadType1 = 126; + const int kUnusedPayloadType2 = 127; + EXPECT_FALSE(FindCodecById(engine_.send_codecs(), kUnusedPayloadType1)); + EXPECT_FALSE(FindCodecById(engine_.send_codecs(), kUnusedPayloadType2)); + + // SSRCs for RTX. + cricket::StreamParams params = + cricket::StreamParams::CreateLegacy(kSsrcs1[0]); + params.AddFidSsrc(kSsrcs1[0], kRtxSsrcs1[0]); + AddSendStream(params); + + // Original payload type for RTX. + cricket::VideoSendParameters parameters; + parameters.codecs.push_back(GetEngineCodec("VP8")); + cricket::VideoCodec rtx_codec(kUnusedPayloadType1, "rtx"); + rtx_codec.SetParam("apt", GetEngineCodec("VP8").id); + parameters.codecs.push_back(rtx_codec); + EXPECT_TRUE(channel_->SetSendParameters(parameters)); + ASSERT_EQ(1U, fake_call_->GetVideoSendStreams().size()); + const webrtc::VideoSendStream::Config& config_before = + fake_call_->GetVideoSendStreams()[0]->GetConfig(); + EXPECT_EQ(kUnusedPayloadType1, config_before.rtp.rtx.payload_type); + ASSERT_EQ(1U, config_before.rtp.rtx.ssrcs.size()); + EXPECT_EQ(kRtxSsrcs1[0], config_before.rtp.rtx.ssrcs[0]); + + // Change payload type for RTX. + parameters.codecs[1].id = kUnusedPayloadType2; + EXPECT_TRUE(channel_->SetSendParameters(parameters)); + ASSERT_EQ(1U, fake_call_->GetVideoSendStreams().size()); + const webrtc::VideoSendStream::Config& config_after = + fake_call_->GetVideoSendStreams()[0]->GetConfig(); + EXPECT_EQ(kUnusedPayloadType2, config_after.rtp.rtx.payload_type); + ASSERT_EQ(1U, config_after.rtp.rtx.ssrcs.size()); + EXPECT_EQ(kRtxSsrcs1[0], config_after.rtp.rtx.ssrcs[0]); +} + +TEST_F(WebRtcVideoChannelTest, SetSendCodecsWithoutFecDisablesFec) { + cricket::VideoSendParameters parameters; + parameters.codecs.push_back(GetEngineCodec("VP8")); + parameters.codecs.push_back(GetEngineCodec("ulpfec")); + ASSERT_TRUE(channel_->SetSendParameters(parameters)); + + FakeVideoSendStream* stream = AddSendStream(); + webrtc::VideoSendStream::Config config = stream->GetConfig().Copy(); + + EXPECT_EQ(GetEngineCodec("ulpfec").id, config.rtp.ulpfec.ulpfec_payload_type); + + parameters.codecs.pop_back(); + ASSERT_TRUE(channel_->SetSendParameters(parameters)); + stream = fake_call_->GetVideoSendStreams()[0]; + ASSERT_TRUE(stream != nullptr); + config = stream->GetConfig().Copy(); + EXPECT_EQ(-1, config.rtp.ulpfec.ulpfec_payload_type) + << "SetSendCodec without ULPFEC should disable current ULPFEC."; +} + +TEST_F(WebRtcVideoChannelFlexfecSendRecvTest, + SetSendCodecsWithoutFecDisablesFec) { + cricket::VideoSendParameters parameters; + parameters.codecs.push_back(GetEngineCodec("VP8")); + parameters.codecs.push_back(GetEngineCodec("flexfec-03")); + ASSERT_TRUE(channel_->SetSendParameters(parameters)); + + FakeVideoSendStream* stream = AddSendStream( + CreatePrimaryWithFecFrStreamParams("cname", kSsrcs1[0], kFlexfecSsrc)); + webrtc::VideoSendStream::Config config = stream->GetConfig().Copy(); + + EXPECT_EQ(GetEngineCodec("flexfec-03").id, config.rtp.flexfec.payload_type); + EXPECT_EQ(kFlexfecSsrc, config.rtp.flexfec.ssrc); + ASSERT_EQ(1U, config.rtp.flexfec.protected_media_ssrcs.size()); + EXPECT_EQ(kSsrcs1[0], config.rtp.flexfec.protected_media_ssrcs[0]); + + parameters.codecs.pop_back(); + ASSERT_TRUE(channel_->SetSendParameters(parameters)); + stream = fake_call_->GetVideoSendStreams()[0]; + ASSERT_TRUE(stream != nullptr); + config = stream->GetConfig().Copy(); + EXPECT_EQ(-1, config.rtp.flexfec.payload_type) + << "SetSendCodec without FlexFEC should disable current FlexFEC."; +} + +TEST_F(WebRtcVideoChannelTest, SetSendCodecsChangesExistingStreams) { + cricket::VideoSendParameters parameters; + cricket::VideoCodec codec(100, "VP8"); + codec.SetParam(kCodecParamMaxQuantization, kDefaultQpMax); + parameters.codecs.push_back(codec); + + ASSERT_TRUE(channel_->SetSendParameters(parameters)); + channel_->SetSend(true); + + FakeVideoSendStream* stream = AddSendStream(); + webrtc::test::FrameForwarder frame_forwarder; + EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, nullptr, &frame_forwarder)); + + std::vector<webrtc::VideoStream> streams = stream->GetVideoStreams(); + EXPECT_EQ(kDefaultQpMax, streams[0].max_qp); + + parameters.codecs.clear(); + codec.SetParam(kCodecParamMaxQuantization, kDefaultQpMax + 1); + parameters.codecs.push_back(codec); + ASSERT_TRUE(channel_->SetSendParameters(parameters)); + streams = fake_call_->GetVideoSendStreams()[0]->GetVideoStreams(); + EXPECT_EQ(kDefaultQpMax + 1, streams[0].max_qp); + EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, nullptr, nullptr)); +} + +TEST_F(WebRtcVideoChannelTest, SetSendCodecsWithBitrates) { + SetSendCodecsShouldWorkForBitrates("100", 100000, "150", 150000, "200", + 200000); +} + +TEST_F(WebRtcVideoChannelTest, SetSendCodecsWithHighMaxBitrate) { + SetSendCodecsShouldWorkForBitrates("", 0, "", -1, "10000", 10000000); + std::vector<webrtc::VideoStream> streams = AddSendStream()->GetVideoStreams(); + ASSERT_EQ(1u, streams.size()); + EXPECT_EQ(10000000, streams[0].max_bitrate_bps); +} + +TEST_F(WebRtcVideoChannelTest, + SetSendCodecsWithoutBitratesUsesCorrectDefaults) { + SetSendCodecsShouldWorkForBitrates("", 0, "", -1, "", -1); +} + +TEST_F(WebRtcVideoChannelTest, SetSendCodecsCapsMinAndStartBitrate) { + SetSendCodecsShouldWorkForBitrates("-1", 0, "-100", -1, "", -1); +} + +TEST_F(WebRtcVideoChannelTest, SetSendCodecsRejectsMaxLessThanMinBitrate) { + send_parameters_.codecs[0].params[kCodecParamMinBitrate] = "300"; + send_parameters_.codecs[0].params[kCodecParamMaxBitrate] = "200"; + EXPECT_FALSE(channel_->SetSendParameters(send_parameters_)); +} + +// Test that when both the codec-specific bitrate params and max_bandwidth_bps +// are present in the same send parameters, the settings are combined correctly. +TEST_F(WebRtcVideoChannelTest, SetSendCodecsWithBitratesAndMaxSendBandwidth) { + send_parameters_.codecs[0].params[kCodecParamMinBitrate] = "100"; + send_parameters_.codecs[0].params[kCodecParamStartBitrate] = "200"; + send_parameters_.codecs[0].params[kCodecParamMaxBitrate] = "300"; + send_parameters_.max_bandwidth_bps = 400000; + // We expect max_bandwidth_bps to take priority, if set. + ExpectSetBitrateParameters(100000, 200000, 400000); + EXPECT_TRUE(channel_->SetSendParameters(send_parameters_)); + // Since the codec isn't changing, start_bitrate_bps should be -1. + ExpectSetBitrateParameters(100000, -1, 350000); + + // Decrease max_bandwidth_bps. + send_parameters_.max_bandwidth_bps = 350000; + EXPECT_TRUE(channel_->SetSendParameters(send_parameters_)); + + // Now try again with the values flipped around. + send_parameters_.codecs[0].params[kCodecParamMaxBitrate] = "400"; + send_parameters_.max_bandwidth_bps = 300000; + ExpectSetBitrateParameters(100000, 200000, 300000); + EXPECT_TRUE(channel_->SetSendParameters(send_parameters_)); + + // If we change the codec max, max_bandwidth_bps should still apply. + send_parameters_.codecs[0].params[kCodecParamMaxBitrate] = "350"; + ExpectSetBitrateParameters(100000, 200000, 300000); + EXPECT_TRUE(channel_->SetSendParameters(send_parameters_)); +} + +TEST_F(WebRtcVideoChannelTest, SetMaxSendBandwidthShouldPreserveOtherBitrates) { + SetSendCodecsShouldWorkForBitrates("100", 100000, "150", 150000, "200", + 200000); + send_parameters_.max_bandwidth_bps = 300000; + // Setting max bitrate should keep previous min bitrate. + // Setting max bitrate should not reset start bitrate. + ExpectSetBitrateParameters(100000, -1, 300000); + EXPECT_TRUE(channel_->SetSendParameters(send_parameters_)); +} + +TEST_F(WebRtcVideoChannelTest, SetMaxSendBandwidthShouldBeRemovable) { + send_parameters_.max_bandwidth_bps = 300000; + ExpectSetMaxBitrate(300000); + EXPECT_TRUE(channel_->SetSendParameters(send_parameters_)); + // -1 means to disable max bitrate (set infinite). + send_parameters_.max_bandwidth_bps = -1; + ExpectSetMaxBitrate(-1); + EXPECT_TRUE(channel_->SetSendParameters(send_parameters_)); +} + +TEST_F(WebRtcVideoChannelTest, SetMaxSendBandwidthAndAddSendStream) { + send_parameters_.max_bandwidth_bps = 99999; + FakeVideoSendStream* stream = AddSendStream(); + ExpectSetMaxBitrate(send_parameters_.max_bandwidth_bps); + ASSERT_TRUE(channel_->SetSendParameters(send_parameters_)); + ASSERT_EQ(1u, stream->GetVideoStreams().size()); + EXPECT_EQ(send_parameters_.max_bandwidth_bps, + stream->GetVideoStreams()[0].max_bitrate_bps); + + send_parameters_.max_bandwidth_bps = 77777; + ExpectSetMaxBitrate(send_parameters_.max_bandwidth_bps); + ASSERT_TRUE(channel_->SetSendParameters(send_parameters_)); + EXPECT_EQ(send_parameters_.max_bandwidth_bps, + stream->GetVideoStreams()[0].max_bitrate_bps); +} + +// Tests that when the codec specific max bitrate and VideoSendParameters +// max_bandwidth_bps are used, that it sets the VideoStream's max bitrate +// appropriately. +TEST_F(WebRtcVideoChannelTest, + MaxBitratePrioritizesVideoSendParametersOverCodecMaxBitrate) { + send_parameters_.codecs[0].params[kCodecParamMinBitrate] = "100"; + send_parameters_.codecs[0].params[kCodecParamStartBitrate] = "200"; + send_parameters_.codecs[0].params[kCodecParamMaxBitrate] = "300"; + send_parameters_.max_bandwidth_bps = -1; + AddSendStream(); + ExpectSetMaxBitrate(300000); + ASSERT_TRUE(channel_->SetSendParameters(send_parameters_)); + + std::vector<FakeVideoSendStream*> video_send_streams = GetFakeSendStreams(); + ASSERT_EQ(1u, video_send_streams.size()); + FakeVideoSendStream* video_send_stream = video_send_streams[0]; + ASSERT_EQ(1u, video_send_streams[0]->GetVideoStreams().size()); + // First the max bitrate is set based upon the codec param. + EXPECT_EQ(300000, + video_send_streams[0]->GetVideoStreams()[0].max_bitrate_bps); + + // The VideoSendParameters max bitrate overrides the codec's. + send_parameters_.max_bandwidth_bps = 500000; + ExpectSetMaxBitrate(send_parameters_.max_bandwidth_bps); + ASSERT_TRUE(channel_->SetSendParameters(send_parameters_)); + ASSERT_EQ(1u, video_send_stream->GetVideoStreams().size()); + EXPECT_EQ(500000, video_send_stream->GetVideoStreams()[0].max_bitrate_bps); +} + +// Tests that when the codec specific max bitrate and RtpParameters +// max_bitrate_bps are used, that it sets the VideoStream's max bitrate +// appropriately. +TEST_F(WebRtcVideoChannelTest, + MaxBitratePrioritizesRtpParametersOverCodecMaxBitrate) { + send_parameters_.codecs[0].params[kCodecParamMinBitrate] = "100"; + send_parameters_.codecs[0].params[kCodecParamStartBitrate] = "200"; + send_parameters_.codecs[0].params[kCodecParamMaxBitrate] = "300"; + send_parameters_.max_bandwidth_bps = -1; + AddSendStream(); + ExpectSetMaxBitrate(300000); + ASSERT_TRUE(channel_->SetSendParameters(send_parameters_)); + + std::vector<FakeVideoSendStream*> video_send_streams = GetFakeSendStreams(); + ASSERT_EQ(1u, video_send_streams.size()); + FakeVideoSendStream* video_send_stream = video_send_streams[0]; + ASSERT_EQ(1u, video_send_stream->GetVideoStreams().size()); + // First the max bitrate is set based upon the codec param. + EXPECT_EQ(300000, video_send_stream->GetVideoStreams()[0].max_bitrate_bps); + + // The RtpParameter max bitrate overrides the codec's. + webrtc::RtpParameters parameters = channel_->GetRtpSendParameters(last_ssrc_); + ASSERT_EQ(1u, parameters.encodings.size()); + parameters.encodings[0].max_bitrate_bps = 500000; + EXPECT_TRUE(channel_->SetRtpSendParameters(last_ssrc_, parameters).ok()); + ASSERT_EQ(1u, video_send_stream->GetVideoStreams().size()); + EXPECT_EQ(parameters.encodings[0].max_bitrate_bps, + video_send_stream->GetVideoStreams()[0].max_bitrate_bps); +} + +TEST_F(WebRtcVideoChannelTest, + MaxBitrateIsMinimumOfMaxSendBandwidthAndMaxEncodingBitrate) { + send_parameters_.max_bandwidth_bps = 99999; + FakeVideoSendStream* stream = AddSendStream(); + ExpectSetMaxBitrate(send_parameters_.max_bandwidth_bps); + ASSERT_TRUE(channel_->SetSendParameters(send_parameters_)); + ASSERT_EQ(1u, stream->GetVideoStreams().size()); + EXPECT_EQ(send_parameters_.max_bandwidth_bps, + stream->GetVideoStreams()[0].max_bitrate_bps); + + // Get and set the rtp encoding parameters. + webrtc::RtpParameters parameters = channel_->GetRtpSendParameters(last_ssrc_); + EXPECT_EQ(1u, parameters.encodings.size()); + + parameters.encodings[0].max_bitrate_bps = 99999 - 1; + EXPECT_TRUE(channel_->SetRtpSendParameters(last_ssrc_, parameters).ok()); + EXPECT_EQ(parameters.encodings[0].max_bitrate_bps, + stream->GetVideoStreams()[0].max_bitrate_bps); + + parameters.encodings[0].max_bitrate_bps = 99999 + 1; + EXPECT_TRUE(channel_->SetRtpSendParameters(last_ssrc_, parameters).ok()); + EXPECT_EQ(send_parameters_.max_bandwidth_bps, + stream->GetVideoStreams()[0].max_bitrate_bps); +} + +TEST_F(WebRtcVideoChannelTest, SetMaxSendBitrateCanIncreaseSenderBitrate) { + cricket::VideoSendParameters parameters; + parameters.codecs.push_back(GetEngineCodec("VP8")); + ASSERT_TRUE(channel_->SetSendParameters(parameters)); + channel_->SetSend(true); + + FakeVideoSendStream* stream = AddSendStream(); + + webrtc::test::FrameForwarder frame_forwarder; + EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, nullptr, &frame_forwarder)); + + std::vector<webrtc::VideoStream> streams = stream->GetVideoStreams(); + int initial_max_bitrate_bps = streams[0].max_bitrate_bps; + EXPECT_GT(initial_max_bitrate_bps, 0); + + parameters.max_bandwidth_bps = initial_max_bitrate_bps * 2; + EXPECT_TRUE(channel_->SetSendParameters(parameters)); + // Insert a frame to update the encoder config. + frame_forwarder.IncomingCapturedFrame(frame_source_.GetFrame()); + streams = stream->GetVideoStreams(); + EXPECT_EQ(initial_max_bitrate_bps * 2, streams[0].max_bitrate_bps); + EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, nullptr, nullptr)); +} + +TEST_F(WebRtcVideoChannelTest, + SetMaxSendBitrateCanIncreaseSimulcastSenderBitrate) { + cricket::VideoSendParameters parameters; + parameters.codecs.push_back(GetEngineCodec("VP8")); + ASSERT_TRUE(channel_->SetSendParameters(parameters)); + channel_->SetSend(true); + + FakeVideoSendStream* stream = AddSendStream( + cricket::CreateSimStreamParams("cname", MAKE_VECTOR(kSsrcs3))); + + // Send a frame to make sure this scales up to >1 stream (simulcast). + webrtc::test::FrameForwarder frame_forwarder; + EXPECT_TRUE(channel_->SetVideoSend(kSsrcs3[0], nullptr, &frame_forwarder)); + frame_forwarder.IncomingCapturedFrame(frame_source_.GetFrame()); + + std::vector<webrtc::VideoStream> streams = stream->GetVideoStreams(); + ASSERT_GT(streams.size(), 1u) + << "Without simulcast this test doesn't make sense."; + int initial_max_bitrate_bps = GetTotalMaxBitrate(streams).bps(); + EXPECT_GT(initial_max_bitrate_bps, 0); + + parameters.max_bandwidth_bps = initial_max_bitrate_bps * 2; + EXPECT_TRUE(channel_->SetSendParameters(parameters)); + // Insert a frame to update the encoder config. + frame_forwarder.IncomingCapturedFrame(frame_source_.GetFrame()); + streams = stream->GetVideoStreams(); + int increased_max_bitrate_bps = GetTotalMaxBitrate(streams).bps(); + EXPECT_EQ(initial_max_bitrate_bps * 2, increased_max_bitrate_bps); + + EXPECT_TRUE(channel_->SetVideoSend(kSsrcs3[0], nullptr, nullptr)); +} + +TEST_F(WebRtcVideoChannelTest, SetSendCodecsWithMaxQuantization) { + static const char* kMaxQuantization = "21"; + cricket::VideoSendParameters parameters; + parameters.codecs.push_back(GetEngineCodec("VP8")); + parameters.codecs[0].params[kCodecParamMaxQuantization] = kMaxQuantization; + EXPECT_TRUE(channel_->SetSendParameters(parameters)); + EXPECT_EQ(atoi(kMaxQuantization), + AddSendStream()->GetVideoStreams().back().max_qp); + + VideoCodec codec; + EXPECT_TRUE(channel_->GetSendCodec(&codec)); + EXPECT_EQ(kMaxQuantization, codec.params[kCodecParamMaxQuantization]); +} + +TEST_F(WebRtcVideoChannelTest, SetSendCodecsRejectBadPayloadTypes) { + // TODO(pbos): Should we only allow the dynamic range? + static const int kIncorrectPayloads[] = {-2, -1, 128, 129}; + cricket::VideoSendParameters parameters; + parameters.codecs.push_back(GetEngineCodec("VP8")); + for (size_t i = 0; i < arraysize(kIncorrectPayloads); ++i) { + parameters.codecs[0].id = kIncorrectPayloads[i]; + EXPECT_FALSE(channel_->SetSendParameters(parameters)) + << "Bad payload type '" << kIncorrectPayloads[i] << "' accepted."; + } +} + +TEST_F(WebRtcVideoChannelTest, SetSendCodecsAcceptAllValidPayloadTypes) { + cricket::VideoSendParameters parameters; + parameters.codecs.push_back(GetEngineCodec("VP8")); + for (int payload_type = 96; payload_type <= 127; ++payload_type) { + parameters.codecs[0].id = payload_type; + EXPECT_TRUE(channel_->SetSendParameters(parameters)) + << "Payload type '" << payload_type << "' rejected."; + } +} + +// Test that setting the a different set of codecs but with an identical front +// codec doesn't result in the stream being recreated. +// This may happen when a subsequent negotiation includes fewer codecs, as a +// result of one of the codecs being rejected. +TEST_F(WebRtcVideoChannelTest, + SetSendCodecsIdenticalFirstCodecDoesntRecreateStream) { + cricket::VideoSendParameters parameters1; + parameters1.codecs.push_back(GetEngineCodec("VP8")); + parameters1.codecs.push_back(GetEngineCodec("VP9")); + EXPECT_TRUE(channel_->SetSendParameters(parameters1)); + + AddSendStream(); + EXPECT_EQ(1, fake_call_->GetNumCreatedSendStreams()); + + cricket::VideoSendParameters parameters2; + parameters2.codecs.push_back(GetEngineCodec("VP8")); + EXPECT_TRUE(channel_->SetSendParameters(parameters2)); + EXPECT_EQ(1, fake_call_->GetNumCreatedSendStreams()); +} + +TEST_F(WebRtcVideoChannelTest, SetRecvCodecsWithOnlyVp8) { + cricket::VideoRecvParameters parameters; + parameters.codecs.push_back(GetEngineCodec("VP8")); + EXPECT_TRUE(channel_->SetRecvParameters(parameters)); +} + +// Test that we set our inbound RTX codecs properly. +TEST_F(WebRtcVideoChannelTest, SetRecvCodecsWithRtx) { + const int kUnusedPayloadType1 = 126; + const int kUnusedPayloadType2 = 127; + EXPECT_FALSE(FindCodecById(engine_.recv_codecs(), kUnusedPayloadType1)); + EXPECT_FALSE(FindCodecById(engine_.recv_codecs(), kUnusedPayloadType2)); + + cricket::VideoRecvParameters parameters; + parameters.codecs.push_back(GetEngineCodec("VP8")); + cricket::VideoCodec rtx_codec(kUnusedPayloadType1, "rtx"); + parameters.codecs.push_back(rtx_codec); + EXPECT_FALSE(channel_->SetRecvParameters(parameters)) + << "RTX codec without associated payload should be rejected."; + + parameters.codecs[1].SetParam("apt", kUnusedPayloadType2); + EXPECT_FALSE(channel_->SetRecvParameters(parameters)) + << "RTX codec with invalid associated payload type should be rejected."; + + parameters.codecs[1].SetParam("apt", GetEngineCodec("VP8").id); + EXPECT_TRUE(channel_->SetRecvParameters(parameters)); + + cricket::VideoCodec rtx_codec2(kUnusedPayloadType2, "rtx"); + rtx_codec2.SetParam("apt", rtx_codec.id); + parameters.codecs.push_back(rtx_codec2); + + EXPECT_FALSE(channel_->SetRecvParameters(parameters)) + << "RTX codec with another RTX as associated payload type should be " + "rejected."; +} + +TEST_F(WebRtcVideoChannelTest, SetRecvCodecsWithPacketization) { + cricket::VideoCodec vp8_codec = GetEngineCodec("VP8"); + vp8_codec.packetization = kPacketizationParamRaw; + + cricket::VideoRecvParameters parameters; + parameters.codecs = {vp8_codec, GetEngineCodec("VP9")}; + EXPECT_TRUE(channel_->SetRecvParameters(parameters)); + + const cricket::StreamParams params = + cricket::StreamParams::CreateLegacy(kSsrcs1[0]); + AddRecvStream(params); + ASSERT_THAT(fake_call_->GetVideoReceiveStreams(), testing::SizeIs(1)); + + const webrtc::VideoReceiveStream::Config& config = + fake_call_->GetVideoReceiveStreams()[0]->GetConfig(); + ASSERT_THAT(config.rtp.raw_payload_types, testing::SizeIs(1)); + EXPECT_EQ(config.rtp.raw_payload_types.count(vp8_codec.id), 1U); +} + +TEST_F(WebRtcVideoChannelTest, SetRecvCodecsWithPacketizationRecreatesStream) { + cricket::VideoRecvParameters parameters; + parameters.codecs = {GetEngineCodec("VP8"), GetEngineCodec("VP9")}; + parameters.codecs.back().packetization = kPacketizationParamRaw; + EXPECT_TRUE(channel_->SetRecvParameters(parameters)); + + const cricket::StreamParams params = + cricket::StreamParams::CreateLegacy(kSsrcs1[0]); + AddRecvStream(params); + ASSERT_THAT(fake_call_->GetVideoReceiveStreams(), testing::SizeIs(1)); + EXPECT_EQ(fake_call_->GetNumCreatedReceiveStreams(), 1); + + parameters.codecs.back().packetization.reset(); + EXPECT_TRUE(channel_->SetRecvParameters(parameters)); + EXPECT_EQ(fake_call_->GetNumCreatedReceiveStreams(), 2); +} + +TEST_F(WebRtcVideoChannelTest, DuplicateUlpfecCodecIsDropped) { + constexpr int kFirstUlpfecPayloadType = 126; + constexpr int kSecondUlpfecPayloadType = 127; + + cricket::VideoRecvParameters parameters; + parameters.codecs.push_back(GetEngineCodec("VP8")); + parameters.codecs.push_back( + cricket::VideoCodec(kFirstUlpfecPayloadType, cricket::kUlpfecCodecName)); + parameters.codecs.push_back( + cricket::VideoCodec(kSecondUlpfecPayloadType, cricket::kUlpfecCodecName)); + ASSERT_TRUE(channel_->SetRecvParameters(parameters)); + + FakeVideoReceiveStream* recv_stream = AddRecvStream(); + EXPECT_EQ(kFirstUlpfecPayloadType, + recv_stream->GetConfig().rtp.ulpfec_payload_type); +} + +TEST_F(WebRtcVideoChannelTest, DuplicateRedCodecIsDropped) { + constexpr int kFirstRedPayloadType = 126; + constexpr int kSecondRedPayloadType = 127; + + cricket::VideoRecvParameters parameters; + parameters.codecs.push_back(GetEngineCodec("VP8")); + parameters.codecs.push_back( + cricket::VideoCodec(kFirstRedPayloadType, cricket::kRedCodecName)); + parameters.codecs.push_back( + cricket::VideoCodec(kSecondRedPayloadType, cricket::kRedCodecName)); + ASSERT_TRUE(channel_->SetRecvParameters(parameters)); + + FakeVideoReceiveStream* recv_stream = AddRecvStream(); + EXPECT_EQ(kFirstRedPayloadType, + recv_stream->GetConfig().rtp.red_payload_type); +} + +TEST_F(WebRtcVideoChannelTest, SetRecvCodecsWithChangedRtxPayloadType) { + const int kUnusedPayloadType1 = 126; + const int kUnusedPayloadType2 = 127; + EXPECT_FALSE(FindCodecById(engine_.recv_codecs(), kUnusedPayloadType1)); + EXPECT_FALSE(FindCodecById(engine_.recv_codecs(), kUnusedPayloadType2)); + + // SSRCs for RTX. + cricket::StreamParams params = + cricket::StreamParams::CreateLegacy(kSsrcs1[0]); + params.AddFidSsrc(kSsrcs1[0], kRtxSsrcs1[0]); + AddRecvStream(params); + + // Original payload type for RTX. + cricket::VideoRecvParameters parameters; + parameters.codecs.push_back(GetEngineCodec("VP8")); + cricket::VideoCodec rtx_codec(kUnusedPayloadType1, "rtx"); + rtx_codec.SetParam("apt", GetEngineCodec("VP8").id); + parameters.codecs.push_back(rtx_codec); + EXPECT_TRUE(channel_->SetRecvParameters(parameters)); + ASSERT_EQ(1U, fake_call_->GetVideoReceiveStreams().size()); + const webrtc::VideoReceiveStream::Config& config_before = + fake_call_->GetVideoReceiveStreams()[0]->GetConfig(); + EXPECT_EQ(1U, config_before.rtp.rtx_associated_payload_types.size()); + const int* payload_type_before = FindKeyByValue( + config_before.rtp.rtx_associated_payload_types, GetEngineCodec("VP8").id); + ASSERT_NE(payload_type_before, nullptr); + EXPECT_EQ(kUnusedPayloadType1, *payload_type_before); + EXPECT_EQ(kRtxSsrcs1[0], config_before.rtp.rtx_ssrc); + + // Change payload type for RTX. + parameters.codecs[1].id = kUnusedPayloadType2; + EXPECT_TRUE(channel_->SetRecvParameters(parameters)); + ASSERT_EQ(1U, fake_call_->GetVideoReceiveStreams().size()); + const webrtc::VideoReceiveStream::Config& config_after = + fake_call_->GetVideoReceiveStreams()[0]->GetConfig(); + EXPECT_EQ(1U, config_after.rtp.rtx_associated_payload_types.size()); + const int* payload_type_after = FindKeyByValue( + config_after.rtp.rtx_associated_payload_types, GetEngineCodec("VP8").id); + ASSERT_NE(payload_type_after, nullptr); + EXPECT_EQ(kUnusedPayloadType2, *payload_type_after); + EXPECT_EQ(kRtxSsrcs1[0], config_after.rtp.rtx_ssrc); +} + +TEST_F(WebRtcVideoChannelTest, SetRecvCodecsDifferentPayloadType) { + cricket::VideoRecvParameters parameters; + parameters.codecs.push_back(GetEngineCodec("VP8")); + parameters.codecs[0].id = 99; + EXPECT_TRUE(channel_->SetRecvParameters(parameters)); +} + +TEST_F(WebRtcVideoChannelTest, SetRecvCodecsAcceptDefaultCodecs) { + cricket::VideoRecvParameters parameters; + parameters.codecs = engine_.recv_codecs(); + EXPECT_TRUE(channel_->SetRecvParameters(parameters)); + + FakeVideoReceiveStream* stream = AddRecvStream(); + const webrtc::VideoReceiveStream::Config& config = stream->GetConfig(); + EXPECT_EQ(engine_.recv_codecs()[0].name, + config.decoders[0].video_format.name); + EXPECT_EQ(engine_.recv_codecs()[0].id, config.decoders[0].payload_type); +} + +TEST_F(WebRtcVideoChannelTest, SetRecvCodecsRejectUnsupportedCodec) { + cricket::VideoRecvParameters parameters; + parameters.codecs.push_back(GetEngineCodec("VP8")); + parameters.codecs.push_back(VideoCodec(101, "WTF3")); + EXPECT_FALSE(channel_->SetRecvParameters(parameters)); +} + +TEST_F(WebRtcVideoChannelTest, SetRecvCodecsAcceptsMultipleVideoCodecs) { + cricket::VideoRecvParameters parameters; + parameters.codecs.push_back(GetEngineCodec("VP8")); + parameters.codecs.push_back(GetEngineCodec("VP9")); + EXPECT_TRUE(channel_->SetRecvParameters(parameters)); +} + +TEST_F(WebRtcVideoChannelTest, SetRecvCodecsWithoutFecDisablesFec) { + cricket::VideoSendParameters send_parameters; + send_parameters.codecs.push_back(GetEngineCodec("VP8")); + send_parameters.codecs.push_back(GetEngineCodec("red")); + send_parameters.codecs.push_back(GetEngineCodec("ulpfec")); + ASSERT_TRUE(channel_->SetSendParameters(send_parameters)); + + FakeVideoReceiveStream* stream = AddRecvStream(); + + EXPECT_EQ(GetEngineCodec("ulpfec").id, + stream->GetConfig().rtp.ulpfec_payload_type); + + cricket::VideoRecvParameters recv_parameters; + recv_parameters.codecs.push_back(GetEngineCodec("VP8")); + ASSERT_TRUE(channel_->SetRecvParameters(recv_parameters)); + stream = fake_call_->GetVideoReceiveStreams()[0]; + ASSERT_TRUE(stream != nullptr); + EXPECT_EQ(-1, stream->GetConfig().rtp.ulpfec_payload_type) + << "SetSendCodec without ULPFEC should disable current ULPFEC."; +} + +TEST_F(WebRtcVideoChannelFlexfecRecvTest, SetRecvParamsWithoutFecDisablesFec) { + AddRecvStream( + CreatePrimaryWithFecFrStreamParams("cname", kSsrcs1[0], kFlexfecSsrc)); + const std::vector<FakeFlexfecReceiveStream*>& streams = + fake_call_->GetFlexfecReceiveStreams(); + + ASSERT_EQ(1U, streams.size()); + const FakeFlexfecReceiveStream* stream = streams.front(); + EXPECT_EQ(GetEngineCodec("flexfec-03").id, stream->GetConfig().payload_type); + EXPECT_EQ(kFlexfecSsrc, stream->GetConfig().remote_ssrc); + ASSERT_EQ(1U, stream->GetConfig().protected_media_ssrcs.size()); + EXPECT_EQ(kSsrcs1[0], stream->GetConfig().protected_media_ssrcs[0]); + + cricket::VideoRecvParameters recv_parameters; + recv_parameters.codecs.push_back(GetEngineCodec("VP8")); + ASSERT_TRUE(channel_->SetRecvParameters(recv_parameters)); + EXPECT_TRUE(streams.empty()) + << "SetSendCodec without FlexFEC should disable current FlexFEC."; +} + +TEST_F(WebRtcVideoChannelTest, SetSendParamsWithFecEnablesFec) { + FakeVideoReceiveStream* stream = AddRecvStream(); + EXPECT_EQ(GetEngineCodec("ulpfec").id, + stream->GetConfig().rtp.ulpfec_payload_type); + + cricket::VideoRecvParameters recv_parameters; + recv_parameters.codecs.push_back(GetEngineCodec("VP8")); + recv_parameters.codecs.push_back(GetEngineCodec("red")); + recv_parameters.codecs.push_back(GetEngineCodec("ulpfec")); + ASSERT_TRUE(channel_->SetRecvParameters(recv_parameters)); + stream = fake_call_->GetVideoReceiveStreams()[0]; + ASSERT_TRUE(stream != nullptr); + EXPECT_EQ(GetEngineCodec("ulpfec").id, + stream->GetConfig().rtp.ulpfec_payload_type) + << "ULPFEC should be enabled on the receive stream."; + + cricket::VideoSendParameters send_parameters; + send_parameters.codecs.push_back(GetEngineCodec("VP8")); + send_parameters.codecs.push_back(GetEngineCodec("red")); + send_parameters.codecs.push_back(GetEngineCodec("ulpfec")); + ASSERT_TRUE(channel_->SetSendParameters(send_parameters)); + stream = fake_call_->GetVideoReceiveStreams()[0]; + EXPECT_EQ(GetEngineCodec("ulpfec").id, + stream->GetConfig().rtp.ulpfec_payload_type) + << "ULPFEC should be enabled on the receive stream."; +} + +TEST_F(WebRtcVideoChannelFlexfecSendRecvTest, + SetSendRecvParamsWithFecEnablesFec) { + AddRecvStream( + CreatePrimaryWithFecFrStreamParams("cname", kSsrcs1[0], kFlexfecSsrc)); + const std::vector<FakeFlexfecReceiveStream*>& streams = + fake_call_->GetFlexfecReceiveStreams(); + + cricket::VideoRecvParameters recv_parameters; + recv_parameters.codecs.push_back(GetEngineCodec("VP8")); + recv_parameters.codecs.push_back(GetEngineCodec("flexfec-03")); + ASSERT_TRUE(channel_->SetRecvParameters(recv_parameters)); + ASSERT_EQ(1U, streams.size()); + const FakeFlexfecReceiveStream* stream_with_recv_params = streams.front(); + EXPECT_EQ(GetEngineCodec("flexfec-03").id, + stream_with_recv_params->GetConfig().payload_type); + EXPECT_EQ(kFlexfecSsrc, stream_with_recv_params->GetConfig().remote_ssrc); + EXPECT_EQ(1U, + stream_with_recv_params->GetConfig().protected_media_ssrcs.size()); + EXPECT_EQ(kSsrcs1[0], + stream_with_recv_params->GetConfig().protected_media_ssrcs[0]); + + cricket::VideoSendParameters send_parameters; + send_parameters.codecs.push_back(GetEngineCodec("VP8")); + send_parameters.codecs.push_back(GetEngineCodec("flexfec-03")); + ASSERT_TRUE(channel_->SetSendParameters(send_parameters)); + ASSERT_EQ(1U, streams.size()); + const FakeFlexfecReceiveStream* stream_with_send_params = streams.front(); + EXPECT_EQ(GetEngineCodec("flexfec-03").id, + stream_with_send_params->GetConfig().payload_type); + EXPECT_EQ(kFlexfecSsrc, stream_with_send_params->GetConfig().remote_ssrc); + EXPECT_EQ(1U, + stream_with_send_params->GetConfig().protected_media_ssrcs.size()); + EXPECT_EQ(kSsrcs1[0], + stream_with_send_params->GetConfig().protected_media_ssrcs[0]); +} + +TEST_F(WebRtcVideoChannelTest, SetRecvCodecsRejectDuplicateFecPayloads) { + cricket::VideoRecvParameters parameters; + parameters.codecs.push_back(GetEngineCodec("VP8")); + parameters.codecs.push_back(GetEngineCodec("red")); + parameters.codecs[1].id = parameters.codecs[0].id; + EXPECT_FALSE(channel_->SetRecvParameters(parameters)); +} + +TEST_F(WebRtcVideoChannelFlexfecRecvTest, + SetRecvCodecsRejectDuplicateFecPayloads) { + cricket::VideoRecvParameters parameters; + parameters.codecs.push_back(GetEngineCodec("VP8")); + parameters.codecs.push_back(GetEngineCodec("flexfec-03")); + parameters.codecs[1].id = parameters.codecs[0].id; + EXPECT_FALSE(channel_->SetRecvParameters(parameters)); +} + +TEST_F(WebRtcVideoChannelTest, SetRecvCodecsRejectDuplicateCodecPayloads) { + cricket::VideoRecvParameters parameters; + parameters.codecs.push_back(GetEngineCodec("VP8")); + parameters.codecs.push_back(GetEngineCodec("VP9")); + parameters.codecs[1].id = parameters.codecs[0].id; + EXPECT_FALSE(channel_->SetRecvParameters(parameters)); +} + +TEST_F(WebRtcVideoChannelTest, + SetRecvCodecsAcceptSameCodecOnMultiplePayloadTypes) { + cricket::VideoRecvParameters parameters; + parameters.codecs.push_back(GetEngineCodec("VP8")); + parameters.codecs.push_back(GetEngineCodec("VP8")); + parameters.codecs[1].id += 1; + EXPECT_TRUE(channel_->SetRecvParameters(parameters)); +} + +// Test that setting the same codecs but with a different order +// doesn't result in the stream being recreated. +TEST_F(WebRtcVideoChannelTest, + SetRecvCodecsDifferentOrderDoesntRecreateStream) { + cricket::VideoRecvParameters parameters1; + parameters1.codecs.push_back(GetEngineCodec("VP8")); + parameters1.codecs.push_back(GetEngineCodec("red")); + EXPECT_TRUE(channel_->SetRecvParameters(parameters1)); + + AddRecvStream(cricket::StreamParams::CreateLegacy(123)); + EXPECT_EQ(1, fake_call_->GetNumCreatedReceiveStreams()); + + cricket::VideoRecvParameters parameters2; + parameters2.codecs.push_back(GetEngineCodec("red")); + parameters2.codecs.push_back(GetEngineCodec("VP8")); + EXPECT_TRUE(channel_->SetRecvParameters(parameters2)); + EXPECT_EQ(1, fake_call_->GetNumCreatedReceiveStreams()); +} + +TEST_F(WebRtcVideoChannelTest, SendStreamNotSendingByDefault) { + EXPECT_FALSE(AddSendStream()->IsSending()); +} + +TEST_F(WebRtcVideoChannelTest, ReceiveStreamReceivingByDefault) { + EXPECT_TRUE(AddRecvStream()->IsReceiving()); +} + +TEST_F(WebRtcVideoChannelTest, SetSend) { + FakeVideoSendStream* stream = AddSendStream(); + EXPECT_FALSE(stream->IsSending()); + + // false->true + EXPECT_TRUE(channel_->SetSend(true)); + EXPECT_TRUE(stream->IsSending()); + // true->true + EXPECT_TRUE(channel_->SetSend(true)); + EXPECT_TRUE(stream->IsSending()); + // true->false + EXPECT_TRUE(channel_->SetSend(false)); + EXPECT_FALSE(stream->IsSending()); + // false->false + EXPECT_TRUE(channel_->SetSend(false)); + EXPECT_FALSE(stream->IsSending()); + + EXPECT_TRUE(channel_->SetSend(true)); + FakeVideoSendStream* new_stream = AddSendStream(); + EXPECT_TRUE(new_stream->IsSending()) + << "Send stream created after SetSend(true) not sending initially."; +} + +// This test verifies DSCP settings are properly applied on video media channel. +TEST_F(WebRtcVideoChannelTest, TestSetDscpOptions) { + std::unique_ptr<cricket::FakeNetworkInterface> network_interface( + new cricket::FakeNetworkInterface); + MediaConfig config; + std::unique_ptr<cricket::WebRtcVideoChannel> channel; + webrtc::RtpParameters parameters; + + channel.reset( + static_cast<cricket::WebRtcVideoChannel*>(engine_.CreateMediaChannel( + call_.get(), config, VideoOptions(), webrtc::CryptoOptions(), + video_bitrate_allocator_factory_.get()))); + channel->SetInterface(network_interface.get(), + webrtc::MediaTransportConfig()); + // Default value when DSCP is disabled should be DSCP_DEFAULT. + EXPECT_EQ(rtc::DSCP_DEFAULT, network_interface->dscp()); + + // Default value when DSCP is enabled is also DSCP_DEFAULT, until it is set + // through rtp parameters. + config.enable_dscp = true; + channel.reset( + static_cast<cricket::WebRtcVideoChannel*>(engine_.CreateMediaChannel( + call_.get(), config, VideoOptions(), webrtc::CryptoOptions(), + video_bitrate_allocator_factory_.get()))); + channel->SetInterface(network_interface.get(), + webrtc::MediaTransportConfig()); + EXPECT_EQ(rtc::DSCP_DEFAULT, network_interface->dscp()); + + // Create a send stream to configure + EXPECT_TRUE(channel->AddSendStream(StreamParams::CreateLegacy(kSsrc))); + parameters = channel->GetRtpSendParameters(kSsrc); + ASSERT_FALSE(parameters.encodings.empty()); + + // Various priorities map to various dscp values. + parameters.encodings[0].network_priority = webrtc::Priority::kHigh; + ASSERT_TRUE(channel->SetRtpSendParameters(kSsrc, parameters).ok()); + EXPECT_EQ(rtc::DSCP_AF41, network_interface->dscp()); + parameters.encodings[0].network_priority = webrtc::Priority::kVeryLow; + ASSERT_TRUE(channel->SetRtpSendParameters(kSsrc, parameters).ok()); + EXPECT_EQ(rtc::DSCP_CS1, network_interface->dscp()); + + // Packets should also self-identify their dscp in PacketOptions. + const uint8_t kData[10] = {0}; + EXPECT_TRUE(static_cast<webrtc::Transport*>(channel.get()) + ->SendRtcp(kData, sizeof(kData))); + EXPECT_EQ(rtc::DSCP_CS1, network_interface->options().dscp); + + // Verify that setting the option to false resets the + // DiffServCodePoint. + config.enable_dscp = false; + channel.reset( + static_cast<cricket::WebRtcVideoChannel*>(engine_.CreateMediaChannel( + call_.get(), config, VideoOptions(), webrtc::CryptoOptions(), + video_bitrate_allocator_factory_.get()))); + channel->SetInterface(network_interface.get(), + webrtc::MediaTransportConfig()); + EXPECT_EQ(rtc::DSCP_DEFAULT, network_interface->dscp()); +} + +// This test verifies that the RTCP reduced size mode is properly applied to +// send video streams. +TEST_F(WebRtcVideoChannelTest, TestSetSendRtcpReducedSize) { + // Create stream, expecting that default mode is "compound". + FakeVideoSendStream* stream1 = AddSendStream(); + EXPECT_EQ(webrtc::RtcpMode::kCompound, stream1->GetConfig().rtp.rtcp_mode); + webrtc::RtpParameters rtp_parameters = + channel_->GetRtpSendParameters(last_ssrc_); + EXPECT_FALSE(rtp_parameters.rtcp.reduced_size); + + // Now enable reduced size mode. + send_parameters_.rtcp.reduced_size = true; + EXPECT_TRUE(channel_->SetSendParameters(send_parameters_)); + stream1 = fake_call_->GetVideoSendStreams()[0]; + EXPECT_EQ(webrtc::RtcpMode::kReducedSize, stream1->GetConfig().rtp.rtcp_mode); + rtp_parameters = channel_->GetRtpSendParameters(last_ssrc_); + EXPECT_TRUE(rtp_parameters.rtcp.reduced_size); + + // Create a new stream and ensure it picks up the reduced size mode. + FakeVideoSendStream* stream2 = AddSendStream(); + EXPECT_EQ(webrtc::RtcpMode::kReducedSize, stream2->GetConfig().rtp.rtcp_mode); +} + +// This test verifies that the RTCP reduced size mode is properly applied to +// receive video streams. +TEST_F(WebRtcVideoChannelTest, TestSetRecvRtcpReducedSize) { + // Create stream, expecting that default mode is "compound". + FakeVideoReceiveStream* stream1 = AddRecvStream(); + EXPECT_EQ(webrtc::RtcpMode::kCompound, stream1->GetConfig().rtp.rtcp_mode); + + // Now enable reduced size mode. + // TODO(deadbeef): Once "recv_parameters" becomes "receiver_parameters", + // the reduced_size flag should come from that. + send_parameters_.rtcp.reduced_size = true; + EXPECT_TRUE(channel_->SetSendParameters(send_parameters_)); + stream1 = fake_call_->GetVideoReceiveStreams()[0]; + EXPECT_EQ(webrtc::RtcpMode::kReducedSize, stream1->GetConfig().rtp.rtcp_mode); + + // Create a new stream and ensure it picks up the reduced size mode. + FakeVideoReceiveStream* stream2 = AddRecvStream(); + EXPECT_EQ(webrtc::RtcpMode::kReducedSize, stream2->GetConfig().rtp.rtcp_mode); +} + +TEST_F(WebRtcVideoChannelTest, OnReadyToSendSignalsNetworkState) { + EXPECT_EQ(webrtc::kNetworkUp, + fake_call_->GetNetworkState(webrtc::MediaType::VIDEO)); + EXPECT_EQ(webrtc::kNetworkUp, + fake_call_->GetNetworkState(webrtc::MediaType::AUDIO)); + + channel_->OnReadyToSend(false); + EXPECT_EQ(webrtc::kNetworkDown, + fake_call_->GetNetworkState(webrtc::MediaType::VIDEO)); + EXPECT_EQ(webrtc::kNetworkUp, + fake_call_->GetNetworkState(webrtc::MediaType::AUDIO)); + + channel_->OnReadyToSend(true); + EXPECT_EQ(webrtc::kNetworkUp, + fake_call_->GetNetworkState(webrtc::MediaType::VIDEO)); + EXPECT_EQ(webrtc::kNetworkUp, + fake_call_->GetNetworkState(webrtc::MediaType::AUDIO)); +} + +TEST_F(WebRtcVideoChannelTest, GetStatsReportsSentCodecName) { + cricket::VideoSendParameters parameters; + parameters.codecs.push_back(GetEngineCodec("VP8")); + EXPECT_TRUE(channel_->SetSendParameters(parameters)); + + AddSendStream(); + + cricket::VideoMediaInfo info; + ASSERT_TRUE(channel_->GetStats(&info)); + EXPECT_EQ("VP8", info.senders[0].codec_name); +} + +TEST_F(WebRtcVideoChannelTest, GetStatsReportsEncoderImplementationName) { + FakeVideoSendStream* stream = AddSendStream(); + webrtc::VideoSendStream::Stats stats; + stats.encoder_implementation_name = "encoder_implementation_name"; + stream->SetStats(stats); + + cricket::VideoMediaInfo info; + ASSERT_TRUE(channel_->GetStats(&info)); + EXPECT_EQ(stats.encoder_implementation_name, + info.senders[0].encoder_implementation_name); +} + +TEST_F(WebRtcVideoChannelTest, GetStatsReportsCpuOveruseMetrics) { + FakeVideoSendStream* stream = AddSendStream(); + webrtc::VideoSendStream::Stats stats; + stats.avg_encode_time_ms = 13; + stats.encode_usage_percent = 42; + stream->SetStats(stats); + + cricket::VideoMediaInfo info; + ASSERT_TRUE(channel_->GetStats(&info)); + EXPECT_EQ(stats.avg_encode_time_ms, info.senders[0].avg_encode_ms); + EXPECT_EQ(stats.encode_usage_percent, info.senders[0].encode_usage_percent); +} + +TEST_F(WebRtcVideoChannelTest, GetStatsReportsFramesEncoded) { + FakeVideoSendStream* stream = AddSendStream(); + webrtc::VideoSendStream::Stats stats; + stats.frames_encoded = 13; + stream->SetStats(stats); + + cricket::VideoMediaInfo info; + ASSERT_TRUE(channel_->GetStats(&info)); + EXPECT_EQ(stats.frames_encoded, info.senders[0].frames_encoded); +} + +TEST_F(WebRtcVideoChannelTest, GetStatsReportsKeyFramesEncoded) { + FakeVideoSendStream* stream = AddSendStream(); + webrtc::VideoSendStream::Stats stats; + stats.substreams[123].frame_counts.key_frames = 10; + stats.substreams[456].frame_counts.key_frames = 87; + stream->SetStats(stats); + + cricket::VideoMediaInfo info; + ASSERT_TRUE(channel_->GetStats(&info)); + EXPECT_EQ(info.senders.size(), 2u); + EXPECT_EQ(10u, info.senders[0].key_frames_encoded); + EXPECT_EQ(87u, info.senders[1].key_frames_encoded); + EXPECT_EQ(97u, info.aggregated_senders[0].key_frames_encoded); +} + +TEST_F(WebRtcVideoChannelTest, GetStatsReportsPerLayerQpSum) { + FakeVideoSendStream* stream = AddSendStream(); + webrtc::VideoSendStream::Stats stats; + stats.substreams[123].qp_sum = 15; + stats.substreams[456].qp_sum = 11; + stream->SetStats(stats); + + cricket::VideoMediaInfo info; + ASSERT_TRUE(channel_->GetStats(&info)); + EXPECT_EQ(info.senders.size(), 2u); + EXPECT_EQ(stats.substreams[123].qp_sum, info.senders[0].qp_sum); + EXPECT_EQ(stats.substreams[456].qp_sum, info.senders[1].qp_sum); + EXPECT_EQ(*info.aggregated_senders[0].qp_sum, 26u); +} + +webrtc::VideoSendStream::Stats GetInitialisedStats() { + webrtc::VideoSendStream::Stats stats; + stats.encoder_implementation_name = "vp"; + stats.input_frame_rate = 1; + stats.encode_frame_rate = 2; + stats.avg_encode_time_ms = 3; + stats.encode_usage_percent = 4; + stats.frames_encoded = 5; + stats.total_encode_time_ms = 6; + stats.frames_dropped_by_capturer = 7; + stats.frames_dropped_by_encoder_queue = 8; + stats.frames_dropped_by_rate_limiter = 9; + stats.frames_dropped_by_congestion_window = 10; + stats.frames_dropped_by_encoder = 11; + stats.target_media_bitrate_bps = 13; + stats.media_bitrate_bps = 14; + stats.suspended = true; + stats.bw_limited_resolution = true; + stats.cpu_limited_resolution = true; + // Not wired. + stats.bw_limited_framerate = true; + // Not wired. + stats.cpu_limited_framerate = true; + stats.quality_limitation_reason = webrtc::QualityLimitationReason::kCpu; + stats.quality_limitation_durations_ms[webrtc::QualityLimitationReason::kCpu] = + 15; + stats.quality_limitation_resolution_changes = 16; + stats.number_of_cpu_adapt_changes = 17; + stats.number_of_quality_adapt_changes = 18; + stats.has_entered_low_resolution = true; + stats.content_type = webrtc::VideoContentType::SCREENSHARE; + stats.frames_sent = 19; + stats.huge_frames_sent = 20; + + return stats; +} + +TEST_F(WebRtcVideoChannelTest, GetAggregatedStatsReportWithoutSubStreams) { + FakeVideoSendStream* stream = AddSendStream(); + auto stats = GetInitialisedStats(); + stream->SetStats(stats); + cricket::VideoMediaInfo video_media_info; + ASSERT_TRUE(channel_->GetStats(&video_media_info)); + EXPECT_EQ(video_media_info.aggregated_senders.size(), 1u); + auto& sender = video_media_info.aggregated_senders[0]; + + // MediaSenderInfo + + EXPECT_EQ(sender.payload_bytes_sent, 0); + EXPECT_EQ(sender.header_and_padding_bytes_sent, 0); + EXPECT_EQ(sender.retransmitted_bytes_sent, 0u); + EXPECT_EQ(sender.packets_sent, 0); + EXPECT_EQ(sender.retransmitted_packets_sent, 0u); + EXPECT_EQ(sender.packets_lost, 0); + EXPECT_EQ(sender.fraction_lost, 0.0f); + EXPECT_EQ(sender.rtt_ms, 0); + EXPECT_EQ(sender.codec_name, DefaultCodec().name); + EXPECT_EQ(sender.codec_payload_type, DefaultCodec().id); + EXPECT_EQ(sender.local_stats.size(), 1u); + EXPECT_EQ(sender.local_stats[0].ssrc, last_ssrc_); + EXPECT_EQ(sender.local_stats[0].timestamp, 0.0f); + EXPECT_EQ(sender.remote_stats.size(), 0u); + EXPECT_EQ(sender.report_block_datas.size(), 0u); + + // VideoSenderInfo + + EXPECT_EQ(sender.ssrc_groups.size(), 0u); + EXPECT_EQ(sender.encoder_implementation_name, + stats.encoder_implementation_name); + // Comes from substream only. + EXPECT_EQ(sender.firs_rcvd, 0); + EXPECT_EQ(sender.plis_rcvd, 0); + EXPECT_EQ(sender.nacks_rcvd, 0); + EXPECT_EQ(sender.send_frame_width, 0); + EXPECT_EQ(sender.send_frame_height, 0); + + EXPECT_EQ(sender.framerate_input, stats.input_frame_rate); + EXPECT_EQ(sender.framerate_sent, stats.encode_frame_rate); + EXPECT_EQ(sender.nominal_bitrate, stats.media_bitrate_bps); + EXPECT_NE(sender.adapt_reason & WebRtcVideoChannel::ADAPTREASON_CPU, 0); + EXPECT_NE(sender.adapt_reason & WebRtcVideoChannel::ADAPTREASON_BANDWIDTH, 0); + EXPECT_EQ(sender.adapt_changes, stats.number_of_cpu_adapt_changes); + EXPECT_EQ(sender.quality_limitation_reason, stats.quality_limitation_reason); + EXPECT_EQ(sender.quality_limitation_durations_ms, + stats.quality_limitation_durations_ms); + EXPECT_EQ(sender.quality_limitation_resolution_changes, + stats.quality_limitation_resolution_changes); + EXPECT_EQ(sender.avg_encode_ms, stats.avg_encode_time_ms); + EXPECT_EQ(sender.encode_usage_percent, stats.encode_usage_percent); + EXPECT_EQ(sender.frames_encoded, stats.frames_encoded); + // Comes from substream only. + EXPECT_EQ(sender.key_frames_encoded, 0u); + + EXPECT_EQ(sender.total_encode_time_ms, stats.total_encode_time_ms); + EXPECT_EQ(sender.total_encoded_bytes_target, + stats.total_encoded_bytes_target); + // Comes from substream only. + EXPECT_EQ(sender.total_packet_send_delay_ms, 0u); + EXPECT_EQ(sender.qp_sum, absl::nullopt); + + EXPECT_EQ(sender.has_entered_low_resolution, + stats.has_entered_low_resolution); + EXPECT_EQ(sender.content_type, webrtc::VideoContentType::SCREENSHARE); + EXPECT_EQ(sender.frames_sent, stats.frames_encoded); + EXPECT_EQ(sender.huge_frames_sent, stats.huge_frames_sent); + EXPECT_EQ(sender.rid, absl::nullopt); +} + +TEST_F(WebRtcVideoChannelTest, GetAggregatedStatsReportForSubStreams) { + FakeVideoSendStream* stream = AddSendStream(); + auto stats = GetInitialisedStats(); + + const uint32_t ssrc_1 = 123u; + const uint32_t ssrc_2 = 456u; + + auto& substream = stats.substreams[ssrc_1]; + substream.frame_counts.key_frames = 1; + substream.frame_counts.delta_frames = 2; + substream.width = 3; + substream.height = 4; + substream.total_bitrate_bps = 5; + substream.retransmit_bitrate_bps = 6; + substream.avg_delay_ms = 7; + substream.max_delay_ms = 8; + substream.total_packet_send_delay_ms = 9; + substream.rtp_stats.transmitted.header_bytes = 10; + substream.rtp_stats.transmitted.padding_bytes = 11; + substream.rtp_stats.retransmitted.payload_bytes = 12; + substream.rtp_stats.retransmitted.packets = 13; + substream.rtcp_packet_type_counts.fir_packets = 14; + substream.rtcp_packet_type_counts.nack_packets = 15; + substream.rtcp_packet_type_counts.pli_packets = 16; + substream.rtcp_stats.packets_lost = 17; + substream.rtcp_stats.fraction_lost = 18; + webrtc::ReportBlockData report_block_data; + report_block_data.AddRoundTripTimeSample(19); + substream.report_block_data = report_block_data; + substream.encode_frame_rate = 20.0; + substream.frames_encoded = 21; + substream.qp_sum = 22; + substream.total_encode_time_ms = 23; + substream.total_encoded_bytes_target = 24; + substream.huge_frames_sent = 25; + + stats.substreams[ssrc_2] = substream; + + stream->SetStats(stats); + + cricket::VideoMediaInfo video_media_info; + ASSERT_TRUE(channel_->GetStats(&video_media_info)); + EXPECT_EQ(video_media_info.aggregated_senders.size(), 1u); + auto& sender = video_media_info.aggregated_senders[0]; + + // MediaSenderInfo + + EXPECT_EQ( + sender.payload_bytes_sent, + static_cast<int64_t>(2u * substream.rtp_stats.transmitted.payload_bytes)); + EXPECT_EQ(sender.header_and_padding_bytes_sent, + static_cast<int64_t>( + 2u * (substream.rtp_stats.transmitted.header_bytes + + substream.rtp_stats.transmitted.padding_bytes))); + EXPECT_EQ(sender.retransmitted_bytes_sent, + 2u * substream.rtp_stats.retransmitted.payload_bytes); + EXPECT_EQ(sender.packets_sent, + static_cast<int>(2 * substream.rtp_stats.transmitted.packets)); + EXPECT_EQ(sender.retransmitted_packets_sent, + 2u * substream.rtp_stats.retransmitted.packets); + EXPECT_EQ(sender.packets_lost, 2 * substream.rtcp_stats.packets_lost); + EXPECT_EQ(sender.fraction_lost, + static_cast<float>(substream.rtcp_stats.fraction_lost) / (1 << 8)); + EXPECT_EQ(sender.rtt_ms, 0); + EXPECT_EQ(sender.codec_name, DefaultCodec().name); + EXPECT_EQ(sender.codec_payload_type, DefaultCodec().id); + EXPECT_EQ(sender.local_stats.size(), 1u); + EXPECT_EQ(sender.local_stats[0].ssrc, last_ssrc_); + EXPECT_EQ(sender.local_stats[0].timestamp, 0.0f); + EXPECT_EQ(sender.remote_stats.size(), 0u); + EXPECT_EQ(sender.report_block_datas.size(), 2u * 1); + + // VideoSenderInfo + + EXPECT_EQ(sender.ssrc_groups.size(), 0u); + EXPECT_EQ(sender.encoder_implementation_name, + stats.encoder_implementation_name); + EXPECT_EQ( + sender.firs_rcvd, + static_cast<int>(2 * substream.rtcp_packet_type_counts.fir_packets)); + EXPECT_EQ( + sender.plis_rcvd, + static_cast<int>(2 * substream.rtcp_packet_type_counts.pli_packets)); + EXPECT_EQ( + sender.nacks_rcvd, + static_cast<int>(2 * substream.rtcp_packet_type_counts.nack_packets)); + EXPECT_EQ(sender.send_frame_width, substream.width); + EXPECT_EQ(sender.send_frame_height, substream.height); + + EXPECT_EQ(sender.framerate_input, stats.input_frame_rate); + EXPECT_EQ(sender.framerate_sent, stats.encode_frame_rate); + EXPECT_EQ(sender.nominal_bitrate, stats.media_bitrate_bps); + EXPECT_NE(sender.adapt_reason & WebRtcVideoChannel::ADAPTREASON_CPU, 0); + EXPECT_NE(sender.adapt_reason & WebRtcVideoChannel::ADAPTREASON_BANDWIDTH, 0); + EXPECT_EQ(sender.adapt_changes, stats.number_of_cpu_adapt_changes); + EXPECT_EQ(sender.quality_limitation_reason, stats.quality_limitation_reason); + EXPECT_EQ(sender.quality_limitation_durations_ms, + stats.quality_limitation_durations_ms); + EXPECT_EQ(sender.quality_limitation_resolution_changes, + stats.quality_limitation_resolution_changes); + EXPECT_EQ(sender.avg_encode_ms, stats.avg_encode_time_ms); + EXPECT_EQ(sender.encode_usage_percent, stats.encode_usage_percent); + EXPECT_EQ(sender.frames_encoded, 2u * substream.frames_encoded); + EXPECT_EQ(sender.key_frames_encoded, 2u * substream.frame_counts.key_frames); + EXPECT_EQ(sender.total_encode_time_ms, 2u * substream.total_encode_time_ms); + EXPECT_EQ(sender.total_encoded_bytes_target, + 2u * substream.total_encoded_bytes_target); + EXPECT_EQ(sender.total_packet_send_delay_ms, + 2u * substream.total_packet_send_delay_ms); + EXPECT_EQ(sender.has_entered_low_resolution, + stats.has_entered_low_resolution); + EXPECT_EQ(sender.qp_sum, 2u * *substream.qp_sum); + EXPECT_EQ(sender.content_type, webrtc::VideoContentType::SCREENSHARE); + EXPECT_EQ(sender.frames_sent, 2u * substream.frames_encoded); + EXPECT_EQ(sender.huge_frames_sent, stats.huge_frames_sent); + EXPECT_EQ(sender.rid, absl::nullopt); +} + +TEST_F(WebRtcVideoChannelTest, GetPerLayerStatsReportForSubStreams) { + FakeVideoSendStream* stream = AddSendStream(); + auto stats = GetInitialisedStats(); + + const uint32_t ssrc_1 = 123u; + const uint32_t ssrc_2 = 456u; + + auto& substream = stats.substreams[ssrc_1]; + substream.frame_counts.key_frames = 1; + substream.frame_counts.delta_frames = 2; + substream.width = 3; + substream.height = 4; + substream.total_bitrate_bps = 5; + substream.retransmit_bitrate_bps = 6; + substream.avg_delay_ms = 7; + substream.max_delay_ms = 8; + substream.total_packet_send_delay_ms = 9; + substream.rtp_stats.transmitted.header_bytes = 10; + substream.rtp_stats.transmitted.padding_bytes = 11; + substream.rtp_stats.retransmitted.payload_bytes = 12; + substream.rtp_stats.retransmitted.packets = 13; + substream.rtcp_packet_type_counts.fir_packets = 14; + substream.rtcp_packet_type_counts.nack_packets = 15; + substream.rtcp_packet_type_counts.pli_packets = 16; + substream.rtcp_stats.packets_lost = 17; + substream.rtcp_stats.fraction_lost = 18; + webrtc::ReportBlockData report_block_data; + report_block_data.AddRoundTripTimeSample(19); + substream.report_block_data = report_block_data; + substream.encode_frame_rate = 20.0; + substream.frames_encoded = 21; + substream.qp_sum = 22; + substream.total_encode_time_ms = 23; + substream.total_encoded_bytes_target = 24; + substream.huge_frames_sent = 25; + + stats.substreams[ssrc_2] = substream; + + stream->SetStats(stats); + + cricket::VideoMediaInfo video_media_info; + ASSERT_TRUE(channel_->GetStats(&video_media_info)); + EXPECT_EQ(video_media_info.senders.size(), 2u); + auto& sender = video_media_info.senders[0]; + + // MediaSenderInfo + + EXPECT_EQ( + sender.payload_bytes_sent, + static_cast<int64_t>(substream.rtp_stats.transmitted.payload_bytes)); + EXPECT_EQ( + sender.header_and_padding_bytes_sent, + static_cast<int64_t>(substream.rtp_stats.transmitted.header_bytes + + substream.rtp_stats.transmitted.padding_bytes)); + EXPECT_EQ(sender.retransmitted_bytes_sent, + substream.rtp_stats.retransmitted.payload_bytes); + EXPECT_EQ(sender.packets_sent, + static_cast<int>(substream.rtp_stats.transmitted.packets)); + EXPECT_EQ(sender.retransmitted_packets_sent, + substream.rtp_stats.retransmitted.packets); + EXPECT_EQ(sender.packets_lost, substream.rtcp_stats.packets_lost); + EXPECT_EQ(sender.fraction_lost, + static_cast<float>(substream.rtcp_stats.fraction_lost) / (1 << 8)); + EXPECT_EQ(sender.rtt_ms, 0); + EXPECT_EQ(sender.codec_name, DefaultCodec().name); + EXPECT_EQ(sender.codec_payload_type, DefaultCodec().id); + EXPECT_EQ(sender.local_stats.size(), 1u); + EXPECT_EQ(sender.local_stats[0].ssrc, ssrc_1); + EXPECT_EQ(sender.local_stats[0].timestamp, 0.0f); + EXPECT_EQ(sender.remote_stats.size(), 0u); + EXPECT_EQ(sender.report_block_datas.size(), 1u); + + // VideoSenderInfo + + EXPECT_EQ(sender.ssrc_groups.size(), 0u); + EXPECT_EQ(sender.encoder_implementation_name, + stats.encoder_implementation_name); + EXPECT_EQ(sender.firs_rcvd, + static_cast<int>(substream.rtcp_packet_type_counts.fir_packets)); + EXPECT_EQ(sender.plis_rcvd, + static_cast<int>(substream.rtcp_packet_type_counts.pli_packets)); + EXPECT_EQ(sender.nacks_rcvd, + static_cast<int>(substream.rtcp_packet_type_counts.nack_packets)); + EXPECT_EQ(sender.send_frame_width, substream.width); + EXPECT_EQ(sender.send_frame_height, substream.height); + + EXPECT_EQ(sender.framerate_input, stats.input_frame_rate); + EXPECT_EQ(sender.framerate_sent, substream.encode_frame_rate); + EXPECT_EQ(sender.nominal_bitrate, stats.media_bitrate_bps); + EXPECT_NE(sender.adapt_reason & WebRtcVideoChannel::ADAPTREASON_CPU, 0); + EXPECT_NE(sender.adapt_reason & WebRtcVideoChannel::ADAPTREASON_BANDWIDTH, 0); + EXPECT_EQ(sender.adapt_changes, stats.number_of_cpu_adapt_changes); + EXPECT_EQ(sender.quality_limitation_reason, stats.quality_limitation_reason); + EXPECT_EQ(sender.quality_limitation_durations_ms, + stats.quality_limitation_durations_ms); + EXPECT_EQ(sender.quality_limitation_resolution_changes, + stats.quality_limitation_resolution_changes); + EXPECT_EQ(sender.avg_encode_ms, stats.avg_encode_time_ms); + EXPECT_EQ(sender.encode_usage_percent, stats.encode_usage_percent); + EXPECT_EQ(sender.frames_encoded, + static_cast<uint32_t>(substream.frames_encoded)); + EXPECT_EQ(sender.key_frames_encoded, + static_cast<uint32_t>(substream.frame_counts.key_frames)); + EXPECT_EQ(sender.total_encode_time_ms, substream.total_encode_time_ms); + EXPECT_EQ(sender.total_encoded_bytes_target, + substream.total_encoded_bytes_target); + EXPECT_EQ(sender.total_packet_send_delay_ms, + substream.total_packet_send_delay_ms); + EXPECT_EQ(sender.has_entered_low_resolution, + stats.has_entered_low_resolution); + EXPECT_EQ(sender.qp_sum, *substream.qp_sum); + EXPECT_EQ(sender.content_type, webrtc::VideoContentType::SCREENSHARE); + EXPECT_EQ(sender.frames_sent, + static_cast<uint32_t>(substream.frames_encoded)); + EXPECT_EQ(sender.huge_frames_sent, substream.huge_frames_sent); + EXPECT_EQ(sender.rid, absl::nullopt); +} + +TEST_F(WebRtcVideoChannelTest, GetStatsReportsUpperResolution) { + FakeVideoSendStream* stream = AddSendStream(); + webrtc::VideoSendStream::Stats stats; + stats.substreams[17].width = 123; + stats.substreams[17].height = 40; + stats.substreams[42].width = 80; + stats.substreams[42].height = 31; + stats.substreams[11].width = 20; + stats.substreams[11].height = 90; + stream->SetStats(stats); + + cricket::VideoMediaInfo info; + ASSERT_TRUE(channel_->GetStats(&info)); + ASSERT_EQ(1u, info.aggregated_senders.size()); + ASSERT_EQ(3u, info.senders.size()); + EXPECT_EQ(123, info.senders[1].send_frame_width); + EXPECT_EQ(40, info.senders[1].send_frame_height); + EXPECT_EQ(80, info.senders[2].send_frame_width); + EXPECT_EQ(31, info.senders[2].send_frame_height); + EXPECT_EQ(20, info.senders[0].send_frame_width); + EXPECT_EQ(90, info.senders[0].send_frame_height); + EXPECT_EQ(123, info.aggregated_senders[0].send_frame_width); + EXPECT_EQ(90, info.aggregated_senders[0].send_frame_height); +} + +TEST_F(WebRtcVideoChannelTest, GetStatsReportsCpuAdaptationStats) { + FakeVideoSendStream* stream = AddSendStream(); + webrtc::VideoSendStream::Stats stats; + stats.number_of_cpu_adapt_changes = 2; + stats.cpu_limited_resolution = true; + stream->SetStats(stats); + + cricket::VideoMediaInfo info; + EXPECT_TRUE(channel_->GetStats(&info)); + ASSERT_EQ(1U, info.senders.size()); + EXPECT_EQ(WebRtcVideoChannel::ADAPTREASON_CPU, info.senders[0].adapt_reason); + EXPECT_EQ(stats.number_of_cpu_adapt_changes, info.senders[0].adapt_changes); +} + +TEST_F(WebRtcVideoChannelTest, GetStatsReportsAdaptationAndBandwidthStats) { + FakeVideoSendStream* stream = AddSendStream(); + webrtc::VideoSendStream::Stats stats; + stats.number_of_cpu_adapt_changes = 2; + stats.cpu_limited_resolution = true; + stats.bw_limited_resolution = true; + stream->SetStats(stats); + + cricket::VideoMediaInfo info; + EXPECT_TRUE(channel_->GetStats(&info)); + ASSERT_EQ(1U, info.senders.size()); + EXPECT_EQ(WebRtcVideoChannel::ADAPTREASON_CPU | + WebRtcVideoChannel::ADAPTREASON_BANDWIDTH, + info.senders[0].adapt_reason); + EXPECT_EQ(stats.number_of_cpu_adapt_changes, info.senders[0].adapt_changes); +} + +TEST(WebRtcVideoChannelHelperTest, MergeInfoAboutOutboundRtpSubstreams) { + const uint32_t kFirstMediaStreamSsrc = 10; + const uint32_t kSecondMediaStreamSsrc = 20; + const uint32_t kRtxSsrc = 30; + const uint32_t kFlexfecSsrc = 40; + std::map<uint32_t, webrtc::VideoSendStream::StreamStats> substreams; + // First kMedia stream. + substreams[kFirstMediaStreamSsrc].type = + webrtc::VideoSendStream::StreamStats::StreamType::kMedia; + substreams[kFirstMediaStreamSsrc].rtp_stats.transmitted.header_bytes = 1; + substreams[kFirstMediaStreamSsrc].rtp_stats.transmitted.padding_bytes = 2; + substreams[kFirstMediaStreamSsrc].rtp_stats.transmitted.payload_bytes = 3; + substreams[kFirstMediaStreamSsrc].rtp_stats.transmitted.packets = 4; + substreams[kFirstMediaStreamSsrc].rtp_stats.retransmitted.header_bytes = 5; + substreams[kFirstMediaStreamSsrc].rtp_stats.retransmitted.padding_bytes = 6; + substreams[kFirstMediaStreamSsrc].rtp_stats.retransmitted.payload_bytes = 7; + substreams[kFirstMediaStreamSsrc].rtp_stats.retransmitted.packets = 8; + substreams[kFirstMediaStreamSsrc].referenced_media_ssrc = absl::nullopt; + substreams[kFirstMediaStreamSsrc].width = 1280; + substreams[kFirstMediaStreamSsrc].height = 720; + // Second kMedia stream. + substreams[kSecondMediaStreamSsrc].type = + webrtc::VideoSendStream::StreamStats::StreamType::kMedia; + substreams[kSecondMediaStreamSsrc].rtp_stats.transmitted.header_bytes = 10; + substreams[kSecondMediaStreamSsrc].rtp_stats.transmitted.padding_bytes = 11; + substreams[kSecondMediaStreamSsrc].rtp_stats.transmitted.payload_bytes = 12; + substreams[kSecondMediaStreamSsrc].rtp_stats.transmitted.packets = 13; + substreams[kSecondMediaStreamSsrc].rtp_stats.retransmitted.header_bytes = 14; + substreams[kSecondMediaStreamSsrc].rtp_stats.retransmitted.padding_bytes = 15; + substreams[kSecondMediaStreamSsrc].rtp_stats.retransmitted.payload_bytes = 16; + substreams[kSecondMediaStreamSsrc].rtp_stats.retransmitted.packets = 17; + substreams[kSecondMediaStreamSsrc].referenced_media_ssrc = absl::nullopt; + substreams[kSecondMediaStreamSsrc].width = 640; + substreams[kSecondMediaStreamSsrc].height = 480; + // kRtx stream referencing the first kMedia stream. + substreams[kRtxSsrc].type = + webrtc::VideoSendStream::StreamStats::StreamType::kRtx; + substreams[kRtxSsrc].rtp_stats.transmitted.header_bytes = 19; + substreams[kRtxSsrc].rtp_stats.transmitted.padding_bytes = 20; + substreams[kRtxSsrc].rtp_stats.transmitted.payload_bytes = 21; + substreams[kRtxSsrc].rtp_stats.transmitted.packets = 22; + substreams[kRtxSsrc].rtp_stats.retransmitted.header_bytes = 23; + substreams[kRtxSsrc].rtp_stats.retransmitted.padding_bytes = 24; + substreams[kRtxSsrc].rtp_stats.retransmitted.payload_bytes = 25; + substreams[kRtxSsrc].rtp_stats.retransmitted.packets = 26; + substreams[kRtxSsrc].referenced_media_ssrc = kFirstMediaStreamSsrc; + // kFlexfec stream referencing the second kMedia stream. + substreams[kFlexfecSsrc].type = + webrtc::VideoSendStream::StreamStats::StreamType::kFlexfec; + substreams[kFlexfecSsrc].rtp_stats.transmitted.header_bytes = 19; + substreams[kFlexfecSsrc].rtp_stats.transmitted.padding_bytes = 20; + substreams[kFlexfecSsrc].rtp_stats.transmitted.payload_bytes = 21; + substreams[kFlexfecSsrc].rtp_stats.transmitted.packets = 22; + substreams[kFlexfecSsrc].rtp_stats.retransmitted.header_bytes = 23; + substreams[kFlexfecSsrc].rtp_stats.retransmitted.padding_bytes = 24; + substreams[kFlexfecSsrc].rtp_stats.retransmitted.payload_bytes = 25; + substreams[kFlexfecSsrc].rtp_stats.retransmitted.packets = 26; + substreams[kFlexfecSsrc].referenced_media_ssrc = kSecondMediaStreamSsrc; + + auto merged_substreams = + MergeInfoAboutOutboundRtpSubstreamsForTesting(substreams); + // Only kMedia substreams remain. + EXPECT_TRUE(merged_substreams.find(kFirstMediaStreamSsrc) != + merged_substreams.end()); + EXPECT_EQ(merged_substreams[kFirstMediaStreamSsrc].type, + webrtc::VideoSendStream::StreamStats::StreamType::kMedia); + EXPECT_TRUE(merged_substreams.find(kSecondMediaStreamSsrc) != + merged_substreams.end()); + EXPECT_EQ(merged_substreams[kSecondMediaStreamSsrc].type, + webrtc::VideoSendStream::StreamStats::StreamType::kMedia); + EXPECT_FALSE(merged_substreams.find(kRtxSsrc) != merged_substreams.end()); + EXPECT_FALSE(merged_substreams.find(kFlexfecSsrc) != merged_substreams.end()); + // Expect kFirstMediaStreamSsrc's rtp_stats to be merged with kRtxSsrc. + webrtc::StreamDataCounters first_media_expected_rtp_stats = + substreams[kFirstMediaStreamSsrc].rtp_stats; + first_media_expected_rtp_stats.Add(substreams[kRtxSsrc].rtp_stats); + EXPECT_EQ(merged_substreams[kFirstMediaStreamSsrc].rtp_stats.transmitted, + first_media_expected_rtp_stats.transmitted); + EXPECT_EQ(merged_substreams[kFirstMediaStreamSsrc].rtp_stats.retransmitted, + first_media_expected_rtp_stats.retransmitted); + // Expect kSecondMediaStreamSsrc' rtp_stats to be merged with kFlexfecSsrc. + webrtc::StreamDataCounters second_media_expected_rtp_stats = + substreams[kSecondMediaStreamSsrc].rtp_stats; + second_media_expected_rtp_stats.Add(substreams[kFlexfecSsrc].rtp_stats); + EXPECT_EQ(merged_substreams[kSecondMediaStreamSsrc].rtp_stats.transmitted, + second_media_expected_rtp_stats.transmitted); + EXPECT_EQ(merged_substreams[kSecondMediaStreamSsrc].rtp_stats.retransmitted, + second_media_expected_rtp_stats.retransmitted); + // Expect other metrics to come from the original kMedia stats. + EXPECT_EQ(merged_substreams[kFirstMediaStreamSsrc].width, + substreams[kFirstMediaStreamSsrc].width); + EXPECT_EQ(merged_substreams[kFirstMediaStreamSsrc].height, + substreams[kFirstMediaStreamSsrc].height); + EXPECT_EQ(merged_substreams[kSecondMediaStreamSsrc].width, + substreams[kSecondMediaStreamSsrc].width); + EXPECT_EQ(merged_substreams[kSecondMediaStreamSsrc].height, + substreams[kSecondMediaStreamSsrc].height); +} + +TEST_F(WebRtcVideoChannelTest, + GetStatsReportsTransmittedAndRetransmittedBytesAndPacketsCorrectly) { + FakeVideoSendStream* stream = AddSendStream(); + webrtc::VideoSendStream::Stats stats; + // Simulcast layer 1, RTP stream. header+padding=10, payload=20, packets=3. + stats.substreams[101].type = + webrtc::VideoSendStream::StreamStats::StreamType::kMedia; + stats.substreams[101].rtp_stats.transmitted.header_bytes = 5; + stats.substreams[101].rtp_stats.transmitted.padding_bytes = 5; + stats.substreams[101].rtp_stats.transmitted.payload_bytes = 20; + stats.substreams[101].rtp_stats.transmitted.packets = 3; + stats.substreams[101].rtp_stats.retransmitted.header_bytes = 0; + stats.substreams[101].rtp_stats.retransmitted.padding_bytes = 0; + stats.substreams[101].rtp_stats.retransmitted.payload_bytes = 0; + stats.substreams[101].rtp_stats.retransmitted.packets = 0; + stats.substreams[101].referenced_media_ssrc = absl::nullopt; + // Simulcast layer 1, RTX stream. header+padding=5, payload=10, packets=1. + stats.substreams[102].type = + webrtc::VideoSendStream::StreamStats::StreamType::kRtx; + stats.substreams[102].rtp_stats.retransmitted.header_bytes = 3; + stats.substreams[102].rtp_stats.retransmitted.padding_bytes = 2; + stats.substreams[102].rtp_stats.retransmitted.payload_bytes = 10; + stats.substreams[102].rtp_stats.retransmitted.packets = 1; + stats.substreams[102].rtp_stats.transmitted = + stats.substreams[102].rtp_stats.retransmitted; + stats.substreams[102].referenced_media_ssrc = 101; + // Simulcast layer 2, RTP stream. header+padding=20, payload=40, packets=7. + stats.substreams[201].type = + webrtc::VideoSendStream::StreamStats::StreamType::kMedia; + stats.substreams[201].rtp_stats.transmitted.header_bytes = 10; + stats.substreams[201].rtp_stats.transmitted.padding_bytes = 10; + stats.substreams[201].rtp_stats.transmitted.payload_bytes = 40; + stats.substreams[201].rtp_stats.transmitted.packets = 7; + stats.substreams[201].rtp_stats.retransmitted.header_bytes = 0; + stats.substreams[201].rtp_stats.retransmitted.padding_bytes = 0; + stats.substreams[201].rtp_stats.retransmitted.payload_bytes = 0; + stats.substreams[201].rtp_stats.retransmitted.packets = 0; + stats.substreams[201].referenced_media_ssrc = absl::nullopt; + // Simulcast layer 2, RTX stream. header+padding=10, payload=20, packets=4. + stats.substreams[202].type = + webrtc::VideoSendStream::StreamStats::StreamType::kRtx; + stats.substreams[202].rtp_stats.retransmitted.header_bytes = 6; + stats.substreams[202].rtp_stats.retransmitted.padding_bytes = 4; + stats.substreams[202].rtp_stats.retransmitted.payload_bytes = 20; + stats.substreams[202].rtp_stats.retransmitted.packets = 4; + stats.substreams[202].rtp_stats.transmitted = + stats.substreams[202].rtp_stats.retransmitted; + stats.substreams[202].referenced_media_ssrc = 201; + // FlexFEC stream associated with the Simulcast layer 2. + // header+padding=15, payload=17, packets=5. + stats.substreams[301].type = + webrtc::VideoSendStream::StreamStats::StreamType::kFlexfec; + stats.substreams[301].rtp_stats.transmitted.header_bytes = 13; + stats.substreams[301].rtp_stats.transmitted.padding_bytes = 2; + stats.substreams[301].rtp_stats.transmitted.payload_bytes = 17; + stats.substreams[301].rtp_stats.transmitted.packets = 5; + stats.substreams[301].rtp_stats.retransmitted.header_bytes = 0; + stats.substreams[301].rtp_stats.retransmitted.padding_bytes = 0; + stats.substreams[301].rtp_stats.retransmitted.payload_bytes = 0; + stats.substreams[301].rtp_stats.retransmitted.packets = 0; + stats.substreams[301].referenced_media_ssrc = 201; + stream->SetStats(stats); + + cricket::VideoMediaInfo info; + ASSERT_TRUE(channel_->GetStats(&info)); + EXPECT_EQ(info.senders.size(), 2u); + EXPECT_EQ(15u, info.senders[0].header_and_padding_bytes_sent); + EXPECT_EQ(30u, info.senders[0].payload_bytes_sent); + EXPECT_EQ(4, info.senders[0].packets_sent); + EXPECT_EQ(10u, info.senders[0].retransmitted_bytes_sent); + EXPECT_EQ(1u, info.senders[0].retransmitted_packets_sent); + + EXPECT_EQ(45u, info.senders[1].header_and_padding_bytes_sent); + EXPECT_EQ(77u, info.senders[1].payload_bytes_sent); + EXPECT_EQ(16, info.senders[1].packets_sent); + EXPECT_EQ(20u, info.senders[1].retransmitted_bytes_sent); + EXPECT_EQ(4u, info.senders[1].retransmitted_packets_sent); +} + +TEST_F(WebRtcVideoChannelTest, + GetStatsTranslatesBandwidthLimitedResolutionCorrectly) { + FakeVideoSendStream* stream = AddSendStream(); + webrtc::VideoSendStream::Stats stats; + stats.bw_limited_resolution = true; + stream->SetStats(stats); + + cricket::VideoMediaInfo info; + EXPECT_TRUE(channel_->GetStats(&info)); + ASSERT_EQ(1U, info.senders.size()); + EXPECT_EQ(WebRtcVideoChannel::ADAPTREASON_BANDWIDTH, + info.senders[0].adapt_reason); +} + +TEST_F(WebRtcVideoChannelTest, GetStatsTranslatesSendRtcpPacketTypesCorrectly) { + FakeVideoSendStream* stream = AddSendStream(); + webrtc::VideoSendStream::Stats stats; + stats.substreams[17].rtcp_packet_type_counts.fir_packets = 2; + stats.substreams[17].rtcp_packet_type_counts.nack_packets = 3; + stats.substreams[17].rtcp_packet_type_counts.pli_packets = 4; + + stats.substreams[42].rtcp_packet_type_counts.fir_packets = 5; + stats.substreams[42].rtcp_packet_type_counts.nack_packets = 7; + stats.substreams[42].rtcp_packet_type_counts.pli_packets = 9; + + stream->SetStats(stats); + + cricket::VideoMediaInfo info; + ASSERT_TRUE(channel_->GetStats(&info)); + EXPECT_EQ(2, info.senders[0].firs_rcvd); + EXPECT_EQ(3, info.senders[0].nacks_rcvd); + EXPECT_EQ(4, info.senders[0].plis_rcvd); + + EXPECT_EQ(5, info.senders[1].firs_rcvd); + EXPECT_EQ(7, info.senders[1].nacks_rcvd); + EXPECT_EQ(9, info.senders[1].plis_rcvd); + + EXPECT_EQ(7, info.aggregated_senders[0].firs_rcvd); + EXPECT_EQ(10, info.aggregated_senders[0].nacks_rcvd); + EXPECT_EQ(13, info.aggregated_senders[0].plis_rcvd); +} + +TEST_F(WebRtcVideoChannelTest, + GetStatsTranslatesReceiveRtcpPacketTypesCorrectly) { + FakeVideoReceiveStream* stream = AddRecvStream(); + webrtc::VideoReceiveStream::Stats stats; + stats.rtcp_packet_type_counts.fir_packets = 2; + stats.rtcp_packet_type_counts.nack_packets = 3; + stats.rtcp_packet_type_counts.pli_packets = 4; + stream->SetStats(stats); + + cricket::VideoMediaInfo info; + ASSERT_TRUE(channel_->GetStats(&info)); + EXPECT_EQ(stats.rtcp_packet_type_counts.fir_packets, + rtc::checked_cast<unsigned int>(info.receivers[0].firs_sent)); + EXPECT_EQ(stats.rtcp_packet_type_counts.nack_packets, + rtc::checked_cast<unsigned int>(info.receivers[0].nacks_sent)); + EXPECT_EQ(stats.rtcp_packet_type_counts.pli_packets, + rtc::checked_cast<unsigned int>(info.receivers[0].plis_sent)); +} + +TEST_F(WebRtcVideoChannelTest, GetStatsTranslatesDecodeStatsCorrectly) { + FakeVideoReceiveStream* stream = AddRecvStream(); + webrtc::VideoReceiveStream::Stats stats; + stats.decoder_implementation_name = "decoder_implementation_name"; + stats.decode_ms = 2; + stats.max_decode_ms = 3; + stats.current_delay_ms = 4; + stats.target_delay_ms = 5; + stats.jitter_buffer_ms = 6; + stats.jitter_buffer_delay_seconds = 60; + stats.jitter_buffer_emitted_count = 6; + stats.min_playout_delay_ms = 7; + stats.render_delay_ms = 8; + stats.width = 9; + stats.height = 10; + stats.frame_counts.key_frames = 11; + stats.frame_counts.delta_frames = 12; + stats.frames_rendered = 13; + stats.frames_decoded = 14; + stats.qp_sum = 15; + stats.total_decode_time_ms = 16; + stream->SetStats(stats); + + cricket::VideoMediaInfo info; + ASSERT_TRUE(channel_->GetStats(&info)); + EXPECT_EQ(stats.decoder_implementation_name, + info.receivers[0].decoder_implementation_name); + EXPECT_EQ(stats.decode_ms, info.receivers[0].decode_ms); + EXPECT_EQ(stats.max_decode_ms, info.receivers[0].max_decode_ms); + EXPECT_EQ(stats.current_delay_ms, info.receivers[0].current_delay_ms); + EXPECT_EQ(stats.target_delay_ms, info.receivers[0].target_delay_ms); + EXPECT_EQ(stats.jitter_buffer_ms, info.receivers[0].jitter_buffer_ms); + EXPECT_EQ(stats.jitter_buffer_delay_seconds, + info.receivers[0].jitter_buffer_delay_seconds); + EXPECT_EQ(stats.jitter_buffer_emitted_count, + info.receivers[0].jitter_buffer_emitted_count); + EXPECT_EQ(stats.min_playout_delay_ms, info.receivers[0].min_playout_delay_ms); + EXPECT_EQ(stats.render_delay_ms, info.receivers[0].render_delay_ms); + EXPECT_EQ(stats.width, info.receivers[0].frame_width); + EXPECT_EQ(stats.height, info.receivers[0].frame_height); + EXPECT_EQ(rtc::checked_cast<unsigned int>(stats.frame_counts.key_frames + + stats.frame_counts.delta_frames), + info.receivers[0].frames_received); + EXPECT_EQ(stats.frames_rendered, info.receivers[0].frames_rendered); + EXPECT_EQ(stats.frames_decoded, info.receivers[0].frames_decoded); + EXPECT_EQ(rtc::checked_cast<unsigned int>(stats.frame_counts.key_frames), + info.receivers[0].key_frames_decoded); + EXPECT_EQ(stats.qp_sum, info.receivers[0].qp_sum); + EXPECT_EQ(stats.total_decode_time_ms, info.receivers[0].total_decode_time_ms); +} + +TEST_F(WebRtcVideoChannelTest, + GetStatsTranslatesInterFrameDelayStatsCorrectly) { + FakeVideoReceiveStream* stream = AddRecvStream(); + webrtc::VideoReceiveStream::Stats stats; + stats.total_inter_frame_delay = 0.123; + stats.total_squared_inter_frame_delay = 0.00456; + stream->SetStats(stats); + + cricket::VideoMediaInfo info; + ASSERT_TRUE(channel_->GetStats(&info)); + EXPECT_EQ(stats.total_inter_frame_delay, + info.receivers[0].total_inter_frame_delay); + EXPECT_EQ(stats.total_squared_inter_frame_delay, + info.receivers[0].total_squared_inter_frame_delay); +} + +TEST_F(WebRtcVideoChannelTest, GetStatsTranslatesReceivePacketStatsCorrectly) { + FakeVideoReceiveStream* stream = AddRecvStream(); + webrtc::VideoReceiveStream::Stats stats; + stats.rtp_stats.packet_counter.payload_bytes = 2; + stats.rtp_stats.packet_counter.header_bytes = 3; + stats.rtp_stats.packet_counter.padding_bytes = 4; + stats.rtp_stats.packet_counter.packets = 5; + stats.rtp_stats.packets_lost = 6; + stream->SetStats(stats); + + cricket::VideoMediaInfo info; + ASSERT_TRUE(channel_->GetStats(&info)); + EXPECT_EQ(stats.rtp_stats.packet_counter.payload_bytes, + rtc::checked_cast<size_t>(info.receivers[0].payload_bytes_rcvd)); + EXPECT_EQ(stats.rtp_stats.packet_counter.packets, + rtc::checked_cast<unsigned int>(info.receivers[0].packets_rcvd)); + EXPECT_EQ(stats.rtp_stats.packets_lost, info.receivers[0].packets_lost); +} + +TEST_F(WebRtcVideoChannelTest, TranslatesCallStatsCorrectly) { + AddSendStream(); + AddSendStream(); + webrtc::Call::Stats stats; + stats.rtt_ms = 123; + fake_call_->SetStats(stats); + + cricket::VideoMediaInfo info; + ASSERT_TRUE(channel_->GetStats(&info)); + ASSERT_EQ(2u, info.senders.size()); + EXPECT_EQ(stats.rtt_ms, info.senders[0].rtt_ms); + EXPECT_EQ(stats.rtt_ms, info.senders[1].rtt_ms); +} + +TEST_F(WebRtcVideoChannelTest, TranslatesSenderBitrateStatsCorrectly) { + FakeVideoSendStream* stream = AddSendStream(); + webrtc::VideoSendStream::Stats stats; + stats.target_media_bitrate_bps = 156; + stats.media_bitrate_bps = 123; + stats.substreams[17].total_bitrate_bps = 1; + stats.substreams[17].retransmit_bitrate_bps = 2; + stats.substreams[42].total_bitrate_bps = 3; + stats.substreams[42].retransmit_bitrate_bps = 4; + stream->SetStats(stats); + + FakeVideoSendStream* stream2 = AddSendStream(); + webrtc::VideoSendStream::Stats stats2; + stats2.target_media_bitrate_bps = 200; + stats2.media_bitrate_bps = 321; + stats2.substreams[13].total_bitrate_bps = 5; + stats2.substreams[13].retransmit_bitrate_bps = 6; + stats2.substreams[21].total_bitrate_bps = 7; + stats2.substreams[21].retransmit_bitrate_bps = 8; + stream2->SetStats(stats2); + + cricket::VideoMediaInfo info; + ASSERT_TRUE(channel_->GetStats(&info)); + ASSERT_EQ(2u, info.aggregated_senders.size()); + ASSERT_EQ(4u, info.senders.size()); + BandwidthEstimationInfo bwe_info; + channel_->FillBitrateInfo(&bwe_info); + // Assuming stream and stream2 corresponds to senders[0] and [1] respectively + // is OK as std::maps are sorted and AddSendStream() gives increasing SSRCs. + EXPECT_EQ(stats.media_bitrate_bps, + info.aggregated_senders[0].nominal_bitrate); + EXPECT_EQ(stats2.media_bitrate_bps, + info.aggregated_senders[1].nominal_bitrate); + EXPECT_EQ(stats.target_media_bitrate_bps + stats2.target_media_bitrate_bps, + bwe_info.target_enc_bitrate); + EXPECT_EQ(stats.media_bitrate_bps + stats2.media_bitrate_bps, + bwe_info.actual_enc_bitrate); + EXPECT_EQ(1 + 3 + 5 + 7, bwe_info.transmit_bitrate) + << "Bandwidth stats should take all streams into account."; + EXPECT_EQ(2 + 4 + 6 + 8, bwe_info.retransmit_bitrate) + << "Bandwidth stats should take all streams into account."; +} + +TEST_F(WebRtcVideoChannelTest, DefaultReceiveStreamReconfiguresToUseRtx) { + EXPECT_TRUE(channel_->SetSendParameters(send_parameters_)); + + const std::vector<uint32_t> ssrcs = MAKE_VECTOR(kSsrcs1); + const std::vector<uint32_t> rtx_ssrcs = MAKE_VECTOR(kRtxSsrcs1); + + ASSERT_EQ(0u, fake_call_->GetVideoReceiveStreams().size()); + const size_t kDataLength = 12; + uint8_t data[kDataLength]; + memset(data, 0, sizeof(data)); + rtc::SetBE32(&data[8], ssrcs[0]); + rtc::CopyOnWriteBuffer packet(data, kDataLength); + channel_->OnPacketReceived(packet, /* packet_time_us */ -1); + + ASSERT_EQ(1u, fake_call_->GetVideoReceiveStreams().size()) + << "No default receive stream created."; + FakeVideoReceiveStream* recv_stream = fake_call_->GetVideoReceiveStreams()[0]; + EXPECT_EQ(0u, recv_stream->GetConfig().rtp.rtx_ssrc) + << "Default receive stream should not have configured RTX"; + + EXPECT_TRUE(channel_->AddRecvStream( + cricket::CreateSimWithRtxStreamParams("cname", ssrcs, rtx_ssrcs))); + ASSERT_EQ(1u, fake_call_->GetVideoReceiveStreams().size()) + << "AddRecvStream should have reconfigured, not added a new receiver."; + recv_stream = fake_call_->GetVideoReceiveStreams()[0]; + EXPECT_FALSE( + recv_stream->GetConfig().rtp.rtx_associated_payload_types.empty()); + EXPECT_TRUE(VerifyRtxReceiveAssociations(recv_stream->GetConfig())) + << "RTX should be mapped for all decoders/payload types."; + EXPECT_TRUE(HasRtxReceiveAssociation(recv_stream->GetConfig(), + GetEngineCodec("red").id)) + << "RTX should be mapped also for the RED payload type"; + EXPECT_EQ(rtx_ssrcs[0], recv_stream->GetConfig().rtp.rtx_ssrc); +} + +TEST_F(WebRtcVideoChannelTest, RejectsAddingStreamsWithMissingSsrcsForRtx) { + EXPECT_TRUE(channel_->SetSendParameters(send_parameters_)); + + const std::vector<uint32_t> ssrcs = MAKE_VECTOR(kSsrcs1); + const std::vector<uint32_t> rtx_ssrcs = MAKE_VECTOR(kRtxSsrcs1); + + StreamParams sp = + cricket::CreateSimWithRtxStreamParams("cname", ssrcs, rtx_ssrcs); + sp.ssrcs = ssrcs; // Without RTXs, this is the important part. + + EXPECT_FALSE(channel_->AddSendStream(sp)); + EXPECT_FALSE(channel_->AddRecvStream(sp)); +} + +TEST_F(WebRtcVideoChannelTest, RejectsAddingStreamsWithOverlappingRtxSsrcs) { + EXPECT_TRUE(channel_->SetSendParameters(send_parameters_)); + + const std::vector<uint32_t> ssrcs = MAKE_VECTOR(kSsrcs1); + const std::vector<uint32_t> rtx_ssrcs = MAKE_VECTOR(kRtxSsrcs1); + + StreamParams sp = + cricket::CreateSimWithRtxStreamParams("cname", ssrcs, rtx_ssrcs); + + EXPECT_TRUE(channel_->AddSendStream(sp)); + EXPECT_TRUE(channel_->AddRecvStream(sp)); + + // The RTX SSRC is already used in previous streams, using it should fail. + sp = cricket::StreamParams::CreateLegacy(rtx_ssrcs[0]); + EXPECT_FALSE(channel_->AddSendStream(sp)); + EXPECT_FALSE(channel_->AddRecvStream(sp)); + + // After removing the original stream this should be fine to add (makes sure + // that RTX ssrcs are not forever taken). + EXPECT_TRUE(channel_->RemoveSendStream(ssrcs[0])); + EXPECT_TRUE(channel_->RemoveRecvStream(ssrcs[0])); + EXPECT_TRUE(channel_->AddSendStream(sp)); + EXPECT_TRUE(channel_->AddRecvStream(sp)); +} + +TEST_F(WebRtcVideoChannelTest, + RejectsAddingStreamsWithOverlappingSimulcastSsrcs) { + static const uint32_t kFirstStreamSsrcs[] = {1, 2, 3}; + static const uint32_t kOverlappingStreamSsrcs[] = {4, 3, 5}; + EXPECT_TRUE(channel_->SetSendParameters(send_parameters_)); + + StreamParams sp = + cricket::CreateSimStreamParams("cname", MAKE_VECTOR(kFirstStreamSsrcs)); + + EXPECT_TRUE(channel_->AddSendStream(sp)); + EXPECT_TRUE(channel_->AddRecvStream(sp)); + + // One of the SSRCs is already used in previous streams, using it should fail. + sp = cricket::CreateSimStreamParams("cname", + MAKE_VECTOR(kOverlappingStreamSsrcs)); + EXPECT_FALSE(channel_->AddSendStream(sp)); + EXPECT_FALSE(channel_->AddRecvStream(sp)); + + // After removing the original stream this should be fine to add (makes sure + // that RTX ssrcs are not forever taken). + EXPECT_TRUE(channel_->RemoveSendStream(kFirstStreamSsrcs[0])); + EXPECT_TRUE(channel_->RemoveRecvStream(kFirstStreamSsrcs[0])); + EXPECT_TRUE(channel_->AddSendStream(sp)); + EXPECT_TRUE(channel_->AddRecvStream(sp)); +} + +TEST_F(WebRtcVideoChannelTest, ReportsSsrcGroupsInStats) { + EXPECT_TRUE(channel_->SetSendParameters(send_parameters_)); + + static const uint32_t kSenderSsrcs[] = {4, 7, 10}; + static const uint32_t kSenderRtxSsrcs[] = {5, 8, 11}; + + StreamParams sender_sp = cricket::CreateSimWithRtxStreamParams( + "cname", MAKE_VECTOR(kSenderSsrcs), MAKE_VECTOR(kSenderRtxSsrcs)); + + EXPECT_TRUE(channel_->AddSendStream(sender_sp)); + + static const uint32_t kReceiverSsrcs[] = {3}; + static const uint32_t kReceiverRtxSsrcs[] = {2}; + + StreamParams receiver_sp = cricket::CreateSimWithRtxStreamParams( + "cname", MAKE_VECTOR(kReceiverSsrcs), MAKE_VECTOR(kReceiverRtxSsrcs)); + EXPECT_TRUE(channel_->AddRecvStream(receiver_sp)); + + cricket::VideoMediaInfo info; + ASSERT_TRUE(channel_->GetStats(&info)); + + ASSERT_EQ(1u, info.senders.size()); + ASSERT_EQ(1u, info.receivers.size()); + + EXPECT_NE(sender_sp.ssrc_groups, receiver_sp.ssrc_groups); + EXPECT_EQ(sender_sp.ssrc_groups, info.senders[0].ssrc_groups); + EXPECT_EQ(receiver_sp.ssrc_groups, info.receivers[0].ssrc_groups); +} + +TEST_F(WebRtcVideoChannelTest, MapsReceivedPayloadTypeToCodecName) { + FakeVideoReceiveStream* stream = AddRecvStream(); + webrtc::VideoReceiveStream::Stats stats; + cricket::VideoMediaInfo info; + + // Report no codec name before receiving. + stream->SetStats(stats); + ASSERT_TRUE(channel_->GetStats(&info)); + EXPECT_STREQ("", info.receivers[0].codec_name.c_str()); + + // Report VP8 if we're receiving it. + stats.current_payload_type = GetEngineCodec("VP8").id; + stream->SetStats(stats); + ASSERT_TRUE(channel_->GetStats(&info)); + EXPECT_STREQ(kVp8CodecName, info.receivers[0].codec_name.c_str()); + + // Report no codec name for unknown playload types. + stats.current_payload_type = 3; + stream->SetStats(stats); + ASSERT_TRUE(channel_->GetStats(&info)); + EXPECT_STREQ("", info.receivers[0].codec_name.c_str()); +} + +// Tests that when we add a stream without SSRCs, but contains a stream_id +// that it is stored and its stream id is later used when the first packet +// arrives to properly create a receive stream with a sync label. +TEST_F(WebRtcVideoChannelTest, RecvUnsignaledSsrcWithSignaledStreamId) { + const char kSyncLabel[] = "sync_label"; + cricket::StreamParams unsignaled_stream; + unsignaled_stream.set_stream_ids({kSyncLabel}); + ASSERT_TRUE(channel_->AddRecvStream(unsignaled_stream)); + // The stream shouldn't have been created at this point because it doesn't + // have any SSRCs. + EXPECT_EQ(0u, fake_call_->GetVideoReceiveStreams().size()); + + // Create and deliver packet. + const size_t kDataLength = 12; + uint8_t data[kDataLength]; + memset(data, 0, sizeof(data)); + rtc::SetBE32(&data[8], kIncomingUnsignalledSsrc); + rtc::CopyOnWriteBuffer packet(data, kDataLength); + channel_->OnPacketReceived(packet, /* packet_time_us */ -1); + + // The stream should now be created with the appropriate sync label. + EXPECT_EQ(1u, fake_call_->GetVideoReceiveStreams().size()); + EXPECT_EQ(kSyncLabel, + fake_call_->GetVideoReceiveStreams()[0]->GetConfig().sync_group); + + // Reset the unsignaled stream to clear the cache. This time when + // a default video receive stream is created it won't have a sync_group. + channel_->ResetUnsignaledRecvStream(); + EXPECT_EQ(0u, fake_call_->GetVideoReceiveStreams().size()); + + channel_->OnPacketReceived(packet, /* packet_time_us */ -1); + EXPECT_EQ(1u, fake_call_->GetVideoReceiveStreams().size()); + EXPECT_TRUE( + fake_call_->GetVideoReceiveStreams()[0]->GetConfig().sync_group.empty()); +} + +TEST_F(WebRtcVideoChannelTest, + ResetUnsignaledRecvStreamDeletesAllDefaultStreams) { + // No receive streams to start with. + EXPECT_TRUE(fake_call_->GetVideoReceiveStreams().empty()); + + // Packet with unsignaled SSRC is received. + const size_t kDataLength = 12; + uint8_t data[kDataLength]; + memset(data, 0, sizeof(data)); + rtc::SetBE32(&data[8], kIncomingUnsignalledSsrc); + rtc::CopyOnWriteBuffer packet(data, kDataLength); + channel_->OnPacketReceived(packet, /* packet_time_us */ -1); + + // Default receive stream created. + const auto& receivers1 = fake_call_->GetVideoReceiveStreams(); + ASSERT_EQ(receivers1.size(), 1u); + EXPECT_EQ(receivers1[0]->GetConfig().rtp.remote_ssrc, + kIncomingUnsignalledSsrc); + + // Stream with another SSRC gets signaled. + channel_->ResetUnsignaledRecvStream(); + constexpr uint32_t kIncomingSignalledSsrc = kIncomingUnsignalledSsrc + 1; + ASSERT_TRUE(channel_->AddRecvStream( + cricket::StreamParams::CreateLegacy(kIncomingSignalledSsrc))); + + // New receiver is for the signaled stream. + const auto& receivers2 = fake_call_->GetVideoReceiveStreams(); + ASSERT_EQ(receivers2.size(), 1u); + EXPECT_EQ(receivers2[0]->GetConfig().rtp.remote_ssrc, kIncomingSignalledSsrc); +} + +// Test BaseMinimumPlayoutDelayMs on receive streams. +TEST_F(WebRtcVideoChannelTest, BaseMinimumPlayoutDelayMs) { + // Test that set won't work for non-existing receive streams. + EXPECT_FALSE(channel_->SetBaseMinimumPlayoutDelayMs(kSsrc + 2, 200)); + // Test that get won't work for non-existing receive streams. + EXPECT_FALSE(channel_->GetBaseMinimumPlayoutDelayMs(kSsrc + 2)); + + EXPECT_TRUE(AddRecvStream()); + // Test that set works for the existing receive stream. + EXPECT_TRUE(channel_->SetBaseMinimumPlayoutDelayMs(last_ssrc_, 200)); + auto* recv_stream = fake_call_->GetVideoReceiveStream(last_ssrc_); + EXPECT_TRUE(recv_stream); + EXPECT_EQ(recv_stream->base_mininum_playout_delay_ms(), 200); + EXPECT_EQ(channel_->GetBaseMinimumPlayoutDelayMs(last_ssrc_).value_or(0), + 200); +} + +// Test BaseMinimumPlayoutDelayMs on unsignaled receive streams. +TEST_F(WebRtcVideoChannelTest, BaseMinimumPlayoutDelayMsUnsignaledRecvStream) { + absl::optional<int> delay_ms; + const FakeVideoReceiveStream* recv_stream; + + // Set default stream with SSRC 0 + EXPECT_TRUE(channel_->SetBaseMinimumPlayoutDelayMs(0, 200)); + EXPECT_EQ(200, channel_->GetBaseMinimumPlayoutDelayMs(0).value_or(0)); + + // Spawn an unsignaled stream by sending a packet, it should inherit + // default delay 200. + const size_t kDataLength = 12; + uint8_t data[kDataLength]; + memset(data, 0, sizeof(data)); + rtc::SetBE32(&data[8], kIncomingUnsignalledSsrc); + rtc::CopyOnWriteBuffer packet(data, kDataLength); + channel_->OnPacketReceived(packet, /* packet_time_us */ -1); + + recv_stream = fake_call_->GetVideoReceiveStream(kIncomingUnsignalledSsrc); + EXPECT_EQ(recv_stream->base_mininum_playout_delay_ms(), 200); + delay_ms = channel_->GetBaseMinimumPlayoutDelayMs(kIncomingUnsignalledSsrc); + EXPECT_EQ(200, delay_ms.value_or(0)); + + // Check that now if we change delay for SSRC 0 it will change delay for the + // default receiving stream as well. + EXPECT_TRUE(channel_->SetBaseMinimumPlayoutDelayMs(0, 300)); + EXPECT_EQ(300, channel_->GetBaseMinimumPlayoutDelayMs(0).value_or(0)); + delay_ms = channel_->GetBaseMinimumPlayoutDelayMs(kIncomingUnsignalledSsrc); + EXPECT_EQ(300, delay_ms.value_or(0)); + recv_stream = fake_call_->GetVideoReceiveStream(kIncomingUnsignalledSsrc); + EXPECT_EQ(recv_stream->base_mininum_playout_delay_ms(), 300); +} + +void WebRtcVideoChannelTest::TestReceiveUnsignaledSsrcPacket( + uint8_t payload_type, + bool expect_created_receive_stream) { + // kRedRtxPayloadType must currently be unused. + EXPECT_FALSE(FindCodecById(engine_.recv_codecs(), kRedRtxPayloadType)); + + // Add a RED RTX codec. + VideoCodec red_rtx_codec = + VideoCodec::CreateRtxCodec(kRedRtxPayloadType, GetEngineCodec("red").id); + recv_parameters_.codecs.push_back(red_rtx_codec); + EXPECT_TRUE(channel_->SetRecvParameters(recv_parameters_)); + + ASSERT_EQ(0u, fake_call_->GetVideoReceiveStreams().size()); + const size_t kDataLength = 12; + uint8_t data[kDataLength]; + memset(data, 0, sizeof(data)); + + rtc::Set8(data, 1, payload_type); + rtc::SetBE32(&data[8], kIncomingUnsignalledSsrc); + rtc::CopyOnWriteBuffer packet(data, kDataLength); + channel_->OnPacketReceived(packet, /* packet_time_us */ -1); + + if (expect_created_receive_stream) { + EXPECT_EQ(1u, fake_call_->GetVideoReceiveStreams().size()) + << "Should have created a receive stream for payload type: " + << payload_type; + } else { + EXPECT_EQ(0u, fake_call_->GetVideoReceiveStreams().size()) + << "Shouldn't have created a receive stream for payload type: " + << payload_type; + } +} + +class WebRtcVideoChannelDiscardUnknownSsrcTest : public WebRtcVideoChannelTest { + public: + WebRtcVideoChannelDiscardUnknownSsrcTest() + : WebRtcVideoChannelTest( + "WebRTC-Video-DiscardPacketsWithUnknownSsrc/Enabled/") {} +}; + +TEST_F(WebRtcVideoChannelDiscardUnknownSsrcTest, NoUnsignalledStreamCreated) { + TestReceiveUnsignaledSsrcPacket(GetEngineCodec("VP8").id, + false /* expect_created_receive_stream */); +} + +TEST_F(WebRtcVideoChannelTest, Vp8PacketCreatesUnsignalledStream) { + TestReceiveUnsignaledSsrcPacket(GetEngineCodec("VP8").id, + true /* expect_created_receive_stream */); +} + +TEST_F(WebRtcVideoChannelTest, Vp9PacketCreatesUnsignalledStream) { + TestReceiveUnsignaledSsrcPacket(GetEngineCodec("VP9").id, + true /* expect_created_receive_stream */); +} + +TEST_F(WebRtcVideoChannelTest, RtxPacketDoesntCreateUnsignalledStream) { + AssignDefaultAptRtxTypes(); + const cricket::VideoCodec vp8 = GetEngineCodec("VP8"); + const int rtx_vp8_payload_type = default_apt_rtx_types_[vp8.id]; + TestReceiveUnsignaledSsrcPacket(rtx_vp8_payload_type, + false /* expect_created_receive_stream */); +} + +TEST_F(WebRtcVideoChannelTest, UlpfecPacketDoesntCreateUnsignalledStream) { + TestReceiveUnsignaledSsrcPacket(GetEngineCodec("ulpfec").id, + false /* expect_created_receive_stream */); +} + +TEST_F(WebRtcVideoChannelFlexfecRecvTest, + FlexfecPacketDoesntCreateUnsignalledStream) { + TestReceiveUnsignaledSsrcPacket(GetEngineCodec("flexfec-03").id, + false /* expect_created_receive_stream */); +} + +TEST_F(WebRtcVideoChannelTest, RedRtxPacketDoesntCreateUnsignalledStream) { + TestReceiveUnsignaledSsrcPacket(kRedRtxPayloadType, + false /* expect_created_receive_stream */); +} + +// Test that receiving any unsignalled SSRC works even if it changes. +// The first unsignalled SSRC received will create a default receive stream. +// Any different unsignalled SSRC received will replace the default. +TEST_F(WebRtcVideoChannelTest, ReceiveDifferentUnsignaledSsrc) { + // Allow receiving VP8, VP9, H264 (if enabled). + cricket::VideoRecvParameters parameters; + parameters.codecs.push_back(GetEngineCodec("VP8")); + parameters.codecs.push_back(GetEngineCodec("VP9")); + +#if defined(WEBRTC_USE_H264) + cricket::VideoCodec H264codec(126, "H264"); + parameters.codecs.push_back(H264codec); +#endif + + EXPECT_TRUE(channel_->SetRecvParameters(parameters)); + // No receive streams yet. + ASSERT_EQ(0u, fake_call_->GetVideoReceiveStreams().size()); + cricket::FakeVideoRenderer renderer; + channel_->SetDefaultSink(&renderer); + + // Receive VP8 packet on first SSRC. + uint8_t data[kMinRtpPacketLen]; + cricket::RtpHeader rtpHeader; + rtpHeader.payload_type = GetEngineCodec("VP8").id; + rtpHeader.seq_num = rtpHeader.timestamp = 0; + rtpHeader.ssrc = kIncomingUnsignalledSsrc + 1; + cricket::SetRtpHeader(data, sizeof(data), rtpHeader); + rtc::CopyOnWriteBuffer packet(data, sizeof(data)); + channel_->OnPacketReceived(packet, /* packet_time_us */ -1); + // VP8 packet should create default receive stream. + ASSERT_EQ(1u, fake_call_->GetVideoReceiveStreams().size()); + FakeVideoReceiveStream* recv_stream = fake_call_->GetVideoReceiveStreams()[0]; + EXPECT_EQ(rtpHeader.ssrc, recv_stream->GetConfig().rtp.remote_ssrc); + // Verify that the receive stream sinks to a renderer. + webrtc::VideoFrame video_frame = + webrtc::VideoFrame::Builder() + .set_video_frame_buffer(CreateBlackFrameBuffer(4, 4)) + .set_timestamp_rtp(100) + .set_timestamp_us(0) + .set_rotation(webrtc::kVideoRotation_0) + .build(); + recv_stream->InjectFrame(video_frame); + EXPECT_EQ(1, renderer.num_rendered_frames()); + + // Receive VP9 packet on second SSRC. + rtpHeader.payload_type = GetEngineCodec("VP9").id; + rtpHeader.ssrc = kIncomingUnsignalledSsrc + 2; + cricket::SetRtpHeader(data, sizeof(data), rtpHeader); + rtc::CopyOnWriteBuffer packet2(data, sizeof(data)); + channel_->OnPacketReceived(packet2, /* packet_time_us */ -1); + // VP9 packet should replace the default receive SSRC. + ASSERT_EQ(1u, fake_call_->GetVideoReceiveStreams().size()); + recv_stream = fake_call_->GetVideoReceiveStreams()[0]; + EXPECT_EQ(rtpHeader.ssrc, recv_stream->GetConfig().rtp.remote_ssrc); + // Verify that the receive stream sinks to a renderer. + webrtc::VideoFrame video_frame2 = + webrtc::VideoFrame::Builder() + .set_video_frame_buffer(CreateBlackFrameBuffer(4, 4)) + .set_timestamp_rtp(200) + .set_timestamp_us(0) + .set_rotation(webrtc::kVideoRotation_0) + .build(); + recv_stream->InjectFrame(video_frame2); + EXPECT_EQ(2, renderer.num_rendered_frames()); + +#if defined(WEBRTC_USE_H264) + // Receive H264 packet on third SSRC. + rtpHeader.payload_type = 126; + rtpHeader.ssrc = kIncomingUnsignalledSsrc + 3; + cricket::SetRtpHeader(data, sizeof(data), rtpHeader); + rtc::CopyOnWriteBuffer packet3(data, sizeof(data)); + channel_->OnPacketReceived(packet3, /* packet_time_us */ -1); + // H264 packet should replace the default receive SSRC. + ASSERT_EQ(1u, fake_call_->GetVideoReceiveStreams().size()); + recv_stream = fake_call_->GetVideoReceiveStreams()[0]; + EXPECT_EQ(rtpHeader.ssrc, recv_stream->GetConfig().rtp.remote_ssrc); + // Verify that the receive stream sinks to a renderer. + webrtc::VideoFrame video_frame3 = + webrtc::VideoFrame::Builder() + .set_video_frame_buffer(CreateBlackFrameBuffer(4, 4)) + .set_timestamp_rtp(300) + .set_timestamp_us(0) + .set_rotation(webrtc::kVideoRotation_0) + .build(); + recv_stream->InjectFrame(video_frame3); + EXPECT_EQ(3, renderer.num_rendered_frames()); +#endif +} + +// This test verifies that when a new default stream is created for a new +// unsignaled SSRC, the new stream does not overwrite any old stream that had +// been the default receive stream before being properly signaled. +TEST_F(WebRtcVideoChannelTest, + NewUnsignaledStreamDoesNotDestroyPreviouslyUnsignaledStream) { + cricket::VideoRecvParameters parameters; + parameters.codecs.push_back(GetEngineCodec("VP8")); + ASSERT_TRUE(channel_->SetRecvParameters(parameters)); + + // No streams signaled and no packets received, so we should not have any + // stream objects created yet. + EXPECT_EQ(0u, fake_call_->GetVideoReceiveStreams().size()); + + // Receive packet on an unsignaled SSRC. + uint8_t data[kMinRtpPacketLen]; + cricket::RtpHeader rtp_header; + rtp_header.payload_type = GetEngineCodec("VP8").id; + rtp_header.seq_num = rtp_header.timestamp = 0; + rtp_header.ssrc = kSsrcs3[0]; + cricket::SetRtpHeader(data, sizeof(data), rtp_header); + rtc::CopyOnWriteBuffer packet(data, sizeof(data)); + channel_->OnPacketReceived(packet, /* packet_time_us */ -1); + // Default receive stream should be created. + ASSERT_EQ(1u, fake_call_->GetVideoReceiveStreams().size()); + FakeVideoReceiveStream* recv_stream0 = + fake_call_->GetVideoReceiveStreams()[0]; + EXPECT_EQ(kSsrcs3[0], recv_stream0->GetConfig().rtp.remote_ssrc); + + // Signal the SSRC. + EXPECT_TRUE( + channel_->AddRecvStream(cricket::StreamParams::CreateLegacy(kSsrcs3[0]))); + ASSERT_EQ(1u, fake_call_->GetVideoReceiveStreams().size()); + recv_stream0 = fake_call_->GetVideoReceiveStreams()[0]; + EXPECT_EQ(kSsrcs3[0], recv_stream0->GetConfig().rtp.remote_ssrc); + + // Receive packet on a different unsignaled SSRC. + rtp_header.ssrc = kSsrcs3[1]; + cricket::SetRtpHeader(data, sizeof(data), rtp_header); + packet.SetData(data, sizeof(data)); + channel_->OnPacketReceived(packet, /* packet_time_us */ -1); + // New default receive stream should be created, but old stream should remain. + ASSERT_EQ(2u, fake_call_->GetVideoReceiveStreams().size()); + EXPECT_EQ(recv_stream0, fake_call_->GetVideoReceiveStreams()[0]); + FakeVideoReceiveStream* recv_stream1 = + fake_call_->GetVideoReceiveStreams()[1]; + EXPECT_EQ(kSsrcs3[1], recv_stream1->GetConfig().rtp.remote_ssrc); +} + +TEST_F(WebRtcVideoChannelTest, CanSetMaxBitrateForExistingStream) { + AddSendStream(); + + webrtc::test::FrameForwarder frame_forwarder; + EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, nullptr, &frame_forwarder)); + EXPECT_TRUE(channel_->SetSend(true)); + frame_forwarder.IncomingCapturedFrame(frame_source_.GetFrame()); + + int default_encoder_bitrate = GetMaxEncoderBitrate(); + EXPECT_GT(default_encoder_bitrate, 1000); + + // TODO(skvlad): Resolve the inconsistency between the interpretation + // of the global bitrate limit for audio and video: + // - Audio: max_bandwidth_bps = 0 - fail the operation, + // max_bandwidth_bps = -1 - remove the bandwidth limit + // - Video: max_bandwidth_bps = 0 - remove the bandwidth limit, + // max_bandwidth_bps = -1 - remove the bandwidth limit + + SetAndExpectMaxBitrate(1000, 0, 1000); + SetAndExpectMaxBitrate(1000, 800, 800); + SetAndExpectMaxBitrate(600, 800, 600); + SetAndExpectMaxBitrate(0, 800, 800); + SetAndExpectMaxBitrate(0, 0, default_encoder_bitrate); + + EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, nullptr, nullptr)); +} + +TEST_F(WebRtcVideoChannelTest, CannotSetMaxBitrateForNonexistentStream) { + webrtc::RtpParameters nonexistent_parameters = + channel_->GetRtpSendParameters(last_ssrc_); + EXPECT_EQ(0u, nonexistent_parameters.encodings.size()); + + nonexistent_parameters.encodings.push_back(webrtc::RtpEncodingParameters()); + EXPECT_FALSE( + channel_->SetRtpSendParameters(last_ssrc_, nonexistent_parameters).ok()); +} + +TEST_F(WebRtcVideoChannelTest, + SetLowMaxBitrateOverwritesVideoStreamMinBitrate) { + FakeVideoSendStream* stream = AddSendStream(); + + webrtc::RtpParameters parameters = channel_->GetRtpSendParameters(last_ssrc_); + EXPECT_EQ(1UL, parameters.encodings.size()); + EXPECT_FALSE(parameters.encodings[0].max_bitrate_bps.has_value()); + EXPECT_TRUE(channel_->SetRtpSendParameters(last_ssrc_, parameters).ok()); + + // Note that this is testing the behavior of the FakeVideoSendStream, which + // also calls to CreateEncoderStreams to get the VideoStreams, so essentially + // we are just testing the behavior of + // EncoderStreamFactory::CreateEncoderStreams. + ASSERT_EQ(1UL, stream->GetVideoStreams().size()); + EXPECT_EQ(webrtc::kDefaultMinVideoBitrateBps, + stream->GetVideoStreams()[0].min_bitrate_bps); + + // Set a low max bitrate & check that VideoStream.min_bitrate_bps is limited + // by this amount. + parameters = channel_->GetRtpSendParameters(last_ssrc_); + int low_max_bitrate_bps = webrtc::kDefaultMinVideoBitrateBps - 1000; + parameters.encodings[0].max_bitrate_bps = low_max_bitrate_bps; + EXPECT_TRUE(channel_->SetRtpSendParameters(last_ssrc_, parameters).ok()); + + ASSERT_EQ(1UL, stream->GetVideoStreams().size()); + EXPECT_EQ(low_max_bitrate_bps, stream->GetVideoStreams()[0].min_bitrate_bps); + EXPECT_EQ(low_max_bitrate_bps, stream->GetVideoStreams()[0].max_bitrate_bps); +} + +TEST_F(WebRtcVideoChannelTest, + SetHighMinBitrateOverwritesVideoStreamMaxBitrate) { + FakeVideoSendStream* stream = AddSendStream(); + + // Note that this is testing the behavior of the FakeVideoSendStream, which + // also calls to CreateEncoderStreams to get the VideoStreams, so essentially + // we are just testing the behavior of + // EncoderStreamFactory::CreateEncoderStreams. + ASSERT_EQ(1UL, stream->GetVideoStreams().size()); + int high_min_bitrate_bps = stream->GetVideoStreams()[0].max_bitrate_bps + 1; + + // Set a high min bitrate and check that max_bitrate_bps is adjusted up. + webrtc::RtpParameters parameters = channel_->GetRtpSendParameters(last_ssrc_); + EXPECT_EQ(1UL, parameters.encodings.size()); + parameters.encodings[0].min_bitrate_bps = high_min_bitrate_bps; + EXPECT_TRUE(channel_->SetRtpSendParameters(last_ssrc_, parameters).ok()); + + ASSERT_EQ(1UL, stream->GetVideoStreams().size()); + EXPECT_EQ(high_min_bitrate_bps, stream->GetVideoStreams()[0].min_bitrate_bps); + EXPECT_EQ(high_min_bitrate_bps, stream->GetVideoStreams()[0].max_bitrate_bps); +} + +TEST_F(WebRtcVideoChannelTest, + SetMinBitrateAboveMaxBitrateLimitAdjustsMinBitrateDown) { + send_parameters_.max_bandwidth_bps = 99999; + FakeVideoSendStream* stream = AddSendStream(); + ExpectSetMaxBitrate(send_parameters_.max_bandwidth_bps); + ASSERT_TRUE(channel_->SetSendParameters(send_parameters_)); + ASSERT_EQ(1UL, stream->GetVideoStreams().size()); + EXPECT_EQ(webrtc::kDefaultMinVideoBitrateBps, + stream->GetVideoStreams()[0].min_bitrate_bps); + EXPECT_EQ(send_parameters_.max_bandwidth_bps, + stream->GetVideoStreams()[0].max_bitrate_bps); + + // Set min bitrate above global max bitrate and check that min_bitrate_bps is + // adjusted down. + webrtc::RtpParameters parameters = channel_->GetRtpSendParameters(last_ssrc_); + EXPECT_EQ(1UL, parameters.encodings.size()); + parameters.encodings[0].min_bitrate_bps = 99999 + 1; + EXPECT_TRUE(channel_->SetRtpSendParameters(last_ssrc_, parameters).ok()); + ASSERT_EQ(1UL, stream->GetVideoStreams().size()); + EXPECT_EQ(send_parameters_.max_bandwidth_bps, + stream->GetVideoStreams()[0].min_bitrate_bps); + EXPECT_EQ(send_parameters_.max_bandwidth_bps, + stream->GetVideoStreams()[0].max_bitrate_bps); +} + +TEST_F(WebRtcVideoChannelTest, SetMaxFramerateOneStream) { + FakeVideoSendStream* stream = AddSendStream(); + + webrtc::RtpParameters parameters = channel_->GetRtpSendParameters(last_ssrc_); + EXPECT_EQ(1UL, parameters.encodings.size()); + EXPECT_FALSE(parameters.encodings[0].max_framerate.has_value()); + EXPECT_TRUE(channel_->SetRtpSendParameters(last_ssrc_, parameters).ok()); + + // Note that this is testing the behavior of the FakeVideoSendStream, which + // also calls to CreateEncoderStreams to get the VideoStreams, so essentially + // we are just testing the behavior of + // EncoderStreamFactory::CreateEncoderStreams. + ASSERT_EQ(1UL, stream->GetVideoStreams().size()); + EXPECT_EQ(kDefaultVideoMaxFramerate, + stream->GetVideoStreams()[0].max_framerate); + + // Set max framerate and check that VideoStream.max_framerate is set. + const int kNewMaxFramerate = kDefaultVideoMaxFramerate - 1; + parameters = channel_->GetRtpSendParameters(last_ssrc_); + parameters.encodings[0].max_framerate = kNewMaxFramerate; + EXPECT_TRUE(channel_->SetRtpSendParameters(last_ssrc_, parameters).ok()); + + ASSERT_EQ(1UL, stream->GetVideoStreams().size()); + EXPECT_EQ(kNewMaxFramerate, stream->GetVideoStreams()[0].max_framerate); +} + +TEST_F(WebRtcVideoChannelTest, SetNumTemporalLayersForSingleStream) { + FakeVideoSendStream* stream = AddSendStream(); + + webrtc::RtpParameters parameters = channel_->GetRtpSendParameters(last_ssrc_); + EXPECT_EQ(1UL, parameters.encodings.size()); + EXPECT_FALSE(parameters.encodings[0].num_temporal_layers.has_value()); + EXPECT_TRUE(channel_->SetRtpSendParameters(last_ssrc_, parameters).ok()); + + // Note that this is testing the behavior of the FakeVideoSendStream, which + // also calls to CreateEncoderStreams to get the VideoStreams, so essentially + // we are just testing the behavior of + // EncoderStreamFactory::CreateEncoderStreams. + ASSERT_EQ(1UL, stream->GetVideoStreams().size()); + EXPECT_FALSE(stream->GetVideoStreams()[0].num_temporal_layers.has_value()); + + // Set temporal layers and check that VideoStream.num_temporal_layers is set. + parameters = channel_->GetRtpSendParameters(last_ssrc_); + parameters.encodings[0].num_temporal_layers = 2; + EXPECT_TRUE(channel_->SetRtpSendParameters(last_ssrc_, parameters).ok()); + + ASSERT_EQ(1UL, stream->GetVideoStreams().size()); + EXPECT_EQ(2UL, stream->GetVideoStreams()[0].num_temporal_layers); +} + +TEST_F(WebRtcVideoChannelTest, + CannotSetRtpSendParametersWithIncorrectNumberOfEncodings) { + AddSendStream(); + webrtc::RtpParameters parameters = channel_->GetRtpSendParameters(last_ssrc_); + // Two or more encodings should result in failure. + parameters.encodings.push_back(webrtc::RtpEncodingParameters()); + EXPECT_FALSE(channel_->SetRtpSendParameters(last_ssrc_, parameters).ok()); + // Zero encodings should also fail. + parameters.encodings.clear(); + EXPECT_FALSE(channel_->SetRtpSendParameters(last_ssrc_, parameters).ok()); +} + +TEST_F(WebRtcVideoChannelTest, + CannotSetSimulcastRtpSendParametersWithIncorrectNumberOfEncodings) { + std::vector<uint32_t> ssrcs = MAKE_VECTOR(kSsrcs3); + StreamParams sp = CreateSimStreamParams("cname", ssrcs); + AddSendStream(sp); + + webrtc::RtpParameters parameters = channel_->GetRtpSendParameters(last_ssrc_); + + // Additional encodings should result in failure. + parameters.encodings.push_back(webrtc::RtpEncodingParameters()); + EXPECT_FALSE(channel_->SetRtpSendParameters(last_ssrc_, parameters).ok()); + // Zero encodings should also fail. + parameters.encodings.clear(); + EXPECT_FALSE(channel_->SetRtpSendParameters(last_ssrc_, parameters).ok()); +} + +// Changing the SSRC through RtpParameters is not allowed. +TEST_F(WebRtcVideoChannelTest, CannotSetSsrcInRtpSendParameters) { + AddSendStream(); + webrtc::RtpParameters parameters = channel_->GetRtpSendParameters(last_ssrc_); + parameters.encodings[0].ssrc = 0xdeadbeef; + EXPECT_FALSE(channel_->SetRtpSendParameters(last_ssrc_, parameters).ok()); +} + +// Tests that when RTCRtpEncodingParameters.bitrate_priority gets set to +// a value <= 0, setting the parameters returns false. +TEST_F(WebRtcVideoChannelTest, SetRtpSendParametersInvalidBitratePriority) { + AddSendStream(); + webrtc::RtpParameters parameters = channel_->GetRtpSendParameters(last_ssrc_); + EXPECT_EQ(1UL, parameters.encodings.size()); + EXPECT_EQ(webrtc::kDefaultBitratePriority, + parameters.encodings[0].bitrate_priority); + + parameters.encodings[0].bitrate_priority = 0; + EXPECT_FALSE(channel_->SetRtpSendParameters(last_ssrc_, parameters).ok()); + parameters.encodings[0].bitrate_priority = -2; + EXPECT_FALSE(channel_->SetRtpSendParameters(last_ssrc_, parameters).ok()); +} + +// Tests when the the RTCRtpEncodingParameters.bitrate_priority gets set +// properly on the VideoChannel and propogates down to the video encoder. +TEST_F(WebRtcVideoChannelTest, SetRtpSendParametersPriorityOneStream) { + AddSendStream(); + webrtc::RtpParameters parameters = channel_->GetRtpSendParameters(last_ssrc_); + EXPECT_EQ(1UL, parameters.encodings.size()); + EXPECT_EQ(webrtc::kDefaultBitratePriority, + parameters.encodings[0].bitrate_priority); + + // Change the value and set it on the VideoChannel. + double new_bitrate_priority = 2.0; + parameters.encodings[0].bitrate_priority = new_bitrate_priority; + EXPECT_TRUE(channel_->SetRtpSendParameters(last_ssrc_, parameters).ok()); + + // Verify that the encoding parameters bitrate_priority is set for the + // VideoChannel. + parameters = channel_->GetRtpSendParameters(last_ssrc_); + EXPECT_EQ(1UL, parameters.encodings.size()); + EXPECT_EQ(new_bitrate_priority, parameters.encodings[0].bitrate_priority); + + // Verify that the new value propagated down to the encoder. + std::vector<FakeVideoSendStream*> video_send_streams = + fake_call_->GetVideoSendStreams(); + EXPECT_EQ(1UL, video_send_streams.size()); + FakeVideoSendStream* video_send_stream = video_send_streams.front(); + // Check that the WebRtcVideoSendStream updated the VideoEncoderConfig + // appropriately. + EXPECT_EQ(new_bitrate_priority, + video_send_stream->GetEncoderConfig().bitrate_priority); + // Check that the vector of VideoStreams also was propagated correctly. Note + // that this is testing the behavior of the FakeVideoSendStream, which mimics + // the calls to CreateEncoderStreams to get the VideoStreams. + EXPECT_EQ(absl::optional<double>(new_bitrate_priority), + video_send_stream->GetVideoStreams()[0].bitrate_priority); +} + +// Tests that the RTCRtpEncodingParameters.bitrate_priority is set for the +// VideoChannel and the value propogates to the video encoder with all simulcast +// streams. +TEST_F(WebRtcVideoChannelTest, SetRtpSendParametersPrioritySimulcastStreams) { + // Create the stream params with multiple ssrcs for simulcast. + const size_t kNumSimulcastStreams = 3; + std::vector<uint32_t> ssrcs = MAKE_VECTOR(kSsrcs3); + StreamParams stream_params = CreateSimStreamParams("cname", ssrcs); + AddSendStream(stream_params); + uint32_t primary_ssrc = stream_params.first_ssrc(); + + // Using the FrameForwarder, we manually send a full size + // frame. This creates multiple VideoStreams for all simulcast layers when + // reconfiguring, and allows us to test this behavior. + webrtc::test::FrameForwarder frame_forwarder; + VideoOptions options; + EXPECT_TRUE(channel_->SetVideoSend(primary_ssrc, &options, &frame_forwarder)); + channel_->SetSend(true); + frame_forwarder.IncomingCapturedFrame(frame_source_.GetFrame( + 1920, 1080, webrtc::VideoRotation::kVideoRotation_0, + rtc::kNumMicrosecsPerSec / 30)); + + // Get and set the rtp encoding parameters. + webrtc::RtpParameters parameters = + channel_->GetRtpSendParameters(primary_ssrc); + EXPECT_EQ(kNumSimulcastStreams, parameters.encodings.size()); + EXPECT_EQ(webrtc::kDefaultBitratePriority, + parameters.encodings[0].bitrate_priority); + // Change the value and set it on the VideoChannel. + double new_bitrate_priority = 2.0; + parameters.encodings[0].bitrate_priority = new_bitrate_priority; + EXPECT_TRUE(channel_->SetRtpSendParameters(primary_ssrc, parameters).ok()); + + // Verify that the encoding parameters priority is set on the VideoChannel. + parameters = channel_->GetRtpSendParameters(primary_ssrc); + EXPECT_EQ(kNumSimulcastStreams, parameters.encodings.size()); + EXPECT_EQ(new_bitrate_priority, parameters.encodings[0].bitrate_priority); + + // Verify that the new value propagated down to the encoder. + std::vector<FakeVideoSendStream*> video_send_streams = + fake_call_->GetVideoSendStreams(); + EXPECT_EQ(1UL, video_send_streams.size()); + FakeVideoSendStream* video_send_stream = video_send_streams.front(); + // Check that the WebRtcVideoSendStream updated the VideoEncoderConfig + // appropriately. + EXPECT_EQ(kNumSimulcastStreams, + video_send_stream->GetEncoderConfig().number_of_streams); + EXPECT_EQ(new_bitrate_priority, + video_send_stream->GetEncoderConfig().bitrate_priority); + // Check that the vector of VideoStreams also propagated correctly. The + // FakeVideoSendStream calls CreateEncoderStreams, and we are testing that + // these are created appropriately for the simulcast case. + EXPECT_EQ(kNumSimulcastStreams, video_send_stream->GetVideoStreams().size()); + EXPECT_EQ(absl::optional<double>(new_bitrate_priority), + video_send_stream->GetVideoStreams()[0].bitrate_priority); + // Since we are only setting bitrate priority per-sender, the other + // VideoStreams should have a bitrate priority of 0. + EXPECT_EQ(absl::nullopt, + video_send_stream->GetVideoStreams()[1].bitrate_priority); + EXPECT_EQ(absl::nullopt, + video_send_stream->GetVideoStreams()[2].bitrate_priority); + EXPECT_TRUE(channel_->SetVideoSend(primary_ssrc, nullptr, nullptr)); +} + +TEST_F(WebRtcVideoChannelTest, + GetAndSetRtpSendParametersScaleResolutionDownByVP8) { + VideoSendParameters parameters; + parameters.codecs.push_back(VideoCodec(kVp8CodecName)); + ASSERT_TRUE(channel_->SetSendParameters(parameters)); + FakeVideoSendStream* stream = SetUpSimulcast(true, false); + + webrtc::test::FrameForwarder frame_forwarder; + FakeFrameSource frame_source(1280, 720, rtc::kNumMicrosecsPerSec / 30); + + VideoOptions options; + EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, &options, &frame_forwarder)); + channel_->SetSend(true); + + // Try layers in natural order (smallest to largest). + { + auto rtp_parameters = channel_->GetRtpSendParameters(last_ssrc_); + ASSERT_EQ(3u, rtp_parameters.encodings.size()); + rtp_parameters.encodings[0].scale_resolution_down_by = 4.0; + rtp_parameters.encodings[1].scale_resolution_down_by = 2.0; + rtp_parameters.encodings[2].scale_resolution_down_by = 1.0; + auto result = channel_->SetRtpSendParameters(last_ssrc_, rtp_parameters); + ASSERT_TRUE(result.ok()); + + frame_forwarder.IncomingCapturedFrame(frame_source.GetFrame()); + + std::vector<webrtc::VideoStream> video_streams = stream->GetVideoStreams(); + ASSERT_EQ(3u, video_streams.size()); + EXPECT_EQ(320u, video_streams[0].width); + EXPECT_EQ(180u, video_streams[0].height); + EXPECT_EQ(640u, video_streams[1].width); + EXPECT_EQ(360u, video_streams[1].height); + EXPECT_EQ(1280u, video_streams[2].width); + EXPECT_EQ(720u, video_streams[2].height); + } + + // Try layers in reverse natural order (largest to smallest). + { + auto rtp_parameters = channel_->GetRtpSendParameters(last_ssrc_); + ASSERT_EQ(3u, rtp_parameters.encodings.size()); + rtp_parameters.encodings[0].scale_resolution_down_by = 1.0; + rtp_parameters.encodings[1].scale_resolution_down_by = 2.0; + rtp_parameters.encodings[2].scale_resolution_down_by = 4.0; + auto result = channel_->SetRtpSendParameters(last_ssrc_, rtp_parameters); + ASSERT_TRUE(result.ok()); + + frame_forwarder.IncomingCapturedFrame(frame_source.GetFrame()); + + std::vector<webrtc::VideoStream> video_streams = stream->GetVideoStreams(); + ASSERT_EQ(3u, video_streams.size()); + EXPECT_EQ(1280u, video_streams[0].width); + EXPECT_EQ(720u, video_streams[0].height); + EXPECT_EQ(640u, video_streams[1].width); + EXPECT_EQ(360u, video_streams[1].height); + EXPECT_EQ(320u, video_streams[2].width); + EXPECT_EQ(180u, video_streams[2].height); + } + + // Try layers in mixed order. + { + auto rtp_parameters = channel_->GetRtpSendParameters(last_ssrc_); + ASSERT_EQ(3u, rtp_parameters.encodings.size()); + rtp_parameters.encodings[0].scale_resolution_down_by = 10.0; + rtp_parameters.encodings[1].scale_resolution_down_by = 2.0; + rtp_parameters.encodings[2].scale_resolution_down_by = 4.0; + auto result = channel_->SetRtpSendParameters(last_ssrc_, rtp_parameters); + ASSERT_TRUE(result.ok()); + + frame_forwarder.IncomingCapturedFrame(frame_source.GetFrame()); + + std::vector<webrtc::VideoStream> video_streams = stream->GetVideoStreams(); + ASSERT_EQ(3u, video_streams.size()); + EXPECT_EQ(128u, video_streams[0].width); + EXPECT_EQ(72u, video_streams[0].height); + EXPECT_EQ(640u, video_streams[1].width); + EXPECT_EQ(360u, video_streams[1].height); + EXPECT_EQ(320u, video_streams[2].width); + EXPECT_EQ(180u, video_streams[2].height); + } + + // Try with a missing scale setting, defaults to 1.0 if any other is set. + { + auto rtp_parameters = channel_->GetRtpSendParameters(last_ssrc_); + ASSERT_EQ(3u, rtp_parameters.encodings.size()); + rtp_parameters.encodings[0].scale_resolution_down_by = 1.0; + rtp_parameters.encodings[1].scale_resolution_down_by.reset(); + rtp_parameters.encodings[2].scale_resolution_down_by = 4.0; + auto result = channel_->SetRtpSendParameters(last_ssrc_, rtp_parameters); + ASSERT_TRUE(result.ok()); + + frame_forwarder.IncomingCapturedFrame(frame_source.GetFrame()); + + std::vector<webrtc::VideoStream> video_streams = stream->GetVideoStreams(); + ASSERT_EQ(3u, video_streams.size()); + EXPECT_EQ(1280u, video_streams[0].width); + EXPECT_EQ(720u, video_streams[0].height); + EXPECT_EQ(1280u, video_streams[1].width); + EXPECT_EQ(720u, video_streams[1].height); + EXPECT_EQ(320u, video_streams[2].width); + EXPECT_EQ(180u, video_streams[2].height); + } + + EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, nullptr, nullptr)); +} + +TEST_F(WebRtcVideoChannelTest, + GetAndSetRtpSendParametersScaleResolutionDownByVP8WithOddResolution) { + // Ensure that the top layer has width and height divisible by 2^3, + // so that the bottom layer has width and height divisible by 2. + // TODO(bugs.webrtc.org/8785): Remove this field trial when we fully trust + // the number of simulcast layers set by the app. + webrtc::test::ScopedFieldTrials field_trial( + "WebRTC-NormalizeSimulcastResolution/Enabled-3/"); + + // Set up WebRtcVideoChannel for 3-layer VP8 simulcast. + VideoSendParameters parameters; + parameters.codecs.push_back(VideoCodec(kVp8CodecName)); + ASSERT_TRUE(channel_->SetSendParameters(parameters)); + FakeVideoSendStream* stream = SetUpSimulcast(true, false); + webrtc::test::FrameForwarder frame_forwarder; + EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, /*options=*/nullptr, + &frame_forwarder)); + channel_->SetSend(true); + + // Set |scale_resolution_down_by|'s. + auto rtp_parameters = channel_->GetRtpSendParameters(last_ssrc_); + ASSERT_EQ(rtp_parameters.encodings.size(), 3u); + rtp_parameters.encodings[0].scale_resolution_down_by = 1.0; + rtp_parameters.encodings[1].scale_resolution_down_by = 2.0; + rtp_parameters.encodings[2].scale_resolution_down_by = 4.0; + const auto result = + channel_->SetRtpSendParameters(last_ssrc_, rtp_parameters); + ASSERT_TRUE(result.ok()); + + // Use a capture resolution whose width and height are not divisible by 2^3. + // (See field trial set at the top of the test.) + FakeFrameSource frame_source(2007, 1207, rtc::kNumMicrosecsPerSec / 30); + frame_forwarder.IncomingCapturedFrame(frame_source.GetFrame()); + + // Ensure the scaling is correct. + const auto video_streams = stream->GetVideoStreams(); + ASSERT_EQ(video_streams.size(), 3u); + // Ensure that we round the capture resolution down for the top layer... + EXPECT_EQ(video_streams[0].width, 2000u); + EXPECT_EQ(video_streams[0].height, 1200u); + EXPECT_EQ(video_streams[1].width, 1000u); + EXPECT_EQ(video_streams[1].height, 600u); + // ...and that the bottom layer has a width/height divisible by 2. + EXPECT_EQ(video_streams[2].width, 500u); + EXPECT_EQ(video_streams[2].height, 300u); + + // Tear down. + EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, nullptr, nullptr)); +} + +TEST_F(WebRtcVideoChannelTest, + GetAndSetRtpSendParametersScaleResolutionDownByH264) { + encoder_factory_->AddSupportedVideoCodecType(kH264CodecName); + VideoSendParameters parameters; + parameters.codecs.push_back(VideoCodec(kH264CodecName)); + ASSERT_TRUE(channel_->SetSendParameters(parameters)); + FakeVideoSendStream* stream = SetUpSimulcast(true, false); + + webrtc::test::FrameForwarder frame_forwarder; + FakeFrameSource frame_source(1280, 720, rtc::kNumMicrosecsPerSec / 30); + + VideoOptions options; + EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, &options, &frame_forwarder)); + channel_->SetSend(true); + + // Try layers in natural order (smallest to largest). + { + auto rtp_parameters = channel_->GetRtpSendParameters(last_ssrc_); + ASSERT_EQ(3u, rtp_parameters.encodings.size()); + rtp_parameters.encodings[0].scale_resolution_down_by = 4.0; + rtp_parameters.encodings[1].scale_resolution_down_by = 2.0; + rtp_parameters.encodings[2].scale_resolution_down_by = 1.0; + auto result = channel_->SetRtpSendParameters(last_ssrc_, rtp_parameters); + ASSERT_TRUE(result.ok()); + + frame_forwarder.IncomingCapturedFrame(frame_source.GetFrame()); + + std::vector<webrtc::VideoStream> video_streams = stream->GetVideoStreams(); + ASSERT_EQ(3u, video_streams.size()); + EXPECT_EQ(320u, video_streams[0].width); + EXPECT_EQ(180u, video_streams[0].height); + EXPECT_EQ(640u, video_streams[1].width); + EXPECT_EQ(360u, video_streams[1].height); + EXPECT_EQ(1280u, video_streams[2].width); + EXPECT_EQ(720u, video_streams[2].height); + } + + // Try layers in reverse natural order (largest to smallest). + { + auto rtp_parameters = channel_->GetRtpSendParameters(last_ssrc_); + ASSERT_EQ(3u, rtp_parameters.encodings.size()); + rtp_parameters.encodings[0].scale_resolution_down_by = 1.0; + rtp_parameters.encodings[1].scale_resolution_down_by = 2.0; + rtp_parameters.encodings[2].scale_resolution_down_by = 4.0; + auto result = channel_->SetRtpSendParameters(last_ssrc_, rtp_parameters); + ASSERT_TRUE(result.ok()); + + frame_forwarder.IncomingCapturedFrame(frame_source.GetFrame()); + + std::vector<webrtc::VideoStream> video_streams = stream->GetVideoStreams(); + ASSERT_EQ(3u, video_streams.size()); + EXPECT_EQ(1280u, video_streams[0].width); + EXPECT_EQ(720u, video_streams[0].height); + EXPECT_EQ(640u, video_streams[1].width); + EXPECT_EQ(360u, video_streams[1].height); + EXPECT_EQ(320u, video_streams[2].width); + EXPECT_EQ(180u, video_streams[2].height); + } + + // Try layers in mixed order. + { + auto rtp_parameters = channel_->GetRtpSendParameters(last_ssrc_); + ASSERT_EQ(3u, rtp_parameters.encodings.size()); + rtp_parameters.encodings[0].scale_resolution_down_by = 10.0; + rtp_parameters.encodings[1].scale_resolution_down_by = 2.0; + rtp_parameters.encodings[2].scale_resolution_down_by = 4.0; + auto result = channel_->SetRtpSendParameters(last_ssrc_, rtp_parameters); + ASSERT_TRUE(result.ok()); + + frame_forwarder.IncomingCapturedFrame(frame_source.GetFrame()); + + std::vector<webrtc::VideoStream> video_streams = stream->GetVideoStreams(); + ASSERT_EQ(3u, video_streams.size()); + EXPECT_EQ(128u, video_streams[0].width); + EXPECT_EQ(72u, video_streams[0].height); + EXPECT_EQ(640u, video_streams[1].width); + EXPECT_EQ(360u, video_streams[1].height); + EXPECT_EQ(320u, video_streams[2].width); + EXPECT_EQ(180u, video_streams[2].height); + } + + // Try with a missing scale setting, defaults to 1.0 if any other is set. + { + auto rtp_parameters = channel_->GetRtpSendParameters(last_ssrc_); + ASSERT_EQ(3u, rtp_parameters.encodings.size()); + rtp_parameters.encodings[0].scale_resolution_down_by = 1.0; + rtp_parameters.encodings[1].scale_resolution_down_by.reset(); + rtp_parameters.encodings[2].scale_resolution_down_by = 4.0; + auto result = channel_->SetRtpSendParameters(last_ssrc_, rtp_parameters); + ASSERT_TRUE(result.ok()); + + frame_forwarder.IncomingCapturedFrame(frame_source.GetFrame()); + + std::vector<webrtc::VideoStream> video_streams = stream->GetVideoStreams(); + ASSERT_EQ(3u, video_streams.size()); + EXPECT_EQ(1280u, video_streams[0].width); + EXPECT_EQ(720u, video_streams[0].height); + EXPECT_EQ(1280u, video_streams[1].width); + EXPECT_EQ(720u, video_streams[1].height); + EXPECT_EQ(320u, video_streams[2].width); + EXPECT_EQ(180u, video_streams[2].height); + } + EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, nullptr, nullptr)); +} + +TEST_F(WebRtcVideoChannelTest, + GetAndSetRtpSendParametersScaleResolutionDownByH264WithOddResolution) { + // Ensure that the top layer has width and height divisible by 2^3, + // so that the bottom layer has width and height divisible by 2. + // TODO(bugs.webrtc.org/8785): Remove this field trial when we fully trust + // the number of simulcast layers set by the app. + webrtc::test::ScopedFieldTrials field_trial( + "WebRTC-NormalizeSimulcastResolution/Enabled-3/"); + + // Set up WebRtcVideoChannel for 3-layer H264 simulcast. + encoder_factory_->AddSupportedVideoCodecType(kH264CodecName); + VideoSendParameters parameters; + parameters.codecs.push_back(VideoCodec(kH264CodecName)); + ASSERT_TRUE(channel_->SetSendParameters(parameters)); + FakeVideoSendStream* stream = SetUpSimulcast(true, false); + webrtc::test::FrameForwarder frame_forwarder; + EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, /*options=*/nullptr, + &frame_forwarder)); + channel_->SetSend(true); + + // Set |scale_resolution_down_by|'s. + auto rtp_parameters = channel_->GetRtpSendParameters(last_ssrc_); + ASSERT_EQ(rtp_parameters.encodings.size(), 3u); + rtp_parameters.encodings[0].scale_resolution_down_by = 1.0; + rtp_parameters.encodings[1].scale_resolution_down_by = 2.0; + rtp_parameters.encodings[2].scale_resolution_down_by = 4.0; + const auto result = + channel_->SetRtpSendParameters(last_ssrc_, rtp_parameters); + ASSERT_TRUE(result.ok()); + + // Use a capture resolution whose width and height are not divisible by 2^3. + // (See field trial set at the top of the test.) + FakeFrameSource frame_source(2007, 1207, rtc::kNumMicrosecsPerSec / 30); + frame_forwarder.IncomingCapturedFrame(frame_source.GetFrame()); + + // Ensure the scaling is correct. + const auto video_streams = stream->GetVideoStreams(); + ASSERT_EQ(video_streams.size(), 3u); + // Ensure that we round the capture resolution down for the top layer... + EXPECT_EQ(video_streams[0].width, 2000u); + EXPECT_EQ(video_streams[0].height, 1200u); + EXPECT_EQ(video_streams[1].width, 1000u); + EXPECT_EQ(video_streams[1].height, 600u); + // ...and that the bottom layer has a width/height divisible by 2. + EXPECT_EQ(video_streams[2].width, 500u); + EXPECT_EQ(video_streams[2].height, 300u); + + // Tear down. + EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, nullptr, nullptr)); +} + +TEST_F(WebRtcVideoChannelTest, GetAndSetRtpSendParametersMaxFramerate) { + const size_t kNumSimulcastStreams = 3; + SetUpSimulcast(true, false); + + // Get and set the rtp encoding parameters. + webrtc::RtpParameters parameters = channel_->GetRtpSendParameters(last_ssrc_); + EXPECT_EQ(kNumSimulcastStreams, parameters.encodings.size()); + for (const auto& encoding : parameters.encodings) { + EXPECT_FALSE(encoding.max_framerate); + } + + // Change the value and set it on the VideoChannel. + parameters.encodings[0].max_framerate = 10; + parameters.encodings[1].max_framerate = 20; + parameters.encodings[2].max_framerate = 25; + EXPECT_TRUE(channel_->SetRtpSendParameters(last_ssrc_, parameters).ok()); + + // Verify that the bitrates are set on the VideoChannel. + parameters = channel_->GetRtpSendParameters(last_ssrc_); + EXPECT_EQ(kNumSimulcastStreams, parameters.encodings.size()); + EXPECT_EQ(10, parameters.encodings[0].max_framerate); + EXPECT_EQ(20, parameters.encodings[1].max_framerate); + EXPECT_EQ(25, parameters.encodings[2].max_framerate); +} + +TEST_F(WebRtcVideoChannelTest, + SetRtpSendParametersNumTemporalLayersFailsForInvalidRange) { + const size_t kNumSimulcastStreams = 3; + SetUpSimulcast(true, false); + + // Get and set the rtp encoding parameters. + webrtc::RtpParameters parameters = channel_->GetRtpSendParameters(last_ssrc_); + EXPECT_EQ(kNumSimulcastStreams, parameters.encodings.size()); + + // Num temporal layers should be in the range [1, kMaxTemporalStreams]. + parameters.encodings[0].num_temporal_layers = 0; + EXPECT_EQ(webrtc::RTCErrorType::INVALID_RANGE, + channel_->SetRtpSendParameters(last_ssrc_, parameters).type()); + parameters.encodings[0].num_temporal_layers = webrtc::kMaxTemporalStreams + 1; + EXPECT_EQ(webrtc::RTCErrorType::INVALID_RANGE, + channel_->SetRtpSendParameters(last_ssrc_, parameters).type()); +} + +TEST_F(WebRtcVideoChannelTest, + SetRtpSendParametersNumTemporalLayersFailsForInvalidModification) { + const size_t kNumSimulcastStreams = 3; + SetUpSimulcast(true, false); + + // Get and set the rtp encoding parameters. + webrtc::RtpParameters parameters = channel_->GetRtpSendParameters(last_ssrc_); + EXPECT_EQ(kNumSimulcastStreams, parameters.encodings.size()); + + // No/all layers should be set. + parameters.encodings[0].num_temporal_layers = 1; + EXPECT_EQ(webrtc::RTCErrorType::INVALID_MODIFICATION, + channel_->SetRtpSendParameters(last_ssrc_, parameters).type()); + + // Different values not supported. + parameters.encodings[0].num_temporal_layers = 1; + parameters.encodings[1].num_temporal_layers = 2; + parameters.encodings[2].num_temporal_layers = 2; + EXPECT_EQ(webrtc::RTCErrorType::INVALID_MODIFICATION, + channel_->SetRtpSendParameters(last_ssrc_, parameters).type()); +} + +TEST_F(WebRtcVideoChannelTest, GetAndSetRtpSendParametersNumTemporalLayers) { + const size_t kNumSimulcastStreams = 3; + SetUpSimulcast(true, false); + + // Get and set the rtp encoding parameters. + webrtc::RtpParameters parameters = channel_->GetRtpSendParameters(last_ssrc_); + EXPECT_EQ(kNumSimulcastStreams, parameters.encodings.size()); + for (const auto& encoding : parameters.encodings) + EXPECT_FALSE(encoding.num_temporal_layers); + + // Change the value and set it on the VideoChannel. + parameters.encodings[0].num_temporal_layers = 3; + parameters.encodings[1].num_temporal_layers = 3; + parameters.encodings[2].num_temporal_layers = 3; + EXPECT_TRUE(channel_->SetRtpSendParameters(last_ssrc_, parameters).ok()); + + // Verify that the number of temporal layers are set on the VideoChannel. + parameters = channel_->GetRtpSendParameters(last_ssrc_); + EXPECT_EQ(kNumSimulcastStreams, parameters.encodings.size()); + EXPECT_EQ(3, parameters.encodings[0].num_temporal_layers); + EXPECT_EQ(3, parameters.encodings[1].num_temporal_layers); + EXPECT_EQ(3, parameters.encodings[2].num_temporal_layers); +} + +TEST_F(WebRtcVideoChannelTest, NumTemporalLayersPropagatedToEncoder) { + const size_t kNumSimulcastStreams = 3; + FakeVideoSendStream* stream = SetUpSimulcast(true, false); + + // Send a full size frame so all simulcast layers are used when reconfiguring. + webrtc::test::FrameForwarder frame_forwarder; + VideoOptions options; + EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, &options, &frame_forwarder)); + channel_->SetSend(true); + frame_forwarder.IncomingCapturedFrame(frame_source_.GetFrame()); + + // Get and set the rtp encoding parameters. + // Change the value and set it on the VideoChannel. + webrtc::RtpParameters parameters = channel_->GetRtpSendParameters(last_ssrc_); + EXPECT_EQ(kNumSimulcastStreams, parameters.encodings.size()); + parameters.encodings[0].num_temporal_layers = 2; + parameters.encodings[1].num_temporal_layers = 2; + parameters.encodings[2].num_temporal_layers = 2; + EXPECT_TRUE(channel_->SetRtpSendParameters(last_ssrc_, parameters).ok()); + + // Verify that the new value is propagated down to the encoder. + // Check that WebRtcVideoSendStream updates VideoEncoderConfig correctly. + EXPECT_EQ(2, stream->num_encoder_reconfigurations()); + webrtc::VideoEncoderConfig encoder_config = stream->GetEncoderConfig().Copy(); + EXPECT_EQ(kNumSimulcastStreams, encoder_config.number_of_streams); + EXPECT_EQ(kNumSimulcastStreams, encoder_config.simulcast_layers.size()); + EXPECT_EQ(2UL, encoder_config.simulcast_layers[0].num_temporal_layers); + EXPECT_EQ(2UL, encoder_config.simulcast_layers[1].num_temporal_layers); + EXPECT_EQ(2UL, encoder_config.simulcast_layers[2].num_temporal_layers); + + // FakeVideoSendStream calls CreateEncoderStreams, test that the vector of + // VideoStreams are created appropriately for the simulcast case. + EXPECT_EQ(kNumSimulcastStreams, stream->GetVideoStreams().size()); + EXPECT_EQ(2UL, stream->GetVideoStreams()[0].num_temporal_layers); + EXPECT_EQ(2UL, stream->GetVideoStreams()[1].num_temporal_layers); + EXPECT_EQ(2UL, stream->GetVideoStreams()[2].num_temporal_layers); + + // No parameter changed, encoder should not be reconfigured. + EXPECT_TRUE(channel_->SetRtpSendParameters(last_ssrc_, parameters).ok()); + EXPECT_EQ(2, stream->num_encoder_reconfigurations()); + + EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, nullptr, nullptr)); +} + +TEST_F(WebRtcVideoChannelTest, + DefaultValuePropagatedToEncoderForUnsetNumTemporalLayers) { + const size_t kDefaultNumTemporalLayers = 3; + const size_t kNumSimulcastStreams = 3; + FakeVideoSendStream* stream = SetUpSimulcast(true, false); + + // Send a full size frame so all simulcast layers are used when reconfiguring. + webrtc::test::FrameForwarder frame_forwarder; + VideoOptions options; + EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, &options, &frame_forwarder)); + channel_->SetSend(true); + frame_forwarder.IncomingCapturedFrame(frame_source_.GetFrame()); + + // Change rtp encoding parameters, num_temporal_layers not changed. + webrtc::RtpParameters parameters = channel_->GetRtpSendParameters(last_ssrc_); + EXPECT_EQ(kNumSimulcastStreams, parameters.encodings.size()); + parameters.encodings[0].min_bitrate_bps = 33000; + EXPECT_TRUE(channel_->SetRtpSendParameters(last_ssrc_, parameters).ok()); + + // Verify that no value is propagated down to the encoder. + webrtc::VideoEncoderConfig encoder_config = stream->GetEncoderConfig().Copy(); + EXPECT_EQ(kNumSimulcastStreams, encoder_config.number_of_streams); + EXPECT_EQ(kNumSimulcastStreams, encoder_config.simulcast_layers.size()); + EXPECT_FALSE(encoder_config.simulcast_layers[0].num_temporal_layers); + EXPECT_FALSE(encoder_config.simulcast_layers[1].num_temporal_layers); + EXPECT_FALSE(encoder_config.simulcast_layers[2].num_temporal_layers); + + // FakeVideoSendStream calls CreateEncoderStreams, test that the vector of + // VideoStreams are created appropriately for the simulcast case. + EXPECT_EQ(kNumSimulcastStreams, stream->GetVideoStreams().size()); + EXPECT_EQ(kDefaultNumTemporalLayers, + stream->GetVideoStreams()[0].num_temporal_layers); + EXPECT_EQ(kDefaultNumTemporalLayers, + stream->GetVideoStreams()[1].num_temporal_layers); + EXPECT_EQ(kDefaultNumTemporalLayers, + stream->GetVideoStreams()[2].num_temporal_layers); + + EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, nullptr, nullptr)); +} + +TEST_F(WebRtcVideoChannelTest, + DefaultValuePropagatedToEncoderForUnsetFramerate) { + const size_t kNumSimulcastStreams = 3; + const std::vector<webrtc::VideoStream> kDefault = GetSimulcastBitrates720p(); + FakeVideoSendStream* stream = SetUpSimulcast(true, false); + + // Send a full size frame so all simulcast layers are used when reconfiguring. + webrtc::test::FrameForwarder frame_forwarder; + VideoOptions options; + EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, &options, &frame_forwarder)); + channel_->SetSend(true); + frame_forwarder.IncomingCapturedFrame(frame_source_.GetFrame()); + + // Get and set the rtp encoding parameters. + // Change the value and set it on the VideoChannel. + webrtc::RtpParameters parameters = channel_->GetRtpSendParameters(last_ssrc_); + EXPECT_EQ(kNumSimulcastStreams, parameters.encodings.size()); + parameters.encodings[0].max_framerate = 15; + parameters.encodings[2].max_framerate = 20; + EXPECT_TRUE(channel_->SetRtpSendParameters(last_ssrc_, parameters).ok()); + + // Verify that the new value propagated down to the encoder. + // Check that WebRtcVideoSendStream updates VideoEncoderConfig correctly. + webrtc::VideoEncoderConfig encoder_config = stream->GetEncoderConfig().Copy(); + EXPECT_EQ(kNumSimulcastStreams, encoder_config.number_of_streams); + EXPECT_EQ(kNumSimulcastStreams, encoder_config.simulcast_layers.size()); + EXPECT_EQ(15, encoder_config.simulcast_layers[0].max_framerate); + EXPECT_EQ(-1, encoder_config.simulcast_layers[1].max_framerate); + EXPECT_EQ(20, encoder_config.simulcast_layers[2].max_framerate); + + // FakeVideoSendStream calls CreateEncoderStreams, test that the vector of + // VideoStreams are created appropriately for the simulcast case. + // The maximum |max_framerate| is used, kDefaultVideoMaxFramerate: 60. + EXPECT_EQ(kNumSimulcastStreams, stream->GetVideoStreams().size()); + EXPECT_EQ(15, stream->GetVideoStreams()[0].max_framerate); + EXPECT_EQ(kDefaultVideoMaxFramerate, + stream->GetVideoStreams()[1].max_framerate); + EXPECT_EQ(20, stream->GetVideoStreams()[2].max_framerate); + + EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, nullptr, nullptr)); +} + +TEST_F(WebRtcVideoChannelTest, GetAndSetRtpSendParametersMinAndMaxBitrate) { + const size_t kNumSimulcastStreams = 3; + SetUpSimulcast(true, false); + + // Get and set the rtp encoding parameters. + webrtc::RtpParameters parameters = channel_->GetRtpSendParameters(last_ssrc_); + EXPECT_EQ(kNumSimulcastStreams, parameters.encodings.size()); + for (const auto& encoding : parameters.encodings) { + EXPECT_FALSE(encoding.min_bitrate_bps); + EXPECT_FALSE(encoding.max_bitrate_bps); + } + + // Change the value and set it on the VideoChannel. + parameters.encodings[0].min_bitrate_bps = 100000; + parameters.encodings[0].max_bitrate_bps = 200000; + parameters.encodings[1].min_bitrate_bps = 300000; + parameters.encodings[1].max_bitrate_bps = 400000; + parameters.encodings[2].min_bitrate_bps = 500000; + parameters.encodings[2].max_bitrate_bps = 600000; + EXPECT_TRUE(channel_->SetRtpSendParameters(last_ssrc_, parameters).ok()); + + // Verify that the bitrates are set on the VideoChannel. + parameters = channel_->GetRtpSendParameters(last_ssrc_); + EXPECT_EQ(kNumSimulcastStreams, parameters.encodings.size()); + EXPECT_EQ(100000, parameters.encodings[0].min_bitrate_bps); + EXPECT_EQ(200000, parameters.encodings[0].max_bitrate_bps); + EXPECT_EQ(300000, parameters.encodings[1].min_bitrate_bps); + EXPECT_EQ(400000, parameters.encodings[1].max_bitrate_bps); + EXPECT_EQ(500000, parameters.encodings[2].min_bitrate_bps); + EXPECT_EQ(600000, parameters.encodings[2].max_bitrate_bps); +} + +TEST_F(WebRtcVideoChannelTest, SetRtpSendParametersFailsWithIncorrectBitrate) { + const size_t kNumSimulcastStreams = 3; + SetUpSimulcast(true, false); + + // Get and set the rtp encoding parameters. + webrtc::RtpParameters parameters = channel_->GetRtpSendParameters(last_ssrc_); + EXPECT_EQ(kNumSimulcastStreams, parameters.encodings.size()); + + // Max bitrate lower than min bitrate should fail. + parameters.encodings[2].min_bitrate_bps = 100000; + parameters.encodings[2].max_bitrate_bps = 100000 - 1; + EXPECT_EQ(webrtc::RTCErrorType::INVALID_RANGE, + channel_->SetRtpSendParameters(last_ssrc_, parameters).type()); +} + +// Test that min and max bitrate values set via RtpParameters are correctly +// propagated to the underlying encoder, and that the target is set to 3/4 of +// the maximum (3/4 was chosen because it's similar to the simulcast defaults +// that are used if no min/max are specified). +TEST_F(WebRtcVideoChannelTest, MinAndMaxSimulcastBitratePropagatedToEncoder) { + const size_t kNumSimulcastStreams = 3; + FakeVideoSendStream* stream = SetUpSimulcast(true, false); + + // Send a full size frame so all simulcast layers are used when reconfiguring. + webrtc::test::FrameForwarder frame_forwarder; + VideoOptions options; + EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, &options, &frame_forwarder)); + channel_->SetSend(true); + frame_forwarder.IncomingCapturedFrame(frame_source_.GetFrame()); + + // Get and set the rtp encoding parameters. + // Change the value and set it on the VideoChannel. + webrtc::RtpParameters parameters = channel_->GetRtpSendParameters(last_ssrc_); + EXPECT_EQ(kNumSimulcastStreams, parameters.encodings.size()); + parameters.encodings[0].min_bitrate_bps = 100000; + parameters.encodings[0].max_bitrate_bps = 200000; + parameters.encodings[1].min_bitrate_bps = 300000; + parameters.encodings[1].max_bitrate_bps = 400000; + parameters.encodings[2].min_bitrate_bps = 500000; + parameters.encodings[2].max_bitrate_bps = 600000; + EXPECT_TRUE(channel_->SetRtpSendParameters(last_ssrc_, parameters).ok()); + + // Verify that the new value propagated down to the encoder. + // Check that WebRtcVideoSendStream updates VideoEncoderConfig correctly. + EXPECT_EQ(2, stream->num_encoder_reconfigurations()); + webrtc::VideoEncoderConfig encoder_config = stream->GetEncoderConfig().Copy(); + EXPECT_EQ(kNumSimulcastStreams, encoder_config.number_of_streams); + EXPECT_EQ(kNumSimulcastStreams, encoder_config.simulcast_layers.size()); + EXPECT_EQ(100000, encoder_config.simulcast_layers[0].min_bitrate_bps); + EXPECT_EQ(200000, encoder_config.simulcast_layers[0].max_bitrate_bps); + EXPECT_EQ(300000, encoder_config.simulcast_layers[1].min_bitrate_bps); + EXPECT_EQ(400000, encoder_config.simulcast_layers[1].max_bitrate_bps); + EXPECT_EQ(500000, encoder_config.simulcast_layers[2].min_bitrate_bps); + EXPECT_EQ(600000, encoder_config.simulcast_layers[2].max_bitrate_bps); + + // FakeVideoSendStream calls CreateEncoderStreams, test that the vector of + // VideoStreams are created appropriately for the simulcast case. + EXPECT_EQ(kNumSimulcastStreams, stream->GetVideoStreams().size()); + // Target bitrate: 200000 * 3 / 4 = 150000. + EXPECT_EQ(100000, stream->GetVideoStreams()[0].min_bitrate_bps); + EXPECT_EQ(150000, stream->GetVideoStreams()[0].target_bitrate_bps); + EXPECT_EQ(200000, stream->GetVideoStreams()[0].max_bitrate_bps); + // Target bitrate: 400000 * 3 / 4 = 300000. + EXPECT_EQ(300000, stream->GetVideoStreams()[1].min_bitrate_bps); + EXPECT_EQ(300000, stream->GetVideoStreams()[1].target_bitrate_bps); + EXPECT_EQ(400000, stream->GetVideoStreams()[1].max_bitrate_bps); + // Target bitrate: 600000 * 3 / 4 = 450000, less than min -> max. + EXPECT_EQ(500000, stream->GetVideoStreams()[2].min_bitrate_bps); + EXPECT_EQ(600000, stream->GetVideoStreams()[2].target_bitrate_bps); + EXPECT_EQ(600000, stream->GetVideoStreams()[2].max_bitrate_bps); + + // No parameter changed, encoder should not be reconfigured. + EXPECT_TRUE(channel_->SetRtpSendParameters(last_ssrc_, parameters).ok()); + EXPECT_EQ(2, stream->num_encoder_reconfigurations()); + + EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, nullptr, nullptr)); +} + +// Test to only specify the min or max bitrate value for a layer via +// RtpParameters. The unspecified min/max and target value should be set to the +// simulcast default that is used if no min/max are specified. +TEST_F(WebRtcVideoChannelTest, MinOrMaxSimulcastBitratePropagatedToEncoder) { + const size_t kNumSimulcastStreams = 3; + const std::vector<webrtc::VideoStream> kDefault = GetSimulcastBitrates720p(); + FakeVideoSendStream* stream = SetUpSimulcast(true, false); + + // Send a full size frame so all simulcast layers are used when reconfiguring. + webrtc::test::FrameForwarder frame_forwarder; + VideoOptions options; + EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, &options, &frame_forwarder)); + channel_->SetSend(true); + frame_forwarder.IncomingCapturedFrame(frame_source_.GetFrame()); + + // Get and set the rtp encoding parameters. + webrtc::RtpParameters parameters = channel_->GetRtpSendParameters(last_ssrc_); + EXPECT_EQ(kNumSimulcastStreams, parameters.encodings.size()); + + // Change the value and set it on the VideoChannel. + // Layer 0: only configure min bitrate. + const int kMinBpsLayer0 = kDefault[0].min_bitrate_bps + 1; + parameters.encodings[0].min_bitrate_bps = kMinBpsLayer0; + // Layer 1: only configure max bitrate. + const int kMaxBpsLayer1 = kDefault[1].max_bitrate_bps - 1; + parameters.encodings[1].max_bitrate_bps = kMaxBpsLayer1; + EXPECT_TRUE(channel_->SetRtpSendParameters(last_ssrc_, parameters).ok()); + + // Verify that the new value propagated down to the encoder. + // Check that WebRtcVideoSendStream updates VideoEncoderConfig correctly. + webrtc::VideoEncoderConfig encoder_config = stream->GetEncoderConfig().Copy(); + EXPECT_EQ(kNumSimulcastStreams, encoder_config.number_of_streams); + EXPECT_EQ(kNumSimulcastStreams, encoder_config.simulcast_layers.size()); + EXPECT_EQ(kMinBpsLayer0, encoder_config.simulcast_layers[0].min_bitrate_bps); + EXPECT_EQ(-1, encoder_config.simulcast_layers[0].max_bitrate_bps); + EXPECT_EQ(-1, encoder_config.simulcast_layers[1].min_bitrate_bps); + EXPECT_EQ(kMaxBpsLayer1, encoder_config.simulcast_layers[1].max_bitrate_bps); + EXPECT_EQ(-1, encoder_config.simulcast_layers[2].min_bitrate_bps); + EXPECT_EQ(-1, encoder_config.simulcast_layers[2].max_bitrate_bps); + + // FakeVideoSendStream calls CreateEncoderStreams, test that the vector of + // VideoStreams are created appropriately for the simulcast case. + EXPECT_EQ(kNumSimulcastStreams, stream->GetVideoStreams().size()); + // Layer 0: min configured bitrate should overwrite min default. + EXPECT_EQ(kMinBpsLayer0, stream->GetVideoStreams()[0].min_bitrate_bps); + EXPECT_EQ(kDefault[0].target_bitrate_bps, + stream->GetVideoStreams()[0].target_bitrate_bps); + EXPECT_EQ(kDefault[0].max_bitrate_bps, + stream->GetVideoStreams()[0].max_bitrate_bps); + // Layer 1: max configured bitrate should overwrite max default. + EXPECT_EQ(kDefault[1].min_bitrate_bps, + stream->GetVideoStreams()[1].min_bitrate_bps); + EXPECT_EQ(kDefault[1].target_bitrate_bps, + stream->GetVideoStreams()[1].target_bitrate_bps); + EXPECT_EQ(kMaxBpsLayer1, stream->GetVideoStreams()[1].max_bitrate_bps); + // Layer 2: min and max bitrate not configured, default expected. + EXPECT_EQ(kDefault[2].min_bitrate_bps, + stream->GetVideoStreams()[2].min_bitrate_bps); + EXPECT_EQ(kDefault[2].target_bitrate_bps, + stream->GetVideoStreams()[2].target_bitrate_bps); + EXPECT_EQ(kDefault[2].max_bitrate_bps, + stream->GetVideoStreams()[2].max_bitrate_bps); + + EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, nullptr, nullptr)); +} + +// Test that specifying the min (or max) bitrate value for a layer via +// RtpParameters above (or below) the simulcast default max (or min) adjusts the +// unspecified values accordingly. +TEST_F(WebRtcVideoChannelTest, SetMinAndMaxSimulcastBitrateAboveBelowDefault) { + const size_t kNumSimulcastStreams = 3; + const std::vector<webrtc::VideoStream> kDefault = GetSimulcastBitrates720p(); + FakeVideoSendStream* stream = SetUpSimulcast(true, false); + + // Send a full size frame so all simulcast layers are used when reconfiguring. + webrtc::test::FrameForwarder frame_forwarder; + VideoOptions options; + EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, &options, &frame_forwarder)); + channel_->SetSend(true); + frame_forwarder.IncomingCapturedFrame(frame_source_.GetFrame()); + + // Get and set the rtp encoding parameters. + webrtc::RtpParameters parameters = channel_->GetRtpSendParameters(last_ssrc_); + EXPECT_EQ(kNumSimulcastStreams, parameters.encodings.size()); + + // Change the value and set it on the VideoChannel. + // For layer 0, set the min bitrate above the default max. + const int kMinBpsLayer0 = kDefault[0].max_bitrate_bps + 1; + parameters.encodings[0].min_bitrate_bps = kMinBpsLayer0; + // For layer 1, set the max bitrate below the default min. + const int kMaxBpsLayer1 = kDefault[1].min_bitrate_bps - 1; + parameters.encodings[1].max_bitrate_bps = kMaxBpsLayer1; + EXPECT_TRUE(channel_->SetRtpSendParameters(last_ssrc_, parameters).ok()); + + // Verify that the new value propagated down to the encoder. + // FakeVideoSendStream calls CreateEncoderStreams, test that the vector of + // VideoStreams are created appropriately for the simulcast case. + EXPECT_EQ(kNumSimulcastStreams, stream->GetVideoStreams().size()); + // Layer 0: Min bitrate above default max (target/max should be adjusted). + EXPECT_EQ(kMinBpsLayer0, stream->GetVideoStreams()[0].min_bitrate_bps); + EXPECT_EQ(kMinBpsLayer0, stream->GetVideoStreams()[0].target_bitrate_bps); + EXPECT_EQ(kMinBpsLayer0, stream->GetVideoStreams()[0].max_bitrate_bps); + // Layer 1: Max bitrate below default min (min/target should be adjusted). + EXPECT_EQ(kMaxBpsLayer1, stream->GetVideoStreams()[1].min_bitrate_bps); + EXPECT_EQ(kMaxBpsLayer1, stream->GetVideoStreams()[1].target_bitrate_bps); + EXPECT_EQ(kMaxBpsLayer1, stream->GetVideoStreams()[1].max_bitrate_bps); + // Layer 2: min and max bitrate not configured, default expected. + EXPECT_EQ(kDefault[2].min_bitrate_bps, + stream->GetVideoStreams()[2].min_bitrate_bps); + EXPECT_EQ(kDefault[2].target_bitrate_bps, + stream->GetVideoStreams()[2].target_bitrate_bps); + EXPECT_EQ(kDefault[2].max_bitrate_bps, + stream->GetVideoStreams()[2].max_bitrate_bps); + + EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, nullptr, nullptr)); +} + +TEST_F(WebRtcVideoChannelTest, BandwidthAboveTotalMaxBitrateGivenToMaxLayer) { + const size_t kNumSimulcastStreams = 3; + const std::vector<webrtc::VideoStream> kDefault = GetSimulcastBitrates720p(); + FakeVideoSendStream* stream = SetUpSimulcast(true, false); + + // Send a full size frame so all simulcast layers are used when reconfiguring. + webrtc::test::FrameForwarder frame_forwarder; + VideoOptions options; + EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, &options, &frame_forwarder)); + channel_->SetSend(true); + frame_forwarder.IncomingCapturedFrame(frame_source_.GetFrame()); + + // Set max bitrate for all but the highest layer. + webrtc::RtpParameters parameters = channel_->GetRtpSendParameters(last_ssrc_); + EXPECT_EQ(kNumSimulcastStreams, parameters.encodings.size()); + parameters.encodings[0].max_bitrate_bps = kDefault[0].max_bitrate_bps; + parameters.encodings[1].max_bitrate_bps = kDefault[1].max_bitrate_bps; + EXPECT_TRUE(channel_->SetRtpSendParameters(last_ssrc_, parameters).ok()); + + // Set max bandwidth equal to total max bitrate. + send_parameters_.max_bandwidth_bps = + GetTotalMaxBitrate(stream->GetVideoStreams()).bps(); + ExpectSetMaxBitrate(send_parameters_.max_bandwidth_bps); + ASSERT_TRUE(channel_->SetSendParameters(send_parameters_)); + + // No bitrate above the total max to give to the highest layer. + EXPECT_EQ(kNumSimulcastStreams, stream->GetVideoStreams().size()); + EXPECT_EQ(kDefault[2].max_bitrate_bps, + stream->GetVideoStreams()[2].max_bitrate_bps); + + // Set max bandwidth above the total max bitrate. + send_parameters_.max_bandwidth_bps = + GetTotalMaxBitrate(stream->GetVideoStreams()).bps() + 1; + ExpectSetMaxBitrate(send_parameters_.max_bandwidth_bps); + ASSERT_TRUE(channel_->SetSendParameters(send_parameters_)); + + // The highest layer has no max bitrate set -> the bitrate above the total + // max should be given to the highest layer. + EXPECT_EQ(kNumSimulcastStreams, stream->GetVideoStreams().size()); + EXPECT_EQ(send_parameters_.max_bandwidth_bps, + GetTotalMaxBitrate(stream->GetVideoStreams()).bps()); + EXPECT_EQ(kDefault[2].max_bitrate_bps + 1, + stream->GetVideoStreams()[2].max_bitrate_bps); + + EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, nullptr, nullptr)); +} + +TEST_F(WebRtcVideoChannelTest, + BandwidthAboveTotalMaxBitrateNotGivenToMaxLayerIfMaxBitrateSet) { + const size_t kNumSimulcastStreams = 3; + const std::vector<webrtc::VideoStream> kDefault = GetSimulcastBitrates720p(); + EXPECT_EQ(kNumSimulcastStreams, kDefault.size()); + FakeVideoSendStream* stream = SetUpSimulcast(true, false); + + // Send a full size frame so all simulcast layers are used when reconfiguring. + webrtc::test::FrameForwarder frame_forwarder; + VideoOptions options; + EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, &options, &frame_forwarder)); + channel_->SetSend(true); + frame_forwarder.IncomingCapturedFrame(frame_source_.GetFrame()); + + // Set max bitrate for the highest layer. + webrtc::RtpParameters parameters = channel_->GetRtpSendParameters(last_ssrc_); + EXPECT_EQ(kNumSimulcastStreams, parameters.encodings.size()); + parameters.encodings[2].max_bitrate_bps = kDefault[2].max_bitrate_bps; + EXPECT_TRUE(channel_->SetRtpSendParameters(last_ssrc_, parameters).ok()); + + // Set max bandwidth above the total max bitrate. + send_parameters_.max_bandwidth_bps = + GetTotalMaxBitrate(stream->GetVideoStreams()).bps() + 1; + ExpectSetMaxBitrate(send_parameters_.max_bandwidth_bps); + ASSERT_TRUE(channel_->SetSendParameters(send_parameters_)); + + // The highest layer has the max bitrate set -> the bitrate above the total + // max should not be given to the highest layer. + EXPECT_EQ(kNumSimulcastStreams, stream->GetVideoStreams().size()); + EXPECT_EQ(*parameters.encodings[2].max_bitrate_bps, + stream->GetVideoStreams()[2].max_bitrate_bps); + + EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, nullptr, nullptr)); +} + +// Test that min and max bitrate values set via RtpParameters are correctly +// propagated to the underlying encoder for a single stream. +TEST_F(WebRtcVideoChannelTest, MinAndMaxBitratePropagatedToEncoder) { + FakeVideoSendStream* stream = AddSendStream(); + EXPECT_TRUE(channel_->SetSend(true)); + EXPECT_TRUE(stream->IsSending()); + + // Set min and max bitrate. + webrtc::RtpParameters parameters = channel_->GetRtpSendParameters(last_ssrc_); + EXPECT_EQ(1u, parameters.encodings.size()); + parameters.encodings[0].min_bitrate_bps = 80000; + parameters.encodings[0].max_bitrate_bps = 150000; + EXPECT_TRUE(channel_->SetRtpSendParameters(last_ssrc_, parameters).ok()); + + // Check that WebRtcVideoSendStream updates VideoEncoderConfig correctly. + webrtc::VideoEncoderConfig encoder_config = stream->GetEncoderConfig().Copy(); + EXPECT_EQ(1u, encoder_config.number_of_streams); + EXPECT_EQ(1u, encoder_config.simulcast_layers.size()); + EXPECT_EQ(80000, encoder_config.simulcast_layers[0].min_bitrate_bps); + EXPECT_EQ(150000, encoder_config.simulcast_layers[0].max_bitrate_bps); + + // FakeVideoSendStream calls CreateEncoderStreams, test that the vector of + // VideoStreams are created appropriately. + EXPECT_EQ(1u, stream->GetVideoStreams().size()); + EXPECT_EQ(80000, stream->GetVideoStreams()[0].min_bitrate_bps); + EXPECT_EQ(150000, stream->GetVideoStreams()[0].target_bitrate_bps); + EXPECT_EQ(150000, stream->GetVideoStreams()[0].max_bitrate_bps); +} + +// Test the default min and max bitrate value are correctly propagated to the +// underlying encoder for a single stream (when the values are not set via +// RtpParameters). +TEST_F(WebRtcVideoChannelTest, DefaultMinAndMaxBitratePropagatedToEncoder) { + FakeVideoSendStream* stream = AddSendStream(); + EXPECT_TRUE(channel_->SetSend(true)); + EXPECT_TRUE(stream->IsSending()); + + // Check that WebRtcVideoSendStream updates VideoEncoderConfig correctly. + webrtc::VideoEncoderConfig encoder_config = stream->GetEncoderConfig().Copy(); + EXPECT_EQ(1u, encoder_config.number_of_streams); + EXPECT_EQ(1u, encoder_config.simulcast_layers.size()); + EXPECT_EQ(-1, encoder_config.simulcast_layers[0].min_bitrate_bps); + EXPECT_EQ(-1, encoder_config.simulcast_layers[0].max_bitrate_bps); + + // FakeVideoSendStream calls CreateEncoderStreams, test that the vector of + // VideoStreams are created appropriately. + EXPECT_EQ(1u, stream->GetVideoStreams().size()); + EXPECT_EQ(webrtc::kDefaultMinVideoBitrateBps, + stream->GetVideoStreams()[0].min_bitrate_bps); + EXPECT_GT(stream->GetVideoStreams()[0].max_bitrate_bps, + stream->GetVideoStreams()[0].min_bitrate_bps); + EXPECT_EQ(stream->GetVideoStreams()[0].max_bitrate_bps, + stream->GetVideoStreams()[0].target_bitrate_bps); +} + +// Test that a stream will not be sending if its encoding is made inactive +// through SetRtpSendParameters. +TEST_F(WebRtcVideoChannelTest, SetRtpSendParametersOneEncodingActive) { + FakeVideoSendStream* stream = AddSendStream(); + EXPECT_TRUE(channel_->SetSend(true)); + EXPECT_TRUE(stream->IsSending()); + + // Get current parameters and change "active" to false. + webrtc::RtpParameters parameters = channel_->GetRtpSendParameters(last_ssrc_); + ASSERT_EQ(1u, parameters.encodings.size()); + ASSERT_TRUE(parameters.encodings[0].active); + parameters.encodings[0].active = false; + EXPECT_TRUE(channel_->SetRtpSendParameters(last_ssrc_, parameters).ok()); + EXPECT_FALSE(stream->IsSending()); + + // Now change it back to active and verify we resume sending. + parameters.encodings[0].active = true; + EXPECT_TRUE(channel_->SetRtpSendParameters(last_ssrc_, parameters).ok()); + EXPECT_TRUE(stream->IsSending()); +} + +// Tests that when active is updated for any simulcast layer then the send +// stream's sending state will be updated and it will be reconfigured with the +// new appropriate active simulcast streams. +TEST_F(WebRtcVideoChannelTest, SetRtpSendParametersMultipleEncodingsActive) { + // Create the stream params with multiple ssrcs for simulcast. + const size_t kNumSimulcastStreams = 3; + std::vector<uint32_t> ssrcs = MAKE_VECTOR(kSsrcs3); + StreamParams stream_params = CreateSimStreamParams("cname", ssrcs); + FakeVideoSendStream* fake_video_send_stream = AddSendStream(stream_params); + uint32_t primary_ssrc = stream_params.first_ssrc(); + + // Using the FrameForwarder, we manually send a full size + // frame. This allows us to test that ReconfigureEncoder is called + // appropriately. + webrtc::test::FrameForwarder frame_forwarder; + VideoOptions options; + EXPECT_TRUE(channel_->SetVideoSend(primary_ssrc, &options, &frame_forwarder)); + channel_->SetSend(true); + frame_forwarder.IncomingCapturedFrame(frame_source_.GetFrame( + 1920, 1080, webrtc::VideoRotation::kVideoRotation_0, + rtc::kNumMicrosecsPerSec / 30)); + + // Check that all encodings are initially active. + webrtc::RtpParameters parameters = + channel_->GetRtpSendParameters(primary_ssrc); + EXPECT_EQ(kNumSimulcastStreams, parameters.encodings.size()); + EXPECT_TRUE(parameters.encodings[0].active); + EXPECT_TRUE(parameters.encodings[1].active); + EXPECT_TRUE(parameters.encodings[2].active); + EXPECT_TRUE(fake_video_send_stream->IsSending()); + + // Only turn on only the middle stream. + parameters.encodings[0].active = false; + parameters.encodings[1].active = true; + parameters.encodings[2].active = false; + EXPECT_TRUE(channel_->SetRtpSendParameters(primary_ssrc, parameters).ok()); + // Verify that the active fields are set on the VideoChannel. + parameters = channel_->GetRtpSendParameters(primary_ssrc); + EXPECT_EQ(kNumSimulcastStreams, parameters.encodings.size()); + EXPECT_FALSE(parameters.encodings[0].active); + EXPECT_TRUE(parameters.encodings[1].active); + EXPECT_FALSE(parameters.encodings[2].active); + // Check that the VideoSendStream is updated appropriately. This means its + // send state was updated and it was reconfigured. + EXPECT_TRUE(fake_video_send_stream->IsSending()); + std::vector<webrtc::VideoStream> simulcast_streams = + fake_video_send_stream->GetVideoStreams(); + EXPECT_EQ(kNumSimulcastStreams, simulcast_streams.size()); + EXPECT_FALSE(simulcast_streams[0].active); + EXPECT_TRUE(simulcast_streams[1].active); + EXPECT_FALSE(simulcast_streams[2].active); + + // Turn off all streams. + parameters.encodings[0].active = false; + parameters.encodings[1].active = false; + parameters.encodings[2].active = false; + EXPECT_TRUE(channel_->SetRtpSendParameters(primary_ssrc, parameters).ok()); + // Verify that the active fields are set on the VideoChannel. + parameters = channel_->GetRtpSendParameters(primary_ssrc); + EXPECT_EQ(kNumSimulcastStreams, parameters.encodings.size()); + EXPECT_FALSE(parameters.encodings[0].active); + EXPECT_FALSE(parameters.encodings[1].active); + EXPECT_FALSE(parameters.encodings[2].active); + // Check that the VideoSendStream is off. + EXPECT_FALSE(fake_video_send_stream->IsSending()); + simulcast_streams = fake_video_send_stream->GetVideoStreams(); + EXPECT_EQ(kNumSimulcastStreams, simulcast_streams.size()); + EXPECT_FALSE(simulcast_streams[0].active); + EXPECT_FALSE(simulcast_streams[1].active); + EXPECT_FALSE(simulcast_streams[2].active); + + EXPECT_TRUE(channel_->SetVideoSend(primary_ssrc, nullptr, nullptr)); +} + +// Test that if a stream is reconfigured (due to a codec change or other +// change) while its encoding is still inactive, it doesn't start sending. +TEST_F(WebRtcVideoChannelTest, + InactiveStreamDoesntStartSendingWhenReconfigured) { + // Set an initial codec list, which will be modified later. + cricket::VideoSendParameters parameters1; + parameters1.codecs.push_back(GetEngineCodec("VP8")); + parameters1.codecs.push_back(GetEngineCodec("VP9")); + EXPECT_TRUE(channel_->SetSendParameters(parameters1)); + + FakeVideoSendStream* stream = AddSendStream(); + EXPECT_TRUE(channel_->SetSend(true)); + EXPECT_TRUE(stream->IsSending()); + + // Get current parameters and change "active" to false. + webrtc::RtpParameters parameters = channel_->GetRtpSendParameters(last_ssrc_); + ASSERT_EQ(1u, parameters.encodings.size()); + ASSERT_TRUE(parameters.encodings[0].active); + parameters.encodings[0].active = false; + EXPECT_EQ(1u, GetFakeSendStreams().size()); + EXPECT_EQ(1, fake_call_->GetNumCreatedSendStreams()); + EXPECT_TRUE(channel_->SetRtpSendParameters(last_ssrc_, parameters).ok()); + EXPECT_FALSE(stream->IsSending()); + + // Reorder the codec list, causing the stream to be reconfigured. + cricket::VideoSendParameters parameters2; + parameters2.codecs.push_back(GetEngineCodec("VP9")); + parameters2.codecs.push_back(GetEngineCodec("VP8")); + EXPECT_TRUE(channel_->SetSendParameters(parameters2)); + auto new_streams = GetFakeSendStreams(); + // Assert that a new underlying stream was created due to the codec change. + // Otherwise, this test isn't testing what it set out to test. + EXPECT_EQ(1u, GetFakeSendStreams().size()); + EXPECT_EQ(2, fake_call_->GetNumCreatedSendStreams()); + + // Verify that we still are not sending anything, due to the inactive + // encoding. + EXPECT_FALSE(new_streams[0]->IsSending()); +} + +// Test that GetRtpSendParameters returns the currently configured codecs. +TEST_F(WebRtcVideoChannelTest, GetRtpSendParametersCodecs) { + AddSendStream(); + cricket::VideoSendParameters parameters; + parameters.codecs.push_back(GetEngineCodec("VP8")); + parameters.codecs.push_back(GetEngineCodec("VP9")); + EXPECT_TRUE(channel_->SetSendParameters(parameters)); + + webrtc::RtpParameters rtp_parameters = + channel_->GetRtpSendParameters(last_ssrc_); + ASSERT_EQ(2u, rtp_parameters.codecs.size()); + EXPECT_EQ(GetEngineCodec("VP8").ToCodecParameters(), + rtp_parameters.codecs[0]); + EXPECT_EQ(GetEngineCodec("VP9").ToCodecParameters(), + rtp_parameters.codecs[1]); +} + +// Test that GetRtpSendParameters returns the currently configured RTCP CNAME. +TEST_F(WebRtcVideoChannelTest, GetRtpSendParametersRtcpCname) { + StreamParams params = StreamParams::CreateLegacy(kSsrc); + params.cname = "rtcpcname"; + AddSendStream(params); + + webrtc::RtpParameters rtp_parameters = channel_->GetRtpSendParameters(kSsrc); + EXPECT_STREQ("rtcpcname", rtp_parameters.rtcp.cname.c_str()); +} + +// Test that RtpParameters for send stream has one encoding and it has +// the correct SSRC. +TEST_F(WebRtcVideoChannelTest, GetRtpSendParametersSsrc) { + AddSendStream(); + + webrtc::RtpParameters rtp_parameters = + channel_->GetRtpSendParameters(last_ssrc_); + ASSERT_EQ(1u, rtp_parameters.encodings.size()); + EXPECT_EQ(last_ssrc_, rtp_parameters.encodings[0].ssrc); +} + +TEST_F(WebRtcVideoChannelTest, DetectRtpSendParameterHeaderExtensionsChange) { + AddSendStream(); + + webrtc::RtpParameters rtp_parameters = + channel_->GetRtpSendParameters(last_ssrc_); + rtp_parameters.header_extensions.emplace_back(); + + EXPECT_NE(0u, rtp_parameters.header_extensions.size()); + + webrtc::RTCError result = + channel_->SetRtpSendParameters(last_ssrc_, rtp_parameters); + EXPECT_EQ(webrtc::RTCErrorType::INVALID_MODIFICATION, result.type()); +} + +TEST_F(WebRtcVideoChannelTest, GetRtpSendParametersDegradationPreference) { + AddSendStream(); + + webrtc::test::FrameForwarder frame_forwarder; + EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, nullptr, &frame_forwarder)); + + webrtc::RtpParameters rtp_parameters = + channel_->GetRtpSendParameters(last_ssrc_); + EXPECT_FALSE(rtp_parameters.degradation_preference.has_value()); + rtp_parameters.degradation_preference = + webrtc::DegradationPreference::MAINTAIN_FRAMERATE; + + EXPECT_TRUE(channel_->SetRtpSendParameters(last_ssrc_, rtp_parameters).ok()); + + webrtc::RtpParameters updated_rtp_parameters = + channel_->GetRtpSendParameters(last_ssrc_); + EXPECT_EQ(updated_rtp_parameters.degradation_preference, + webrtc::DegradationPreference::MAINTAIN_FRAMERATE); + + // Remove the source since it will be destroyed before the channel + EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, nullptr, nullptr)); +} + +// Test that if we set/get parameters multiple times, we get the same results. +TEST_F(WebRtcVideoChannelTest, SetAndGetRtpSendParameters) { + AddSendStream(); + cricket::VideoSendParameters parameters; + parameters.codecs.push_back(GetEngineCodec("VP8")); + parameters.codecs.push_back(GetEngineCodec("VP9")); + EXPECT_TRUE(channel_->SetSendParameters(parameters)); + + webrtc::RtpParameters initial_params = + channel_->GetRtpSendParameters(last_ssrc_); + + // We should be able to set the params we just got. + EXPECT_TRUE(channel_->SetRtpSendParameters(last_ssrc_, initial_params).ok()); + + // ... And this shouldn't change the params returned by GetRtpSendParameters. + EXPECT_EQ(initial_params, channel_->GetRtpSendParameters(last_ssrc_)); +} + +// Test that GetRtpReceiveParameters returns the currently configured codecs. +TEST_F(WebRtcVideoChannelTest, GetRtpReceiveParametersCodecs) { + AddRecvStream(); + cricket::VideoRecvParameters parameters; + parameters.codecs.push_back(GetEngineCodec("VP8")); + parameters.codecs.push_back(GetEngineCodec("VP9")); + EXPECT_TRUE(channel_->SetRecvParameters(parameters)); + + webrtc::RtpParameters rtp_parameters = + channel_->GetRtpReceiveParameters(last_ssrc_); + ASSERT_EQ(2u, rtp_parameters.codecs.size()); + EXPECT_EQ(GetEngineCodec("VP8").ToCodecParameters(), + rtp_parameters.codecs[0]); + EXPECT_EQ(GetEngineCodec("VP9").ToCodecParameters(), + rtp_parameters.codecs[1]); +} + +#if defined(WEBRTC_USE_H264) +TEST_F(WebRtcVideoChannelTest, GetRtpReceiveFmtpSprop) { +#else +TEST_F(WebRtcVideoChannelTest, DISABLED_GetRtpReceiveFmtpSprop) { +#endif + cricket::VideoRecvParameters parameters; + cricket::VideoCodec kH264sprop1(101, "H264"); + kH264sprop1.SetParam(kH264FmtpSpropParameterSets, "uvw"); + parameters.codecs.push_back(kH264sprop1); + cricket::VideoCodec kH264sprop2(102, "H264"); + kH264sprop2.SetParam(kH264FmtpSpropParameterSets, "xyz"); + parameters.codecs.push_back(kH264sprop2); + EXPECT_TRUE(channel_->SetRecvParameters(parameters)); + + FakeVideoReceiveStream* recv_stream = AddRecvStream(); + const webrtc::VideoReceiveStream::Config& cfg = recv_stream->GetConfig(); + webrtc::RtpParameters rtp_parameters = + channel_->GetRtpReceiveParameters(last_ssrc_); + ASSERT_EQ(2u, rtp_parameters.codecs.size()); + EXPECT_EQ(kH264sprop1.ToCodecParameters(), rtp_parameters.codecs[0]); + ASSERT_EQ(2u, cfg.decoders.size()); + EXPECT_EQ(101, cfg.decoders[0].payload_type); + EXPECT_EQ("H264", cfg.decoders[0].video_format.name); + const auto it0 = + cfg.decoders[0].video_format.parameters.find(kH264FmtpSpropParameterSets); + ASSERT_TRUE(it0 != cfg.decoders[0].video_format.parameters.end()); + EXPECT_EQ("uvw", it0->second); + + EXPECT_EQ(102, cfg.decoders[1].payload_type); + EXPECT_EQ("H264", cfg.decoders[1].video_format.name); + const auto it1 = + cfg.decoders[1].video_format.parameters.find(kH264FmtpSpropParameterSets); + ASSERT_TRUE(it1 != cfg.decoders[1].video_format.parameters.end()); + EXPECT_EQ("xyz", it1->second); +} + +// Test that RtpParameters for receive stream has one encoding and it has +// the correct SSRC. +TEST_F(WebRtcVideoChannelTest, GetRtpReceiveParametersSsrc) { + AddRecvStream(); + + webrtc::RtpParameters rtp_parameters = + channel_->GetRtpReceiveParameters(last_ssrc_); + ASSERT_EQ(1u, rtp_parameters.encodings.size()); + EXPECT_EQ(last_ssrc_, rtp_parameters.encodings[0].ssrc); +} + +// Test that if we set/get parameters multiple times, we get the same results. +TEST_F(WebRtcVideoChannelTest, SetAndGetRtpReceiveParameters) { + AddRecvStream(); + cricket::VideoRecvParameters parameters; + parameters.codecs.push_back(GetEngineCodec("VP8")); + parameters.codecs.push_back(GetEngineCodec("VP9")); + EXPECT_TRUE(channel_->SetRecvParameters(parameters)); + + webrtc::RtpParameters initial_params = + channel_->GetRtpReceiveParameters(last_ssrc_); + + // ... And this shouldn't change the params returned by + // GetRtpReceiveParameters. + EXPECT_EQ(initial_params, channel_->GetRtpReceiveParameters(last_ssrc_)); +} + +// Test that GetDefaultRtpReceiveParameters returns parameters correctly when +// SSRCs aren't signaled. It should always return an empty +// "RtpEncodingParameters", even after a packet is received and the unsignaled +// SSRC is known. +TEST_F(WebRtcVideoChannelTest, + GetDefaultRtpReceiveParametersWithUnsignaledSsrc) { + // Call necessary methods to configure receiving a default stream as + // soon as it arrives. + cricket::VideoRecvParameters parameters; + parameters.codecs.push_back(GetEngineCodec("VP8")); + parameters.codecs.push_back(GetEngineCodec("VP9")); + EXPECT_TRUE(channel_->SetRecvParameters(parameters)); + + // Call GetRtpReceiveParameters before configured to receive an unsignaled + // stream. Should return nothing. + EXPECT_EQ(webrtc::RtpParameters(), + channel_->GetDefaultRtpReceiveParameters()); + + // Set a sink for an unsignaled stream. + cricket::FakeVideoRenderer renderer; + channel_->SetDefaultSink(&renderer); + + // Call GetDefaultRtpReceiveParameters before the SSRC is known. + webrtc::RtpParameters rtp_parameters = + channel_->GetDefaultRtpReceiveParameters(); + ASSERT_EQ(1u, rtp_parameters.encodings.size()); + EXPECT_FALSE(rtp_parameters.encodings[0].ssrc); + + // Receive VP8 packet. + uint8_t data[kMinRtpPacketLen]; + cricket::RtpHeader rtpHeader; + rtpHeader.payload_type = GetEngineCodec("VP8").id; + rtpHeader.seq_num = rtpHeader.timestamp = 0; + rtpHeader.ssrc = kIncomingUnsignalledSsrc; + cricket::SetRtpHeader(data, sizeof(data), rtpHeader); + rtc::CopyOnWriteBuffer packet(data, sizeof(data)); + channel_->OnPacketReceived(packet, /* packet_time_us */ -1); + + // The |ssrc| member should still be unset. + rtp_parameters = channel_->GetDefaultRtpReceiveParameters(); + ASSERT_EQ(1u, rtp_parameters.encodings.size()); + EXPECT_FALSE(rtp_parameters.encodings[0].ssrc); +} + +void WebRtcVideoChannelTest::TestReceiverLocalSsrcConfiguration( + bool receiver_first) { + EXPECT_TRUE(channel_->SetSendParameters(send_parameters_)); + + const uint32_t kSenderSsrc = 0xC0FFEE; + const uint32_t kSecondSenderSsrc = 0xBADCAFE; + const uint32_t kReceiverSsrc = 0x4711; + const uint32_t kExpectedDefaultReceiverSsrc = 1; + + if (receiver_first) { + AddRecvStream(StreamParams::CreateLegacy(kReceiverSsrc)); + std::vector<FakeVideoReceiveStream*> receive_streams = + fake_call_->GetVideoReceiveStreams(); + ASSERT_EQ(1u, receive_streams.size()); + // Default local SSRC when we have no sender. + EXPECT_EQ(kExpectedDefaultReceiverSsrc, + receive_streams[0]->GetConfig().rtp.local_ssrc); + } + AddSendStream(StreamParams::CreateLegacy(kSenderSsrc)); + if (!receiver_first) + AddRecvStream(StreamParams::CreateLegacy(kReceiverSsrc)); + std::vector<FakeVideoReceiveStream*> receive_streams = + fake_call_->GetVideoReceiveStreams(); + ASSERT_EQ(1u, receive_streams.size()); + EXPECT_EQ(kSenderSsrc, receive_streams[0]->GetConfig().rtp.local_ssrc); + + // Removing first sender should fall back to another (in this case the second) + // local send stream's SSRC. + AddSendStream(StreamParams::CreateLegacy(kSecondSenderSsrc)); + ASSERT_TRUE(channel_->RemoveSendStream(kSenderSsrc)); + receive_streams = fake_call_->GetVideoReceiveStreams(); + ASSERT_EQ(1u, receive_streams.size()); + EXPECT_EQ(kSecondSenderSsrc, receive_streams[0]->GetConfig().rtp.local_ssrc); + + // Removing the last sender should fall back to default local SSRC. + ASSERT_TRUE(channel_->RemoveSendStream(kSecondSenderSsrc)); + receive_streams = fake_call_->GetVideoReceiveStreams(); + ASSERT_EQ(1u, receive_streams.size()); + EXPECT_EQ(kExpectedDefaultReceiverSsrc, + receive_streams[0]->GetConfig().rtp.local_ssrc); +} + +TEST_F(WebRtcVideoChannelTest, ConfiguresLocalSsrc) { + TestReceiverLocalSsrcConfiguration(false); +} + +TEST_F(WebRtcVideoChannelTest, ConfiguresLocalSsrcOnExistingReceivers) { + TestReceiverLocalSsrcConfiguration(true); +} + +class WebRtcVideoChannelSimulcastTest : public ::testing::Test { + public: + WebRtcVideoChannelSimulcastTest() + : fake_call_(), + encoder_factory_(new cricket::FakeWebRtcVideoEncoderFactory), + decoder_factory_(new cricket::FakeWebRtcVideoDecoderFactory), + mock_rate_allocator_factory_( + std::make_unique<webrtc::MockVideoBitrateAllocatorFactory>()), + engine_(std::unique_ptr<cricket::FakeWebRtcVideoEncoderFactory>( + encoder_factory_), + std::unique_ptr<cricket::FakeWebRtcVideoDecoderFactory>( + decoder_factory_)), + last_ssrc_(0) {} + + void SetUp() override { + encoder_factory_->AddSupportedVideoCodecType("VP8"); + decoder_factory_->AddSupportedVideoCodecType("VP8"); + channel_.reset(engine_.CreateMediaChannel( + &fake_call_, GetMediaConfig(), VideoOptions(), webrtc::CryptoOptions(), + mock_rate_allocator_factory_.get())); + channel_->OnReadyToSend(true); + last_ssrc_ = 123; + } + + protected: + void VerifySimulcastSettings(const VideoCodec& codec, + int capture_width, + int capture_height, + size_t num_configured_streams, + size_t expected_num_streams, + bool screenshare, + bool conference_mode) { + cricket::VideoSendParameters parameters; + parameters.codecs.push_back(codec); + parameters.conference_mode = conference_mode; + ASSERT_TRUE(channel_->SetSendParameters(parameters)); + + std::vector<uint32_t> ssrcs = MAKE_VECTOR(kSsrcs3); + RTC_DCHECK(num_configured_streams <= ssrcs.size()); + ssrcs.resize(num_configured_streams); + + AddSendStream(CreateSimStreamParams("cname", ssrcs)); + // Send a full-size frame to trigger a stream reconfiguration to use all + // expected simulcast layers. + webrtc::test::FrameForwarder frame_forwarder; + cricket::FakeFrameSource frame_source(capture_width, capture_height, + rtc::kNumMicrosecsPerSec / 30); + + VideoOptions options; + if (screenshare) + options.is_screencast = screenshare; + EXPECT_TRUE( + channel_->SetVideoSend(ssrcs.front(), &options, &frame_forwarder)); + // Fetch the latest stream since SetVideoSend() may recreate it if the + // screen content setting is changed. + FakeVideoSendStream* stream = fake_call_.GetVideoSendStreams().front(); + channel_->SetSend(true); + frame_forwarder.IncomingCapturedFrame(frame_source.GetFrame()); + + auto rtp_parameters = channel_->GetRtpSendParameters(kSsrcs3[0]); + EXPECT_EQ(num_configured_streams, rtp_parameters.encodings.size()); + + std::vector<webrtc::VideoStream> video_streams = stream->GetVideoStreams(); + ASSERT_EQ(expected_num_streams, video_streams.size()); + EXPECT_LE(expected_num_streams, stream->GetConfig().rtp.ssrcs.size()); + + std::vector<webrtc::VideoStream> expected_streams; + if (num_configured_streams > 1 || conference_mode) { + expected_streams = GetSimulcastConfig( + /*min_layers=*/1, num_configured_streams, capture_width, + capture_height, webrtc::kDefaultBitratePriority, kDefaultQpMax, + screenshare && conference_mode, true); + if (screenshare && conference_mode) { + for (const webrtc::VideoStream& stream : expected_streams) { + // Never scale screen content. + EXPECT_EQ(stream.width, rtc::checked_cast<size_t>(capture_width)); + EXPECT_EQ(stream.height, rtc::checked_cast<size_t>(capture_height)); + } + } + } else { + webrtc::VideoStream stream; + stream.width = capture_width; + stream.height = capture_height; + stream.max_framerate = kDefaultVideoMaxFramerate; + stream.min_bitrate_bps = webrtc::kDefaultMinVideoBitrateBps; + stream.target_bitrate_bps = stream.max_bitrate_bps = + GetMaxDefaultBitrateBps(capture_width, capture_height); + stream.max_qp = kDefaultQpMax; + expected_streams.push_back(stream); + } + + ASSERT_EQ(expected_streams.size(), video_streams.size()); + + size_t num_streams = video_streams.size(); + int total_max_bitrate_bps = 0; + for (size_t i = 0; i < num_streams; ++i) { + EXPECT_EQ(expected_streams[i].width, video_streams[i].width); + EXPECT_EQ(expected_streams[i].height, video_streams[i].height); + + EXPECT_GT(video_streams[i].max_framerate, 0); + EXPECT_EQ(expected_streams[i].max_framerate, + video_streams[i].max_framerate); + + EXPECT_GT(video_streams[i].min_bitrate_bps, 0); + EXPECT_EQ(expected_streams[i].min_bitrate_bps, + video_streams[i].min_bitrate_bps); + + EXPECT_GT(video_streams[i].target_bitrate_bps, 0); + EXPECT_EQ(expected_streams[i].target_bitrate_bps, + video_streams[i].target_bitrate_bps); + + EXPECT_GT(video_streams[i].max_bitrate_bps, 0); + EXPECT_EQ(expected_streams[i].max_bitrate_bps, + video_streams[i].max_bitrate_bps); + + EXPECT_GT(video_streams[i].max_qp, 0); + EXPECT_EQ(expected_streams[i].max_qp, video_streams[i].max_qp); + + EXPECT_EQ(num_configured_streams > 1 || conference_mode, + expected_streams[i].num_temporal_layers.has_value()); + + if (conference_mode) { + EXPECT_EQ(expected_streams[i].num_temporal_layers, + video_streams[i].num_temporal_layers); + } + + if (i == num_streams - 1) { + total_max_bitrate_bps += video_streams[i].max_bitrate_bps; + } else { + total_max_bitrate_bps += video_streams[i].target_bitrate_bps; + } + } + + EXPECT_TRUE(channel_->SetVideoSend(ssrcs.front(), nullptr, nullptr)); + } + + FakeVideoSendStream* AddSendStream() { + return AddSendStream(StreamParams::CreateLegacy(last_ssrc_++)); + } + + FakeVideoSendStream* AddSendStream(const StreamParams& sp) { + size_t num_streams = fake_call_.GetVideoSendStreams().size(); + EXPECT_TRUE(channel_->AddSendStream(sp)); + std::vector<FakeVideoSendStream*> streams = + fake_call_.GetVideoSendStreams(); + EXPECT_EQ(num_streams + 1, streams.size()); + return streams[streams.size() - 1]; + } + + std::vector<FakeVideoSendStream*> GetFakeSendStreams() { + return fake_call_.GetVideoSendStreams(); + } + + FakeVideoReceiveStream* AddRecvStream() { + return AddRecvStream(StreamParams::CreateLegacy(last_ssrc_++)); + } + + FakeVideoReceiveStream* AddRecvStream(const StreamParams& sp) { + size_t num_streams = fake_call_.GetVideoReceiveStreams().size(); + EXPECT_TRUE(channel_->AddRecvStream(sp)); + std::vector<FakeVideoReceiveStream*> streams = + fake_call_.GetVideoReceiveStreams(); + EXPECT_EQ(num_streams + 1, streams.size()); + return streams[streams.size() - 1]; + } + + webrtc::RtcEventLogNull event_log_; + FakeCall fake_call_; + cricket::FakeWebRtcVideoEncoderFactory* encoder_factory_; + cricket::FakeWebRtcVideoDecoderFactory* decoder_factory_; + std::unique_ptr<webrtc::MockVideoBitrateAllocatorFactory> + mock_rate_allocator_factory_; + WebRtcVideoEngine engine_; + std::unique_ptr<VideoMediaChannel> channel_; + uint32_t last_ssrc_; +}; + +TEST_F(WebRtcVideoChannelSimulcastTest, SetSendCodecsWith2SimulcastStreams) { + VerifySimulcastSettings(cricket::VideoCodec("VP8"), 640, 360, 2, 2, false, + true); +} + +TEST_F(WebRtcVideoChannelSimulcastTest, SetSendCodecsWith3SimulcastStreams) { + VerifySimulcastSettings(cricket::VideoCodec("VP8"), 1280, 720, 3, 3, false, + true); +} + +// Test that we normalize send codec format size in simulcast. +TEST_F(WebRtcVideoChannelSimulcastTest, SetSendCodecsWithOddSizeInSimulcast) { + VerifySimulcastSettings(cricket::VideoCodec("VP8"), 541, 271, 2, 2, false, + true); +} + +TEST_F(WebRtcVideoChannelSimulcastTest, SetSendCodecsForScreenshare) { + VerifySimulcastSettings(cricket::VideoCodec("VP8"), 1280, 720, 3, 3, true, + false); +} + +TEST_F(WebRtcVideoChannelSimulcastTest, SetSendCodecsForSimulcastScreenshare) { + VerifySimulcastSettings(cricket::VideoCodec("VP8"), 1280, 720, 3, 2, true, + true); +} + +TEST_F(WebRtcVideoChannelSimulcastTest, SimulcastScreenshareWithoutConference) { + VerifySimulcastSettings(cricket::VideoCodec("VP8"), 1280, 720, 3, 3, true, + false); +} + +TEST_F(WebRtcVideoChannelBaseTest, GetSources) { + EXPECT_THAT(channel_->GetSources(kSsrc), IsEmpty()); + + channel_->SetDefaultSink(&renderer_); + EXPECT_TRUE(SetDefaultCodec()); + EXPECT_TRUE(SetSend(true)); + EXPECT_EQ(renderer_.num_rendered_frames(), 0); + + // Send and receive one frame. + SendFrame(); + EXPECT_FRAME_WAIT(1, kVideoWidth, kVideoHeight, kTimeout); + + EXPECT_THAT(channel_->GetSources(kSsrc - 1), IsEmpty()); + EXPECT_THAT(channel_->GetSources(kSsrc), SizeIs(1)); + EXPECT_THAT(channel_->GetSources(kSsrc + 1), IsEmpty()); + + webrtc::RtpSource source = channel_->GetSources(kSsrc)[0]; + EXPECT_EQ(source.source_id(), kSsrc); + EXPECT_EQ(source.source_type(), webrtc::RtpSourceType::SSRC); + int64_t rtp_timestamp_1 = source.rtp_timestamp(); + int64_t timestamp_ms_1 = source.timestamp_ms(); + + // Send and receive another frame. + SendFrame(); + EXPECT_FRAME_WAIT(2, kVideoWidth, kVideoHeight, kTimeout); + + EXPECT_THAT(channel_->GetSources(kSsrc - 1), IsEmpty()); + EXPECT_THAT(channel_->GetSources(kSsrc), SizeIs(1)); + EXPECT_THAT(channel_->GetSources(kSsrc + 1), IsEmpty()); + + source = channel_->GetSources(kSsrc)[0]; + EXPECT_EQ(source.source_id(), kSsrc); + EXPECT_EQ(source.source_type(), webrtc::RtpSourceType::SSRC); + int64_t rtp_timestamp_2 = source.rtp_timestamp(); + int64_t timestamp_ms_2 = source.timestamp_ms(); + + EXPECT_GT(rtp_timestamp_2, rtp_timestamp_1); + EXPECT_GT(timestamp_ms_2, timestamp_ms_1); +} + +TEST_F(WebRtcVideoChannelTest, SetsRidsOnSendStream) { + StreamParams sp = CreateSimStreamParams("cname", {123, 456, 789}); + + std::vector<std::string> rids = {"f", "h", "q"}; + std::vector<cricket::RidDescription> rid_descriptions; + for (const auto& rid : rids) { + rid_descriptions.emplace_back(rid, cricket::RidDirection::kSend); + } + sp.set_rids(rid_descriptions); + + ASSERT_TRUE(channel_->AddSendStream(sp)); + const auto& streams = fake_call_->GetVideoSendStreams(); + ASSERT_EQ(1u, streams.size()); + auto stream = streams[0]; + ASSERT_NE(stream, nullptr); + const auto& config = stream->GetConfig(); + EXPECT_THAT(config.rtp.rids, ElementsAreArray(rids)); +} + +} // namespace cricket diff --git a/media/engine/webrtc_voice_engine.cc b/media/engine/webrtc_voice_engine.cc new file mode 100644 index 0000000000..85c72804c1 --- /dev/null +++ b/media/engine/webrtc_voice_engine.cc @@ -0,0 +1,2392 @@ +/* + * Copyright (c) 2004 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "media/engine/webrtc_voice_engine.h" + +#include <algorithm> +#include <cstdio> +#include <functional> +#include <string> +#include <utility> +#include <vector> + +#include "absl/algorithm/container.h" +#include "absl/strings/match.h" +#include "api/audio_codecs/audio_codec_pair_id.h" +#include "api/call/audio_sink.h" +#include "media/base/audio_source.h" +#include "media/base/media_constants.h" +#include "media/base/stream_params.h" +#include "media/engine/adm_helpers.h" +#include "media/engine/payload_type_mapper.h" +#include "media/engine/webrtc_media_engine.h" +#include "modules/audio_device/audio_device_impl.h" +#include "modules/audio_mixer/audio_mixer_impl.h" +#include "modules/audio_processing/aec_dump/aec_dump_factory.h" +#include "modules/audio_processing/include/audio_processing.h" +#include "rtc_base/arraysize.h" +#include "rtc_base/byte_order.h" +#include "rtc_base/constructor_magic.h" +#include "rtc_base/experiments/field_trial_parser.h" +#include "rtc_base/experiments/field_trial_units.h" +#include "rtc_base/helpers.h" +#include "rtc_base/logging.h" +#include "rtc_base/race_checker.h" +#include "rtc_base/strings/audio_format_to_string.h" +#include "rtc_base/strings/string_builder.h" +#include "rtc_base/third_party/base64/base64.h" +#include "rtc_base/trace_event.h" +#include "system_wrappers/include/field_trial.h" +#include "system_wrappers/include/metrics.h" + +namespace cricket { +namespace { + +constexpr size_t kMaxUnsignaledRecvStreams = 4; + +constexpr int kNackRtpHistoryMs = 5000; + +const int kMinTelephoneEventCode = 0; // RFC4733 (Section 2.3.1) +const int kMaxTelephoneEventCode = 255; + +const int kMinPayloadType = 0; +const int kMaxPayloadType = 127; + +class ProxySink : public webrtc::AudioSinkInterface { + public: + explicit ProxySink(AudioSinkInterface* sink) : sink_(sink) { + RTC_DCHECK(sink); + } + + void OnData(const Data& audio) override { sink_->OnData(audio); } + + private: + webrtc::AudioSinkInterface* sink_; +}; + +bool ValidateStreamParams(const StreamParams& sp) { + if (sp.ssrcs.empty()) { + RTC_DLOG(LS_ERROR) << "No SSRCs in stream parameters: " << sp.ToString(); + return false; + } + if (sp.ssrcs.size() > 1) { + RTC_DLOG(LS_ERROR) << "Multiple SSRCs in stream parameters: " + << sp.ToString(); + return false; + } + return true; +} + +// Dumps an AudioCodec in RFC 2327-ish format. +std::string ToString(const AudioCodec& codec) { + rtc::StringBuilder ss; + ss << codec.name << "/" << codec.clockrate << "/" << codec.channels; + if (!codec.params.empty()) { + ss << " {"; + for (const auto& param : codec.params) { + ss << " " << param.first << "=" << param.second; + } + ss << " }"; + } + ss << " (" << codec.id << ")"; + return ss.Release(); +} + +bool IsCodec(const AudioCodec& codec, const char* ref_name) { + return absl::EqualsIgnoreCase(codec.name, ref_name); +} + +bool FindCodec(const std::vector<AudioCodec>& codecs, + const AudioCodec& codec, + AudioCodec* found_codec) { + for (const AudioCodec& c : codecs) { + if (c.Matches(codec)) { + if (found_codec != NULL) { + *found_codec = c; + } + return true; + } + } + return false; +} + +bool VerifyUniquePayloadTypes(const std::vector<AudioCodec>& codecs) { + if (codecs.empty()) { + return true; + } + std::vector<int> payload_types; + absl::c_transform(codecs, std::back_inserter(payload_types), + [](const AudioCodec& codec) { return codec.id; }); + absl::c_sort(payload_types); + return absl::c_adjacent_find(payload_types) == payload_types.end(); +} + +absl::optional<std::string> GetAudioNetworkAdaptorConfig( + const AudioOptions& options) { + if (options.audio_network_adaptor && *options.audio_network_adaptor && + options.audio_network_adaptor_config) { + // Turn on audio network adaptor only when |options_.audio_network_adaptor| + // equals true and |options_.audio_network_adaptor_config| has a value. + return options.audio_network_adaptor_config; + } + return absl::nullopt; +} + +// Returns its smallest positive argument. If neither argument is positive, +// returns an arbitrary nonpositive value. +int MinPositive(int a, int b) { + if (a <= 0) { + return b; + } + if (b <= 0) { + return a; + } + return std::min(a, b); +} + +// |max_send_bitrate_bps| is the bitrate from "b=" in SDP. +// |rtp_max_bitrate_bps| is the bitrate from RtpSender::SetParameters. +absl::optional<int> ComputeSendBitrate(int max_send_bitrate_bps, + absl::optional<int> rtp_max_bitrate_bps, + const webrtc::AudioCodecSpec& spec) { + // If application-configured bitrate is set, take minimum of that and SDP + // bitrate. + const int bps = rtp_max_bitrate_bps + ? MinPositive(max_send_bitrate_bps, *rtp_max_bitrate_bps) + : max_send_bitrate_bps; + if (bps <= 0) { + return spec.info.default_bitrate_bps; + } + + if (bps < spec.info.min_bitrate_bps) { + // If codec is not multi-rate and |bps| is less than the fixed bitrate then + // fail. If codec is not multi-rate and |bps| exceeds or equal the fixed + // bitrate then ignore. + RTC_LOG(LS_ERROR) << "Failed to set codec " << spec.format.name + << " to bitrate " << bps + << " bps" + ", requires at least " + << spec.info.min_bitrate_bps << " bps."; + return absl::nullopt; + } + + if (spec.info.HasFixedBitrate()) { + return spec.info.default_bitrate_bps; + } else { + // If codec is multi-rate then just set the bitrate. + return std::min(bps, spec.info.max_bitrate_bps); + } +} + +} // namespace + +WebRtcVoiceEngine::WebRtcVoiceEngine( + webrtc::TaskQueueFactory* task_queue_factory, + webrtc::AudioDeviceModule* adm, + const rtc::scoped_refptr<webrtc::AudioEncoderFactory>& encoder_factory, + const rtc::scoped_refptr<webrtc::AudioDecoderFactory>& decoder_factory, + rtc::scoped_refptr<webrtc::AudioMixer> audio_mixer, + rtc::scoped_refptr<webrtc::AudioProcessing> audio_processing) + : task_queue_factory_(task_queue_factory), + adm_(adm), + encoder_factory_(encoder_factory), + decoder_factory_(decoder_factory), + audio_mixer_(audio_mixer), + apm_(audio_processing) { + // This may be called from any thread, so detach thread checkers. + worker_thread_checker_.Detach(); + signal_thread_checker_.Detach(); + RTC_LOG(LS_INFO) << "WebRtcVoiceEngine::WebRtcVoiceEngine"; + RTC_DCHECK(decoder_factory); + RTC_DCHECK(encoder_factory); + // The rest of our initialization will happen in Init. +} + +WebRtcVoiceEngine::~WebRtcVoiceEngine() { + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + RTC_LOG(LS_INFO) << "WebRtcVoiceEngine::~WebRtcVoiceEngine"; + if (initialized_) { + StopAecDump(); + + // Stop AudioDevice. + adm()->StopPlayout(); + adm()->StopRecording(); + adm()->RegisterAudioCallback(nullptr); + adm()->Terminate(); + } +} + +void WebRtcVoiceEngine::Init() { + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + RTC_LOG(LS_INFO) << "WebRtcVoiceEngine::Init"; + + // TaskQueue expects to be created/destroyed on the same thread. + low_priority_worker_queue_.reset( + new rtc::TaskQueue(task_queue_factory_->CreateTaskQueue( + "rtc-low-prio", webrtc::TaskQueueFactory::Priority::LOW))); + + // Load our audio codec lists. + RTC_LOG(LS_VERBOSE) << "Supported send codecs in order of preference:"; + send_codecs_ = CollectCodecs(encoder_factory_->GetSupportedEncoders()); + for (const AudioCodec& codec : send_codecs_) { + RTC_LOG(LS_VERBOSE) << ToString(codec); + } + + RTC_LOG(LS_VERBOSE) << "Supported recv codecs in order of preference:"; + recv_codecs_ = CollectCodecs(decoder_factory_->GetSupportedDecoders()); + for (const AudioCodec& codec : recv_codecs_) { + RTC_LOG(LS_VERBOSE) << ToString(codec); + } + +#if defined(WEBRTC_INCLUDE_INTERNAL_AUDIO_DEVICE) + // No ADM supplied? Create a default one. + if (!adm_) { + adm_ = webrtc::AudioDeviceModule::Create( + webrtc::AudioDeviceModule::kPlatformDefaultAudio, task_queue_factory_); + } +#endif // WEBRTC_INCLUDE_INTERNAL_AUDIO_DEVICE + RTC_CHECK(adm()); + webrtc::adm_helpers::Init(adm()); + + // Set up AudioState. + { + webrtc::AudioState::Config config; + if (audio_mixer_) { + config.audio_mixer = audio_mixer_; + } else { + config.audio_mixer = webrtc::AudioMixerImpl::Create(); + } + config.audio_processing = apm_; + config.audio_device_module = adm_; + audio_state_ = webrtc::AudioState::Create(config); + } + + // Connect the ADM to our audio path. + adm()->RegisterAudioCallback(audio_state()->audio_transport()); + + // Set default engine options. + { + AudioOptions options; + options.echo_cancellation = true; + options.auto_gain_control = true; + options.noise_suppression = true; + options.highpass_filter = true; + options.stereo_swapping = false; + options.audio_jitter_buffer_max_packets = 200; + options.audio_jitter_buffer_fast_accelerate = false; + options.audio_jitter_buffer_min_delay_ms = 0; + options.audio_jitter_buffer_enable_rtx_handling = false; + options.typing_detection = true; + options.experimental_agc = false; + options.experimental_ns = false; + options.residual_echo_detector = true; + bool error = ApplyOptions(options); + RTC_DCHECK(error); + } + + initialized_ = true; +} + +rtc::scoped_refptr<webrtc::AudioState> WebRtcVoiceEngine::GetAudioState() + const { + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + return audio_state_; +} + +VoiceMediaChannel* WebRtcVoiceEngine::CreateMediaChannel( + webrtc::Call* call, + const MediaConfig& config, + const AudioOptions& options, + const webrtc::CryptoOptions& crypto_options) { + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + return new WebRtcVoiceMediaChannel(this, config, options, crypto_options, + call); +} + +bool WebRtcVoiceEngine::ApplyOptions(const AudioOptions& options_in) { + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + RTC_LOG(LS_INFO) << "WebRtcVoiceEngine::ApplyOptions: " + << options_in.ToString(); + AudioOptions options = options_in; // The options are modified below. + + // Set and adjust echo canceller options. + // Use desktop AEC by default, when not using hardware AEC. + bool use_mobile_software_aec = false; + +#if defined(WEBRTC_IOS) + if (options.ios_force_software_aec_HACK && + *options.ios_force_software_aec_HACK) { + // EC may be forced on for a device known to have non-functioning platform + // AEC. + options.echo_cancellation = true; + RTC_LOG(LS_WARNING) + << "Force software AEC on iOS. May conflict with platform AEC."; + } else { + // On iOS, VPIO provides built-in EC. + options.echo_cancellation = false; + RTC_LOG(LS_INFO) << "Always disable AEC on iOS. Use built-in instead."; + } +#elif defined(WEBRTC_ANDROID) + use_mobile_software_aec = true; +#endif + +// Set and adjust noise suppressor options. +#if defined(WEBRTC_IOS) + // On iOS, VPIO provides built-in NS. + options.noise_suppression = false; + options.typing_detection = false; + options.experimental_ns = false; + RTC_LOG(LS_INFO) << "Always disable NS on iOS. Use built-in instead."; +#elif defined(WEBRTC_ANDROID) + options.typing_detection = false; + options.experimental_ns = false; +#endif + +// Set and adjust gain control options. +#if defined(WEBRTC_IOS) + // On iOS, VPIO provides built-in AGC. + options.auto_gain_control = false; + options.experimental_agc = false; + RTC_LOG(LS_INFO) << "Always disable AGC on iOS. Use built-in instead."; +#elif defined(WEBRTC_ANDROID) + options.experimental_agc = false; +#endif + +#if defined(WEBRTC_IOS) || defined(WEBRTC_ANDROID) + // Turn off the gain control if specified by the field trial. + // The purpose of the field trial is to reduce the amount of resampling + // performed inside the audio processing module on mobile platforms by + // whenever possible turning off the fixed AGC mode and the high-pass filter. + // (https://bugs.chromium.org/p/webrtc/issues/detail?id=6181). + if (webrtc::field_trial::IsEnabled( + "WebRTC-Audio-MinimizeResamplingOnMobile")) { + options.auto_gain_control = false; + RTC_LOG(LS_INFO) << "Disable AGC according to field trial."; + if (!(options.noise_suppression.value_or(false) || + options.echo_cancellation.value_or(false))) { + // If possible, turn off the high-pass filter. + RTC_LOG(LS_INFO) + << "Disable high-pass filter in response to field trial."; + options.highpass_filter = false; + } + } +#endif + + if (options.echo_cancellation) { + // Check if platform supports built-in EC. Currently only supported on + // Android and in combination with Java based audio layer. + // TODO(henrika): investigate possibility to support built-in EC also + // in combination with Open SL ES audio. + const bool built_in_aec = adm()->BuiltInAECIsAvailable(); + if (built_in_aec) { + // Built-in EC exists on this device. Enable/Disable it according to the + // echo_cancellation audio option. + const bool enable_built_in_aec = *options.echo_cancellation; + if (adm()->EnableBuiltInAEC(enable_built_in_aec) == 0 && + enable_built_in_aec) { + // Disable internal software EC if built-in EC is enabled, + // i.e., replace the software EC with the built-in EC. + options.echo_cancellation = false; + RTC_LOG(LS_INFO) + << "Disabling EC since built-in EC will be used instead"; + } + } + } + + if (options.auto_gain_control) { + bool built_in_agc_avaliable = adm()->BuiltInAGCIsAvailable(); + if (built_in_agc_avaliable) { + if (adm()->EnableBuiltInAGC(*options.auto_gain_control) == 0 && + *options.auto_gain_control) { + // Disable internal software AGC if built-in AGC is enabled, + // i.e., replace the software AGC with the built-in AGC. + options.auto_gain_control = false; + RTC_LOG(LS_INFO) + << "Disabling AGC since built-in AGC will be used instead"; + } + } + } + + if (options.noise_suppression) { + if (adm()->BuiltInNSIsAvailable()) { + bool builtin_ns = *options.noise_suppression; + if (adm()->EnableBuiltInNS(builtin_ns) == 0 && builtin_ns) { + // Disable internal software NS if built-in NS is enabled, + // i.e., replace the software NS with the built-in NS. + options.noise_suppression = false; + RTC_LOG(LS_INFO) + << "Disabling NS since built-in NS will be used instead"; + } + } + } + + if (options.stereo_swapping) { + RTC_LOG(LS_INFO) << "Stereo swapping enabled? " << *options.stereo_swapping; + audio_state()->SetStereoChannelSwapping(*options.stereo_swapping); + } + + if (options.audio_jitter_buffer_max_packets) { + RTC_LOG(LS_INFO) << "NetEq capacity is " + << *options.audio_jitter_buffer_max_packets; + audio_jitter_buffer_max_packets_ = + std::max(20, *options.audio_jitter_buffer_max_packets); + } + if (options.audio_jitter_buffer_fast_accelerate) { + RTC_LOG(LS_INFO) << "NetEq fast mode? " + << *options.audio_jitter_buffer_fast_accelerate; + audio_jitter_buffer_fast_accelerate_ = + *options.audio_jitter_buffer_fast_accelerate; + } + if (options.audio_jitter_buffer_min_delay_ms) { + RTC_LOG(LS_INFO) << "NetEq minimum delay is " + << *options.audio_jitter_buffer_min_delay_ms; + audio_jitter_buffer_min_delay_ms_ = + *options.audio_jitter_buffer_min_delay_ms; + } + if (options.audio_jitter_buffer_enable_rtx_handling) { + RTC_LOG(LS_INFO) << "NetEq handle reordered packets? " + << *options.audio_jitter_buffer_enable_rtx_handling; + audio_jitter_buffer_enable_rtx_handling_ = + *options.audio_jitter_buffer_enable_rtx_handling; + } + + webrtc::AudioProcessing* ap = apm(); + if (!ap) { + RTC_LOG(LS_INFO) + << "No audio processing module present. No software-provided effects " + "(AEC, NS, AGC, ...) are activated"; + return true; + } + + webrtc::Config config; + + if (options.experimental_ns) { + experimental_ns_ = options.experimental_ns; + } + if (experimental_ns_) { + RTC_LOG(LS_INFO) << "Experimental ns is enabled? " << *experimental_ns_; + config.Set<webrtc::ExperimentalNs>( + new webrtc::ExperimentalNs(*experimental_ns_)); + } + + webrtc::AudioProcessing::Config apm_config = ap->GetConfig(); + + if (options.echo_cancellation) { + apm_config.echo_canceller.enabled = *options.echo_cancellation; + apm_config.echo_canceller.mobile_mode = use_mobile_software_aec; + } + + if (options.auto_gain_control) { + const bool enabled = *options.auto_gain_control; + apm_config.gain_controller1.enabled = enabled; +#if defined(WEBRTC_IOS) || defined(WEBRTC_ANDROID) + apm_config.gain_controller1.mode = + apm_config.gain_controller1.kFixedDigital; +#else + apm_config.gain_controller1.mode = + apm_config.gain_controller1.kAdaptiveAnalog; +#endif + constexpr int kMinVolumeLevel = 0; + constexpr int kMaxVolumeLevel = 255; + apm_config.gain_controller1.analog_level_minimum = kMinVolumeLevel; + apm_config.gain_controller1.analog_level_maximum = kMaxVolumeLevel; + } + if (options.tx_agc_target_dbov) { + apm_config.gain_controller1.target_level_dbfs = *options.tx_agc_target_dbov; + } + if (options.tx_agc_digital_compression_gain) { + apm_config.gain_controller1.compression_gain_db = + *options.tx_agc_digital_compression_gain; + } + if (options.tx_agc_limiter) { + apm_config.gain_controller1.enable_limiter = *options.tx_agc_limiter; + } + + if (options.highpass_filter) { + apm_config.high_pass_filter.enabled = *options.highpass_filter; + } + + if (options.residual_echo_detector) { + apm_config.residual_echo_detector.enabled = *options.residual_echo_detector; + } + + if (options.noise_suppression) { + const bool enabled = *options.noise_suppression; + apm_config.noise_suppression.enabled = enabled; + apm_config.noise_suppression.level = + webrtc::AudioProcessing::Config::NoiseSuppression::Level::kHigh; + RTC_LOG(LS_INFO) << "NS set to " << enabled; + } + + if (options.typing_detection) { + RTC_LOG(LS_INFO) << "Typing detection is enabled? " + << *options.typing_detection; + apm_config.voice_detection.enabled = *options.typing_detection; + } + + ap->SetExtraOptions(config); + ap->ApplyConfig(apm_config); + return true; +} + +const std::vector<AudioCodec>& WebRtcVoiceEngine::send_codecs() const { + RTC_DCHECK(signal_thread_checker_.IsCurrent()); + return send_codecs_; +} + +const std::vector<AudioCodec>& WebRtcVoiceEngine::recv_codecs() const { + RTC_DCHECK(signal_thread_checker_.IsCurrent()); + return recv_codecs_; +} + +std::vector<webrtc::RtpHeaderExtensionCapability> +WebRtcVoiceEngine::GetRtpHeaderExtensions() const { + RTC_DCHECK(signal_thread_checker_.IsCurrent()); + std::vector<webrtc::RtpHeaderExtensionCapability> result; + int id = 1; + for (const auto& uri : + {webrtc::RtpExtension::kAudioLevelUri, + webrtc::RtpExtension::kAbsSendTimeUri, + webrtc::RtpExtension::kTransportSequenceNumberUri, + webrtc::RtpExtension::kMidUri, webrtc::RtpExtension::kRidUri, + webrtc::RtpExtension::kRepairedRidUri}) { + result.emplace_back(uri, id++, webrtc::RtpTransceiverDirection::kSendRecv); + } + return result; +} + +void WebRtcVoiceEngine::RegisterChannel(WebRtcVoiceMediaChannel* channel) { + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + RTC_DCHECK(channel); + channels_.push_back(channel); +} + +void WebRtcVoiceEngine::UnregisterChannel(WebRtcVoiceMediaChannel* channel) { + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + auto it = absl::c_find(channels_, channel); + RTC_DCHECK(it != channels_.end()); + channels_.erase(it); +} + +bool WebRtcVoiceEngine::StartAecDump(webrtc::FileWrapper file, + int64_t max_size_bytes) { + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + + webrtc::AudioProcessing* ap = apm(); + if (!ap) { + RTC_LOG(LS_WARNING) + << "Attempting to start aecdump when no audio processing module is " + "present, hence no aecdump is started."; + return false; + } + + return ap->CreateAndAttachAecDump(file.Release(), max_size_bytes, + low_priority_worker_queue_.get()); +} + +void WebRtcVoiceEngine::StopAecDump() { + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + webrtc::AudioProcessing* ap = apm(); + if (ap) { + ap->DetachAecDump(); + } else { + RTC_LOG(LS_WARNING) << "Attempting to stop aecdump when no audio " + "processing module is present"; + } +} + +webrtc::AudioDeviceModule* WebRtcVoiceEngine::adm() { + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + RTC_DCHECK(adm_); + return adm_.get(); +} + +webrtc::AudioProcessing* WebRtcVoiceEngine::apm() const { + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + return apm_.get(); +} + +webrtc::AudioState* WebRtcVoiceEngine::audio_state() { + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + RTC_DCHECK(audio_state_); + return audio_state_.get(); +} + +std::vector<AudioCodec> WebRtcVoiceEngine::CollectCodecs( + const std::vector<webrtc::AudioCodecSpec>& specs) const { + PayloadTypeMapper mapper; + std::vector<AudioCodec> out; + + // Only generate CN payload types for these clockrates: + std::map<int, bool, std::greater<int>> generate_cn = { + {8000, false}, {16000, false}, {32000, false}}; + // Only generate telephone-event payload types for these clockrates: + std::map<int, bool, std::greater<int>> generate_dtmf = { + {8000, false}, {16000, false}, {32000, false}, {48000, false}}; + + auto map_format = [&mapper](const webrtc::SdpAudioFormat& format, + std::vector<AudioCodec>* out) { + absl::optional<AudioCodec> opt_codec = mapper.ToAudioCodec(format); + if (opt_codec) { + if (out) { + out->push_back(*opt_codec); + } + } else { + RTC_LOG(LS_ERROR) << "Unable to assign payload type to format: " + << rtc::ToString(format); + } + + return opt_codec; + }; + + for (const auto& spec : specs) { + // We need to do some extra stuff before adding the main codecs to out. + absl::optional<AudioCodec> opt_codec = map_format(spec.format, nullptr); + if (opt_codec) { + AudioCodec& codec = *opt_codec; + if (spec.info.supports_network_adaption) { + codec.AddFeedbackParam( + FeedbackParam(kRtcpFbParamTransportCc, kParamValueEmpty)); + } + + if (spec.info.allow_comfort_noise) { + // Generate a CN entry if the decoder allows it and we support the + // clockrate. + auto cn = generate_cn.find(spec.format.clockrate_hz); + if (cn != generate_cn.end()) { + cn->second = true; + } + } + + // Generate a telephone-event entry if we support the clockrate. + auto dtmf = generate_dtmf.find(spec.format.clockrate_hz); + if (dtmf != generate_dtmf.end()) { + dtmf->second = true; + } + + out.push_back(codec); + } + } + + // Add CN codecs after "proper" audio codecs. + for (const auto& cn : generate_cn) { + if (cn.second) { + map_format({kCnCodecName, cn.first, 1}, &out); + } + } + + // Add telephone-event codecs last. + for (const auto& dtmf : generate_dtmf) { + if (dtmf.second) { + map_format({kDtmfCodecName, dtmf.first, 1}, &out); + } + } + + return out; +} + +class WebRtcVoiceMediaChannel::WebRtcAudioSendStream + : public AudioSource::Sink { + public: + WebRtcAudioSendStream( + uint32_t ssrc, + const std::string& mid, + const std::string& c_name, + const std::string track_id, + const absl::optional<webrtc::AudioSendStream::Config::SendCodecSpec>& + send_codec_spec, + bool extmap_allow_mixed, + const std::vector<webrtc::RtpExtension>& extensions, + int max_send_bitrate_bps, + int rtcp_report_interval_ms, + const absl::optional<std::string>& audio_network_adaptor_config, + webrtc::Call* call, + webrtc::Transport* send_transport, + const rtc::scoped_refptr<webrtc::AudioEncoderFactory>& encoder_factory, + const absl::optional<webrtc::AudioCodecPairId> codec_pair_id, + rtc::scoped_refptr<webrtc::FrameEncryptorInterface> frame_encryptor, + const webrtc::CryptoOptions& crypto_options) + : call_(call), + config_(send_transport), + max_send_bitrate_bps_(max_send_bitrate_bps), + rtp_parameters_(CreateRtpParametersWithOneEncoding()) { + RTC_DCHECK(call); + RTC_DCHECK(encoder_factory); + config_.rtp.ssrc = ssrc; + config_.rtp.mid = mid; + config_.rtp.c_name = c_name; + config_.rtp.extmap_allow_mixed = extmap_allow_mixed; + config_.rtp.extensions = extensions; + config_.has_dscp = + rtp_parameters_.encodings[0].network_priority != webrtc::Priority::kLow; + config_.audio_network_adaptor_config = audio_network_adaptor_config; + config_.encoder_factory = encoder_factory; + config_.codec_pair_id = codec_pair_id; + config_.track_id = track_id; + config_.frame_encryptor = frame_encryptor; + config_.crypto_options = crypto_options; + config_.rtcp_report_interval_ms = rtcp_report_interval_ms; + rtp_parameters_.encodings[0].ssrc = ssrc; + rtp_parameters_.rtcp.cname = c_name; + rtp_parameters_.header_extensions = extensions; + + if (send_codec_spec) { + UpdateSendCodecSpec(*send_codec_spec); + } + + stream_ = call_->CreateAudioSendStream(config_); + } + + ~WebRtcAudioSendStream() override { + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + ClearSource(); + call_->DestroyAudioSendStream(stream_); + } + + void SetSendCodecSpec( + const webrtc::AudioSendStream::Config::SendCodecSpec& send_codec_spec) { + UpdateSendCodecSpec(send_codec_spec); + ReconfigureAudioSendStream(); + } + + void SetRtpExtensions(const std::vector<webrtc::RtpExtension>& extensions) { + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + config_.rtp.extensions = extensions; + rtp_parameters_.header_extensions = extensions; + ReconfigureAudioSendStream(); + } + + void SetExtmapAllowMixed(bool extmap_allow_mixed) { + config_.rtp.extmap_allow_mixed = extmap_allow_mixed; + ReconfigureAudioSendStream(); + } + + void SetMid(const std::string& mid) { + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + if (config_.rtp.mid == mid) { + return; + } + config_.rtp.mid = mid; + ReconfigureAudioSendStream(); + } + + void SetFrameEncryptor( + rtc::scoped_refptr<webrtc::FrameEncryptorInterface> frame_encryptor) { + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + config_.frame_encryptor = frame_encryptor; + ReconfigureAudioSendStream(); + } + + void SetAudioNetworkAdaptorConfig( + const absl::optional<std::string>& audio_network_adaptor_config) { + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + if (config_.audio_network_adaptor_config == audio_network_adaptor_config) { + return; + } + config_.audio_network_adaptor_config = audio_network_adaptor_config; + UpdateAllowedBitrateRange(); + ReconfigureAudioSendStream(); + } + + bool SetMaxSendBitrate(int bps) { + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + RTC_DCHECK(config_.send_codec_spec); + RTC_DCHECK(audio_codec_spec_); + auto send_rate = ComputeSendBitrate( + bps, rtp_parameters_.encodings[0].max_bitrate_bps, *audio_codec_spec_); + + if (!send_rate) { + return false; + } + + max_send_bitrate_bps_ = bps; + + if (send_rate != config_.send_codec_spec->target_bitrate_bps) { + config_.send_codec_spec->target_bitrate_bps = send_rate; + ReconfigureAudioSendStream(); + } + return true; + } + + bool SendTelephoneEvent(int payload_type, + int payload_freq, + int event, + int duration_ms) { + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + RTC_DCHECK(stream_); + return stream_->SendTelephoneEvent(payload_type, payload_freq, event, + duration_ms); + } + + void SetSend(bool send) { + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + send_ = send; + UpdateSendState(); + } + + void SetMuted(bool muted) { + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + RTC_DCHECK(stream_); + stream_->SetMuted(muted); + muted_ = muted; + } + + bool muted() const { + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + return muted_; + } + + webrtc::AudioSendStream::Stats GetStats(bool has_remote_tracks) const { + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + RTC_DCHECK(stream_); + return stream_->GetStats(has_remote_tracks); + } + + // Starts the sending by setting ourselves as a sink to the AudioSource to + // get data callbacks. + // This method is called on the libjingle worker thread. + // TODO(xians): Make sure Start() is called only once. + void SetSource(AudioSource* source) { + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + RTC_DCHECK(source); + if (source_) { + RTC_DCHECK(source_ == source); + return; + } + source->SetSink(this); + source_ = source; + UpdateSendState(); + } + + // Stops sending by setting the sink of the AudioSource to nullptr. No data + // callback will be received after this method. + // This method is called on the libjingle worker thread. + void ClearSource() { + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + if (source_) { + source_->SetSink(nullptr); + source_ = nullptr; + } + UpdateSendState(); + } + + // AudioSource::Sink implementation. + // This method is called on the audio thread. + void OnData(const void* audio_data, + int bits_per_sample, + int sample_rate, + size_t number_of_channels, + size_t number_of_frames, + absl::optional<int64_t> absolute_capture_timestamp_ms) override { + RTC_DCHECK_EQ(16, bits_per_sample); + RTC_CHECK_RUNS_SERIALIZED(&audio_capture_race_checker_); + RTC_DCHECK(stream_); + std::unique_ptr<webrtc::AudioFrame> audio_frame(new webrtc::AudioFrame()); + audio_frame->UpdateFrame( + audio_frame->timestamp_, static_cast<const int16_t*>(audio_data), + number_of_frames, sample_rate, audio_frame->speech_type_, + audio_frame->vad_activity_, number_of_channels); + // TODO(bugs.webrtc.org/10739): add dcheck that + // |absolute_capture_timestamp_ms| always receives a value. + if (absolute_capture_timestamp_ms) { + audio_frame->set_absolute_capture_timestamp_ms( + *absolute_capture_timestamp_ms); + } + stream_->SendAudioData(std::move(audio_frame)); + } + + // Callback from the |source_| when it is going away. In case Start() has + // never been called, this callback won't be triggered. + void OnClose() override { + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + // Set |source_| to nullptr to make sure no more callback will get into + // the source. + source_ = nullptr; + UpdateSendState(); + } + + const webrtc::RtpParameters& rtp_parameters() const { + return rtp_parameters_; + } + + webrtc::RTCError SetRtpParameters(const webrtc::RtpParameters& parameters) { + webrtc::RTCError error = CheckRtpParametersInvalidModificationAndValues( + rtp_parameters_, parameters); + if (!error.ok()) { + return error; + } + + absl::optional<int> send_rate; + if (audio_codec_spec_) { + send_rate = ComputeSendBitrate(max_send_bitrate_bps_, + parameters.encodings[0].max_bitrate_bps, + *audio_codec_spec_); + if (!send_rate) { + return webrtc::RTCError(webrtc::RTCErrorType::INTERNAL_ERROR); + } + } + + const absl::optional<int> old_rtp_max_bitrate = + rtp_parameters_.encodings[0].max_bitrate_bps; + double old_priority = rtp_parameters_.encodings[0].bitrate_priority; + webrtc::Priority old_dscp = rtp_parameters_.encodings[0].network_priority; + rtp_parameters_ = parameters; + config_.bitrate_priority = rtp_parameters_.encodings[0].bitrate_priority; + config_.has_dscp = (rtp_parameters_.encodings[0].network_priority != + webrtc::Priority::kLow); + + bool reconfigure_send_stream = + (rtp_parameters_.encodings[0].max_bitrate_bps != old_rtp_max_bitrate) || + (rtp_parameters_.encodings[0].bitrate_priority != old_priority) || + (rtp_parameters_.encodings[0].network_priority != old_dscp); + if (rtp_parameters_.encodings[0].max_bitrate_bps != old_rtp_max_bitrate) { + // Update the bitrate range. + if (send_rate) { + config_.send_codec_spec->target_bitrate_bps = send_rate; + } + UpdateAllowedBitrateRange(); + } + if (reconfigure_send_stream) { + ReconfigureAudioSendStream(); + } + + rtp_parameters_.rtcp.cname = config_.rtp.c_name; + rtp_parameters_.rtcp.reduced_size = false; + + // parameters.encodings[0].active could have changed. + UpdateSendState(); + return webrtc::RTCError::OK(); + } + + void SetEncoderToPacketizerFrameTransformer( + rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer) { + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + config_.frame_transformer = std::move(frame_transformer); + ReconfigureAudioSendStream(); + } + + private: + void UpdateSendState() { + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + RTC_DCHECK(stream_); + RTC_DCHECK_EQ(1UL, rtp_parameters_.encodings.size()); + if (send_ && source_ != nullptr && rtp_parameters_.encodings[0].active) { + stream_->Start(); + } else { // !send || source_ = nullptr + stream_->Stop(); + } + } + + void UpdateAllowedBitrateRange() { + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + // The order of precedence, from lowest to highest is: + // - a reasonable default of 32kbps min/max + // - fixed target bitrate from codec spec + // - bitrate configured in the rtp_parameter encodings settings + const int kDefaultBitrateBps = 32000; + config_.min_bitrate_bps = kDefaultBitrateBps; + config_.max_bitrate_bps = kDefaultBitrateBps; + + if (config_.send_codec_spec && + config_.send_codec_spec->target_bitrate_bps) { + config_.min_bitrate_bps = *config_.send_codec_spec->target_bitrate_bps; + config_.max_bitrate_bps = *config_.send_codec_spec->target_bitrate_bps; + } + + if (rtp_parameters_.encodings[0].min_bitrate_bps) { + config_.min_bitrate_bps = *rtp_parameters_.encodings[0].min_bitrate_bps; + } + if (rtp_parameters_.encodings[0].max_bitrate_bps) { + config_.max_bitrate_bps = *rtp_parameters_.encodings[0].max_bitrate_bps; + } + } + + void UpdateSendCodecSpec( + const webrtc::AudioSendStream::Config::SendCodecSpec& send_codec_spec) { + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + config_.send_codec_spec = send_codec_spec; + auto info = + config_.encoder_factory->QueryAudioEncoder(send_codec_spec.format); + RTC_DCHECK(info); + // If a specific target bitrate has been set for the stream, use that as + // the new default bitrate when computing send bitrate. + if (send_codec_spec.target_bitrate_bps) { + info->default_bitrate_bps = std::max( + info->min_bitrate_bps, + std::min(info->max_bitrate_bps, *send_codec_spec.target_bitrate_bps)); + } + + audio_codec_spec_.emplace( + webrtc::AudioCodecSpec{send_codec_spec.format, *info}); + + config_.send_codec_spec->target_bitrate_bps = ComputeSendBitrate( + max_send_bitrate_bps_, rtp_parameters_.encodings[0].max_bitrate_bps, + *audio_codec_spec_); + + UpdateAllowedBitrateRange(); + } + + void ReconfigureAudioSendStream() { + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + RTC_DCHECK(stream_); + stream_->Reconfigure(config_); + } + + rtc::ThreadChecker worker_thread_checker_; + rtc::RaceChecker audio_capture_race_checker_; + webrtc::Call* call_ = nullptr; + webrtc::AudioSendStream::Config config_; + // The stream is owned by WebRtcAudioSendStream and may be reallocated if + // configuration changes. + webrtc::AudioSendStream* stream_ = nullptr; + + // Raw pointer to AudioSource owned by LocalAudioTrackHandler. + // PeerConnection will make sure invalidating the pointer before the object + // goes away. + AudioSource* source_ = nullptr; + bool send_ = false; + bool muted_ = false; + int max_send_bitrate_bps_; + webrtc::RtpParameters rtp_parameters_; + absl::optional<webrtc::AudioCodecSpec> audio_codec_spec_; + + RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(WebRtcAudioSendStream); +}; + +class WebRtcVoiceMediaChannel::WebRtcAudioReceiveStream { + public: + WebRtcAudioReceiveStream( + uint32_t remote_ssrc, + uint32_t local_ssrc, + bool use_transport_cc, + bool use_nack, + const std::vector<std::string>& stream_ids, + const std::vector<webrtc::RtpExtension>& extensions, + webrtc::Call* call, + webrtc::Transport* rtcp_send_transport, + const rtc::scoped_refptr<webrtc::AudioDecoderFactory>& decoder_factory, + const std::map<int, webrtc::SdpAudioFormat>& decoder_map, + absl::optional<webrtc::AudioCodecPairId> codec_pair_id, + size_t jitter_buffer_max_packets, + bool jitter_buffer_fast_accelerate, + int jitter_buffer_min_delay_ms, + bool jitter_buffer_enable_rtx_handling, + rtc::scoped_refptr<webrtc::FrameDecryptorInterface> frame_decryptor, + const webrtc::CryptoOptions& crypto_options, + rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer) + : call_(call), config_() { + RTC_DCHECK(call); + config_.rtp.remote_ssrc = remote_ssrc; + config_.rtp.local_ssrc = local_ssrc; + config_.rtp.transport_cc = use_transport_cc; + config_.rtp.nack.rtp_history_ms = use_nack ? kNackRtpHistoryMs : 0; + config_.rtp.extensions = extensions; + config_.rtcp_send_transport = rtcp_send_transport; + config_.jitter_buffer_max_packets = jitter_buffer_max_packets; + config_.jitter_buffer_fast_accelerate = jitter_buffer_fast_accelerate; + config_.jitter_buffer_min_delay_ms = jitter_buffer_min_delay_ms; + config_.jitter_buffer_enable_rtx_handling = + jitter_buffer_enable_rtx_handling; + if (!stream_ids.empty()) { + config_.sync_group = stream_ids[0]; + } + config_.decoder_factory = decoder_factory; + config_.decoder_map = decoder_map; + config_.codec_pair_id = codec_pair_id; + config_.frame_decryptor = frame_decryptor; + config_.crypto_options = crypto_options; + config_.frame_transformer = std::move(frame_transformer); + RecreateAudioReceiveStream(); + } + + ~WebRtcAudioReceiveStream() { + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + call_->DestroyAudioReceiveStream(stream_); + } + + void SetFrameDecryptor( + rtc::scoped_refptr<webrtc::FrameDecryptorInterface> frame_decryptor) { + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + config_.frame_decryptor = frame_decryptor; + RecreateAudioReceiveStream(); + } + + void SetLocalSsrc(uint32_t local_ssrc) { + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + if (local_ssrc != config_.rtp.local_ssrc) { + config_.rtp.local_ssrc = local_ssrc; + RecreateAudioReceiveStream(); + } + } + + void SetUseTransportCcAndRecreateStream(bool use_transport_cc, + bool use_nack) { + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + config_.rtp.transport_cc = use_transport_cc; + config_.rtp.nack.rtp_history_ms = use_nack ? kNackRtpHistoryMs : 0; + ReconfigureAudioReceiveStream(); + } + + void SetRtpExtensionsAndRecreateStream( + const std::vector<webrtc::RtpExtension>& extensions) { + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + config_.rtp.extensions = extensions; + RecreateAudioReceiveStream(); + } + + // Set a new payload type -> decoder map. + void SetDecoderMap(const std::map<int, webrtc::SdpAudioFormat>& decoder_map) { + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + config_.decoder_map = decoder_map; + ReconfigureAudioReceiveStream(); + } + + void MaybeRecreateAudioReceiveStream( + const std::vector<std::string>& stream_ids) { + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + std::string sync_group; + if (!stream_ids.empty()) { + sync_group = stream_ids[0]; + } + if (config_.sync_group != sync_group) { + RTC_LOG(LS_INFO) << "Recreating AudioReceiveStream for SSRC=" + << config_.rtp.remote_ssrc + << " because of sync group change."; + config_.sync_group = sync_group; + RecreateAudioReceiveStream(); + } + } + + webrtc::AudioReceiveStream::Stats GetStats() const { + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + RTC_DCHECK(stream_); + return stream_->GetStats(); + } + + void SetRawAudioSink(std::unique_ptr<webrtc::AudioSinkInterface> sink) { + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + // Need to update the stream's sink first; once raw_audio_sink_ is + // reassigned, whatever was in there before is destroyed. + stream_->SetSink(sink.get()); + raw_audio_sink_ = std::move(sink); + } + + void SetOutputVolume(double volume) { + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + output_volume_ = volume; + stream_->SetGain(volume); + } + + void SetPlayout(bool playout) { + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + RTC_DCHECK(stream_); + if (playout) { + stream_->Start(); + } else { + stream_->Stop(); + } + playout_ = playout; + } + + bool SetBaseMinimumPlayoutDelayMs(int delay_ms) { + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + RTC_DCHECK(stream_); + if (stream_->SetBaseMinimumPlayoutDelayMs(delay_ms)) { + // Memorize only valid delay because during stream recreation it will be + // passed to the constructor and it must be valid value. + config_.jitter_buffer_min_delay_ms = delay_ms; + return true; + } else { + RTC_LOG(LS_ERROR) << "Failed to SetBaseMinimumPlayoutDelayMs" + " on AudioReceiveStream on SSRC=" + << config_.rtp.remote_ssrc + << " with delay_ms=" << delay_ms; + return false; + } + } + + int GetBaseMinimumPlayoutDelayMs() const { + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + RTC_DCHECK(stream_); + return stream_->GetBaseMinimumPlayoutDelayMs(); + } + + std::vector<webrtc::RtpSource> GetSources() { + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + RTC_DCHECK(stream_); + return stream_->GetSources(); + } + + webrtc::RtpParameters GetRtpParameters() const { + webrtc::RtpParameters rtp_parameters; + rtp_parameters.encodings.emplace_back(); + rtp_parameters.encodings[0].ssrc = config_.rtp.remote_ssrc; + rtp_parameters.header_extensions = config_.rtp.extensions; + + return rtp_parameters; + } + + void SetDepacketizerToDecoderFrameTransformer( + rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer) { + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + config_.frame_transformer = std::move(frame_transformer); + ReconfigureAudioReceiveStream(); + } + + private: + void RecreateAudioReceiveStream() { + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + if (stream_) { + call_->DestroyAudioReceiveStream(stream_); + } + stream_ = call_->CreateAudioReceiveStream(config_); + RTC_CHECK(stream_); + stream_->SetGain(output_volume_); + SetPlayout(playout_); + stream_->SetSink(raw_audio_sink_.get()); + } + + void ReconfigureAudioReceiveStream() { + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + RTC_DCHECK(stream_); + stream_->Reconfigure(config_); + } + + rtc::ThreadChecker worker_thread_checker_; + webrtc::Call* call_ = nullptr; + webrtc::AudioReceiveStream::Config config_; + // The stream is owned by WebRtcAudioReceiveStream and may be reallocated if + // configuration changes. + webrtc::AudioReceiveStream* stream_ = nullptr; + bool playout_ = false; + float output_volume_ = 1.0; + std::unique_ptr<webrtc::AudioSinkInterface> raw_audio_sink_; + + RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(WebRtcAudioReceiveStream); +}; + +WebRtcVoiceMediaChannel::WebRtcVoiceMediaChannel( + WebRtcVoiceEngine* engine, + const MediaConfig& config, + const AudioOptions& options, + const webrtc::CryptoOptions& crypto_options, + webrtc::Call* call) + : VoiceMediaChannel(config), + engine_(engine), + call_(call), + audio_config_(config.audio), + crypto_options_(crypto_options) { + RTC_LOG(LS_VERBOSE) << "WebRtcVoiceMediaChannel::WebRtcVoiceMediaChannel"; + RTC_DCHECK(call); + engine->RegisterChannel(this); + SetOptions(options); +} + +WebRtcVoiceMediaChannel::~WebRtcVoiceMediaChannel() { + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + RTC_LOG(LS_VERBOSE) << "WebRtcVoiceMediaChannel::~WebRtcVoiceMediaChannel"; + // TODO(solenberg): Should be able to delete the streams directly, without + // going through RemoveNnStream(), once stream objects handle + // all (de)configuration. + while (!send_streams_.empty()) { + RemoveSendStream(send_streams_.begin()->first); + } + while (!recv_streams_.empty()) { + RemoveRecvStream(recv_streams_.begin()->first); + } + engine()->UnregisterChannel(this); +} + +bool WebRtcVoiceMediaChannel::SetSendParameters( + const AudioSendParameters& params) { + TRACE_EVENT0("webrtc", "WebRtcVoiceMediaChannel::SetSendParameters"); + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + RTC_LOG(LS_INFO) << "WebRtcVoiceMediaChannel::SetSendParameters: " + << params.ToString(); + // TODO(pthatcher): Refactor this to be more clean now that we have + // all the information at once. + + if (!SetSendCodecs(params.codecs)) { + return false; + } + + if (!ValidateRtpExtensions(params.extensions)) { + return false; + } + + if (ExtmapAllowMixed() != params.extmap_allow_mixed) { + SetExtmapAllowMixed(params.extmap_allow_mixed); + for (auto& it : send_streams_) { + it.second->SetExtmapAllowMixed(params.extmap_allow_mixed); + } + } + + std::vector<webrtc::RtpExtension> filtered_extensions = FilterRtpExtensions( + params.extensions, webrtc::RtpExtension::IsSupportedForAudio, true); + if (send_rtp_extensions_ != filtered_extensions) { + send_rtp_extensions_.swap(filtered_extensions); + for (auto& it : send_streams_) { + it.second->SetRtpExtensions(send_rtp_extensions_); + } + } + if (!params.mid.empty()) { + mid_ = params.mid; + for (auto& it : send_streams_) { + it.second->SetMid(params.mid); + } + } + + if (!SetMaxSendBitrate(params.max_bandwidth_bps)) { + return false; + } + return SetOptions(params.options); +} + +bool WebRtcVoiceMediaChannel::SetRecvParameters( + const AudioRecvParameters& params) { + TRACE_EVENT0("webrtc", "WebRtcVoiceMediaChannel::SetRecvParameters"); + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + RTC_LOG(LS_INFO) << "WebRtcVoiceMediaChannel::SetRecvParameters: " + << params.ToString(); + // TODO(pthatcher): Refactor this to be more clean now that we have + // all the information at once. + + if (!SetRecvCodecs(params.codecs)) { + return false; + } + + if (!ValidateRtpExtensions(params.extensions)) { + return false; + } + std::vector<webrtc::RtpExtension> filtered_extensions = FilterRtpExtensions( + params.extensions, webrtc::RtpExtension::IsSupportedForAudio, false); + if (recv_rtp_extensions_ != filtered_extensions) { + recv_rtp_extensions_.swap(filtered_extensions); + for (auto& it : recv_streams_) { + it.second->SetRtpExtensionsAndRecreateStream(recv_rtp_extensions_); + } + } + return true; +} + +webrtc::RtpParameters WebRtcVoiceMediaChannel::GetRtpSendParameters( + uint32_t ssrc) const { + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + auto it = send_streams_.find(ssrc); + if (it == send_streams_.end()) { + RTC_LOG(LS_WARNING) << "Attempting to get RTP send parameters for stream " + "with ssrc " + << ssrc << " which doesn't exist."; + return webrtc::RtpParameters(); + } + + webrtc::RtpParameters rtp_params = it->second->rtp_parameters(); + // Need to add the common list of codecs to the send stream-specific + // RTP parameters. + for (const AudioCodec& codec : send_codecs_) { + rtp_params.codecs.push_back(codec.ToCodecParameters()); + } + return rtp_params; +} + +webrtc::RTCError WebRtcVoiceMediaChannel::SetRtpSendParameters( + uint32_t ssrc, + const webrtc::RtpParameters& parameters) { + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + auto it = send_streams_.find(ssrc); + if (it == send_streams_.end()) { + RTC_LOG(LS_WARNING) << "Attempting to set RTP send parameters for stream " + "with ssrc " + << ssrc << " which doesn't exist."; + return webrtc::RTCError(webrtc::RTCErrorType::INTERNAL_ERROR); + } + + // TODO(deadbeef): Handle setting parameters with a list of codecs in a + // different order (which should change the send codec). + webrtc::RtpParameters current_parameters = GetRtpSendParameters(ssrc); + if (current_parameters.codecs != parameters.codecs) { + RTC_DLOG(LS_ERROR) << "Using SetParameters to change the set of codecs " + "is not currently supported."; + return webrtc::RTCError(webrtc::RTCErrorType::UNSUPPORTED_PARAMETER); + } + + if (!parameters.encodings.empty()) { + // Note that these values come from: + // https://tools.ietf.org/html/draft-ietf-tsvwg-rtcweb-qos-16#section-5 + rtc::DiffServCodePoint new_dscp = rtc::DSCP_DEFAULT; + switch (parameters.encodings[0].network_priority) { + case webrtc::Priority::kVeryLow: + new_dscp = rtc::DSCP_CS1; + break; + case webrtc::Priority::kLow: + new_dscp = rtc::DSCP_DEFAULT; + break; + case webrtc::Priority::kMedium: + new_dscp = rtc::DSCP_EF; + break; + case webrtc::Priority::kHigh: + new_dscp = rtc::DSCP_EF; + break; + } + SetPreferredDscp(new_dscp); + } + + // TODO(minyue): The following legacy actions go into + // |WebRtcAudioSendStream::SetRtpParameters()| which is called at the end, + // though there are two difference: + // 1. |WebRtcVoiceMediaChannel::SetChannelSendParameters()| only calls + // |SetSendCodec| while |WebRtcAudioSendStream::SetRtpParameters()| calls + // |SetSendCodecs|. The outcome should be the same. + // 2. AudioSendStream can be recreated. + + // Codecs are handled at the WebRtcVoiceMediaChannel level. + webrtc::RtpParameters reduced_params = parameters; + reduced_params.codecs.clear(); + return it->second->SetRtpParameters(reduced_params); +} + +webrtc::RtpParameters WebRtcVoiceMediaChannel::GetRtpReceiveParameters( + uint32_t ssrc) const { + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + webrtc::RtpParameters rtp_params; + auto it = recv_streams_.find(ssrc); + if (it == recv_streams_.end()) { + RTC_LOG(LS_WARNING) + << "Attempting to get RTP receive parameters for stream " + "with ssrc " + << ssrc << " which doesn't exist."; + return webrtc::RtpParameters(); + } + rtp_params = it->second->GetRtpParameters(); + + for (const AudioCodec& codec : recv_codecs_) { + rtp_params.codecs.push_back(codec.ToCodecParameters()); + } + return rtp_params; +} + +webrtc::RtpParameters WebRtcVoiceMediaChannel::GetDefaultRtpReceiveParameters() + const { + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + webrtc::RtpParameters rtp_params; + if (!default_sink_) { + RTC_LOG(LS_WARNING) << "Attempting to get RTP parameters for the default, " + "unsignaled audio receive stream, but not yet " + "configured to receive such a stream."; + return rtp_params; + } + rtp_params.encodings.emplace_back(); + + for (const AudioCodec& codec : recv_codecs_) { + rtp_params.codecs.push_back(codec.ToCodecParameters()); + } + return rtp_params; +} + +bool WebRtcVoiceMediaChannel::SetOptions(const AudioOptions& options) { + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + RTC_LOG(LS_INFO) << "Setting voice channel options: " << options.ToString(); + + // We retain all of the existing options, and apply the given ones + // on top. This means there is no way to "clear" options such that + // they go back to the engine default. + options_.SetAll(options); + if (!engine()->ApplyOptions(options_)) { + RTC_LOG(LS_WARNING) + << "Failed to apply engine options during channel SetOptions."; + return false; + } + + absl::optional<std::string> audio_network_adaptor_config = + GetAudioNetworkAdaptorConfig(options_); + for (auto& it : send_streams_) { + it.second->SetAudioNetworkAdaptorConfig(audio_network_adaptor_config); + } + + RTC_LOG(LS_INFO) << "Set voice channel options. Current options: " + << options_.ToString(); + return true; +} + +bool WebRtcVoiceMediaChannel::SetRecvCodecs( + const std::vector<AudioCodec>& codecs) { + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + + // Set the payload types to be used for incoming media. + RTC_LOG(LS_INFO) << "Setting receive voice codecs."; + + if (!VerifyUniquePayloadTypes(codecs)) { + RTC_LOG(LS_ERROR) << "Codec payload types overlap."; + return false; + } + + // Create a payload type -> SdpAudioFormat map with all the decoders. Fail + // unless the factory claims to support all decoders. + std::map<int, webrtc::SdpAudioFormat> decoder_map; + for (const AudioCodec& codec : codecs) { + // Log a warning if a codec's payload type is changing. This used to be + // treated as an error. It's abnormal, but not really illegal. + AudioCodec old_codec; + if (FindCodec(recv_codecs_, codec, &old_codec) && + old_codec.id != codec.id) { + RTC_LOG(LS_WARNING) << codec.name << " mapped to a second payload type (" + << codec.id << ", was already mapped to " + << old_codec.id << ")"; + } + auto format = AudioCodecToSdpAudioFormat(codec); + if (!IsCodec(codec, "cn") && !IsCodec(codec, "telephone-event") && + !engine()->decoder_factory_->IsSupportedDecoder(format)) { + RTC_LOG(LS_ERROR) << "Unsupported codec: " << rtc::ToString(format); + return false; + } + // We allow adding new codecs but don't allow changing the payload type of + // codecs that are already configured since we might already be receiving + // packets with that payload type. See RFC3264, Section 8.3.2. + // TODO(deadbeef): Also need to check for clashes with previously mapped + // payload types, and not just currently mapped ones. For example, this + // should be illegal: + // 1. {100: opus/48000/2, 101: ISAC/16000} + // 2. {100: opus/48000/2} + // 3. {100: opus/48000/2, 101: ISAC/32000} + // Though this check really should happen at a higher level, since this + // conflict could happen between audio and video codecs. + auto existing = decoder_map_.find(codec.id); + if (existing != decoder_map_.end() && !existing->second.Matches(format)) { + RTC_LOG(LS_ERROR) << "Attempting to use payload type " << codec.id + << " for " << codec.name + << ", but it is already used for " + << existing->second.name; + return false; + } + decoder_map.insert({codec.id, std::move(format)}); + } + + if (decoder_map == decoder_map_) { + // There's nothing new to configure. + return true; + } + + if (playout_) { + // Receive codecs can not be changed while playing. So we temporarily + // pause playout. + ChangePlayout(false); + } + + decoder_map_ = std::move(decoder_map); + for (auto& kv : recv_streams_) { + kv.second->SetDecoderMap(decoder_map_); + } + recv_codecs_ = codecs; + + if (desired_playout_ && !playout_) { + ChangePlayout(desired_playout_); + } + return true; +} + +// Utility function called from SetSendParameters() to extract current send +// codec settings from the given list of codecs (originally from SDP). Both send +// and receive streams may be reconfigured based on the new settings. +bool WebRtcVoiceMediaChannel::SetSendCodecs( + const std::vector<AudioCodec>& codecs) { + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + dtmf_payload_type_ = absl::nullopt; + dtmf_payload_freq_ = -1; + + // Validate supplied codecs list. + for (const AudioCodec& codec : codecs) { + // TODO(solenberg): Validate more aspects of input - that payload types + // don't overlap, remove redundant/unsupported codecs etc - + // the same way it is done for RtpHeaderExtensions. + if (codec.id < kMinPayloadType || codec.id > kMaxPayloadType) { + RTC_LOG(LS_WARNING) << "Codec payload type out of range: " + << ToString(codec); + return false; + } + } + + // Find PT of telephone-event codec with lowest clockrate, as a fallback, in + // case we don't have a DTMF codec with a rate matching the send codec's, or + // if this function returns early. + std::vector<AudioCodec> dtmf_codecs; + for (const AudioCodec& codec : codecs) { + if (IsCodec(codec, kDtmfCodecName)) { + dtmf_codecs.push_back(codec); + if (!dtmf_payload_type_ || codec.clockrate < dtmf_payload_freq_) { + dtmf_payload_type_ = codec.id; + dtmf_payload_freq_ = codec.clockrate; + } + } + } + + // Scan through the list to figure out the codec to use for sending. + absl::optional<webrtc::AudioSendStream::Config::SendCodecSpec> + send_codec_spec; + webrtc::BitrateConstraints bitrate_config; + absl::optional<webrtc::AudioCodecInfo> voice_codec_info; + for (const AudioCodec& voice_codec : codecs) { + if (!(IsCodec(voice_codec, kCnCodecName) || + IsCodec(voice_codec, kDtmfCodecName) || + IsCodec(voice_codec, kRedCodecName))) { + webrtc::SdpAudioFormat format(voice_codec.name, voice_codec.clockrate, + voice_codec.channels, voice_codec.params); + + voice_codec_info = engine()->encoder_factory_->QueryAudioEncoder(format); + if (!voice_codec_info) { + RTC_LOG(LS_WARNING) << "Unknown codec " << ToString(voice_codec); + continue; + } + + send_codec_spec = webrtc::AudioSendStream::Config::SendCodecSpec( + voice_codec.id, format); + if (voice_codec.bitrate > 0) { + send_codec_spec->target_bitrate_bps = voice_codec.bitrate; + } + send_codec_spec->transport_cc_enabled = HasTransportCc(voice_codec); + send_codec_spec->nack_enabled = HasNack(voice_codec); + bitrate_config = GetBitrateConfigForCodec(voice_codec); + break; + } + } + + if (!send_codec_spec) { + return false; + } + + RTC_DCHECK(voice_codec_info); + if (voice_codec_info->allow_comfort_noise) { + // Loop through the codecs list again to find the CN codec. + // TODO(solenberg): Break out into a separate function? + for (const AudioCodec& cn_codec : codecs) { + if (IsCodec(cn_codec, kCnCodecName) && + cn_codec.clockrate == send_codec_spec->format.clockrate_hz && + cn_codec.channels == voice_codec_info->num_channels) { + if (cn_codec.channels != 1) { + RTC_LOG(LS_WARNING) + << "CN #channels " << cn_codec.channels << " not supported."; + } else if (cn_codec.clockrate != 8000 && cn_codec.clockrate != 16000 && + cn_codec.clockrate != 32000) { + RTC_LOG(LS_WARNING) + << "CN frequency " << cn_codec.clockrate << " not supported."; + } else { + send_codec_spec->cng_payload_type = cn_codec.id; + } + break; + } + } + + // Find the telephone-event PT exactly matching the preferred send codec. + for (const AudioCodec& dtmf_codec : dtmf_codecs) { + if (dtmf_codec.clockrate == send_codec_spec->format.clockrate_hz) { + dtmf_payload_type_ = dtmf_codec.id; + dtmf_payload_freq_ = dtmf_codec.clockrate; + break; + } + } + } + + if (send_codec_spec_ != send_codec_spec) { + send_codec_spec_ = std::move(send_codec_spec); + // Apply new settings to all streams. + for (const auto& kv : send_streams_) { + kv.second->SetSendCodecSpec(*send_codec_spec_); + } + } else { + // If the codec isn't changing, set the start bitrate to -1 which means + // "unchanged" so that BWE isn't affected. + bitrate_config.start_bitrate_bps = -1; + } + call_->GetTransportControllerSend()->SetSdpBitrateParameters(bitrate_config); + + // Check if the transport cc feedback or NACK status has changed on the + // preferred send codec, and in that case reconfigure all receive streams. + if (recv_transport_cc_enabled_ != send_codec_spec_->transport_cc_enabled || + recv_nack_enabled_ != send_codec_spec_->nack_enabled) { + RTC_LOG(LS_INFO) << "Recreate all the receive streams because the send " + "codec has changed."; + recv_transport_cc_enabled_ = send_codec_spec_->transport_cc_enabled; + recv_nack_enabled_ = send_codec_spec_->nack_enabled; + for (auto& kv : recv_streams_) { + kv.second->SetUseTransportCcAndRecreateStream(recv_transport_cc_enabled_, + recv_nack_enabled_); + } + } + + send_codecs_ = codecs; + return true; +} + +void WebRtcVoiceMediaChannel::SetPlayout(bool playout) { + desired_playout_ = playout; + return ChangePlayout(desired_playout_); +} + +void WebRtcVoiceMediaChannel::ChangePlayout(bool playout) { + TRACE_EVENT0("webrtc", "WebRtcVoiceMediaChannel::ChangePlayout"); + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + if (playout_ == playout) { + return; + } + + for (const auto& kv : recv_streams_) { + kv.second->SetPlayout(playout); + } + playout_ = playout; +} + +void WebRtcVoiceMediaChannel::SetSend(bool send) { + TRACE_EVENT0("webrtc", "WebRtcVoiceMediaChannel::SetSend"); + if (send_ == send) { + return; + } + + // Apply channel specific options, and initialize the ADM for recording (this + // may take time on some platforms, e.g. Android). + if (send) { + engine()->ApplyOptions(options_); + + // InitRecording() may return an error if the ADM is already recording. + if (!engine()->adm()->RecordingIsInitialized() && + !engine()->adm()->Recording()) { + if (engine()->adm()->InitRecording() != 0) { + RTC_LOG(LS_WARNING) << "Failed to initialize recording"; + } + } + } + + // Change the settings on each send channel. + for (auto& kv : send_streams_) { + kv.second->SetSend(send); + } + + send_ = send; +} + +bool WebRtcVoiceMediaChannel::SetAudioSend(uint32_t ssrc, + bool enable, + const AudioOptions* options, + AudioSource* source) { + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + // TODO(solenberg): The state change should be fully rolled back if any one of + // these calls fail. + if (!SetLocalSource(ssrc, source)) { + return false; + } + if (!MuteStream(ssrc, !enable)) { + return false; + } + if (enable && options) { + return SetOptions(*options); + } + return true; +} + +bool WebRtcVoiceMediaChannel::AddSendStream(const StreamParams& sp) { + TRACE_EVENT0("webrtc", "WebRtcVoiceMediaChannel::AddSendStream"); + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + RTC_LOG(LS_INFO) << "AddSendStream: " << sp.ToString(); + + uint32_t ssrc = sp.first_ssrc(); + RTC_DCHECK(0 != ssrc); + + if (send_streams_.find(ssrc) != send_streams_.end()) { + RTC_LOG(LS_ERROR) << "Stream already exists with ssrc " << ssrc; + return false; + } + + absl::optional<std::string> audio_network_adaptor_config = + GetAudioNetworkAdaptorConfig(options_); + WebRtcAudioSendStream* stream = new WebRtcAudioSendStream( + ssrc, mid_, sp.cname, sp.id, send_codec_spec_, ExtmapAllowMixed(), + send_rtp_extensions_, max_send_bitrate_bps_, + audio_config_.rtcp_report_interval_ms, audio_network_adaptor_config, + call_, this, engine()->encoder_factory_, codec_pair_id_, nullptr, + crypto_options_); + send_streams_.insert(std::make_pair(ssrc, stream)); + + // At this point the stream's local SSRC has been updated. If it is the first + // send stream, make sure that all the receive streams are updated with the + // same SSRC in order to send receiver reports. + if (send_streams_.size() == 1) { + receiver_reports_ssrc_ = ssrc; + for (const auto& kv : recv_streams_) { + // TODO(solenberg): Allow applications to set the RTCP SSRC of receive + // streams instead, so we can avoid reconfiguring the streams here. + kv.second->SetLocalSsrc(ssrc); + } + } + + send_streams_[ssrc]->SetSend(send_); + return true; +} + +bool WebRtcVoiceMediaChannel::RemoveSendStream(uint32_t ssrc) { + TRACE_EVENT0("webrtc", "WebRtcVoiceMediaChannel::RemoveSendStream"); + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + RTC_LOG(LS_INFO) << "RemoveSendStream: " << ssrc; + + auto it = send_streams_.find(ssrc); + if (it == send_streams_.end()) { + RTC_LOG(LS_WARNING) << "Try to remove stream with ssrc " << ssrc + << " which doesn't exist."; + return false; + } + + it->second->SetSend(false); + + // TODO(solenberg): If we're removing the receiver_reports_ssrc_ stream, find + // the first active send stream and use that instead, reassociating receive + // streams. + + delete it->second; + send_streams_.erase(it); + if (send_streams_.empty()) { + SetSend(false); + } + return true; +} + +bool WebRtcVoiceMediaChannel::AddRecvStream(const StreamParams& sp) { + TRACE_EVENT0("webrtc", "WebRtcVoiceMediaChannel::AddRecvStream"); + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + RTC_LOG(LS_INFO) << "AddRecvStream: " << sp.ToString(); + + if (!sp.has_ssrcs()) { + // This is a StreamParam with unsignaled SSRCs. Store it, so it can be used + // later when we know the SSRCs on the first packet arrival. + unsignaled_stream_params_ = sp; + return true; + } + + if (!ValidateStreamParams(sp)) { + return false; + } + + const uint32_t ssrc = sp.first_ssrc(); + + // If this stream was previously received unsignaled, we promote it, possibly + // recreating the AudioReceiveStream, if stream ids have changed. + if (MaybeDeregisterUnsignaledRecvStream(ssrc)) { + recv_streams_[ssrc]->MaybeRecreateAudioReceiveStream(sp.stream_ids()); + return true; + } + + if (recv_streams_.find(ssrc) != recv_streams_.end()) { + RTC_LOG(LS_ERROR) << "Stream already exists with ssrc " << ssrc; + return false; + } + + // Create a new channel for receiving audio data. + recv_streams_.insert(std::make_pair( + ssrc, new WebRtcAudioReceiveStream( + ssrc, receiver_reports_ssrc_, recv_transport_cc_enabled_, + recv_nack_enabled_, sp.stream_ids(), recv_rtp_extensions_, + call_, this, engine()->decoder_factory_, decoder_map_, + codec_pair_id_, engine()->audio_jitter_buffer_max_packets_, + engine()->audio_jitter_buffer_fast_accelerate_, + engine()->audio_jitter_buffer_min_delay_ms_, + engine()->audio_jitter_buffer_enable_rtx_handling_, + unsignaled_frame_decryptor_, crypto_options_, nullptr))); + recv_streams_[ssrc]->SetPlayout(playout_); + + return true; +} + +bool WebRtcVoiceMediaChannel::RemoveRecvStream(uint32_t ssrc) { + TRACE_EVENT0("webrtc", "WebRtcVoiceMediaChannel::RemoveRecvStream"); + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + RTC_LOG(LS_INFO) << "RemoveRecvStream: " << ssrc; + + const auto it = recv_streams_.find(ssrc); + if (it == recv_streams_.end()) { + RTC_LOG(LS_WARNING) << "Try to remove stream with ssrc " << ssrc + << " which doesn't exist."; + return false; + } + + MaybeDeregisterUnsignaledRecvStream(ssrc); + + it->second->SetRawAudioSink(nullptr); + delete it->second; + recv_streams_.erase(it); + return true; +} + +void WebRtcVoiceMediaChannel::ResetUnsignaledRecvStream() { + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + RTC_LOG(LS_INFO) << "ResetUnsignaledRecvStream."; + unsignaled_stream_params_ = StreamParams(); +} + +bool WebRtcVoiceMediaChannel::SetLocalSource(uint32_t ssrc, + AudioSource* source) { + auto it = send_streams_.find(ssrc); + if (it == send_streams_.end()) { + if (source) { + // Return an error if trying to set a valid source with an invalid ssrc. + RTC_LOG(LS_ERROR) << "SetLocalSource failed with ssrc " << ssrc; + return false; + } + + // The channel likely has gone away, do nothing. + return true; + } + + if (source) { + it->second->SetSource(source); + } else { + it->second->ClearSource(); + } + + return true; +} + +bool WebRtcVoiceMediaChannel::SetOutputVolume(uint32_t ssrc, double volume) { + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + const auto it = recv_streams_.find(ssrc); + if (it == recv_streams_.end()) { + RTC_LOG(LS_WARNING) << "SetOutputVolume: no recv stream " << ssrc; + return false; + } + it->second->SetOutputVolume(volume); + RTC_LOG(LS_INFO) << "SetOutputVolume() to " << volume + << " for recv stream with ssrc " << ssrc; + return true; +} + +bool WebRtcVoiceMediaChannel::SetDefaultOutputVolume(double volume) { + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + default_recv_volume_ = volume; + for (uint32_t ssrc : unsignaled_recv_ssrcs_) { + const auto it = recv_streams_.find(ssrc); + if (it == recv_streams_.end()) { + RTC_LOG(LS_WARNING) << "SetDefaultOutputVolume: no recv stream " << ssrc; + return false; + } + it->second->SetOutputVolume(volume); + RTC_LOG(LS_INFO) << "SetDefaultOutputVolume() to " << volume + << " for recv stream with ssrc " << ssrc; + } + return true; +} + +bool WebRtcVoiceMediaChannel::SetBaseMinimumPlayoutDelayMs(uint32_t ssrc, + int delay_ms) { + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + std::vector<uint32_t> ssrcs(1, ssrc); + // SSRC of 0 represents the default receive stream. + if (ssrc == 0) { + default_recv_base_minimum_delay_ms_ = delay_ms; + ssrcs = unsignaled_recv_ssrcs_; + } + for (uint32_t ssrc : ssrcs) { + const auto it = recv_streams_.find(ssrc); + if (it == recv_streams_.end()) { + RTC_LOG(LS_WARNING) << "SetBaseMinimumPlayoutDelayMs: no recv stream " + << ssrc; + return false; + } + it->second->SetBaseMinimumPlayoutDelayMs(delay_ms); + RTC_LOG(LS_INFO) << "SetBaseMinimumPlayoutDelayMs() to " << delay_ms + << " for recv stream with ssrc " << ssrc; + } + return true; +} + +absl::optional<int> WebRtcVoiceMediaChannel::GetBaseMinimumPlayoutDelayMs( + uint32_t ssrc) const { + // SSRC of 0 represents the default receive stream. + if (ssrc == 0) { + return default_recv_base_minimum_delay_ms_; + } + + const auto it = recv_streams_.find(ssrc); + + if (it != recv_streams_.end()) { + return it->second->GetBaseMinimumPlayoutDelayMs(); + } + return absl::nullopt; +} + +bool WebRtcVoiceMediaChannel::CanInsertDtmf() { + return dtmf_payload_type_.has_value() && send_; +} + +void WebRtcVoiceMediaChannel::SetFrameDecryptor( + uint32_t ssrc, + rtc::scoped_refptr<webrtc::FrameDecryptorInterface> frame_decryptor) { + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + auto matching_stream = recv_streams_.find(ssrc); + if (matching_stream != recv_streams_.end()) { + matching_stream->second->SetFrameDecryptor(frame_decryptor); + } + // Handle unsignaled frame decryptors. + if (ssrc == 0) { + unsignaled_frame_decryptor_ = frame_decryptor; + } +} + +void WebRtcVoiceMediaChannel::SetFrameEncryptor( + uint32_t ssrc, + rtc::scoped_refptr<webrtc::FrameEncryptorInterface> frame_encryptor) { + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + auto matching_stream = send_streams_.find(ssrc); + if (matching_stream != send_streams_.end()) { + matching_stream->second->SetFrameEncryptor(frame_encryptor); + } +} + +bool WebRtcVoiceMediaChannel::InsertDtmf(uint32_t ssrc, + int event, + int duration) { + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + RTC_LOG(LS_INFO) << "WebRtcVoiceMediaChannel::InsertDtmf"; + if (!CanInsertDtmf()) { + return false; + } + + // Figure out which WebRtcAudioSendStream to send the event on. + auto it = ssrc != 0 ? send_streams_.find(ssrc) : send_streams_.begin(); + if (it == send_streams_.end()) { + RTC_LOG(LS_WARNING) << "The specified ssrc " << ssrc << " is not in use."; + return false; + } + if (event < kMinTelephoneEventCode || event > kMaxTelephoneEventCode) { + RTC_LOG(LS_WARNING) << "DTMF event code " << event << " out of range."; + return false; + } + RTC_DCHECK_NE(-1, dtmf_payload_freq_); + return it->second->SendTelephoneEvent(*dtmf_payload_type_, dtmf_payload_freq_, + event, duration); +} + +void WebRtcVoiceMediaChannel::OnPacketReceived(rtc::CopyOnWriteBuffer packet, + int64_t packet_time_us) { + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + + webrtc::PacketReceiver::DeliveryStatus delivery_result = + call_->Receiver()->DeliverPacket(webrtc::MediaType::AUDIO, packet, + packet_time_us); + + if (delivery_result != webrtc::PacketReceiver::DELIVERY_UNKNOWN_SSRC) { + return; + } + + // Create an unsignaled receive stream for this previously not received ssrc. + // If there already is N unsignaled receive streams, delete the oldest. + // See: https://bugs.chromium.org/p/webrtc/issues/detail?id=5208 + uint32_t ssrc = 0; + if (!GetRtpSsrc(packet.cdata(), packet.size(), &ssrc)) { + return; + } + RTC_DCHECK(!absl::c_linear_search(unsignaled_recv_ssrcs_, ssrc)); + + // Add new stream. + StreamParams sp = unsignaled_stream_params_; + sp.ssrcs.push_back(ssrc); + RTC_LOG(LS_INFO) << "Creating unsignaled receive stream for SSRC=" << ssrc; + if (!AddRecvStream(sp)) { + RTC_LOG(LS_WARNING) << "Could not create unsignaled receive stream."; + return; + } + unsignaled_recv_ssrcs_.push_back(ssrc); + RTC_HISTOGRAM_COUNTS_LINEAR("WebRTC.Audio.NumOfUnsignaledStreams", + unsignaled_recv_ssrcs_.size(), 1, 100, 101); + + // Remove oldest unsignaled stream, if we have too many. + if (unsignaled_recv_ssrcs_.size() > kMaxUnsignaledRecvStreams) { + uint32_t remove_ssrc = unsignaled_recv_ssrcs_.front(); + RTC_DLOG(LS_INFO) << "Removing unsignaled receive stream with SSRC=" + << remove_ssrc; + RemoveRecvStream(remove_ssrc); + } + RTC_DCHECK_GE(kMaxUnsignaledRecvStreams, unsignaled_recv_ssrcs_.size()); + + SetOutputVolume(ssrc, default_recv_volume_); + SetBaseMinimumPlayoutDelayMs(ssrc, default_recv_base_minimum_delay_ms_); + + // The default sink can only be attached to one stream at a time, so we hook + // it up to the *latest* unsignaled stream we've seen, in order to support the + // case where the SSRC of one unsignaled stream changes. + if (default_sink_) { + for (uint32_t drop_ssrc : unsignaled_recv_ssrcs_) { + auto it = recv_streams_.find(drop_ssrc); + it->second->SetRawAudioSink(nullptr); + } + std::unique_ptr<webrtc::AudioSinkInterface> proxy_sink( + new ProxySink(default_sink_.get())); + SetRawAudioSink(ssrc, std::move(proxy_sink)); + } + + delivery_result = call_->Receiver()->DeliverPacket(webrtc::MediaType::AUDIO, + packet, packet_time_us); + RTC_DCHECK_NE(webrtc::PacketReceiver::DELIVERY_UNKNOWN_SSRC, delivery_result); +} + +void WebRtcVoiceMediaChannel::OnNetworkRouteChanged( + const std::string& transport_name, + const rtc::NetworkRoute& network_route) { + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + call_->GetTransportControllerSend()->OnNetworkRouteChanged(transport_name, + network_route); + call_->OnAudioTransportOverheadChanged(network_route.packet_overhead); +} + +bool WebRtcVoiceMediaChannel::MuteStream(uint32_t ssrc, bool muted) { + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + const auto it = send_streams_.find(ssrc); + if (it == send_streams_.end()) { + RTC_LOG(LS_WARNING) << "The specified ssrc " << ssrc << " is not in use."; + return false; + } + it->second->SetMuted(muted); + + // TODO(solenberg): + // We set the AGC to mute state only when all the channels are muted. + // This implementation is not ideal, instead we should signal the AGC when + // the mic channel is muted/unmuted. We can't do it today because there + // is no good way to know which stream is mapping to the mic channel. + bool all_muted = muted; + for (const auto& kv : send_streams_) { + all_muted = all_muted && kv.second->muted(); + } + webrtc::AudioProcessing* ap = engine()->apm(); + if (ap) { + ap->set_output_will_be_muted(all_muted); + } + + return true; +} + +bool WebRtcVoiceMediaChannel::SetMaxSendBitrate(int bps) { + RTC_LOG(LS_INFO) << "WebRtcVoiceMediaChannel::SetMaxSendBitrate."; + max_send_bitrate_bps_ = bps; + bool success = true; + for (const auto& kv : send_streams_) { + if (!kv.second->SetMaxSendBitrate(max_send_bitrate_bps_)) { + success = false; + } + } + return success; +} + +void WebRtcVoiceMediaChannel::OnReadyToSend(bool ready) { + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + RTC_LOG(LS_VERBOSE) << "OnReadyToSend: " << (ready ? "Ready." : "Not ready."); + call_->SignalChannelNetworkState( + webrtc::MediaType::AUDIO, + ready ? webrtc::kNetworkUp : webrtc::kNetworkDown); +} + +bool WebRtcVoiceMediaChannel::GetStats(VoiceMediaInfo* info) { + TRACE_EVENT0("webrtc", "WebRtcVoiceMediaChannel::GetStats"); + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + RTC_DCHECK(info); + + // Get SSRC and stats for each sender. + RTC_DCHECK_EQ(info->senders.size(), 0U); + for (const auto& stream : send_streams_) { + webrtc::AudioSendStream::Stats stats = + stream.second->GetStats(recv_streams_.size() > 0); + VoiceSenderInfo sinfo; + sinfo.add_ssrc(stats.local_ssrc); + sinfo.payload_bytes_sent = stats.payload_bytes_sent; + sinfo.header_and_padding_bytes_sent = stats.header_and_padding_bytes_sent; + sinfo.retransmitted_bytes_sent = stats.retransmitted_bytes_sent; + sinfo.packets_sent = stats.packets_sent; + sinfo.retransmitted_packets_sent = stats.retransmitted_packets_sent; + sinfo.packets_lost = stats.packets_lost; + sinfo.fraction_lost = stats.fraction_lost; + sinfo.codec_name = stats.codec_name; + sinfo.codec_payload_type = stats.codec_payload_type; + sinfo.jitter_ms = stats.jitter_ms; + sinfo.rtt_ms = stats.rtt_ms; + sinfo.audio_level = stats.audio_level; + sinfo.total_input_energy = stats.total_input_energy; + sinfo.total_input_duration = stats.total_input_duration; + sinfo.typing_noise_detected = (send_ ? stats.typing_noise_detected : false); + sinfo.ana_statistics = stats.ana_statistics; + sinfo.apm_statistics = stats.apm_statistics; + sinfo.report_block_datas = std::move(stats.report_block_datas); + info->senders.push_back(sinfo); + } + + // Get SSRC and stats for each receiver. + RTC_DCHECK_EQ(info->receivers.size(), 0U); + for (const auto& stream : recv_streams_) { + uint32_t ssrc = stream.first; + // When SSRCs are unsignaled, there's only one audio MediaStreamTrack, but + // multiple RTP streams can be received over time (if the SSRC changes for + // whatever reason). We only want the RTCMediaStreamTrackStats to represent + // the stats for the most recent stream (the one whose audio is actually + // routed to the MediaStreamTrack), so here we ignore any unsignaled SSRCs + // except for the most recent one (last in the vector). This is somewhat of + // a hack, and means you don't get *any* stats for these inactive streams, + // but it's slightly better than the previous behavior, which was "highest + // SSRC wins". + // See: https://bugs.chromium.org/p/webrtc/issues/detail?id=8158 + if (!unsignaled_recv_ssrcs_.empty()) { + auto end_it = --unsignaled_recv_ssrcs_.end(); + if (absl::linear_search(unsignaled_recv_ssrcs_.begin(), end_it, ssrc)) { + continue; + } + } + webrtc::AudioReceiveStream::Stats stats = stream.second->GetStats(); + VoiceReceiverInfo rinfo; + rinfo.add_ssrc(stats.remote_ssrc); + rinfo.payload_bytes_rcvd = stats.payload_bytes_rcvd; + rinfo.header_and_padding_bytes_rcvd = stats.header_and_padding_bytes_rcvd; + rinfo.packets_rcvd = stats.packets_rcvd; + rinfo.fec_packets_received = stats.fec_packets_received; + rinfo.fec_packets_discarded = stats.fec_packets_discarded; + rinfo.packets_lost = stats.packets_lost; + rinfo.codec_name = stats.codec_name; + rinfo.codec_payload_type = stats.codec_payload_type; + rinfo.jitter_ms = stats.jitter_ms; + rinfo.jitter_buffer_ms = stats.jitter_buffer_ms; + rinfo.jitter_buffer_preferred_ms = stats.jitter_buffer_preferred_ms; + rinfo.delay_estimate_ms = stats.delay_estimate_ms; + rinfo.audio_level = stats.audio_level; + rinfo.total_output_energy = stats.total_output_energy; + rinfo.total_samples_received = stats.total_samples_received; + rinfo.total_output_duration = stats.total_output_duration; + rinfo.concealed_samples = stats.concealed_samples; + rinfo.silent_concealed_samples = stats.silent_concealed_samples; + rinfo.concealment_events = stats.concealment_events; + rinfo.jitter_buffer_delay_seconds = stats.jitter_buffer_delay_seconds; + rinfo.jitter_buffer_emitted_count = stats.jitter_buffer_emitted_count; + rinfo.jitter_buffer_target_delay_seconds = + stats.jitter_buffer_target_delay_seconds; + rinfo.inserted_samples_for_deceleration = + stats.inserted_samples_for_deceleration; + rinfo.removed_samples_for_acceleration = + stats.removed_samples_for_acceleration; + rinfo.expand_rate = stats.expand_rate; + rinfo.speech_expand_rate = stats.speech_expand_rate; + rinfo.secondary_decoded_rate = stats.secondary_decoded_rate; + rinfo.secondary_discarded_rate = stats.secondary_discarded_rate; + rinfo.accelerate_rate = stats.accelerate_rate; + rinfo.preemptive_expand_rate = stats.preemptive_expand_rate; + rinfo.delayed_packet_outage_samples = stats.delayed_packet_outage_samples; + rinfo.decoding_calls_to_silence_generator = + stats.decoding_calls_to_silence_generator; + rinfo.decoding_calls_to_neteq = stats.decoding_calls_to_neteq; + rinfo.decoding_normal = stats.decoding_normal; + rinfo.decoding_plc = stats.decoding_plc; + rinfo.decoding_codec_plc = stats.decoding_codec_plc; + rinfo.decoding_cng = stats.decoding_cng; + rinfo.decoding_plc_cng = stats.decoding_plc_cng; + rinfo.decoding_muted_output = stats.decoding_muted_output; + rinfo.capture_start_ntp_time_ms = stats.capture_start_ntp_time_ms; + rinfo.last_packet_received_timestamp_ms = + stats.last_packet_received_timestamp_ms; + rinfo.estimated_playout_ntp_timestamp_ms = + stats.estimated_playout_ntp_timestamp_ms; + rinfo.jitter_buffer_flushes = stats.jitter_buffer_flushes; + rinfo.relative_packet_arrival_delay_seconds = + stats.relative_packet_arrival_delay_seconds; + rinfo.interruption_count = stats.interruption_count; + rinfo.total_interruption_duration_ms = stats.total_interruption_duration_ms; + + info->receivers.push_back(rinfo); + } + + // Get codec info + for (const AudioCodec& codec : send_codecs_) { + webrtc::RtpCodecParameters codec_params = codec.ToCodecParameters(); + info->send_codecs.insert( + std::make_pair(codec_params.payload_type, std::move(codec_params))); + } + for (const AudioCodec& codec : recv_codecs_) { + webrtc::RtpCodecParameters codec_params = codec.ToCodecParameters(); + info->receive_codecs.insert( + std::make_pair(codec_params.payload_type, std::move(codec_params))); + } + info->device_underrun_count = engine_->adm()->GetPlayoutUnderrunCount(); + + return true; +} + +void WebRtcVoiceMediaChannel::SetRawAudioSink( + uint32_t ssrc, + std::unique_ptr<webrtc::AudioSinkInterface> sink) { + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + RTC_LOG(LS_VERBOSE) << "WebRtcVoiceMediaChannel::SetRawAudioSink: ssrc:" + << ssrc << " " << (sink ? "(ptr)" : "NULL"); + const auto it = recv_streams_.find(ssrc); + if (it == recv_streams_.end()) { + RTC_LOG(LS_WARNING) << "SetRawAudioSink: no recv stream " << ssrc; + return; + } + it->second->SetRawAudioSink(std::move(sink)); +} + +void WebRtcVoiceMediaChannel::SetDefaultRawAudioSink( + std::unique_ptr<webrtc::AudioSinkInterface> sink) { + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + RTC_LOG(LS_VERBOSE) << "WebRtcVoiceMediaChannel::SetDefaultRawAudioSink:"; + if (!unsignaled_recv_ssrcs_.empty()) { + std::unique_ptr<webrtc::AudioSinkInterface> proxy_sink( + sink ? new ProxySink(sink.get()) : nullptr); + SetRawAudioSink(unsignaled_recv_ssrcs_.back(), std::move(proxy_sink)); + } + default_sink_ = std::move(sink); +} + +std::vector<webrtc::RtpSource> WebRtcVoiceMediaChannel::GetSources( + uint32_t ssrc) const { + auto it = recv_streams_.find(ssrc); + if (it == recv_streams_.end()) { + RTC_LOG(LS_ERROR) << "Attempting to get contributing sources for SSRC:" + << ssrc << " which doesn't exist."; + return std::vector<webrtc::RtpSource>(); + } + return it->second->GetSources(); +} + +void WebRtcVoiceMediaChannel::SetEncoderToPacketizerFrameTransformer( + uint32_t ssrc, + rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer) { + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + auto matching_stream = send_streams_.find(ssrc); + if (matching_stream == send_streams_.end()) { + RTC_LOG(LS_INFO) << "Attempting to set frame transformer for SSRC:" << ssrc + << " which doesn't exist."; + return; + } + matching_stream->second->SetEncoderToPacketizerFrameTransformer( + std::move(frame_transformer)); +} + +void WebRtcVoiceMediaChannel::SetDepacketizerToDecoderFrameTransformer( + uint32_t ssrc, + rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer) { + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + auto matching_stream = recv_streams_.find(ssrc); + if (matching_stream == recv_streams_.end()) { + RTC_LOG(LS_INFO) << "Attempting to set frame transformer for SSRC:" << ssrc + << " which doesn't exist."; + return; + } + matching_stream->second->SetDepacketizerToDecoderFrameTransformer( + std::move(frame_transformer)); +} + +bool WebRtcVoiceMediaChannel::MaybeDeregisterUnsignaledRecvStream( + uint32_t ssrc) { + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + auto it = absl::c_find(unsignaled_recv_ssrcs_, ssrc); + if (it != unsignaled_recv_ssrcs_.end()) { + unsignaled_recv_ssrcs_.erase(it); + return true; + } + return false; +} +} // namespace cricket diff --git a/media/engine/webrtc_voice_engine.h b/media/engine/webrtc_voice_engine.h new file mode 100644 index 0000000000..86a7a495fe --- /dev/null +++ b/media/engine/webrtc_voice_engine.h @@ -0,0 +1,339 @@ +/* + * Copyright (c) 2004 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MEDIA_ENGINE_WEBRTC_VOICE_ENGINE_H_ +#define MEDIA_ENGINE_WEBRTC_VOICE_ENGINE_H_ + +#include <map> +#include <memory> +#include <string> +#include <vector> + +#include "api/audio_codecs/audio_encoder_factory.h" +#include "api/scoped_refptr.h" +#include "api/task_queue/task_queue_factory.h" +#include "api/transport/rtp/rtp_source.h" +#include "call/audio_state.h" +#include "call/call.h" +#include "media/base/media_engine.h" +#include "media/base/rtp_utils.h" +#include "rtc_base/buffer.h" +#include "rtc_base/constructor_magic.h" +#include "rtc_base/network_route.h" +#include "rtc_base/task_queue.h" +#include "rtc_base/thread_checker.h" + +namespace cricket { + +class AudioDeviceModule; +class AudioMixer; +class AudioSource; +class WebRtcVoiceMediaChannel; + +// WebRtcVoiceEngine is a class to be used with CompositeMediaEngine. +// It uses the WebRtc VoiceEngine library for audio handling. +class WebRtcVoiceEngine final : public VoiceEngineInterface { + friend class WebRtcVoiceMediaChannel; + + public: + WebRtcVoiceEngine( + webrtc::TaskQueueFactory* task_queue_factory, + webrtc::AudioDeviceModule* adm, + const rtc::scoped_refptr<webrtc::AudioEncoderFactory>& encoder_factory, + const rtc::scoped_refptr<webrtc::AudioDecoderFactory>& decoder_factory, + rtc::scoped_refptr<webrtc::AudioMixer> audio_mixer, + rtc::scoped_refptr<webrtc::AudioProcessing> audio_processing); + ~WebRtcVoiceEngine() override; + + // Does initialization that needs to occur on the worker thread. + void Init() override; + + rtc::scoped_refptr<webrtc::AudioState> GetAudioState() const override; + VoiceMediaChannel* CreateMediaChannel( + webrtc::Call* call, + const MediaConfig& config, + const AudioOptions& options, + const webrtc::CryptoOptions& crypto_options) override; + + const std::vector<AudioCodec>& send_codecs() const override; + const std::vector<AudioCodec>& recv_codecs() const override; + std::vector<webrtc::RtpHeaderExtensionCapability> GetRtpHeaderExtensions() + const override; + + // For tracking WebRtc channels. Needed because we have to pause them + // all when switching devices. + // May only be called by WebRtcVoiceMediaChannel. + void RegisterChannel(WebRtcVoiceMediaChannel* channel); + void UnregisterChannel(WebRtcVoiceMediaChannel* channel); + + // Starts AEC dump using an existing file. A maximum file size in bytes can be + // specified. When the maximum file size is reached, logging is stopped and + // the file is closed. If max_size_bytes is set to <= 0, no limit will be + // used. + bool StartAecDump(webrtc::FileWrapper file, int64_t max_size_bytes) override; + + // Stops AEC dump. + void StopAecDump() override; + + private: + // Every option that is "set" will be applied. Every option not "set" will be + // ignored. This allows us to selectively turn on and off different options + // easily at any time. + bool ApplyOptions(const AudioOptions& options); + + int CreateVoEChannel(); + + webrtc::TaskQueueFactory* const task_queue_factory_; + std::unique_ptr<rtc::TaskQueue> low_priority_worker_queue_; + + webrtc::AudioDeviceModule* adm(); + webrtc::AudioProcessing* apm() const; + webrtc::AudioState* audio_state(); + + std::vector<AudioCodec> CollectCodecs( + const std::vector<webrtc::AudioCodecSpec>& specs) const; + + rtc::ThreadChecker signal_thread_checker_; + rtc::ThreadChecker worker_thread_checker_; + + // The audio device module. + rtc::scoped_refptr<webrtc::AudioDeviceModule> adm_; + rtc::scoped_refptr<webrtc::AudioEncoderFactory> encoder_factory_; + rtc::scoped_refptr<webrtc::AudioDecoderFactory> decoder_factory_; + rtc::scoped_refptr<webrtc::AudioMixer> audio_mixer_; + // The audio processing module. + rtc::scoped_refptr<webrtc::AudioProcessing> apm_; + // The primary instance of WebRtc VoiceEngine. + rtc::scoped_refptr<webrtc::AudioState> audio_state_; + std::vector<AudioCodec> send_codecs_; + std::vector<AudioCodec> recv_codecs_; + std::vector<WebRtcVoiceMediaChannel*> channels_; + bool is_dumping_aec_ = false; + bool initialized_ = false; + + // Cache experimental_ns and apply in case they are missing in the audio + // options. We need to do this because SetExtraOptions() will revert to + // defaults for options which are not provided. + absl::optional<bool> experimental_ns_; + // Jitter buffer settings for new streams. + size_t audio_jitter_buffer_max_packets_ = 200; + bool audio_jitter_buffer_fast_accelerate_ = false; + int audio_jitter_buffer_min_delay_ms_ = 0; + bool audio_jitter_buffer_enable_rtx_handling_ = false; + + RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(WebRtcVoiceEngine); +}; + +// WebRtcVoiceMediaChannel is an implementation of VoiceMediaChannel that uses +// WebRtc Voice Engine. +class WebRtcVoiceMediaChannel final : public VoiceMediaChannel, + public webrtc::Transport { + public: + WebRtcVoiceMediaChannel(WebRtcVoiceEngine* engine, + const MediaConfig& config, + const AudioOptions& options, + const webrtc::CryptoOptions& crypto_options, + webrtc::Call* call); + ~WebRtcVoiceMediaChannel() override; + + const AudioOptions& options() const { return options_; } + + bool SetSendParameters(const AudioSendParameters& params) override; + bool SetRecvParameters(const AudioRecvParameters& params) override; + webrtc::RtpParameters GetRtpSendParameters(uint32_t ssrc) const override; + webrtc::RTCError SetRtpSendParameters( + uint32_t ssrc, + const webrtc::RtpParameters& parameters) override; + webrtc::RtpParameters GetRtpReceiveParameters(uint32_t ssrc) const override; + webrtc::RtpParameters GetDefaultRtpReceiveParameters() const override; + + void SetPlayout(bool playout) override; + void SetSend(bool send) override; + bool SetAudioSend(uint32_t ssrc, + bool enable, + const AudioOptions* options, + AudioSource* source) override; + bool AddSendStream(const StreamParams& sp) override; + bool RemoveSendStream(uint32_t ssrc) override; + bool AddRecvStream(const StreamParams& sp) override; + bool RemoveRecvStream(uint32_t ssrc) override; + void ResetUnsignaledRecvStream() override; + + // E2EE Frame API + // Set a frame decryptor to a particular ssrc that will intercept all + // incoming audio payloads and attempt to decrypt them before forwarding the + // result. + void SetFrameDecryptor(uint32_t ssrc, + rtc::scoped_refptr<webrtc::FrameDecryptorInterface> + frame_decryptor) override; + // Set a frame encryptor to a particular ssrc that will intercept all + // outgoing audio payloads frames and attempt to encrypt them and forward the + // result to the packetizer. + void SetFrameEncryptor(uint32_t ssrc, + rtc::scoped_refptr<webrtc::FrameEncryptorInterface> + frame_encryptor) override; + + bool SetOutputVolume(uint32_t ssrc, double volume) override; + // Applies the new volume to current and future unsignaled streams. + bool SetDefaultOutputVolume(double volume) override; + + bool SetBaseMinimumPlayoutDelayMs(uint32_t ssrc, int delay_ms) override; + absl::optional<int> GetBaseMinimumPlayoutDelayMs( + uint32_t ssrc) const override; + + bool CanInsertDtmf() override; + bool InsertDtmf(uint32_t ssrc, int event, int duration) override; + + void OnPacketReceived(rtc::CopyOnWriteBuffer packet, + int64_t packet_time_us) override; + void OnNetworkRouteChanged(const std::string& transport_name, + const rtc::NetworkRoute& network_route) override; + void OnReadyToSend(bool ready) override; + bool GetStats(VoiceMediaInfo* info) override; + + // Set the audio sink for an existing stream. + void SetRawAudioSink( + uint32_t ssrc, + std::unique_ptr<webrtc::AudioSinkInterface> sink) override; + // Will set the audio sink on the latest unsignaled stream, future or + // current. Only one stream at a time will use the sink. + void SetDefaultRawAudioSink( + std::unique_ptr<webrtc::AudioSinkInterface> sink) override; + + std::vector<webrtc::RtpSource> GetSources(uint32_t ssrc) const override; + + // Sets a frame transformer between encoder and packetizer, to transform + // encoded frames before sending them out the network. + void SetEncoderToPacketizerFrameTransformer( + uint32_t ssrc, + rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer) + override; + void SetDepacketizerToDecoderFrameTransformer( + uint32_t ssrc, + rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer) + override; + + // implements Transport interface + bool SendRtp(const uint8_t* data, + size_t len, + const webrtc::PacketOptions& options) override { + rtc::CopyOnWriteBuffer packet(data, len, kMaxRtpPacketLen); + rtc::PacketOptions rtc_options; + rtc_options.packet_id = options.packet_id; + if (DscpEnabled()) { + rtc_options.dscp = PreferredDscp(); + } + rtc_options.info_signaled_after_sent.included_in_feedback = + options.included_in_feedback; + rtc_options.info_signaled_after_sent.included_in_allocation = + options.included_in_allocation; + return VoiceMediaChannel::SendPacket(&packet, rtc_options); + } + + bool SendRtcp(const uint8_t* data, size_t len) override { + rtc::CopyOnWriteBuffer packet(data, len, kMaxRtpPacketLen); + rtc::PacketOptions rtc_options; + if (DscpEnabled()) { + rtc_options.dscp = PreferredDscp(); + } + + return VoiceMediaChannel::SendRtcp(&packet, rtc_options); + } + + private: + bool SetOptions(const AudioOptions& options); + bool SetRecvCodecs(const std::vector<AudioCodec>& codecs); + bool SetSendCodecs(const std::vector<AudioCodec>& codecs); + bool SetLocalSource(uint32_t ssrc, AudioSource* source); + bool MuteStream(uint32_t ssrc, bool mute); + + WebRtcVoiceEngine* engine() { return engine_; } + void ChangePlayout(bool playout); + int CreateVoEChannel(); + bool DeleteVoEChannel(int channel); + bool SetMaxSendBitrate(int bps); + void SetupRecording(); + // Check if 'ssrc' is an unsignaled stream, and if so mark it as not being + // unsignaled anymore (i.e. it is now removed, or signaled), and return true. + bool MaybeDeregisterUnsignaledRecvStream(uint32_t ssrc); + + rtc::ThreadChecker worker_thread_checker_; + + WebRtcVoiceEngine* const engine_ = nullptr; + std::vector<AudioCodec> send_codecs_; + + // TODO(kwiberg): decoder_map_ and recv_codecs_ store the exact same + // information, in slightly different formats. Eliminate recv_codecs_. + std::map<int, webrtc::SdpAudioFormat> decoder_map_; + std::vector<AudioCodec> recv_codecs_; + + int max_send_bitrate_bps_ = 0; + AudioOptions options_; + absl::optional<int> dtmf_payload_type_; + int dtmf_payload_freq_ = -1; + bool recv_transport_cc_enabled_ = false; + bool recv_nack_enabled_ = false; + bool desired_playout_ = false; + bool playout_ = false; + bool send_ = false; + webrtc::Call* const call_ = nullptr; + + const MediaConfig::Audio audio_config_; + + // Queue of unsignaled SSRCs; oldest at the beginning. + std::vector<uint32_t> unsignaled_recv_ssrcs_; + + // This is a stream param that comes from the remote description, but wasn't + // signaled with any a=ssrc lines. It holds the information that was signaled + // before the unsignaled receive stream is created when the first packet is + // received. + StreamParams unsignaled_stream_params_; + + // Volume for unsignaled streams, which may be set before the stream exists. + double default_recv_volume_ = 1.0; + + // Delay for unsignaled streams, which may be set before the stream exists. + int default_recv_base_minimum_delay_ms_ = 0; + + // Sink for latest unsignaled stream - may be set before the stream exists. + std::unique_ptr<webrtc::AudioSinkInterface> default_sink_; + // Default SSRC to use for RTCP receiver reports in case of no signaled + // send streams. See: https://code.google.com/p/webrtc/issues/detail?id=4740 + // and https://code.google.com/p/chromium/issues/detail?id=547661 + uint32_t receiver_reports_ssrc_ = 0xFA17FA17u; + + class WebRtcAudioSendStream; + std::map<uint32_t, WebRtcAudioSendStream*> send_streams_; + std::vector<webrtc::RtpExtension> send_rtp_extensions_; + std::string mid_; + + class WebRtcAudioReceiveStream; + std::map<uint32_t, WebRtcAudioReceiveStream*> recv_streams_; + std::vector<webrtc::RtpExtension> recv_rtp_extensions_; + + absl::optional<webrtc::AudioSendStream::Config::SendCodecSpec> + send_codec_spec_; + + // TODO(kwiberg): Per-SSRC codec pair IDs? + const webrtc::AudioCodecPairId codec_pair_id_ = + webrtc::AudioCodecPairId::Create(); + + // Per peer connection crypto options that last for the lifetime of the peer + // connection. + const webrtc::CryptoOptions crypto_options_; + // Unsignaled streams have an option to have a frame decryptor set on them. + rtc::scoped_refptr<webrtc::FrameDecryptorInterface> + unsignaled_frame_decryptor_; + + RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(WebRtcVoiceMediaChannel); +}; +} // namespace cricket + +#endif // MEDIA_ENGINE_WEBRTC_VOICE_ENGINE_H_ diff --git a/media/engine/webrtc_voice_engine_unittest.cc b/media/engine/webrtc_voice_engine_unittest.cc new file mode 100644 index 0000000000..e7ebf8940f --- /dev/null +++ b/media/engine/webrtc_voice_engine_unittest.cc @@ -0,0 +1,3784 @@ +/* + * Copyright (c) 2008 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "media/engine/webrtc_voice_engine.h" + +#include <memory> +#include <utility> + +#include "absl/memory/memory.h" +#include "absl/strings/match.h" +#include "api/audio_codecs/builtin_audio_decoder_factory.h" +#include "api/audio_codecs/builtin_audio_encoder_factory.h" +#include "api/rtc_event_log/rtc_event_log.h" +#include "api/rtp_parameters.h" +#include "api/scoped_refptr.h" +#include "api/task_queue/default_task_queue_factory.h" +#include "api/transport/field_trial_based_config.h" +#include "call/call.h" +#include "media/base/fake_media_engine.h" +#include "media/base/fake_network_interface.h" +#include "media/base/fake_rtp.h" +#include "media/base/media_constants.h" +#include "media/engine/fake_webrtc_call.h" +#include "modules/audio_device/include/mock_audio_device.h" +#include "modules/audio_processing/include/mock_audio_processing.h" +#include "rtc_base/arraysize.h" +#include "rtc_base/byte_order.h" +#include "rtc_base/numerics/safe_conversions.h" +#include "test/field_trial.h" +#include "test/gtest.h" +#include "test/mock_audio_decoder_factory.h" +#include "test/mock_audio_encoder_factory.h" + +using ::testing::_; +using ::testing::ContainerEq; +using ::testing::Contains; +using ::testing::Field; +using ::testing::Return; +using ::testing::ReturnPointee; +using ::testing::SaveArg; +using ::testing::StrictMock; + +namespace { +using webrtc::BitrateConstraints; + +constexpr uint32_t kMaxUnsignaledRecvStreams = 4; + +const cricket::AudioCodec kPcmuCodec(0, "PCMU", 8000, 64000, 1); +const cricket::AudioCodec kIsacCodec(103, "ISAC", 16000, 32000, 1); +const cricket::AudioCodec kOpusCodec(111, "opus", 48000, 32000, 2); +const cricket::AudioCodec kG722CodecVoE(9, "G722", 16000, 64000, 1); +const cricket::AudioCodec kG722CodecSdp(9, "G722", 8000, 64000, 1); +const cricket::AudioCodec kCn8000Codec(13, "CN", 8000, 0, 1); +const cricket::AudioCodec kCn16000Codec(105, "CN", 16000, 0, 1); +const cricket::AudioCodec kTelephoneEventCodec1(106, + "telephone-event", + 8000, + 0, + 1); +const cricket::AudioCodec kTelephoneEventCodec2(107, + "telephone-event", + 32000, + 0, + 1); + +const uint32_t kSsrc0 = 0; +const uint32_t kSsrc1 = 1; +const uint32_t kSsrcX = 0x99; +const uint32_t kSsrcY = 0x17; +const uint32_t kSsrcZ = 0x42; +const uint32_t kSsrcW = 0x02; +const uint32_t kSsrcs4[] = {11, 200, 30, 44}; + +constexpr int kRtpHistoryMs = 5000; + +constexpr webrtc::AudioProcessing::Config::GainController1::Mode + kDefaultAgcMode = +#if defined(WEBRTC_IOS) || defined(WEBRTC_ANDROID) + webrtc::AudioProcessing::Config::GainController1::kFixedDigital; +#else + webrtc::AudioProcessing::Config::GainController1::kAdaptiveAnalog; +#endif + +constexpr webrtc::AudioProcessing::Config::NoiseSuppression::Level + kDefaultNsLevel = + webrtc::AudioProcessing::Config::NoiseSuppression::Level::kHigh; + +void AdmSetupExpectations(webrtc::test::MockAudioDeviceModule* adm) { + RTC_DCHECK(adm); + + // Setup. + EXPECT_CALL(*adm, Init()).WillOnce(Return(0)); + EXPECT_CALL(*adm, RegisterAudioCallback(_)).WillOnce(Return(0)); +#if defined(WEBRTC_WIN) + EXPECT_CALL( + *adm, + SetPlayoutDevice( + ::testing::Matcher<webrtc::AudioDeviceModule::WindowsDeviceType>( + webrtc::AudioDeviceModule::kDefaultCommunicationDevice))) + .WillOnce(Return(0)); +#else + EXPECT_CALL(*adm, SetPlayoutDevice(0)).WillOnce(Return(0)); +#endif // #if defined(WEBRTC_WIN) + EXPECT_CALL(*adm, InitSpeaker()).WillOnce(Return(0)); + EXPECT_CALL(*adm, StereoPlayoutIsAvailable(::testing::_)).WillOnce(Return(0)); + EXPECT_CALL(*adm, SetStereoPlayout(false)).WillOnce(Return(0)); +#if defined(WEBRTC_WIN) + EXPECT_CALL( + *adm, + SetRecordingDevice( + ::testing::Matcher<webrtc::AudioDeviceModule::WindowsDeviceType>( + webrtc::AudioDeviceModule::kDefaultCommunicationDevice))) + .WillOnce(Return(0)); +#else + EXPECT_CALL(*adm, SetRecordingDevice(0)).WillOnce(Return(0)); +#endif // #if defined(WEBRTC_WIN) + EXPECT_CALL(*adm, InitMicrophone()).WillOnce(Return(0)); + EXPECT_CALL(*adm, StereoRecordingIsAvailable(::testing::_)) + .WillOnce(Return(0)); + EXPECT_CALL(*adm, SetStereoRecording(false)).WillOnce(Return(0)); + EXPECT_CALL(*adm, BuiltInAECIsAvailable()).WillOnce(Return(false)); + EXPECT_CALL(*adm, BuiltInAGCIsAvailable()).WillOnce(Return(false)); + EXPECT_CALL(*adm, BuiltInNSIsAvailable()).WillOnce(Return(false)); + + // Teardown. + EXPECT_CALL(*adm, StopPlayout()).WillOnce(Return(0)); + EXPECT_CALL(*adm, StopRecording()).WillOnce(Return(0)); + EXPECT_CALL(*adm, RegisterAudioCallback(nullptr)).WillOnce(Return(0)); + EXPECT_CALL(*adm, Terminate()).WillOnce(Return(0)); +} +} // namespace + +// Tests that our stub library "works". +TEST(WebRtcVoiceEngineTestStubLibrary, StartupShutdown) { + for (bool use_null_apm : {false, true}) { + std::unique_ptr<webrtc::TaskQueueFactory> task_queue_factory = + webrtc::CreateDefaultTaskQueueFactory(); + rtc::scoped_refptr<webrtc::test::MockAudioDeviceModule> adm = + webrtc::test::MockAudioDeviceModule::CreateStrict(); + AdmSetupExpectations(adm); + rtc::scoped_refptr<StrictMock<webrtc::test::MockAudioProcessing>> apm = + use_null_apm ? nullptr + : new rtc::RefCountedObject< + StrictMock<webrtc::test::MockAudioProcessing>>(); + + webrtc::AudioProcessing::Config apm_config; + if (!use_null_apm) { + EXPECT_CALL(*apm, GetConfig()).WillRepeatedly(ReturnPointee(&apm_config)); + EXPECT_CALL(*apm, ApplyConfig(_)).WillRepeatedly(SaveArg<0>(&apm_config)); + EXPECT_CALL(*apm, SetExtraOptions(::testing::_)); + EXPECT_CALL(*apm, DetachAecDump()); + } + { + cricket::WebRtcVoiceEngine engine( + task_queue_factory.get(), adm, + webrtc::MockAudioEncoderFactory::CreateUnusedFactory(), + webrtc::MockAudioDecoderFactory::CreateUnusedFactory(), nullptr, apm); + engine.Init(); + } + } +} + +class FakeAudioSink : public webrtc::AudioSinkInterface { + public: + void OnData(const Data& audio) override {} +}; + +class FakeAudioSource : public cricket::AudioSource { + void SetSink(Sink* sink) override {} +}; + +class WebRtcVoiceEngineTestFake : public ::testing::TestWithParam<bool> { + public: + WebRtcVoiceEngineTestFake() : WebRtcVoiceEngineTestFake("") {} + + explicit WebRtcVoiceEngineTestFake(const char* field_trials) + : use_null_apm_(GetParam()), + task_queue_factory_(webrtc::CreateDefaultTaskQueueFactory()), + adm_(webrtc::test::MockAudioDeviceModule::CreateStrict()), + apm_(use_null_apm_ + ? nullptr + : new rtc::RefCountedObject< + StrictMock<webrtc::test::MockAudioProcessing>>()), + call_(), + override_field_trials_(field_trials) { + // AudioDeviceModule. + AdmSetupExpectations(adm_); + + if (!use_null_apm_) { + // AudioProcessing. + EXPECT_CALL(*apm_, GetConfig()) + .WillRepeatedly(ReturnPointee(&apm_config_)); + EXPECT_CALL(*apm_, ApplyConfig(_)) + .WillRepeatedly(SaveArg<0>(&apm_config_)); + EXPECT_CALL(*apm_, SetExtraOptions(::testing::_)); + EXPECT_CALL(*apm_, DetachAecDump()); + } + + // Default Options. + // TODO(kwiberg): We should use mock factories here, but a bunch of + // the tests here probe the specific set of codecs provided by the builtin + // factories. Those tests should probably be moved elsewhere. + auto encoder_factory = webrtc::CreateBuiltinAudioEncoderFactory(); + auto decoder_factory = webrtc::CreateBuiltinAudioDecoderFactory(); + engine_.reset(new cricket::WebRtcVoiceEngine( + task_queue_factory_.get(), adm_, encoder_factory, decoder_factory, + nullptr, apm_)); + engine_->Init(); + send_parameters_.codecs.push_back(kPcmuCodec); + recv_parameters_.codecs.push_back(kPcmuCodec); + + if (!use_null_apm_) { + // Default Options. + VerifyEchoCancellationSettings(/*enabled=*/true); + EXPECT_TRUE(IsHighPassFilterEnabled()); + EXPECT_TRUE(IsTypingDetectionEnabled()); + EXPECT_TRUE(apm_config_.noise_suppression.enabled); + EXPECT_EQ(apm_config_.noise_suppression.level, kDefaultNsLevel); + VerifyGainControlEnabledCorrectly(); + VerifyGainControlDefaultSettings(); + } + } + + bool SetupChannel() { + if (!use_null_apm_) { + EXPECT_CALL(*apm_, SetExtraOptions(::testing::_)); + } + channel_ = engine_->CreateMediaChannel(&call_, cricket::MediaConfig(), + cricket::AudioOptions(), + webrtc::CryptoOptions()); + return (channel_ != nullptr); + } + + bool SetupRecvStream() { + if (!SetupChannel()) { + return false; + } + return AddRecvStream(kSsrcX); + } + + bool SetupSendStream() { + return SetupSendStream(cricket::StreamParams::CreateLegacy(kSsrcX)); + } + + bool SetupSendStream(const cricket::StreamParams& sp) { + if (!SetupChannel()) { + return false; + } + if (!channel_->AddSendStream(sp)) { + return false; + } + if (!use_null_apm_) { + EXPECT_CALL(*apm_, set_output_will_be_muted(false)); + } + return channel_->SetAudioSend(kSsrcX, true, nullptr, &fake_source_); + } + + bool AddRecvStream(uint32_t ssrc) { + EXPECT_TRUE(channel_); + return channel_->AddRecvStream(cricket::StreamParams::CreateLegacy(ssrc)); + } + + void SetupForMultiSendStream() { + EXPECT_TRUE(SetupSendStream()); + // Remove stream added in Setup. + EXPECT_TRUE(call_.GetAudioSendStream(kSsrcX)); + EXPECT_TRUE(channel_->RemoveSendStream(kSsrcX)); + // Verify the channel does not exist. + EXPECT_FALSE(call_.GetAudioSendStream(kSsrcX)); + } + + void DeliverPacket(const void* data, int len) { + rtc::CopyOnWriteBuffer packet(reinterpret_cast<const uint8_t*>(data), len); + channel_->OnPacketReceived(packet, /* packet_time_us */ -1); + } + + void TearDown() override { delete channel_; } + + const cricket::FakeAudioSendStream& GetSendStream(uint32_t ssrc) { + const auto* send_stream = call_.GetAudioSendStream(ssrc); + EXPECT_TRUE(send_stream); + return *send_stream; + } + + const cricket::FakeAudioReceiveStream& GetRecvStream(uint32_t ssrc) { + const auto* recv_stream = call_.GetAudioReceiveStream(ssrc); + EXPECT_TRUE(recv_stream); + return *recv_stream; + } + + const webrtc::AudioSendStream::Config& GetSendStreamConfig(uint32_t ssrc) { + return GetSendStream(ssrc).GetConfig(); + } + + const webrtc::AudioReceiveStream::Config& GetRecvStreamConfig(uint32_t ssrc) { + return GetRecvStream(ssrc).GetConfig(); + } + + void SetSend(bool enable) { + ASSERT_TRUE(channel_); + if (enable) { + EXPECT_CALL(*adm_, RecordingIsInitialized()).WillOnce(Return(false)); + EXPECT_CALL(*adm_, Recording()).WillOnce(Return(false)); + EXPECT_CALL(*adm_, InitRecording()).WillOnce(Return(0)); + if (!use_null_apm_) { + EXPECT_CALL(*apm_, SetExtraOptions(::testing::_)); + } + } + channel_->SetSend(enable); + } + + void SetSendParameters(const cricket::AudioSendParameters& params) { + if (!use_null_apm_) { + EXPECT_CALL(*apm_, SetExtraOptions(::testing::_)); + } + ASSERT_TRUE(channel_); + EXPECT_TRUE(channel_->SetSendParameters(params)); + } + + void SetAudioSend(uint32_t ssrc, + bool enable, + cricket::AudioSource* source, + const cricket::AudioOptions* options = nullptr) { + ASSERT_TRUE(channel_); + if (!use_null_apm_) { + EXPECT_CALL(*apm_, set_output_will_be_muted(!enable)); + if (enable && options) { + EXPECT_CALL(*apm_, SetExtraOptions(::testing::_)); + } + } + EXPECT_TRUE(channel_->SetAudioSend(ssrc, enable, options, source)); + } + + void TestInsertDtmf(uint32_t ssrc, + bool caller, + const cricket::AudioCodec& codec) { + EXPECT_TRUE(SetupChannel()); + if (caller) { + // If this is a caller, local description will be applied and add the + // send stream. + EXPECT_TRUE( + channel_->AddSendStream(cricket::StreamParams::CreateLegacy(kSsrcX))); + } + + // Test we can only InsertDtmf when the other side supports telephone-event. + SetSendParameters(send_parameters_); + SetSend(true); + EXPECT_FALSE(channel_->CanInsertDtmf()); + EXPECT_FALSE(channel_->InsertDtmf(ssrc, 1, 111)); + send_parameters_.codecs.push_back(codec); + SetSendParameters(send_parameters_); + EXPECT_TRUE(channel_->CanInsertDtmf()); + + if (!caller) { + // If this is callee, there's no active send channel yet. + EXPECT_FALSE(channel_->InsertDtmf(ssrc, 2, 123)); + EXPECT_TRUE( + channel_->AddSendStream(cricket::StreamParams::CreateLegacy(kSsrcX))); + } + + // Check we fail if the ssrc is invalid. + EXPECT_FALSE(channel_->InsertDtmf(-1, 1, 111)); + + // Test send. + cricket::FakeAudioSendStream::TelephoneEvent telephone_event = + GetSendStream(kSsrcX).GetLatestTelephoneEvent(); + EXPECT_EQ(-1, telephone_event.payload_type); + EXPECT_TRUE(channel_->InsertDtmf(ssrc, 2, 123)); + telephone_event = GetSendStream(kSsrcX).GetLatestTelephoneEvent(); + EXPECT_EQ(codec.id, telephone_event.payload_type); + EXPECT_EQ(codec.clockrate, telephone_event.payload_frequency); + EXPECT_EQ(2, telephone_event.event_code); + EXPECT_EQ(123, telephone_event.duration_ms); + } + + void TestExtmapAllowMixedCaller(bool extmap_allow_mixed) { + // For a caller, the answer will be applied in set remote description + // where SetSendParameters() is called. + EXPECT_TRUE(SetupChannel()); + EXPECT_TRUE( + channel_->AddSendStream(cricket::StreamParams::CreateLegacy(kSsrcX))); + send_parameters_.extmap_allow_mixed = extmap_allow_mixed; + SetSendParameters(send_parameters_); + const webrtc::AudioSendStream::Config& config = GetSendStreamConfig(kSsrcX); + EXPECT_EQ(extmap_allow_mixed, config.rtp.extmap_allow_mixed); + } + + void TestExtmapAllowMixedCallee(bool extmap_allow_mixed) { + // For a callee, the answer will be applied in set local description + // where SetExtmapAllowMixed() and AddSendStream() are called. + EXPECT_TRUE(SetupChannel()); + channel_->SetExtmapAllowMixed(extmap_allow_mixed); + EXPECT_TRUE( + channel_->AddSendStream(cricket::StreamParams::CreateLegacy(kSsrcX))); + + const webrtc::AudioSendStream::Config& config = GetSendStreamConfig(kSsrcX); + EXPECT_EQ(extmap_allow_mixed, config.rtp.extmap_allow_mixed); + } + + // Test that send bandwidth is set correctly. + // |codec| is the codec under test. + // |max_bitrate| is a parameter to set to SetMaxSendBandwidth(). + // |expected_result| is the expected result from SetMaxSendBandwidth(). + // |expected_bitrate| is the expected audio bitrate afterward. + void TestMaxSendBandwidth(const cricket::AudioCodec& codec, + int max_bitrate, + bool expected_result, + int expected_bitrate) { + cricket::AudioSendParameters parameters; + parameters.codecs.push_back(codec); + parameters.max_bandwidth_bps = max_bitrate; + if (expected_result) { + SetSendParameters(parameters); + } else { + EXPECT_FALSE(channel_->SetSendParameters(parameters)); + } + EXPECT_EQ(expected_bitrate, GetCodecBitrate(kSsrcX)); + } + + // Sets the per-stream maximum bitrate limit for the specified SSRC. + bool SetMaxBitrateForStream(int32_t ssrc, int bitrate) { + webrtc::RtpParameters parameters = channel_->GetRtpSendParameters(ssrc); + EXPECT_EQ(1UL, parameters.encodings.size()); + + parameters.encodings[0].max_bitrate_bps = bitrate; + return channel_->SetRtpSendParameters(ssrc, parameters).ok(); + } + + void SetGlobalMaxBitrate(const cricket::AudioCodec& codec, int bitrate) { + cricket::AudioSendParameters send_parameters; + send_parameters.codecs.push_back(codec); + send_parameters.max_bandwidth_bps = bitrate; + SetSendParameters(send_parameters); + } + + void CheckSendCodecBitrate(int32_t ssrc, + const char expected_name[], + int expected_bitrate) { + const auto& spec = GetSendStreamConfig(ssrc).send_codec_spec; + EXPECT_EQ(expected_name, spec->format.name); + EXPECT_EQ(expected_bitrate, spec->target_bitrate_bps); + } + + absl::optional<int> GetCodecBitrate(int32_t ssrc) { + return GetSendStreamConfig(ssrc).send_codec_spec->target_bitrate_bps; + } + + const absl::optional<std::string>& GetAudioNetworkAdaptorConfig( + int32_t ssrc) { + return GetSendStreamConfig(ssrc).audio_network_adaptor_config; + } + + void SetAndExpectMaxBitrate(const cricket::AudioCodec& codec, + int global_max, + int stream_max, + bool expected_result, + int expected_codec_bitrate) { + // Clear the bitrate limit from the previous test case. + EXPECT_TRUE(SetMaxBitrateForStream(kSsrcX, -1)); + + // Attempt to set the requested bitrate limits. + SetGlobalMaxBitrate(codec, global_max); + EXPECT_EQ(expected_result, SetMaxBitrateForStream(kSsrcX, stream_max)); + + // Verify that reading back the parameters gives results + // consistent with the Set() result. + webrtc::RtpParameters resulting_parameters = + channel_->GetRtpSendParameters(kSsrcX); + EXPECT_EQ(1UL, resulting_parameters.encodings.size()); + EXPECT_EQ(expected_result ? stream_max : -1, + resulting_parameters.encodings[0].max_bitrate_bps); + + // Verify that the codec settings have the expected bitrate. + EXPECT_EQ(expected_codec_bitrate, GetCodecBitrate(kSsrcX)); + } + + void SetSendCodecsShouldWorkForBitrates(const char* min_bitrate_kbps, + int expected_min_bitrate_bps, + const char* start_bitrate_kbps, + int expected_start_bitrate_bps, + const char* max_bitrate_kbps, + int expected_max_bitrate_bps) { + EXPECT_TRUE(SetupSendStream()); + auto& codecs = send_parameters_.codecs; + codecs.clear(); + codecs.push_back(kOpusCodec); + codecs[0].params[cricket::kCodecParamMinBitrate] = min_bitrate_kbps; + codecs[0].params[cricket::kCodecParamStartBitrate] = start_bitrate_kbps; + codecs[0].params[cricket::kCodecParamMaxBitrate] = max_bitrate_kbps; + EXPECT_CALL(*call_.GetMockTransportControllerSend(), + SetSdpBitrateParameters( + AllOf(Field(&BitrateConstraints::min_bitrate_bps, + expected_min_bitrate_bps), + Field(&BitrateConstraints::start_bitrate_bps, + expected_start_bitrate_bps), + Field(&BitrateConstraints::max_bitrate_bps, + expected_max_bitrate_bps)))); + + SetSendParameters(send_parameters_); + } + + void TestSetSendRtpHeaderExtensions(const std::string& ext) { + EXPECT_TRUE(SetupSendStream()); + + // Ensure extensions are off by default. + EXPECT_EQ(0u, GetSendStreamConfig(kSsrcX).rtp.extensions.size()); + + // Ensure unknown extensions won't cause an error. + send_parameters_.extensions.push_back( + webrtc::RtpExtension("urn:ietf:params:unknownextention", 1)); + SetSendParameters(send_parameters_); + EXPECT_EQ(0u, GetSendStreamConfig(kSsrcX).rtp.extensions.size()); + + // Ensure extensions stay off with an empty list of headers. + send_parameters_.extensions.clear(); + SetSendParameters(send_parameters_); + EXPECT_EQ(0u, GetSendStreamConfig(kSsrcX).rtp.extensions.size()); + + // Ensure extension is set properly. + const int id = 1; + send_parameters_.extensions.push_back(webrtc::RtpExtension(ext, id)); + SetSendParameters(send_parameters_); + EXPECT_EQ(1u, GetSendStreamConfig(kSsrcX).rtp.extensions.size()); + EXPECT_EQ(ext, GetSendStreamConfig(kSsrcX).rtp.extensions[0].uri); + EXPECT_EQ(id, GetSendStreamConfig(kSsrcX).rtp.extensions[0].id); + + // Ensure extension is set properly on new stream. + EXPECT_TRUE( + channel_->AddSendStream(cricket::StreamParams::CreateLegacy(kSsrcY))); + EXPECT_NE(call_.GetAudioSendStream(kSsrcX), + call_.GetAudioSendStream(kSsrcY)); + EXPECT_EQ(1u, GetSendStreamConfig(kSsrcY).rtp.extensions.size()); + EXPECT_EQ(ext, GetSendStreamConfig(kSsrcY).rtp.extensions[0].uri); + EXPECT_EQ(id, GetSendStreamConfig(kSsrcY).rtp.extensions[0].id); + + // Ensure all extensions go back off with an empty list. + send_parameters_.codecs.push_back(kPcmuCodec); + send_parameters_.extensions.clear(); + SetSendParameters(send_parameters_); + EXPECT_EQ(0u, GetSendStreamConfig(kSsrcX).rtp.extensions.size()); + EXPECT_EQ(0u, GetSendStreamConfig(kSsrcY).rtp.extensions.size()); + } + + void TestSetRecvRtpHeaderExtensions(const std::string& ext) { + EXPECT_TRUE(SetupRecvStream()); + + // Ensure extensions are off by default. + EXPECT_EQ(0u, GetRecvStreamConfig(kSsrcX).rtp.extensions.size()); + + // Ensure unknown extensions won't cause an error. + recv_parameters_.extensions.push_back( + webrtc::RtpExtension("urn:ietf:params:unknownextention", 1)); + EXPECT_TRUE(channel_->SetRecvParameters(recv_parameters_)); + EXPECT_EQ(0u, GetRecvStreamConfig(kSsrcX).rtp.extensions.size()); + + // Ensure extensions stay off with an empty list of headers. + recv_parameters_.extensions.clear(); + EXPECT_TRUE(channel_->SetRecvParameters(recv_parameters_)); + EXPECT_EQ(0u, GetRecvStreamConfig(kSsrcX).rtp.extensions.size()); + + // Ensure extension is set properly. + const int id = 2; + recv_parameters_.extensions.push_back(webrtc::RtpExtension(ext, id)); + EXPECT_TRUE(channel_->SetRecvParameters(recv_parameters_)); + EXPECT_EQ(1u, GetRecvStreamConfig(kSsrcX).rtp.extensions.size()); + EXPECT_EQ(ext, GetRecvStreamConfig(kSsrcX).rtp.extensions[0].uri); + EXPECT_EQ(id, GetRecvStreamConfig(kSsrcX).rtp.extensions[0].id); + + // Ensure extension is set properly on new stream. + EXPECT_TRUE(AddRecvStream(kSsrcY)); + EXPECT_NE(call_.GetAudioReceiveStream(kSsrcX), + call_.GetAudioReceiveStream(kSsrcY)); + EXPECT_EQ(1u, GetRecvStreamConfig(kSsrcY).rtp.extensions.size()); + EXPECT_EQ(ext, GetRecvStreamConfig(kSsrcY).rtp.extensions[0].uri); + EXPECT_EQ(id, GetRecvStreamConfig(kSsrcY).rtp.extensions[0].id); + + // Ensure all extensions go back off with an empty list. + recv_parameters_.extensions.clear(); + EXPECT_TRUE(channel_->SetRecvParameters(recv_parameters_)); + EXPECT_EQ(0u, GetRecvStreamConfig(kSsrcX).rtp.extensions.size()); + EXPECT_EQ(0u, GetRecvStreamConfig(kSsrcY).rtp.extensions.size()); + } + + webrtc::AudioSendStream::Stats GetAudioSendStreamStats() const { + webrtc::AudioSendStream::Stats stats; + stats.local_ssrc = 12; + stats.payload_bytes_sent = 345; + stats.header_and_padding_bytes_sent = 56; + stats.packets_sent = 678; + stats.packets_lost = 9012; + stats.fraction_lost = 34.56f; + stats.codec_name = "codec_name_send"; + stats.codec_payload_type = 42; + stats.jitter_ms = 12; + stats.rtt_ms = 345; + stats.audio_level = 678; + stats.apm_statistics.delay_median_ms = 234; + stats.apm_statistics.delay_standard_deviation_ms = 567; + stats.apm_statistics.echo_return_loss = 890; + stats.apm_statistics.echo_return_loss_enhancement = 1234; + stats.apm_statistics.residual_echo_likelihood = 0.432f; + stats.apm_statistics.residual_echo_likelihood_recent_max = 0.6f; + stats.ana_statistics.bitrate_action_counter = 321; + stats.ana_statistics.channel_action_counter = 432; + stats.ana_statistics.dtx_action_counter = 543; + stats.ana_statistics.fec_action_counter = 654; + stats.ana_statistics.frame_length_increase_counter = 765; + stats.ana_statistics.frame_length_decrease_counter = 876; + stats.ana_statistics.uplink_packet_loss_fraction = 987.0; + stats.typing_noise_detected = true; + return stats; + } + void SetAudioSendStreamStats() { + for (auto* s : call_.GetAudioSendStreams()) { + s->SetStats(GetAudioSendStreamStats()); + } + } + void VerifyVoiceSenderInfo(const cricket::VoiceSenderInfo& info, + bool is_sending) { + const auto stats = GetAudioSendStreamStats(); + EXPECT_EQ(info.ssrc(), stats.local_ssrc); + EXPECT_EQ(info.payload_bytes_sent, stats.payload_bytes_sent); + EXPECT_EQ(info.header_and_padding_bytes_sent, + stats.header_and_padding_bytes_sent); + EXPECT_EQ(info.packets_sent, stats.packets_sent); + EXPECT_EQ(info.packets_lost, stats.packets_lost); + EXPECT_EQ(info.fraction_lost, stats.fraction_lost); + EXPECT_EQ(info.codec_name, stats.codec_name); + EXPECT_EQ(info.codec_payload_type, stats.codec_payload_type); + EXPECT_EQ(info.jitter_ms, stats.jitter_ms); + EXPECT_EQ(info.rtt_ms, stats.rtt_ms); + EXPECT_EQ(info.audio_level, stats.audio_level); + EXPECT_EQ(info.apm_statistics.delay_median_ms, + stats.apm_statistics.delay_median_ms); + EXPECT_EQ(info.apm_statistics.delay_standard_deviation_ms, + stats.apm_statistics.delay_standard_deviation_ms); + EXPECT_EQ(info.apm_statistics.echo_return_loss, + stats.apm_statistics.echo_return_loss); + EXPECT_EQ(info.apm_statistics.echo_return_loss_enhancement, + stats.apm_statistics.echo_return_loss_enhancement); + EXPECT_EQ(info.apm_statistics.residual_echo_likelihood, + stats.apm_statistics.residual_echo_likelihood); + EXPECT_EQ(info.apm_statistics.residual_echo_likelihood_recent_max, + stats.apm_statistics.residual_echo_likelihood_recent_max); + EXPECT_EQ(info.ana_statistics.bitrate_action_counter, + stats.ana_statistics.bitrate_action_counter); + EXPECT_EQ(info.ana_statistics.channel_action_counter, + stats.ana_statistics.channel_action_counter); + EXPECT_EQ(info.ana_statistics.dtx_action_counter, + stats.ana_statistics.dtx_action_counter); + EXPECT_EQ(info.ana_statistics.fec_action_counter, + stats.ana_statistics.fec_action_counter); + EXPECT_EQ(info.ana_statistics.frame_length_increase_counter, + stats.ana_statistics.frame_length_increase_counter); + EXPECT_EQ(info.ana_statistics.frame_length_decrease_counter, + stats.ana_statistics.frame_length_decrease_counter); + EXPECT_EQ(info.ana_statistics.uplink_packet_loss_fraction, + stats.ana_statistics.uplink_packet_loss_fraction); + EXPECT_EQ(info.typing_noise_detected, + stats.typing_noise_detected && is_sending); + } + + webrtc::AudioReceiveStream::Stats GetAudioReceiveStreamStats() const { + webrtc::AudioReceiveStream::Stats stats; + stats.remote_ssrc = 123; + stats.payload_bytes_rcvd = 456; + stats.header_and_padding_bytes_rcvd = 67; + stats.packets_rcvd = 768; + stats.packets_lost = 101; + stats.codec_name = "codec_name_recv"; + stats.codec_payload_type = 42; + stats.jitter_ms = 901; + stats.jitter_buffer_ms = 234; + stats.jitter_buffer_preferred_ms = 567; + stats.delay_estimate_ms = 890; + stats.audio_level = 1234; + stats.total_samples_received = 5678901; + stats.concealed_samples = 234; + stats.concealment_events = 12; + stats.jitter_buffer_delay_seconds = 34; + stats.jitter_buffer_emitted_count = 77; + stats.expand_rate = 5.67f; + stats.speech_expand_rate = 8.90f; + stats.secondary_decoded_rate = 1.23f; + stats.secondary_discarded_rate = 0.12f; + stats.accelerate_rate = 4.56f; + stats.preemptive_expand_rate = 7.89f; + stats.decoding_calls_to_silence_generator = 12; + stats.decoding_calls_to_neteq = 345; + stats.decoding_normal = 67890; + stats.decoding_plc = 1234; + stats.decoding_codec_plc = 1236; + stats.decoding_cng = 5678; + stats.decoding_plc_cng = 9012; + stats.decoding_muted_output = 3456; + stats.capture_start_ntp_time_ms = 7890; + return stats; + } + void SetAudioReceiveStreamStats() { + for (auto* s : call_.GetAudioReceiveStreams()) { + s->SetStats(GetAudioReceiveStreamStats()); + } + } + void VerifyVoiceReceiverInfo(const cricket::VoiceReceiverInfo& info) { + const auto stats = GetAudioReceiveStreamStats(); + EXPECT_EQ(info.ssrc(), stats.remote_ssrc); + EXPECT_EQ(info.payload_bytes_rcvd, stats.payload_bytes_rcvd); + EXPECT_EQ(info.header_and_padding_bytes_rcvd, + stats.header_and_padding_bytes_rcvd); + EXPECT_EQ(rtc::checked_cast<unsigned int>(info.packets_rcvd), + stats.packets_rcvd); + EXPECT_EQ(rtc::checked_cast<unsigned int>(info.packets_lost), + stats.packets_lost); + EXPECT_EQ(info.codec_name, stats.codec_name); + EXPECT_EQ(info.codec_payload_type, stats.codec_payload_type); + EXPECT_EQ(rtc::checked_cast<unsigned int>(info.jitter_ms), stats.jitter_ms); + EXPECT_EQ(rtc::checked_cast<unsigned int>(info.jitter_buffer_ms), + stats.jitter_buffer_ms); + EXPECT_EQ(rtc::checked_cast<unsigned int>(info.jitter_buffer_preferred_ms), + stats.jitter_buffer_preferred_ms); + EXPECT_EQ(rtc::checked_cast<unsigned int>(info.delay_estimate_ms), + stats.delay_estimate_ms); + EXPECT_EQ(info.audio_level, stats.audio_level); + EXPECT_EQ(info.total_samples_received, stats.total_samples_received); + EXPECT_EQ(info.concealed_samples, stats.concealed_samples); + EXPECT_EQ(info.concealment_events, stats.concealment_events); + EXPECT_EQ(info.jitter_buffer_delay_seconds, + stats.jitter_buffer_delay_seconds); + EXPECT_EQ(info.jitter_buffer_emitted_count, + stats.jitter_buffer_emitted_count); + EXPECT_EQ(info.expand_rate, stats.expand_rate); + EXPECT_EQ(info.speech_expand_rate, stats.speech_expand_rate); + EXPECT_EQ(info.secondary_decoded_rate, stats.secondary_decoded_rate); + EXPECT_EQ(info.secondary_discarded_rate, stats.secondary_discarded_rate); + EXPECT_EQ(info.accelerate_rate, stats.accelerate_rate); + EXPECT_EQ(info.preemptive_expand_rate, stats.preemptive_expand_rate); + EXPECT_EQ(info.decoding_calls_to_silence_generator, + stats.decoding_calls_to_silence_generator); + EXPECT_EQ(info.decoding_calls_to_neteq, stats.decoding_calls_to_neteq); + EXPECT_EQ(info.decoding_normal, stats.decoding_normal); + EXPECT_EQ(info.decoding_plc, stats.decoding_plc); + EXPECT_EQ(info.decoding_codec_plc, stats.decoding_codec_plc); + EXPECT_EQ(info.decoding_cng, stats.decoding_cng); + EXPECT_EQ(info.decoding_plc_cng, stats.decoding_plc_cng); + EXPECT_EQ(info.decoding_muted_output, stats.decoding_muted_output); + EXPECT_EQ(info.capture_start_ntp_time_ms, stats.capture_start_ntp_time_ms); + } + void VerifyVoiceSendRecvCodecs(const cricket::VoiceMediaInfo& info) const { + EXPECT_EQ(send_parameters_.codecs.size(), info.send_codecs.size()); + for (const cricket::AudioCodec& codec : send_parameters_.codecs) { + ASSERT_EQ(info.send_codecs.count(codec.id), 1U); + EXPECT_EQ(info.send_codecs.find(codec.id)->second, + codec.ToCodecParameters()); + } + EXPECT_EQ(recv_parameters_.codecs.size(), info.receive_codecs.size()); + for (const cricket::AudioCodec& codec : recv_parameters_.codecs) { + ASSERT_EQ(info.receive_codecs.count(codec.id), 1U); + EXPECT_EQ(info.receive_codecs.find(codec.id)->second, + codec.ToCodecParameters()); + } + } + + void VerifyGainControlEnabledCorrectly() { + EXPECT_TRUE(apm_config_.gain_controller1.enabled); + EXPECT_EQ(kDefaultAgcMode, apm_config_.gain_controller1.mode); + EXPECT_EQ(0, apm_config_.gain_controller1.analog_level_minimum); + EXPECT_EQ(255, apm_config_.gain_controller1.analog_level_maximum); + } + + void VerifyGainControlDefaultSettings() { + EXPECT_EQ(3, apm_config_.gain_controller1.target_level_dbfs); + EXPECT_EQ(9, apm_config_.gain_controller1.compression_gain_db); + EXPECT_TRUE(apm_config_.gain_controller1.enable_limiter); + } + + void VerifyEchoCancellationSettings(bool enabled) { + constexpr bool kDefaultUseAecm = +#if defined(WEBRTC_ANDROID) + true; +#else + false; +#endif + EXPECT_EQ(apm_config_.echo_canceller.enabled, enabled); + EXPECT_EQ(apm_config_.echo_canceller.mobile_mode, kDefaultUseAecm); + } + + bool IsHighPassFilterEnabled() { + return apm_config_.high_pass_filter.enabled; + } + + bool IsTypingDetectionEnabled() { + return apm_config_.voice_detection.enabled; + } + + protected: + const bool use_null_apm_; + std::unique_ptr<webrtc::TaskQueueFactory> task_queue_factory_; + rtc::scoped_refptr<webrtc::test::MockAudioDeviceModule> adm_; + rtc::scoped_refptr<StrictMock<webrtc::test::MockAudioProcessing>> apm_; + cricket::FakeCall call_; + std::unique_ptr<cricket::WebRtcVoiceEngine> engine_; + cricket::VoiceMediaChannel* channel_ = nullptr; + cricket::AudioSendParameters send_parameters_; + cricket::AudioRecvParameters recv_parameters_; + FakeAudioSource fake_source_; + webrtc::AudioProcessing::Config apm_config_; + + private: + webrtc::test::ScopedFieldTrials override_field_trials_; +}; + +INSTANTIATE_TEST_SUITE_P(TestBothWithAndWithoutNullApm, + WebRtcVoiceEngineTestFake, + ::testing::Values(false, true)); + +// Tests that we can create and destroy a channel. +TEST_P(WebRtcVoiceEngineTestFake, CreateMediaChannel) { + EXPECT_TRUE(SetupChannel()); +} + +// Test that we can add a send stream and that it has the correct defaults. +TEST_P(WebRtcVoiceEngineTestFake, CreateSendStream) { + EXPECT_TRUE(SetupChannel()); + EXPECT_TRUE( + channel_->AddSendStream(cricket::StreamParams::CreateLegacy(kSsrcX))); + const webrtc::AudioSendStream::Config& config = GetSendStreamConfig(kSsrcX); + EXPECT_EQ(kSsrcX, config.rtp.ssrc); + EXPECT_EQ("", config.rtp.c_name); + EXPECT_EQ(0u, config.rtp.extensions.size()); + EXPECT_EQ(static_cast<cricket::WebRtcVoiceMediaChannel*>(channel_), + config.send_transport); +} + +// Test that we can add a receive stream and that it has the correct defaults. +TEST_P(WebRtcVoiceEngineTestFake, CreateRecvStream) { + EXPECT_TRUE(SetupChannel()); + EXPECT_TRUE(AddRecvStream(kSsrcX)); + const webrtc::AudioReceiveStream::Config& config = + GetRecvStreamConfig(kSsrcX); + EXPECT_EQ(kSsrcX, config.rtp.remote_ssrc); + EXPECT_EQ(0xFA17FA17, config.rtp.local_ssrc); + EXPECT_FALSE(config.rtp.transport_cc); + EXPECT_EQ(0u, config.rtp.extensions.size()); + EXPECT_EQ(static_cast<cricket::WebRtcVoiceMediaChannel*>(channel_), + config.rtcp_send_transport); + EXPECT_EQ("", config.sync_group); +} + +TEST_P(WebRtcVoiceEngineTestFake, OpusSupportsTransportCc) { + const std::vector<cricket::AudioCodec>& codecs = engine_->send_codecs(); + bool opus_found = false; + for (const cricket::AudioCodec& codec : codecs) { + if (codec.name == "opus") { + EXPECT_TRUE(HasTransportCc(codec)); + opus_found = true; + } + } + EXPECT_TRUE(opus_found); +} + +// Test that we set our inbound codecs properly, including changing PT. +TEST_P(WebRtcVoiceEngineTestFake, SetRecvCodecs) { + EXPECT_TRUE(SetupChannel()); + cricket::AudioRecvParameters parameters; + parameters.codecs.push_back(kIsacCodec); + parameters.codecs.push_back(kPcmuCodec); + parameters.codecs.push_back(kTelephoneEventCodec1); + parameters.codecs.push_back(kTelephoneEventCodec2); + parameters.codecs[0].id = 106; // collide with existing CN 32k + parameters.codecs[2].id = 126; + EXPECT_TRUE(channel_->SetRecvParameters(parameters)); + EXPECT_TRUE(AddRecvStream(kSsrcX)); + EXPECT_THAT(GetRecvStreamConfig(kSsrcX).decoder_map, + (ContainerEq<std::map<int, webrtc::SdpAudioFormat>>( + {{0, {"PCMU", 8000, 1}}, + {106, {"ISAC", 16000, 1}}, + {126, {"telephone-event", 8000, 1}}, + {107, {"telephone-event", 32000, 1}}}))); +} + +// Test that we fail to set an unknown inbound codec. +TEST_P(WebRtcVoiceEngineTestFake, SetRecvCodecsUnsupportedCodec) { + EXPECT_TRUE(SetupChannel()); + cricket::AudioRecvParameters parameters; + parameters.codecs.push_back(kIsacCodec); + parameters.codecs.push_back(cricket::AudioCodec(127, "XYZ", 32000, 0, 1)); + EXPECT_FALSE(channel_->SetRecvParameters(parameters)); +} + +// Test that we fail if we have duplicate types in the inbound list. +TEST_P(WebRtcVoiceEngineTestFake, SetRecvCodecsDuplicatePayloadType) { + EXPECT_TRUE(SetupChannel()); + cricket::AudioRecvParameters parameters; + parameters.codecs.push_back(kIsacCodec); + parameters.codecs.push_back(kCn16000Codec); + parameters.codecs[1].id = kIsacCodec.id; + EXPECT_FALSE(channel_->SetRecvParameters(parameters)); +} + +// Test that we can decode OPUS without stereo parameters. +TEST_P(WebRtcVoiceEngineTestFake, SetRecvCodecsWithOpusNoStereo) { + EXPECT_TRUE(SetupChannel()); + cricket::AudioRecvParameters parameters; + parameters.codecs.push_back(kIsacCodec); + parameters.codecs.push_back(kPcmuCodec); + parameters.codecs.push_back(kOpusCodec); + EXPECT_TRUE(channel_->SetRecvParameters(parameters)); + EXPECT_TRUE(AddRecvStream(kSsrcX)); + EXPECT_THAT(GetRecvStreamConfig(kSsrcX).decoder_map, + (ContainerEq<std::map<int, webrtc::SdpAudioFormat>>( + {{0, {"PCMU", 8000, 1}}, + {103, {"ISAC", 16000, 1}}, + {111, {"opus", 48000, 2}}}))); +} + +// Test that we can decode OPUS with stereo = 0. +TEST_P(WebRtcVoiceEngineTestFake, SetRecvCodecsWithOpus0Stereo) { + EXPECT_TRUE(SetupChannel()); + cricket::AudioRecvParameters parameters; + parameters.codecs.push_back(kIsacCodec); + parameters.codecs.push_back(kPcmuCodec); + parameters.codecs.push_back(kOpusCodec); + parameters.codecs[2].params["stereo"] = "0"; + EXPECT_TRUE(channel_->SetRecvParameters(parameters)); + EXPECT_TRUE(AddRecvStream(kSsrcX)); + EXPECT_THAT(GetRecvStreamConfig(kSsrcX).decoder_map, + (ContainerEq<std::map<int, webrtc::SdpAudioFormat>>( + {{0, {"PCMU", 8000, 1}}, + {103, {"ISAC", 16000, 1}}, + {111, {"opus", 48000, 2, {{"stereo", "0"}}}}}))); +} + +// Test that we can decode OPUS with stereo = 1. +TEST_P(WebRtcVoiceEngineTestFake, SetRecvCodecsWithOpus1Stereo) { + EXPECT_TRUE(SetupChannel()); + cricket::AudioRecvParameters parameters; + parameters.codecs.push_back(kIsacCodec); + parameters.codecs.push_back(kPcmuCodec); + parameters.codecs.push_back(kOpusCodec); + parameters.codecs[2].params["stereo"] = "1"; + EXPECT_TRUE(channel_->SetRecvParameters(parameters)); + EXPECT_TRUE(AddRecvStream(kSsrcX)); + EXPECT_THAT(GetRecvStreamConfig(kSsrcX).decoder_map, + (ContainerEq<std::map<int, webrtc::SdpAudioFormat>>( + {{0, {"PCMU", 8000, 1}}, + {103, {"ISAC", 16000, 1}}, + {111, {"opus", 48000, 2, {{"stereo", "1"}}}}}))); +} + +// Test that changes to recv codecs are applied to all streams. +TEST_P(WebRtcVoiceEngineTestFake, SetRecvCodecsWithMultipleStreams) { + EXPECT_TRUE(SetupChannel()); + cricket::AudioRecvParameters parameters; + parameters.codecs.push_back(kIsacCodec); + parameters.codecs.push_back(kPcmuCodec); + parameters.codecs.push_back(kTelephoneEventCodec1); + parameters.codecs.push_back(kTelephoneEventCodec2); + parameters.codecs[0].id = 106; // collide with existing CN 32k + parameters.codecs[2].id = 126; + EXPECT_TRUE(channel_->SetRecvParameters(parameters)); + for (const auto& ssrc : {kSsrcX, kSsrcY}) { + EXPECT_TRUE(AddRecvStream(ssrc)); + EXPECT_THAT(GetRecvStreamConfig(ssrc).decoder_map, + (ContainerEq<std::map<int, webrtc::SdpAudioFormat>>( + {{0, {"PCMU", 8000, 1}}, + {106, {"ISAC", 16000, 1}}, + {126, {"telephone-event", 8000, 1}}, + {107, {"telephone-event", 32000, 1}}}))); + } +} + +TEST_P(WebRtcVoiceEngineTestFake, SetRecvCodecsAfterAddingStreams) { + EXPECT_TRUE(SetupRecvStream()); + cricket::AudioRecvParameters parameters; + parameters.codecs.push_back(kIsacCodec); + parameters.codecs[0].id = 106; // collide with existing CN 32k + EXPECT_TRUE(channel_->SetRecvParameters(parameters)); + + const auto& dm = GetRecvStreamConfig(kSsrcX).decoder_map; + ASSERT_EQ(1u, dm.count(106)); + EXPECT_EQ(webrtc::SdpAudioFormat("isac", 16000, 1), dm.at(106)); +} + +// Test that we can apply the same set of codecs again while playing. +TEST_P(WebRtcVoiceEngineTestFake, SetRecvCodecsWhilePlaying) { + EXPECT_TRUE(SetupRecvStream()); + cricket::AudioRecvParameters parameters; + parameters.codecs.push_back(kIsacCodec); + parameters.codecs.push_back(kCn16000Codec); + EXPECT_TRUE(channel_->SetRecvParameters(parameters)); + channel_->SetPlayout(true); + EXPECT_TRUE(channel_->SetRecvParameters(parameters)); + + // Remapping a payload type to a different codec should fail. + parameters.codecs[0] = kOpusCodec; + parameters.codecs[0].id = kIsacCodec.id; + EXPECT_FALSE(channel_->SetRecvParameters(parameters)); + EXPECT_TRUE(GetRecvStream(kSsrcX).started()); +} + +// Test that we can add a codec while playing. +TEST_P(WebRtcVoiceEngineTestFake, AddRecvCodecsWhilePlaying) { + EXPECT_TRUE(SetupRecvStream()); + cricket::AudioRecvParameters parameters; + parameters.codecs.push_back(kIsacCodec); + parameters.codecs.push_back(kCn16000Codec); + EXPECT_TRUE(channel_->SetRecvParameters(parameters)); + channel_->SetPlayout(true); + + parameters.codecs.push_back(kOpusCodec); + EXPECT_TRUE(channel_->SetRecvParameters(parameters)); + EXPECT_TRUE(GetRecvStream(kSsrcX).started()); +} + +// Test that we accept adding the same codec with a different payload type. +// See: https://bugs.chromium.org/p/webrtc/issues/detail?id=5847 +TEST_P(WebRtcVoiceEngineTestFake, ChangeRecvCodecPayloadType) { + EXPECT_TRUE(SetupRecvStream()); + cricket::AudioRecvParameters parameters; + parameters.codecs.push_back(kIsacCodec); + EXPECT_TRUE(channel_->SetRecvParameters(parameters)); + + ++parameters.codecs[0].id; + EXPECT_TRUE(channel_->SetRecvParameters(parameters)); +} + +TEST_P(WebRtcVoiceEngineTestFake, SetSendBandwidthAuto) { + EXPECT_TRUE(SetupSendStream()); + + // Test that when autobw is enabled, bitrate is kept as the default + // value. autobw is enabled for the following tests because the target + // bitrate is <= 0. + + // ISAC, default bitrate == 32000. + TestMaxSendBandwidth(kIsacCodec, 0, true, 32000); + + // PCMU, default bitrate == 64000. + TestMaxSendBandwidth(kPcmuCodec, -1, true, 64000); + + // opus, default bitrate == 32000 in mono. + TestMaxSendBandwidth(kOpusCodec, -1, true, 32000); +} + +TEST_P(WebRtcVoiceEngineTestFake, SetMaxSendBandwidthMultiRateAsCaller) { + EXPECT_TRUE(SetupSendStream()); + + // ISAC, default bitrate == 32000. + TestMaxSendBandwidth(kIsacCodec, 16000, true, 16000); + // Rates above the max (56000) should be capped. + TestMaxSendBandwidth(kIsacCodec, 100000, true, 32000); + + // opus, default bitrate == 64000. + TestMaxSendBandwidth(kOpusCodec, 96000, true, 96000); + TestMaxSendBandwidth(kOpusCodec, 48000, true, 48000); + // Rates above the max (510000) should be capped. + TestMaxSendBandwidth(kOpusCodec, 600000, true, 510000); +} + +TEST_P(WebRtcVoiceEngineTestFake, SetMaxSendBandwidthFixedRateAsCaller) { + EXPECT_TRUE(SetupSendStream()); + + // Test that we can only set a maximum bitrate for a fixed-rate codec + // if it's bigger than the fixed rate. + + // PCMU, fixed bitrate == 64000. + TestMaxSendBandwidth(kPcmuCodec, 0, true, 64000); + TestMaxSendBandwidth(kPcmuCodec, 1, false, 64000); + TestMaxSendBandwidth(kPcmuCodec, 128000, true, 64000); + TestMaxSendBandwidth(kPcmuCodec, 32000, false, 64000); + TestMaxSendBandwidth(kPcmuCodec, 64000, true, 64000); + TestMaxSendBandwidth(kPcmuCodec, 63999, false, 64000); + TestMaxSendBandwidth(kPcmuCodec, 64001, true, 64000); +} + +TEST_P(WebRtcVoiceEngineTestFake, SetMaxSendBandwidthMultiRateAsCallee) { + EXPECT_TRUE(SetupChannel()); + const int kDesiredBitrate = 128000; + cricket::AudioSendParameters parameters; + parameters.codecs = engine_->send_codecs(); + parameters.max_bandwidth_bps = kDesiredBitrate; + SetSendParameters(parameters); + + EXPECT_TRUE( + channel_->AddSendStream(cricket::StreamParams::CreateLegacy(kSsrcX))); + + EXPECT_EQ(kDesiredBitrate, GetCodecBitrate(kSsrcX)); +} + +// Test that bitrate cannot be set for CBR codecs. +// Bitrate is ignored if it is higher than the fixed bitrate. +// Bitrate less then the fixed bitrate is an error. +TEST_P(WebRtcVoiceEngineTestFake, SetMaxSendBandwidthCbr) { + EXPECT_TRUE(SetupSendStream()); + + // PCMU, default bitrate == 64000. + SetSendParameters(send_parameters_); + EXPECT_EQ(64000, GetCodecBitrate(kSsrcX)); + + send_parameters_.max_bandwidth_bps = 128000; + SetSendParameters(send_parameters_); + EXPECT_EQ(64000, GetCodecBitrate(kSsrcX)); + + send_parameters_.max_bandwidth_bps = 128; + EXPECT_FALSE(channel_->SetSendParameters(send_parameters_)); + EXPECT_EQ(64000, GetCodecBitrate(kSsrcX)); +} + +// Test that the per-stream bitrate limit and the global +// bitrate limit both apply. +TEST_P(WebRtcVoiceEngineTestFake, SetMaxBitratePerStream) { + EXPECT_TRUE(SetupSendStream()); + + // opus, default bitrate == 32000. + SetAndExpectMaxBitrate(kOpusCodec, 0, 0, true, 32000); + SetAndExpectMaxBitrate(kOpusCodec, 48000, 0, true, 48000); + SetAndExpectMaxBitrate(kOpusCodec, 48000, 64000, true, 48000); + SetAndExpectMaxBitrate(kOpusCodec, 64000, 48000, true, 48000); + + // CBR codecs allow both maximums to exceed the bitrate. + SetAndExpectMaxBitrate(kPcmuCodec, 0, 0, true, 64000); + SetAndExpectMaxBitrate(kPcmuCodec, 64001, 0, true, 64000); + SetAndExpectMaxBitrate(kPcmuCodec, 0, 64001, true, 64000); + SetAndExpectMaxBitrate(kPcmuCodec, 64001, 64001, true, 64000); + + // CBR codecs don't allow per stream maximums to be too low. + SetAndExpectMaxBitrate(kPcmuCodec, 0, 63999, false, 64000); + SetAndExpectMaxBitrate(kPcmuCodec, 64001, 63999, false, 64000); +} + +// Test that an attempt to set RtpParameters for a stream that does not exist +// fails. +TEST_P(WebRtcVoiceEngineTestFake, CannotSetMaxBitrateForNonexistentStream) { + EXPECT_TRUE(SetupChannel()); + webrtc::RtpParameters nonexistent_parameters = + channel_->GetRtpSendParameters(kSsrcX); + EXPECT_EQ(0u, nonexistent_parameters.encodings.size()); + + nonexistent_parameters.encodings.push_back(webrtc::RtpEncodingParameters()); + EXPECT_FALSE( + channel_->SetRtpSendParameters(kSsrcX, nonexistent_parameters).ok()); +} + +TEST_P(WebRtcVoiceEngineTestFake, + CannotSetRtpSendParametersWithIncorrectNumberOfEncodings) { + // This test verifies that setting RtpParameters succeeds only if + // the structure contains exactly one encoding. + // TODO(skvlad): Update this test when we start supporting setting parameters + // for each encoding individually. + + EXPECT_TRUE(SetupSendStream()); + webrtc::RtpParameters parameters = channel_->GetRtpSendParameters(kSsrcX); + // Two or more encodings should result in failure. + parameters.encodings.push_back(webrtc::RtpEncodingParameters()); + EXPECT_FALSE(channel_->SetRtpSendParameters(kSsrcX, parameters).ok()); + // Zero encodings should also fail. + parameters.encodings.clear(); + EXPECT_FALSE(channel_->SetRtpSendParameters(kSsrcX, parameters).ok()); +} + +// Changing the SSRC through RtpParameters is not allowed. +TEST_P(WebRtcVoiceEngineTestFake, CannotSetSsrcInRtpSendParameters) { + EXPECT_TRUE(SetupSendStream()); + webrtc::RtpParameters parameters = channel_->GetRtpSendParameters(kSsrcX); + parameters.encodings[0].ssrc = 0xdeadbeef; + EXPECT_FALSE(channel_->SetRtpSendParameters(kSsrcX, parameters).ok()); +} + +// Test that a stream will not be sending if its encoding is made +// inactive through SetRtpSendParameters. +TEST_P(WebRtcVoiceEngineTestFake, SetRtpParametersEncodingsActive) { + EXPECT_TRUE(SetupSendStream()); + SetSend(true); + EXPECT_TRUE(GetSendStream(kSsrcX).IsSending()); + // Get current parameters and change "active" to false. + webrtc::RtpParameters parameters = channel_->GetRtpSendParameters(kSsrcX); + ASSERT_EQ(1u, parameters.encodings.size()); + ASSERT_TRUE(parameters.encodings[0].active); + parameters.encodings[0].active = false; + EXPECT_TRUE(channel_->SetRtpSendParameters(kSsrcX, parameters).ok()); + EXPECT_FALSE(GetSendStream(kSsrcX).IsSending()); + + // Now change it back to active and verify we resume sending. + // This should occur even when other parameters are updated. + parameters.encodings[0].active = true; + parameters.encodings[0].max_bitrate_bps = absl::optional<int>(6000); + EXPECT_TRUE(channel_->SetRtpSendParameters(kSsrcX, parameters).ok()); + EXPECT_TRUE(GetSendStream(kSsrcX).IsSending()); +} + +// Test that SetRtpSendParameters configures the correct encoding channel for +// each SSRC. +TEST_P(WebRtcVoiceEngineTestFake, RtpParametersArePerStream) { + SetupForMultiSendStream(); + // Create send streams. + for (uint32_t ssrc : kSsrcs4) { + EXPECT_TRUE( + channel_->AddSendStream(cricket::StreamParams::CreateLegacy(ssrc))); + } + // Configure one stream to be limited by the stream config, another to be + // limited by the global max, and the third one with no per-stream limit + // (still subject to the global limit). + SetGlobalMaxBitrate(kOpusCodec, 32000); + EXPECT_TRUE(SetMaxBitrateForStream(kSsrcs4[0], 24000)); + EXPECT_TRUE(SetMaxBitrateForStream(kSsrcs4[1], 48000)); + EXPECT_TRUE(SetMaxBitrateForStream(kSsrcs4[2], -1)); + + EXPECT_EQ(24000, GetCodecBitrate(kSsrcs4[0])); + EXPECT_EQ(32000, GetCodecBitrate(kSsrcs4[1])); + EXPECT_EQ(32000, GetCodecBitrate(kSsrcs4[2])); + + // Remove the global cap; the streams should switch to their respective + // maximums (or remain unchanged if there was no other limit on them.) + SetGlobalMaxBitrate(kOpusCodec, -1); + EXPECT_EQ(24000, GetCodecBitrate(kSsrcs4[0])); + EXPECT_EQ(48000, GetCodecBitrate(kSsrcs4[1])); + EXPECT_EQ(32000, GetCodecBitrate(kSsrcs4[2])); +} + +// Test that GetRtpSendParameters returns the currently configured codecs. +TEST_P(WebRtcVoiceEngineTestFake, GetRtpSendParametersCodecs) { + EXPECT_TRUE(SetupSendStream()); + cricket::AudioSendParameters parameters; + parameters.codecs.push_back(kIsacCodec); + parameters.codecs.push_back(kPcmuCodec); + SetSendParameters(parameters); + + webrtc::RtpParameters rtp_parameters = channel_->GetRtpSendParameters(kSsrcX); + ASSERT_EQ(2u, rtp_parameters.codecs.size()); + EXPECT_EQ(kIsacCodec.ToCodecParameters(), rtp_parameters.codecs[0]); + EXPECT_EQ(kPcmuCodec.ToCodecParameters(), rtp_parameters.codecs[1]); +} + +// Test that GetRtpSendParameters returns the currently configured RTCP CNAME. +TEST_P(WebRtcVoiceEngineTestFake, GetRtpSendParametersRtcpCname) { + cricket::StreamParams params = cricket::StreamParams::CreateLegacy(kSsrcX); + params.cname = "rtcpcname"; + EXPECT_TRUE(SetupSendStream(params)); + + webrtc::RtpParameters rtp_parameters = channel_->GetRtpSendParameters(kSsrcX); + EXPECT_STREQ("rtcpcname", rtp_parameters.rtcp.cname.c_str()); +} + +TEST_P(WebRtcVoiceEngineTestFake, + DetectRtpSendParameterHeaderExtensionsChange) { + EXPECT_TRUE(SetupSendStream()); + + webrtc::RtpParameters rtp_parameters = channel_->GetRtpSendParameters(kSsrcX); + rtp_parameters.header_extensions.emplace_back(); + + EXPECT_NE(0u, rtp_parameters.header_extensions.size()); + + webrtc::RTCError result = + channel_->SetRtpSendParameters(kSsrcX, rtp_parameters); + EXPECT_EQ(webrtc::RTCErrorType::INVALID_MODIFICATION, result.type()); +} + +// Test that GetRtpSendParameters returns an SSRC. +TEST_P(WebRtcVoiceEngineTestFake, GetRtpSendParametersSsrc) { + EXPECT_TRUE(SetupSendStream()); + webrtc::RtpParameters rtp_parameters = channel_->GetRtpSendParameters(kSsrcX); + ASSERT_EQ(1u, rtp_parameters.encodings.size()); + EXPECT_EQ(kSsrcX, rtp_parameters.encodings[0].ssrc); +} + +// Test that if we set/get parameters multiple times, we get the same results. +TEST_P(WebRtcVoiceEngineTestFake, SetAndGetRtpSendParameters) { + EXPECT_TRUE(SetupSendStream()); + cricket::AudioSendParameters parameters; + parameters.codecs.push_back(kIsacCodec); + parameters.codecs.push_back(kPcmuCodec); + SetSendParameters(parameters); + + webrtc::RtpParameters initial_params = channel_->GetRtpSendParameters(kSsrcX); + + // We should be able to set the params we just got. + EXPECT_TRUE(channel_->SetRtpSendParameters(kSsrcX, initial_params).ok()); + + // ... And this shouldn't change the params returned by GetRtpSendParameters. + webrtc::RtpParameters new_params = channel_->GetRtpSendParameters(kSsrcX); + EXPECT_EQ(initial_params, channel_->GetRtpSendParameters(kSsrcX)); +} + +// Test that max_bitrate_bps in send stream config gets updated correctly when +// SetRtpSendParameters is called. +TEST_P(WebRtcVoiceEngineTestFake, SetRtpSendParameterUpdatesMaxBitrate) { + webrtc::test::ScopedFieldTrials override_field_trials( + "WebRTC-Audio-SendSideBwe/Enabled/"); + EXPECT_TRUE(SetupSendStream()); + cricket::AudioSendParameters send_parameters; + send_parameters.codecs.push_back(kOpusCodec); + SetSendParameters(send_parameters); + + webrtc::RtpParameters rtp_parameters = channel_->GetRtpSendParameters(kSsrcX); + // Expect empty on parameters.encodings[0].max_bitrate_bps; + EXPECT_FALSE(rtp_parameters.encodings[0].max_bitrate_bps); + + constexpr int kMaxBitrateBps = 6000; + rtp_parameters.encodings[0].max_bitrate_bps = kMaxBitrateBps; + EXPECT_TRUE(channel_->SetRtpSendParameters(kSsrcX, rtp_parameters).ok()); + + const int max_bitrate = GetSendStreamConfig(kSsrcX).max_bitrate_bps; + EXPECT_EQ(max_bitrate, kMaxBitrateBps); +} + +// Tests that when RTCRtpEncodingParameters.bitrate_priority gets set to +// a value <= 0, setting the parameters returns false. +TEST_P(WebRtcVoiceEngineTestFake, SetRtpSendParameterInvalidBitratePriority) { + EXPECT_TRUE(SetupSendStream()); + webrtc::RtpParameters rtp_parameters = channel_->GetRtpSendParameters(kSsrcX); + EXPECT_EQ(1UL, rtp_parameters.encodings.size()); + EXPECT_EQ(webrtc::kDefaultBitratePriority, + rtp_parameters.encodings[0].bitrate_priority); + + rtp_parameters.encodings[0].bitrate_priority = 0; + EXPECT_FALSE(channel_->SetRtpSendParameters(kSsrcX, rtp_parameters).ok()); + rtp_parameters.encodings[0].bitrate_priority = -1.0; + EXPECT_FALSE(channel_->SetRtpSendParameters(kSsrcX, rtp_parameters).ok()); +} + +// Test that the bitrate_priority in the send stream config gets updated when +// SetRtpSendParameters is set for the VoiceMediaChannel. +TEST_P(WebRtcVoiceEngineTestFake, SetRtpSendParameterUpdatesBitratePriority) { + EXPECT_TRUE(SetupSendStream()); + webrtc::RtpParameters rtp_parameters = channel_->GetRtpSendParameters(kSsrcX); + + EXPECT_EQ(1UL, rtp_parameters.encodings.size()); + EXPECT_EQ(webrtc::kDefaultBitratePriority, + rtp_parameters.encodings[0].bitrate_priority); + double new_bitrate_priority = 2.0; + rtp_parameters.encodings[0].bitrate_priority = new_bitrate_priority; + EXPECT_TRUE(channel_->SetRtpSendParameters(kSsrcX, rtp_parameters).ok()); + + // The priority should get set for both the audio channel's rtp parameters + // and the audio send stream's audio config. + EXPECT_EQ( + new_bitrate_priority, + channel_->GetRtpSendParameters(kSsrcX).encodings[0].bitrate_priority); + EXPECT_EQ(new_bitrate_priority, GetSendStreamConfig(kSsrcX).bitrate_priority); +} + +// Test that GetRtpReceiveParameters returns the currently configured codecs. +TEST_P(WebRtcVoiceEngineTestFake, GetRtpReceiveParametersCodecs) { + EXPECT_TRUE(SetupRecvStream()); + cricket::AudioRecvParameters parameters; + parameters.codecs.push_back(kIsacCodec); + parameters.codecs.push_back(kPcmuCodec); + EXPECT_TRUE(channel_->SetRecvParameters(parameters)); + + webrtc::RtpParameters rtp_parameters = + channel_->GetRtpReceiveParameters(kSsrcX); + ASSERT_EQ(2u, rtp_parameters.codecs.size()); + EXPECT_EQ(kIsacCodec.ToCodecParameters(), rtp_parameters.codecs[0]); + EXPECT_EQ(kPcmuCodec.ToCodecParameters(), rtp_parameters.codecs[1]); +} + +// Test that GetRtpReceiveParameters returns an SSRC. +TEST_P(WebRtcVoiceEngineTestFake, GetRtpReceiveParametersSsrc) { + EXPECT_TRUE(SetupRecvStream()); + webrtc::RtpParameters rtp_parameters = + channel_->GetRtpReceiveParameters(kSsrcX); + ASSERT_EQ(1u, rtp_parameters.encodings.size()); + EXPECT_EQ(kSsrcX, rtp_parameters.encodings[0].ssrc); +} + +// Test that if we set/get parameters multiple times, we get the same results. +TEST_P(WebRtcVoiceEngineTestFake, SetAndGetRtpReceiveParameters) { + EXPECT_TRUE(SetupRecvStream()); + cricket::AudioRecvParameters parameters; + parameters.codecs.push_back(kIsacCodec); + parameters.codecs.push_back(kPcmuCodec); + EXPECT_TRUE(channel_->SetRecvParameters(parameters)); + + webrtc::RtpParameters initial_params = + channel_->GetRtpReceiveParameters(kSsrcX); + + // ... And this shouldn't change the params returned by + // GetRtpReceiveParameters. + webrtc::RtpParameters new_params = channel_->GetRtpReceiveParameters(kSsrcX); + EXPECT_EQ(initial_params, channel_->GetRtpReceiveParameters(kSsrcX)); +} + +// Test that GetRtpReceiveParameters returns parameters correctly when SSRCs +// aren't signaled. It should return an empty "RtpEncodingParameters" when +// configured to receive an unsignaled stream and no packets have been received +// yet, and start returning the SSRC once a packet has been received. +TEST_P(WebRtcVoiceEngineTestFake, GetRtpReceiveParametersWithUnsignaledSsrc) { + ASSERT_TRUE(SetupChannel()); + // Call necessary methods to configure receiving a default stream as + // soon as it arrives. + cricket::AudioRecvParameters parameters; + parameters.codecs.push_back(kIsacCodec); + parameters.codecs.push_back(kPcmuCodec); + EXPECT_TRUE(channel_->SetRecvParameters(parameters)); + + // Call GetDefaultRtpReceiveParameters before configured to receive an + // unsignaled stream. Should return nothing. + EXPECT_EQ(webrtc::RtpParameters(), + channel_->GetDefaultRtpReceiveParameters()); + + // Set a sink for an unsignaled stream. + std::unique_ptr<FakeAudioSink> fake_sink(new FakeAudioSink()); + channel_->SetDefaultRawAudioSink(std::move(fake_sink)); + + // Call GetDefaultRtpReceiveParameters before the SSRC is known. + webrtc::RtpParameters rtp_parameters = + channel_->GetDefaultRtpReceiveParameters(); + ASSERT_EQ(1u, rtp_parameters.encodings.size()); + EXPECT_FALSE(rtp_parameters.encodings[0].ssrc); + + // Receive PCMU packet (SSRC=1). + DeliverPacket(kPcmuFrame, sizeof(kPcmuFrame)); + + // The |ssrc| member should still be unset. + rtp_parameters = channel_->GetDefaultRtpReceiveParameters(); + ASSERT_EQ(1u, rtp_parameters.encodings.size()); + EXPECT_FALSE(rtp_parameters.encodings[0].ssrc); +} + +// Test that we apply codecs properly. +TEST_P(WebRtcVoiceEngineTestFake, SetSendCodecs) { + EXPECT_TRUE(SetupSendStream()); + cricket::AudioSendParameters parameters; + parameters.codecs.push_back(kIsacCodec); + parameters.codecs.push_back(kPcmuCodec); + parameters.codecs.push_back(kCn8000Codec); + parameters.codecs[0].id = 96; + parameters.codecs[0].bitrate = 22000; + SetSendParameters(parameters); + const auto& send_codec_spec = *GetSendStreamConfig(kSsrcX).send_codec_spec; + EXPECT_EQ(96, send_codec_spec.payload_type); + EXPECT_EQ(22000, send_codec_spec.target_bitrate_bps); + EXPECT_STRCASEEQ("ISAC", send_codec_spec.format.name.c_str()); + EXPECT_NE(send_codec_spec.format.clockrate_hz, 8000); + EXPECT_EQ(absl::nullopt, send_codec_spec.cng_payload_type); + EXPECT_FALSE(channel_->CanInsertDtmf()); +} + +// Test that WebRtcVoiceEngine reconfigures, rather than recreates its +// AudioSendStream. +TEST_P(WebRtcVoiceEngineTestFake, DontRecreateSendStream) { + EXPECT_TRUE(SetupSendStream()); + cricket::AudioSendParameters parameters; + parameters.codecs.push_back(kIsacCodec); + parameters.codecs.push_back(kPcmuCodec); + parameters.codecs.push_back(kCn8000Codec); + parameters.codecs[0].id = 96; + parameters.codecs[0].bitrate = 48000; + const int initial_num = call_.GetNumCreatedSendStreams(); + SetSendParameters(parameters); + EXPECT_EQ(initial_num, call_.GetNumCreatedSendStreams()); + // Calling SetSendCodec again with same codec which is already set. + // In this case media channel shouldn't send codec to VoE. + SetSendParameters(parameters); + EXPECT_EQ(initial_num, call_.GetNumCreatedSendStreams()); +} + +// TODO(ossu): Revisit if these tests need to be here, now that these kinds of +// tests should be available in AudioEncoderOpusTest. + +// Test that if clockrate is not 48000 for opus, we fail. +TEST_P(WebRtcVoiceEngineTestFake, SetSendCodecOpusBadClockrate) { + EXPECT_TRUE(SetupSendStream()); + cricket::AudioSendParameters parameters; + parameters.codecs.push_back(kOpusCodec); + parameters.codecs[0].bitrate = 0; + parameters.codecs[0].clockrate = 50000; + EXPECT_FALSE(channel_->SetSendParameters(parameters)); +} + +// Test that if channels=0 for opus, we fail. +TEST_P(WebRtcVoiceEngineTestFake, SetSendCodecOpusBad0ChannelsNoStereo) { + EXPECT_TRUE(SetupSendStream()); + cricket::AudioSendParameters parameters; + parameters.codecs.push_back(kOpusCodec); + parameters.codecs[0].bitrate = 0; + parameters.codecs[0].channels = 0; + EXPECT_FALSE(channel_->SetSendParameters(parameters)); +} + +// Test that if channels=0 for opus, we fail. +TEST_P(WebRtcVoiceEngineTestFake, SetSendCodecOpusBad0Channels1Stereo) { + EXPECT_TRUE(SetupSendStream()); + cricket::AudioSendParameters parameters; + parameters.codecs.push_back(kOpusCodec); + parameters.codecs[0].bitrate = 0; + parameters.codecs[0].channels = 0; + parameters.codecs[0].params["stereo"] = "1"; + EXPECT_FALSE(channel_->SetSendParameters(parameters)); +} + +// Test that if channel is 1 for opus and there's no stereo, we fail. +TEST_P(WebRtcVoiceEngineTestFake, SetSendCodecOpus1ChannelNoStereo) { + EXPECT_TRUE(SetupSendStream()); + cricket::AudioSendParameters parameters; + parameters.codecs.push_back(kOpusCodec); + parameters.codecs[0].bitrate = 0; + parameters.codecs[0].channels = 1; + EXPECT_FALSE(channel_->SetSendParameters(parameters)); +} + +// Test that if channel is 1 for opus and stereo=0, we fail. +TEST_P(WebRtcVoiceEngineTestFake, SetSendCodecOpusBad1Channel0Stereo) { + EXPECT_TRUE(SetupSendStream()); + cricket::AudioSendParameters parameters; + parameters.codecs.push_back(kOpusCodec); + parameters.codecs[0].bitrate = 0; + parameters.codecs[0].channels = 1; + parameters.codecs[0].params["stereo"] = "0"; + EXPECT_FALSE(channel_->SetSendParameters(parameters)); +} + +// Test that if channel is 1 for opus and stereo=1, we fail. +TEST_P(WebRtcVoiceEngineTestFake, SetSendCodecOpusBad1Channel1Stereo) { + EXPECT_TRUE(SetupSendStream()); + cricket::AudioSendParameters parameters; + parameters.codecs.push_back(kOpusCodec); + parameters.codecs[0].bitrate = 0; + parameters.codecs[0].channels = 1; + parameters.codecs[0].params["stereo"] = "1"; + EXPECT_FALSE(channel_->SetSendParameters(parameters)); +} + +// Test that with bitrate=0 and no stereo, bitrate is 32000. +TEST_P(WebRtcVoiceEngineTestFake, SetSendCodecOpusGood0BitrateNoStereo) { + EXPECT_TRUE(SetupSendStream()); + cricket::AudioSendParameters parameters; + parameters.codecs.push_back(kOpusCodec); + parameters.codecs[0].bitrate = 0; + SetSendParameters(parameters); + CheckSendCodecBitrate(kSsrcX, "opus", 32000); +} + +// Test that with bitrate=0 and stereo=0, bitrate is 32000. +TEST_P(WebRtcVoiceEngineTestFake, SetSendCodecOpusGood0Bitrate0Stereo) { + EXPECT_TRUE(SetupSendStream()); + cricket::AudioSendParameters parameters; + parameters.codecs.push_back(kOpusCodec); + parameters.codecs[0].bitrate = 0; + parameters.codecs[0].params["stereo"] = "0"; + SetSendParameters(parameters); + CheckSendCodecBitrate(kSsrcX, "opus", 32000); +} + +// Test that with bitrate=invalid and stereo=0, bitrate is 32000. +TEST_P(WebRtcVoiceEngineTestFake, SetSendCodecOpusGoodXBitrate0Stereo) { + EXPECT_TRUE(SetupSendStream()); + cricket::AudioSendParameters parameters; + parameters.codecs.push_back(kOpusCodec); + parameters.codecs[0].params["stereo"] = "0"; + // bitrate that's out of the range between 6000 and 510000 will be clamped. + parameters.codecs[0].bitrate = 5999; + SetSendParameters(parameters); + CheckSendCodecBitrate(kSsrcX, "opus", 6000); + + parameters.codecs[0].bitrate = 510001; + SetSendParameters(parameters); + CheckSendCodecBitrate(kSsrcX, "opus", 510000); +} + +// Test that with bitrate=0 and stereo=1, bitrate is 64000. +TEST_P(WebRtcVoiceEngineTestFake, SetSendCodecOpusGood0Bitrate1Stereo) { + EXPECT_TRUE(SetupSendStream()); + cricket::AudioSendParameters parameters; + parameters.codecs.push_back(kOpusCodec); + parameters.codecs[0].bitrate = 0; + parameters.codecs[0].params["stereo"] = "1"; + SetSendParameters(parameters); + CheckSendCodecBitrate(kSsrcX, "opus", 64000); +} + +// Test that with bitrate=invalid and stereo=1, bitrate is 64000. +TEST_P(WebRtcVoiceEngineTestFake, SetSendCodecOpusGoodXBitrate1Stereo) { + EXPECT_TRUE(SetupSendStream()); + cricket::AudioSendParameters parameters; + parameters.codecs.push_back(kOpusCodec); + parameters.codecs[0].params["stereo"] = "1"; + // bitrate that's out of the range between 6000 and 510000 will be clamped. + parameters.codecs[0].bitrate = 5999; + SetSendParameters(parameters); + CheckSendCodecBitrate(kSsrcX, "opus", 6000); + + parameters.codecs[0].bitrate = 510001; + SetSendParameters(parameters); + CheckSendCodecBitrate(kSsrcX, "opus", 510000); +} + +// Test that with bitrate=N and stereo unset, bitrate is N. +TEST_P(WebRtcVoiceEngineTestFake, SetSendCodecOpusGoodNBitrateNoStereo) { + EXPECT_TRUE(SetupSendStream()); + cricket::AudioSendParameters parameters; + parameters.codecs.push_back(kOpusCodec); + parameters.codecs[0].bitrate = 96000; + SetSendParameters(parameters); + const auto& spec = *GetSendStreamConfig(kSsrcX).send_codec_spec; + EXPECT_EQ(111, spec.payload_type); + EXPECT_EQ(96000, spec.target_bitrate_bps); + EXPECT_EQ("opus", spec.format.name); + EXPECT_EQ(2u, spec.format.num_channels); + EXPECT_EQ(48000, spec.format.clockrate_hz); +} + +// Test that with bitrate=N and stereo=0, bitrate is N. +TEST_P(WebRtcVoiceEngineTestFake, SetSendCodecOpusGoodNBitrate0Stereo) { + EXPECT_TRUE(SetupSendStream()); + cricket::AudioSendParameters parameters; + parameters.codecs.push_back(kOpusCodec); + parameters.codecs[0].bitrate = 30000; + parameters.codecs[0].params["stereo"] = "0"; + SetSendParameters(parameters); + CheckSendCodecBitrate(kSsrcX, "opus", 30000); +} + +// Test that with bitrate=N and without any parameters, bitrate is N. +TEST_P(WebRtcVoiceEngineTestFake, SetSendCodecOpusGoodNBitrateNoParameters) { + EXPECT_TRUE(SetupSendStream()); + cricket::AudioSendParameters parameters; + parameters.codecs.push_back(kOpusCodec); + parameters.codecs[0].bitrate = 30000; + SetSendParameters(parameters); + CheckSendCodecBitrate(kSsrcX, "opus", 30000); +} + +// Test that with bitrate=N and stereo=1, bitrate is N. +TEST_P(WebRtcVoiceEngineTestFake, SetSendCodecOpusGoodNBitrate1Stereo) { + EXPECT_TRUE(SetupSendStream()); + cricket::AudioSendParameters parameters; + parameters.codecs.push_back(kOpusCodec); + parameters.codecs[0].bitrate = 30000; + parameters.codecs[0].params["stereo"] = "1"; + SetSendParameters(parameters); + CheckSendCodecBitrate(kSsrcX, "opus", 30000); +} + +TEST_P(WebRtcVoiceEngineTestFake, SetSendCodecsWithBitrates) { + SetSendCodecsShouldWorkForBitrates("100", 100000, "150", 150000, "200", + 200000); +} + +TEST_P(WebRtcVoiceEngineTestFake, SetSendCodecsWithHighMaxBitrate) { + SetSendCodecsShouldWorkForBitrates("", 0, "", -1, "10000", 10000000); +} + +TEST_P(WebRtcVoiceEngineTestFake, + SetSendCodecsWithoutBitratesUsesCorrectDefaults) { + SetSendCodecsShouldWorkForBitrates("", 0, "", -1, "", -1); +} + +TEST_P(WebRtcVoiceEngineTestFake, SetSendCodecsCapsMinAndStartBitrate) { + SetSendCodecsShouldWorkForBitrates("-1", 0, "-100", -1, "", -1); +} + +TEST_P(WebRtcVoiceEngineTestFake, SetMaxSendBandwidthForAudioDoesntAffectBwe) { + SetSendCodecsShouldWorkForBitrates("100", 100000, "150", 150000, "200", + 200000); + send_parameters_.max_bandwidth_bps = 100000; + // Setting max bitrate should keep previous min bitrate + // Setting max bitrate should not reset start bitrate. + EXPECT_CALL(*call_.GetMockTransportControllerSend(), + SetSdpBitrateParameters( + AllOf(Field(&BitrateConstraints::min_bitrate_bps, 100000), + Field(&BitrateConstraints::start_bitrate_bps, -1), + Field(&BitrateConstraints::max_bitrate_bps, 200000)))); + SetSendParameters(send_parameters_); +} + +// Test that we can enable NACK with opus as callee. +TEST_P(WebRtcVoiceEngineTestFake, SetSendCodecEnableNackAsCallee) { + EXPECT_TRUE(SetupRecvStream()); + cricket::AudioSendParameters parameters; + parameters.codecs.push_back(kOpusCodec); + parameters.codecs[0].AddFeedbackParam(cricket::FeedbackParam( + cricket::kRtcpFbParamNack, cricket::kParamValueEmpty)); + EXPECT_EQ(0, GetRecvStreamConfig(kSsrcX).rtp.nack.rtp_history_ms); + SetSendParameters(parameters); + // NACK should be enabled even with no send stream. + EXPECT_EQ(kRtpHistoryMs, GetRecvStreamConfig(kSsrcX).rtp.nack.rtp_history_ms); + + EXPECT_TRUE( + channel_->AddSendStream(cricket::StreamParams::CreateLegacy(kSsrcX))); +} + +// Test that we can enable NACK on receive streams. +TEST_P(WebRtcVoiceEngineTestFake, SetSendCodecEnableNackRecvStreams) { + EXPECT_TRUE(SetupSendStream()); + EXPECT_TRUE(AddRecvStream(kSsrcY)); + cricket::AudioSendParameters parameters; + parameters.codecs.push_back(kOpusCodec); + parameters.codecs[0].AddFeedbackParam(cricket::FeedbackParam( + cricket::kRtcpFbParamNack, cricket::kParamValueEmpty)); + EXPECT_EQ(0, GetRecvStreamConfig(kSsrcY).rtp.nack.rtp_history_ms); + SetSendParameters(parameters); + EXPECT_EQ(kRtpHistoryMs, GetRecvStreamConfig(kSsrcY).rtp.nack.rtp_history_ms); +} + +// Test that we can disable NACK on receive streams. +TEST_P(WebRtcVoiceEngineTestFake, SetSendCodecDisableNackRecvStreams) { + EXPECT_TRUE(SetupSendStream()); + EXPECT_TRUE(AddRecvStream(kSsrcY)); + cricket::AudioSendParameters parameters; + parameters.codecs.push_back(kOpusCodec); + parameters.codecs[0].AddFeedbackParam(cricket::FeedbackParam( + cricket::kRtcpFbParamNack, cricket::kParamValueEmpty)); + SetSendParameters(parameters); + EXPECT_EQ(kRtpHistoryMs, GetRecvStreamConfig(kSsrcY).rtp.nack.rtp_history_ms); + + parameters.codecs.clear(); + parameters.codecs.push_back(kOpusCodec); + SetSendParameters(parameters); + EXPECT_EQ(0, GetRecvStreamConfig(kSsrcY).rtp.nack.rtp_history_ms); +} + +// Test that NACK is enabled on a new receive stream. +TEST_P(WebRtcVoiceEngineTestFake, AddRecvStreamEnableNack) { + EXPECT_TRUE(SetupSendStream()); + cricket::AudioSendParameters parameters; + parameters.codecs.push_back(kIsacCodec); + parameters.codecs.push_back(kCn16000Codec); + parameters.codecs[0].AddFeedbackParam(cricket::FeedbackParam( + cricket::kRtcpFbParamNack, cricket::kParamValueEmpty)); + SetSendParameters(parameters); + + EXPECT_TRUE(AddRecvStream(kSsrcY)); + EXPECT_EQ(kRtpHistoryMs, GetRecvStreamConfig(kSsrcY).rtp.nack.rtp_history_ms); + EXPECT_TRUE(AddRecvStream(kSsrcZ)); + EXPECT_EQ(kRtpHistoryMs, GetRecvStreamConfig(kSsrcZ).rtp.nack.rtp_history_ms); +} + +TEST_P(WebRtcVoiceEngineTestFake, TransportCcCanBeEnabledAndDisabled) { + EXPECT_TRUE(SetupChannel()); + cricket::AudioSendParameters send_parameters; + send_parameters.codecs.push_back(kOpusCodec); + EXPECT_TRUE(send_parameters.codecs[0].feedback_params.params().empty()); + SetSendParameters(send_parameters); + + cricket::AudioRecvParameters recv_parameters; + recv_parameters.codecs.push_back(kIsacCodec); + EXPECT_TRUE(channel_->SetRecvParameters(recv_parameters)); + EXPECT_TRUE(AddRecvStream(kSsrcX)); + ASSERT_TRUE(call_.GetAudioReceiveStream(kSsrcX) != nullptr); + EXPECT_FALSE( + call_.GetAudioReceiveStream(kSsrcX)->GetConfig().rtp.transport_cc); + + send_parameters.codecs = engine_->send_codecs(); + SetSendParameters(send_parameters); + ASSERT_TRUE(call_.GetAudioReceiveStream(kSsrcX) != nullptr); + EXPECT_TRUE( + call_.GetAudioReceiveStream(kSsrcX)->GetConfig().rtp.transport_cc); +} + +// Test that we can switch back and forth between Opus and ISAC with CN. +TEST_P(WebRtcVoiceEngineTestFake, SetSendCodecsIsacOpusSwitching) { + EXPECT_TRUE(SetupSendStream()); + + cricket::AudioSendParameters opus_parameters; + opus_parameters.codecs.push_back(kOpusCodec); + SetSendParameters(opus_parameters); + { + const auto& spec = *GetSendStreamConfig(kSsrcX).send_codec_spec; + EXPECT_EQ(111, spec.payload_type); + EXPECT_STRCASEEQ("opus", spec.format.name.c_str()); + } + + cricket::AudioSendParameters isac_parameters; + isac_parameters.codecs.push_back(kIsacCodec); + isac_parameters.codecs.push_back(kCn16000Codec); + isac_parameters.codecs.push_back(kOpusCodec); + SetSendParameters(isac_parameters); + { + const auto& spec = *GetSendStreamConfig(kSsrcX).send_codec_spec; + EXPECT_EQ(103, spec.payload_type); + EXPECT_STRCASEEQ("ISAC", spec.format.name.c_str()); + } + + SetSendParameters(opus_parameters); + { + const auto& spec = *GetSendStreamConfig(kSsrcX).send_codec_spec; + EXPECT_EQ(111, spec.payload_type); + EXPECT_STRCASEEQ("opus", spec.format.name.c_str()); + } +} + +// Test that we handle various ways of specifying bitrate. +TEST_P(WebRtcVoiceEngineTestFake, SetSendCodecsBitrate) { + EXPECT_TRUE(SetupSendStream()); + cricket::AudioSendParameters parameters; + parameters.codecs.push_back(kIsacCodec); // bitrate == 32000 + SetSendParameters(parameters); + { + const auto& spec = *GetSendStreamConfig(kSsrcX).send_codec_spec; + EXPECT_EQ(103, spec.payload_type); + EXPECT_STRCASEEQ("ISAC", spec.format.name.c_str()); + EXPECT_EQ(32000, spec.target_bitrate_bps); + } + + parameters.codecs[0].bitrate = 0; // bitrate == default + SetSendParameters(parameters); + { + const auto& spec = *GetSendStreamConfig(kSsrcX).send_codec_spec; + EXPECT_EQ(103, spec.payload_type); + EXPECT_STRCASEEQ("ISAC", spec.format.name.c_str()); + EXPECT_EQ(32000, spec.target_bitrate_bps); + } + parameters.codecs[0].bitrate = 28000; // bitrate == 28000 + SetSendParameters(parameters); + { + const auto& spec = *GetSendStreamConfig(kSsrcX).send_codec_spec; + EXPECT_EQ(103, spec.payload_type); + EXPECT_STRCASEEQ("ISAC", spec.format.name.c_str()); + EXPECT_EQ(28000, spec.target_bitrate_bps); + } + + parameters.codecs[0] = kPcmuCodec; // bitrate == 64000 + SetSendParameters(parameters); + { + const auto& spec = *GetSendStreamConfig(kSsrcX).send_codec_spec; + EXPECT_EQ(0, spec.payload_type); + EXPECT_STRCASEEQ("PCMU", spec.format.name.c_str()); + EXPECT_EQ(64000, spec.target_bitrate_bps); + } + + parameters.codecs[0].bitrate = 0; // bitrate == default + SetSendParameters(parameters); + { + const auto& spec = *GetSendStreamConfig(kSsrcX).send_codec_spec; + EXPECT_EQ(0, spec.payload_type); + EXPECT_STREQ("PCMU", spec.format.name.c_str()); + EXPECT_EQ(64000, spec.target_bitrate_bps); + } + + parameters.codecs[0] = kOpusCodec; + parameters.codecs[0].bitrate = 0; // bitrate == default + SetSendParameters(parameters); + { + const auto& spec = *GetSendStreamConfig(kSsrcX).send_codec_spec; + EXPECT_EQ(111, spec.payload_type); + EXPECT_STREQ("opus", spec.format.name.c_str()); + EXPECT_EQ(32000, spec.target_bitrate_bps); + } +} + +// Test that we fail if no codecs are specified. +TEST_P(WebRtcVoiceEngineTestFake, SetSendCodecsNoCodecs) { + EXPECT_TRUE(SetupSendStream()); + cricket::AudioSendParameters parameters; + EXPECT_FALSE(channel_->SetSendParameters(parameters)); +} + +// Test that we can set send codecs even with telephone-event codec as the first +// one on the list. +TEST_P(WebRtcVoiceEngineTestFake, SetSendCodecsDTMFOnTop) { + EXPECT_TRUE(SetupSendStream()); + cricket::AudioSendParameters parameters; + parameters.codecs.push_back(kTelephoneEventCodec1); + parameters.codecs.push_back(kIsacCodec); + parameters.codecs.push_back(kPcmuCodec); + parameters.codecs[0].id = 98; // DTMF + parameters.codecs[1].id = 96; + SetSendParameters(parameters); + const auto& spec = *GetSendStreamConfig(kSsrcX).send_codec_spec; + EXPECT_EQ(96, spec.payload_type); + EXPECT_STRCASEEQ("ISAC", spec.format.name.c_str()); + SetSend(true); + EXPECT_TRUE(channel_->CanInsertDtmf()); +} + +// Test that CanInsertDtmf() is governed by the send flag +TEST_P(WebRtcVoiceEngineTestFake, DTMFControlledBySendFlag) { + EXPECT_TRUE(SetupSendStream()); + cricket::AudioSendParameters parameters; + parameters.codecs.push_back(kTelephoneEventCodec1); + parameters.codecs.push_back(kPcmuCodec); + parameters.codecs[0].id = 98; // DTMF + parameters.codecs[1].id = 96; + SetSendParameters(parameters); + EXPECT_FALSE(channel_->CanInsertDtmf()); + SetSend(true); + EXPECT_TRUE(channel_->CanInsertDtmf()); + SetSend(false); + EXPECT_FALSE(channel_->CanInsertDtmf()); +} + +// Test that payload type range is limited for telephone-event codec. +TEST_P(WebRtcVoiceEngineTestFake, SetSendCodecsDTMFPayloadTypeOutOfRange) { + EXPECT_TRUE(SetupSendStream()); + cricket::AudioSendParameters parameters; + parameters.codecs.push_back(kTelephoneEventCodec2); + parameters.codecs.push_back(kIsacCodec); + parameters.codecs[0].id = 0; // DTMF + parameters.codecs[1].id = 96; + SetSendParameters(parameters); + SetSend(true); + EXPECT_TRUE(channel_->CanInsertDtmf()); + parameters.codecs[0].id = 128; // DTMF + EXPECT_FALSE(channel_->SetSendParameters(parameters)); + EXPECT_FALSE(channel_->CanInsertDtmf()); + parameters.codecs[0].id = 127; + SetSendParameters(parameters); + EXPECT_TRUE(channel_->CanInsertDtmf()); + parameters.codecs[0].id = -1; // DTMF + EXPECT_FALSE(channel_->SetSendParameters(parameters)); + EXPECT_FALSE(channel_->CanInsertDtmf()); +} + +// Test that we can set send codecs even with CN codec as the first +// one on the list. +TEST_P(WebRtcVoiceEngineTestFake, SetSendCodecsCNOnTop) { + EXPECT_TRUE(SetupSendStream()); + cricket::AudioSendParameters parameters; + parameters.codecs.push_back(kCn16000Codec); + parameters.codecs.push_back(kIsacCodec); + parameters.codecs.push_back(kPcmuCodec); + parameters.codecs[0].id = 98; // wideband CN + parameters.codecs[1].id = 96; + SetSendParameters(parameters); + const auto& send_codec_spec = *GetSendStreamConfig(kSsrcX).send_codec_spec; + EXPECT_EQ(96, send_codec_spec.payload_type); + EXPECT_STRCASEEQ("ISAC", send_codec_spec.format.name.c_str()); + EXPECT_EQ(98, send_codec_spec.cng_payload_type); +} + +// Test that we set VAD and DTMF types correctly as caller. +TEST_P(WebRtcVoiceEngineTestFake, SetSendCodecsCNandDTMFAsCaller) { + EXPECT_TRUE(SetupSendStream()); + cricket::AudioSendParameters parameters; + parameters.codecs.push_back(kIsacCodec); + parameters.codecs.push_back(kPcmuCodec); + // TODO(juberti): cn 32000 + parameters.codecs.push_back(kCn16000Codec); + parameters.codecs.push_back(kCn8000Codec); + parameters.codecs.push_back(kTelephoneEventCodec1); + parameters.codecs[0].id = 96; + parameters.codecs[2].id = 97; // wideband CN + parameters.codecs[4].id = 98; // DTMF + SetSendParameters(parameters); + const auto& send_codec_spec = *GetSendStreamConfig(kSsrcX).send_codec_spec; + EXPECT_EQ(96, send_codec_spec.payload_type); + EXPECT_STRCASEEQ("ISAC", send_codec_spec.format.name.c_str()); + EXPECT_EQ(1u, send_codec_spec.format.num_channels); + EXPECT_EQ(97, send_codec_spec.cng_payload_type); + SetSend(true); + EXPECT_TRUE(channel_->CanInsertDtmf()); +} + +// Test that we set VAD and DTMF types correctly as callee. +TEST_P(WebRtcVoiceEngineTestFake, SetSendCodecsCNandDTMFAsCallee) { + EXPECT_TRUE(SetupChannel()); + cricket::AudioSendParameters parameters; + parameters.codecs.push_back(kIsacCodec); + parameters.codecs.push_back(kPcmuCodec); + // TODO(juberti): cn 32000 + parameters.codecs.push_back(kCn16000Codec); + parameters.codecs.push_back(kCn8000Codec); + parameters.codecs.push_back(kTelephoneEventCodec2); + parameters.codecs[0].id = 96; + parameters.codecs[2].id = 97; // wideband CN + parameters.codecs[4].id = 98; // DTMF + SetSendParameters(parameters); + EXPECT_TRUE( + channel_->AddSendStream(cricket::StreamParams::CreateLegacy(kSsrcX))); + + const auto& send_codec_spec = *GetSendStreamConfig(kSsrcX).send_codec_spec; + EXPECT_EQ(96, send_codec_spec.payload_type); + EXPECT_STRCASEEQ("ISAC", send_codec_spec.format.name.c_str()); + EXPECT_EQ(1u, send_codec_spec.format.num_channels); + EXPECT_EQ(97, send_codec_spec.cng_payload_type); + SetSend(true); + EXPECT_TRUE(channel_->CanInsertDtmf()); +} + +// Test that we only apply VAD if we have a CN codec that matches the +// send codec clockrate. +TEST_P(WebRtcVoiceEngineTestFake, SetSendCodecsCNNoMatch) { + EXPECT_TRUE(SetupSendStream()); + cricket::AudioSendParameters parameters; + // Set ISAC(16K) and CN(16K). VAD should be activated. + parameters.codecs.push_back(kIsacCodec); + parameters.codecs.push_back(kCn16000Codec); + parameters.codecs[1].id = 97; + SetSendParameters(parameters); + { + const auto& send_codec_spec = *GetSendStreamConfig(kSsrcX).send_codec_spec; + EXPECT_STRCASEEQ("ISAC", send_codec_spec.format.name.c_str()); + EXPECT_EQ(1u, send_codec_spec.format.num_channels); + EXPECT_EQ(97, send_codec_spec.cng_payload_type); + } + // Set PCMU(8K) and CN(16K). VAD should not be activated. + parameters.codecs[0] = kPcmuCodec; + SetSendParameters(parameters); + { + const auto& send_codec_spec = *GetSendStreamConfig(kSsrcX).send_codec_spec; + EXPECT_STRCASEEQ("PCMU", send_codec_spec.format.name.c_str()); + EXPECT_EQ(absl::nullopt, send_codec_spec.cng_payload_type); + } + // Set PCMU(8K) and CN(8K). VAD should be activated. + parameters.codecs[1] = kCn8000Codec; + SetSendParameters(parameters); + { + const auto& send_codec_spec = *GetSendStreamConfig(kSsrcX).send_codec_spec; + EXPECT_STRCASEEQ("PCMU", send_codec_spec.format.name.c_str()); + EXPECT_EQ(1u, send_codec_spec.format.num_channels); + EXPECT_EQ(13, send_codec_spec.cng_payload_type); + } + // Set ISAC(16K) and CN(8K). VAD should not be activated. + parameters.codecs[0] = kIsacCodec; + SetSendParameters(parameters); + { + const auto& send_codec_spec = *GetSendStreamConfig(kSsrcX).send_codec_spec; + EXPECT_STRCASEEQ("ISAC", send_codec_spec.format.name.c_str()); + EXPECT_EQ(absl::nullopt, send_codec_spec.cng_payload_type); + } +} + +// Test that we perform case-insensitive matching of codec names. +TEST_P(WebRtcVoiceEngineTestFake, SetSendCodecsCaseInsensitive) { + EXPECT_TRUE(SetupSendStream()); + cricket::AudioSendParameters parameters; + parameters.codecs.push_back(kIsacCodec); + parameters.codecs.push_back(kPcmuCodec); + parameters.codecs.push_back(kCn16000Codec); + parameters.codecs.push_back(kCn8000Codec); + parameters.codecs.push_back(kTelephoneEventCodec1); + parameters.codecs[0].name = "iSaC"; + parameters.codecs[0].id = 96; + parameters.codecs[2].id = 97; // wideband CN + parameters.codecs[4].id = 98; // DTMF + SetSendParameters(parameters); + const auto& send_codec_spec = *GetSendStreamConfig(kSsrcX).send_codec_spec; + EXPECT_EQ(96, send_codec_spec.payload_type); + EXPECT_STRCASEEQ("ISAC", send_codec_spec.format.name.c_str()); + EXPECT_EQ(1u, send_codec_spec.format.num_channels); + EXPECT_EQ(97, send_codec_spec.cng_payload_type); + SetSend(true); + EXPECT_TRUE(channel_->CanInsertDtmf()); +} + +class WebRtcVoiceEngineWithSendSideBweTest : public WebRtcVoiceEngineTestFake { + public: + WebRtcVoiceEngineWithSendSideBweTest() + : WebRtcVoiceEngineTestFake("WebRTC-Audio-SendSideBwe/Enabled/") {} +}; + +TEST_P(WebRtcVoiceEngineWithSendSideBweTest, + SupportsTransportSequenceNumberHeaderExtension) { + const std::vector<webrtc::RtpExtension> header_extensions = + GetDefaultEnabledRtpHeaderExtensions(*engine_); + EXPECT_THAT(header_extensions, + Contains(::testing::Field( + "uri", &RtpExtension::uri, + webrtc::RtpExtension::kTransportSequenceNumberUri))); +} + +// Test support for audio level header extension. +TEST_P(WebRtcVoiceEngineTestFake, SendAudioLevelHeaderExtensions) { + TestSetSendRtpHeaderExtensions(webrtc::RtpExtension::kAudioLevelUri); +} +TEST_P(WebRtcVoiceEngineTestFake, RecvAudioLevelHeaderExtensions) { + TestSetRecvRtpHeaderExtensions(webrtc::RtpExtension::kAudioLevelUri); +} + +// Test support for transport sequence number header extension. +TEST_P(WebRtcVoiceEngineTestFake, SendTransportSequenceNumberHeaderExtensions) { + TestSetSendRtpHeaderExtensions( + webrtc::RtpExtension::kTransportSequenceNumberUri); +} +TEST_P(WebRtcVoiceEngineTestFake, RecvTransportSequenceNumberHeaderExtensions) { + TestSetRecvRtpHeaderExtensions( + webrtc::RtpExtension::kTransportSequenceNumberUri); +} + +// Test that we can create a channel and start sending on it. +TEST_P(WebRtcVoiceEngineTestFake, Send) { + EXPECT_TRUE(SetupSendStream()); + SetSendParameters(send_parameters_); + SetSend(true); + EXPECT_TRUE(GetSendStream(kSsrcX).IsSending()); + SetSend(false); + EXPECT_FALSE(GetSendStream(kSsrcX).IsSending()); +} + +// Test that a channel will send if and only if it has a source and is enabled +// for sending. +TEST_P(WebRtcVoiceEngineTestFake, SendStateWithAndWithoutSource) { + EXPECT_TRUE(SetupSendStream()); + SetSendParameters(send_parameters_); + SetAudioSend(kSsrcX, true, nullptr); + SetSend(true); + EXPECT_FALSE(GetSendStream(kSsrcX).IsSending()); + SetAudioSend(kSsrcX, true, &fake_source_); + EXPECT_TRUE(GetSendStream(kSsrcX).IsSending()); + SetAudioSend(kSsrcX, true, nullptr); + EXPECT_FALSE(GetSendStream(kSsrcX).IsSending()); +} + +// Test that a channel is muted/unmuted. +TEST_P(WebRtcVoiceEngineTestFake, SendStateMuteUnmute) { + EXPECT_TRUE(SetupSendStream()); + SetSendParameters(send_parameters_); + EXPECT_FALSE(GetSendStream(kSsrcX).muted()); + SetAudioSend(kSsrcX, true, nullptr); + EXPECT_FALSE(GetSendStream(kSsrcX).muted()); + SetAudioSend(kSsrcX, false, nullptr); + EXPECT_TRUE(GetSendStream(kSsrcX).muted()); +} + +// Test that SetSendParameters() does not alter a stream's send state. +TEST_P(WebRtcVoiceEngineTestFake, SendStateWhenStreamsAreRecreated) { + EXPECT_TRUE(SetupSendStream()); + EXPECT_FALSE(GetSendStream(kSsrcX).IsSending()); + + // Turn on sending. + SetSend(true); + EXPECT_TRUE(GetSendStream(kSsrcX).IsSending()); + + // Changing RTP header extensions will recreate the AudioSendStream. + send_parameters_.extensions.push_back( + webrtc::RtpExtension(webrtc::RtpExtension::kAudioLevelUri, 12)); + SetSendParameters(send_parameters_); + EXPECT_TRUE(GetSendStream(kSsrcX).IsSending()); + + // Turn off sending. + SetSend(false); + EXPECT_FALSE(GetSendStream(kSsrcX).IsSending()); + + // Changing RTP header extensions will recreate the AudioSendStream. + send_parameters_.extensions.clear(); + SetSendParameters(send_parameters_); + EXPECT_FALSE(GetSendStream(kSsrcX).IsSending()); +} + +// Test that we can create a channel and start playing out on it. +TEST_P(WebRtcVoiceEngineTestFake, Playout) { + EXPECT_TRUE(SetupRecvStream()); + EXPECT_TRUE(channel_->SetRecvParameters(recv_parameters_)); + channel_->SetPlayout(true); + EXPECT_TRUE(GetRecvStream(kSsrcX).started()); + channel_->SetPlayout(false); + EXPECT_FALSE(GetRecvStream(kSsrcX).started()); +} + +// Test that we can add and remove send streams. +TEST_P(WebRtcVoiceEngineTestFake, CreateAndDeleteMultipleSendStreams) { + SetupForMultiSendStream(); + + // Set the global state for sending. + SetSend(true); + + for (uint32_t ssrc : kSsrcs4) { + EXPECT_TRUE( + channel_->AddSendStream(cricket::StreamParams::CreateLegacy(ssrc))); + SetAudioSend(ssrc, true, &fake_source_); + // Verify that we are in a sending state for all the created streams. + EXPECT_TRUE(GetSendStream(ssrc).IsSending()); + } + EXPECT_EQ(arraysize(kSsrcs4), call_.GetAudioSendStreams().size()); + + // Delete the send streams. + for (uint32_t ssrc : kSsrcs4) { + EXPECT_TRUE(channel_->RemoveSendStream(ssrc)); + EXPECT_FALSE(call_.GetAudioSendStream(ssrc)); + EXPECT_FALSE(channel_->RemoveSendStream(ssrc)); + } + EXPECT_EQ(0u, call_.GetAudioSendStreams().size()); +} + +// Test SetSendCodecs correctly configure the codecs in all send streams. +TEST_P(WebRtcVoiceEngineTestFake, SetSendCodecsWithMultipleSendStreams) { + SetupForMultiSendStream(); + + // Create send streams. + for (uint32_t ssrc : kSsrcs4) { + EXPECT_TRUE( + channel_->AddSendStream(cricket::StreamParams::CreateLegacy(ssrc))); + } + + cricket::AudioSendParameters parameters; + // Set ISAC(16K) and CN(16K). VAD should be activated. + parameters.codecs.push_back(kIsacCodec); + parameters.codecs.push_back(kCn16000Codec); + parameters.codecs[1].id = 97; + SetSendParameters(parameters); + + // Verify ISAC and VAD are corrected configured on all send channels. + for (uint32_t ssrc : kSsrcs4) { + ASSERT_TRUE(call_.GetAudioSendStream(ssrc) != nullptr); + const auto& send_codec_spec = + *call_.GetAudioSendStream(ssrc)->GetConfig().send_codec_spec; + EXPECT_STRCASEEQ("ISAC", send_codec_spec.format.name.c_str()); + EXPECT_EQ(1u, send_codec_spec.format.num_channels); + EXPECT_EQ(97, send_codec_spec.cng_payload_type); + } + + // Change to PCMU(8K) and CN(16K). + parameters.codecs[0] = kPcmuCodec; + SetSendParameters(parameters); + for (uint32_t ssrc : kSsrcs4) { + ASSERT_TRUE(call_.GetAudioSendStream(ssrc) != nullptr); + const auto& send_codec_spec = + *call_.GetAudioSendStream(ssrc)->GetConfig().send_codec_spec; + EXPECT_STRCASEEQ("PCMU", send_codec_spec.format.name.c_str()); + EXPECT_EQ(absl::nullopt, send_codec_spec.cng_payload_type); + } +} + +// Test we can SetSend on all send streams correctly. +TEST_P(WebRtcVoiceEngineTestFake, SetSendWithMultipleSendStreams) { + SetupForMultiSendStream(); + + // Create the send channels and they should be a "not sending" date. + for (uint32_t ssrc : kSsrcs4) { + EXPECT_TRUE( + channel_->AddSendStream(cricket::StreamParams::CreateLegacy(ssrc))); + SetAudioSend(ssrc, true, &fake_source_); + EXPECT_FALSE(GetSendStream(ssrc).IsSending()); + } + + // Set the global state for starting sending. + SetSend(true); + for (uint32_t ssrc : kSsrcs4) { + // Verify that we are in a sending state for all the send streams. + EXPECT_TRUE(GetSendStream(ssrc).IsSending()); + } + + // Set the global state for stopping sending. + SetSend(false); + for (uint32_t ssrc : kSsrcs4) { + // Verify that we are in a stop state for all the send streams. + EXPECT_FALSE(GetSendStream(ssrc).IsSending()); + } +} + +// Test we can set the correct statistics on all send streams. +TEST_P(WebRtcVoiceEngineTestFake, GetStatsWithMultipleSendStreams) { + SetupForMultiSendStream(); + + // Create send streams. + for (uint32_t ssrc : kSsrcs4) { + EXPECT_TRUE( + channel_->AddSendStream(cricket::StreamParams::CreateLegacy(ssrc))); + } + + // Create a receive stream to check that none of the send streams end up in + // the receive stream stats. + EXPECT_TRUE(AddRecvStream(kSsrcY)); + + // We need send codec to be set to get all stats. + SetSendParameters(send_parameters_); + EXPECT_TRUE(channel_->SetRecvParameters(recv_parameters_)); + SetAudioSendStreamStats(); + + // Check stats for the added streams. + { + EXPECT_CALL(*adm_, GetPlayoutUnderrunCount()).WillOnce(Return(0)); + cricket::VoiceMediaInfo info; + EXPECT_EQ(true, channel_->GetStats(&info)); + + // We have added 4 send streams. We should see empty stats for all. + EXPECT_EQ(static_cast<size_t>(arraysize(kSsrcs4)), info.senders.size()); + for (const auto& sender : info.senders) { + VerifyVoiceSenderInfo(sender, false); + } + VerifyVoiceSendRecvCodecs(info); + + // We have added one receive stream. We should see empty stats. + EXPECT_EQ(info.receivers.size(), 1u); + EXPECT_EQ(info.receivers[0].ssrc(), 0u); + } + + // Remove the kSsrcY stream. No receiver stats. + { + cricket::VoiceMediaInfo info; + EXPECT_TRUE(channel_->RemoveRecvStream(kSsrcY)); + EXPECT_CALL(*adm_, GetPlayoutUnderrunCount()).WillOnce(Return(0)); + EXPECT_EQ(true, channel_->GetStats(&info)); + EXPECT_EQ(static_cast<size_t>(arraysize(kSsrcs4)), info.senders.size()); + EXPECT_EQ(0u, info.receivers.size()); + } + + // Deliver a new packet - a default receive stream should be created and we + // should see stats again. + { + cricket::VoiceMediaInfo info; + DeliverPacket(kPcmuFrame, sizeof(kPcmuFrame)); + SetAudioReceiveStreamStats(); + EXPECT_CALL(*adm_, GetPlayoutUnderrunCount()).WillOnce(Return(0)); + EXPECT_EQ(true, channel_->GetStats(&info)); + EXPECT_EQ(static_cast<size_t>(arraysize(kSsrcs4)), info.senders.size()); + EXPECT_EQ(1u, info.receivers.size()); + VerifyVoiceReceiverInfo(info.receivers[0]); + VerifyVoiceSendRecvCodecs(info); + } +} + +// Test that we can add and remove receive streams, and do proper send/playout. +// We can receive on multiple streams while sending one stream. +TEST_P(WebRtcVoiceEngineTestFake, PlayoutWithMultipleStreams) { + EXPECT_TRUE(SetupSendStream()); + + // Start playout without a receive stream. + SetSendParameters(send_parameters_); + channel_->SetPlayout(true); + + // Adding another stream should enable playout on the new stream only. + EXPECT_TRUE(AddRecvStream(kSsrcY)); + SetSend(true); + EXPECT_TRUE(GetSendStream(kSsrcX).IsSending()); + + // Make sure only the new stream is played out. + EXPECT_TRUE(GetRecvStream(kSsrcY).started()); + + // Adding yet another stream should have stream 2 and 3 enabled for playout. + EXPECT_TRUE(AddRecvStream(kSsrcZ)); + EXPECT_TRUE(GetRecvStream(kSsrcY).started()); + EXPECT_TRUE(GetRecvStream(kSsrcZ).started()); + + // Stop sending. + SetSend(false); + EXPECT_FALSE(GetSendStream(kSsrcX).IsSending()); + + // Stop playout. + channel_->SetPlayout(false); + EXPECT_FALSE(GetRecvStream(kSsrcY).started()); + EXPECT_FALSE(GetRecvStream(kSsrcZ).started()); + + // Restart playout and make sure recv streams are played out. + channel_->SetPlayout(true); + EXPECT_TRUE(GetRecvStream(kSsrcY).started()); + EXPECT_TRUE(GetRecvStream(kSsrcZ).started()); + + // Now remove the recv streams. + EXPECT_TRUE(channel_->RemoveRecvStream(kSsrcZ)); + EXPECT_TRUE(channel_->RemoveRecvStream(kSsrcY)); +} + +TEST_P(WebRtcVoiceEngineTestFake, TxAgcConfigViaOptions) { + EXPECT_TRUE(SetupSendStream()); + EXPECT_CALL(*adm_, BuiltInAGCIsAvailable()) + .Times(::testing::AtLeast(1)) + .WillRepeatedly(Return(false)); + + if (!use_null_apm_) { + // Ensure default options. + VerifyGainControlEnabledCorrectly(); + VerifyGainControlDefaultSettings(); + } + + const auto& agc_config = apm_config_.gain_controller1; + + send_parameters_.options.auto_gain_control = false; + SetSendParameters(send_parameters_); + if (!use_null_apm_) { + EXPECT_FALSE(agc_config.enabled); + } + send_parameters_.options.auto_gain_control = absl::nullopt; + + send_parameters_.options.tx_agc_target_dbov = 5; + SetSendParameters(send_parameters_); + if (!use_null_apm_) { + EXPECT_EQ(5, agc_config.target_level_dbfs); + } + send_parameters_.options.tx_agc_target_dbov = absl::nullopt; + + send_parameters_.options.tx_agc_digital_compression_gain = 10; + SetSendParameters(send_parameters_); + if (!use_null_apm_) { + EXPECT_EQ(10, agc_config.compression_gain_db); + } + send_parameters_.options.tx_agc_digital_compression_gain = absl::nullopt; + + send_parameters_.options.tx_agc_limiter = false; + SetSendParameters(send_parameters_); + if (!use_null_apm_) { + EXPECT_FALSE(agc_config.enable_limiter); + } + send_parameters_.options.tx_agc_limiter = absl::nullopt; + + SetSendParameters(send_parameters_); + if (!use_null_apm_) { + // Expect all options to have been preserved. + EXPECT_FALSE(agc_config.enabled); + EXPECT_EQ(5, agc_config.target_level_dbfs); + EXPECT_EQ(10, agc_config.compression_gain_db); + EXPECT_FALSE(agc_config.enable_limiter); + } +} + +TEST_P(WebRtcVoiceEngineTestFake, SetAudioNetworkAdaptorViaOptions) { + EXPECT_TRUE(SetupSendStream()); + send_parameters_.options.audio_network_adaptor = true; + send_parameters_.options.audio_network_adaptor_config = {"1234"}; + SetSendParameters(send_parameters_); + EXPECT_EQ(send_parameters_.options.audio_network_adaptor_config, + GetAudioNetworkAdaptorConfig(kSsrcX)); +} + +TEST_P(WebRtcVoiceEngineTestFake, AudioSendResetAudioNetworkAdaptor) { + EXPECT_TRUE(SetupSendStream()); + send_parameters_.options.audio_network_adaptor = true; + send_parameters_.options.audio_network_adaptor_config = {"1234"}; + SetSendParameters(send_parameters_); + EXPECT_EQ(send_parameters_.options.audio_network_adaptor_config, + GetAudioNetworkAdaptorConfig(kSsrcX)); + cricket::AudioOptions options; + options.audio_network_adaptor = false; + SetAudioSend(kSsrcX, true, nullptr, &options); + EXPECT_EQ(absl::nullopt, GetAudioNetworkAdaptorConfig(kSsrcX)); +} + +TEST_P(WebRtcVoiceEngineTestFake, AudioNetworkAdaptorNotGetOverridden) { + EXPECT_TRUE(SetupSendStream()); + send_parameters_.options.audio_network_adaptor = true; + send_parameters_.options.audio_network_adaptor_config = {"1234"}; + SetSendParameters(send_parameters_); + EXPECT_EQ(send_parameters_.options.audio_network_adaptor_config, + GetAudioNetworkAdaptorConfig(kSsrcX)); + const int initial_num = call_.GetNumCreatedSendStreams(); + cricket::AudioOptions options; + options.audio_network_adaptor = absl::nullopt; + // Unvalued |options.audio_network_adaptor|.should not reset audio network + // adaptor. + SetAudioSend(kSsrcX, true, nullptr, &options); + // AudioSendStream not expected to be recreated. + EXPECT_EQ(initial_num, call_.GetNumCreatedSendStreams()); + EXPECT_EQ(send_parameters_.options.audio_network_adaptor_config, + GetAudioNetworkAdaptorConfig(kSsrcX)); +} + +class WebRtcVoiceEngineWithSendSideBweWithOverheadTest + : public WebRtcVoiceEngineTestFake { + public: + WebRtcVoiceEngineWithSendSideBweWithOverheadTest() + : WebRtcVoiceEngineTestFake( + "WebRTC-Audio-SendSideBwe/Enabled/WebRTC-Audio-Allocation/" + "min:6000bps,max:32000bps/WebRTC-SendSideBwe-WithOverhead/" + "Enabled/") {} +}; + +// Test that we can set the outgoing SSRC properly. +// SSRC is set in SetupSendStream() by calling AddSendStream. +TEST_P(WebRtcVoiceEngineTestFake, SetSendSsrc) { + EXPECT_TRUE(SetupSendStream()); + EXPECT_TRUE(call_.GetAudioSendStream(kSsrcX)); +} + +TEST_P(WebRtcVoiceEngineTestFake, GetStats) { + // Setup. We need send codec to be set to get all stats. + EXPECT_TRUE(SetupSendStream()); + // SetupSendStream adds a send stream with kSsrcX, so the receive + // stream has to use a different SSRC. + EXPECT_TRUE(AddRecvStream(kSsrcY)); + SetSendParameters(send_parameters_); + EXPECT_TRUE(channel_->SetRecvParameters(recv_parameters_)); + SetAudioSendStreamStats(); + + // Check stats for the added streams. + { + EXPECT_CALL(*adm_, GetPlayoutUnderrunCount()).WillOnce(Return(0)); + cricket::VoiceMediaInfo info; + EXPECT_EQ(true, channel_->GetStats(&info)); + + // We have added one send stream. We should see the stats we've set. + EXPECT_EQ(1u, info.senders.size()); + VerifyVoiceSenderInfo(info.senders[0], false); + // We have added one receive stream. We should see empty stats. + EXPECT_EQ(info.receivers.size(), 1u); + EXPECT_EQ(info.receivers[0].ssrc(), 0u); + } + + // Start sending - this affects some reported stats. + { + cricket::VoiceMediaInfo info; + SetSend(true); + EXPECT_CALL(*adm_, GetPlayoutUnderrunCount()).WillOnce(Return(0)); + EXPECT_EQ(true, channel_->GetStats(&info)); + VerifyVoiceSenderInfo(info.senders[0], true); + VerifyVoiceSendRecvCodecs(info); + } + + // Remove the kSsrcY stream. No receiver stats. + { + cricket::VoiceMediaInfo info; + EXPECT_TRUE(channel_->RemoveRecvStream(kSsrcY)); + EXPECT_CALL(*adm_, GetPlayoutUnderrunCount()).WillOnce(Return(0)); + EXPECT_EQ(true, channel_->GetStats(&info)); + EXPECT_EQ(1u, info.senders.size()); + EXPECT_EQ(0u, info.receivers.size()); + } + + // Deliver a new packet - a default receive stream should be created and we + // should see stats again. + { + cricket::VoiceMediaInfo info; + DeliverPacket(kPcmuFrame, sizeof(kPcmuFrame)); + SetAudioReceiveStreamStats(); + EXPECT_CALL(*adm_, GetPlayoutUnderrunCount()).WillOnce(Return(0)); + EXPECT_EQ(true, channel_->GetStats(&info)); + EXPECT_EQ(1u, info.senders.size()); + EXPECT_EQ(1u, info.receivers.size()); + VerifyVoiceReceiverInfo(info.receivers[0]); + VerifyVoiceSendRecvCodecs(info); + } +} + +// Test that we can set the outgoing SSRC properly with multiple streams. +// SSRC is set in SetupSendStream() by calling AddSendStream. +TEST_P(WebRtcVoiceEngineTestFake, SetSendSsrcWithMultipleStreams) { + EXPECT_TRUE(SetupSendStream()); + EXPECT_TRUE(call_.GetAudioSendStream(kSsrcX)); + EXPECT_TRUE(AddRecvStream(kSsrcY)); + EXPECT_EQ(kSsrcX, GetRecvStreamConfig(kSsrcY).rtp.local_ssrc); +} + +// Test that the local SSRC is the same on sending and receiving channels if the +// receive channel is created before the send channel. +TEST_P(WebRtcVoiceEngineTestFake, SetSendSsrcAfterCreatingReceiveChannel) { + EXPECT_TRUE(SetupChannel()); + EXPECT_TRUE(AddRecvStream(kSsrcY)); + EXPECT_TRUE( + channel_->AddSendStream(cricket::StreamParams::CreateLegacy(kSsrcX))); + EXPECT_TRUE(call_.GetAudioSendStream(kSsrcX)); + EXPECT_EQ(kSsrcX, GetRecvStreamConfig(kSsrcY).rtp.local_ssrc); +} + +// Test that we can properly receive packets. +TEST_P(WebRtcVoiceEngineTestFake, Recv) { + EXPECT_TRUE(SetupChannel()); + EXPECT_TRUE(AddRecvStream(1)); + DeliverPacket(kPcmuFrame, sizeof(kPcmuFrame)); + + EXPECT_TRUE( + GetRecvStream(1).VerifyLastPacket(kPcmuFrame, sizeof(kPcmuFrame))); +} + +// Test that we can properly receive packets on multiple streams. +TEST_P(WebRtcVoiceEngineTestFake, RecvWithMultipleStreams) { + EXPECT_TRUE(SetupChannel()); + const uint32_t ssrc1 = 1; + const uint32_t ssrc2 = 2; + const uint32_t ssrc3 = 3; + EXPECT_TRUE(AddRecvStream(ssrc1)); + EXPECT_TRUE(AddRecvStream(ssrc2)); + EXPECT_TRUE(AddRecvStream(ssrc3)); + // Create packets with the right SSRCs. + unsigned char packets[4][sizeof(kPcmuFrame)]; + for (size_t i = 0; i < arraysize(packets); ++i) { + memcpy(packets[i], kPcmuFrame, sizeof(kPcmuFrame)); + rtc::SetBE32(packets[i] + 8, static_cast<uint32_t>(i)); + } + + const cricket::FakeAudioReceiveStream& s1 = GetRecvStream(ssrc1); + const cricket::FakeAudioReceiveStream& s2 = GetRecvStream(ssrc2); + const cricket::FakeAudioReceiveStream& s3 = GetRecvStream(ssrc3); + + EXPECT_EQ(s1.received_packets(), 0); + EXPECT_EQ(s2.received_packets(), 0); + EXPECT_EQ(s3.received_packets(), 0); + + DeliverPacket(packets[0], sizeof(packets[0])); + EXPECT_EQ(s1.received_packets(), 0); + EXPECT_EQ(s2.received_packets(), 0); + EXPECT_EQ(s3.received_packets(), 0); + + DeliverPacket(packets[1], sizeof(packets[1])); + EXPECT_EQ(s1.received_packets(), 1); + EXPECT_TRUE(s1.VerifyLastPacket(packets[1], sizeof(packets[1]))); + EXPECT_EQ(s2.received_packets(), 0); + EXPECT_EQ(s3.received_packets(), 0); + + DeliverPacket(packets[2], sizeof(packets[2])); + EXPECT_EQ(s1.received_packets(), 1); + EXPECT_EQ(s2.received_packets(), 1); + EXPECT_TRUE(s2.VerifyLastPacket(packets[2], sizeof(packets[2]))); + EXPECT_EQ(s3.received_packets(), 0); + + DeliverPacket(packets[3], sizeof(packets[3])); + EXPECT_EQ(s1.received_packets(), 1); + EXPECT_EQ(s2.received_packets(), 1); + EXPECT_EQ(s3.received_packets(), 1); + EXPECT_TRUE(s3.VerifyLastPacket(packets[3], sizeof(packets[3]))); + + EXPECT_TRUE(channel_->RemoveRecvStream(ssrc3)); + EXPECT_TRUE(channel_->RemoveRecvStream(ssrc2)); + EXPECT_TRUE(channel_->RemoveRecvStream(ssrc1)); +} + +// Test that receiving on an unsignaled stream works (a stream is created). +TEST_P(WebRtcVoiceEngineTestFake, RecvUnsignaled) { + EXPECT_TRUE(SetupChannel()); + EXPECT_EQ(0u, call_.GetAudioReceiveStreams().size()); + + DeliverPacket(kPcmuFrame, sizeof(kPcmuFrame)); + + EXPECT_EQ(1u, call_.GetAudioReceiveStreams().size()); + EXPECT_TRUE( + GetRecvStream(kSsrc1).VerifyLastPacket(kPcmuFrame, sizeof(kPcmuFrame))); +} + +// Tests that when we add a stream without SSRCs, but contains a stream_id +// that it is stored and its stream id is later used when the first packet +// arrives to properly create a receive stream with a sync label. +TEST_P(WebRtcVoiceEngineTestFake, RecvUnsignaledSsrcWithSignaledStreamId) { + const char kSyncLabel[] = "sync_label"; + EXPECT_TRUE(SetupChannel()); + cricket::StreamParams unsignaled_stream; + unsignaled_stream.set_stream_ids({kSyncLabel}); + ASSERT_TRUE(channel_->AddRecvStream(unsignaled_stream)); + // The stream shouldn't have been created at this point because it doesn't + // have any SSRCs. + EXPECT_EQ(0u, call_.GetAudioReceiveStreams().size()); + + DeliverPacket(kPcmuFrame, sizeof(kPcmuFrame)); + + EXPECT_EQ(1u, call_.GetAudioReceiveStreams().size()); + EXPECT_TRUE( + GetRecvStream(kSsrc1).VerifyLastPacket(kPcmuFrame, sizeof(kPcmuFrame))); + EXPECT_EQ(kSyncLabel, GetRecvStream(kSsrc1).GetConfig().sync_group); + + // Remset the unsignaled stream to clear the cached parameters. If a new + // default unsignaled receive stream is created it will not have a sync group. + channel_->ResetUnsignaledRecvStream(); + channel_->RemoveRecvStream(kSsrc1); + + DeliverPacket(kPcmuFrame, sizeof(kPcmuFrame)); + + EXPECT_EQ(1u, call_.GetAudioReceiveStreams().size()); + EXPECT_TRUE( + GetRecvStream(kSsrc1).VerifyLastPacket(kPcmuFrame, sizeof(kPcmuFrame))); + EXPECT_TRUE(GetRecvStream(kSsrc1).GetConfig().sync_group.empty()); +} + +// Test that receiving N unsignaled stream works (streams will be created), and +// that packets are forwarded to them all. +TEST_P(WebRtcVoiceEngineTestFake, RecvMultipleUnsignaled) { + EXPECT_TRUE(SetupChannel()); + unsigned char packet[sizeof(kPcmuFrame)]; + memcpy(packet, kPcmuFrame, sizeof(kPcmuFrame)); + + // Note that SSRC = 0 is not supported. + for (uint32_t ssrc = 1; ssrc < (1 + kMaxUnsignaledRecvStreams); ++ssrc) { + rtc::SetBE32(&packet[8], ssrc); + DeliverPacket(packet, sizeof(packet)); + + // Verify we have one new stream for each loop iteration. + EXPECT_EQ(ssrc, call_.GetAudioReceiveStreams().size()); + EXPECT_EQ(1, GetRecvStream(ssrc).received_packets()); + EXPECT_TRUE(GetRecvStream(ssrc).VerifyLastPacket(packet, sizeof(packet))); + } + + // Sending on the same SSRCs again should not create new streams. + for (uint32_t ssrc = 1; ssrc < (1 + kMaxUnsignaledRecvStreams); ++ssrc) { + rtc::SetBE32(&packet[8], ssrc); + DeliverPacket(packet, sizeof(packet)); + + EXPECT_EQ(kMaxUnsignaledRecvStreams, call_.GetAudioReceiveStreams().size()); + EXPECT_EQ(2, GetRecvStream(ssrc).received_packets()); + EXPECT_TRUE(GetRecvStream(ssrc).VerifyLastPacket(packet, sizeof(packet))); + } + + // Send on another SSRC, the oldest unsignaled stream (SSRC=1) is replaced. + constexpr uint32_t kAnotherSsrc = 667; + rtc::SetBE32(&packet[8], kAnotherSsrc); + DeliverPacket(packet, sizeof(packet)); + + const auto& streams = call_.GetAudioReceiveStreams(); + EXPECT_EQ(kMaxUnsignaledRecvStreams, streams.size()); + size_t i = 0; + for (uint32_t ssrc = 2; ssrc < (1 + kMaxUnsignaledRecvStreams); ++ssrc, ++i) { + EXPECT_EQ(ssrc, streams[i]->GetConfig().rtp.remote_ssrc); + EXPECT_EQ(2, streams[i]->received_packets()); + } + EXPECT_EQ(kAnotherSsrc, streams[i]->GetConfig().rtp.remote_ssrc); + EXPECT_EQ(1, streams[i]->received_packets()); + // Sanity check that we've checked all streams. + EXPECT_EQ(kMaxUnsignaledRecvStreams, (i + 1)); +} + +// Test that a default channel is created even after a signaled stream has been +// added, and that this stream will get any packets for unknown SSRCs. +TEST_P(WebRtcVoiceEngineTestFake, RecvUnsignaledAfterSignaled) { + EXPECT_TRUE(SetupChannel()); + unsigned char packet[sizeof(kPcmuFrame)]; + memcpy(packet, kPcmuFrame, sizeof(kPcmuFrame)); + + // Add a known stream, send packet and verify we got it. + const uint32_t signaled_ssrc = 1; + rtc::SetBE32(&packet[8], signaled_ssrc); + EXPECT_TRUE(AddRecvStream(signaled_ssrc)); + DeliverPacket(packet, sizeof(packet)); + EXPECT_TRUE( + GetRecvStream(signaled_ssrc).VerifyLastPacket(packet, sizeof(packet))); + EXPECT_EQ(1u, call_.GetAudioReceiveStreams().size()); + + // Note that the first unknown SSRC cannot be 0, because we only support + // creating receive streams for SSRC!=0. + const uint32_t unsignaled_ssrc = 7011; + rtc::SetBE32(&packet[8], unsignaled_ssrc); + DeliverPacket(packet, sizeof(packet)); + EXPECT_TRUE( + GetRecvStream(unsignaled_ssrc).VerifyLastPacket(packet, sizeof(packet))); + EXPECT_EQ(2u, call_.GetAudioReceiveStreams().size()); + + DeliverPacket(packet, sizeof(packet)); + EXPECT_EQ(2, GetRecvStream(unsignaled_ssrc).received_packets()); + + rtc::SetBE32(&packet[8], signaled_ssrc); + DeliverPacket(packet, sizeof(packet)); + EXPECT_EQ(2, GetRecvStream(signaled_ssrc).received_packets()); + EXPECT_EQ(2u, call_.GetAudioReceiveStreams().size()); +} + +// Two tests to verify that adding a receive stream with the same SSRC as a +// previously added unsignaled stream will only recreate underlying stream +// objects if the stream parameters have changed. +TEST_P(WebRtcVoiceEngineTestFake, AddRecvStreamAfterUnsignaled_NoRecreate) { + EXPECT_TRUE(SetupChannel()); + + // Spawn unsignaled stream with SSRC=1. + DeliverPacket(kPcmuFrame, sizeof(kPcmuFrame)); + EXPECT_EQ(1u, call_.GetAudioReceiveStreams().size()); + EXPECT_TRUE( + GetRecvStream(1).VerifyLastPacket(kPcmuFrame, sizeof(kPcmuFrame))); + + // Verify that the underlying stream object in Call is not recreated when a + // stream with SSRC=1 is added. + const auto& streams = call_.GetAudioReceiveStreams(); + EXPECT_EQ(1u, streams.size()); + int audio_receive_stream_id = streams.front()->id(); + EXPECT_TRUE(AddRecvStream(1)); + EXPECT_EQ(1u, streams.size()); + EXPECT_EQ(audio_receive_stream_id, streams.front()->id()); +} + +TEST_P(WebRtcVoiceEngineTestFake, AddRecvStreamAfterUnsignaled_Recreate) { + EXPECT_TRUE(SetupChannel()); + + // Spawn unsignaled stream with SSRC=1. + DeliverPacket(kPcmuFrame, sizeof(kPcmuFrame)); + EXPECT_EQ(1u, call_.GetAudioReceiveStreams().size()); + EXPECT_TRUE( + GetRecvStream(1).VerifyLastPacket(kPcmuFrame, sizeof(kPcmuFrame))); + + // Verify that the underlying stream object in Call *is* recreated when a + // stream with SSRC=1 is added, and which has changed stream parameters. + const auto& streams = call_.GetAudioReceiveStreams(); + EXPECT_EQ(1u, streams.size()); + int audio_receive_stream_id = streams.front()->id(); + cricket::StreamParams stream_params; + stream_params.ssrcs.push_back(1); + stream_params.set_stream_ids({"stream_id"}); + EXPECT_TRUE(channel_->AddRecvStream(stream_params)); + EXPECT_EQ(1u, streams.size()); + EXPECT_NE(audio_receive_stream_id, streams.front()->id()); +} + +// Test that AddRecvStream creates new stream. +TEST_P(WebRtcVoiceEngineTestFake, AddRecvStream) { + EXPECT_TRUE(SetupRecvStream()); + EXPECT_TRUE(AddRecvStream(1)); +} + +// Test that after adding a recv stream, we do not decode more codecs than +// those previously passed into SetRecvCodecs. +TEST_P(WebRtcVoiceEngineTestFake, AddRecvStreamUnsupportedCodec) { + EXPECT_TRUE(SetupSendStream()); + cricket::AudioRecvParameters parameters; + parameters.codecs.push_back(kIsacCodec); + parameters.codecs.push_back(kPcmuCodec); + EXPECT_TRUE(channel_->SetRecvParameters(parameters)); + EXPECT_TRUE(AddRecvStream(kSsrcX)); + EXPECT_THAT(GetRecvStreamConfig(kSsrcX).decoder_map, + (ContainerEq<std::map<int, webrtc::SdpAudioFormat>>( + {{0, {"PCMU", 8000, 1}}, {103, {"ISAC", 16000, 1}}}))); +} + +// Test that we properly clean up any streams that were added, even if +// not explicitly removed. +TEST_P(WebRtcVoiceEngineTestFake, StreamCleanup) { + EXPECT_TRUE(SetupSendStream()); + SetSendParameters(send_parameters_); + EXPECT_TRUE(AddRecvStream(1)); + EXPECT_TRUE(AddRecvStream(2)); + + EXPECT_EQ(1u, call_.GetAudioSendStreams().size()); + EXPECT_EQ(2u, call_.GetAudioReceiveStreams().size()); + delete channel_; + channel_ = NULL; + EXPECT_EQ(0u, call_.GetAudioSendStreams().size()); + EXPECT_EQ(0u, call_.GetAudioReceiveStreams().size()); +} + +TEST_P(WebRtcVoiceEngineTestFake, TestAddRecvStreamSuccessWithZeroSsrc) { + EXPECT_TRUE(SetupSendStream()); + EXPECT_TRUE(AddRecvStream(0)); +} + +TEST_P(WebRtcVoiceEngineTestFake, TestAddRecvStreamFailWithSameSsrc) { + EXPECT_TRUE(SetupChannel()); + EXPECT_TRUE(AddRecvStream(1)); + EXPECT_FALSE(AddRecvStream(1)); +} + +// Test the InsertDtmf on default send stream as caller. +TEST_P(WebRtcVoiceEngineTestFake, InsertDtmfOnDefaultSendStreamAsCaller) { + TestInsertDtmf(0, true, kTelephoneEventCodec1); +} + +// Test the InsertDtmf on default send stream as callee +TEST_P(WebRtcVoiceEngineTestFake, InsertDtmfOnDefaultSendStreamAsCallee) { + TestInsertDtmf(0, false, kTelephoneEventCodec2); +} + +// Test the InsertDtmf on specified send stream as caller. +TEST_P(WebRtcVoiceEngineTestFake, InsertDtmfOnSendStreamAsCaller) { + TestInsertDtmf(kSsrcX, true, kTelephoneEventCodec2); +} + +// Test the InsertDtmf on specified send stream as callee. +TEST_P(WebRtcVoiceEngineTestFake, InsertDtmfOnSendStreamAsCallee) { + TestInsertDtmf(kSsrcX, false, kTelephoneEventCodec1); +} + +// Test propagation of extmap allow mixed setting. +TEST_P(WebRtcVoiceEngineTestFake, SetExtmapAllowMixedAsCaller) { + TestExtmapAllowMixedCaller(/*extmap_allow_mixed=*/true); +} +TEST_P(WebRtcVoiceEngineTestFake, SetExtmapAllowMixedDisabledAsCaller) { + TestExtmapAllowMixedCaller(/*extmap_allow_mixed=*/false); +} +TEST_P(WebRtcVoiceEngineTestFake, SetExtmapAllowMixedAsCallee) { + TestExtmapAllowMixedCallee(/*extmap_allow_mixed=*/true); +} +TEST_P(WebRtcVoiceEngineTestFake, SetExtmapAllowMixedDisabledAsCallee) { + TestExtmapAllowMixedCallee(/*extmap_allow_mixed=*/false); +} + +TEST_P(WebRtcVoiceEngineTestFake, SetAudioOptions) { + EXPECT_TRUE(SetupSendStream()); + EXPECT_TRUE(AddRecvStream(kSsrcY)); + EXPECT_CALL(*adm_, BuiltInAECIsAvailable()) + .Times(8) + .WillRepeatedly(Return(false)); + EXPECT_CALL(*adm_, BuiltInAGCIsAvailable()) + .Times(4) + .WillRepeatedly(Return(false)); + EXPECT_CALL(*adm_, BuiltInNSIsAvailable()) + .Times(2) + .WillRepeatedly(Return(false)); + + EXPECT_EQ(200u, GetRecvStreamConfig(kSsrcY).jitter_buffer_max_packets); + EXPECT_FALSE(GetRecvStreamConfig(kSsrcY).jitter_buffer_fast_accelerate); + + // Nothing set in AudioOptions, so everything should be as default. + send_parameters_.options = cricket::AudioOptions(); + SetSendParameters(send_parameters_); + if (!use_null_apm_) { + VerifyEchoCancellationSettings(/*enabled=*/true); + EXPECT_TRUE(IsHighPassFilterEnabled()); + EXPECT_TRUE(IsTypingDetectionEnabled()); + } + EXPECT_EQ(200u, GetRecvStreamConfig(kSsrcY).jitter_buffer_max_packets); + EXPECT_FALSE(GetRecvStreamConfig(kSsrcY).jitter_buffer_fast_accelerate); + + // Turn typing detection off. + send_parameters_.options.typing_detection = false; + SetSendParameters(send_parameters_); + if (!use_null_apm_) { + EXPECT_FALSE(IsTypingDetectionEnabled()); + } + + // Leave typing detection unchanged, but non-default. + send_parameters_.options.typing_detection = absl::nullopt; + SetSendParameters(send_parameters_); + if (!use_null_apm_) { + EXPECT_FALSE(IsTypingDetectionEnabled()); + } + + // Turn typing detection on. + send_parameters_.options.typing_detection = true; + SetSendParameters(send_parameters_); + if (!use_null_apm_) { + EXPECT_TRUE(IsTypingDetectionEnabled()); + } + + // Turn echo cancellation off + send_parameters_.options.echo_cancellation = false; + SetSendParameters(send_parameters_); + if (!use_null_apm_) { + VerifyEchoCancellationSettings(/*enabled=*/false); + } + + // Turn echo cancellation back on, with settings, and make sure + // nothing else changed. + send_parameters_.options.echo_cancellation = true; + SetSendParameters(send_parameters_); + if (!use_null_apm_) { + VerifyEchoCancellationSettings(/*enabled=*/true); + } + + // Turn off echo cancellation and delay agnostic aec. + send_parameters_.options.echo_cancellation = false; + SetSendParameters(send_parameters_); + if (!use_null_apm_) { + VerifyEchoCancellationSettings(/*enabled=*/false); + } + + // Restore AEC to be on to work with the following tests. + send_parameters_.options.echo_cancellation = true; + SetSendParameters(send_parameters_); + + // Turn off AGC + send_parameters_.options.auto_gain_control = false; + SetSendParameters(send_parameters_); + if (!use_null_apm_) { + VerifyEchoCancellationSettings(/*enabled=*/true); + EXPECT_FALSE(apm_config_.gain_controller1.enabled); + } + + // Turn AGC back on + send_parameters_.options.auto_gain_control = true; + SetSendParameters(send_parameters_); + if (!use_null_apm_) { + VerifyEchoCancellationSettings(/*enabled=*/true); + EXPECT_TRUE(apm_config_.gain_controller1.enabled); + } + + // Turn off other options. + send_parameters_.options.noise_suppression = false; + send_parameters_.options.highpass_filter = false; + send_parameters_.options.stereo_swapping = true; + SetSendParameters(send_parameters_); + if (!use_null_apm_) { + VerifyEchoCancellationSettings(/*enabled=*/true); + EXPECT_FALSE(IsHighPassFilterEnabled()); + EXPECT_TRUE(apm_config_.gain_controller1.enabled); + EXPECT_FALSE(apm_config_.noise_suppression.enabled); + EXPECT_EQ(apm_config_.noise_suppression.level, kDefaultNsLevel); + } + + // Set options again to ensure it has no impact. + SetSendParameters(send_parameters_); + if (!use_null_apm_) { + VerifyEchoCancellationSettings(/*enabled=*/true); + EXPECT_TRUE(apm_config_.gain_controller1.enabled); + EXPECT_FALSE(apm_config_.noise_suppression.enabled); + EXPECT_EQ(apm_config_.noise_suppression.level, kDefaultNsLevel); + } +} + +TEST_P(WebRtcVoiceEngineTestFake, SetOptionOverridesViaChannels) { + EXPECT_TRUE(SetupSendStream()); + EXPECT_CALL(*adm_, BuiltInAECIsAvailable()) + .Times(use_null_apm_ ? 4 : 8) + .WillRepeatedly(Return(false)); + EXPECT_CALL(*adm_, BuiltInAGCIsAvailable()) + .Times(use_null_apm_ ? 7 : 8) + .WillRepeatedly(Return(false)); + EXPECT_CALL(*adm_, BuiltInNSIsAvailable()) + .Times(use_null_apm_ ? 5 : 8) + .WillRepeatedly(Return(false)); + EXPECT_CALL(*adm_, RecordingIsInitialized()) + .Times(2) + .WillRepeatedly(Return(false)); + + EXPECT_CALL(*adm_, Recording()).Times(2).WillRepeatedly(Return(false)); + EXPECT_CALL(*adm_, InitRecording()).Times(2).WillRepeatedly(Return(0)); + if (!use_null_apm_) { + EXPECT_CALL(*apm_, SetExtraOptions(::testing::_)).Times(10); + } + + std::unique_ptr<cricket::WebRtcVoiceMediaChannel> channel1( + static_cast<cricket::WebRtcVoiceMediaChannel*>( + engine_->CreateMediaChannel(&call_, cricket::MediaConfig(), + cricket::AudioOptions(), + webrtc::CryptoOptions()))); + std::unique_ptr<cricket::WebRtcVoiceMediaChannel> channel2( + static_cast<cricket::WebRtcVoiceMediaChannel*>( + engine_->CreateMediaChannel(&call_, cricket::MediaConfig(), + cricket::AudioOptions(), + webrtc::CryptoOptions()))); + + // Have to add a stream to make SetSend work. + cricket::StreamParams stream1; + stream1.ssrcs.push_back(1); + channel1->AddSendStream(stream1); + cricket::StreamParams stream2; + stream2.ssrcs.push_back(2); + channel2->AddSendStream(stream2); + + // AEC and AGC and NS + cricket::AudioSendParameters parameters_options_all = send_parameters_; + parameters_options_all.options.echo_cancellation = true; + parameters_options_all.options.auto_gain_control = true; + parameters_options_all.options.noise_suppression = true; + EXPECT_TRUE(channel1->SetSendParameters(parameters_options_all)); + if (!use_null_apm_) { + VerifyEchoCancellationSettings(/*enabled=*/true); + VerifyGainControlEnabledCorrectly(); + EXPECT_TRUE(apm_config_.noise_suppression.enabled); + EXPECT_EQ(apm_config_.noise_suppression.level, kDefaultNsLevel); + EXPECT_EQ(parameters_options_all.options, channel1->options()); + EXPECT_TRUE(channel2->SetSendParameters(parameters_options_all)); + VerifyEchoCancellationSettings(/*enabled=*/true); + VerifyGainControlEnabledCorrectly(); + EXPECT_EQ(parameters_options_all.options, channel2->options()); + } + + // unset NS + cricket::AudioSendParameters parameters_options_no_ns = send_parameters_; + parameters_options_no_ns.options.noise_suppression = false; + EXPECT_TRUE(channel1->SetSendParameters(parameters_options_no_ns)); + cricket::AudioOptions expected_options = parameters_options_all.options; + if (!use_null_apm_) { + VerifyEchoCancellationSettings(/*enabled=*/true); + EXPECT_FALSE(apm_config_.noise_suppression.enabled); + EXPECT_EQ(apm_config_.noise_suppression.level, kDefaultNsLevel); + VerifyGainControlEnabledCorrectly(); + expected_options.echo_cancellation = true; + expected_options.auto_gain_control = true; + expected_options.noise_suppression = false; + EXPECT_EQ(expected_options, channel1->options()); + } + + // unset AGC + cricket::AudioSendParameters parameters_options_no_agc = send_parameters_; + parameters_options_no_agc.options.auto_gain_control = false; + EXPECT_TRUE(channel2->SetSendParameters(parameters_options_no_agc)); + if (!use_null_apm_) { + VerifyEchoCancellationSettings(/*enabled=*/true); + EXPECT_FALSE(apm_config_.gain_controller1.enabled); + EXPECT_TRUE(apm_config_.noise_suppression.enabled); + EXPECT_EQ(apm_config_.noise_suppression.level, kDefaultNsLevel); + expected_options.echo_cancellation = true; + expected_options.auto_gain_control = false; + expected_options.noise_suppression = true; + EXPECT_EQ(expected_options, channel2->options()); + } + + EXPECT_TRUE(channel_->SetSendParameters(parameters_options_all)); + if (!use_null_apm_) { + VerifyEchoCancellationSettings(/*enabled=*/true); + VerifyGainControlEnabledCorrectly(); + EXPECT_TRUE(apm_config_.noise_suppression.enabled); + EXPECT_EQ(apm_config_.noise_suppression.level, kDefaultNsLevel); + } + + channel1->SetSend(true); + if (!use_null_apm_) { + VerifyEchoCancellationSettings(/*enabled=*/true); + VerifyGainControlEnabledCorrectly(); + EXPECT_FALSE(apm_config_.noise_suppression.enabled); + EXPECT_EQ(apm_config_.noise_suppression.level, kDefaultNsLevel); + } + + channel2->SetSend(true); + if (!use_null_apm_) { + VerifyEchoCancellationSettings(/*enabled=*/true); + EXPECT_FALSE(apm_config_.gain_controller1.enabled); + EXPECT_TRUE(apm_config_.noise_suppression.enabled); + EXPECT_EQ(apm_config_.noise_suppression.level, kDefaultNsLevel); + } + + // Make sure settings take effect while we are sending. + cricket::AudioSendParameters parameters_options_no_agc_nor_ns = + send_parameters_; + parameters_options_no_agc_nor_ns.options.auto_gain_control = false; + parameters_options_no_agc_nor_ns.options.noise_suppression = false; + EXPECT_TRUE(channel2->SetSendParameters(parameters_options_no_agc_nor_ns)); + if (!use_null_apm_) { + VerifyEchoCancellationSettings(/*enabled=*/true); + EXPECT_FALSE(apm_config_.gain_controller1.enabled); + EXPECT_FALSE(apm_config_.noise_suppression.enabled); + EXPECT_EQ(apm_config_.noise_suppression.level, kDefaultNsLevel); + expected_options.echo_cancellation = true; + expected_options.auto_gain_control = false; + expected_options.noise_suppression = false; + EXPECT_EQ(expected_options, channel2->options()); + } +} + +// This test verifies DSCP settings are properly applied on voice media channel. +TEST_P(WebRtcVoiceEngineTestFake, TestSetDscpOptions) { + EXPECT_TRUE(SetupSendStream()); + cricket::FakeNetworkInterface network_interface; + cricket::MediaConfig config; + std::unique_ptr<cricket::WebRtcVoiceMediaChannel> channel; + webrtc::RtpParameters parameters; + + if (!use_null_apm_) { + EXPECT_CALL(*apm_, SetExtraOptions(::testing::_)).Times(3); + } + + channel.reset(static_cast<cricket::WebRtcVoiceMediaChannel*>( + engine_->CreateMediaChannel(&call_, config, cricket::AudioOptions(), + webrtc::CryptoOptions()))); + channel->SetInterface(&network_interface, webrtc::MediaTransportConfig()); + // Default value when DSCP is disabled should be DSCP_DEFAULT. + EXPECT_EQ(rtc::DSCP_DEFAULT, network_interface.dscp()); + + config.enable_dscp = true; + channel.reset(static_cast<cricket::WebRtcVoiceMediaChannel*>( + engine_->CreateMediaChannel(&call_, config, cricket::AudioOptions(), + webrtc::CryptoOptions()))); + channel->SetInterface(&network_interface, webrtc::MediaTransportConfig()); + EXPECT_EQ(rtc::DSCP_DEFAULT, network_interface.dscp()); + + // Create a send stream to configure + EXPECT_TRUE( + channel->AddSendStream(cricket::StreamParams::CreateLegacy(kSsrcZ))); + parameters = channel->GetRtpSendParameters(kSsrcZ); + ASSERT_FALSE(parameters.encodings.empty()); + + // Various priorities map to various dscp values. + parameters.encodings[0].network_priority = webrtc::Priority::kHigh; + ASSERT_TRUE(channel->SetRtpSendParameters(kSsrcZ, parameters).ok()); + EXPECT_EQ(rtc::DSCP_EF, network_interface.dscp()); + parameters.encodings[0].network_priority = webrtc::Priority::kVeryLow; + ASSERT_TRUE(channel->SetRtpSendParameters(kSsrcZ, parameters).ok()); + EXPECT_EQ(rtc::DSCP_CS1, network_interface.dscp()); + + // Packets should also self-identify their dscp in PacketOptions. + const uint8_t kData[10] = {0}; + EXPECT_TRUE(channel->SendRtcp(kData, sizeof(kData))); + EXPECT_EQ(rtc::DSCP_CS1, network_interface.options().dscp); + + // Verify that setting the option to false resets the + // DiffServCodePoint. + config.enable_dscp = false; + channel.reset(static_cast<cricket::WebRtcVoiceMediaChannel*>( + engine_->CreateMediaChannel(&call_, config, cricket::AudioOptions(), + webrtc::CryptoOptions()))); + channel->SetInterface(&network_interface, webrtc::MediaTransportConfig()); + // Default value when DSCP is disabled should be DSCP_DEFAULT. + EXPECT_EQ(rtc::DSCP_DEFAULT, network_interface.dscp()); + + channel->SetInterface(nullptr, webrtc::MediaTransportConfig()); +} + +TEST_P(WebRtcVoiceEngineTestFake, SetOutputVolume) { + EXPECT_TRUE(SetupChannel()); + EXPECT_FALSE(channel_->SetOutputVolume(kSsrcY, 0.5)); + cricket::StreamParams stream; + stream.ssrcs.push_back(kSsrcY); + EXPECT_TRUE(channel_->AddRecvStream(stream)); + EXPECT_DOUBLE_EQ(1, GetRecvStream(kSsrcY).gain()); + EXPECT_TRUE(channel_->SetOutputVolume(kSsrcY, 3)); + EXPECT_DOUBLE_EQ(3, GetRecvStream(kSsrcY).gain()); +} + +TEST_P(WebRtcVoiceEngineTestFake, SetOutputVolumeUnsignaledRecvStream) { + EXPECT_TRUE(SetupChannel()); + + // Spawn an unsignaled stream by sending a packet - gain should be 1. + DeliverPacket(kPcmuFrame, sizeof(kPcmuFrame)); + EXPECT_DOUBLE_EQ(1, GetRecvStream(kSsrc1).gain()); + + // Should remember the volume "2" which will be set on new unsignaled streams, + // and also set the gain to 2 on existing unsignaled streams. + EXPECT_TRUE(channel_->SetDefaultOutputVolume(2)); + EXPECT_DOUBLE_EQ(2, GetRecvStream(kSsrc1).gain()); + + // Spawn an unsignaled stream by sending a packet - gain should be 2. + unsigned char pcmuFrame2[sizeof(kPcmuFrame)]; + memcpy(pcmuFrame2, kPcmuFrame, sizeof(kPcmuFrame)); + rtc::SetBE32(&pcmuFrame2[8], kSsrcX); + DeliverPacket(pcmuFrame2, sizeof(pcmuFrame2)); + EXPECT_DOUBLE_EQ(2, GetRecvStream(kSsrcX).gain()); + + // Setting gain for all unsignaled streams. + EXPECT_TRUE(channel_->SetDefaultOutputVolume(3)); + if (kMaxUnsignaledRecvStreams > 1) { + EXPECT_DOUBLE_EQ(3, GetRecvStream(kSsrc1).gain()); + } + EXPECT_DOUBLE_EQ(3, GetRecvStream(kSsrcX).gain()); + + // Setting gain on an individual stream affects only that. + EXPECT_TRUE(channel_->SetOutputVolume(kSsrcX, 4)); + if (kMaxUnsignaledRecvStreams > 1) { + EXPECT_DOUBLE_EQ(3, GetRecvStream(kSsrc1).gain()); + } + EXPECT_DOUBLE_EQ(4, GetRecvStream(kSsrcX).gain()); +} + +TEST_P(WebRtcVoiceEngineTestFake, BaseMinimumPlayoutDelayMs) { + EXPECT_TRUE(SetupChannel()); + EXPECT_FALSE(channel_->SetBaseMinimumPlayoutDelayMs(kSsrcY, 200)); + EXPECT_FALSE(channel_->GetBaseMinimumPlayoutDelayMs(kSsrcY).has_value()); + + cricket::StreamParams stream; + stream.ssrcs.push_back(kSsrcY); + EXPECT_TRUE(channel_->AddRecvStream(stream)); + EXPECT_EQ(0, GetRecvStream(kSsrcY).base_mininum_playout_delay_ms()); + EXPECT_TRUE(channel_->SetBaseMinimumPlayoutDelayMs(kSsrcY, 300)); + EXPECT_EQ(300, GetRecvStream(kSsrcY).base_mininum_playout_delay_ms()); +} + +TEST_P(WebRtcVoiceEngineTestFake, + BaseMinimumPlayoutDelayMsUnsignaledRecvStream) { + // Here base minimum delay is abbreviated to delay in comments for shortness. + EXPECT_TRUE(SetupChannel()); + + // Spawn an unsignaled stream by sending a packet - delay should be 0. + DeliverPacket(kPcmuFrame, sizeof(kPcmuFrame)); + EXPECT_EQ(0, channel_->GetBaseMinimumPlayoutDelayMs(kSsrc1).value_or(-1)); + // Check that it doesn't provide default values for unknown ssrc. + EXPECT_FALSE(channel_->GetBaseMinimumPlayoutDelayMs(kSsrcY).has_value()); + + // Check that default value for unsignaled streams is 0. + EXPECT_EQ(0, channel_->GetBaseMinimumPlayoutDelayMs(kSsrc0).value_or(-1)); + + // Should remember the delay 100 which will be set on new unsignaled streams, + // and also set the delay to 100 on existing unsignaled streams. + EXPECT_TRUE(channel_->SetBaseMinimumPlayoutDelayMs(kSsrc0, 100)); + EXPECT_EQ(100, channel_->GetBaseMinimumPlayoutDelayMs(kSsrc0).value_or(-1)); + // Check that it doesn't provide default values for unknown ssrc. + EXPECT_FALSE(channel_->GetBaseMinimumPlayoutDelayMs(kSsrcY).has_value()); + + // Spawn an unsignaled stream by sending a packet - delay should be 100. + unsigned char pcmuFrame2[sizeof(kPcmuFrame)]; + memcpy(pcmuFrame2, kPcmuFrame, sizeof(kPcmuFrame)); + rtc::SetBE32(&pcmuFrame2[8], kSsrcX); + DeliverPacket(pcmuFrame2, sizeof(pcmuFrame2)); + EXPECT_EQ(100, channel_->GetBaseMinimumPlayoutDelayMs(kSsrcX).value_or(-1)); + + // Setting delay with SSRC=0 should affect all unsignaled streams. + EXPECT_TRUE(channel_->SetBaseMinimumPlayoutDelayMs(kSsrc0, 300)); + if (kMaxUnsignaledRecvStreams > 1) { + EXPECT_EQ(300, channel_->GetBaseMinimumPlayoutDelayMs(kSsrc1).value_or(-1)); + } + EXPECT_EQ(300, channel_->GetBaseMinimumPlayoutDelayMs(kSsrcX).value_or(-1)); + + // Setting delay on an individual stream affects only that. + EXPECT_TRUE(channel_->SetBaseMinimumPlayoutDelayMs(kSsrcX, 400)); + if (kMaxUnsignaledRecvStreams > 1) { + EXPECT_EQ(300, channel_->GetBaseMinimumPlayoutDelayMs(kSsrc1).value_or(-1)); + } + EXPECT_EQ(400, channel_->GetBaseMinimumPlayoutDelayMs(kSsrcX).value_or(-1)); + EXPECT_EQ(300, channel_->GetBaseMinimumPlayoutDelayMs(kSsrc0).value_or(-1)); + // Check that it doesn't provide default values for unknown ssrc. + EXPECT_FALSE(channel_->GetBaseMinimumPlayoutDelayMs(kSsrcY).has_value()); +} + +TEST_P(WebRtcVoiceEngineTestFake, SetsSyncGroupFromStreamId) { + const uint32_t kAudioSsrc = 123; + const std::string kStreamId = "AvSyncLabel"; + + EXPECT_TRUE(SetupSendStream()); + cricket::StreamParams sp = cricket::StreamParams::CreateLegacy(kAudioSsrc); + sp.set_stream_ids({kStreamId}); + // Creating two channels to make sure that sync label is set properly for both + // the default voice channel and following ones. + EXPECT_TRUE(channel_->AddRecvStream(sp)); + sp.ssrcs[0] += 1; + EXPECT_TRUE(channel_->AddRecvStream(sp)); + + ASSERT_EQ(2u, call_.GetAudioReceiveStreams().size()); + EXPECT_EQ(kStreamId, + call_.GetAudioReceiveStream(kAudioSsrc)->GetConfig().sync_group) + << "SyncGroup should be set based on stream id"; + EXPECT_EQ(kStreamId, + call_.GetAudioReceiveStream(kAudioSsrc + 1)->GetConfig().sync_group) + << "SyncGroup should be set based on stream id"; +} + +// TODO(solenberg): Remove, once recv streams are configured through Call. +// (This is then covered by TestSetRecvRtpHeaderExtensions.) +TEST_P(WebRtcVoiceEngineTestFake, ConfiguresAudioReceiveStreamRtpExtensions) { + // Test that setting the header extensions results in the expected state + // changes on an associated Call. + std::vector<uint32_t> ssrcs; + ssrcs.push_back(223); + ssrcs.push_back(224); + + EXPECT_TRUE(SetupSendStream()); + SetSendParameters(send_parameters_); + for (uint32_t ssrc : ssrcs) { + EXPECT_TRUE( + channel_->AddRecvStream(cricket::StreamParams::CreateLegacy(ssrc))); + } + + EXPECT_EQ(2u, call_.GetAudioReceiveStreams().size()); + for (uint32_t ssrc : ssrcs) { + const auto* s = call_.GetAudioReceiveStream(ssrc); + EXPECT_NE(nullptr, s); + EXPECT_EQ(0u, s->GetConfig().rtp.extensions.size()); + } + + // Set up receive extensions. + const std::vector<webrtc::RtpExtension> header_extensions = + GetDefaultEnabledRtpHeaderExtensions(*engine_); + cricket::AudioRecvParameters recv_parameters; + recv_parameters.extensions = header_extensions; + channel_->SetRecvParameters(recv_parameters); + EXPECT_EQ(2u, call_.GetAudioReceiveStreams().size()); + for (uint32_t ssrc : ssrcs) { + const auto* s = call_.GetAudioReceiveStream(ssrc); + EXPECT_NE(nullptr, s); + const auto& s_exts = s->GetConfig().rtp.extensions; + EXPECT_EQ(header_extensions.size(), s_exts.size()); + for (const auto& e_ext : header_extensions) { + for (const auto& s_ext : s_exts) { + if (e_ext.id == s_ext.id) { + EXPECT_EQ(e_ext.uri, s_ext.uri); + } + } + } + } + + // Disable receive extensions. + channel_->SetRecvParameters(cricket::AudioRecvParameters()); + for (uint32_t ssrc : ssrcs) { + const auto* s = call_.GetAudioReceiveStream(ssrc); + EXPECT_NE(nullptr, s); + EXPECT_EQ(0u, s->GetConfig().rtp.extensions.size()); + } +} + +TEST_P(WebRtcVoiceEngineTestFake, DeliverAudioPacket_Call) { + // Test that packets are forwarded to the Call when configured accordingly. + const uint32_t kAudioSsrc = 1; + rtc::CopyOnWriteBuffer kPcmuPacket(kPcmuFrame, sizeof(kPcmuFrame)); + static const unsigned char kRtcp[] = { + 0x80, 0xc9, 0x00, 0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; + rtc::CopyOnWriteBuffer kRtcpPacket(kRtcp, sizeof(kRtcp)); + + EXPECT_TRUE(SetupSendStream()); + cricket::WebRtcVoiceMediaChannel* media_channel = + static_cast<cricket::WebRtcVoiceMediaChannel*>(channel_); + SetSendParameters(send_parameters_); + EXPECT_TRUE(media_channel->AddRecvStream( + cricket::StreamParams::CreateLegacy(kAudioSsrc))); + + EXPECT_EQ(1u, call_.GetAudioReceiveStreams().size()); + const cricket::FakeAudioReceiveStream* s = + call_.GetAudioReceiveStream(kAudioSsrc); + EXPECT_EQ(0, s->received_packets()); + channel_->OnPacketReceived(kPcmuPacket, /* packet_time_us */ -1); + EXPECT_EQ(1, s->received_packets()); +} + +// All receive channels should be associated with the first send channel, +// since they do not send RTCP SR. +TEST_P(WebRtcVoiceEngineTestFake, AssociateFirstSendChannel_SendCreatedFirst) { + EXPECT_TRUE(SetupSendStream()); + EXPECT_TRUE(AddRecvStream(kSsrcY)); + EXPECT_EQ(kSsrcX, GetRecvStreamConfig(kSsrcY).rtp.local_ssrc); + EXPECT_TRUE( + channel_->AddSendStream(cricket::StreamParams::CreateLegacy(kSsrcZ))); + EXPECT_EQ(kSsrcX, GetRecvStreamConfig(kSsrcY).rtp.local_ssrc); + EXPECT_TRUE(AddRecvStream(kSsrcW)); + EXPECT_EQ(kSsrcX, GetRecvStreamConfig(kSsrcW).rtp.local_ssrc); +} + +TEST_P(WebRtcVoiceEngineTestFake, AssociateFirstSendChannel_RecvCreatedFirst) { + EXPECT_TRUE(SetupRecvStream()); + EXPECT_EQ(0xFA17FA17u, GetRecvStreamConfig(kSsrcX).rtp.local_ssrc); + EXPECT_TRUE( + channel_->AddSendStream(cricket::StreamParams::CreateLegacy(kSsrcY))); + EXPECT_EQ(kSsrcY, GetRecvStreamConfig(kSsrcX).rtp.local_ssrc); + EXPECT_TRUE(AddRecvStream(kSsrcZ)); + EXPECT_EQ(kSsrcY, GetRecvStreamConfig(kSsrcZ).rtp.local_ssrc); + EXPECT_TRUE( + channel_->AddSendStream(cricket::StreamParams::CreateLegacy(kSsrcW))); + EXPECT_EQ(kSsrcY, GetRecvStreamConfig(kSsrcX).rtp.local_ssrc); + EXPECT_EQ(kSsrcY, GetRecvStreamConfig(kSsrcZ).rtp.local_ssrc); +} + +TEST_P(WebRtcVoiceEngineTestFake, SetRawAudioSink) { + EXPECT_TRUE(SetupChannel()); + std::unique_ptr<FakeAudioSink> fake_sink_1(new FakeAudioSink()); + std::unique_ptr<FakeAudioSink> fake_sink_2(new FakeAudioSink()); + + // Setting the sink before a recv stream exists should do nothing. + channel_->SetRawAudioSink(kSsrcX, std::move(fake_sink_1)); + EXPECT_TRUE(AddRecvStream(kSsrcX)); + EXPECT_EQ(nullptr, GetRecvStream(kSsrcX).sink()); + + // Now try actually setting the sink. + channel_->SetRawAudioSink(kSsrcX, std::move(fake_sink_2)); + EXPECT_NE(nullptr, GetRecvStream(kSsrcX).sink()); + + // Now try resetting it. + channel_->SetRawAudioSink(kSsrcX, nullptr); + EXPECT_EQ(nullptr, GetRecvStream(kSsrcX).sink()); +} + +TEST_P(WebRtcVoiceEngineTestFake, SetRawAudioSinkUnsignaledRecvStream) { + EXPECT_TRUE(SetupChannel()); + std::unique_ptr<FakeAudioSink> fake_sink_1(new FakeAudioSink()); + std::unique_ptr<FakeAudioSink> fake_sink_2(new FakeAudioSink()); + std::unique_ptr<FakeAudioSink> fake_sink_3(new FakeAudioSink()); + std::unique_ptr<FakeAudioSink> fake_sink_4(new FakeAudioSink()); + + // Should be able to set a default sink even when no stream exists. + channel_->SetDefaultRawAudioSink(std::move(fake_sink_1)); + + // Spawn an unsignaled stream by sending a packet - it should be assigned the + // default sink. + DeliverPacket(kPcmuFrame, sizeof(kPcmuFrame)); + EXPECT_NE(nullptr, GetRecvStream(kSsrc1).sink()); + + // Try resetting the default sink. + channel_->SetDefaultRawAudioSink(nullptr); + EXPECT_EQ(nullptr, GetRecvStream(kSsrc1).sink()); + + // Try setting the default sink while the default stream exists. + channel_->SetDefaultRawAudioSink(std::move(fake_sink_2)); + EXPECT_NE(nullptr, GetRecvStream(kSsrc1).sink()); + + // If we remove and add a default stream, it should get the same sink. + EXPECT_TRUE(channel_->RemoveRecvStream(kSsrc1)); + DeliverPacket(kPcmuFrame, sizeof(kPcmuFrame)); + EXPECT_NE(nullptr, GetRecvStream(kSsrc1).sink()); + + // Spawn another unsignaled stream - it should be assigned the default sink + // and the previous unsignaled stream should lose it. + unsigned char pcmuFrame2[sizeof(kPcmuFrame)]; + memcpy(pcmuFrame2, kPcmuFrame, sizeof(kPcmuFrame)); + rtc::SetBE32(&pcmuFrame2[8], kSsrcX); + DeliverPacket(pcmuFrame2, sizeof(pcmuFrame2)); + if (kMaxUnsignaledRecvStreams > 1) { + EXPECT_EQ(nullptr, GetRecvStream(kSsrc1).sink()); + } + EXPECT_NE(nullptr, GetRecvStream(kSsrcX).sink()); + + // Reset the default sink - the second unsignaled stream should lose it. + channel_->SetDefaultRawAudioSink(nullptr); + if (kMaxUnsignaledRecvStreams > 1) { + EXPECT_EQ(nullptr, GetRecvStream(kSsrc1).sink()); + } + EXPECT_EQ(nullptr, GetRecvStream(kSsrcX).sink()); + + // Try setting the default sink while two streams exists. + channel_->SetDefaultRawAudioSink(std::move(fake_sink_3)); + if (kMaxUnsignaledRecvStreams > 1) { + EXPECT_EQ(nullptr, GetRecvStream(kSsrc1).sink()); + } + EXPECT_NE(nullptr, GetRecvStream(kSsrcX).sink()); + + // Try setting the sink for the first unsignaled stream using its known SSRC. + channel_->SetRawAudioSink(kSsrc1, std::move(fake_sink_4)); + if (kMaxUnsignaledRecvStreams > 1) { + EXPECT_NE(nullptr, GetRecvStream(kSsrc1).sink()); + } + EXPECT_NE(nullptr, GetRecvStream(kSsrcX).sink()); + if (kMaxUnsignaledRecvStreams > 1) { + EXPECT_NE(GetRecvStream(kSsrc1).sink(), GetRecvStream(kSsrcX).sink()); + } +} + +// Test that, just like the video channel, the voice channel communicates the +// network state to the call. +TEST_P(WebRtcVoiceEngineTestFake, OnReadyToSendSignalsNetworkState) { + EXPECT_TRUE(SetupChannel()); + + EXPECT_EQ(webrtc::kNetworkUp, + call_.GetNetworkState(webrtc::MediaType::AUDIO)); + EXPECT_EQ(webrtc::kNetworkUp, + call_.GetNetworkState(webrtc::MediaType::VIDEO)); + + channel_->OnReadyToSend(false); + EXPECT_EQ(webrtc::kNetworkDown, + call_.GetNetworkState(webrtc::MediaType::AUDIO)); + EXPECT_EQ(webrtc::kNetworkUp, + call_.GetNetworkState(webrtc::MediaType::VIDEO)); + + channel_->OnReadyToSend(true); + EXPECT_EQ(webrtc::kNetworkUp, + call_.GetNetworkState(webrtc::MediaType::AUDIO)); + EXPECT_EQ(webrtc::kNetworkUp, + call_.GetNetworkState(webrtc::MediaType::VIDEO)); +} + +// Test that playout is still started after changing parameters +TEST_P(WebRtcVoiceEngineTestFake, PreservePlayoutWhenRecreateRecvStream) { + SetupRecvStream(); + channel_->SetPlayout(true); + EXPECT_TRUE(GetRecvStream(kSsrcX).started()); + + // Changing RTP header extensions will recreate the AudioReceiveStream. + cricket::AudioRecvParameters parameters; + parameters.extensions.push_back( + webrtc::RtpExtension(webrtc::RtpExtension::kAudioLevelUri, 12)); + channel_->SetRecvParameters(parameters); + + EXPECT_TRUE(GetRecvStream(kSsrcX).started()); +} + +// Tests when GetSources is called with non-existing ssrc, it will return an +// empty list of RtpSource without crashing. +TEST_P(WebRtcVoiceEngineTestFake, GetSourcesWithNonExistingSsrc) { + // Setup an recv stream with |kSsrcX|. + SetupRecvStream(); + cricket::WebRtcVoiceMediaChannel* media_channel = + static_cast<cricket::WebRtcVoiceMediaChannel*>(channel_); + // Call GetSources with |kSsrcY| which doesn't exist. + std::vector<webrtc::RtpSource> sources = media_channel->GetSources(kSsrcY); + EXPECT_EQ(0u, sources.size()); +} + +// Tests that the library initializes and shuts down properly. +TEST(WebRtcVoiceEngineTest, StartupShutdown) { + for (bool use_null_apm : {false, true}) { + // If the VoiceEngine wants to gather available codecs early, that's fine + // but we never want it to create a decoder at this stage. + std::unique_ptr<webrtc::TaskQueueFactory> task_queue_factory = + webrtc::CreateDefaultTaskQueueFactory(); + rtc::scoped_refptr<webrtc::test::MockAudioDeviceModule> adm = + webrtc::test::MockAudioDeviceModule::CreateNice(); + rtc::scoped_refptr<webrtc::AudioProcessing> apm = + use_null_apm ? nullptr : webrtc::AudioProcessingBuilder().Create(); + cricket::WebRtcVoiceEngine engine( + task_queue_factory.get(), adm, + webrtc::MockAudioEncoderFactory::CreateUnusedFactory(), + webrtc::MockAudioDecoderFactory::CreateUnusedFactory(), nullptr, apm); + engine.Init(); + webrtc::RtcEventLogNull event_log; + webrtc::Call::Config call_config(&event_log); + webrtc::FieldTrialBasedConfig field_trials; + call_config.trials = &field_trials; + call_config.task_queue_factory = task_queue_factory.get(); + auto call = absl::WrapUnique(webrtc::Call::Create(call_config)); + cricket::VoiceMediaChannel* channel = engine.CreateMediaChannel( + call.get(), cricket::MediaConfig(), cricket::AudioOptions(), + webrtc::CryptoOptions()); + EXPECT_TRUE(channel != nullptr); + delete channel; + } +} + +// Tests that reference counting on the external ADM is correct. +TEST(WebRtcVoiceEngineTest, StartupShutdownWithExternalADM) { + for (bool use_null_apm : {false, true}) { + std::unique_ptr<webrtc::TaskQueueFactory> task_queue_factory = + webrtc::CreateDefaultTaskQueueFactory(); + rtc::scoped_refptr<rtc::RefCountedObject< + ::testing::NiceMock<webrtc::test::MockAudioDeviceModule>>> + adm(new rtc::RefCountedObject< + ::testing::NiceMock<webrtc::test::MockAudioDeviceModule>>()); + { + rtc::scoped_refptr<webrtc::AudioProcessing> apm = + use_null_apm ? nullptr : webrtc::AudioProcessingBuilder().Create(); + cricket::WebRtcVoiceEngine engine( + task_queue_factory.get(), adm, + webrtc::MockAudioEncoderFactory::CreateUnusedFactory(), + webrtc::MockAudioDecoderFactory::CreateUnusedFactory(), nullptr, apm); + engine.Init(); + webrtc::RtcEventLogNull event_log; + webrtc::Call::Config call_config(&event_log); + webrtc::FieldTrialBasedConfig field_trials; + call_config.trials = &field_trials; + call_config.task_queue_factory = task_queue_factory.get(); + auto call = absl::WrapUnique(webrtc::Call::Create(call_config)); + cricket::VoiceMediaChannel* channel = engine.CreateMediaChannel( + call.get(), cricket::MediaConfig(), cricket::AudioOptions(), + webrtc::CryptoOptions()); + EXPECT_TRUE(channel != nullptr); + delete channel; + } + // The engine/channel should have dropped their references. + EXPECT_TRUE(adm->HasOneRef()); + } +} + +// Verify the payload id of common audio codecs, including CN, ISAC, and G722. +TEST(WebRtcVoiceEngineTest, HasCorrectPayloadTypeMapping) { + for (bool use_null_apm : {false, true}) { + std::unique_ptr<webrtc::TaskQueueFactory> task_queue_factory = + webrtc::CreateDefaultTaskQueueFactory(); + // TODO(ossu): Why are the payload types of codecs with non-static payload + // type assignments checked here? It shouldn't really matter. + rtc::scoped_refptr<webrtc::test::MockAudioDeviceModule> adm = + webrtc::test::MockAudioDeviceModule::CreateNice(); + rtc::scoped_refptr<webrtc::AudioProcessing> apm = + use_null_apm ? nullptr : webrtc::AudioProcessingBuilder().Create(); + cricket::WebRtcVoiceEngine engine( + task_queue_factory.get(), adm, + webrtc::MockAudioEncoderFactory::CreateUnusedFactory(), + webrtc::MockAudioDecoderFactory::CreateUnusedFactory(), nullptr, apm); + engine.Init(); + for (const cricket::AudioCodec& codec : engine.send_codecs()) { + auto is_codec = [&codec](const char* name, int clockrate = 0) { + return absl::EqualsIgnoreCase(codec.name, name) && + (clockrate == 0 || codec.clockrate == clockrate); + }; + if (is_codec("CN", 16000)) { + EXPECT_EQ(105, codec.id); + } else if (is_codec("CN", 32000)) { + EXPECT_EQ(106, codec.id); + } else if (is_codec("ISAC", 16000)) { + EXPECT_EQ(103, codec.id); + } else if (is_codec("ISAC", 32000)) { + EXPECT_EQ(104, codec.id); + } else if (is_codec("G722", 8000)) { + EXPECT_EQ(9, codec.id); + } else if (is_codec("telephone-event", 8000)) { + EXPECT_EQ(126, codec.id); + // TODO(solenberg): 16k, 32k, 48k DTMF should be dynamically assigned. + // Remove these checks once both send and receive side assigns payload + // types dynamically. + } else if (is_codec("telephone-event", 16000)) { + EXPECT_EQ(113, codec.id); + } else if (is_codec("telephone-event", 32000)) { + EXPECT_EQ(112, codec.id); + } else if (is_codec("telephone-event", 48000)) { + EXPECT_EQ(110, codec.id); + } else if (is_codec("opus")) { + EXPECT_EQ(111, codec.id); + ASSERT_TRUE(codec.params.find("minptime") != codec.params.end()); + EXPECT_EQ("10", codec.params.find("minptime")->second); + ASSERT_TRUE(codec.params.find("useinbandfec") != codec.params.end()); + EXPECT_EQ("1", codec.params.find("useinbandfec")->second); + } + } + } +} + +// Tests that VoE supports at least 32 channels +TEST(WebRtcVoiceEngineTest, Has32Channels) { + for (bool use_null_apm : {false, true}) { + std::unique_ptr<webrtc::TaskQueueFactory> task_queue_factory = + webrtc::CreateDefaultTaskQueueFactory(); + rtc::scoped_refptr<webrtc::test::MockAudioDeviceModule> adm = + webrtc::test::MockAudioDeviceModule::CreateNice(); + rtc::scoped_refptr<webrtc::AudioProcessing> apm = + use_null_apm ? nullptr : webrtc::AudioProcessingBuilder().Create(); + cricket::WebRtcVoiceEngine engine( + task_queue_factory.get(), adm, + webrtc::MockAudioEncoderFactory::CreateUnusedFactory(), + webrtc::MockAudioDecoderFactory::CreateUnusedFactory(), nullptr, apm); + engine.Init(); + webrtc::RtcEventLogNull event_log; + webrtc::Call::Config call_config(&event_log); + webrtc::FieldTrialBasedConfig field_trials; + call_config.trials = &field_trials; + call_config.task_queue_factory = task_queue_factory.get(); + auto call = absl::WrapUnique(webrtc::Call::Create(call_config)); + + cricket::VoiceMediaChannel* channels[32]; + size_t num_channels = 0; + while (num_channels < arraysize(channels)) { + cricket::VoiceMediaChannel* channel = engine.CreateMediaChannel( + call.get(), cricket::MediaConfig(), cricket::AudioOptions(), + webrtc::CryptoOptions()); + if (!channel) + break; + channels[num_channels++] = channel; + } + + size_t expected = arraysize(channels); + EXPECT_EQ(expected, num_channels); + + while (num_channels > 0) { + delete channels[--num_channels]; + } + } +} + +// Test that we set our preferred codecs properly. +TEST(WebRtcVoiceEngineTest, SetRecvCodecs) { + for (bool use_null_apm : {false, true}) { + std::unique_ptr<webrtc::TaskQueueFactory> task_queue_factory = + webrtc::CreateDefaultTaskQueueFactory(); + // TODO(ossu): I'm not sure of the intent of this test. It's either: + // - Check that our builtin codecs are usable by Channel. + // - The codecs provided by the engine is usable by Channel. + // It does not check that the codecs in the RecvParameters are actually + // what we sent in - though it's probably reasonable to expect so, if + // SetRecvParameters returns true. + // I think it will become clear once audio decoder injection is completed. + rtc::scoped_refptr<webrtc::test::MockAudioDeviceModule> adm = + webrtc::test::MockAudioDeviceModule::CreateNice(); + rtc::scoped_refptr<webrtc::AudioProcessing> apm = + use_null_apm ? nullptr : webrtc::AudioProcessingBuilder().Create(); + cricket::WebRtcVoiceEngine engine( + task_queue_factory.get(), adm, + webrtc::MockAudioEncoderFactory::CreateUnusedFactory(), + webrtc::CreateBuiltinAudioDecoderFactory(), nullptr, apm); + engine.Init(); + webrtc::RtcEventLogNull event_log; + webrtc::Call::Config call_config(&event_log); + webrtc::FieldTrialBasedConfig field_trials; + call_config.trials = &field_trials; + call_config.task_queue_factory = task_queue_factory.get(); + auto call = absl::WrapUnique(webrtc::Call::Create(call_config)); + cricket::WebRtcVoiceMediaChannel channel( + &engine, cricket::MediaConfig(), cricket::AudioOptions(), + webrtc::CryptoOptions(), call.get()); + cricket::AudioRecvParameters parameters; + parameters.codecs = engine.recv_codecs(); + EXPECT_TRUE(channel.SetRecvParameters(parameters)); + } +} + +TEST(WebRtcVoiceEngineTest, CollectRecvCodecs) { + for (bool use_null_apm : {false, true}) { + std::vector<webrtc::AudioCodecSpec> specs; + webrtc::AudioCodecSpec spec1{{"codec1", 48000, 2, {{"param1", "value1"}}}, + {48000, 2, 16000, 10000, 20000}}; + spec1.info.allow_comfort_noise = false; + spec1.info.supports_network_adaption = true; + specs.push_back(spec1); + webrtc::AudioCodecSpec spec2{{"codec2", 32000, 1}, {32000, 1, 32000}}; + spec2.info.allow_comfort_noise = false; + specs.push_back(spec2); + specs.push_back(webrtc::AudioCodecSpec{ + {"codec3", 16000, 1, {{"param1", "value1b"}, {"param2", "value2"}}}, + {16000, 1, 13300}}); + specs.push_back( + webrtc::AudioCodecSpec{{"codec4", 8000, 1}, {8000, 1, 64000}}); + specs.push_back( + webrtc::AudioCodecSpec{{"codec5", 8000, 2}, {8000, 1, 64000}}); + + std::unique_ptr<webrtc::TaskQueueFactory> task_queue_factory = + webrtc::CreateDefaultTaskQueueFactory(); + rtc::scoped_refptr<webrtc::MockAudioEncoderFactory> unused_encoder_factory = + webrtc::MockAudioEncoderFactory::CreateUnusedFactory(); + rtc::scoped_refptr<webrtc::MockAudioDecoderFactory> mock_decoder_factory = + new rtc::RefCountedObject<webrtc::MockAudioDecoderFactory>; + EXPECT_CALL(*mock_decoder_factory.get(), GetSupportedDecoders()) + .WillOnce(Return(specs)); + rtc::scoped_refptr<webrtc::test::MockAudioDeviceModule> adm = + webrtc::test::MockAudioDeviceModule::CreateNice(); + + rtc::scoped_refptr<webrtc::AudioProcessing> apm = + use_null_apm ? nullptr : webrtc::AudioProcessingBuilder().Create(); + cricket::WebRtcVoiceEngine engine(task_queue_factory.get(), adm, + unused_encoder_factory, + mock_decoder_factory, nullptr, apm); + engine.Init(); + auto codecs = engine.recv_codecs(); + EXPECT_EQ(11u, codecs.size()); + + // Rather than just ASSERTing that there are enough codecs, ensure that we + // can check the actual values safely, to provide better test results. + auto get_codec = [&codecs](size_t index) -> const cricket::AudioCodec& { + static const cricket::AudioCodec missing_codec(0, "<missing>", 0, 0, 0); + if (codecs.size() > index) + return codecs[index]; + return missing_codec; + }; + + // Ensure the general codecs are generated first and in order. + for (size_t i = 0; i != specs.size(); ++i) { + EXPECT_EQ(specs[i].format.name, get_codec(i).name); + EXPECT_EQ(specs[i].format.clockrate_hz, get_codec(i).clockrate); + EXPECT_EQ(specs[i].format.num_channels, get_codec(i).channels); + EXPECT_EQ(specs[i].format.parameters, get_codec(i).params); + } + + // Find the index of a codec, or -1 if not found, so that we can easily + // check supplementary codecs are ordered after the general codecs. + auto find_codec = [&codecs](const webrtc::SdpAudioFormat& format) -> int { + for (size_t i = 0; i != codecs.size(); ++i) { + const cricket::AudioCodec& codec = codecs[i]; + if (absl::EqualsIgnoreCase(codec.name, format.name) && + codec.clockrate == format.clockrate_hz && + codec.channels == format.num_channels) { + return rtc::checked_cast<int>(i); + } + } + return -1; + }; + + // Ensure all supplementary codecs are generated last. Their internal + // ordering is not important. Without this cast, the comparison turned + // unsigned and, thus, failed for -1. + const int num_specs = static_cast<int>(specs.size()); + EXPECT_GE(find_codec({"cn", 8000, 1}), num_specs); + EXPECT_GE(find_codec({"cn", 16000, 1}), num_specs); + EXPECT_EQ(find_codec({"cn", 32000, 1}), -1); + EXPECT_GE(find_codec({"telephone-event", 8000, 1}), num_specs); + EXPECT_GE(find_codec({"telephone-event", 16000, 1}), num_specs); + EXPECT_GE(find_codec({"telephone-event", 32000, 1}), num_specs); + EXPECT_GE(find_codec({"telephone-event", 48000, 1}), num_specs); + } +} diff --git a/media/sctp/OWNERS.webrtc b/media/sctp/OWNERS.webrtc new file mode 100644 index 0000000000..a32f041ac8 --- /dev/null +++ b/media/sctp/OWNERS.webrtc @@ -0,0 +1 @@ +deadbeef@webrtc.org diff --git a/media/sctp/noop.cc b/media/sctp/noop.cc new file mode 100644 index 0000000000..a3523b18b2 --- /dev/null +++ b/media/sctp/noop.cc @@ -0,0 +1,13 @@ +/* + * Copyright 2017 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// This file is only needed to make ninja happy on some platforms. +// On some platforms it is not possible to link an rtc_static_library +// without any source file listed in the GN target. diff --git a/media/sctp/sctp_transport.cc b/media/sctp/sctp_transport.cc new file mode 100644 index 0000000000..40061a6048 --- /dev/null +++ b/media/sctp/sctp_transport.cc @@ -0,0 +1,1277 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include <errno.h> +namespace { +// Some ERRNO values get re-#defined to WSA* equivalents in some talk/ +// headers. We save the original ones in an enum. +enum PreservedErrno { + SCTP_EINPROGRESS = EINPROGRESS, + SCTP_EWOULDBLOCK = EWOULDBLOCK +}; +} // namespace + +#include <stdarg.h> +#include <stdio.h> + +#include <memory> + +#include "absl/algorithm/container.h" +#include "absl/base/attributes.h" +#include "absl/types/optional.h" +#include "media/base/codec.h" +#include "media/base/media_channel.h" +#include "media/base/media_constants.h" +#include "media/base/stream_params.h" +#include "media/sctp/sctp_transport.h" +#include "p2p/base/dtls_transport_internal.h" // For PF_NORMAL +#include "rtc_base/arraysize.h" +#include "rtc_base/copy_on_write_buffer.h" +#include "rtc_base/critical_section.h" +#include "rtc_base/helpers.h" +#include "rtc_base/logging.h" +#include "rtc_base/numerics/safe_conversions.h" +#include "rtc_base/string_utils.h" +#include "rtc_base/thread_checker.h" +#include "rtc_base/trace_event.h" +#include "usrsctplib/usrsctp.h" + +namespace { + +// The biggest SCTP packet. Starting from a 'safe' wire MTU value of 1280, +// take off 80 bytes for DTLS/TURN/TCP/IP overhead. +static constexpr size_t kSctpMtu = 1200; + +// Set the initial value of the static SCTP Data Engines reference count. +ABSL_CONST_INIT int g_usrsctp_usage_count = 0; +ABSL_CONST_INIT rtc::GlobalLock g_usrsctp_lock_; + +// DataMessageType is used for the SCTP "Payload Protocol Identifier", as +// defined in http://tools.ietf.org/html/rfc4960#section-14.4 +// +// For the list of IANA approved values see: +// http://www.iana.org/assignments/sctp-parameters/sctp-parameters.xml +// The value is not used by SCTP itself. It indicates the protocol running +// on top of SCTP. +enum PayloadProtocolIdentifier { + PPID_NONE = 0, // No protocol is specified. + // Matches the PPIDs in mozilla source and + // https://datatracker.ietf.org/doc/draft-ietf-rtcweb-data-protocol Sec. 9 + // They're not yet assigned by IANA. + PPID_CONTROL = 50, + PPID_BINARY_PARTIAL = 52, + PPID_BINARY_LAST = 53, + PPID_TEXT_PARTIAL = 54, + PPID_TEXT_LAST = 51 +}; + +// Helper for logging SCTP messages. +#if defined(__GNUC__) +__attribute__((__format__(__printf__, 1, 2))) +#endif +void DebugSctpPrintf(const char* format, ...) { +#if RTC_DCHECK_IS_ON + char s[255]; + va_list ap; + va_start(ap, format); + vsnprintf(s, sizeof(s), format, ap); + RTC_LOG(LS_INFO) << "SCTP: " << s; + va_end(ap); +#endif +} + +// Get the PPID to use for the terminating fragment of this type. +PayloadProtocolIdentifier GetPpid(cricket::DataMessageType type) { + switch (type) { + default: + case cricket::DMT_NONE: + return PPID_NONE; + case cricket::DMT_CONTROL: + return PPID_CONTROL; + case cricket::DMT_BINARY: + return PPID_BINARY_LAST; + case cricket::DMT_TEXT: + return PPID_TEXT_LAST; + } +} + +bool GetDataMediaType(PayloadProtocolIdentifier ppid, + cricket::DataMessageType* dest) { + RTC_DCHECK(dest != NULL); + switch (ppid) { + case PPID_BINARY_PARTIAL: + case PPID_BINARY_LAST: + *dest = cricket::DMT_BINARY; + return true; + + case PPID_TEXT_PARTIAL: + case PPID_TEXT_LAST: + *dest = cricket::DMT_TEXT; + return true; + + case PPID_CONTROL: + *dest = cricket::DMT_CONTROL; + return true; + + case PPID_NONE: + *dest = cricket::DMT_NONE; + return true; + + default: + return false; + } +} + +// Log the packet in text2pcap format, if log level is at LS_VERBOSE. +// +// In order to turn these logs into a pcap file you can use, first filter the +// "SCTP_PACKET" log lines: +// +// cat chrome_debug.log | grep SCTP_PACKET > filtered.log +// +// Then run through text2pcap: +// +// text2pcap -n -l 248 -D -t '%H:%M:%S.' filtered.log filtered.pcapng +// +// Command flag information: +// -n: Outputs to a pcapng file, can specify inbound/outbound packets. +// -l: Specifies the link layer header type. 248 means SCTP. See: +// http://www.tcpdump.org/linktypes.html +// -D: Text before packet specifies if it is inbound or outbound. +// -t: Time format. +// +// Why do all this? Because SCTP goes over DTLS, which is encrypted. So just +// getting a normal packet capture won't help you, unless you have the DTLS +// keying material. +void VerboseLogPacket(const void* data, size_t length, int direction) { + if (RTC_LOG_CHECK_LEVEL(LS_VERBOSE) && length > 0) { + char* dump_buf; + // Some downstream project uses an older version of usrsctp that expects + // a non-const "void*" as first parameter when dumping the packet, so we + // need to cast the const away here to avoid a compiler error. + if ((dump_buf = usrsctp_dumppacket(const_cast<void*>(data), length, + direction)) != NULL) { + RTC_LOG(LS_VERBOSE) << dump_buf; + usrsctp_freedumpbuffer(dump_buf); + } + } +} + +// Creates the sctp_sendv_spa struct used for setting flags in the +// sctp_sendv() call. +sctp_sendv_spa CreateSctpSendParams(const cricket::SendDataParams& params) { + struct sctp_sendv_spa spa = {0}; + spa.sendv_flags |= SCTP_SEND_SNDINFO_VALID; + spa.sendv_sndinfo.snd_sid = params.sid; + spa.sendv_sndinfo.snd_ppid = rtc::HostToNetwork32(GetPpid(params.type)); + // Explicitly marking the EOR flag turns the usrsctp_sendv call below into a + // non atomic operation. This means that the sctp lib might only accept the + // message partially. This is done in order to improve throughput, so that we + // don't have to wait for an empty buffer to send the max message length, for + // example. + spa.sendv_sndinfo.snd_flags |= SCTP_EOR; + + // Ordered implies reliable. + if (!params.ordered) { + spa.sendv_sndinfo.snd_flags |= SCTP_UNORDERED; + if (params.max_rtx_count >= 0 || params.max_rtx_ms == 0) { + spa.sendv_flags |= SCTP_SEND_PRINFO_VALID; + spa.sendv_prinfo.pr_policy = SCTP_PR_SCTP_RTX; + spa.sendv_prinfo.pr_value = params.max_rtx_count; + } else { + spa.sendv_flags |= SCTP_SEND_PRINFO_VALID; + spa.sendv_prinfo.pr_policy = SCTP_PR_SCTP_TTL; + spa.sendv_prinfo.pr_value = params.max_rtx_ms; + } + } + return spa; +} +} // namespace + +namespace cricket { + +// Handles global init/deinit, and mapping from usrsctp callbacks to +// SctpTransport calls. +class SctpTransport::UsrSctpWrapper { + public: + static void InitializeUsrSctp() { + RTC_LOG(LS_INFO) << __FUNCTION__; + // First argument is udp_encapsulation_port, which is not releveant for our + // AF_CONN use of sctp. + usrsctp_init(0, &UsrSctpWrapper::OnSctpOutboundPacket, &DebugSctpPrintf); + + // To turn on/off detailed SCTP debugging. You will also need to have the + // SCTP_DEBUG cpp defines flag, which can be turned on in media/BUILD.gn. + // usrsctp_sysctl_set_sctp_debug_on(SCTP_DEBUG_ALL); + + // TODO(ldixon): Consider turning this on/off. + usrsctp_sysctl_set_sctp_ecn_enable(0); + + // This is harmless, but we should find out when the library default + // changes. + int send_size = usrsctp_sysctl_get_sctp_sendspace(); + if (send_size != kSctpSendBufferSize) { + RTC_LOG(LS_ERROR) << "Got different send size than expected: " + << send_size; + } + + // TODO(ldixon): Consider turning this on/off. + // This is not needed right now (we don't do dynamic address changes): + // If SCTP Auto-ASCONF is enabled, the peer is informed automatically + // when a new address is added or removed. This feature is enabled by + // default. + // usrsctp_sysctl_set_sctp_auto_asconf(0); + + // TODO(ldixon): Consider turning this on/off. + // Add a blackhole sysctl. Setting it to 1 results in no ABORTs + // being sent in response to INITs, setting it to 2 results + // in no ABORTs being sent for received OOTB packets. + // This is similar to the TCP sysctl. + // + // See: http://lakerest.net/pipermail/sctp-coders/2012-January/009438.html + // See: http://svnweb.freebsd.org/base?view=revision&revision=229805 + // usrsctp_sysctl_set_sctp_blackhole(2); + + // Set the number of default outgoing streams. This is the number we'll + // send in the SCTP INIT message. + usrsctp_sysctl_set_sctp_nr_outgoing_streams_default(kMaxSctpStreams); + } + + static void UninitializeUsrSctp() { + RTC_LOG(LS_INFO) << __FUNCTION__; + // usrsctp_finish() may fail if it's called too soon after the transports + // are + // closed. Wait and try again until it succeeds for up to 3 seconds. + for (size_t i = 0; i < 300; ++i) { + if (usrsctp_finish() == 0) { + return; + } + + rtc::Thread::SleepMs(10); + } + RTC_LOG(LS_ERROR) << "Failed to shutdown usrsctp."; + } + + static void IncrementUsrSctpUsageCount() { + rtc::GlobalLockScope lock(&g_usrsctp_lock_); + if (!g_usrsctp_usage_count) { + InitializeUsrSctp(); + } + ++g_usrsctp_usage_count; + } + + static void DecrementUsrSctpUsageCount() { + rtc::GlobalLockScope lock(&g_usrsctp_lock_); + --g_usrsctp_usage_count; + if (!g_usrsctp_usage_count) { + UninitializeUsrSctp(); + } + } + + // This is the callback usrsctp uses when there's data to send on the network + // that has been wrapped appropriatly for the SCTP protocol. + static int OnSctpOutboundPacket(void* addr, + void* data, + size_t length, + uint8_t tos, + uint8_t set_df) { + SctpTransport* transport = static_cast<SctpTransport*>(addr); + RTC_LOG(LS_VERBOSE) << "global OnSctpOutboundPacket():" + "addr: " + << addr << "; length: " << length + << "; tos: " << rtc::ToHex(tos) + << "; set_df: " << rtc::ToHex(set_df); + + VerboseLogPacket(data, length, SCTP_DUMP_OUTBOUND); + // Note: We have to copy the data; the caller will delete it. + rtc::CopyOnWriteBuffer buf(reinterpret_cast<uint8_t*>(data), length); + // TODO(deadbeef): Why do we need an AsyncInvoke here? We're already on the + // right thread and don't need to unwind the stack. + transport->invoker_.AsyncInvoke<void>( + RTC_FROM_HERE, transport->network_thread_, + rtc::Bind(&SctpTransport::OnPacketFromSctpToNetwork, transport, buf)); + return 0; + } + + // This is the callback called from usrsctp when data has been received, after + // a packet has been interpreted and parsed by usrsctp and found to contain + // payload data. It is called by a usrsctp thread. It is assumed this function + // will free the memory used by 'data'. + static int OnSctpInboundPacket(struct socket* sock, + union sctp_sockstore addr, + void* data, + size_t length, + struct sctp_rcvinfo rcv, + int flags, + void* ulp_info) { + SctpTransport* transport = static_cast<SctpTransport*>(ulp_info); + // Post data to the transport's receiver thread (copying it). + // TODO(ldixon): Unclear if copy is needed as this method is responsible for + // memory cleanup. But this does simplify code. + const PayloadProtocolIdentifier ppid = + static_cast<PayloadProtocolIdentifier>( + rtc::NetworkToHost32(rcv.rcv_ppid)); + DataMessageType type = DMT_NONE; + if (!GetDataMediaType(ppid, &type) && !(flags & MSG_NOTIFICATION)) { + // It's neither a notification nor a recognized data packet. Drop it. + RTC_LOG(LS_ERROR) << "Received an unknown PPID " << ppid + << " on an SCTP packet. Dropping."; + free(data); + } else { + ReceiveDataParams params; + + params.sid = rcv.rcv_sid; + params.seq_num = rcv.rcv_ssn; + params.timestamp = rcv.rcv_tsn; + params.type = type; + + // Expect only continuation messages belonging to the same sid, the sctp + // stack should ensure this. + if ((transport->partial_incoming_message_.size() != 0) && + (rcv.rcv_sid != transport->partial_params_.sid)) { + // A message with a new sid, but haven't seen the EOR for the + // previous message. Deliver the previous partial message to avoid + // merging messages from different sid's. + transport->invoker_.AsyncInvoke<void>( + RTC_FROM_HERE, transport->network_thread_, + rtc::Bind(&SctpTransport::OnInboundPacketFromSctpToTransport, + transport, transport->partial_incoming_message_, + transport->partial_params_, transport->partial_flags_)); + + transport->partial_incoming_message_.Clear(); + } + + transport->partial_incoming_message_.AppendData( + reinterpret_cast<uint8_t*>(data), length); + transport->partial_params_ = params; + transport->partial_flags_ = flags; + + free(data); + + // Merge partial messages until they exceed the maximum send buffer size. + // This enables messages from a single send to be delivered in a single + // callback. Larger messages (originating from other implementations) will + // still be delivered in chunks. + if (!(flags & MSG_EOR) && + (transport->partial_incoming_message_.size() < kSctpSendBufferSize)) { + return 1; + } + + if (!(flags & MSG_EOR)) { + // TODO(bugs.webrtc.org/7774): We currently chunk messages if they are + // >= kSctpSendBufferSize. The better thing to do here is buffer up to + // the size negotiated in the SDP, and if a larger message is received + // close the channel and report the error. See discussion in the bug. + RTC_LOG(LS_WARNING) << "Chunking SCTP message without the EOR bit set."; + } + + // The ownership of the packet transfers to |invoker_|. Using + // CopyOnWriteBuffer is the most convenient way to do this. + transport->invoker_.AsyncInvoke<void>( + RTC_FROM_HERE, transport->network_thread_, + rtc::Bind(&SctpTransport::OnInboundPacketFromSctpToTransport, + transport, transport->partial_incoming_message_, params, + flags)); + + transport->partial_incoming_message_.Clear(); + } + return 1; + } + + static SctpTransport* GetTransportFromSocket(struct socket* sock) { + struct sockaddr* addrs = nullptr; + int naddrs = usrsctp_getladdrs(sock, 0, &addrs); + if (naddrs <= 0 || addrs[0].sa_family != AF_CONN) { + return nullptr; + } + // usrsctp_getladdrs() returns the addresses bound to this socket, which + // contains the SctpTransport* as sconn_addr. Read the pointer, + // then free the list of addresses once we have the pointer. We only open + // AF_CONN sockets, and they should all have the sconn_addr set to the + // pointer that created them, so [0] is as good as any other. + struct sockaddr_conn* sconn = + reinterpret_cast<struct sockaddr_conn*>(&addrs[0]); + SctpTransport* transport = + reinterpret_cast<SctpTransport*>(sconn->sconn_addr); + usrsctp_freeladdrs(addrs); + + return transport; + } + + static int SendThresholdCallback(struct socket* sock, uint32_t sb_free) { + // Fired on our I/O thread. SctpTransport::OnPacketReceived() gets + // a packet containing acknowledgments, which goes into usrsctp_conninput, + // and then back here. + SctpTransport* transport = GetTransportFromSocket(sock); + if (!transport) { + RTC_LOG(LS_ERROR) + << "SendThresholdCallback: Failed to get transport for socket " + << sock; + return 0; + } + transport->OnSendThresholdCallback(); + return 0; + } +}; + +SctpTransport::SctpTransport(rtc::Thread* network_thread, + rtc::PacketTransportInternal* transport) + : network_thread_(network_thread), + transport_(transport), + was_ever_writable_(transport ? transport->writable() : false) { + RTC_DCHECK(network_thread_); + RTC_DCHECK_RUN_ON(network_thread_); + ConnectTransportSignals(); +} + +SctpTransport::~SctpTransport() { + // Close abruptly; no reset procedure. + CloseSctpSocket(); + // It's not strictly necessary to reset these fields to nullptr, + // but having these fields set to nullptr is a clear indication that + // object was destructed. There was a bug in usrsctp when it + // invoked OnSctpOutboundPacket callback for destructed SctpTransport, + // which caused obscure SIGSEGV on access to these fields, + // having this fields set to nullptr will make it easier to understand + // that SctpTransport was destructed and "use-after-free" bug happen. + // SIGSEGV error triggered on dereference these pointers will also + // be easier to understand due to 0x0 address. All of this assumes + // that ASAN is not enabled to detect "use-after-free", which is + // currently default configuration. + network_thread_ = nullptr; + transport_ = nullptr; +} + +void SctpTransport::SetDtlsTransport(rtc::PacketTransportInternal* transport) { + RTC_DCHECK_RUN_ON(network_thread_); + DisconnectTransportSignals(); + transport_ = transport; + ConnectTransportSignals(); + if (!was_ever_writable_ && transport && transport->writable()) { + was_ever_writable_ = true; + // New transport is writable, now we can start the SCTP connection if Start + // was called already. + if (started_) { + RTC_DCHECK(!sock_); + Connect(); + } + } +} + +bool SctpTransport::Start(int local_sctp_port, + int remote_sctp_port, + int max_message_size) { + RTC_DCHECK_RUN_ON(network_thread_); + if (local_sctp_port == -1) { + local_sctp_port = kSctpDefaultPort; + } + if (remote_sctp_port == -1) { + remote_sctp_port = kSctpDefaultPort; + } + if (max_message_size > kSctpSendBufferSize) { + RTC_LOG(LS_ERROR) << "Max message size of " << max_message_size + << " is larger than send bufffer size " + << kSctpSendBufferSize; + return false; + } + if (max_message_size < 1) { + RTC_LOG(LS_ERROR) << "Max message size of " << max_message_size + << " is too small"; + return false; + } + // We allow changing max_message_size with a second Start() call, + // but not changing the port numbers. + max_message_size_ = max_message_size; + if (started_) { + if (local_sctp_port != local_port_ || remote_sctp_port != remote_port_) { + RTC_LOG(LS_ERROR) + << "Can't change SCTP port after SCTP association formed."; + return false; + } + return true; + } + local_port_ = local_sctp_port; + remote_port_ = remote_sctp_port; + started_ = true; + RTC_DCHECK(!sock_); + // Only try to connect if the DTLS transport has been writable before + // (indicating that the DTLS handshake is complete). + if (was_ever_writable_) { + return Connect(); + } + return true; +} + +bool SctpTransport::OpenStream(int sid) { + RTC_DCHECK_RUN_ON(network_thread_); + if (sid > kMaxSctpSid) { + RTC_LOG(LS_WARNING) << debug_name_ + << "->OpenStream(...): " + "Not adding data stream " + "with sid=" + << sid << " because sid is too high."; + return false; + } + auto it = stream_status_by_sid_.find(sid); + if (it == stream_status_by_sid_.end()) { + stream_status_by_sid_[sid] = StreamStatus(); + return true; + } + if (it->second.is_open()) { + RTC_LOG(LS_WARNING) << debug_name_ + << "->OpenStream(...): " + "Not adding data stream " + "with sid=" + << sid << " because stream is already open."; + return false; + } else { + RTC_LOG(LS_WARNING) << debug_name_ + << "->OpenStream(...): " + "Not adding data stream " + " with sid=" + << sid << " because stream is still closing."; + return false; + } +} + +bool SctpTransport::ResetStream(int sid) { + RTC_DCHECK_RUN_ON(network_thread_); + + auto it = stream_status_by_sid_.find(sid); + if (it == stream_status_by_sid_.end() || !it->second.is_open()) { + RTC_LOG(LS_WARNING) << debug_name_ << "->ResetStream(" << sid + << "): stream not open."; + return false; + } + + RTC_LOG(LS_VERBOSE) << debug_name_ << "->ResetStream(" << sid + << "): " + "Queuing RE-CONFIG chunk."; + it->second.closure_initiated = true; + + // Signal our stream-reset logic that it should try to send now, if it can. + SendQueuedStreamResets(); + + // The stream will actually get removed when we get the acknowledgment. + return true; +} + +bool SctpTransport::SendData(const SendDataParams& params, + const rtc::CopyOnWriteBuffer& payload, + SendDataResult* result) { + RTC_DCHECK_RUN_ON(network_thread_); + + if (partial_outgoing_message_.has_value()) { + if (result) { + *result = SDR_BLOCK; + } + // Ready to send should get set only when SendData() call gets blocked. + ready_to_send_data_ = false; + return false; + } + size_t payload_size = payload.size(); + OutgoingMessage message(payload, params); + SendDataResult send_message_result = SendMessageInternal(&message); + if (result) { + *result = send_message_result; + } + if (payload_size == message.size()) { + // Nothing was sent. + return false; + } + // If any data is sent, we accept the message. In the case that data was + // partially accepted by the sctp library, the remaining is buffered. This + // ensures the client does not resend the message. + RTC_DCHECK_LT(message.size(), payload_size); + if (message.size() > 0) { + RTC_DCHECK(!partial_outgoing_message_.has_value()); + RTC_DLOG(LS_VERBOSE) << "Partially sent message. Buffering the remaining" + << message.size() << "/" << payload_size << " bytes."; + + partial_outgoing_message_.emplace(message); + } + return true; +} + +SendDataResult SctpTransport::SendMessageInternal(OutgoingMessage* message) { + RTC_DCHECK_RUN_ON(network_thread_); + if (!sock_) { + RTC_LOG(LS_WARNING) << debug_name_ + << "->SendMessageInternal(...): " + "Not sending packet with sid=" + << message->send_params().sid + << " len=" << message->size() << " before Start()."; + return SDR_ERROR; + } + if (message->send_params().type != DMT_CONTROL) { + auto it = stream_status_by_sid_.find(message->send_params().sid); + if (it == stream_status_by_sid_.end() || !it->second.is_open()) { + RTC_LOG(LS_WARNING) + << debug_name_ + << "->SendMessageInternal(...): " + "Not sending data because sid is unknown or closing: " + << message->send_params().sid; + return SDR_ERROR; + } + } + if (message->size() > static_cast<size_t>(max_message_size_)) { + RTC_LOG(LS_ERROR) << "Attempting to send message of size " + << message->size() << " which is larger than limit " + << max_message_size_; + return SDR_ERROR; + } + + // Send data using SCTP. + sctp_sendv_spa spa = CreateSctpSendParams(message->send_params()); + // Note: this send call is not atomic because the EOR bit is set. This means + // that usrsctp can partially accept this message and it is our duty to buffer + // the rest. + ssize_t send_res = usrsctp_sendv( + sock_, message->data(), message->size(), NULL, 0, &spa, + rtc::checked_cast<socklen_t>(sizeof(spa)), SCTP_SENDV_SPA, 0); + if (send_res < 0) { + if (errno == SCTP_EWOULDBLOCK) { + ready_to_send_data_ = false; + RTC_LOG(LS_INFO) << debug_name_ + << "->SendMessageInternal(...): EWOULDBLOCK returned"; + return SDR_BLOCK; + } + + RTC_LOG_ERRNO(LS_ERROR) << "ERROR:" << debug_name_ + << "->SendMessageInternal(...): " + " usrsctp_sendv: "; + return SDR_ERROR; + } + + size_t amount_sent = static_cast<size_t>(send_res); + RTC_DCHECK_LE(amount_sent, message->size()); + message->Advance(amount_sent); + // Only way out now is success. + return SDR_SUCCESS; +} + +bool SctpTransport::ReadyToSendData() { + RTC_DCHECK_RUN_ON(network_thread_); + return ready_to_send_data_; +} + +void SctpTransport::ConnectTransportSignals() { + RTC_DCHECK_RUN_ON(network_thread_); + if (!transport_) { + return; + } + transport_->SignalWritableState.connect(this, + &SctpTransport::OnWritableState); + transport_->SignalReadPacket.connect(this, &SctpTransport::OnPacketRead); + transport_->SignalClosed.connect(this, &SctpTransport::OnClosed); +} + +void SctpTransport::DisconnectTransportSignals() { + RTC_DCHECK_RUN_ON(network_thread_); + if (!transport_) { + return; + } + transport_->SignalWritableState.disconnect(this); + transport_->SignalReadPacket.disconnect(this); + transport_->SignalClosed.disconnect(this); +} + +bool SctpTransport::Connect() { + RTC_DCHECK_RUN_ON(network_thread_); + RTC_LOG(LS_VERBOSE) << debug_name_ << "->Connect()."; + + // If we already have a socket connection (which shouldn't ever happen), just + // return. + RTC_DCHECK(!sock_); + if (sock_) { + RTC_LOG(LS_ERROR) << debug_name_ + << "->Connect(): Ignored as socket " + "is already established."; + return true; + } + + // If no socket (it was closed) try to start it again. This can happen when + // the socket we are connecting to closes, does an sctp shutdown handshake, + // or behaves unexpectedly causing us to perform a CloseSctpSocket. + if (!OpenSctpSocket()) { + return false; + } + + // Note: conversion from int to uint16_t happens on assignment. + sockaddr_conn local_sconn = GetSctpSockAddr(local_port_); + if (usrsctp_bind(sock_, reinterpret_cast<sockaddr*>(&local_sconn), + sizeof(local_sconn)) < 0) { + RTC_LOG_ERRNO(LS_ERROR) + << debug_name_ << "->Connect(): " << ("Failed usrsctp_bind"); + CloseSctpSocket(); + return false; + } + + // Note: conversion from int to uint16_t happens on assignment. + sockaddr_conn remote_sconn = GetSctpSockAddr(remote_port_); + int connect_result = usrsctp_connect( + sock_, reinterpret_cast<sockaddr*>(&remote_sconn), sizeof(remote_sconn)); + if (connect_result < 0 && errno != SCTP_EINPROGRESS) { + RTC_LOG_ERRNO(LS_ERROR) << debug_name_ + << "->Connect(): " + "Failed usrsctp_connect. got errno=" + << errno << ", but wanted " << SCTP_EINPROGRESS; + CloseSctpSocket(); + return false; + } + // Set the MTU and disable MTU discovery. + // We can only do this after usrsctp_connect or it has no effect. + sctp_paddrparams params = {}; + memcpy(¶ms.spp_address, &remote_sconn, sizeof(remote_sconn)); + params.spp_flags = SPP_PMTUD_DISABLE; + // The MTU value provided specifies the space available for chunks in the + // packet, so we subtract the SCTP header size. + params.spp_pathmtu = kSctpMtu - sizeof(struct sctp_common_header); + if (usrsctp_setsockopt(sock_, IPPROTO_SCTP, SCTP_PEER_ADDR_PARAMS, ¶ms, + sizeof(params))) { + RTC_LOG_ERRNO(LS_ERROR) << debug_name_ + << "->Connect(): " + "Failed to set SCTP_PEER_ADDR_PARAMS."; + } + // Since this is a fresh SCTP association, we'll always start out with empty + // queues, so "ReadyToSendData" should be true. + SetReadyToSendData(); + return true; +} + +bool SctpTransport::OpenSctpSocket() { + RTC_DCHECK_RUN_ON(network_thread_); + if (sock_) { + RTC_LOG(LS_WARNING) << debug_name_ + << "->OpenSctpSocket(): " + "Ignoring attempt to re-create existing socket."; + return false; + } + + UsrSctpWrapper::IncrementUsrSctpUsageCount(); + + // If kSctpSendBufferSize isn't reflective of reality, we log an error, but we + // still have to do something reasonable here. Look up what the buffer's real + // size is and set our threshold to something reasonable. + static const int kSendThreshold = usrsctp_sysctl_get_sctp_sendspace() / 2; + + sock_ = usrsctp_socket( + AF_CONN, SOCK_STREAM, IPPROTO_SCTP, &UsrSctpWrapper::OnSctpInboundPacket, + &UsrSctpWrapper::SendThresholdCallback, kSendThreshold, this); + if (!sock_) { + RTC_LOG_ERRNO(LS_ERROR) << debug_name_ + << "->OpenSctpSocket(): " + "Failed to create SCTP socket."; + UsrSctpWrapper::DecrementUsrSctpUsageCount(); + return false; + } + + if (!ConfigureSctpSocket()) { + usrsctp_close(sock_); + sock_ = nullptr; + UsrSctpWrapper::DecrementUsrSctpUsageCount(); + return false; + } + // Register this class as an address for usrsctp. This is used by SCTP to + // direct the packets received (by the created socket) to this class. + usrsctp_register_address(this); + return true; +} + +bool SctpTransport::ConfigureSctpSocket() { + RTC_DCHECK_RUN_ON(network_thread_); + RTC_DCHECK(sock_); + // Make the socket non-blocking. Connect, close, shutdown etc will not block + // the thread waiting for the socket operation to complete. + if (usrsctp_set_non_blocking(sock_, 1) < 0) { + RTC_LOG_ERRNO(LS_ERROR) << debug_name_ + << "->ConfigureSctpSocket(): " + "Failed to set SCTP to non blocking."; + return false; + } + + // This ensures that the usrsctp close call deletes the association. This + // prevents usrsctp from calling OnSctpOutboundPacket with references to + // this class as the address. + linger linger_opt; + linger_opt.l_onoff = 1; + linger_opt.l_linger = 0; + if (usrsctp_setsockopt(sock_, SOL_SOCKET, SO_LINGER, &linger_opt, + sizeof(linger_opt))) { + RTC_LOG_ERRNO(LS_ERROR) << debug_name_ + << "->ConfigureSctpSocket(): " + "Failed to set SO_LINGER."; + return false; + } + + // Enable stream ID resets. + struct sctp_assoc_value stream_rst; + stream_rst.assoc_id = SCTP_ALL_ASSOC; + stream_rst.assoc_value = 1; + if (usrsctp_setsockopt(sock_, IPPROTO_SCTP, SCTP_ENABLE_STREAM_RESET, + &stream_rst, sizeof(stream_rst))) { + RTC_LOG_ERRNO(LS_ERROR) << debug_name_ + << "->ConfigureSctpSocket(): " + "Failed to set SCTP_ENABLE_STREAM_RESET."; + return false; + } + + // Nagle. + uint32_t nodelay = 1; + if (usrsctp_setsockopt(sock_, IPPROTO_SCTP, SCTP_NODELAY, &nodelay, + sizeof(nodelay))) { + RTC_LOG_ERRNO(LS_ERROR) << debug_name_ + << "->ConfigureSctpSocket(): " + "Failed to set SCTP_NODELAY."; + return false; + } + + // Explicit EOR. + uint32_t eor = 1; + if (usrsctp_setsockopt(sock_, IPPROTO_SCTP, SCTP_EXPLICIT_EOR, &eor, + sizeof(eor))) { + RTC_LOG_ERRNO(LS_ERROR) << debug_name_ + << "->ConfigureSctpSocket(): " + "Failed to set SCTP_EXPLICIT_EOR."; + return false; + } + + // Subscribe to SCTP event notifications. + int event_types[] = {SCTP_ASSOC_CHANGE, SCTP_PEER_ADDR_CHANGE, + SCTP_SEND_FAILED_EVENT, SCTP_SENDER_DRY_EVENT, + SCTP_STREAM_RESET_EVENT}; + struct sctp_event event = {0}; + event.se_assoc_id = SCTP_ALL_ASSOC; + event.se_on = 1; + for (size_t i = 0; i < arraysize(event_types); i++) { + event.se_type = event_types[i]; + if (usrsctp_setsockopt(sock_, IPPROTO_SCTP, SCTP_EVENT, &event, + sizeof(event)) < 0) { + RTC_LOG_ERRNO(LS_ERROR) << debug_name_ + << "->ConfigureSctpSocket(): " + "Failed to set SCTP_EVENT type: " + << event.se_type; + return false; + } + } + return true; +} + +void SctpTransport::CloseSctpSocket() { + RTC_DCHECK_RUN_ON(network_thread_); + if (sock_) { + // We assume that SO_LINGER option is set to close the association when + // close is called. This means that any pending packets in usrsctp will be + // discarded instead of being sent. + usrsctp_close(sock_); + sock_ = nullptr; + usrsctp_deregister_address(this); + UsrSctpWrapper::DecrementUsrSctpUsageCount(); + ready_to_send_data_ = false; + } +} + +bool SctpTransport::SendQueuedStreamResets() { + RTC_DCHECK_RUN_ON(network_thread_); + + // Figure out how many streams need to be reset. We need to do this so we can + // allocate the right amount of memory for the sctp_reset_streams structure. + size_t num_streams = absl::c_count_if( + stream_status_by_sid_, + [](const std::map<uint32_t, StreamStatus>::value_type& stream) { + return stream.second.need_outgoing_reset(); + }); + if (num_streams == 0) { + // Nothing to reset. + return true; + } + + RTC_LOG(LS_VERBOSE) << "SendQueuedStreamResets[" << debug_name_ + << "]: Resetting " << num_streams << " outgoing streams."; + + const size_t num_bytes = + sizeof(struct sctp_reset_streams) + (num_streams * sizeof(uint16_t)); + std::vector<uint8_t> reset_stream_buf(num_bytes, 0); + struct sctp_reset_streams* resetp = + reinterpret_cast<sctp_reset_streams*>(&reset_stream_buf[0]); + resetp->srs_assoc_id = SCTP_ALL_ASSOC; + resetp->srs_flags = SCTP_STREAM_RESET_OUTGOING; + resetp->srs_number_streams = rtc::checked_cast<uint16_t>(num_streams); + int result_idx = 0; + + for (const std::map<uint32_t, StreamStatus>::value_type& stream : + stream_status_by_sid_) { + if (!stream.second.need_outgoing_reset()) { + continue; + } + resetp->srs_stream_list[result_idx++] = stream.first; + } + + int ret = + usrsctp_setsockopt(sock_, IPPROTO_SCTP, SCTP_RESET_STREAMS, resetp, + rtc::checked_cast<socklen_t>(reset_stream_buf.size())); + if (ret < 0) { + // Note that usrsctp only lets us have one reset in progress at a time + // (even though multiple streams can be reset at once). If this happens, + // SendQueuedStreamResets will end up called after the current in-progress + // reset finishes, in OnStreamResetEvent. + RTC_LOG_ERRNO(LS_WARNING) << debug_name_ + << "->SendQueuedStreamResets(): " + "Failed to send a stream reset for " + << num_streams << " streams"; + return false; + } + + // Since the usrsctp call completed successfully, update our stream status + // map to note that we started the outgoing reset. + for (auto it = stream_status_by_sid_.begin(); + it != stream_status_by_sid_.end(); ++it) { + if (it->second.need_outgoing_reset()) { + it->second.outgoing_reset_initiated = true; + } + } + return true; +} + +void SctpTransport::SetReadyToSendData() { + RTC_DCHECK_RUN_ON(network_thread_); + if (!ready_to_send_data_) { + ready_to_send_data_ = true; + SignalReadyToSendData(); + } +} + +bool SctpTransport::SendBufferedMessage() { + RTC_DCHECK_RUN_ON(network_thread_); + RTC_DCHECK(partial_outgoing_message_.has_value()); + RTC_DLOG(LS_VERBOSE) << "Sending partially buffered message of size " + << partial_outgoing_message_->size() << "."; + + SendMessageInternal(&partial_outgoing_message_.value()); + if (partial_outgoing_message_->size() > 0) { + // Still need to finish sending the message. + return false; + } + RTC_DCHECK_EQ(0u, partial_outgoing_message_->size()); + partial_outgoing_message_.reset(); + return true; +} + +void SctpTransport::OnWritableState(rtc::PacketTransportInternal* transport) { + RTC_DCHECK_RUN_ON(network_thread_); + RTC_DCHECK_EQ(transport_, transport); + if (!was_ever_writable_ && transport->writable()) { + was_ever_writable_ = true; + if (started_) { + Connect(); + } + } +} + +// Called by network interface when a packet has been received. +void SctpTransport::OnPacketRead(rtc::PacketTransportInternal* transport, + const char* data, + size_t len, + const int64_t& /* packet_time_us */, + int flags) { + RTC_DCHECK_RUN_ON(network_thread_); + RTC_DCHECK_EQ(transport_, transport); + TRACE_EVENT0("webrtc", "SctpTransport::OnPacketRead"); + + if (flags & PF_SRTP_BYPASS) { + // We are only interested in SCTP packets. + return; + } + + RTC_LOG(LS_VERBOSE) << debug_name_ + << "->OnPacketRead(...): " + " length=" + << len << ", started: " << started_; + // Only give receiving packets to usrsctp after if connected. This enables two + // peers to each make a connect call, but for them not to receive an INIT + // packet before they have called connect; least the last receiver of the INIT + // packet will have called connect, and a connection will be established. + if (sock_) { + // Pass received packet to SCTP stack. Once processed by usrsctp, the data + // will be will be given to the global OnSctpInboundData, and then, + // marshalled by the AsyncInvoker. + VerboseLogPacket(data, len, SCTP_DUMP_INBOUND); + usrsctp_conninput(this, data, len, 0); + } else { + // TODO(ldixon): Consider caching the packet for very slightly better + // reliability. + } +} + +void SctpTransport::OnClosed(rtc::PacketTransportInternal* transport) { + SignalClosedAbruptly(); +} + +void SctpTransport::OnSendThresholdCallback() { + RTC_DCHECK_RUN_ON(network_thread_); + if (partial_outgoing_message_.has_value()) { + if (!SendBufferedMessage()) { + // Did not finish sending the buffered message. + return; + } + } + SetReadyToSendData(); +} + +sockaddr_conn SctpTransport::GetSctpSockAddr(int port) { + sockaddr_conn sconn = {0}; + sconn.sconn_family = AF_CONN; +#ifdef HAVE_SCONN_LEN + sconn.sconn_len = sizeof(sockaddr_conn); +#endif + // Note: conversion from int to uint16_t happens here. + sconn.sconn_port = rtc::HostToNetwork16(port); + sconn.sconn_addr = this; + return sconn; +} + +void SctpTransport::OnPacketFromSctpToNetwork( + const rtc::CopyOnWriteBuffer& buffer) { + RTC_DCHECK_RUN_ON(network_thread_); + if (buffer.size() > (kSctpMtu)) { + RTC_LOG(LS_ERROR) << debug_name_ + << "->OnPacketFromSctpToNetwork(...): " + "SCTP seems to have made a packet that is bigger " + "than its official MTU: " + << buffer.size() << " vs max of " << kSctpMtu; + } + TRACE_EVENT0("webrtc", "SctpTransport::OnPacketFromSctpToNetwork"); + + // Don't create noise by trying to send a packet when the DTLS transport isn't + // even writable. + if (!transport_ || !transport_->writable()) { + return; + } + + // Bon voyage. + transport_->SendPacket(buffer.data<char>(), buffer.size(), + rtc::PacketOptions(), PF_NORMAL); +} + +void SctpTransport::OnInboundPacketFromSctpToTransport( + const rtc::CopyOnWriteBuffer& buffer, + ReceiveDataParams params, + int flags) { + RTC_DCHECK_RUN_ON(network_thread_); + RTC_LOG(LS_VERBOSE) << debug_name_ + << "->OnInboundPacketFromSctpToTransport(...): " + "Received SCTP data:" + " sid=" + << params.sid + << " notification: " << (flags & MSG_NOTIFICATION) + << " length=" << buffer.size(); + // Sending a packet with data == NULL (no data) is SCTPs "close the + // connection" message. This sets sock_ = NULL; + if (!buffer.size() || !buffer.data()) { + RTC_LOG(LS_INFO) << debug_name_ + << "->OnInboundPacketFromSctpToTransport(...): " + "No data, closing."; + return; + } + if (flags & MSG_NOTIFICATION) { + OnNotificationFromSctp(buffer); + } else { + OnDataFromSctpToTransport(params, buffer); + } +} + +void SctpTransport::OnDataFromSctpToTransport( + const ReceiveDataParams& params, + const rtc::CopyOnWriteBuffer& buffer) { + RTC_DCHECK_RUN_ON(network_thread_); + RTC_LOG(LS_VERBOSE) << debug_name_ + << "->OnDataFromSctpToTransport(...): " + "Posting with length: " + << buffer.size() << " on stream " << params.sid; + // Reports all received messages to upper layers, no matter whether the sid + // is known. + SignalDataReceived(params, buffer); +} + +void SctpTransport::OnNotificationFromSctp( + const rtc::CopyOnWriteBuffer& buffer) { + RTC_DCHECK_RUN_ON(network_thread_); + const sctp_notification& notification = + reinterpret_cast<const sctp_notification&>(*buffer.data()); + RTC_DCHECK(notification.sn_header.sn_length == buffer.size()); + + // TODO(ldixon): handle notifications appropriately. + switch (notification.sn_header.sn_type) { + case SCTP_ASSOC_CHANGE: + RTC_LOG(LS_VERBOSE) << "SCTP_ASSOC_CHANGE"; + OnNotificationAssocChange(notification.sn_assoc_change); + break; + case SCTP_REMOTE_ERROR: + RTC_LOG(LS_INFO) << "SCTP_REMOTE_ERROR"; + break; + case SCTP_SHUTDOWN_EVENT: + RTC_LOG(LS_INFO) << "SCTP_SHUTDOWN_EVENT"; + break; + case SCTP_ADAPTATION_INDICATION: + RTC_LOG(LS_INFO) << "SCTP_ADAPTATION_INDICATION"; + break; + case SCTP_PARTIAL_DELIVERY_EVENT: + RTC_LOG(LS_INFO) << "SCTP_PARTIAL_DELIVERY_EVENT"; + break; + case SCTP_AUTHENTICATION_EVENT: + RTC_LOG(LS_INFO) << "SCTP_AUTHENTICATION_EVENT"; + break; + case SCTP_SENDER_DRY_EVENT: + RTC_LOG(LS_VERBOSE) << "SCTP_SENDER_DRY_EVENT"; + SetReadyToSendData(); + break; + // TODO(ldixon): Unblock after congestion. + case SCTP_NOTIFICATIONS_STOPPED_EVENT: + RTC_LOG(LS_INFO) << "SCTP_NOTIFICATIONS_STOPPED_EVENT"; + break; + case SCTP_SEND_FAILED_EVENT: { + const struct sctp_send_failed_event& ssfe = + notification.sn_send_failed_event; + RTC_LOG(LS_WARNING) << "SCTP_SEND_FAILED_EVENT: message with" + " PPID = " + << rtc::NetworkToHost32(ssfe.ssfe_info.snd_ppid) + << " SID = " << ssfe.ssfe_info.snd_sid + << " flags = " << rtc::ToHex(ssfe.ssfe_info.snd_flags) + << " failed to sent due to error = " + << rtc::ToHex(ssfe.ssfe_error); + break; + } + case SCTP_STREAM_RESET_EVENT: + OnStreamResetEvent(¬ification.sn_strreset_event); + break; + case SCTP_ASSOC_RESET_EVENT: + RTC_LOG(LS_INFO) << "SCTP_ASSOC_RESET_EVENT"; + break; + case SCTP_STREAM_CHANGE_EVENT: + RTC_LOG(LS_INFO) << "SCTP_STREAM_CHANGE_EVENT"; + // An acknowledgment we get after our stream resets have gone through, + // if they've failed. We log the message, but don't react -- we don't + // keep around the last-transmitted set of SSIDs we wanted to close for + // error recovery. It doesn't seem likely to occur, and if so, likely + // harmless within the lifetime of a single SCTP association. + break; + case SCTP_PEER_ADDR_CHANGE: + RTC_LOG(LS_INFO) << "SCTP_PEER_ADDR_CHANGE"; + break; + default: + RTC_LOG(LS_WARNING) << "Unknown SCTP event: " + << notification.sn_header.sn_type; + break; + } +} + +void SctpTransport::OnNotificationAssocChange(const sctp_assoc_change& change) { + RTC_DCHECK_RUN_ON(network_thread_); + switch (change.sac_state) { + case SCTP_COMM_UP: + RTC_LOG(LS_VERBOSE) << "Association change SCTP_COMM_UP, stream # is " + << change.sac_outbound_streams << " outbound, " + << change.sac_inbound_streams << " inbound."; + max_outbound_streams_ = change.sac_outbound_streams; + max_inbound_streams_ = change.sac_inbound_streams; + SignalAssociationChangeCommunicationUp(); + break; + case SCTP_COMM_LOST: + RTC_LOG(LS_INFO) << "Association change SCTP_COMM_LOST"; + break; + case SCTP_RESTART: + RTC_LOG(LS_INFO) << "Association change SCTP_RESTART"; + break; + case SCTP_SHUTDOWN_COMP: + RTC_LOG(LS_INFO) << "Association change SCTP_SHUTDOWN_COMP"; + break; + case SCTP_CANT_STR_ASSOC: + RTC_LOG(LS_INFO) << "Association change SCTP_CANT_STR_ASSOC"; + break; + default: + RTC_LOG(LS_INFO) << "Association change UNKNOWN"; + break; + } +} + +void SctpTransport::OnStreamResetEvent( + const struct sctp_stream_reset_event* evt) { + RTC_DCHECK_RUN_ON(network_thread_); + + // This callback indicates that a reset is complete for incoming and/or + // outgoing streams. The reset may have been initiated by us or the remote + // side. + const int num_sids = (evt->strreset_length - sizeof(*evt)) / + sizeof(evt->strreset_stream_list[0]); + + if (evt->strreset_flags & SCTP_STREAM_RESET_FAILED) { + // OK, just try sending any previously sent stream resets again. The stream + // IDs sent over when the RESET_FIALED flag is set seem to be garbage + // values. Ignore them. + for (std::map<uint32_t, StreamStatus>::value_type& stream : + stream_status_by_sid_) { + stream.second.outgoing_reset_initiated = false; + } + SendQueuedStreamResets(); + // TODO(deadbeef): If this happens, the entire SCTP association is in quite + // crippled state. The SCTP session should be dismantled, and the WebRTC + // connectivity errored because is clear that the distant party is not + // playing ball: malforms the transported data. + return; + } + + // Loop over the received events and properly update the StreamStatus map. + for (int i = 0; i < num_sids; i++) { + const uint32_t sid = evt->strreset_stream_list[i]; + auto it = stream_status_by_sid_.find(sid); + if (it == stream_status_by_sid_.end()) { + // This stream is unknown. Sometimes this can be from a + // RESET_FAILED-related retransmit. + RTC_LOG(LS_VERBOSE) << "SCTP_STREAM_RESET_EVENT(" << debug_name_ + << "): Unknown sid " << sid; + continue; + } + StreamStatus& status = it->second; + + if (evt->strreset_flags & SCTP_STREAM_RESET_INCOMING_SSN) { + RTC_LOG(LS_VERBOSE) << "SCTP_STREAM_RESET_INCOMING_SSN(" << debug_name_ + << "): sid " << sid; + status.incoming_reset_complete = true; + // If we receive an incoming stream reset and we haven't started the + // closing procedure ourselves, this means the remote side started the + // closing procedure; fire a signal so that the relevant data channel + // can change to "closing" (we still need to reset the outgoing stream + // before it changes to "closed"). + if (!status.closure_initiated) { + SignalClosingProcedureStartedRemotely(sid); + } + } + if (evt->strreset_flags & SCTP_STREAM_RESET_OUTGOING_SSN) { + RTC_LOG(LS_VERBOSE) << "SCTP_STREAM_RESET_OUTGOING_SSN(" << debug_name_ + << "): sid " << sid; + status.outgoing_reset_complete = true; + } + + // If this reset completes the closing procedure, remove the stream from + // our map so we can consider it closed, and fire a signal such that the + // relevant DataChannel will change its state to "closed" and its ID can be + // re-used. + if (status.reset_complete()) { + stream_status_by_sid_.erase(it); + SignalClosingProcedureComplete(sid); + } + } + + // Always try to send any queued resets because this call indicates that the + // last outgoing or incoming reset has made some progress. + SendQueuedStreamResets(); +} + +} // namespace cricket diff --git a/media/sctp/sctp_transport.h b/media/sctp/sctp_transport.h new file mode 100644 index 0000000000..d346cfc71f --- /dev/null +++ b/media/sctp/sctp_transport.h @@ -0,0 +1,290 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MEDIA_SCTP_SCTP_TRANSPORT_H_ +#define MEDIA_SCTP_SCTP_TRANSPORT_H_ + +#include <errno.h> + +#include <map> +#include <memory> +#include <set> +#include <string> +#include <vector> + +#include "absl/types/optional.h" +#include "rtc_base/async_invoker.h" +#include "rtc_base/buffer.h" +#include "rtc_base/constructor_magic.h" +#include "rtc_base/copy_on_write_buffer.h" +#include "rtc_base/third_party/sigslot/sigslot.h" +#include "rtc_base/thread.h" +// For SendDataParams/ReceiveDataParams. +#include "media/base/media_channel.h" +#include "media/sctp/sctp_transport_internal.h" + +// Defined by "usrsctplib/usrsctp.h" +struct sockaddr_conn; +struct sctp_assoc_change; +struct sctp_stream_reset_event; +struct sctp_sendv_spa; + +// Defined by <sys/socket.h> +struct socket; +namespace cricket { + +// Holds data to be passed on to a transport. +struct SctpInboundPacket; + +// From transport calls, data flows like this: +// [network thread (although it can in princple be another thread)] +// 1. SctpTransport::SendData(data) +// 2. usrsctp_sendv(data) +// [network thread returns; sctp thread then calls the following] +// 3. OnSctpOutboundPacket(wrapped_data) +// [sctp thread returns having async invoked on the network thread] +// 4. SctpTransport::OnPacketFromSctpToNetwork(wrapped_data) +// 5. DtlsTransport::SendPacket(wrapped_data) +// 6. ... across network ... a packet is sent back ... +// 7. SctpTransport::OnPacketReceived(wrapped_data) +// 8. usrsctp_conninput(wrapped_data) +// [network thread returns; sctp thread then calls the following] +// 9. OnSctpInboundData(data) +// [sctp thread returns having async invoked on the network thread] +// 10. SctpTransport::OnInboundPacketFromSctpToTransport(inboundpacket) +// 11. SctpTransport::OnDataFromSctpToTransport(data) +// 12. SctpTransport::SignalDataReceived(data) +// [from the same thread, methods registered/connected to +// SctpTransport are called with the recieved data] +class SctpTransport : public SctpTransportInternal, + public sigslot::has_slots<> { + public: + // |network_thread| is where packets will be processed and callbacks from + // this transport will be posted, and is the only thread on which public + // methods can be called. + // |transport| is not required (can be null). + SctpTransport(rtc::Thread* network_thread, + rtc::PacketTransportInternal* transport); + ~SctpTransport() override; + + // SctpTransportInternal overrides (see sctptransportinternal.h for comments). + void SetDtlsTransport(rtc::PacketTransportInternal* transport) override; + bool Start(int local_port, int remote_port, int max_message_size) override; + bool OpenStream(int sid) override; + bool ResetStream(int sid) override; + bool SendData(const SendDataParams& params, + const rtc::CopyOnWriteBuffer& payload, + SendDataResult* result = nullptr) override; + bool ReadyToSendData() override; + int max_message_size() const override { return max_message_size_; } + absl::optional<int> max_outbound_streams() const override { + return max_outbound_streams_; + } + absl::optional<int> max_inbound_streams() const override { + return max_inbound_streams_; + } + void set_debug_name_for_testing(const char* debug_name) override { + debug_name_ = debug_name; + } + + // Exposed to allow Post call from c-callbacks. + // TODO(deadbeef): Remove this or at least make it return a const pointer. + rtc::Thread* network_thread() const { return network_thread_; } + + private: + // A message to be sent by the sctp library. This class is used to track the + // progress of writing a single message to the sctp library in the presence of + // partial writes. In this case, the Advance() function is provided in order + // to advance over what has already been accepted by the sctp library and + // avoid copying the remaining partial message buffer. + class OutgoingMessage { + public: + OutgoingMessage(const rtc::CopyOnWriteBuffer& buffer, + const SendDataParams& send_params) + : buffer_(buffer), send_params_(send_params) {} + + // Advances the buffer by the incremented amount. Must not advance further + // than the current data size. + void Advance(size_t increment) { + RTC_DCHECK_LE(increment + offset_, buffer_.size()); + offset_ += increment; + } + + size_t size() const { return buffer_.size() - offset_; } + + const void* data() const { return buffer_.data() + offset_; } + + SendDataParams send_params() const { return send_params_; } + + private: + const rtc::CopyOnWriteBuffer buffer_; + const SendDataParams send_params_; + size_t offset_ = 0; + }; + + void ConnectTransportSignals(); + void DisconnectTransportSignals(); + + // Creates the socket and connects. + bool Connect(); + + // Returns false when opening the socket failed. + bool OpenSctpSocket(); + // Helpet method to set socket options. + bool ConfigureSctpSocket(); + // Sets |sock_ |to nullptr. + void CloseSctpSocket(); + + // Sends a SCTP_RESET_STREAM for all streams in closing_ssids_. + bool SendQueuedStreamResets(); + + // Sets the "ready to send" flag and fires signal if needed. + void SetReadyToSendData(); + + // Sends the outgoing buffered message that was only partially accepted by the + // sctp lib because it did not have enough space. Returns true if the entire + // buffered message was accepted by the sctp lib. + bool SendBufferedMessage(); + + // Tries to send the |payload| on the usrsctp lib. The message will be + // advanced by the amount that was sent. + SendDataResult SendMessageInternal(OutgoingMessage* message); + + // Callbacks from DTLS transport. + void OnWritableState(rtc::PacketTransportInternal* transport); + virtual void OnPacketRead(rtc::PacketTransportInternal* transport, + const char* data, + size_t len, + const int64_t& packet_time_us, + int flags); + void OnClosed(rtc::PacketTransportInternal* transport); + + // Methods related to usrsctp callbacks. + void OnSendThresholdCallback(); + sockaddr_conn GetSctpSockAddr(int port); + + // Called using |invoker_| to send packet on the network. + void OnPacketFromSctpToNetwork(const rtc::CopyOnWriteBuffer& buffer); + // Called using |invoker_| to decide what to do with the packet. + // The |flags| parameter is used by SCTP to distinguish notification packets + // from other types of packets. + void OnInboundPacketFromSctpToTransport(const rtc::CopyOnWriteBuffer& buffer, + ReceiveDataParams params, + int flags); + void OnDataFromSctpToTransport(const ReceiveDataParams& params, + const rtc::CopyOnWriteBuffer& buffer); + void OnNotificationFromSctp(const rtc::CopyOnWriteBuffer& buffer); + void OnNotificationAssocChange(const sctp_assoc_change& change); + + void OnStreamResetEvent(const struct sctp_stream_reset_event* evt); + + // Responsible for marshalling incoming data to the transports listeners, and + // outgoing data to the network interface. + rtc::Thread* network_thread_; + // Helps pass inbound/outbound packets asynchronously to the network thread. + rtc::AsyncInvoker invoker_; + // Underlying DTLS transport. + rtc::PacketTransportInternal* transport_ = nullptr; + + // Track the data received from usrsctp between callbacks until the EOR bit + // arrives. + rtc::CopyOnWriteBuffer partial_incoming_message_; + ReceiveDataParams partial_params_; + int partial_flags_; + // A message that was attempted to be sent, but was only partially accepted by + // usrsctp lib with usrsctp_sendv() because it cannot buffer the full message. + // This occurs because we explicitly set the EOR bit when sending, so + // usrsctp_sendv() is not atomic. + absl::optional<OutgoingMessage> partial_outgoing_message_; + + bool was_ever_writable_ = false; + int local_port_ = kSctpDefaultPort; + int remote_port_ = kSctpDefaultPort; + int max_message_size_ = kSctpSendBufferSize; + struct socket* sock_ = nullptr; // The socket created by usrsctp_socket(...). + + // Has Start been called? Don't create SCTP socket until it has. + bool started_ = false; + // Are we ready to queue data (SCTP socket created, and not blocked due to + // congestion control)? Different than |transport_|'s "ready to send". + bool ready_to_send_data_ = false; + + // Used to keep track of the status of each stream (or rather, each pair of + // incoming/outgoing streams with matching IDs). It's specifically used to + // keep track of the status of resets, but more information could be put here + // later. + // + // See datachannel.h for a summary of the closing procedure. + struct StreamStatus { + // Closure initiated by application via ResetStream? Note that + // this may be true while outgoing_reset_initiated is false if the outgoing + // reset needed to be queued. + bool closure_initiated = false; + // Whether we've initiated the outgoing stream reset via + // SCTP_RESET_STREAMS. + bool outgoing_reset_initiated = false; + // Whether usrsctp has indicated that the incoming/outgoing streams have + // been reset. It's expected that the peer will reset its outgoing stream + // (our incoming stream) after receiving the reset for our outgoing stream, + // though older versions of chromium won't do this. See crbug.com/559394 + // for context. + bool outgoing_reset_complete = false; + bool incoming_reset_complete = false; + + // Some helper methods to improve code readability. + bool is_open() const { + return !closure_initiated && !incoming_reset_complete && + !outgoing_reset_complete; + } + // We need to send an outgoing reset if the application has closed the data + // channel, or if we received a reset of the incoming stream from the + // remote endpoint, indicating the data channel was closed remotely. + bool need_outgoing_reset() const { + return (incoming_reset_complete || closure_initiated) && + !outgoing_reset_initiated; + } + bool reset_complete() const { + return outgoing_reset_complete && incoming_reset_complete; + } + }; + + // Entries should only be removed from this map if |reset_complete| is + // true. + std::map<uint32_t, StreamStatus> stream_status_by_sid_; + + // A static human-readable name for debugging messages. + const char* debug_name_ = "SctpTransport"; + // Hides usrsctp interactions from this header file. + class UsrSctpWrapper; + // Number of channels negotiated. Not set before negotiation completes. + absl::optional<int> max_outbound_streams_; + absl::optional<int> max_inbound_streams_; + + RTC_DISALLOW_COPY_AND_ASSIGN(SctpTransport); +}; + +class SctpTransportFactory : public SctpTransportInternalFactory { + public: + explicit SctpTransportFactory(rtc::Thread* network_thread) + : network_thread_(network_thread) {} + + std::unique_ptr<SctpTransportInternal> CreateSctpTransport( + rtc::PacketTransportInternal* transport) override { + return std::unique_ptr<SctpTransportInternal>( + new SctpTransport(network_thread_, transport)); + } + + private: + rtc::Thread* network_thread_; +}; + +} // namespace cricket + +#endif // MEDIA_SCTP_SCTP_TRANSPORT_H_ diff --git a/media/sctp/sctp_transport_internal.h b/media/sctp/sctp_transport_internal.h new file mode 100644 index 0000000000..b0e0e0f7e6 --- /dev/null +++ b/media/sctp/sctp_transport_internal.h @@ -0,0 +1,159 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MEDIA_SCTP_SCTP_TRANSPORT_INTERNAL_H_ +#define MEDIA_SCTP_SCTP_TRANSPORT_INTERNAL_H_ + +// TODO(deadbeef): Move SCTP code out of media/, and make it not depend on +// anything in media/. + +#include <memory> +#include <string> +#include <vector> + +#include "rtc_base/copy_on_write_buffer.h" +#include "rtc_base/thread.h" +// For SendDataParams/ReceiveDataParams. +// TODO(deadbeef): Use something else for SCTP. It's confusing that we use an +// SSRC field for SID. +#include "media/base/media_channel.h" +#include "p2p/base/packet_transport_internal.h" + +namespace cricket { + +// Constants that are important to API users +// The size of the SCTP association send buffer. 256kB, the usrsctp default. +constexpr int kSctpSendBufferSize = 256 * 1024; + +// The number of outgoing streams that we'll negotiate. Since stream IDs (SIDs) +// are 0-based, the highest usable SID is 1023. +// +// It's recommended to use the maximum of 65535 in: +// https://tools.ietf.org/html/draft-ietf-rtcweb-data-channel-13#section-6.2 +// However, we use 1024 in order to save memory. usrsctp allocates 104 bytes +// for each pair of incoming/outgoing streams (on a 64-bit system), so 65535 +// streams would waste ~6MB. +// +// Note: "max" and "min" here are inclusive. +constexpr uint16_t kMaxSctpStreams = 1024; +constexpr uint16_t kMaxSctpSid = kMaxSctpStreams - 1; +constexpr uint16_t kMinSctpSid = 0; + +// This is the default SCTP port to use. It is passed along the wire and the +// connectee and connector must be using the same port. It is not related to the +// ports at the IP level. (Corresponds to: sockaddr_conn.sconn_port in +// usrsctp.h) +const int kSctpDefaultPort = 5000; + +// Abstract SctpTransport interface for use internally (by PeerConnection etc.). +// Exists to allow mock/fake SctpTransports to be created. +class SctpTransportInternal { + public: + virtual ~SctpTransportInternal() {} + + // Changes what underlying DTLS transport is uses. Used when switching which + // bundled transport the SctpTransport uses. + virtual void SetDtlsTransport(rtc::PacketTransportInternal* transport) = 0; + + // When Start is called, connects as soon as possible; this can be called + // before DTLS completes, in which case the connection will begin when DTLS + // completes. This method can be called multiple times, though not if either + // of the ports are changed. + // + // |local_sctp_port| and |remote_sctp_port| are passed along the wire and the + // listener and connector must be using the same port. They are not related + // to the ports at the IP level. If set to -1, we default to + // kSctpDefaultPort. + // |max_message_size_| sets the max message size on the connection. + // It must be smaller than or equal to kSctpSendBufferSize. + // It can be changed by a secons Start() call. + // + // TODO(deadbeef): Support calling Start with different local/remote ports + // and create a new association? Not clear if this is something we need to + // support though. See: https://github.com/w3c/webrtc-pc/issues/979 + virtual bool Start(int local_sctp_port, + int remote_sctp_port, + int max_message_size) = 0; + + // NOTE: Initially there was a "Stop" method here, but it was never used, so + // it was removed. + + // Informs SctpTransport that |sid| will start being used. Returns false if + // it is impossible to use |sid|, or if it's already in use. + // Until calling this, can't send data using |sid|. + // TODO(deadbeef): Actually implement the "returns false if |sid| can't be + // used" part. See: + // https://bugs.chromium.org/p/chromium/issues/detail?id=619849 + virtual bool OpenStream(int sid) = 0; + // The inverse of OpenStream. Begins the closing procedure, which will + // eventually result in SignalClosingProcedureComplete on the side that + // initiates it, and both SignalClosingProcedureStartedRemotely and + // SignalClosingProcedureComplete on the other side. + virtual bool ResetStream(int sid) = 0; + // Send data down this channel (will be wrapped as SCTP packets then given to + // usrsctp that will then post the network interface). + // Returns true iff successful data somewhere on the send-queue/network. + // Uses |params.ssrc| as the SCTP sid. + virtual bool SendData(const SendDataParams& params, + const rtc::CopyOnWriteBuffer& payload, + SendDataResult* result = nullptr) = 0; + + // Indicates when the SCTP socket is created and not blocked by congestion + // control. This changes to false when SDR_BLOCK is returned from SendData, + // and + // changes to true when SignalReadyToSendData is fired. The underlying DTLS/ + // ICE channels may be unwritable while ReadyToSendData is true, because data + // can still be queued in usrsctp. + virtual bool ReadyToSendData() = 0; + // Returns the current max message size, set with Start(). + virtual int max_message_size() const = 0; + // Returns the current negotiated max # of outbound streams. + // Will return absl::nullopt if negotiation is incomplete. + virtual absl::optional<int> max_outbound_streams() const = 0; + // Returns the current negotiated max # of inbound streams. + virtual absl::optional<int> max_inbound_streams() const = 0; + + sigslot::signal0<> SignalReadyToSendData; + sigslot::signal0<> SignalAssociationChangeCommunicationUp; + // ReceiveDataParams includes SID, seq num, timestamp, etc. CopyOnWriteBuffer + // contains message payload. + sigslot::signal2<const ReceiveDataParams&, const rtc::CopyOnWriteBuffer&> + SignalDataReceived; + // Parameter is SID; fired when we receive an incoming stream reset on an + // open stream, indicating that the other side started the closing procedure. + // After resetting the outgoing stream, SignalClosingProcedureComplete will + // fire too. + sigslot::signal1<int> SignalClosingProcedureStartedRemotely; + // Parameter is SID; fired when closing procedure is complete (both incoming + // and outgoing streams reset). + sigslot::signal1<int> SignalClosingProcedureComplete; + // Fired when the underlying DTLS transport has closed due to an error + // or an incoming DTLS disconnect. + sigslot::signal0<> SignalClosedAbruptly; + + // Helper for debugging. + virtual void set_debug_name_for_testing(const char* debug_name) = 0; +}; + +// Factory class which can be used to allow fake SctpTransports to be injected +// for testing. Or, theoretically, SctpTransportInternal implementations that +// use something other than usrsctp. +class SctpTransportInternalFactory { + public: + virtual ~SctpTransportInternalFactory() {} + + // Create an SCTP transport using |channel| for the underlying transport. + virtual std::unique_ptr<SctpTransportInternal> CreateSctpTransport( + rtc::PacketTransportInternal* channel) = 0; +}; + +} // namespace cricket + +#endif // MEDIA_SCTP_SCTP_TRANSPORT_INTERNAL_H_ diff --git a/media/sctp/sctp_transport_reliability_unittest.cc b/media/sctp/sctp_transport_reliability_unittest.cc new file mode 100644 index 0000000000..af9ddfeba7 --- /dev/null +++ b/media/sctp/sctp_transport_reliability_unittest.cc @@ -0,0 +1,826 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "media/sctp/sctp_transport.h" + +#include <memory> +#include <queue> +#include <string> + +#include "media/sctp/sctp_transport_internal.h" +#include "rtc_base/copy_on_write_buffer.h" +#include "rtc_base/gunit.h" +#include "rtc_base/logging.h" +#include "rtc_base/random.h" +#include "rtc_base/thread.h" +#include "test/gtest.h" + +namespace { + +static constexpr int kDefaultTimeout = 10000; // 10 seconds. +static constexpr int kTransport1Port = 15001; +static constexpr int kTransport2Port = 25002; +static constexpr int kLogPerMessagesCount = 100; + +/** + * An simple packet transport implementation which can be + * configured to simulate uniform random packet loss and + * configurable random packet delay and reordering. + */ +class SimulatedPacketTransport final : public rtc::PacketTransportInternal { + public: + SimulatedPacketTransport(std::string name, + rtc::Thread* transport_thread, + uint8_t packet_loss_percents, + uint16_t avg_send_delay_millis) + : transport_name_(name), + transport_thread_(transport_thread), + packet_loss_percents_(packet_loss_percents), + avg_send_delay_millis_(avg_send_delay_millis), + random_(42) { + RTC_DCHECK(transport_thread_); + RTC_DCHECK_LE(packet_loss_percents_, 100); + RTC_DCHECK_RUN_ON(transport_thread_); + } + + ~SimulatedPacketTransport() override { + RTC_DCHECK_RUN_ON(transport_thread_); + auto destination = destination_.load(); + if (destination != nullptr) { + invoker_.Flush(destination->transport_thread_); + } + invoker_.Flush(transport_thread_); + destination_ = nullptr; + SignalWritableState(this); + } + + const std::string& transport_name() const override { return transport_name_; } + + bool writable() const override { return destination_ != nullptr; } + + bool receiving() const override { return true; } + + int SendPacket(const char* data, + size_t len, + const rtc::PacketOptions& options, + int flags = 0) { + RTC_DCHECK_RUN_ON(transport_thread_); + auto destination = destination_.load(); + if (destination == nullptr) { + return -1; + } + if (random_.Rand(100) < packet_loss_percents_) { + // silent packet loss + return 0; + } + rtc::CopyOnWriteBuffer buffer(data, len); + auto send_job = [this, flags, buffer = std::move(buffer)] { + auto destination = destination_.load(); + if (destination == nullptr) { + return; + } + destination->SignalReadPacket( + destination, reinterpret_cast<const char*>(buffer.data()), + buffer.size(), rtc::Time(), flags); + }; + // Introduce random send delay in range [0 .. 2 * avg_send_delay_millis_] + // millis, which will also work as random packet reordering mechanism. + uint16_t actual_send_delay = avg_send_delay_millis_; + int16_t reorder_delay = + avg_send_delay_millis_ * + std::min(1.0, std::max(-1.0, random_.Gaussian(0, 0.5))); + actual_send_delay += reorder_delay; + + if (actual_send_delay > 0) { + invoker_.AsyncInvokeDelayed<void>(RTC_FROM_HERE, + destination->transport_thread_, + std::move(send_job), actual_send_delay); + } else { + invoker_.AsyncInvoke<void>(RTC_FROM_HERE, destination->transport_thread_, + std::move(send_job)); + } + return 0; + } + + int SetOption(rtc::Socket::Option opt, int value) override { return 0; } + + bool GetOption(rtc::Socket::Option opt, int* value) override { return false; } + + int GetError() override { return 0; } + + absl::optional<rtc::NetworkRoute> network_route() const override { + return absl::nullopt; + } + + void SetDestination(SimulatedPacketTransport* destination) { + RTC_DCHECK_RUN_ON(transport_thread_); + if (destination == this) { + return; + } + destination_ = destination; + SignalWritableState(this); + } + + private: + const std::string transport_name_; + rtc::Thread* const transport_thread_; + const uint8_t packet_loss_percents_; + const uint16_t avg_send_delay_millis_; + std::atomic<SimulatedPacketTransport*> destination_ ATOMIC_VAR_INIT(nullptr); + rtc::AsyncInvoker invoker_; + webrtc::Random random_; + RTC_DISALLOW_COPY_AND_ASSIGN(SimulatedPacketTransport); +}; + +/** + * A helper class to send specified number of messages + * over SctpTransport with SCTP reliability settings + * provided by user. The reliability settings are specified + * by passing a template instance of SendDataParams. + * When .sid field inside SendDataParams is specified to + * negative value it means that actual .sid will be + * assigned by sender itself, .sid will be assigned from + * range [cricket::kMinSctpSid; cricket::kMaxSctpSid]. + * The wide range of sids are used to possibly trigger + * more execution paths inside usrsctp. + */ +class SctpDataSender final { + public: + SctpDataSender(rtc::Thread* thread, + cricket::SctpTransport* transport, + uint64_t target_messages_count, + cricket::SendDataParams send_params, + uint32_t sender_id) + : thread_(thread), + transport_(transport), + target_messages_count_(target_messages_count), + send_params_(send_params), + sender_id_(sender_id) { + RTC_DCHECK(thread_); + RTC_DCHECK(transport_); + } + + void Start() { + invoker_.AsyncInvoke<void>(RTC_FROM_HERE, thread_, [this] { + if (started_) { + RTC_LOG(LS_INFO) << sender_id_ << " sender is already started"; + return; + } + started_ = true; + SendNextMessage(); + }); + } + + uint64_t BytesSentCount() const { return num_bytes_sent_; } + + uint64_t MessagesSentCount() const { return num_messages_sent_; } + + absl::optional<std::string> GetLastError() { + absl::optional<std::string> result = absl::nullopt; + thread_->Invoke<void>(RTC_FROM_HERE, + [this, &result] { result = last_error_; }); + return result; + } + + bool WaitForCompletion(int give_up_after_ms) { + return sent_target_messages_count_.Wait(give_up_after_ms, kDefaultTimeout); + } + + private: + void SendNextMessage() { + RTC_DCHECK_RUN_ON(thread_); + if (!started_ || num_messages_sent_ >= target_messages_count_) { + sent_target_messages_count_.Set(); + return; + } + + if (num_messages_sent_ % kLogPerMessagesCount == 0) { + RTC_LOG(LS_INFO) << sender_id_ << " sender will try send message " + << (num_messages_sent_ + 1) << " out of " + << target_messages_count_; + } + + cricket::SendDataParams params(send_params_); + if (params.sid < 0) { + params.sid = cricket::kMinSctpSid + + (num_messages_sent_ % cricket::kMaxSctpStreams); + } + + cricket::SendDataResult result; + transport_->SendData(params, payload_, &result); + switch (result) { + case cricket::SDR_BLOCK: + // retry after timeout + invoker_.AsyncInvokeDelayed<void>( + RTC_FROM_HERE, thread_, + rtc::Bind(&SctpDataSender::SendNextMessage, this), 500); + break; + case cricket::SDR_SUCCESS: + // send next + num_bytes_sent_ += payload_.size(); + ++num_messages_sent_; + invoker_.AsyncInvoke<void>( + RTC_FROM_HERE, thread_, + rtc::Bind(&SctpDataSender::SendNextMessage, this)); + break; + case cricket::SDR_ERROR: + // give up + last_error_ = "SctpTransport::SendData error returned"; + sent_target_messages_count_.Set(); + break; + } + } + + rtc::Thread* const thread_; + cricket::SctpTransport* const transport_; + const uint64_t target_messages_count_; + const cricket::SendDataParams send_params_; + const uint32_t sender_id_; + rtc::CopyOnWriteBuffer payload_{std::string(1400, '.').c_str(), 1400}; + std::atomic<bool> started_ ATOMIC_VAR_INIT(false); + rtc::AsyncInvoker invoker_; + std::atomic<uint64_t> num_messages_sent_ ATOMIC_VAR_INIT(0); + rtc::Event sent_target_messages_count_{true, false}; + std::atomic<uint64_t> num_bytes_sent_ ATOMIC_VAR_INIT(0); + absl::optional<std::string> last_error_; + RTC_DISALLOW_COPY_AND_ASSIGN(SctpDataSender); +}; + +/** + * A helper class which counts number of received messages + * and bytes over SctpTransport. Also allow waiting until + * specified number of messages received. + */ +class SctpDataReceiver final : public sigslot::has_slots<> { + public: + explicit SctpDataReceiver(uint32_t receiver_id, + uint64_t target_messages_count) + : receiver_id_(receiver_id), + target_messages_count_(target_messages_count) {} + + void OnDataReceived(const cricket::ReceiveDataParams& params, + const rtc::CopyOnWriteBuffer& data) { + num_bytes_received_ += data.size(); + if (++num_messages_received_ == target_messages_count_) { + received_target_messages_count_.Set(); + } + + if (num_messages_received_ % kLogPerMessagesCount == 0) { + RTC_LOG(INFO) << receiver_id_ << " receiver got " + << num_messages_received_ << " messages"; + } + } + + uint64_t MessagesReceivedCount() const { return num_messages_received_; } + + uint64_t BytesReceivedCount() const { return num_bytes_received_; } + + bool WaitForMessagesReceived(int timeout_millis) { + return received_target_messages_count_.Wait(timeout_millis); + } + + private: + std::atomic<uint64_t> num_messages_received_ ATOMIC_VAR_INIT(0); + std::atomic<uint64_t> num_bytes_received_ ATOMIC_VAR_INIT(0); + rtc::Event received_target_messages_count_{true, false}; + const uint32_t receiver_id_; + const uint64_t target_messages_count_; + RTC_DISALLOW_COPY_AND_ASSIGN(SctpDataReceiver); +}; + +/** + * Simple class to manage set of threads. + */ +class ThreadPool final { + public: + explicit ThreadPool(size_t threads_count) : random_(42) { + RTC_DCHECK(threads_count > 0); + threads_.reserve(threads_count); + for (size_t i = 0; i < threads_count; i++) { + auto thread = rtc::Thread::Create(); + thread->SetName("Thread #" + rtc::ToString(i + 1) + " from Pool", this); + thread->Start(); + threads_.emplace_back(std::move(thread)); + } + } + + rtc::Thread* GetRandomThread() { + return threads_[random_.Rand(0U, threads_.size() - 1)].get(); + } + + private: + webrtc::Random random_; + std::vector<std::unique_ptr<rtc::Thread>> threads_; + RTC_DISALLOW_COPY_AND_ASSIGN(ThreadPool); +}; + +/** + * Represents single ping-pong test over SctpTransport. + * User can specify target number of message for bidirectional + * send, underlying transport packets loss and average packet delay + * and SCTP delivery settings. + */ +class SctpPingPong final { + public: + SctpPingPong(uint32_t id, + uint16_t port1, + uint16_t port2, + rtc::Thread* transport_thread1, + rtc::Thread* transport_thread2, + uint32_t messages_count, + uint8_t packet_loss_percents, + uint16_t avg_send_delay_millis, + cricket::SendDataParams send_params) + : id_(id), + port1_(port1), + port2_(port2), + transport_thread1_(transport_thread1), + transport_thread2_(transport_thread2), + messages_count_(messages_count), + packet_loss_percents_(packet_loss_percents), + avg_send_delay_millis_(avg_send_delay_millis), + send_params_(send_params) { + RTC_DCHECK(transport_thread1_ != nullptr); + RTC_DCHECK(transport_thread2_ != nullptr); + } + + virtual ~SctpPingPong() { + transport_thread1_->Invoke<void>(RTC_FROM_HERE, [this] { + data_sender1_.reset(); + sctp_transport1_->SetDtlsTransport(nullptr); + packet_transport1_->SetDestination(nullptr); + }); + transport_thread2_->Invoke<void>(RTC_FROM_HERE, [this] { + data_sender2_.reset(); + sctp_transport2_->SetDtlsTransport(nullptr); + packet_transport2_->SetDestination(nullptr); + }); + transport_thread1_->Invoke<void>(RTC_FROM_HERE, [this] { + sctp_transport1_.reset(); + data_receiver1_.reset(); + packet_transport1_.reset(); + }); + transport_thread2_->Invoke<void>(RTC_FROM_HERE, [this] { + sctp_transport2_.reset(); + data_receiver2_.reset(); + packet_transport2_.reset(); + }); + } + + bool Start() { + CreateTwoConnectedSctpTransportsWithAllStreams(); + + { + rtc::CritScope cs(&lock_); + if (!errors_list_.empty()) { + return false; + } + } + + data_sender1_.reset(new SctpDataSender(transport_thread1_, + sctp_transport1_.get(), + messages_count_, send_params_, id_)); + data_sender2_.reset(new SctpDataSender(transport_thread2_, + sctp_transport2_.get(), + messages_count_, send_params_, id_)); + data_sender1_->Start(); + data_sender2_->Start(); + return true; + } + + std::vector<std::string> GetErrorsList() const { + std::vector<std::string> result; + { + rtc::CritScope cs(&lock_); + result = errors_list_; + } + return result; + } + + void WaitForCompletion(int32_t timeout_millis) { + if (data_sender1_ == nullptr) { + ReportError("SctpPingPong id = " + rtc::ToString(id_) + + ", sender 1 is not created"); + return; + } + if (data_sender2_ == nullptr) { + ReportError("SctpPingPong id = " + rtc::ToString(id_) + + ", sender 2 is not created"); + return; + } + + if (!data_sender1_->WaitForCompletion(timeout_millis)) { + ReportError("SctpPingPong id = " + rtc::ToString(id_) + + ", sender 1 failed to complete within " + + rtc::ToString(timeout_millis) + " millis"); + return; + } + + auto sender1_error = data_sender1_->GetLastError(); + if (sender1_error.has_value()) { + ReportError("SctpPingPong id = " + rtc::ToString(id_) + + ", sender 1 error: " + sender1_error.value()); + return; + } + + if (!data_sender2_->WaitForCompletion(timeout_millis)) { + ReportError("SctpPingPong id = " + rtc::ToString(id_) + + ", sender 2 failed to complete within " + + rtc::ToString(timeout_millis) + " millis"); + return; + } + + auto sender2_error = data_sender2_->GetLastError(); + if (sender2_error.has_value()) { + ReportError("SctpPingPong id = " + rtc::ToString(id_) + + ", sender 2 error: " + sender1_error.value()); + return; + } + + if ((data_sender1_->MessagesSentCount() != messages_count_)) { + ReportError("SctpPingPong id = " + rtc::ToString(id_) + + ", sender 1 sent only " + + rtc::ToString(data_sender1_->MessagesSentCount()) + + " out of " + rtc::ToString(messages_count_)); + return; + } + + if ((data_sender2_->MessagesSentCount() != messages_count_)) { + ReportError("SctpPingPong id = " + rtc::ToString(id_) + + ", sender 2 sent only " + + rtc::ToString(data_sender2_->MessagesSentCount()) + + " out of " + rtc::ToString(messages_count_)); + return; + } + + if (!data_receiver1_->WaitForMessagesReceived(timeout_millis)) { + ReportError("SctpPingPong id = " + rtc::ToString(id_) + + ", receiver 1 did not complete within " + + rtc::ToString(messages_count_)); + return; + } + + if (!data_receiver2_->WaitForMessagesReceived(timeout_millis)) { + ReportError("SctpPingPong id = " + rtc::ToString(id_) + + ", receiver 2 did not complete within " + + rtc::ToString(messages_count_)); + return; + } + + if (data_receiver1_->BytesReceivedCount() != + data_sender2_->BytesSentCount()) { + ReportError( + "SctpPingPong id = " + rtc::ToString(id_) + ", receiver 1 received " + + rtc::ToString(data_receiver1_->BytesReceivedCount()) + + " bytes, but sender 2 send " + + rtc::ToString(rtc::ToString(data_sender2_->BytesSentCount()))); + return; + } + + if (data_receiver2_->BytesReceivedCount() != + data_sender1_->BytesSentCount()) { + ReportError( + "SctpPingPong id = " + rtc::ToString(id_) + ", receiver 2 received " + + rtc::ToString(data_receiver2_->BytesReceivedCount()) + + " bytes, but sender 1 send " + + rtc::ToString(rtc::ToString(data_sender1_->BytesSentCount()))); + return; + } + + RTC_LOG(LS_INFO) << "SctpPingPong id = " << id_ << " is done"; + } + + private: + void CreateTwoConnectedSctpTransportsWithAllStreams() { + transport_thread1_->Invoke<void>(RTC_FROM_HERE, [this] { + packet_transport1_.reset(new SimulatedPacketTransport( + "SctpPingPong id = " + rtc::ToString(id_) + ", packet transport 1", + transport_thread1_, packet_loss_percents_, avg_send_delay_millis_)); + data_receiver1_.reset(new SctpDataReceiver(id_, messages_count_)); + sctp_transport1_.reset(new cricket::SctpTransport( + transport_thread1_, packet_transport1_.get())); + sctp_transport1_->set_debug_name_for_testing("sctp transport 1"); + + sctp_transport1_->SignalDataReceived.connect( + data_receiver1_.get(), &SctpDataReceiver::OnDataReceived); + + for (uint32_t i = cricket::kMinSctpSid; i <= cricket::kMaxSctpSid; i++) { + if (!sctp_transport1_->OpenStream(i)) { + ReportError("SctpPingPong id = " + rtc::ToString(id_) + + ", sctp transport 1 stream " + rtc::ToString(i) + + " failed to open"); + break; + } + } + }); + + transport_thread2_->Invoke<void>(RTC_FROM_HERE, [this] { + packet_transport2_.reset(new SimulatedPacketTransport( + "SctpPingPong id = " + rtc::ToString(id_) + "packet transport 2", + transport_thread2_, packet_loss_percents_, avg_send_delay_millis_)); + data_receiver2_.reset(new SctpDataReceiver(id_, messages_count_)); + sctp_transport2_.reset(new cricket::SctpTransport( + transport_thread2_, packet_transport2_.get())); + sctp_transport2_->set_debug_name_for_testing("sctp transport 2"); + sctp_transport2_->SignalDataReceived.connect( + data_receiver2_.get(), &SctpDataReceiver::OnDataReceived); + + for (uint32_t i = cricket::kMinSctpSid; i <= cricket::kMaxSctpSid; i++) { + if (!sctp_transport2_->OpenStream(i)) { + ReportError("SctpPingPong id = " + rtc::ToString(id_) + + ", sctp transport 2 stream " + rtc::ToString(i) + + " failed to open"); + break; + } + } + }); + + transport_thread1_->Invoke<void>(RTC_FROM_HERE, [this] { + packet_transport1_->SetDestination(packet_transport2_.get()); + }); + transport_thread2_->Invoke<void>(RTC_FROM_HERE, [this] { + packet_transport2_->SetDestination(packet_transport1_.get()); + }); + + transport_thread1_->Invoke<void>(RTC_FROM_HERE, [this] { + if (!sctp_transport1_->Start(port1_, port2_, + cricket::kSctpSendBufferSize)) { + ReportError("SctpPingPong id = " + rtc::ToString(id_) + + ", failed to start sctp transport 1"); + } + }); + + transport_thread2_->Invoke<void>(RTC_FROM_HERE, [this] { + if (!sctp_transport2_->Start(port2_, port1_, + cricket::kSctpSendBufferSize)) { + ReportError("SctpPingPong id = " + rtc::ToString(id_) + + ", failed to start sctp transport 2"); + } + }); + } + + void ReportError(std::string error) { + rtc::CritScope cs(&lock_); + errors_list_.push_back(std::move(error)); + } + + std::unique_ptr<SimulatedPacketTransport> packet_transport1_; + std::unique_ptr<SimulatedPacketTransport> packet_transport2_; + std::unique_ptr<SctpDataReceiver> data_receiver1_; + std::unique_ptr<SctpDataReceiver> data_receiver2_; + std::unique_ptr<cricket::SctpTransport> sctp_transport1_; + std::unique_ptr<cricket::SctpTransport> sctp_transport2_; + std::unique_ptr<SctpDataSender> data_sender1_; + std::unique_ptr<SctpDataSender> data_sender2_; + rtc::CriticalSection lock_; + std::vector<std::string> errors_list_ RTC_GUARDED_BY(lock_); + + const uint32_t id_; + const uint16_t port1_; + const uint16_t port2_; + rtc::Thread* const transport_thread1_; + rtc::Thread* const transport_thread2_; + const uint32_t messages_count_; + const uint8_t packet_loss_percents_; + const uint16_t avg_send_delay_millis_; + const cricket::SendDataParams send_params_; + RTC_DISALLOW_COPY_AND_ASSIGN(SctpPingPong); +}; + +/** + * Helper function to calculate max number of milliseconds + * allowed for test to run based on test configuration. + */ +constexpr int32_t GetExecutionTimeLimitInMillis(uint32_t total_messages, + uint8_t packet_loss_percents) { + return std::min<int64_t>( + std::numeric_limits<int32_t>::max(), + std::max<int64_t>( + 1LL * total_messages * 100 * + std::max(1, packet_loss_percents * packet_loss_percents), + kDefaultTimeout)); +} + +} // namespace + +namespace cricket { + +/** + * The set of tests intended to check usrsctp reliability on + * stress conditions: multiple sockets, concurrent access, + * lossy network link. It was observed in the past that + * usrsctp might misbehave in concurrent environment + * under load on lossy networks: deadlocks and memory corruption + * issues might happen in non-basic usage scenarios. + * It's recommended to run this test whenever usrsctp version + * used is updated to verify it properly works in stress + * conditions under higher than usual load. + * It is also recommended to enable ASAN when these tests + * are executed, so whenever memory bug is happen inside usrsctp, + * it will be easier to understand what went wrong with ASAN + * provided diagnostics information. + * The tests cases currently disabled by default due to + * long execution time and due to unresolved issue inside + * `usrsctp` library detected by try-bots with ThreadSanitizer. + */ +class UsrSctpReliabilityTest : public ::testing::Test {}; + +/** + * A simple test which send multiple messages over reliable + * connection, usefull to verify test infrastructure works. + * Execution time is less than 1 second. + */ +TEST_F(UsrSctpReliabilityTest, + DISABLED_AllMessagesAreDeliveredOverReliableConnection) { + auto thread1 = rtc::Thread::Create(); + auto thread2 = rtc::Thread::Create(); + thread1->Start(); + thread2->Start(); + constexpr uint8_t packet_loss_percents = 0; + constexpr uint16_t avg_send_delay_millis = 10; + constexpr uint32_t messages_count = 100; + constexpr int32_t wait_timeout = + GetExecutionTimeLimitInMillis(messages_count, packet_loss_percents); + static_assert(wait_timeout > 0, + "Timeout computation must produce positive value"); + + cricket::SendDataParams send_params; + send_params.sid = -1; + send_params.ordered = true; + send_params.reliable = true; + send_params.max_rtx_count = 0; + send_params.max_rtx_ms = 0; + + SctpPingPong test(1, kTransport1Port, kTransport2Port, thread1.get(), + thread2.get(), messages_count, packet_loss_percents, + avg_send_delay_millis, send_params); + EXPECT_TRUE(test.Start()) << rtc::join(test.GetErrorsList(), ';'); + test.WaitForCompletion(wait_timeout); + auto errors_list = test.GetErrorsList(); + EXPECT_TRUE(errors_list.empty()) << rtc::join(errors_list, ';'); +} + +/** + * A test to verify that multiple messages can be reliably delivered + * over lossy network when usrsctp configured to guarantee reliably + * and in order delivery. + * The test case is disabled by default because it takes + * long time to run. + * Execution time is about 2.5 minutes. + */ +TEST_F(UsrSctpReliabilityTest, + DISABLED_AllMessagesAreDeliveredOverLossyConnectionReliableAndInOrder) { + auto thread1 = rtc::Thread::Create(); + auto thread2 = rtc::Thread::Create(); + thread1->Start(); + thread2->Start(); + constexpr uint8_t packet_loss_percents = 5; + constexpr uint16_t avg_send_delay_millis = 16; + constexpr uint32_t messages_count = 10000; + constexpr int32_t wait_timeout = + GetExecutionTimeLimitInMillis(messages_count, packet_loss_percents); + static_assert(wait_timeout > 0, + "Timeout computation must produce positive value"); + + cricket::SendDataParams send_params; + send_params.sid = -1; + send_params.ordered = true; + send_params.reliable = true; + send_params.max_rtx_count = 0; + send_params.max_rtx_ms = 0; + + SctpPingPong test(1, kTransport1Port, kTransport2Port, thread1.get(), + thread2.get(), messages_count, packet_loss_percents, + avg_send_delay_millis, send_params); + + EXPECT_TRUE(test.Start()) << rtc::join(test.GetErrorsList(), ';'); + test.WaitForCompletion(wait_timeout); + auto errors_list = test.GetErrorsList(); + EXPECT_TRUE(errors_list.empty()) << rtc::join(errors_list, ';'); +} + +/** + * A test to verify that multiple messages can be reliably delivered + * over lossy network when usrsctp configured to retransmit lost + * packets. + * The test case is disabled by default because it takes + * long time to run. + * Execution time is about 2.5 minutes. + */ +TEST_F(UsrSctpReliabilityTest, + DISABLED_AllMessagesAreDeliveredOverLossyConnectionWithRetries) { + auto thread1 = rtc::Thread::Create(); + auto thread2 = rtc::Thread::Create(); + thread1->Start(); + thread2->Start(); + constexpr uint8_t packet_loss_percents = 5; + constexpr uint16_t avg_send_delay_millis = 16; + constexpr uint32_t messages_count = 10000; + constexpr int32_t wait_timeout = + GetExecutionTimeLimitInMillis(messages_count, packet_loss_percents); + static_assert(wait_timeout > 0, + "Timeout computation must produce positive value"); + + cricket::SendDataParams send_params; + send_params.sid = -1; + send_params.ordered = false; + send_params.reliable = false; + send_params.max_rtx_count = INT_MAX; + send_params.max_rtx_ms = INT_MAX; + + SctpPingPong test(1, kTransport1Port, kTransport2Port, thread1.get(), + thread2.get(), messages_count, packet_loss_percents, + avg_send_delay_millis, send_params); + + EXPECT_TRUE(test.Start()) << rtc::join(test.GetErrorsList(), ';'); + test.WaitForCompletion(wait_timeout); + auto errors_list = test.GetErrorsList(); + EXPECT_TRUE(errors_list.empty()) << rtc::join(errors_list, ';'); +} + +/** + * This is kind of reliability stress-test of usrsctp to verify + * that all messages are delivered when multiple usrsctp + * sockets used concurrently and underlying transport is lossy. + * + * It was observed in the past that in stress condtions usrsctp + * might encounter deadlock and memory corruption bugs: + * https://github.com/sctplab/usrsctp/issues/325 + * + * It is recoomended to run this test whenever usrsctp version + * used by WebRTC is updated. + * + * The test case is disabled by default because it takes + * long time to run. + * Execution time of this test is about 1-2 hours. + */ +TEST_F(UsrSctpReliabilityTest, + DISABLED_AllMessagesAreDeliveredOverLossyConnectionConcurrentTests) { + ThreadPool pool(16); + + cricket::SendDataParams send_params; + send_params.sid = -1; + send_params.ordered = true; + send_params.reliable = true; + send_params.max_rtx_count = 0; + send_params.max_rtx_ms = 0; + constexpr uint32_t base_sctp_port = 5000; + + // The constants value below were experimentally chosen + // to have reasonable execution time and to reproduce + // particular deadlock issue inside usrsctp: + // https://github.com/sctplab/usrsctp/issues/325 + // The constants values may be adjusted next time + // some other issue inside usrsctp need to be debugged. + constexpr uint32_t messages_count = 200; + constexpr uint8_t packet_loss_percents = 5; + constexpr uint16_t avg_send_delay_millis = 0; + constexpr uint32_t parallel_ping_pongs = 16 * 1024; + constexpr uint32_t total_ping_pong_tests = 16 * parallel_ping_pongs; + + constexpr int32_t wait_timeout = GetExecutionTimeLimitInMillis( + total_ping_pong_tests * messages_count, packet_loss_percents); + static_assert(wait_timeout > 0, + "Timeout computation must produce positive value"); + + std::queue<std::unique_ptr<SctpPingPong>> tests; + + for (uint32_t i = 0; i < total_ping_pong_tests; i++) { + uint32_t port1 = + base_sctp_port + (2 * i) % (UINT16_MAX - base_sctp_port - 1); + + auto test = std::make_unique<SctpPingPong>( + i, port1, port1 + 1, pool.GetRandomThread(), pool.GetRandomThread(), + messages_count, packet_loss_percents, avg_send_delay_millis, + send_params); + + EXPECT_TRUE(test->Start()) << rtc::join(test->GetErrorsList(), ';'); + tests.emplace(std::move(test)); + + while (tests.size() >= parallel_ping_pongs) { + auto& oldest_test = tests.front(); + oldest_test->WaitForCompletion(wait_timeout); + + auto errors_list = oldest_test->GetErrorsList(); + EXPECT_TRUE(errors_list.empty()) << rtc::join(errors_list, ';'); + tests.pop(); + } + } + + while (!tests.empty()) { + auto& oldest_test = tests.front(); + oldest_test->WaitForCompletion(wait_timeout); + + auto errors_list = oldest_test->GetErrorsList(); + EXPECT_TRUE(errors_list.empty()) << rtc::join(errors_list, ';'); + tests.pop(); + } +} + +} // namespace cricket diff --git a/media/sctp/sctp_transport_unittest.cc b/media/sctp/sctp_transport_unittest.cc new file mode 100644 index 0000000000..ff3f2d70a9 --- /dev/null +++ b/media/sctp/sctp_transport_unittest.cc @@ -0,0 +1,717 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "media/sctp/sctp_transport.h" + +#include <stdio.h> +#include <string.h> + +#include <memory> +#include <string> +#include <vector> + +#include "absl/algorithm/container.h" +#include "media/sctp/sctp_transport_internal.h" +#include "p2p/base/fake_dtls_transport.h" +#include "rtc_base/copy_on_write_buffer.h" +#include "rtc_base/gunit.h" +#include "rtc_base/logging.h" +#include "rtc_base/thread.h" +#include "test/gtest.h" + +namespace { +static const int kDefaultTimeout = 10000; // 10 seconds. +// Use ports other than the default 5000 for testing. +static const int kTransport1Port = 5001; +static const int kTransport2Port = 5002; +} // namespace + +namespace cricket { + +// This is essentially a buffer to hold recieved data. It stores only the last +// received data. Calling OnDataReceived twice overwrites old data with the +// newer one. +// TODO(ldixon): Implement constraints, and allow new data to be added to old +// instead of replacing it. +class SctpFakeDataReceiver : public sigslot::has_slots<> { + public: + SctpFakeDataReceiver() : received_(false) {} + + void Clear() { + received_ = false; + last_data_ = ""; + last_params_ = ReceiveDataParams(); + num_messages_received_ = 0; + } + + void OnDataReceived(const ReceiveDataParams& params, + const rtc::CopyOnWriteBuffer& data) { + num_messages_received_++; + received_ = true; + last_data_ = std::string(data.data<char>(), data.size()); + last_params_ = params; + } + + bool received() const { return received_; } + std::string last_data() const { return last_data_; } + ReceiveDataParams last_params() const { return last_params_; } + size_t num_messages_received() const { return num_messages_received_; } + + private: + bool received_; + std::string last_data_; + size_t num_messages_received_ = 0; + ReceiveDataParams last_params_; +}; + +class SctpTransportObserver : public sigslot::has_slots<> { + public: + explicit SctpTransportObserver(SctpTransport* transport) { + transport->SignalClosingProcedureComplete.connect( + this, &SctpTransportObserver::OnClosingProcedureComplete); + transport->SignalReadyToSendData.connect( + this, &SctpTransportObserver::OnReadyToSend); + } + + int StreamCloseCount(int stream) { + return absl::c_count(closed_streams_, stream); + } + + bool WasStreamClosed(int stream) { + return absl::c_linear_search(closed_streams_, stream); + } + + bool ReadyToSend() { return ready_to_send_; } + + private: + void OnClosingProcedureComplete(int stream) { + closed_streams_.push_back(stream); + } + void OnReadyToSend() { ready_to_send_ = true; } + + std::vector<int> closed_streams_; + bool ready_to_send_ = false; +}; + +// Helper class used to immediately attempt to reopen a stream as soon as it's +// been closed. +class SignalTransportClosedReopener : public sigslot::has_slots<> { + public: + SignalTransportClosedReopener(SctpTransport* transport, SctpTransport* peer) + : transport_(transport), peer_(peer) {} + + int StreamCloseCount(int stream) { return absl::c_count(streams_, stream); } + + private: + void OnStreamClosed(int stream) { + transport_->OpenStream(stream); + peer_->OpenStream(stream); + streams_.push_back(stream); + } + + SctpTransport* transport_; + SctpTransport* peer_; + std::vector<int> streams_; +}; + +// SCTP Data Engine testing framework. +class SctpTransportTest : public ::testing::Test, public sigslot::has_slots<> { + protected: + // usrsctp uses the NSS random number generator on non-Android platforms, + // so we need to initialize SSL. + static void SetUpTestSuite() {} + + void SetupConnectedTransportsWithTwoStreams() { + SetupConnectedTransportsWithTwoStreams(kTransport1Port, kTransport2Port); + } + + void SetupConnectedTransportsWithTwoStreams(int port1, int port2) { + fake_dtls1_.reset(new FakeDtlsTransport("fake dtls 1", 0)); + fake_dtls2_.reset(new FakeDtlsTransport("fake dtls 2", 0)); + recv1_.reset(new SctpFakeDataReceiver()); + recv2_.reset(new SctpFakeDataReceiver()); + transport1_.reset(CreateTransport(fake_dtls1_.get(), recv1_.get())); + transport1_->set_debug_name_for_testing("transport1"); + transport1_->SignalReadyToSendData.connect( + this, &SctpTransportTest::OnChan1ReadyToSend); + transport2_.reset(CreateTransport(fake_dtls2_.get(), recv2_.get())); + transport2_->set_debug_name_for_testing("transport2"); + transport2_->SignalReadyToSendData.connect( + this, &SctpTransportTest::OnChan2ReadyToSend); + // Setup two connected transports ready to send and receive. + bool asymmetric = false; + fake_dtls1_->SetDestination(fake_dtls2_.get(), asymmetric); + + RTC_LOG(LS_VERBOSE) << "Transport setup ----------------------------- "; + AddStream(1); + AddStream(2); + + RTC_LOG(LS_VERBOSE) + << "Connect the transports -----------------------------"; + // Both transports need to have started (with matching ports) for an + // association to be formed. + transport1_->Start(port1, port2, kSctpSendBufferSize); + transport2_->Start(port2, port1, kSctpSendBufferSize); + } + + bool AddStream(int sid) { + bool ret = true; + ret = ret && transport1_->OpenStream(sid); + ret = ret && transport2_->OpenStream(sid); + return ret; + } + + SctpTransport* CreateTransport(FakeDtlsTransport* fake_dtls, + SctpFakeDataReceiver* recv) { + SctpTransport* transport = + new SctpTransport(rtc::Thread::Current(), fake_dtls); + // When data is received, pass it to the SctpFakeDataReceiver. + transport->SignalDataReceived.connect( + recv, &SctpFakeDataReceiver::OnDataReceived); + return transport; + } + + bool SendData(SctpTransport* chan, + int sid, + const std::string& msg, + SendDataResult* result, + bool ordered = false) { + SendDataParams params; + params.sid = sid; + params.ordered = ordered; + + return chan->SendData(params, rtc::CopyOnWriteBuffer(&msg[0], msg.length()), + result); + } + + bool ReceivedData(const SctpFakeDataReceiver* recv, + int sid, + const std::string& msg) { + return (recv->received() && recv->last_params().sid == sid && + recv->last_data() == msg); + } + + bool ProcessMessagesUntilIdle() { + rtc::Thread* thread = rtc::Thread::Current(); + while (!thread->empty()) { + rtc::Message msg; + if (thread->Get(&msg, rtc::Thread::kForever)) { + thread->Dispatch(&msg); + } + } + return !thread->IsQuitting(); + } + + SctpTransport* transport1() { return transport1_.get(); } + SctpTransport* transport2() { return transport2_.get(); } + SctpFakeDataReceiver* receiver1() { return recv1_.get(); } + SctpFakeDataReceiver* receiver2() { return recv2_.get(); } + FakeDtlsTransport* fake_dtls1() { return fake_dtls1_.get(); } + FakeDtlsTransport* fake_dtls2() { return fake_dtls2_.get(); } + + int transport1_ready_to_send_count() { + return transport1_ready_to_send_count_; + } + int transport2_ready_to_send_count() { + return transport2_ready_to_send_count_; + } + + private: + std::unique_ptr<FakeDtlsTransport> fake_dtls1_; + std::unique_ptr<FakeDtlsTransport> fake_dtls2_; + std::unique_ptr<SctpFakeDataReceiver> recv1_; + std::unique_ptr<SctpFakeDataReceiver> recv2_; + std::unique_ptr<SctpTransport> transport1_; + std::unique_ptr<SctpTransport> transport2_; + + int transport1_ready_to_send_count_ = 0; + int transport2_ready_to_send_count_ = 0; + + void OnChan1ReadyToSend() { ++transport1_ready_to_send_count_; } + void OnChan2ReadyToSend() { ++transport2_ready_to_send_count_; } +}; + +// Test that data can be sent end-to-end when an SCTP transport starts with one +// transport (which is unwritable), and then switches to another transport. A +// common scenario due to how BUNDLE works. +TEST_F(SctpTransportTest, SwitchDtlsTransport) { + FakeDtlsTransport black_hole("black hole", 0); + FakeDtlsTransport fake_dtls1("fake dtls 1", 0); + FakeDtlsTransport fake_dtls2("fake dtls 2", 0); + SctpFakeDataReceiver recv1; + SctpFakeDataReceiver recv2; + + // Construct transport1 with the "black hole" transport. + std::unique_ptr<SctpTransport> transport1( + CreateTransport(&black_hole, &recv1)); + std::unique_ptr<SctpTransport> transport2( + CreateTransport(&fake_dtls2, &recv2)); + + // Add a stream. + transport1->OpenStream(1); + transport2->OpenStream(1); + + // Tell them both to start (though transport1_ is connected to black_hole). + transport1->Start(kTransport1Port, kTransport2Port, kSctpSendBufferSize); + transport2->Start(kTransport2Port, kTransport1Port, kSctpSendBufferSize); + + // Switch transport1_ to the normal fake_dtls1_ transport. + transport1->SetDtlsTransport(&fake_dtls1); + + // Connect the two fake DTLS transports. + bool asymmetric = false; + fake_dtls1.SetDestination(&fake_dtls2, asymmetric); + + // Make sure we end up able to send data. + SendDataResult result; + ASSERT_TRUE(SendData(transport1.get(), 1, "foo", &result)); + ASSERT_TRUE(SendData(transport2.get(), 1, "bar", &result)); + EXPECT_TRUE_WAIT(ReceivedData(&recv2, 1, "foo"), kDefaultTimeout); + EXPECT_TRUE_WAIT(ReceivedData(&recv1, 1, "bar"), kDefaultTimeout); + + // Setting a null DtlsTransport should work. This could happen when an SCTP + // data section is rejected. + transport1->SetDtlsTransport(nullptr); +} + +// Calling Start twice shouldn't do anything bad, if with the same parameters. +TEST_F(SctpTransportTest, DuplicateStartCallsIgnored) { + SetupConnectedTransportsWithTwoStreams(); + EXPECT_TRUE(transport1()->Start(kTransport1Port, kTransport2Port, + kSctpSendBufferSize)); + + // Make sure we can still send/recv data. + SendDataResult result; + ASSERT_TRUE(SendData(transport1(), 1, "foo", &result)); + ASSERT_TRUE(SendData(transport2(), 1, "bar", &result)); + EXPECT_TRUE_WAIT(ReceivedData(receiver2(), 1, "foo"), kDefaultTimeout); + EXPECT_TRUE_WAIT(ReceivedData(receiver1(), 1, "bar"), kDefaultTimeout); +} + +// Calling Start a second time with a different port should fail. +TEST_F(SctpTransportTest, CallingStartWithDifferentPortFails) { + SetupConnectedTransportsWithTwoStreams(); + EXPECT_FALSE(transport1()->Start(kTransport1Port, 1234, kSctpSendBufferSize)); + EXPECT_FALSE(transport1()->Start(1234, kTransport2Port, kSctpSendBufferSize)); +} + +// A value of -1 for the local/remote port should be treated as the default +// (5000). +TEST_F(SctpTransportTest, NegativeOnePortTreatedAsDefault) { + FakeDtlsTransport fake_dtls1("fake dtls 1", 0); + FakeDtlsTransport fake_dtls2("fake dtls 2", 0); + SctpFakeDataReceiver recv1; + SctpFakeDataReceiver recv2; + std::unique_ptr<SctpTransport> transport1( + CreateTransport(&fake_dtls1, &recv1)); + std::unique_ptr<SctpTransport> transport2( + CreateTransport(&fake_dtls2, &recv2)); + + // Add a stream. + transport1->OpenStream(1); + transport2->OpenStream(1); + + // Tell them both to start, giving one transport the default port and the + // other transport -1. + transport1->Start(kSctpDefaultPort, kSctpDefaultPort, kSctpSendBufferSize); + transport2->Start(-1, -1, kSctpSendBufferSize); + + // Connect the two fake DTLS transports. + bool asymmetric = false; + fake_dtls1.SetDestination(&fake_dtls2, asymmetric); + + // Make sure we end up able to send data. + SendDataResult result; + ASSERT_TRUE(SendData(transport1.get(), 1, "foo", &result)); + ASSERT_TRUE(SendData(transport2.get(), 1, "bar", &result)); + EXPECT_TRUE_WAIT(ReceivedData(&recv2, 1, "foo"), kDefaultTimeout); + EXPECT_TRUE_WAIT(ReceivedData(&recv1, 1, "bar"), kDefaultTimeout); +} + +TEST_F(SctpTransportTest, OpenStreamWithAlreadyOpenedStreamFails) { + FakeDtlsTransport fake_dtls("fake dtls", 0); + SctpFakeDataReceiver recv; + std::unique_ptr<SctpTransport> transport(CreateTransport(&fake_dtls, &recv)); + EXPECT_TRUE(transport->OpenStream(1)); + EXPECT_FALSE(transport->OpenStream(1)); +} + +TEST_F(SctpTransportTest, ResetStreamWithAlreadyResetStreamFails) { + FakeDtlsTransport fake_dtls("fake dtls", 0); + SctpFakeDataReceiver recv; + std::unique_ptr<SctpTransport> transport(CreateTransport(&fake_dtls, &recv)); + EXPECT_TRUE(transport->OpenStream(1)); + EXPECT_TRUE(transport->ResetStream(1)); + EXPECT_FALSE(transport->ResetStream(1)); +} + +// Test that SignalReadyToSendData is fired after Start has been called and the +// DTLS transport is writable. +TEST_F(SctpTransportTest, SignalReadyToSendDataAfterDtlsWritable) { + FakeDtlsTransport fake_dtls("fake dtls", 0); + SctpFakeDataReceiver recv; + std::unique_ptr<SctpTransport> transport(CreateTransport(&fake_dtls, &recv)); + SctpTransportObserver observer(transport.get()); + + transport->Start(kSctpDefaultPort, kSctpDefaultPort, kSctpSendBufferSize); + fake_dtls.SetWritable(true); + EXPECT_TRUE_WAIT(observer.ReadyToSend(), kDefaultTimeout); +} + +// Run the below tests using both ordered and unordered mode. +class SctpTransportTestWithOrdered + : public SctpTransportTest, + public ::testing::WithParamInterface<bool> {}; + +// Tests that a small message gets buffered and later sent by the SctpTransport +// when the sctp library only accepts the message partially. +TEST_P(SctpTransportTestWithOrdered, SendSmallBufferedOutgoingMessage) { + bool ordered = GetParam(); + SetupConnectedTransportsWithTwoStreams(); + // Wait for initial SCTP association to be formed. + EXPECT_EQ_WAIT(1, transport1_ready_to_send_count(), kDefaultTimeout); + // Make the fake transport unwritable so that messages pile up for the SCTP + // socket. + fake_dtls1()->SetWritable(false); + SendDataResult result; + + // Fill almost all of sctp library's send buffer. + ASSERT_TRUE(SendData(transport1(), /*sid=*/1, + std::string(kSctpSendBufferSize - 1, 'a'), &result, + ordered)); + + std::string buffered_message("hello hello"); + // SctpTransport accepts this message by buffering part of it. + ASSERT_TRUE( + SendData(transport1(), /*sid=*/1, buffered_message, &result, ordered)); + ASSERT_TRUE(transport1()->ReadyToSendData()); + + // Sending anything else should block now. + ASSERT_FALSE( + SendData(transport1(), /*sid=*/1, "hello again", &result, ordered)); + ASSERT_EQ(SDR_BLOCK, result); + ASSERT_FALSE(transport1()->ReadyToSendData()); + + // Make sure the ready-to-send count hasn't changed. + EXPECT_EQ(1, transport1_ready_to_send_count()); + // Make the transport writable again and expect a "SignalReadyToSendData" at + // some point after sending the buffered message. + fake_dtls1()->SetWritable(true); + EXPECT_EQ_WAIT(2, transport1_ready_to_send_count(), kDefaultTimeout); + EXPECT_TRUE_WAIT(ReceivedData(receiver2(), 1, buffered_message), + kDefaultTimeout); + EXPECT_EQ(2u, receiver2()->num_messages_received()); +} + +// Tests that a large message gets buffered and later sent by the SctpTransport +// when the sctp library only accepts the message partially. +TEST_P(SctpTransportTestWithOrdered, SendLargeBufferedOutgoingMessage) { + bool ordered = GetParam(); + SetupConnectedTransportsWithTwoStreams(); + // Wait for initial SCTP association to be formed. + EXPECT_EQ_WAIT(1, transport1_ready_to_send_count(), kDefaultTimeout); + // Make the fake transport unwritable so that messages pile up for the SCTP + // socket. + fake_dtls1()->SetWritable(false); + SendDataResult result; + + // Fill almost all of sctp library's send buffer. + ASSERT_TRUE(SendData(transport1(), /*sid=*/1, + std::string(kSctpSendBufferSize / 2, 'a'), &result, + ordered)); + + std::string buffered_message(kSctpSendBufferSize, 'b'); + // SctpTransport accepts this message by buffering the second half. + ASSERT_TRUE( + SendData(transport1(), /*sid=*/1, buffered_message, &result, ordered)); + ASSERT_TRUE(transport1()->ReadyToSendData()); + + // Sending anything else should block now. + ASSERT_FALSE( + SendData(transport1(), /*sid=*/1, "hello again", &result, ordered)); + ASSERT_EQ(SDR_BLOCK, result); + ASSERT_FALSE(transport1()->ReadyToSendData()); + + // Make sure the ready-to-send count hasn't changed. + EXPECT_EQ(1, transport1_ready_to_send_count()); + // Make the transport writable again and expect a "SignalReadyToSendData" at + // some point. + fake_dtls1()->SetWritable(true); + EXPECT_EQ_WAIT(2, transport1_ready_to_send_count(), kDefaultTimeout); + EXPECT_TRUE_WAIT(ReceivedData(receiver2(), 1, buffered_message), + kDefaultTimeout); + EXPECT_EQ(2u, receiver2()->num_messages_received()); +} + +TEST_P(SctpTransportTestWithOrdered, SendData) { + bool ordered = GetParam(); + SetupConnectedTransportsWithTwoStreams(); + + SendDataResult result; + RTC_LOG(LS_VERBOSE) + << "transport1 sending: 'hello?' -----------------------------"; + ASSERT_TRUE(SendData(transport1(), 1, "hello?", &result, ordered)); + EXPECT_EQ(SDR_SUCCESS, result); + EXPECT_TRUE_WAIT(ReceivedData(receiver2(), 1, "hello?"), kDefaultTimeout); + RTC_LOG(LS_VERBOSE) << "recv2.received=" << receiver2()->received() + << ", recv2.last_params.sid=" + << receiver2()->last_params().sid + << ", recv2.last_params.timestamp=" + << receiver2()->last_params().timestamp + << ", recv2.last_params.seq_num=" + << receiver2()->last_params().seq_num + << ", recv2.last_data=" << receiver2()->last_data(); + + RTC_LOG(LS_VERBOSE) + << "transport2 sending: 'hi transport1' -----------------------------"; + ASSERT_TRUE(SendData(transport2(), 2, "hi transport1", &result, ordered)); + EXPECT_EQ(SDR_SUCCESS, result); + EXPECT_TRUE_WAIT(ReceivedData(receiver1(), 2, "hi transport1"), + kDefaultTimeout); + RTC_LOG(LS_VERBOSE) << "recv1.received=" << receiver1()->received() + << ", recv1.last_params.sid=" + << receiver1()->last_params().sid + << ", recv1.last_params.timestamp=" + << receiver1()->last_params().timestamp + << ", recv1.last_params.seq_num=" + << receiver1()->last_params().seq_num + << ", recv1.last_data=" << receiver1()->last_data(); +} + +// Sends a lot of large messages at once and verifies SDR_BLOCK is returned. +TEST_P(SctpTransportTestWithOrdered, SendDataBlocked) { + SetupConnectedTransportsWithTwoStreams(); + + SendDataResult result; + SendDataParams params; + params.sid = 1; + params.ordered = GetParam(); + + std::vector<char> buffer(1024 * 64, 0); + + for (size_t i = 0; i < 100; ++i) { + transport1()->SendData( + params, rtc::CopyOnWriteBuffer(&buffer[0], buffer.size()), &result); + if (result == SDR_BLOCK) + break; + } + + EXPECT_EQ(SDR_BLOCK, result); +} + +// Test that after an SCTP socket's buffer is filled, SignalReadyToSendData +// is fired after it begins to be drained. +TEST_P(SctpTransportTestWithOrdered, SignalReadyToSendDataAfterBlocked) { + SetupConnectedTransportsWithTwoStreams(); + // Wait for initial SCTP association to be formed. + EXPECT_EQ_WAIT(1, transport1_ready_to_send_count(), kDefaultTimeout); + // Make the fake transport unwritable so that messages pile up for the SCTP + // socket. + fake_dtls1()->SetWritable(false); + // Send messages until we get EWOULDBLOCK. + static const size_t kMaxMessages = 1024; + SendDataParams params; + params.sid = 1; + params.ordered = GetParam(); + rtc::CopyOnWriteBuffer buf(1024); + memset(buf.data<uint8_t>(), 0, 1024); + SendDataResult result; + size_t message_count = 0; + for (; message_count < kMaxMessages; ++message_count) { + if (!transport1()->SendData(params, buf, &result) && result == SDR_BLOCK) { + break; + } + } + ASSERT_NE(kMaxMessages, message_count) + << "Sent max number of messages without getting SDR_BLOCK?"; + // Make sure the ready-to-send count hasn't changed. + EXPECT_EQ(1, transport1_ready_to_send_count()); + // Make the transport writable again and expect a "SignalReadyToSendData" at + // some point. + fake_dtls1()->SetWritable(true); + EXPECT_EQ_WAIT(2, transport1_ready_to_send_count(), kDefaultTimeout); + EXPECT_EQ_WAIT(message_count, receiver2()->num_messages_received(), + kDefaultTimeout); +} + +INSTANTIATE_TEST_SUITE_P(SctpTransportTest, + SctpTransportTestWithOrdered, + ::testing::Bool()); + +// This is a regression test that fails with earlier versions of SCTP in +// unordered mode. See bugs.webrtc.org/10939. +TEST_F(SctpTransportTest, SendsLargeDataBufferedBySctpLib) { + SetupConnectedTransportsWithTwoStreams(); + // Wait for initial SCTP association to be formed. + EXPECT_EQ_WAIT(1, transport1_ready_to_send_count(), kDefaultTimeout); + // Make the fake transport unwritable so that messages pile up for the SCTP + // socket. + fake_dtls1()->SetWritable(false); + + SendDataResult result; + std::string buffered_message(kSctpSendBufferSize - 1, 'a'); + ASSERT_TRUE(SendData(transport1(), 1, buffered_message, &result, false)); + + fake_dtls1()->SetWritable(true); + EXPECT_EQ_WAIT(1, transport1_ready_to_send_count(), kDefaultTimeout); + EXPECT_TRUE_WAIT(ReceivedData(receiver2(), 1, buffered_message), + kDefaultTimeout); +} + +// Trying to send data for a nonexistent stream should fail. +TEST_F(SctpTransportTest, SendDataWithNonexistentStreamFails) { + SetupConnectedTransportsWithTwoStreams(); + SendDataResult result; + EXPECT_FALSE(SendData(transport2(), 123, "some data", &result)); + EXPECT_EQ(SDR_ERROR, result); +} + +TEST_F(SctpTransportTest, SendDataHighPorts) { + SetupConnectedTransportsWithTwoStreams(32768, 32769); + + SendDataResult result; + ASSERT_TRUE(SendData(transport1(), 1, "hello?", &result)); + EXPECT_EQ(SDR_SUCCESS, result); + EXPECT_TRUE_WAIT(ReceivedData(receiver2(), 1, "hello?"), kDefaultTimeout); + + ASSERT_TRUE(SendData(transport2(), 2, "hi transport1", &result)); + EXPECT_EQ(SDR_SUCCESS, result); + EXPECT_TRUE_WAIT(ReceivedData(receiver1(), 2, "hi transport1"), + kDefaultTimeout); +} + +TEST_F(SctpTransportTest, ClosesRemoteStream) { + SetupConnectedTransportsWithTwoStreams(); + SctpTransportObserver transport1_observer(transport1()); + SctpTransportObserver transport2_observer(transport2()); + + SendDataResult result; + ASSERT_TRUE(SendData(transport1(), 1, "hello?", &result)); + EXPECT_EQ(SDR_SUCCESS, result); + EXPECT_TRUE_WAIT(ReceivedData(receiver2(), 1, "hello?"), kDefaultTimeout); + ASSERT_TRUE(SendData(transport2(), 2, "hi transport1", &result)); + EXPECT_EQ(SDR_SUCCESS, result); + EXPECT_TRUE_WAIT(ReceivedData(receiver1(), 2, "hi transport1"), + kDefaultTimeout); + + // Close stream 1 on transport 1. Transport 2 should notify us. + transport1()->ResetStream(1); + EXPECT_TRUE_WAIT(transport2_observer.WasStreamClosed(1), kDefaultTimeout); +} + +TEST_F(SctpTransportTest, ClosesTwoRemoteStreams) { + SetupConnectedTransportsWithTwoStreams(); + AddStream(3); + SctpTransportObserver transport1_observer(transport1()); + SctpTransportObserver transport2_observer(transport2()); + + SendDataResult result; + ASSERT_TRUE(SendData(transport1(), 1, "hello?", &result)); + EXPECT_EQ(SDR_SUCCESS, result); + EXPECT_TRUE_WAIT(ReceivedData(receiver2(), 1, "hello?"), kDefaultTimeout); + ASSERT_TRUE(SendData(transport2(), 2, "hi transport1", &result)); + EXPECT_EQ(SDR_SUCCESS, result); + EXPECT_TRUE_WAIT(ReceivedData(receiver1(), 2, "hi transport1"), + kDefaultTimeout); + + // Close two streams on one side. + transport2()->ResetStream(2); + transport2()->ResetStream(3); + EXPECT_TRUE_WAIT(transport2_observer.WasStreamClosed(2), kDefaultTimeout); + EXPECT_TRUE_WAIT(transport2_observer.WasStreamClosed(3), kDefaultTimeout); +} + +TEST_F(SctpTransportTest, ClosesStreamsOnBothSides) { + SetupConnectedTransportsWithTwoStreams(); + AddStream(3); + AddStream(4); + SctpTransportObserver transport1_observer(transport1()); + SctpTransportObserver transport2_observer(transport2()); + + SendDataResult result; + ASSERT_TRUE(SendData(transport1(), 1, "hello?", &result)); + EXPECT_EQ(SDR_SUCCESS, result); + EXPECT_TRUE_WAIT(ReceivedData(receiver2(), 1, "hello?"), kDefaultTimeout); + ASSERT_TRUE(SendData(transport2(), 2, "hi transport1", &result)); + EXPECT_EQ(SDR_SUCCESS, result); + EXPECT_TRUE_WAIT(ReceivedData(receiver1(), 2, "hi transport1"), + kDefaultTimeout); + + // Close one stream on transport1(), while closing three streams on + // transport2(). They will conflict (only one side can close anything at a + // time, apparently). Test the resolution of the conflict. + transport1()->ResetStream(1); + + transport2()->ResetStream(2); + transport2()->ResetStream(3); + transport2()->ResetStream(4); + EXPECT_TRUE_WAIT(transport2_observer.WasStreamClosed(1), kDefaultTimeout); + EXPECT_TRUE_WAIT(transport1_observer.WasStreamClosed(2), kDefaultTimeout); + EXPECT_TRUE_WAIT(transport1_observer.WasStreamClosed(3), kDefaultTimeout); + EXPECT_TRUE_WAIT(transport1_observer.WasStreamClosed(4), kDefaultTimeout); +} + +TEST_F(SctpTransportTest, RefusesHighNumberedTransports) { + SetupConnectedTransportsWithTwoStreams(); + EXPECT_TRUE(AddStream(kMaxSctpSid)); + EXPECT_FALSE(AddStream(kMaxSctpSid + 1)); +} + +TEST_F(SctpTransportTest, ReusesAStream) { + // Shut down transport 1, then open it up again for reuse. + SetupConnectedTransportsWithTwoStreams(); + SendDataResult result; + SctpTransportObserver transport2_observer(transport2()); + + ASSERT_TRUE(SendData(transport1(), 1, "hello?", &result)); + EXPECT_EQ(SDR_SUCCESS, result); + EXPECT_TRUE_WAIT(ReceivedData(receiver2(), 1, "hello?"), kDefaultTimeout); + + transport1()->ResetStream(1); + EXPECT_TRUE_WAIT(transport2_observer.WasStreamClosed(1), kDefaultTimeout); + // Transport 1 is gone now. + + // Create a new transport 1. + AddStream(1); + ASSERT_TRUE(SendData(transport1(), 1, "hi?", &result)); + EXPECT_EQ(SDR_SUCCESS, result); + EXPECT_TRUE_WAIT(ReceivedData(receiver2(), 1, "hi?"), kDefaultTimeout); + transport1()->ResetStream(1); + EXPECT_EQ_WAIT(2, transport2_observer.StreamCloseCount(1), kDefaultTimeout); +} + +TEST_F(SctpTransportTest, RejectsTooLargeMessageSize) { + FakeDtlsTransport fake_dtls("fake dtls", 0); + SctpFakeDataReceiver recv; + std::unique_ptr<SctpTransport> transport(CreateTransport(&fake_dtls, &recv)); + + EXPECT_FALSE(transport->Start(kSctpDefaultPort, kSctpDefaultPort, + kSctpSendBufferSize + 1)); +} + +TEST_F(SctpTransportTest, RejectsTooSmallMessageSize) { + FakeDtlsTransport fake_dtls("fake dtls", 0); + SctpFakeDataReceiver recv; + std::unique_ptr<SctpTransport> transport(CreateTransport(&fake_dtls, &recv)); + + EXPECT_FALSE(transport->Start(kSctpDefaultPort, kSctpDefaultPort, 0)); +} + +TEST_F(SctpTransportTest, RejectsSendTooLargeMessages) { + SetupConnectedTransportsWithTwoStreams(); + // Use "Start" to reduce the max message size + transport1()->Start(kTransport1Port, kTransport2Port, 10); + EXPECT_EQ(10, transport1()->max_message_size()); + const char eleven_characters[] = "12345678901"; + SendDataResult result; + EXPECT_FALSE(SendData(transport1(), 1, eleven_characters, &result)); +} + +} // namespace cricket |