diff options
author | philipel <philipel@webrtc.org> | 2022-05-30 12:41:30 +0200 |
---|---|---|
committer | WebRTC LUCI CQ <webrtc-scoped@luci-project-accounts.iam.gserviceaccount.com> | 2022-06-10 09:41:54 +0000 |
commit | da12e10aba4d12e7a6fb3882dc667901c9e17aa2 (patch) | |
tree | 64a4630254c8903fec36d788e8a06420d5e78d49 | |
parent | 6cbb8f69077e70c76712ba5d0dcc53a984e0b9ad (diff) | |
download | webrtc-da12e10aba4d12e7a6fb3882dc667901c9e17aa2.tar.gz |
Remove legacy RtpVideoStreamReceiver.
Bug: none
Change-Id: I434a56980f4d6c68381abae973cd846c71441b08
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/236846
Reviewed-by: Erik Språng <sprang@webrtc.org>
Commit-Queue: Philip Eliasson <philipel@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#37177}
-rw-r--r-- | video/BUILD.gn | 3 | ||||
-rw-r--r-- | video/rtp_video_stream_receiver.cc | 1278 | ||||
-rw-r--r-- | video/rtp_video_stream_receiver.h | 436 | ||||
-rw-r--r-- | video/rtp_video_stream_receiver_unittest.cc | 1238 |
4 files changed, 0 insertions, 2955 deletions
diff --git a/video/BUILD.gn b/video/BUILD.gn index 74eb3bbe2d..737e02853b 100644 --- a/video/BUILD.gn +++ b/video/BUILD.gn @@ -172,8 +172,6 @@ rtc_source_set("video_legacy") { "call_stats.h", "receive_statistics_proxy.cc", "receive_statistics_proxy.h", - "rtp_video_stream_receiver.cc", - "rtp_video_stream_receiver.h", "video_quality_observer.cc", "video_quality_observer.h", ] @@ -1007,7 +1005,6 @@ if (rtc_include_tests) { sources = [ "call_stats_unittest.cc", "receive_statistics_proxy_unittest.cc", - "rtp_video_stream_receiver_unittest.cc", ] deps = [ ":video_legacy", diff --git a/video/rtp_video_stream_receiver.cc b/video/rtp_video_stream_receiver.cc deleted file mode 100644 index b09e5fc5a3..0000000000 --- a/video/rtp_video_stream_receiver.cc +++ /dev/null @@ -1,1278 +0,0 @@ -/* - * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "video/rtp_video_stream_receiver.h" - -#include <algorithm> -#include <limits> -#include <memory> -#include <utility> -#include <vector> - -#include "absl/algorithm/container.h" -#include "absl/memory/memory.h" -#include "absl/types/optional.h" -#include "api/field_trials_view.h" -#include "media/base/media_constants.h" -#include "modules/pacing/packet_router.h" -#include "modules/remote_bitrate_estimator/include/remote_bitrate_estimator.h" -#include "modules/rtp_rtcp/include/receive_statistics.h" -#include "modules/rtp_rtcp/include/rtp_cvo.h" -#include "modules/rtp_rtcp/include/ulpfec_receiver.h" -#include "modules/rtp_rtcp/source/create_video_rtp_depacketizer.h" -#include "modules/rtp_rtcp/source/rtp_dependency_descriptor_extension.h" -#include "modules/rtp_rtcp/source/rtp_format.h" -#include "modules/rtp_rtcp/source/rtp_generic_frame_descriptor.h" -#include "modules/rtp_rtcp/source/rtp_generic_frame_descriptor_extension.h" -#include "modules/rtp_rtcp/source/rtp_header_extensions.h" -#include "modules/rtp_rtcp/source/rtp_packet_received.h" -#include "modules/rtp_rtcp/source/rtp_rtcp_config.h" -#include "modules/rtp_rtcp/source/rtp_rtcp_impl2.h" -#include "modules/rtp_rtcp/source/video_rtp_depacketizer.h" -#include "modules/rtp_rtcp/source/video_rtp_depacketizer_raw.h" -#include "modules/utility/include/process_thread.h" -#include "modules/video_coding/deprecated/nack_module.h" -#include "modules/video_coding/frame_object.h" -#include "modules/video_coding/h264_sprop_parameter_sets.h" -#include "modules/video_coding/h264_sps_pps_tracker.h" -#include "modules/video_coding/packet_buffer.h" -#include "rtc_base/checks.h" -#include "rtc_base/location.h" -#include "rtc_base/logging.h" -#include "rtc_base/strings/string_builder.h" -#include "system_wrappers/include/metrics.h" -#include "system_wrappers/include/ntp_time.h" -#include "video/receive_statistics_proxy.h" - -namespace webrtc { - -namespace { -// TODO(philipel): Change kPacketBufferStartSize back to 32 in M63 see: -// crbug.com/752886 -constexpr int kPacketBufferStartSize = 512; -constexpr int kPacketBufferMaxSize = 2048; - -int PacketBufferMaxSize(const FieldTrialsView& field_trials) { - // The group here must be a positive power of 2, in which case that is used as - // size. All other values shall result in the default value being used. - const std::string group_name = - field_trials.Lookup("WebRTC-PacketBufferMaxSize"); - int packet_buffer_max_size = kPacketBufferMaxSize; - if (!group_name.empty() && - (sscanf(group_name.c_str(), "%d", &packet_buffer_max_size) != 1 || - packet_buffer_max_size <= 0 || - // Verify that the number is a positive power of 2. - (packet_buffer_max_size & (packet_buffer_max_size - 1)) != 0)) { - RTC_LOG(LS_WARNING) << "Invalid packet buffer max size: " << group_name; - packet_buffer_max_size = kPacketBufferMaxSize; - } - return packet_buffer_max_size; -} - -std::unique_ptr<RtpRtcp> CreateRtpRtcpModule( - Clock* clock, - ReceiveStatistics* receive_statistics, - Transport* outgoing_transport, - RtcpRttStats* rtt_stats, - RtcpPacketTypeCounterObserver* rtcp_packet_type_counter_observer, - RtcpCnameCallback* rtcp_cname_callback, - bool non_sender_rtt_measurement, - uint32_t local_ssrc) { - RtpRtcpInterface::Configuration configuration; - configuration.clock = clock; - configuration.audio = false; - configuration.receiver_only = true; - configuration.receive_statistics = receive_statistics; - configuration.outgoing_transport = outgoing_transport; - configuration.rtt_stats = rtt_stats; - configuration.rtcp_packet_type_counter_observer = - rtcp_packet_type_counter_observer; - configuration.rtcp_cname_callback = rtcp_cname_callback; - configuration.local_media_ssrc = local_ssrc; - configuration.non_sender_rtt_measurement = non_sender_rtt_measurement; - - std::unique_ptr<RtpRtcp> rtp_rtcp = RtpRtcp::DEPRECATED_Create(configuration); - rtp_rtcp->SetRTCPStatus(RtcpMode::kCompound); - - return rtp_rtcp; -} - -static const int kPacketLogIntervalMs = 10000; - -} // namespace - -RtpVideoStreamReceiver::RtcpFeedbackBuffer::RtcpFeedbackBuffer( - KeyFrameRequestSender* key_frame_request_sender, - NackSender* nack_sender, - LossNotificationSender* loss_notification_sender) - : key_frame_request_sender_(key_frame_request_sender), - nack_sender_(nack_sender), - loss_notification_sender_(loss_notification_sender), - request_key_frame_(false) { - RTC_DCHECK(key_frame_request_sender_); - RTC_DCHECK(nack_sender_); - RTC_DCHECK(loss_notification_sender_); -} - -void RtpVideoStreamReceiver::RtcpFeedbackBuffer::RequestKeyFrame() { - MutexLock lock(&mutex_); - request_key_frame_ = true; -} - -void RtpVideoStreamReceiver::RtcpFeedbackBuffer::SendNack( - const std::vector<uint16_t>& sequence_numbers, - bool buffering_allowed) { - RTC_DCHECK(!sequence_numbers.empty()); - MutexLock lock(&mutex_); - nack_sequence_numbers_.insert(nack_sequence_numbers_.end(), - sequence_numbers.cbegin(), - sequence_numbers.cend()); - if (!buffering_allowed) { - // Note that while *buffering* is not allowed, *batching* is, meaning that - // previously buffered messages may be sent along with the current message. - SendRtcpFeedback(ConsumeRtcpFeedbackLocked()); - } -} - -void RtpVideoStreamReceiver::RtcpFeedbackBuffer::SendLossNotification( - uint16_t last_decoded_seq_num, - uint16_t last_received_seq_num, - bool decodability_flag, - bool buffering_allowed) { - RTC_DCHECK(buffering_allowed); - MutexLock lock(&mutex_); - RTC_DCHECK(!lntf_state_) - << "SendLossNotification() called twice in a row with no call to " - "SendBufferedRtcpFeedback() in between."; - lntf_state_ = absl::make_optional<LossNotificationState>( - last_decoded_seq_num, last_received_seq_num, decodability_flag); -} - -void RtpVideoStreamReceiver::RtcpFeedbackBuffer::SendBufferedRtcpFeedback() { - SendRtcpFeedback(ConsumeRtcpFeedback()); -} - -RtpVideoStreamReceiver::RtcpFeedbackBuffer::ConsumedRtcpFeedback -RtpVideoStreamReceiver::RtcpFeedbackBuffer::ConsumeRtcpFeedback() { - MutexLock lock(&mutex_); - return ConsumeRtcpFeedbackLocked(); -} - -RtpVideoStreamReceiver::RtcpFeedbackBuffer::ConsumedRtcpFeedback -RtpVideoStreamReceiver::RtcpFeedbackBuffer::ConsumeRtcpFeedbackLocked() { - ConsumedRtcpFeedback feedback; - std::swap(feedback.request_key_frame, request_key_frame_); - std::swap(feedback.nack_sequence_numbers, nack_sequence_numbers_); - std::swap(feedback.lntf_state, lntf_state_); - return feedback; -} - -void RtpVideoStreamReceiver::RtcpFeedbackBuffer::SendRtcpFeedback( - ConsumedRtcpFeedback feedback) { - if (feedback.lntf_state) { - // If either a NACK or a key frame request is sent, we should buffer - // the LNTF and wait for them (NACK or key frame request) to trigger - // the compound feedback message. - // Otherwise, the LNTF should be sent out immediately. - const bool buffering_allowed = - feedback.request_key_frame || !feedback.nack_sequence_numbers.empty(); - - loss_notification_sender_->SendLossNotification( - feedback.lntf_state->last_decoded_seq_num, - feedback.lntf_state->last_received_seq_num, - feedback.lntf_state->decodability_flag, buffering_allowed); - } - - if (feedback.request_key_frame) { - key_frame_request_sender_->RequestKeyFrame(); - } else if (!feedback.nack_sequence_numbers.empty()) { - nack_sender_->SendNack(feedback.nack_sequence_numbers, true); - } -} - -// DEPRECATED -RtpVideoStreamReceiver::RtpVideoStreamReceiver( - Clock* clock, - Transport* transport, - RtcpRttStats* rtt_stats, - PacketRouter* packet_router, - const VideoReceiveStreamInterface::Config* config, - ReceiveStatistics* rtp_receive_statistics, - ReceiveStatisticsProxy* receive_stats_proxy, - ProcessThread* process_thread, - NackSender* nack_sender, - KeyFrameRequestSender* keyframe_request_sender, - OnCompleteFrameCallback* complete_frame_callback, - rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor, - rtc::scoped_refptr<FrameTransformerInterface> frame_transformer, - const FieldTrialsView* field_trials) - : RtpVideoStreamReceiver(clock, - transport, - rtt_stats, - packet_router, - config, - rtp_receive_statistics, - receive_stats_proxy, - receive_stats_proxy, - process_thread, - nack_sender, - keyframe_request_sender, - complete_frame_callback, - frame_decryptor, - frame_transformer, - field_trials) {} - -RtpVideoStreamReceiver::RtpVideoStreamReceiver( - Clock* clock, - Transport* transport, - RtcpRttStats* rtt_stats, - PacketRouter* packet_router, - const VideoReceiveStreamInterface::Config* config, - ReceiveStatistics* rtp_receive_statistics, - RtcpPacketTypeCounterObserver* rtcp_packet_type_counter_observer, - RtcpCnameCallback* rtcp_cname_callback, - ProcessThread* process_thread, - NackSender* nack_sender, - KeyFrameRequestSender* keyframe_request_sender, - OnCompleteFrameCallback* complete_frame_callback, - rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor, - rtc::scoped_refptr<FrameTransformerInterface> frame_transformer, - const FieldTrialsView* field_trials) - : field_trials_(field_trials ? *field_trials : owned_field_trials_), - clock_(clock), - config_(*config), - packet_router_(packet_router), - process_thread_(process_thread), - ntp_estimator_(clock), - rtp_header_extensions_(config_.rtp.extensions), - forced_playout_delay_max_ms_("max_ms", absl::nullopt), - forced_playout_delay_min_ms_("min_ms", absl::nullopt), - rtp_receive_statistics_(rtp_receive_statistics), - ulpfec_receiver_(UlpfecReceiver::Create(config->rtp.remote_ssrc, - this, - config->rtp.extensions)), - receiving_(false), - last_packet_log_ms_(-1), - rtp_rtcp_(CreateRtpRtcpModule( - clock, - rtp_receive_statistics_, - transport, - rtt_stats, - rtcp_packet_type_counter_observer, - rtcp_cname_callback, - config_.rtp.rtcp_xr.receiver_reference_time_report, - config_.rtp.local_ssrc)), - complete_frame_callback_(complete_frame_callback), - keyframe_request_sender_(keyframe_request_sender), - // TODO(bugs.webrtc.org/10336): Let `rtcp_feedback_buffer_` communicate - // directly with `rtp_rtcp_`. - rtcp_feedback_buffer_(this, nack_sender, this), - packet_buffer_(kPacketBufferStartSize, - PacketBufferMaxSize(field_trials_)), - reference_finder_(std::make_unique<RtpFrameReferenceFinder>()), - has_received_frame_(false), - frames_decryptable_(false), - absolute_capture_time_interpolator_(clock) { - constexpr bool remb_candidate = true; - if (packet_router_) - packet_router_->AddReceiveRtpModule(rtp_rtcp_.get(), remb_candidate); - - RTC_DCHECK(config_.rtp.rtcp_mode != RtcpMode::kOff) - << "A stream should not be configured with RTCP disabled. This value is " - "reserved for internal usage."; - // TODO(pbos): What's an appropriate local_ssrc for receive-only streams? - RTC_DCHECK(config_.rtp.local_ssrc != 0); - RTC_DCHECK(config_.rtp.remote_ssrc != config_.rtp.local_ssrc); - - rtp_rtcp_->SetRTCPStatus(config_.rtp.rtcp_mode); - rtp_rtcp_->SetRemoteSSRC(config_.rtp.remote_ssrc); - - static const int kMaxPacketAgeToNack = 450; - const int max_reordering_threshold = (config_.rtp.nack.rtp_history_ms > 0) - ? kMaxPacketAgeToNack - : kDefaultMaxReorderingThreshold; - rtp_receive_statistics_->SetMaxReorderingThreshold(config_.rtp.remote_ssrc, - max_reordering_threshold); - // TODO(nisse): For historic reasons, we applied the above - // max_reordering_threshold also for RTX stats, which makes little sense since - // we don't NACK rtx packets. Consider deleting the below block, and rely on - // the default threshold. - if (config_.rtp.rtx_ssrc) { - rtp_receive_statistics_->SetMaxReorderingThreshold( - config_.rtp.rtx_ssrc, max_reordering_threshold); - } - ParseFieldTrial( - {&forced_playout_delay_max_ms_, &forced_playout_delay_min_ms_}, - field_trials_.Lookup("WebRTC-ForcePlayoutDelay")); - - process_thread_->RegisterModule(rtp_rtcp_.get(), RTC_FROM_HERE); - - if (config_.rtp.lntf.enabled) { - loss_notification_controller_ = - std::make_unique<LossNotificationController>(&rtcp_feedback_buffer_, - &rtcp_feedback_buffer_); - } - - if (config_.rtp.nack.rtp_history_ms != 0) { - nack_module_ = std::make_unique<DEPRECATED_NackModule>( - clock_, &rtcp_feedback_buffer_, &rtcp_feedback_buffer_, field_trials_); - process_thread_->RegisterModule(nack_module_.get(), RTC_FROM_HERE); - } - - // Only construct the encrypted receiver if frame encryption is enabled. - if (config_.crypto_options.sframe.require_frame_encryption) { - buffered_frame_decryptor_ = - std::make_unique<BufferedFrameDecryptor>(this, this, field_trials_); - if (frame_decryptor != nullptr) { - buffered_frame_decryptor_->SetFrameDecryptor(std::move(frame_decryptor)); - } - } - - if (frame_transformer) { - frame_transformer_delegate_ = - rtc::make_ref_counted<RtpVideoStreamReceiverFrameTransformerDelegate>( - this, std::move(frame_transformer), rtc::Thread::Current(), - config_.rtp.remote_ssrc); - frame_transformer_delegate_->Init(); - } -} - -RtpVideoStreamReceiver::~RtpVideoStreamReceiver() { - RTC_DCHECK(secondary_sinks_.empty()); - - if (nack_module_) { - process_thread_->DeRegisterModule(nack_module_.get()); - } - - process_thread_->DeRegisterModule(rtp_rtcp_.get()); - - if (packet_router_) - packet_router_->RemoveReceiveRtpModule(rtp_rtcp_.get()); - UpdateHistograms(); - if (frame_transformer_delegate_) - frame_transformer_delegate_->Reset(); -} - -void RtpVideoStreamReceiver::AddReceiveCodec( - uint8_t payload_type, - VideoCodecType codec_type, - const std::map<std::string, std::string>& codec_params, - bool raw_payload) { - if (codec_params.count(cricket::kH264FmtpSpsPpsIdrInKeyframe) || - field_trials_.IsEnabled("WebRTC-SpsPpsIdrIsH264Keyframe")) { - MutexLock lock(&packet_buffer_lock_); - packet_buffer_.ForceSpsPpsIdrIsH264Keyframe(); - } - payload_type_map_.emplace( - payload_type, raw_payload ? std::make_unique<VideoRtpDepacketizerRaw>() - : CreateVideoRtpDepacketizer(codec_type)); - pt_codec_params_.emplace(payload_type, codec_params); -} - -absl::optional<Syncable::Info> RtpVideoStreamReceiver::GetSyncInfo() const { - Syncable::Info info; - if (rtp_rtcp_->RemoteNTP(&info.capture_time_ntp_secs, - &info.capture_time_ntp_frac, - /*rtcp_arrival_time_secs=*/nullptr, - /*rtcp_arrival_time_frac=*/nullptr, - &info.capture_time_source_clock) != 0) { - return absl::nullopt; - } - { - MutexLock lock(&sync_info_lock_); - if (!last_received_rtp_timestamp_ || !last_received_rtp_system_time_) { - return absl::nullopt; - } - info.latest_received_capture_timestamp = *last_received_rtp_timestamp_; - info.latest_receive_time_ms = last_received_rtp_system_time_->ms(); - } - - // Leaves info.current_delay_ms uninitialized. - return info; -} - -RtpVideoStreamReceiver::ParseGenericDependenciesResult -RtpVideoStreamReceiver::ParseGenericDependenciesExtension( - const RtpPacketReceived& rtp_packet, - RTPVideoHeader* video_header) { - if (rtp_packet.HasExtension<RtpDependencyDescriptorExtension>()) { - webrtc::DependencyDescriptor dependency_descriptor; - if (!rtp_packet.GetExtension<RtpDependencyDescriptorExtension>( - video_structure_.get(), &dependency_descriptor)) { - // Descriptor is there, but failed to parse. Either it is invalid, - // or too old packet (after relevant video_structure_ changed), - // or too new packet (before relevant video_structure_ arrived). - // Drop such packet to be on the safe side. - // TODO(bugs.webrtc.org/10342): Stash too new packet. - RTC_LOG(LS_WARNING) << "ssrc: " << rtp_packet.Ssrc() - << " Failed to parse dependency descriptor."; - return kDropPacket; - } - if (dependency_descriptor.attached_structure != nullptr && - !dependency_descriptor.first_packet_in_frame) { - RTC_LOG(LS_WARNING) << "ssrc: " << rtp_packet.Ssrc() - << "Invalid dependency descriptor: structure " - "attached to non first packet of a frame."; - return kDropPacket; - } - video_header->is_first_packet_in_frame = - dependency_descriptor.first_packet_in_frame; - video_header->is_last_packet_in_frame = - dependency_descriptor.last_packet_in_frame; - - int64_t frame_id = - frame_id_unwrapper_.Unwrap(dependency_descriptor.frame_number); - auto& generic_descriptor_info = video_header->generic.emplace(); - generic_descriptor_info.frame_id = frame_id; - generic_descriptor_info.spatial_index = - dependency_descriptor.frame_dependencies.spatial_id; - generic_descriptor_info.temporal_index = - dependency_descriptor.frame_dependencies.temporal_id; - for (int fdiff : dependency_descriptor.frame_dependencies.frame_diffs) { - generic_descriptor_info.dependencies.push_back(frame_id - fdiff); - } - generic_descriptor_info.decode_target_indications = - dependency_descriptor.frame_dependencies.decode_target_indications; - if (dependency_descriptor.resolution) { - video_header->width = dependency_descriptor.resolution->Width(); - video_header->height = dependency_descriptor.resolution->Height(); - } - - // FrameDependencyStructure is sent in dependency descriptor of the first - // packet of a key frame and required for parsed dependency descriptor in - // all the following packets until next key frame. - // Save it if there is a (potentially) new structure. - if (dependency_descriptor.attached_structure) { - RTC_DCHECK(dependency_descriptor.first_packet_in_frame); - if (video_structure_frame_id_ > frame_id) { - RTC_LOG(LS_WARNING) - << "Arrived key frame with id " << frame_id << " and structure id " - << dependency_descriptor.attached_structure->structure_id - << " is older than the latest received key frame with id " - << *video_structure_frame_id_ << " and structure id " - << video_structure_->structure_id; - return kDropPacket; - } - video_structure_ = std::move(dependency_descriptor.attached_structure); - video_structure_frame_id_ = frame_id; - video_header->frame_type = VideoFrameType::kVideoFrameKey; - } else { - video_header->frame_type = VideoFrameType::kVideoFrameDelta; - } - return kHasGenericDescriptor; - } - - RtpGenericFrameDescriptor generic_frame_descriptor; - if (!rtp_packet.GetExtension<RtpGenericFrameDescriptorExtension00>( - &generic_frame_descriptor)) { - return kNoGenericDescriptor; - } - - video_header->is_first_packet_in_frame = - generic_frame_descriptor.FirstPacketInSubFrame(); - video_header->is_last_packet_in_frame = - generic_frame_descriptor.LastPacketInSubFrame(); - - if (generic_frame_descriptor.FirstPacketInSubFrame()) { - video_header->frame_type = - generic_frame_descriptor.FrameDependenciesDiffs().empty() - ? VideoFrameType::kVideoFrameKey - : VideoFrameType::kVideoFrameDelta; - - auto& generic_descriptor_info = video_header->generic.emplace(); - int64_t frame_id = - frame_id_unwrapper_.Unwrap(generic_frame_descriptor.FrameId()); - generic_descriptor_info.frame_id = frame_id; - generic_descriptor_info.spatial_index = - generic_frame_descriptor.SpatialLayer(); - generic_descriptor_info.temporal_index = - generic_frame_descriptor.TemporalLayer(); - for (uint16_t fdiff : generic_frame_descriptor.FrameDependenciesDiffs()) { - generic_descriptor_info.dependencies.push_back(frame_id - fdiff); - } - } - video_header->width = generic_frame_descriptor.Width(); - video_header->height = generic_frame_descriptor.Height(); - return kHasGenericDescriptor; -} - -void RtpVideoStreamReceiver::OnReceivedPayloadData( - rtc::CopyOnWriteBuffer codec_payload, - const RtpPacketReceived& rtp_packet, - const RTPVideoHeader& video) { - RTC_DCHECK_RUN_ON(&worker_task_checker_); - - auto packet = - std::make_unique<video_coding::PacketBuffer::Packet>(rtp_packet, video); - - RTPVideoHeader& video_header = packet->video_header; - video_header.rotation = kVideoRotation_0; - video_header.content_type = VideoContentType::UNSPECIFIED; - video_header.video_timing.flags = VideoSendTiming::kInvalid; - video_header.is_last_packet_in_frame |= rtp_packet.Marker(); - - if (const auto* vp9_header = - absl::get_if<RTPVideoHeaderVP9>(&video_header.video_type_header)) { - video_header.is_last_packet_in_frame |= vp9_header->end_of_frame; - video_header.is_first_packet_in_frame |= vp9_header->beginning_of_frame; - } - - rtp_packet.GetExtension<VideoOrientation>(&video_header.rotation); - rtp_packet.GetExtension<VideoContentTypeExtension>( - &video_header.content_type); - rtp_packet.GetExtension<VideoTimingExtension>(&video_header.video_timing); - if (forced_playout_delay_max_ms_ && forced_playout_delay_min_ms_) { - video_header.playout_delay.max_ms = *forced_playout_delay_max_ms_; - video_header.playout_delay.min_ms = *forced_playout_delay_min_ms_; - } else { - rtp_packet.GetExtension<PlayoutDelayLimits>(&video_header.playout_delay); - } - - ParseGenericDependenciesResult generic_descriptor_state = - ParseGenericDependenciesExtension(rtp_packet, &video_header); - - if (!rtp_packet.recovered()) { - UpdatePacketReceiveTimestamps( - rtp_packet, video_header.frame_type == VideoFrameType::kVideoFrameKey); - } - - if (generic_descriptor_state == kDropPacket) - return; - - // Color space should only be transmitted in the last packet of a frame, - // therefore, neglect it otherwise so that last_color_space_ is not reset by - // mistake. - if (video_header.is_last_packet_in_frame) { - video_header.color_space = rtp_packet.GetExtension<ColorSpaceExtension>(); - if (video_header.color_space || - video_header.frame_type == VideoFrameType::kVideoFrameKey) { - // Store color space since it's only transmitted when changed or for key - // frames. Color space will be cleared if a key frame is transmitted - // without color space information. - last_color_space_ = video_header.color_space; - } else if (last_color_space_) { - video_header.color_space = last_color_space_; - } - } - video_header.video_frame_tracking_id = - rtp_packet.GetExtension<VideoFrameTrackingIdExtension>(); - - if (loss_notification_controller_) { - if (rtp_packet.recovered()) { - // TODO(bugs.webrtc.org/10336): Implement support for reordering. - RTC_LOG(LS_INFO) - << "LossNotificationController does not support reordering."; - } else if (generic_descriptor_state == kNoGenericDescriptor) { - RTC_LOG(LS_WARNING) << "LossNotificationController requires generic " - "frame descriptor, but it is missing."; - } else { - if (video_header.is_first_packet_in_frame) { - RTC_DCHECK(video_header.generic); - LossNotificationController::FrameDetails frame; - frame.is_keyframe = - video_header.frame_type == VideoFrameType::kVideoFrameKey; - frame.frame_id = video_header.generic->frame_id; - frame.frame_dependencies = video_header.generic->dependencies; - loss_notification_controller_->OnReceivedPacket( - rtp_packet.SequenceNumber(), &frame); - } else { - loss_notification_controller_->OnReceivedPacket( - rtp_packet.SequenceNumber(), nullptr); - } - } - } - - if (nack_module_) { - const bool is_keyframe = - video_header.is_first_packet_in_frame && - video_header.frame_type == VideoFrameType::kVideoFrameKey; - - packet->times_nacked = nack_module_->OnReceivedPacket( - rtp_packet.SequenceNumber(), is_keyframe, rtp_packet.recovered()); - } else { - packet->times_nacked = -1; - } - - if (codec_payload.size() == 0) { - NotifyReceiverOfEmptyPacket(packet->seq_num); - rtcp_feedback_buffer_.SendBufferedRtcpFeedback(); - return; - } - - if (packet->codec() == kVideoCodecH264) { - // Only when we start to receive packets will we know what payload type - // that will be used. When we know the payload type insert the correct - // sps/pps into the tracker. - if (packet->payload_type != last_payload_type_) { - last_payload_type_ = packet->payload_type; - InsertSpsPpsIntoTracker(packet->payload_type); - } - - video_coding::H264SpsPpsTracker::FixedBitstream fixed = - tracker_.CopyAndFixBitstream( - rtc::MakeArrayView(codec_payload.cdata(), codec_payload.size()), - &packet->video_header); - - switch (fixed.action) { - case video_coding::H264SpsPpsTracker::kRequestKeyframe: - rtcp_feedback_buffer_.RequestKeyFrame(); - rtcp_feedback_buffer_.SendBufferedRtcpFeedback(); - [[fallthrough]]; - case video_coding::H264SpsPpsTracker::kDrop: - return; - case video_coding::H264SpsPpsTracker::kInsert: - packet->video_payload = std::move(fixed.bitstream); - break; - } - - } else { - packet->video_payload = std::move(codec_payload); - } - - rtcp_feedback_buffer_.SendBufferedRtcpFeedback(); - frame_counter_.Add(packet->timestamp); - video_coding::PacketBuffer::InsertResult insert_result; - { - MutexLock lock(&packet_buffer_lock_); - int64_t unwrapped_rtp_seq_num = - rtp_seq_num_unwrapper_.Unwrap(rtp_packet.SequenceNumber()); - auto& packet_info = - packet_infos_ - .emplace( - unwrapped_rtp_seq_num, - RtpPacketInfo( - rtp_packet.Ssrc(), rtp_packet.Csrcs(), - rtp_packet.Timestamp(), - /*audio_level=*/absl::nullopt, - rtp_packet.GetExtension<AbsoluteCaptureTimeExtension>(), - /*receive_time_ms=*/clock_->TimeInMilliseconds())) - .first->second; - - // Try to extrapolate absolute capture time if it is missing. - absl::optional<AbsoluteCaptureTime> absolute_capture_time = - absolute_capture_time_interpolator_.OnReceivePacket( - AbsoluteCaptureTimeInterpolator::GetSource(packet_info.ssrc(), - packet_info.csrcs()), - packet_info.rtp_timestamp(), - // Assume frequency is the same one for all video frames. - kVideoPayloadTypeFrequency, packet_info.absolute_capture_time()); - packet_info.set_absolute_capture_time(absolute_capture_time); - - if (absolute_capture_time.has_value()) { - packet_info.set_local_capture_clock_offset( - capture_clock_offset_updater_.AdjustEstimatedCaptureClockOffset( - absolute_capture_time->estimated_capture_clock_offset)); - } - - insert_result = packet_buffer_.InsertPacket(std::move(packet)); - } - OnInsertedPacket(std::move(insert_result)); -} - -void RtpVideoStreamReceiver::OnRecoveredPacket(const uint8_t* rtp_packet, - size_t rtp_packet_length) { - RtpPacketReceived packet; - if (!packet.Parse(rtp_packet, rtp_packet_length)) - return; - if (packet.PayloadType() == config_.rtp.red_payload_type) { - RTC_LOG(LS_WARNING) << "Discarding recovered packet with RED encapsulation"; - return; - } - - packet.IdentifyExtensions(rtp_header_extensions_); - packet.set_payload_type_frequency(kVideoPayloadTypeFrequency); - // TODO(nisse): UlpfecReceiverImpl::ProcessReceivedFec passes both - // original (decapsulated) media packets and recovered packets to - // this callback. We need a way to distinguish, for setting - // packet.recovered() correctly. Ideally, move RED decapsulation out - // of the Ulpfec implementation. - - ReceivePacket(packet); -} - -// This method handles both regular RTP packets and packets recovered -// via FlexFEC. -void RtpVideoStreamReceiver::OnRtpPacket(const RtpPacketReceived& packet) { - RTC_DCHECK_RUN_ON(&worker_task_checker_); - - if (!receiving_) { - return; - } - - ReceivePacket(packet); - - // Update receive statistics after ReceivePacket. - // Receive statistics will be reset if the payload type changes (make sure - // that the first packet is included in the stats). - if (!packet.recovered()) { - rtp_receive_statistics_->OnRtpPacket(packet); - } - - for (RtpPacketSinkInterface* secondary_sink : secondary_sinks_) { - secondary_sink->OnRtpPacket(packet); - } -} - -void RtpVideoStreamReceiver::RequestKeyFrame() { - // TODO(bugs.webrtc.org/10336): Allow the sender to ignore key frame requests - // issued by anything other than the LossNotificationController if it (the - // sender) is relying on LNTF alone. - if (keyframe_request_sender_) { - keyframe_request_sender_->RequestKeyFrame(); - } else { - rtp_rtcp_->SendPictureLossIndication(); - } -} - -void RtpVideoStreamReceiver::SendLossNotification( - uint16_t last_decoded_seq_num, - uint16_t last_received_seq_num, - bool decodability_flag, - bool buffering_allowed) { - RTC_DCHECK(config_.rtp.lntf.enabled); - rtp_rtcp_->SendLossNotification(last_decoded_seq_num, last_received_seq_num, - decodability_flag, buffering_allowed); -} - -bool RtpVideoStreamReceiver::IsUlpfecEnabled() const { - return config_.rtp.ulpfec_payload_type != -1; -} - -bool RtpVideoStreamReceiver::IsRetransmissionsEnabled() const { - return config_.rtp.nack.rtp_history_ms > 0; -} - -void RtpVideoStreamReceiver::RequestPacketRetransmit( - const std::vector<uint16_t>& sequence_numbers) { - rtp_rtcp_->SendNack(sequence_numbers); -} - -bool RtpVideoStreamReceiver::IsDecryptable() const { - return frames_decryptable_.load(); -} - -void RtpVideoStreamReceiver::OnInsertedPacket( - video_coding::PacketBuffer::InsertResult result) { - std::vector<std::unique_ptr<RtpFrameObject>> assembled_frames; - { - MutexLock lock(&packet_buffer_lock_); - video_coding::PacketBuffer::Packet* first_packet = nullptr; - int max_nack_count; - int64_t min_recv_time; - int64_t max_recv_time; - std::vector<rtc::ArrayView<const uint8_t>> payloads; - RtpPacketInfos::vector_type packet_infos; - - bool frame_boundary = true; - for (auto& packet : result.packets) { - // PacketBuffer promisses frame boundaries are correctly set on each - // packet. Document that assumption with the DCHECKs. - RTC_DCHECK_EQ(frame_boundary, packet->is_first_packet_in_frame()); - int64_t unwrapped_rtp_seq_num = - rtp_seq_num_unwrapper_.Unwrap(packet->seq_num); - RTC_DCHECK(packet_infos_.count(unwrapped_rtp_seq_num) > 0); - RtpPacketInfo& packet_info = packet_infos_[unwrapped_rtp_seq_num]; - if (packet->is_first_packet_in_frame()) { - first_packet = packet.get(); - max_nack_count = packet->times_nacked; - min_recv_time = packet_info.receive_time().ms(); - max_recv_time = packet_info.receive_time().ms(); - } else { - max_nack_count = std::max(max_nack_count, packet->times_nacked); - min_recv_time = - std::min(min_recv_time, packet_info.receive_time().ms()); - max_recv_time = - std::max(max_recv_time, packet_info.receive_time().ms()); - } - payloads.emplace_back(packet->video_payload); - packet_infos.push_back(packet_info); - - frame_boundary = packet->is_last_packet_in_frame(); - if (packet->is_last_packet_in_frame()) { - auto depacketizer_it = - payload_type_map_.find(first_packet->payload_type); - RTC_CHECK(depacketizer_it != payload_type_map_.end()); - - rtc::scoped_refptr<EncodedImageBuffer> bitstream = - depacketizer_it->second->AssembleFrame(payloads); - if (!bitstream) { - // Failed to assemble a frame. Discard and continue. - continue; - } - - const video_coding::PacketBuffer::Packet& last_packet = *packet; - assembled_frames.push_back(std::make_unique<RtpFrameObject>( - first_packet->seq_num, // - last_packet.seq_num, // - last_packet.marker_bit, // - max_nack_count, // - min_recv_time, // - max_recv_time, // - first_packet->timestamp, // - ntp_estimator_.Estimate(first_packet->timestamp), // - last_packet.video_header.video_timing, // - first_packet->payload_type, // - first_packet->codec(), // - last_packet.video_header.rotation, // - last_packet.video_header.content_type, // - first_packet->video_header, // - last_packet.video_header.color_space, // - RtpPacketInfos(std::move(packet_infos)), // - std::move(bitstream))); - payloads.clear(); - packet_infos.clear(); - } - } - RTC_DCHECK(frame_boundary); - - if (result.buffer_cleared) { - packet_infos_.clear(); - } - } // packet_buffer_lock_ - - if (result.buffer_cleared) { - { - MutexLock lock(&sync_info_lock_); - last_received_rtp_system_time_.reset(); - last_received_keyframe_rtp_system_time_.reset(); - last_received_keyframe_rtp_timestamp_.reset(); - } - RequestKeyFrame(); - } - - for (auto& frame : assembled_frames) { - OnAssembledFrame(std::move(frame)); - } -} - -void RtpVideoStreamReceiver::OnAssembledFrame( - std::unique_ptr<RtpFrameObject> frame) { - RTC_DCHECK_RUN_ON(&network_tc_); - RTC_DCHECK(frame); - - const absl::optional<RTPVideoHeader::GenericDescriptorInfo>& descriptor = - frame->GetRtpVideoHeader().generic; - - if (loss_notification_controller_ && descriptor) { - loss_notification_controller_->OnAssembledFrame( - frame->first_seq_num(), descriptor->frame_id, - absl::c_linear_search(descriptor->decode_target_indications, - DecodeTargetIndication::kDiscardable), - descriptor->dependencies); - } - - // If frames arrive before a key frame, they would not be decodable. - // In that case, request a key frame ASAP. - if (!has_received_frame_) { - if (frame->FrameType() != VideoFrameType::kVideoFrameKey) { - // `loss_notification_controller_`, if present, would have already - // requested a key frame when the first packet for the non-key frame - // had arrived, so no need to replicate the request. - if (!loss_notification_controller_) { - RequestKeyFrame(); - } - } - has_received_frame_ = true; - } - - MutexLock lock(&reference_finder_lock_); - // Reset `reference_finder_` if `frame` is new and the codec have changed. - if (current_codec_) { - bool frame_is_newer = - AheadOf(frame->Timestamp(), last_assembled_frame_rtp_timestamp_); - - if (frame->codec_type() != current_codec_) { - if (frame_is_newer) { - // When we reset the `reference_finder_` we don't want new picture ids - // to overlap with old picture ids. To ensure that doesn't happen we - // start from the `last_completed_picture_id_` and add an offset in - // case of reordering. - reference_finder_ = std::make_unique<RtpFrameReferenceFinder>( - last_completed_picture_id_ + std::numeric_limits<uint16_t>::max()); - current_codec_ = frame->codec_type(); - } else { - // Old frame from before the codec switch, discard it. - return; - } - } - - if (frame_is_newer) { - last_assembled_frame_rtp_timestamp_ = frame->Timestamp(); - } - } else { - current_codec_ = frame->codec_type(); - last_assembled_frame_rtp_timestamp_ = frame->Timestamp(); - } - - if (buffered_frame_decryptor_ != nullptr) { - buffered_frame_decryptor_->ManageEncryptedFrame(std::move(frame)); - } else if (frame_transformer_delegate_) { - frame_transformer_delegate_->TransformFrame(std::move(frame)); - } else { - OnCompleteFrames(reference_finder_->ManageFrame(std::move(frame))); - } -} - -void RtpVideoStreamReceiver::OnCompleteFrames( - RtpFrameReferenceFinder::ReturnVector frames) { - { - MutexLock lock(&last_seq_num_mutex_); - for (const auto& frame : frames) { - RtpFrameObject* rtp_frame = static_cast<RtpFrameObject*>(frame.get()); - last_seq_num_for_pic_id_[rtp_frame->Id()] = rtp_frame->last_seq_num(); - } - } - for (auto& frame : frames) { - last_completed_picture_id_ = - std::max(last_completed_picture_id_, frame->Id()); - complete_frame_callback_->OnCompleteFrame(std::move(frame)); - } -} - -void RtpVideoStreamReceiver::OnDecryptedFrame( - std::unique_ptr<RtpFrameObject> frame) { - MutexLock lock(&reference_finder_lock_); - OnCompleteFrames(reference_finder_->ManageFrame(std::move(frame))); -} - -void RtpVideoStreamReceiver::OnDecryptionStatusChange( - FrameDecryptorInterface::Status status) { - frames_decryptable_.store( - (status == FrameDecryptorInterface::Status::kOk) || - (status == FrameDecryptorInterface::Status::kRecoverable)); -} - -void RtpVideoStreamReceiver::SetFrameDecryptor( - rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor) { - RTC_DCHECK_RUN_ON(&network_tc_); - if (buffered_frame_decryptor_ == nullptr) { - buffered_frame_decryptor_ = - std::make_unique<BufferedFrameDecryptor>(this, this, field_trials_); - } - buffered_frame_decryptor_->SetFrameDecryptor(std::move(frame_decryptor)); -} - -void RtpVideoStreamReceiver::SetDepacketizerToDecoderFrameTransformer( - rtc::scoped_refptr<FrameTransformerInterface> frame_transformer) { - RTC_DCHECK_RUN_ON(&network_tc_); - frame_transformer_delegate_ = - rtc::make_ref_counted<RtpVideoStreamReceiverFrameTransformerDelegate>( - this, std::move(frame_transformer), rtc::Thread::Current(), - config_.rtp.remote_ssrc); - frame_transformer_delegate_->Init(); -} - -void RtpVideoStreamReceiver::UpdateRtt(int64_t max_rtt_ms) { - if (nack_module_) - nack_module_->UpdateRtt(max_rtt_ms); -} - -absl::optional<int64_t> RtpVideoStreamReceiver::LastReceivedPacketMs() const { - MutexLock lock(&sync_info_lock_); - if (last_received_rtp_system_time_) { - return absl::optional<int64_t>(last_received_rtp_system_time_->ms()); - } - return absl::nullopt; -} - -absl::optional<int64_t> RtpVideoStreamReceiver::LastReceivedKeyframePacketMs() - const { - MutexLock lock(&sync_info_lock_); - if (last_received_keyframe_rtp_system_time_) { - return absl::optional<int64_t>( - last_received_keyframe_rtp_system_time_->ms()); - } - return absl::nullopt; -} - -void RtpVideoStreamReceiver::AddSecondarySink(RtpPacketSinkInterface* sink) { - RTC_DCHECK_RUN_ON(&worker_task_checker_); - RTC_DCHECK(!absl::c_linear_search(secondary_sinks_, sink)); - secondary_sinks_.push_back(sink); -} - -void RtpVideoStreamReceiver::RemoveSecondarySink( - const RtpPacketSinkInterface* sink) { - RTC_DCHECK_RUN_ON(&worker_task_checker_); - auto it = absl::c_find(secondary_sinks_, sink); - if (it == secondary_sinks_.end()) { - // We might be rolling-back a call whose setup failed mid-way. In such a - // case, it's simpler to remove "everything" rather than remember what - // has already been added. - RTC_LOG(LS_WARNING) << "Removal of unknown sink."; - return; - } - secondary_sinks_.erase(it); -} - -void RtpVideoStreamReceiver::ManageFrame( - std::unique_ptr<RtpFrameObject> frame) { - MutexLock lock(&reference_finder_lock_); - OnCompleteFrames(reference_finder_->ManageFrame(std::move(frame))); -} - -void RtpVideoStreamReceiver::ReceivePacket(const RtpPacketReceived& packet) { - if (packet.payload_size() == 0) { - // Padding or keep-alive packet. - // TODO(nisse): Could drop empty packets earlier, but need to figure out how - // they should be counted in stats. - NotifyReceiverOfEmptyPacket(packet.SequenceNumber()); - return; - } - if (packet.PayloadType() == config_.rtp.red_payload_type) { - ParseAndHandleEncapsulatingHeader(packet); - return; - } - - const auto type_it = payload_type_map_.find(packet.PayloadType()); - if (type_it == payload_type_map_.end()) { - return; - } - absl::optional<VideoRtpDepacketizer::ParsedRtpPayload> parsed_payload = - type_it->second->Parse(packet.PayloadBuffer()); - if (parsed_payload == absl::nullopt) { - RTC_LOG(LS_WARNING) << "Failed parsing payload."; - return; - } - - OnReceivedPayloadData(std::move(parsed_payload->video_payload), packet, - parsed_payload->video_header); -} - -void RtpVideoStreamReceiver::ParseAndHandleEncapsulatingHeader( - const RtpPacketReceived& packet) { - RTC_DCHECK_RUN_ON(&worker_task_checker_); - if (packet.PayloadType() == config_.rtp.red_payload_type && - packet.payload_size() > 0) { - if (packet.payload()[0] == config_.rtp.ulpfec_payload_type) { - // Notify video_receiver about received FEC packets to avoid NACKing these - // packets. - NotifyReceiverOfEmptyPacket(packet.SequenceNumber()); - } - if (!ulpfec_receiver_->AddReceivedRedPacket( - packet, config_.rtp.ulpfec_payload_type)) { - return; - } - ulpfec_receiver_->ProcessReceivedFec(); - } -} - -// In the case of a video stream without picture ids and no rtx the -// RtpFrameReferenceFinder will need to know about padding to -// correctly calculate frame references. -void RtpVideoStreamReceiver::NotifyReceiverOfEmptyPacket(uint16_t seq_num) { - { - MutexLock lock(&reference_finder_lock_); - OnCompleteFrames(reference_finder_->PaddingReceived(seq_num)); - } - - video_coding::PacketBuffer::InsertResult insert_result; - { - MutexLock lock(&packet_buffer_lock_); - insert_result = packet_buffer_.InsertPadding(seq_num); - } - OnInsertedPacket(std::move(insert_result)); - - if (nack_module_) { - nack_module_->OnReceivedPacket(seq_num, /* is_keyframe = */ false, - /* is _recovered = */ false); - } - if (loss_notification_controller_) { - // TODO(bugs.webrtc.org/10336): Handle empty packets. - RTC_LOG(LS_WARNING) - << "LossNotificationController does not expect empty packets."; - } -} - -bool RtpVideoStreamReceiver::DeliverRtcp(const uint8_t* rtcp_packet, - size_t rtcp_packet_length) { - RTC_DCHECK_RUN_ON(&worker_task_checker_); - - if (!receiving_) { - return false; - } - - rtp_rtcp_->IncomingRtcpPacket(rtcp_packet, rtcp_packet_length); - - int64_t rtt = 0; - rtp_rtcp_->RTT(config_.rtp.remote_ssrc, &rtt, nullptr, nullptr, nullptr); - if (rtt == 0) { - // Waiting for valid rtt. - return true; - } - uint32_t ntp_secs = 0; - uint32_t ntp_frac = 0; - uint32_t rtp_timestamp = 0; - uint32_t received_ntp_secs = 0; - uint32_t received_ntp_frac = 0; - if (rtp_rtcp_->RemoteNTP(&ntp_secs, &ntp_frac, &received_ntp_secs, - &received_ntp_frac, &rtp_timestamp) != 0) { - // Waiting for RTCP. - return true; - } - NtpTime received_ntp(received_ntp_secs, received_ntp_frac); - int64_t time_since_received = - clock_->CurrentNtpInMilliseconds() - received_ntp.ToMs(); - // Don't use old SRs to estimate time. - if (time_since_received <= 1) { - ntp_estimator_.UpdateRtcpTimestamp( - TimeDelta::Millis(rtt), NtpTime(ntp_secs, ntp_frac), rtp_timestamp); - absl::optional<int64_t> remote_to_local_clock_offset = - ntp_estimator_.EstimateRemoteToLocalClockOffset(); - if (remote_to_local_clock_offset.has_value()) { - capture_clock_offset_updater_.SetRemoteToLocalClockOffset( - *remote_to_local_clock_offset); - } - } - - return true; -} - -void RtpVideoStreamReceiver::FrameContinuous(int64_t picture_id) { - if (!nack_module_) - return; - - int seq_num = -1; - { - MutexLock lock(&last_seq_num_mutex_); - auto seq_num_it = last_seq_num_for_pic_id_.find(picture_id); - if (seq_num_it != last_seq_num_for_pic_id_.end()) - seq_num = seq_num_it->second; - } - if (seq_num != -1) - nack_module_->ClearUpTo(seq_num); -} - -void RtpVideoStreamReceiver::FrameDecoded(int64_t picture_id) { - int seq_num = -1; - { - MutexLock lock(&last_seq_num_mutex_); - auto seq_num_it = last_seq_num_for_pic_id_.find(picture_id); - if (seq_num_it != last_seq_num_for_pic_id_.end()) { - seq_num = seq_num_it->second; - last_seq_num_for_pic_id_.erase(last_seq_num_for_pic_id_.begin(), - ++seq_num_it); - } - } - if (seq_num != -1) { - { - MutexLock lock(&packet_buffer_lock_); - packet_buffer_.ClearTo(seq_num); - int64_t unwrapped_rtp_seq_num = rtp_seq_num_unwrapper_.Unwrap(seq_num); - packet_infos_.erase(packet_infos_.begin(), - packet_infos_.upper_bound(unwrapped_rtp_seq_num)); - } - MutexLock lock(&reference_finder_lock_); - reference_finder_->ClearTo(seq_num); - } -} - -void RtpVideoStreamReceiver::SignalNetworkState(NetworkState state) { - rtp_rtcp_->SetRTCPStatus(state == kNetworkUp ? config_.rtp.rtcp_mode - : RtcpMode::kOff); -} - -void RtpVideoStreamReceiver::StartReceive() { - RTC_DCHECK_RUN_ON(&worker_task_checker_); - receiving_ = true; -} - -void RtpVideoStreamReceiver::StopReceive() { - RTC_DCHECK_RUN_ON(&worker_task_checker_); - receiving_ = false; -} - -void RtpVideoStreamReceiver::UpdateHistograms() { - FecPacketCounter counter = ulpfec_receiver_->GetPacketCounter(); - if (counter.first_packet_time_ms == -1) - return; - - int64_t elapsed_sec = - (clock_->TimeInMilliseconds() - counter.first_packet_time_ms) / 1000; - if (elapsed_sec < metrics::kMinRunTimeInSeconds) - return; - - if (counter.num_packets > 0) { - RTC_HISTOGRAM_PERCENTAGE( - "WebRTC.Video.ReceivedFecPacketsInPercent", - static_cast<int>(counter.num_fec_packets * 100 / counter.num_packets)); - } - if (counter.num_fec_packets > 0) { - RTC_HISTOGRAM_PERCENTAGE("WebRTC.Video.RecoveredMediaPacketsInPercentOfFec", - static_cast<int>(counter.num_recovered_packets * - 100 / counter.num_fec_packets)); - } - if (config_.rtp.ulpfec_payload_type != -1) { - RTC_HISTOGRAM_COUNTS_10000( - "WebRTC.Video.FecBitrateReceivedInKbps", - static_cast<int>(counter.num_bytes * 8 / elapsed_sec / 1000)); - } -} - -void RtpVideoStreamReceiver::InsertSpsPpsIntoTracker(uint8_t payload_type) { - auto codec_params_it = pt_codec_params_.find(payload_type); - if (codec_params_it == pt_codec_params_.end()) - return; - - RTC_LOG(LS_INFO) << "Found out of band supplied codec parameters for" - " payload type: " - << static_cast<int>(payload_type); - - H264SpropParameterSets sprop_decoder; - auto sprop_base64_it = - codec_params_it->second.find(cricket::kH264FmtpSpropParameterSets); - - if (sprop_base64_it == codec_params_it->second.end()) - return; - - if (!sprop_decoder.DecodeSprop(sprop_base64_it->second.c_str())) - return; - - tracker_.InsertSpsPpsNalus(sprop_decoder.sps_nalu(), - sprop_decoder.pps_nalu()); -} - -void RtpVideoStreamReceiver::UpdatePacketReceiveTimestamps( - const RtpPacketReceived& packet, - bool is_keyframe) { - Timestamp now = clock_->CurrentTime(); - { - MutexLock lock(&sync_info_lock_); - if (is_keyframe || - last_received_keyframe_rtp_timestamp_ == packet.Timestamp()) { - last_received_keyframe_rtp_timestamp_ = packet.Timestamp(); - last_received_keyframe_rtp_system_time_ = now; - } - last_received_rtp_system_time_ = now; - last_received_rtp_timestamp_ = packet.Timestamp(); - } - - // Periodically log the RTP header of incoming packets. - if (now.ms() - last_packet_log_ms_ > kPacketLogIntervalMs) { - rtc::StringBuilder ss; - ss << "Packet received on SSRC: " << packet.Ssrc() - << " with payload type: " << static_cast<int>(packet.PayloadType()) - << ", timestamp: " << packet.Timestamp() - << ", sequence number: " << packet.SequenceNumber() - << ", arrival time: " << ToString(packet.arrival_time()); - int32_t time_offset; - if (packet.GetExtension<TransmissionOffset>(&time_offset)) { - ss << ", toffset: " << time_offset; - } - uint32_t send_time; - if (packet.GetExtension<AbsoluteSendTime>(&send_time)) { - ss << ", abs send time: " << send_time; - } - RTC_LOG(LS_INFO) << ss.str(); - last_packet_log_ms_ = now.ms(); - } -} - -} // namespace webrtc diff --git a/video/rtp_video_stream_receiver.h b/video/rtp_video_stream_receiver.h deleted file mode 100644 index 457885778a..0000000000 --- a/video/rtp_video_stream_receiver.h +++ /dev/null @@ -1,436 +0,0 @@ -/* - * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#ifndef VIDEO_RTP_VIDEO_STREAM_RECEIVER_H_ -#define VIDEO_RTP_VIDEO_STREAM_RECEIVER_H_ - -#include <atomic> -#include <list> -#include <map> -#include <memory> -#include <string> -#include <vector> - -#include "absl/types/optional.h" -#include "api/array_view.h" -#include "api/crypto/frame_decryptor_interface.h" -#include "api/sequence_checker.h" -#include "api/transport/field_trial_based_config.h" -#include "api/units/timestamp.h" -#include "api/video/color_space.h" -#include "api/video/video_codec_type.h" -#include "call/rtp_packet_sink_interface.h" -#include "call/syncable.h" -#include "call/video_receive_stream.h" -#include "modules/rtp_rtcp/include/receive_statistics.h" -#include "modules/rtp_rtcp/include/remote_ntp_time_estimator.h" -#include "modules/rtp_rtcp/include/rtp_header_extension_map.h" -#include "modules/rtp_rtcp/include/rtp_rtcp.h" -#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" -#include "modules/rtp_rtcp/source/absolute_capture_time_interpolator.h" -#include "modules/rtp_rtcp/source/capture_clock_offset_updater.h" -#include "modules/rtp_rtcp/source/rtp_dependency_descriptor_extension.h" -#include "modules/rtp_rtcp/source/rtp_packet_received.h" -#include "modules/rtp_rtcp/source/rtp_video_header.h" -#include "modules/rtp_rtcp/source/video_rtp_depacketizer.h" -#include "modules/video_coding/h264_sps_pps_tracker.h" -#include "modules/video_coding/loss_notification_controller.h" -#include "modules/video_coding/packet_buffer.h" -#include "modules/video_coding/rtp_frame_reference_finder.h" -#include "rtc_base/experiments/field_trial_parser.h" -#include "rtc_base/numerics/sequence_number_util.h" -#include "rtc_base/synchronization/mutex.h" -#include "rtc_base/system/no_unique_address.h" -#include "rtc_base/thread_annotations.h" -#include "video/buffered_frame_decryptor.h" -#include "video/rtp_video_stream_receiver_frame_transformer_delegate.h" -#include "video/unique_timestamp_counter.h" - -namespace webrtc { - -class DEPRECATED_NackModule; -class PacketRouter; -class ProcessThread; -class ReceiveStatistics; -class ReceiveStatisticsProxy; -class RtcpRttStats; -class RtpPacketReceived; -class Transport; -class UlpfecReceiver; - -class RtpVideoStreamReceiver : public LossNotificationSender, - public RecoveredPacketReceiver, - public RtpPacketSinkInterface, - public KeyFrameRequestSender, - public OnDecryptedFrameCallback, - public OnDecryptionStatusChangeCallback, - public RtpVideoFrameReceiver { - public: - // A complete frame is a frame which has received all its packets and all its - // references are known. - class OnCompleteFrameCallback { - public: - virtual ~OnCompleteFrameCallback() {} - virtual void OnCompleteFrame(std::unique_ptr<EncodedFrame> frame) = 0; - }; - - // DEPRECATED due to dependency on ReceiveStatisticsProxy. - RtpVideoStreamReceiver( - Clock* clock, - Transport* transport, - RtcpRttStats* rtt_stats, - // The packet router is optional; if provided, the RtpRtcp module for this - // stream is registered as a candidate for sending REMB and transport - // feedback. - PacketRouter* packet_router, - const VideoReceiveStreamInterface::Config* config, - ReceiveStatistics* rtp_receive_statistics, - ReceiveStatisticsProxy* receive_stats_proxy, - ProcessThread* process_thread, - NackSender* nack_sender, - // The KeyFrameRequestSender is optional; if not provided, key frame - // requests are sent via the internal RtpRtcp module. - KeyFrameRequestSender* keyframe_request_sender, - OnCompleteFrameCallback* complete_frame_callback, - rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor, - rtc::scoped_refptr<FrameTransformerInterface> frame_transformer, - const FieldTrialsView* field_trials = nullptr); - - RtpVideoStreamReceiver( - Clock* clock, - Transport* transport, - RtcpRttStats* rtt_stats, - // The packet router is optional; if provided, the RtpRtcp module for this - // stream is registered as a candidate for sending REMB and transport - // feedback. - PacketRouter* packet_router, - const VideoReceiveStreamInterface::Config* config, - ReceiveStatistics* rtp_receive_statistics, - RtcpPacketTypeCounterObserver* rtcp_packet_type_counter_observer, - RtcpCnameCallback* rtcp_cname_callback, - ProcessThread* process_thread, - NackSender* nack_sender, - // The KeyFrameRequestSender is optional; if not provided, key frame - // requests are sent via the internal RtpRtcp module. - KeyFrameRequestSender* keyframe_request_sender, - OnCompleteFrameCallback* complete_frame_callback, - rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor, - rtc::scoped_refptr<FrameTransformerInterface> frame_transformer, - const FieldTrialsView* field_trials = nullptr); - ~RtpVideoStreamReceiver() override; - - void AddReceiveCodec(uint8_t payload_type, - VideoCodecType codec_type, - const std::map<std::string, std::string>& codec_params, - bool raw_payload); - - void StartReceive(); - void StopReceive(); - - // Produces the transport-related timestamps; current_delay_ms is left unset. - absl::optional<Syncable::Info> GetSyncInfo() const; - - bool DeliverRtcp(const uint8_t* rtcp_packet, size_t rtcp_packet_length); - - void FrameContinuous(int64_t seq_num); - - void FrameDecoded(int64_t seq_num); - - void SignalNetworkState(NetworkState state); - - // Returns number of different frames seen. - int GetUniqueFramesSeen() const { - RTC_DCHECK_RUN_ON(&worker_task_checker_); - return frame_counter_.GetUniqueSeen(); - } - - // Implements RtpPacketSinkInterface. - void OnRtpPacket(const RtpPacketReceived& packet) override; - - // Public only for tests. - void OnReceivedPayloadData(rtc::CopyOnWriteBuffer codec_payload, - const RtpPacketReceived& rtp_packet, - const RTPVideoHeader& video); - - // Implements RecoveredPacketReceiver. - void OnRecoveredPacket(const uint8_t* packet, size_t packet_length) override; - - // Send an RTCP keyframe request. - void RequestKeyFrame() override; - - // Implements LossNotificationSender. - void SendLossNotification(uint16_t last_decoded_seq_num, - uint16_t last_received_seq_num, - bool decodability_flag, - bool buffering_allowed) override; - - bool IsUlpfecEnabled() const; - bool IsRetransmissionsEnabled() const; - - // Returns true if a decryptor is attached and frames can be decrypted. - // Updated by OnDecryptionStatusChangeCallback. Note this refers to Frame - // Decryption not SRTP. - bool IsDecryptable() const; - - // Don't use, still experimental. - void RequestPacketRetransmit(const std::vector<uint16_t>& sequence_numbers); - - void OnCompleteFrames(RtpFrameReferenceFinder::ReturnVector frames); - - // Implements OnDecryptedFrameCallback. - void OnDecryptedFrame(std::unique_ptr<RtpFrameObject> frame) override; - - // Implements OnDecryptionStatusChangeCallback. - void OnDecryptionStatusChange( - FrameDecryptorInterface::Status status) override; - - // Optionally set a frame decryptor after a stream has started. This will not - // reset the decoder state. - void SetFrameDecryptor( - rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor); - - // Sets a frame transformer after a stream has started, if no transformer - // has previously been set. Does not reset the decoder state. - void SetDepacketizerToDecoderFrameTransformer( - rtc::scoped_refptr<FrameTransformerInterface> frame_transformer); - - // Called by VideoReceiveStreamInterface when stats are updated. - void UpdateRtt(int64_t max_rtt_ms); - - absl::optional<int64_t> LastReceivedPacketMs() const; - absl::optional<int64_t> LastReceivedKeyframePacketMs() const; - - // RtpDemuxer only forwards a given RTP packet to one sink. However, some - // sinks, such as FlexFEC, might wish to be informed of all of the packets - // a given sink receives (or any set of sinks). They may do so by registering - // themselves as secondary sinks. - void AddSecondarySink(RtpPacketSinkInterface* sink); - void RemoveSecondarySink(const RtpPacketSinkInterface* sink); - - private: - // Implements RtpVideoFrameReceiver. - void ManageFrame(std::unique_ptr<RtpFrameObject> frame) override; - - // Used for buffering RTCP feedback messages and sending them all together. - // Note: - // 1. Key frame requests and NACKs are mutually exclusive, with the - // former taking precedence over the latter. - // 2. Loss notifications are orthogonal to either. (That is, may be sent - // alongside either.) - class RtcpFeedbackBuffer : public KeyFrameRequestSender, - public NackSender, - public LossNotificationSender { - public: - RtcpFeedbackBuffer(KeyFrameRequestSender* key_frame_request_sender, - NackSender* nack_sender, - LossNotificationSender* loss_notification_sender); - - ~RtcpFeedbackBuffer() override = default; - - // KeyFrameRequestSender implementation. - void RequestKeyFrame() RTC_LOCKS_EXCLUDED(mutex_) override; - - // NackSender implementation. - void SendNack(const std::vector<uint16_t>& sequence_numbers, - bool buffering_allowed) RTC_LOCKS_EXCLUDED(mutex_) override; - - // LossNotificationSender implementation. - void SendLossNotification(uint16_t last_decoded_seq_num, - uint16_t last_received_seq_num, - bool decodability_flag, - bool buffering_allowed) - RTC_LOCKS_EXCLUDED(mutex_) override; - - // Send all RTCP feedback messages buffered thus far. - void SendBufferedRtcpFeedback() RTC_LOCKS_EXCLUDED(mutex_); - - private: - // LNTF-related state. - struct LossNotificationState { - LossNotificationState(uint16_t last_decoded_seq_num, - uint16_t last_received_seq_num, - bool decodability_flag) - : last_decoded_seq_num(last_decoded_seq_num), - last_received_seq_num(last_received_seq_num), - decodability_flag(decodability_flag) {} - - uint16_t last_decoded_seq_num; - uint16_t last_received_seq_num; - bool decodability_flag; - }; - struct ConsumedRtcpFeedback { - bool request_key_frame = false; - std::vector<uint16_t> nack_sequence_numbers; - absl::optional<LossNotificationState> lntf_state; - }; - - ConsumedRtcpFeedback ConsumeRtcpFeedback() RTC_LOCKS_EXCLUDED(mutex_); - ConsumedRtcpFeedback ConsumeRtcpFeedbackLocked() - RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); - // This method is called both with and without mutex_ held. - void SendRtcpFeedback(ConsumedRtcpFeedback feedback); - - KeyFrameRequestSender* const key_frame_request_sender_; - NackSender* const nack_sender_; - LossNotificationSender* const loss_notification_sender_; - - // NACKs are accessible from two threads due to nack_module_ being a module. - Mutex mutex_; - - // Key-frame-request-related state. - bool request_key_frame_ RTC_GUARDED_BY(mutex_); - - // NACK-related state. - std::vector<uint16_t> nack_sequence_numbers_ RTC_GUARDED_BY(mutex_); - - absl::optional<LossNotificationState> lntf_state_ RTC_GUARDED_BY(mutex_); - }; - enum ParseGenericDependenciesResult { - kDropPacket, - kHasGenericDescriptor, - kNoGenericDescriptor - }; - - // Entry point doing non-stats work for a received packet. Called - // for the same packet both before and after RED decapsulation. - void ReceivePacket(const RtpPacketReceived& packet); - // Parses and handles RED headers. - // This function assumes that it's being called from only one thread. - void ParseAndHandleEncapsulatingHeader(const RtpPacketReceived& packet); - void NotifyReceiverOfEmptyPacket(uint16_t seq_num); - void UpdateHistograms(); - bool IsRedEnabled() const; - void InsertSpsPpsIntoTracker(uint8_t payload_type); - void OnInsertedPacket(video_coding::PacketBuffer::InsertResult result); - ParseGenericDependenciesResult ParseGenericDependenciesExtension( - const RtpPacketReceived& rtp_packet, - RTPVideoHeader* video_header) RTC_RUN_ON(worker_task_checker_); - void OnAssembledFrame(std::unique_ptr<RtpFrameObject> frame) - RTC_LOCKS_EXCLUDED(packet_buffer_lock_); - void UpdatePacketReceiveTimestamps(const RtpPacketReceived& packet, - bool is_keyframe) - RTC_RUN_ON(worker_task_checker_); - - const FieldTrialsView& field_trials_; - FieldTrialBasedConfig owned_field_trials_; - - Clock* const clock_; - // Ownership of this object lies with VideoReceiveStreamInterface, which owns - // `this`. - const VideoReceiveStreamInterface::Config& config_; - PacketRouter* const packet_router_; - ProcessThread* const process_thread_; - - RemoteNtpTimeEstimator ntp_estimator_; - - RtpHeaderExtensionMap rtp_header_extensions_; - // Set by the field trial WebRTC-ForcePlayoutDelay to override any playout - // delay that is specified in the received packets. - FieldTrialOptional<int> forced_playout_delay_max_ms_; - FieldTrialOptional<int> forced_playout_delay_min_ms_; - ReceiveStatistics* const rtp_receive_statistics_; - std::unique_ptr<UlpfecReceiver> ulpfec_receiver_; - - RTC_NO_UNIQUE_ADDRESS SequenceChecker worker_task_checker_; - bool receiving_ RTC_GUARDED_BY(worker_task_checker_); - int64_t last_packet_log_ms_ RTC_GUARDED_BY(worker_task_checker_); - - const std::unique_ptr<RtpRtcp> rtp_rtcp_; - - OnCompleteFrameCallback* complete_frame_callback_; - KeyFrameRequestSender* const keyframe_request_sender_; - - RtcpFeedbackBuffer rtcp_feedback_buffer_; - std::unique_ptr<DEPRECATED_NackModule> nack_module_; - std::unique_ptr<LossNotificationController> loss_notification_controller_; - - mutable Mutex packet_buffer_lock_; - video_coding::PacketBuffer packet_buffer_ RTC_GUARDED_BY(packet_buffer_lock_); - UniqueTimestampCounter frame_counter_ RTC_GUARDED_BY(worker_task_checker_); - SeqNumUnwrapper<uint16_t> frame_id_unwrapper_ - RTC_GUARDED_BY(worker_task_checker_); - - // Video structure provided in the dependency descriptor in a first packet - // of a key frame. It is required to parse dependency descriptor in the - // following delta packets. - std::unique_ptr<FrameDependencyStructure> video_structure_ - RTC_GUARDED_BY(worker_task_checker_); - // Frame id of the last frame with the attached video structure. - // absl::nullopt when `video_structure_ == nullptr`; - absl::optional<int64_t> video_structure_frame_id_ - RTC_GUARDED_BY(worker_task_checker_); - - Mutex reference_finder_lock_; - std::unique_ptr<RtpFrameReferenceFinder> reference_finder_ - RTC_GUARDED_BY(reference_finder_lock_); - absl::optional<VideoCodecType> current_codec_; - uint32_t last_assembled_frame_rtp_timestamp_; - - Mutex last_seq_num_mutex_; - std::map<int64_t, uint16_t> last_seq_num_for_pic_id_ - RTC_GUARDED_BY(last_seq_num_mutex_); - video_coding::H264SpsPpsTracker tracker_; - - // Maps payload id to the depacketizer. - std::map<uint8_t, std::unique_ptr<VideoRtpDepacketizer>> payload_type_map_; - - // TODO(johan): Remove pt_codec_params_ once - // https://bugs.chromium.org/p/webrtc/issues/detail?id=6883 is resolved. - // Maps a payload type to a map of out-of-band supplied codec parameters. - std::map<uint8_t, std::map<std::string, std::string>> pt_codec_params_; - int16_t last_payload_type_ = -1; - - bool has_received_frame_; - - std::vector<RtpPacketSinkInterface*> secondary_sinks_ - RTC_GUARDED_BY(worker_task_checker_); - - // Info for GetSyncInfo is updated on network or worker thread, and queried on - // the worker thread. - mutable Mutex sync_info_lock_; - absl::optional<uint32_t> last_received_rtp_timestamp_ - RTC_GUARDED_BY(sync_info_lock_); - absl::optional<uint32_t> last_received_keyframe_rtp_timestamp_ - RTC_GUARDED_BY(sync_info_lock_); - absl::optional<Timestamp> last_received_rtp_system_time_ - RTC_GUARDED_BY(sync_info_lock_); - absl::optional<Timestamp> last_received_keyframe_rtp_system_time_ - RTC_GUARDED_BY(sync_info_lock_); - - // Used to validate the buffered frame decryptor is always run on the correct - // thread. - SequenceChecker network_tc_; - // Handles incoming encrypted frames and forwards them to the - // rtp_reference_finder if they are decryptable. - std::unique_ptr<BufferedFrameDecryptor> buffered_frame_decryptor_ - RTC_PT_GUARDED_BY(network_tc_); - std::atomic<bool> frames_decryptable_; - absl::optional<ColorSpace> last_color_space_; - - AbsoluteCaptureTimeInterpolator absolute_capture_time_interpolator_ - RTC_GUARDED_BY(worker_task_checker_); - - CaptureClockOffsetUpdater capture_clock_offset_updater_ - RTC_GUARDED_BY(worker_task_checker_); - - int64_t last_completed_picture_id_ = 0; - - rtc::scoped_refptr<RtpVideoStreamReceiverFrameTransformerDelegate> - frame_transformer_delegate_; - - SeqNumUnwrapper<uint16_t> rtp_seq_num_unwrapper_ - RTC_GUARDED_BY(packet_buffer_lock_); - std::map<int64_t, RtpPacketInfo> packet_infos_ - RTC_GUARDED_BY(packet_buffer_lock_); -}; - -} // namespace webrtc - -#endif // VIDEO_RTP_VIDEO_STREAM_RECEIVER_H_ diff --git a/video/rtp_video_stream_receiver_unittest.cc b/video/rtp_video_stream_receiver_unittest.cc deleted file mode 100644 index bfe9a4cc89..0000000000 --- a/video/rtp_video_stream_receiver_unittest.cc +++ /dev/null @@ -1,1238 +0,0 @@ -/* - * Copyright 2017 The WebRTC Project Authors. All rights reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "video/rtp_video_stream_receiver.h" - -#include <memory> -#include <utility> - -#include "api/video/video_codec_type.h" -#include "api/video/video_frame_type.h" -#include "call/test/mock_rtp_packet_sink_interface.h" -#include "common_video/h264/h264_common.h" -#include "media/base/media_constants.h" -#include "modules/rtp_rtcp/source/rtp_descriptor_authentication.h" -#include "modules/rtp_rtcp/source/rtp_format.h" -#include "modules/rtp_rtcp/source/rtp_format_vp9.h" -#include "modules/rtp_rtcp/source/rtp_generic_frame_descriptor.h" -#include "modules/rtp_rtcp/source/rtp_generic_frame_descriptor_extension.h" -#include "modules/rtp_rtcp/source/rtp_header_extensions.h" -#include "modules/rtp_rtcp/source/rtp_packet_received.h" -#include "modules/rtp_rtcp/source/rtp_packet_to_send.h" -#include "modules/utility/include/process_thread.h" -#include "modules/video_coding/frame_object.h" -#include "modules/video_coding/include/video_coding_defines.h" -#include "modules/video_coding/rtp_frame_reference_finder.h" -#include "rtc_base/byte_buffer.h" -#include "rtc_base/logging.h" -#include "system_wrappers/include/clock.h" -#include "test/gmock.h" -#include "test/gtest.h" -#include "test/mock_frame_transformer.h" -#include "test/mock_transport.h" -#include "test/scoped_key_value_config.h" - -using ::testing::_; -using ::testing::ElementsAre; -using ::testing::Invoke; -using ::testing::SizeIs; -using ::testing::Values; - -namespace webrtc { - -namespace { - -const uint8_t kH264StartCode[] = {0x00, 0x00, 0x00, 0x01}; - -std::vector<uint64_t> GetAbsoluteCaptureTimestamps(const EncodedFrame* frame) { - std::vector<uint64_t> result; - for (const auto& packet_info : frame->PacketInfos()) { - if (packet_info.absolute_capture_time()) { - result.push_back( - packet_info.absolute_capture_time()->absolute_capture_timestamp); - } - } - return result; -} - -RTPVideoHeader GetGenericVideoHeader(VideoFrameType frame_type) { - RTPVideoHeader video_header; - video_header.is_first_packet_in_frame = true; - video_header.is_last_packet_in_frame = true; - video_header.codec = kVideoCodecGeneric; - video_header.frame_type = frame_type; - return video_header; -} - -class MockNackSender : public NackSender { - public: - MOCK_METHOD(void, - SendNack, - (const std::vector<uint16_t>& sequence_numbers, - bool buffering_allowed), - (override)); -}; - -class MockKeyFrameRequestSender : public KeyFrameRequestSender { - public: - MOCK_METHOD(void, RequestKeyFrame, (), (override)); -}; - -class MockOnCompleteFrameCallback - : public RtpVideoStreamReceiver::OnCompleteFrameCallback { - public: - MOCK_METHOD(void, DoOnCompleteFrame, (EncodedFrame*), ()); - MOCK_METHOD(void, DoOnCompleteFrameFailNullptr, (EncodedFrame*), ()); - MOCK_METHOD(void, DoOnCompleteFrameFailLength, (EncodedFrame*), ()); - MOCK_METHOD(void, DoOnCompleteFrameFailBitstream, (EncodedFrame*), ()); - void OnCompleteFrame(std::unique_ptr<EncodedFrame> frame) override { - if (!frame) { - DoOnCompleteFrameFailNullptr(nullptr); - return; - } - EXPECT_EQ(buffer_.Length(), frame->size()); - if (buffer_.Length() != frame->size()) { - DoOnCompleteFrameFailLength(frame.get()); - return; - } - if (frame->size() != buffer_.Length() || - memcmp(buffer_.Data(), frame->data(), buffer_.Length()) != 0) { - DoOnCompleteFrameFailBitstream(frame.get()); - return; - } - DoOnCompleteFrame(frame.get()); - } - - void ClearExpectedBitstream() { buffer_.Clear(); } - - void AppendExpectedBitstream(const uint8_t data[], size_t size_in_bytes) { - // TODO(Johan): Let rtc::ByteBuffer handle uint8_t* instead of char*. - buffer_.WriteBytes(reinterpret_cast<const char*>(data), size_in_bytes); - } - rtc::ByteBufferWriter buffer_; -}; - -constexpr uint32_t kSsrc = 111; -constexpr uint16_t kSequenceNumber = 222; -constexpr int kPayloadType = 100; -constexpr int kRedPayloadType = 125; - -std::unique_ptr<RtpPacketReceived> CreateRtpPacketReceived() { - auto packet = std::make_unique<RtpPacketReceived>(); - packet->SetSsrc(kSsrc); - packet->SetSequenceNumber(kSequenceNumber); - packet->SetPayloadType(kPayloadType); - return packet; -} - -MATCHER_P(SamePacketAs, other, "") { - return arg.Ssrc() == other.Ssrc() && - arg.SequenceNumber() == other.SequenceNumber(); -} - -} // namespace - -class RtpVideoStreamReceiverTest : public ::testing::Test { - public: - RtpVideoStreamReceiverTest() : RtpVideoStreamReceiverTest("") {} - explicit RtpVideoStreamReceiverTest(std::string field_trials) - : field_trials_(field_trials), - config_(CreateConfig()), - process_thread_(ProcessThread::Create("TestThread")) { - rtp_receive_statistics_ = - ReceiveStatistics::Create(Clock::GetRealTimeClock()); - rtp_video_stream_receiver_ = std::make_unique<RtpVideoStreamReceiver>( - Clock::GetRealTimeClock(), &mock_transport_, nullptr, nullptr, &config_, - rtp_receive_statistics_.get(), nullptr, nullptr, process_thread_.get(), - &mock_nack_sender_, &mock_key_frame_request_sender_, - &mock_on_complete_frame_callback_, nullptr, nullptr, &field_trials_); - rtp_video_stream_receiver_->AddReceiveCodec(kPayloadType, - kVideoCodecGeneric, {}, - /*raw_payload=*/false); - } - - RTPVideoHeader GetDefaultH264VideoHeader() { - RTPVideoHeader video_header; - video_header.codec = kVideoCodecH264; - video_header.video_type_header.emplace<RTPVideoHeaderH264>(); - return video_header; - } - - // TODO(Johan): refactor h264_sps_pps_tracker_unittests.cc to avoid duplicate - // code. - void AddSps(RTPVideoHeader* video_header, - uint8_t sps_id, - rtc::CopyOnWriteBuffer* data) { - NaluInfo info; - info.type = H264::NaluType::kSps; - info.sps_id = sps_id; - info.pps_id = -1; - data->AppendData<uint8_t, 2>({H264::NaluType::kSps, sps_id}); - auto& h264 = absl::get<RTPVideoHeaderH264>(video_header->video_type_header); - h264.nalus[h264.nalus_length++] = info; - } - - void AddPps(RTPVideoHeader* video_header, - uint8_t sps_id, - uint8_t pps_id, - rtc::CopyOnWriteBuffer* data) { - NaluInfo info; - info.type = H264::NaluType::kPps; - info.sps_id = sps_id; - info.pps_id = pps_id; - data->AppendData<uint8_t, 2>({H264::NaluType::kPps, pps_id}); - auto& h264 = absl::get<RTPVideoHeaderH264>(video_header->video_type_header); - h264.nalus[h264.nalus_length++] = info; - } - - void AddIdr(RTPVideoHeader* video_header, int pps_id) { - NaluInfo info; - info.type = H264::NaluType::kIdr; - info.sps_id = -1; - info.pps_id = pps_id; - auto& h264 = absl::get<RTPVideoHeaderH264>(video_header->video_type_header); - h264.nalus[h264.nalus_length++] = info; - } - - protected: - static VideoReceiveStreamInterface::Config CreateConfig() { - VideoReceiveStreamInterface::Config config(nullptr); - config.rtp.remote_ssrc = 1111; - config.rtp.local_ssrc = 2222; - config.rtp.red_payload_type = kRedPayloadType; - return config; - } - - webrtc::test::ScopedKeyValueConfig field_trials_; - VideoReceiveStreamInterface::Config config_; - MockNackSender mock_nack_sender_; - MockKeyFrameRequestSender mock_key_frame_request_sender_; - MockTransport mock_transport_; - MockOnCompleteFrameCallback mock_on_complete_frame_callback_; - std::unique_ptr<ProcessThread> process_thread_; - std::unique_ptr<ReceiveStatistics> rtp_receive_statistics_; - std::unique_ptr<RtpVideoStreamReceiver> rtp_video_stream_receiver_; -}; - -TEST_F(RtpVideoStreamReceiverTest, CacheColorSpaceFromLastPacketOfKeyframe) { - // Test that color space is cached from the last packet of a key frame and - // that it's not reset by padding packets without color space. - constexpr int kVp9PayloadType = 99; - const ColorSpace kColorSpace( - ColorSpace::PrimaryID::kFILM, ColorSpace::TransferID::kBT2020_12, - ColorSpace::MatrixID::kBT2020_NCL, ColorSpace::RangeID::kFull); - const std::vector<uint8_t> kKeyFramePayload = {0, 1, 2, 3, 4, 5, - 6, 7, 8, 9, 10}; - const std::vector<uint8_t> kDeltaFramePayload = {0, 1, 2, 3, 4}; - - // Anonymous helper class that generates received packets. - class { - public: - void SetPayload(const std::vector<uint8_t>& payload, - VideoFrameType video_frame_type) { - video_frame_type_ = video_frame_type; - RtpPacketizer::PayloadSizeLimits pay_load_size_limits; - // Reduce max payload length to make sure the key frame generates two - // packets. - pay_load_size_limits.max_payload_len = 8; - RTPVideoHeaderVP9 rtp_video_header_vp9; - rtp_video_header_vp9.InitRTPVideoHeaderVP9(); - rtp_video_header_vp9.inter_pic_predicted = - (video_frame_type == VideoFrameType::kVideoFrameDelta); - rtp_packetizer_ = std::make_unique<RtpPacketizerVp9>( - payload, pay_load_size_limits, rtp_video_header_vp9); - } - - size_t NumPackets() { return rtp_packetizer_->NumPackets(); } - void SetColorSpace(const ColorSpace& color_space) { - color_space_ = color_space; - } - - RtpPacketReceived NextPacket() { - RtpHeaderExtensionMap extension_map; - extension_map.Register<ColorSpaceExtension>(1); - RtpPacketToSend packet_to_send(&extension_map); - packet_to_send.SetSequenceNumber(sequence_number_++); - packet_to_send.SetSsrc(kSsrc); - packet_to_send.SetPayloadType(kVp9PayloadType); - bool include_color_space = - (rtp_packetizer_->NumPackets() == 1u && - video_frame_type_ == VideoFrameType::kVideoFrameKey); - if (include_color_space) { - EXPECT_TRUE( - packet_to_send.SetExtension<ColorSpaceExtension>(color_space_)); - } - rtp_packetizer_->NextPacket(&packet_to_send); - - RtpPacketReceived received_packet(&extension_map); - received_packet.Parse(packet_to_send.data(), packet_to_send.size()); - return received_packet; - } - - private: - uint16_t sequence_number_ = 0; - VideoFrameType video_frame_type_; - ColorSpace color_space_; - std::unique_ptr<RtpPacketizer> rtp_packetizer_; - } received_packet_generator; - received_packet_generator.SetColorSpace(kColorSpace); - - // Prepare the receiver for VP9. - std::map<std::string, std::string> codec_params; - rtp_video_stream_receiver_->AddReceiveCodec(kVp9PayloadType, kVideoCodecVP9, - codec_params, - /*raw_payload=*/false); - - // Generate key frame packets. - received_packet_generator.SetPayload(kKeyFramePayload, - VideoFrameType::kVideoFrameKey); - EXPECT_EQ(received_packet_generator.NumPackets(), 2u); - RtpPacketReceived key_frame_packet1 = received_packet_generator.NextPacket(); - RtpPacketReceived key_frame_packet2 = received_packet_generator.NextPacket(); - - // Generate delta frame packet. - received_packet_generator.SetPayload(kDeltaFramePayload, - VideoFrameType::kVideoFrameDelta); - EXPECT_EQ(received_packet_generator.NumPackets(), 1u); - RtpPacketReceived delta_frame_packet = received_packet_generator.NextPacket(); - - rtp_video_stream_receiver_->StartReceive(); - mock_on_complete_frame_callback_.AppendExpectedBitstream( - kKeyFramePayload.data(), kKeyFramePayload.size()); - - // Send the key frame and expect a callback with color space information. - EXPECT_FALSE(key_frame_packet1.GetExtension<ColorSpaceExtension>()); - EXPECT_TRUE(key_frame_packet2.GetExtension<ColorSpaceExtension>()); - rtp_video_stream_receiver_->OnRtpPacket(key_frame_packet1); - EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_)) - .WillOnce(Invoke([kColorSpace](EncodedFrame* frame) { - ASSERT_TRUE(frame->EncodedImage().ColorSpace()); - EXPECT_EQ(*frame->EncodedImage().ColorSpace(), kColorSpace); - })); - rtp_video_stream_receiver_->OnRtpPacket(key_frame_packet2); - // Resend the first key frame packet to simulate padding for example. - rtp_video_stream_receiver_->OnRtpPacket(key_frame_packet1); - - mock_on_complete_frame_callback_.ClearExpectedBitstream(); - mock_on_complete_frame_callback_.AppendExpectedBitstream( - kDeltaFramePayload.data(), kDeltaFramePayload.size()); - - // Expect delta frame to have color space set even though color space not - // included in the RTP packet. - EXPECT_FALSE(delta_frame_packet.GetExtension<ColorSpaceExtension>()); - EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_)) - .WillOnce(Invoke([kColorSpace](EncodedFrame* frame) { - ASSERT_TRUE(frame->EncodedImage().ColorSpace()); - EXPECT_EQ(*frame->EncodedImage().ColorSpace(), kColorSpace); - })); - rtp_video_stream_receiver_->OnRtpPacket(delta_frame_packet); -} - -TEST_F(RtpVideoStreamReceiverTest, GenericKeyFrame) { - RtpPacketReceived rtp_packet; - rtc::CopyOnWriteBuffer data({'1', '2', '3', '4'}); - rtp_packet.SetPayloadType(kPayloadType); - rtp_packet.SetSequenceNumber(1); - RTPVideoHeader video_header = - GetGenericVideoHeader(VideoFrameType::kVideoFrameKey); - mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data(), - data.size()); - EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_)); - rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet, - video_header); -} - -TEST_F(RtpVideoStreamReceiverTest, PacketInfoIsPropagatedIntoVideoFrames) { - constexpr uint64_t kAbsoluteCaptureTimestamp = 12; - constexpr int kId0 = 1; - - RtpHeaderExtensionMap extension_map; - extension_map.Register<AbsoluteCaptureTimeExtension>(kId0); - RtpPacketReceived rtp_packet(&extension_map); - rtp_packet.SetPayloadType(kPayloadType); - rtc::CopyOnWriteBuffer data({'1', '2', '3', '4'}); - rtp_packet.SetSequenceNumber(1); - rtp_packet.SetTimestamp(1); - rtp_packet.SetSsrc(kSsrc); - rtp_packet.SetExtension<AbsoluteCaptureTimeExtension>( - AbsoluteCaptureTime{kAbsoluteCaptureTimestamp, - /*estimated_capture_clock_offset=*/absl::nullopt}); - - RTPVideoHeader video_header = - GetGenericVideoHeader(VideoFrameType::kVideoFrameKey); - mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data(), - data.size()); - EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_)) - .WillOnce(Invoke([kAbsoluteCaptureTimestamp](EncodedFrame* frame) { - EXPECT_THAT(GetAbsoluteCaptureTimestamps(frame), - ElementsAre(kAbsoluteCaptureTimestamp)); - })); - rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet, - video_header); -} - -TEST_F(RtpVideoStreamReceiverTest, - MissingAbsoluteCaptureTimeIsFilledWithExtrapolatedValue) { - constexpr uint64_t kAbsoluteCaptureTimestamp = 12; - constexpr int kId0 = 1; - - RtpHeaderExtensionMap extension_map; - extension_map.Register<AbsoluteCaptureTimeExtension>(kId0); - RtpPacketReceived rtp_packet(&extension_map); - rtp_packet.SetPayloadType(kPayloadType); - - rtc::CopyOnWriteBuffer data({'1', '2', '3', '4'}); - uint16_t sequence_number = 1; - uint32_t rtp_timestamp = 1; - rtp_packet.SetSequenceNumber(sequence_number); - rtp_packet.SetTimestamp(rtp_timestamp); - rtp_packet.SetSsrc(kSsrc); - rtp_packet.SetExtension<AbsoluteCaptureTimeExtension>( - AbsoluteCaptureTime{kAbsoluteCaptureTimestamp, - /*estimated_capture_clock_offset=*/absl::nullopt}); - - RTPVideoHeader video_header = - GetGenericVideoHeader(VideoFrameType::kVideoFrameKey); - mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data(), - data.size()); - EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_)); - rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet, - video_header); - - // Rtp packet without absolute capture time. - rtp_packet = RtpPacketReceived(&extension_map); - rtp_packet.SetPayloadType(kPayloadType); - rtp_packet.SetSequenceNumber(++sequence_number); - rtp_packet.SetTimestamp(++rtp_timestamp); - rtp_packet.SetSsrc(kSsrc); - - // There is no absolute capture time in the second packet. - // Expect rtp video stream receiver to extrapolate it for the resulting video - // frame using absolute capture time from the previous packet. - EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_)) - .WillOnce(Invoke([](EncodedFrame* frame) { - EXPECT_THAT(GetAbsoluteCaptureTimestamps(frame), SizeIs(1)); - })); - rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet, - video_header); -} - -TEST_F(RtpVideoStreamReceiverTest, NoInfiniteRecursionOnEncapsulatedRedPacket) { - const std::vector<uint8_t> data({ - 0x80, // RTP version. - kRedPayloadType, // Payload type. - 0, 0, 0, 0, 0, 0, // Don't care. - 0, 0, 0x4, 0x57, // SSRC - kRedPayloadType, // RED header. - 0, 0, 0, 0, 0 // Don't care. - }); - RtpPacketReceived packet; - EXPECT_TRUE(packet.Parse(data.data(), data.size())); - rtp_video_stream_receiver_->StartReceive(); - rtp_video_stream_receiver_->OnRtpPacket(packet); -} - -TEST_F(RtpVideoStreamReceiverTest, - DropsPacketWithRedPayloadTypeAndEmptyPayload) { - const uint8_t kRedPayloadType = 125; - config_.rtp.red_payload_type = kRedPayloadType; - SetUp(); // re-create rtp_video_stream_receiver with red payload type. - // clang-format off - const uint8_t data[] = { - 0x80, // RTP version. - kRedPayloadType, // Payload type. - 0, 0, 0, 0, 0, 0, // Don't care. - 0, 0, 0x4, 0x57, // SSRC - // Empty rtp payload. - }; - // clang-format on - RtpPacketReceived packet; - // Manually convert to CopyOnWriteBuffer to be sure capacity == size - // and asan bot can catch read buffer overflow. - EXPECT_TRUE(packet.Parse(rtc::CopyOnWriteBuffer(data))); - rtp_video_stream_receiver_->StartReceive(); - rtp_video_stream_receiver_->OnRtpPacket(packet); - // Expect asan doesn't find anything. -} - -TEST_F(RtpVideoStreamReceiverTest, GenericKeyFrameBitstreamError) { - RtpPacketReceived rtp_packet; - rtp_packet.SetPayloadType(kPayloadType); - rtc::CopyOnWriteBuffer data({'1', '2', '3', '4'}); - rtp_packet.SetSequenceNumber(1); - RTPVideoHeader video_header = - GetGenericVideoHeader(VideoFrameType::kVideoFrameKey); - constexpr uint8_t expected_bitsteam[] = {1, 2, 3, 0xff}; - mock_on_complete_frame_callback_.AppendExpectedBitstream( - expected_bitsteam, sizeof(expected_bitsteam)); - EXPECT_CALL(mock_on_complete_frame_callback_, - DoOnCompleteFrameFailBitstream(_)); - rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet, - video_header); -} - -class RtpVideoStreamReceiverTestH264 - : public RtpVideoStreamReceiverTest, - public ::testing::WithParamInterface<std::string> { - protected: - RtpVideoStreamReceiverTestH264() : RtpVideoStreamReceiverTest(GetParam()) {} -}; - -INSTANTIATE_TEST_SUITE_P(SpsPpsIdrIsKeyframe, - RtpVideoStreamReceiverTestH264, - Values("", "WebRTC-SpsPpsIdrIsH264Keyframe/Enabled/")); - -TEST_P(RtpVideoStreamReceiverTestH264, InBandSpsPps) { - rtc::CopyOnWriteBuffer sps_data; - RtpPacketReceived rtp_packet; - RTPVideoHeader sps_video_header = GetDefaultH264VideoHeader(); - AddSps(&sps_video_header, 0, &sps_data); - rtp_packet.SetSequenceNumber(0); - rtp_packet.SetPayloadType(kPayloadType); - sps_video_header.is_first_packet_in_frame = true; - sps_video_header.frame_type = VideoFrameType::kEmptyFrame; - mock_on_complete_frame_callback_.AppendExpectedBitstream( - kH264StartCode, sizeof(kH264StartCode)); - mock_on_complete_frame_callback_.AppendExpectedBitstream(sps_data.data(), - sps_data.size()); - rtp_video_stream_receiver_->OnReceivedPayloadData(sps_data, rtp_packet, - sps_video_header); - - rtc::CopyOnWriteBuffer pps_data; - RTPVideoHeader pps_video_header = GetDefaultH264VideoHeader(); - AddPps(&pps_video_header, 0, 1, &pps_data); - rtp_packet.SetSequenceNumber(1); - pps_video_header.is_first_packet_in_frame = true; - pps_video_header.frame_type = VideoFrameType::kEmptyFrame; - mock_on_complete_frame_callback_.AppendExpectedBitstream( - kH264StartCode, sizeof(kH264StartCode)); - mock_on_complete_frame_callback_.AppendExpectedBitstream(pps_data.data(), - pps_data.size()); - rtp_video_stream_receiver_->OnReceivedPayloadData(pps_data, rtp_packet, - pps_video_header); - - rtc::CopyOnWriteBuffer idr_data; - RTPVideoHeader idr_video_header = GetDefaultH264VideoHeader(); - AddIdr(&idr_video_header, 1); - rtp_packet.SetSequenceNumber(2); - idr_video_header.is_first_packet_in_frame = true; - idr_video_header.is_last_packet_in_frame = true; - idr_video_header.frame_type = VideoFrameType::kVideoFrameKey; - const uint8_t idr[] = {0x65, 1, 2, 3}; - idr_data.AppendData(idr); - mock_on_complete_frame_callback_.AppendExpectedBitstream( - kH264StartCode, sizeof(kH264StartCode)); - mock_on_complete_frame_callback_.AppendExpectedBitstream(idr_data.data(), - idr_data.size()); - EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_)); - rtp_video_stream_receiver_->OnReceivedPayloadData(idr_data, rtp_packet, - idr_video_header); -} - -TEST_P(RtpVideoStreamReceiverTestH264, OutOfBandFmtpSpsPps) { - constexpr int kPayloadType = 99; - std::map<std::string, std::string> codec_params; - // Example parameter sets from https://tools.ietf.org/html/rfc3984#section-8.2 - // . - codec_params.insert( - {cricket::kH264FmtpSpropParameterSets, "Z0IACpZTBYmI,aMljiA=="}); - rtp_video_stream_receiver_->AddReceiveCodec(kPayloadType, kVideoCodecH264, - codec_params, - /*raw_payload=*/false); - const uint8_t binary_sps[] = {0x67, 0x42, 0x00, 0x0a, 0x96, - 0x53, 0x05, 0x89, 0x88}; - mock_on_complete_frame_callback_.AppendExpectedBitstream( - kH264StartCode, sizeof(kH264StartCode)); - mock_on_complete_frame_callback_.AppendExpectedBitstream(binary_sps, - sizeof(binary_sps)); - const uint8_t binary_pps[] = {0x68, 0xc9, 0x63, 0x88}; - mock_on_complete_frame_callback_.AppendExpectedBitstream( - kH264StartCode, sizeof(kH264StartCode)); - mock_on_complete_frame_callback_.AppendExpectedBitstream(binary_pps, - sizeof(binary_pps)); - - RtpPacketReceived rtp_packet; - RTPVideoHeader video_header = GetDefaultH264VideoHeader(); - AddIdr(&video_header, 0); - rtp_packet.SetPayloadType(kPayloadType); - rtp_packet.SetSequenceNumber(2); - video_header.is_first_packet_in_frame = true; - video_header.is_last_packet_in_frame = true; - video_header.codec = kVideoCodecH264; - video_header.frame_type = VideoFrameType::kVideoFrameKey; - rtc::CopyOnWriteBuffer data({'1', '2', '3'}); - mock_on_complete_frame_callback_.AppendExpectedBitstream( - kH264StartCode, sizeof(kH264StartCode)); - mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data(), - data.size()); - EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_)); - rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet, - video_header); -} - -TEST_P(RtpVideoStreamReceiverTestH264, ForceSpsPpsIdrIsKeyframe) { - constexpr int kPayloadType = 99; - std::map<std::string, std::string> codec_params; - if (GetParam() == - "") { // Forcing can be done either with field trial or codec_params. - codec_params.insert({cricket::kH264FmtpSpsPpsIdrInKeyframe, ""}); - } - rtp_video_stream_receiver_->AddReceiveCodec(kPayloadType, kVideoCodecH264, - codec_params, - /*raw_payload=*/false); - rtc::CopyOnWriteBuffer sps_data; - RtpPacketReceived rtp_packet; - RTPVideoHeader sps_video_header = GetDefaultH264VideoHeader(); - AddSps(&sps_video_header, 0, &sps_data); - rtp_packet.SetSequenceNumber(0); - rtp_packet.SetPayloadType(kPayloadType); - sps_video_header.is_first_packet_in_frame = true; - sps_video_header.frame_type = VideoFrameType::kEmptyFrame; - mock_on_complete_frame_callback_.AppendExpectedBitstream( - kH264StartCode, sizeof(kH264StartCode)); - mock_on_complete_frame_callback_.AppendExpectedBitstream(sps_data.data(), - sps_data.size()); - rtp_video_stream_receiver_->OnReceivedPayloadData(sps_data, rtp_packet, - sps_video_header); - - rtc::CopyOnWriteBuffer pps_data; - RTPVideoHeader pps_video_header = GetDefaultH264VideoHeader(); - AddPps(&pps_video_header, 0, 1, &pps_data); - rtp_packet.SetSequenceNumber(1); - pps_video_header.is_first_packet_in_frame = true; - pps_video_header.frame_type = VideoFrameType::kEmptyFrame; - mock_on_complete_frame_callback_.AppendExpectedBitstream( - kH264StartCode, sizeof(kH264StartCode)); - mock_on_complete_frame_callback_.AppendExpectedBitstream(pps_data.data(), - pps_data.size()); - rtp_video_stream_receiver_->OnReceivedPayloadData(pps_data, rtp_packet, - pps_video_header); - - rtc::CopyOnWriteBuffer idr_data; - RTPVideoHeader idr_video_header = GetDefaultH264VideoHeader(); - AddIdr(&idr_video_header, 1); - rtp_packet.SetSequenceNumber(2); - idr_video_header.is_first_packet_in_frame = true; - idr_video_header.is_last_packet_in_frame = true; - idr_video_header.frame_type = VideoFrameType::kVideoFrameKey; - const uint8_t idr[] = {0x65, 1, 2, 3}; - idr_data.AppendData(idr); - mock_on_complete_frame_callback_.AppendExpectedBitstream( - kH264StartCode, sizeof(kH264StartCode)); - mock_on_complete_frame_callback_.AppendExpectedBitstream(idr_data.data(), - idr_data.size()); - EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame) - .WillOnce( - [&](EncodedFrame* frame) { EXPECT_TRUE(frame->is_keyframe()); }); - rtp_video_stream_receiver_->OnReceivedPayloadData(idr_data, rtp_packet, - idr_video_header); - mock_on_complete_frame_callback_.ClearExpectedBitstream(); - mock_on_complete_frame_callback_.AppendExpectedBitstream( - kH264StartCode, sizeof(kH264StartCode)); - mock_on_complete_frame_callback_.AppendExpectedBitstream(idr_data.data(), - idr_data.size()); - rtp_packet.SetSequenceNumber(3); - EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame) - .WillOnce( - [&](EncodedFrame* frame) { EXPECT_FALSE(frame->is_keyframe()); }); - rtp_video_stream_receiver_->OnReceivedPayloadData(idr_data, rtp_packet, - idr_video_header); -} - -TEST_F(RtpVideoStreamReceiverTest, PaddingInMediaStream) { - RtpPacketReceived rtp_packet; - RTPVideoHeader video_header = GetDefaultH264VideoHeader(); - rtc::CopyOnWriteBuffer data({'1', '2', '3'}); - rtp_packet.SetPayloadType(kPayloadType); - rtp_packet.SetSequenceNumber(2); - video_header.is_first_packet_in_frame = true; - video_header.is_last_packet_in_frame = true; - video_header.codec = kVideoCodecGeneric; - video_header.frame_type = VideoFrameType::kVideoFrameKey; - mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data(), - data.size()); - - EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_)); - rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet, - video_header); - - rtp_packet.SetSequenceNumber(3); - rtp_video_stream_receiver_->OnReceivedPayloadData({}, rtp_packet, - video_header); - - rtp_packet.SetSequenceNumber(4); - EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_)); - video_header.frame_type = VideoFrameType::kVideoFrameDelta; - rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet, - video_header); - - rtp_packet.SetSequenceNumber(6); - rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet, - video_header); - - EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_)); - rtp_packet.SetSequenceNumber(5); - rtp_video_stream_receiver_->OnReceivedPayloadData({}, rtp_packet, - video_header); -} - -TEST_F(RtpVideoStreamReceiverTest, RequestKeyframeIfFirstFrameIsDelta) { - RtpPacketReceived rtp_packet; - rtp_packet.SetPayloadType(kPayloadType); - rtc::CopyOnWriteBuffer data({'1', '2', '3', '4'}); - rtp_packet.SetSequenceNumber(1); - RTPVideoHeader video_header = - GetGenericVideoHeader(VideoFrameType::kVideoFrameDelta); - EXPECT_CALL(mock_key_frame_request_sender_, RequestKeyFrame()); - rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet, - video_header); -} - -TEST_F(RtpVideoStreamReceiverTest, RequestKeyframeWhenPacketBufferGetsFull) { - constexpr int kPacketBufferMaxSize = 2048; - - RtpPacketReceived rtp_packet; - rtp_packet.SetPayloadType(kPayloadType); - rtc::CopyOnWriteBuffer data({'1', '2', '3', '4'}); - RTPVideoHeader video_header = - GetGenericVideoHeader(VideoFrameType::kVideoFrameDelta); - // Incomplete frames so that the packet buffer is filling up. - video_header.is_last_packet_in_frame = false; - uint16_t start_sequence_number = 1234; - rtp_packet.SetSequenceNumber(start_sequence_number); - while (rtp_packet.SequenceNumber() - start_sequence_number < - kPacketBufferMaxSize) { - rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet, - video_header); - rtp_packet.SetSequenceNumber(rtp_packet.SequenceNumber() + 2); - } - - EXPECT_CALL(mock_key_frame_request_sender_, RequestKeyFrame()); - rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet, - video_header); -} - -TEST_F(RtpVideoStreamReceiverTest, SecondarySinksGetRtpNotifications) { - rtp_video_stream_receiver_->StartReceive(); - - MockRtpPacketSink secondary_sink_1; - MockRtpPacketSink secondary_sink_2; - - rtp_video_stream_receiver_->AddSecondarySink(&secondary_sink_1); - rtp_video_stream_receiver_->AddSecondarySink(&secondary_sink_2); - - auto rtp_packet = CreateRtpPacketReceived(); - EXPECT_CALL(secondary_sink_1, OnRtpPacket(SamePacketAs(*rtp_packet))); - EXPECT_CALL(secondary_sink_2, OnRtpPacket(SamePacketAs(*rtp_packet))); - - rtp_video_stream_receiver_->OnRtpPacket(*rtp_packet); - - // Test tear-down. - rtp_video_stream_receiver_->StopReceive(); - rtp_video_stream_receiver_->RemoveSecondarySink(&secondary_sink_1); - rtp_video_stream_receiver_->RemoveSecondarySink(&secondary_sink_2); -} - -TEST_F(RtpVideoStreamReceiverTest, RemovedSecondarySinksGetNoRtpNotifications) { - rtp_video_stream_receiver_->StartReceive(); - - MockRtpPacketSink secondary_sink; - - rtp_video_stream_receiver_->AddSecondarySink(&secondary_sink); - rtp_video_stream_receiver_->RemoveSecondarySink(&secondary_sink); - - auto rtp_packet = CreateRtpPacketReceived(); - - EXPECT_CALL(secondary_sink, OnRtpPacket(_)).Times(0); - - rtp_video_stream_receiver_->OnRtpPacket(*rtp_packet); - - // Test tear-down. - rtp_video_stream_receiver_->StopReceive(); -} - -TEST_F(RtpVideoStreamReceiverTest, - OnlyRemovedSecondarySinksExcludedFromNotifications) { - rtp_video_stream_receiver_->StartReceive(); - - MockRtpPacketSink kept_secondary_sink; - MockRtpPacketSink removed_secondary_sink; - - rtp_video_stream_receiver_->AddSecondarySink(&kept_secondary_sink); - rtp_video_stream_receiver_->AddSecondarySink(&removed_secondary_sink); - rtp_video_stream_receiver_->RemoveSecondarySink(&removed_secondary_sink); - - auto rtp_packet = CreateRtpPacketReceived(); - EXPECT_CALL(kept_secondary_sink, OnRtpPacket(SamePacketAs(*rtp_packet))); - - rtp_video_stream_receiver_->OnRtpPacket(*rtp_packet); - - // Test tear-down. - rtp_video_stream_receiver_->StopReceive(); - rtp_video_stream_receiver_->RemoveSecondarySink(&kept_secondary_sink); -} - -TEST_F(RtpVideoStreamReceiverTest, - SecondariesOfNonStartedStreamGetNoNotifications) { - // Explicitly showing that the stream is not in the `started` state, - // regardless of whether streams start out `started` or `stopped`. - rtp_video_stream_receiver_->StopReceive(); - - MockRtpPacketSink secondary_sink; - rtp_video_stream_receiver_->AddSecondarySink(&secondary_sink); - - auto rtp_packet = CreateRtpPacketReceived(); - EXPECT_CALL(secondary_sink, OnRtpPacket(_)).Times(0); - - rtp_video_stream_receiver_->OnRtpPacket(*rtp_packet); - - // Test tear-down. - rtp_video_stream_receiver_->RemoveSecondarySink(&secondary_sink); -} - -TEST_F(RtpVideoStreamReceiverTest, ParseGenericDescriptorOnePacket) { - const std::vector<uint8_t> data = {0, 1, 2, 3, 4}; - const int kSpatialIndex = 1; - - rtp_video_stream_receiver_->StartReceive(); - - RtpHeaderExtensionMap extension_map; - extension_map.Register<RtpGenericFrameDescriptorExtension00>(5); - RtpPacketReceived rtp_packet(&extension_map); - rtp_packet.SetPayloadType(kPayloadType); - - RtpGenericFrameDescriptor generic_descriptor; - generic_descriptor.SetFirstPacketInSubFrame(true); - generic_descriptor.SetLastPacketInSubFrame(true); - generic_descriptor.SetFrameId(100); - generic_descriptor.SetSpatialLayersBitmask(1 << kSpatialIndex); - generic_descriptor.AddFrameDependencyDiff(90); - generic_descriptor.AddFrameDependencyDiff(80); - ASSERT_TRUE(rtp_packet.SetExtension<RtpGenericFrameDescriptorExtension00>( - generic_descriptor)); - - uint8_t* payload = rtp_packet.SetPayloadSize(data.size()); - memcpy(payload, data.data(), data.size()); - // The first byte is the header, so we ignore the first byte of `data`. - mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data() + 1, - data.size() - 1); - - rtp_packet.SetMarker(true); - rtp_packet.SetPayloadType(kPayloadType); - rtp_packet.SetSequenceNumber(1); - - EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame) - .WillOnce(Invoke([kSpatialIndex](EncodedFrame* frame) { - EXPECT_EQ(frame->num_references, 2U); - EXPECT_EQ(frame->references[0], frame->Id() - 90); - EXPECT_EQ(frame->references[1], frame->Id() - 80); - EXPECT_EQ(frame->SpatialIndex(), kSpatialIndex); - EXPECT_THAT(frame->PacketInfos(), SizeIs(1)); - })); - - rtp_video_stream_receiver_->OnRtpPacket(rtp_packet); -} - -TEST_F(RtpVideoStreamReceiverTest, ParseGenericDescriptorTwoPackets) { - const std::vector<uint8_t> data = {0, 1, 2, 3, 4}; - const int kSpatialIndex = 1; - - rtp_video_stream_receiver_->StartReceive(); - - RtpHeaderExtensionMap extension_map; - extension_map.Register<RtpGenericFrameDescriptorExtension00>(5); - RtpPacketReceived first_packet(&extension_map); - - RtpGenericFrameDescriptor first_packet_descriptor; - first_packet_descriptor.SetFirstPacketInSubFrame(true); - first_packet_descriptor.SetLastPacketInSubFrame(false); - first_packet_descriptor.SetFrameId(100); - first_packet_descriptor.SetSpatialLayersBitmask(1 << kSpatialIndex); - first_packet_descriptor.SetResolution(480, 360); - ASSERT_TRUE(first_packet.SetExtension<RtpGenericFrameDescriptorExtension00>( - first_packet_descriptor)); - - uint8_t* first_packet_payload = first_packet.SetPayloadSize(data.size()); - memcpy(first_packet_payload, data.data(), data.size()); - // The first byte is the header, so we ignore the first byte of `data`. - mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data() + 1, - data.size() - 1); - - first_packet.SetPayloadType(kPayloadType); - first_packet.SetSequenceNumber(1); - rtp_video_stream_receiver_->OnRtpPacket(first_packet); - - RtpPacketReceived second_packet(&extension_map); - RtpGenericFrameDescriptor second_packet_descriptor; - second_packet_descriptor.SetFirstPacketInSubFrame(false); - second_packet_descriptor.SetLastPacketInSubFrame(true); - ASSERT_TRUE(second_packet.SetExtension<RtpGenericFrameDescriptorExtension00>( - second_packet_descriptor)); - - second_packet.SetMarker(true); - second_packet.SetPayloadType(kPayloadType); - second_packet.SetSequenceNumber(2); - - uint8_t* second_packet_payload = second_packet.SetPayloadSize(data.size()); - memcpy(second_packet_payload, data.data(), data.size()); - // The first byte is the header, so we ignore the first byte of `data`. - mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data() + 1, - data.size() - 1); - - EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame) - .WillOnce(Invoke([kSpatialIndex](EncodedFrame* frame) { - EXPECT_EQ(frame->num_references, 0U); - EXPECT_EQ(frame->SpatialIndex(), kSpatialIndex); - EXPECT_EQ(frame->EncodedImage()._encodedWidth, 480u); - EXPECT_EQ(frame->EncodedImage()._encodedHeight, 360u); - EXPECT_THAT(frame->PacketInfos(), SizeIs(2)); - })); - - rtp_video_stream_receiver_->OnRtpPacket(second_packet); -} - -TEST_F(RtpVideoStreamReceiverTest, ParseGenericDescriptorRawPayload) { - const std::vector<uint8_t> data = {0, 1, 2, 3, 4}; - const int kRawPayloadType = 123; - - rtp_video_stream_receiver_->AddReceiveCodec(kRawPayloadType, - kVideoCodecGeneric, {}, - /*raw_payload=*/true); - rtp_video_stream_receiver_->StartReceive(); - - RtpHeaderExtensionMap extension_map; - extension_map.Register<RtpGenericFrameDescriptorExtension00>(5); - RtpPacketReceived rtp_packet(&extension_map); - - RtpGenericFrameDescriptor generic_descriptor; - generic_descriptor.SetFirstPacketInSubFrame(true); - generic_descriptor.SetLastPacketInSubFrame(true); - ASSERT_TRUE(rtp_packet.SetExtension<RtpGenericFrameDescriptorExtension00>( - generic_descriptor)); - - uint8_t* payload = rtp_packet.SetPayloadSize(data.size()); - memcpy(payload, data.data(), data.size()); - mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data(), - data.size()); - - rtp_packet.SetMarker(true); - rtp_packet.SetPayloadType(kRawPayloadType); - rtp_packet.SetSequenceNumber(1); - - EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame); - rtp_video_stream_receiver_->OnRtpPacket(rtp_packet); -} - -TEST_F(RtpVideoStreamReceiverTest, UnwrapsFrameId) { - const std::vector<uint8_t> data = {0, 1, 2, 3, 4}; - const int kPayloadType = 123; - - rtp_video_stream_receiver_->AddReceiveCodec(kPayloadType, kVideoCodecGeneric, - {}, - /*raw_payload=*/true); - rtp_video_stream_receiver_->StartReceive(); - RtpHeaderExtensionMap extension_map; - extension_map.Register<RtpGenericFrameDescriptorExtension00>(5); - - uint16_t rtp_sequence_number = 1; - auto inject_packet = [&](uint16_t wrapped_frame_id) { - RtpPacketReceived rtp_packet(&extension_map); - - RtpGenericFrameDescriptor generic_descriptor; - generic_descriptor.SetFirstPacketInSubFrame(true); - generic_descriptor.SetLastPacketInSubFrame(true); - generic_descriptor.SetFrameId(wrapped_frame_id); - ASSERT_TRUE(rtp_packet.SetExtension<RtpGenericFrameDescriptorExtension00>( - generic_descriptor)); - - uint8_t* payload = rtp_packet.SetPayloadSize(data.size()); - ASSERT_TRUE(payload); - memcpy(payload, data.data(), data.size()); - mock_on_complete_frame_callback_.ClearExpectedBitstream(); - mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data(), - data.size()); - rtp_packet.SetMarker(true); - rtp_packet.SetPayloadType(kPayloadType); - rtp_packet.SetSequenceNumber(++rtp_sequence_number); - rtp_video_stream_receiver_->OnRtpPacket(rtp_packet); - }; - - int64_t first_picture_id; - EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame) - .WillOnce([&](EncodedFrame* frame) { first_picture_id = frame->Id(); }); - inject_packet(/*wrapped_frame_id=*/0xffff); - - EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame) - .WillOnce([&](EncodedFrame* frame) { - EXPECT_EQ(frame->Id() - first_picture_id, 3); - }); - inject_packet(/*wrapped_frame_id=*/0x0002); -} - -class RtpVideoStreamReceiverDependencyDescriptorTest - : public RtpVideoStreamReceiverTest { - public: - RtpVideoStreamReceiverDependencyDescriptorTest() { - rtp_video_stream_receiver_->AddReceiveCodec(payload_type_, - kVideoCodecGeneric, {}, - /*raw_payload=*/true); - extension_map_.Register<RtpDependencyDescriptorExtension>(7); - rtp_video_stream_receiver_->StartReceive(); - } - - // Returns some valid structure for the DependencyDescriptors. - // First template of that structure always fit for a key frame. - static FrameDependencyStructure CreateStreamStructure() { - FrameDependencyStructure stream_structure; - stream_structure.num_decode_targets = 1; - stream_structure.templates = { - FrameDependencyTemplate().Dtis("S"), - FrameDependencyTemplate().Dtis("S").FrameDiffs({1}), - }; - return stream_structure; - } - - void InjectPacketWith(const FrameDependencyStructure& stream_structure, - const DependencyDescriptor& dependency_descriptor) { - const std::vector<uint8_t> data = {0, 1, 2, 3, 4}; - RtpPacketReceived rtp_packet(&extension_map_); - ASSERT_TRUE(rtp_packet.SetExtension<RtpDependencyDescriptorExtension>( - stream_structure, dependency_descriptor)); - uint8_t* payload = rtp_packet.SetPayloadSize(data.size()); - ASSERT_TRUE(payload); - memcpy(payload, data.data(), data.size()); - mock_on_complete_frame_callback_.ClearExpectedBitstream(); - mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data(), - data.size()); - rtp_packet.SetMarker(true); - rtp_packet.SetPayloadType(payload_type_); - rtp_packet.SetSequenceNumber(++rtp_sequence_number_); - rtp_video_stream_receiver_->OnRtpPacket(rtp_packet); - } - - private: - const int payload_type_ = 123; - RtpHeaderExtensionMap extension_map_; - uint16_t rtp_sequence_number_ = 321; -}; - -TEST_F(RtpVideoStreamReceiverDependencyDescriptorTest, UnwrapsFrameId) { - FrameDependencyStructure stream_structure = CreateStreamStructure(); - - DependencyDescriptor keyframe_descriptor; - keyframe_descriptor.attached_structure = - std::make_unique<FrameDependencyStructure>(stream_structure); - keyframe_descriptor.frame_dependencies = stream_structure.templates[0]; - keyframe_descriptor.frame_number = 0xfff0; - // DependencyDescriptor doesn't support reordering delta frame before - // keyframe. Thus feed a key frame first, then test reodered delta frames. - int64_t first_picture_id; - EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame) - .WillOnce([&](EncodedFrame* frame) { first_picture_id = frame->Id(); }); - InjectPacketWith(stream_structure, keyframe_descriptor); - - DependencyDescriptor deltaframe1_descriptor; - deltaframe1_descriptor.frame_dependencies = stream_structure.templates[1]; - deltaframe1_descriptor.frame_number = 0xfffe; - - DependencyDescriptor deltaframe2_descriptor; - deltaframe1_descriptor.frame_dependencies = stream_structure.templates[1]; - deltaframe2_descriptor.frame_number = 0x0002; - - // Parser should unwrap frame ids correctly even if packets were reordered by - // the network. - EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame) - .WillOnce([&](EncodedFrame* frame) { - // 0x0002 - 0xfff0 - EXPECT_EQ(frame->Id() - first_picture_id, 18); - }) - .WillOnce([&](EncodedFrame* frame) { - // 0xfffe - 0xfff0 - EXPECT_EQ(frame->Id() - first_picture_id, 14); - }); - InjectPacketWith(stream_structure, deltaframe2_descriptor); - InjectPacketWith(stream_structure, deltaframe1_descriptor); -} - -TEST_F(RtpVideoStreamReceiverDependencyDescriptorTest, - DropsLateDeltaFramePacketWithDependencyDescriptorExtension) { - FrameDependencyStructure stream_structure1 = CreateStreamStructure(); - FrameDependencyStructure stream_structure2 = CreateStreamStructure(); - // Make sure template ids for these two structures do not collide: - // adjust structure_id (that is also used as template id offset). - stream_structure1.structure_id = 13; - stream_structure2.structure_id = - stream_structure1.structure_id + stream_structure1.templates.size(); - - DependencyDescriptor keyframe1_descriptor; - keyframe1_descriptor.attached_structure = - std::make_unique<FrameDependencyStructure>(stream_structure1); - keyframe1_descriptor.frame_dependencies = stream_structure1.templates[0]; - keyframe1_descriptor.frame_number = 1; - EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame); - InjectPacketWith(stream_structure1, keyframe1_descriptor); - - // Pass in 2nd key frame with different structure. - DependencyDescriptor keyframe2_descriptor; - keyframe2_descriptor.attached_structure = - std::make_unique<FrameDependencyStructure>(stream_structure2); - keyframe2_descriptor.frame_dependencies = stream_structure2.templates[0]; - keyframe2_descriptor.frame_number = 3; - EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame); - InjectPacketWith(stream_structure2, keyframe2_descriptor); - - // Pass in late delta frame that uses structure of the 1st key frame. - DependencyDescriptor deltaframe_descriptor; - deltaframe_descriptor.frame_dependencies = stream_structure1.templates[0]; - deltaframe_descriptor.frame_number = 2; - EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame).Times(0); - InjectPacketWith(stream_structure1, deltaframe_descriptor); -} - -TEST_F(RtpVideoStreamReceiverDependencyDescriptorTest, - DropsLateKeyFramePacketWithDependencyDescriptorExtension) { - FrameDependencyStructure stream_structure1 = CreateStreamStructure(); - FrameDependencyStructure stream_structure2 = CreateStreamStructure(); - // Make sure template ids for these two structures do not collide: - // adjust structure_id (that is also used as template id offset). - stream_structure1.structure_id = 13; - stream_structure2.structure_id = - stream_structure1.structure_id + stream_structure1.templates.size(); - - DependencyDescriptor keyframe1_descriptor; - keyframe1_descriptor.attached_structure = - std::make_unique<FrameDependencyStructure>(stream_structure1); - keyframe1_descriptor.frame_dependencies = stream_structure1.templates[0]; - keyframe1_descriptor.frame_number = 1; - - DependencyDescriptor keyframe2_descriptor; - keyframe2_descriptor.attached_structure = - std::make_unique<FrameDependencyStructure>(stream_structure2); - keyframe2_descriptor.frame_dependencies = stream_structure2.templates[0]; - keyframe2_descriptor.frame_number = 3; - - EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame) - .WillOnce( - [&](EncodedFrame* frame) { EXPECT_EQ(frame->Id() & 0xFFFF, 3); }); - InjectPacketWith(stream_structure2, keyframe2_descriptor); - InjectPacketWith(stream_structure1, keyframe1_descriptor); - - // Pass in delta frame that uses structure of the 2nd key frame. Late key - // frame shouldn't block it. - DependencyDescriptor deltaframe_descriptor; - deltaframe_descriptor.frame_dependencies = stream_structure2.templates[0]; - deltaframe_descriptor.frame_number = 4; - EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame) - .WillOnce( - [&](EncodedFrame* frame) { EXPECT_EQ(frame->Id() & 0xFFFF, 4); }); - InjectPacketWith(stream_structure2, deltaframe_descriptor); -} - -#if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) -using RtpVideoStreamReceiverDeathTest = RtpVideoStreamReceiverTest; -TEST_F(RtpVideoStreamReceiverDeathTest, RepeatedSecondarySinkDisallowed) { - MockRtpPacketSink secondary_sink; - - rtp_video_stream_receiver_->AddSecondarySink(&secondary_sink); - EXPECT_DEATH(rtp_video_stream_receiver_->AddSecondarySink(&secondary_sink), - ""); - - // Test tear-down. - rtp_video_stream_receiver_->RemoveSecondarySink(&secondary_sink); -} -#endif - -TEST_F(RtpVideoStreamReceiverTest, TransformFrame) { - auto mock_frame_transformer = - rtc::make_ref_counted<testing::NiceMock<MockFrameTransformer>>(); - EXPECT_CALL(*mock_frame_transformer, - RegisterTransformedFrameSinkCallback(_, config_.rtp.remote_ssrc)); - auto receiver = std::make_unique<RtpVideoStreamReceiver>( - Clock::GetRealTimeClock(), &mock_transport_, nullptr, nullptr, &config_, - rtp_receive_statistics_.get(), nullptr, nullptr, process_thread_.get(), - &mock_nack_sender_, nullptr, &mock_on_complete_frame_callback_, nullptr, - mock_frame_transformer, &field_trials_); - receiver->AddReceiveCodec(kPayloadType, kVideoCodecGeneric, {}, - /*raw_payload=*/false); - - RtpPacketReceived rtp_packet; - rtp_packet.SetPayloadType(kPayloadType); - rtc::CopyOnWriteBuffer data({'1', '2', '3', '4'}); - rtp_packet.SetSequenceNumber(1); - RTPVideoHeader video_header = - GetGenericVideoHeader(VideoFrameType::kVideoFrameKey); - mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data(), - data.size()); - EXPECT_CALL(*mock_frame_transformer, Transform(_)); - receiver->OnReceivedPayloadData(data, rtp_packet, video_header); - - EXPECT_CALL(*mock_frame_transformer, - UnregisterTransformedFrameSinkCallback(config_.rtp.remote_ssrc)); - receiver = nullptr; -} - -// Test default behavior and when playout delay is overridden by field trial. -const VideoPlayoutDelay kTransmittedPlayoutDelay = {100, 200}; -const VideoPlayoutDelay kForcedPlayoutDelay = {70, 90}; -struct PlayoutDelayOptions { - std::string field_trial; - VideoPlayoutDelay expected_delay; -}; -const PlayoutDelayOptions kDefaultBehavior = { - /*field_trial=*/"", /*expected_delay=*/kTransmittedPlayoutDelay}; -const PlayoutDelayOptions kOverridePlayoutDelay = { - /*field_trial=*/"WebRTC-ForcePlayoutDelay/min_ms:70,max_ms:90/", - /*expected_delay=*/kForcedPlayoutDelay}; - -class RtpVideoStreamReceiverTestPlayoutDelay - : public RtpVideoStreamReceiverTest, - public ::testing::WithParamInterface<PlayoutDelayOptions> { - protected: - RtpVideoStreamReceiverTestPlayoutDelay() - : RtpVideoStreamReceiverTest(GetParam().field_trial) {} -}; - -INSTANTIATE_TEST_SUITE_P(PlayoutDelay, - RtpVideoStreamReceiverTestPlayoutDelay, - Values(kDefaultBehavior, kOverridePlayoutDelay)); - -TEST_P(RtpVideoStreamReceiverTestPlayoutDelay, PlayoutDelay) { - rtc::CopyOnWriteBuffer payload_data({'1', '2', '3', '4'}); - RtpHeaderExtensionMap extension_map; - extension_map.Register<PlayoutDelayLimits>(1); - RtpPacketToSend packet_to_send(&extension_map); - packet_to_send.SetPayloadType(kPayloadType); - packet_to_send.SetSequenceNumber(1); - - // Set playout delay on outgoing packet. - EXPECT_TRUE(packet_to_send.SetExtension<PlayoutDelayLimits>( - kTransmittedPlayoutDelay)); - uint8_t* payload = packet_to_send.AllocatePayload(payload_data.size()); - memcpy(payload, payload_data.data(), payload_data.size()); - - RtpPacketReceived received_packet(&extension_map); - received_packet.Parse(packet_to_send.data(), packet_to_send.size()); - - RTPVideoHeader video_header = - GetGenericVideoHeader(VideoFrameType::kVideoFrameKey); - mock_on_complete_frame_callback_.AppendExpectedBitstream(payload_data.data(), - payload_data.size()); - // Expect the playout delay of encoded frame to be the same as the transmitted - // playout delay unless it was overridden by a field trial. - EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_)) - .WillOnce(Invoke([expected_playout_delay = - GetParam().expected_delay](EncodedFrame* frame) { - EXPECT_EQ(frame->EncodedImage().playout_delay_, expected_playout_delay); - })); - rtp_video_stream_receiver_->OnReceivedPayloadData( - received_packet.PayloadBuffer(), received_packet, video_header); -} - -} // namespace webrtc |