From 8e0b15b584b4822a75f70288b6072eb94f1610d2 Mon Sep 17 00:00:00 2001 From: Sebastian Jansson Date: Wed, 18 Apr 2018 19:19:22 +0200 Subject: Moves VideoSendStreamImpl to a separate file. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares for adding unit tests for VideoSendStreamImpl. Bug: None Change-Id: I488041b09f4a455ce4cf1bdc7b8163ef6ad19a8a Reviewed-on: https://webrtc-review.googlesource.com/70782 Reviewed-by: Benjamin Wright Reviewed-by: Erik SprÃ¥ng Commit-Queue: Sebastian Jansson Cr-Commit-Position: refs/heads/master@{#22926} --- video/BUILD.gn | 2 + video/video_send_stream.cc | 1188 +-------------------------------------- video/video_send_stream.h | 1 - video/video_send_stream_impl.cc | 1031 +++++++++++++++++++++++++++++++++ video/video_send_stream_impl.h | 201 +++++++ 5 files changed, 1235 insertions(+), 1188 deletions(-) create mode 100644 video/video_send_stream_impl.cc create mode 100644 video/video_send_stream_impl.h (limited to 'video') diff --git a/video/BUILD.gn b/video/BUILD.gn index 309db781ce..732b83a7ef 100644 --- a/video/BUILD.gn +++ b/video/BUILD.gn @@ -42,6 +42,8 @@ rtc_static_library("video") { "video_receive_stream.h", "video_send_stream.cc", "video_send_stream.h", + "video_send_stream_impl.cc", + "video_send_stream_impl.h", "video_stream_decoder.cc", "video_stream_decoder.h", "video_stream_encoder.cc", diff --git a/video/video_send_stream.cc b/video/video_send_stream.cc index 4152efdf8b..02e8b17f9d 100644 --- a/video/video_send_stream.cc +++ b/video/video_send_stream.cc @@ -9,236 +9,16 @@ */ #include "video/video_send_stream.h" -#include -#include -#include -#include -#include #include -#include -#include "call/rtp_transport_controller_send_interface.h" -#include "call/video_send_stream.h" -#include "common_types.h" // NOLINT(build/include) -#include "common_video/include/video_bitrate_allocator.h" -#include "modules/bitrate_controller/include/bitrate_controller.h" -#include "modules/pacing/packet_router.h" -#include "modules/rtp_rtcp/include/rtp_rtcp.h" #include "modules/rtp_rtcp/source/rtp_sender.h" -#include "modules/utility/include/process_thread.h" -#include "modules/video_coding/codecs/vp9/include/vp9_globals.h" -#include "modules/video_coding/fec_controller_default.h" -#include "modules/video_coding/utility/ivf_file_writer.h" -#include "rtc_base/checks.h" -#include "rtc_base/experiments/alr_experiment.h" -#include "rtc_base/file.h" -#include "rtc_base/location.h" -#include "rtc_base/logging.h" -#include "rtc_base/numerics/safe_conversions.h" -#include "rtc_base/trace_event.h" -#include "rtc_base/weak_ptr.h" -#include "system_wrappers/include/field_trial.h" -#include "video/call_stats.h" +#include "video/video_send_stream_impl.h" namespace webrtc { -static const int kMinSendSidePacketHistorySize = 600; - -// Assume an average video stream has around 3 packets per frame (1 mbps / 30 -// fps / 1400B) A sequence number set with size 5500 will be able to store -// packet sequence number for at least last 60 seconds. -static const int kSendSideSeqNumSetMaxSize = 5500; namespace { -// We don't do MTU discovery, so assume that we have the standard ethernet MTU. -const size_t kPathMTU = 1500; - -std::vector CreateRtpRtcpModules( - const VideoSendStream::Config& config, - RtcpIntraFrameObserver* intra_frame_callback, - RtcpBandwidthObserver* bandwidth_callback, - RtpTransportControllerSendInterface* transport, - RtcpRttStats* rtt_stats, - FlexfecSender* flexfec_sender, - SendStatisticsProxy* stats_proxy, - SendDelayStats* send_delay_stats, - RtcEventLog* event_log, - RateLimiter* retransmission_rate_limiter, - OverheadObserver* overhead_observer, - RtpKeepAliveConfig keepalive_config) { - RTC_DCHECK_GT(config.rtp.ssrcs.size(), 0); - RtpRtcp::Configuration configuration; - configuration.audio = false; - configuration.receiver_only = false; - configuration.outgoing_transport = config.send_transport; - configuration.intra_frame_callback = intra_frame_callback; - configuration.bandwidth_callback = bandwidth_callback; - configuration.transport_feedback_callback = - transport->transport_feedback_observer(); - configuration.rtt_stats = rtt_stats; - configuration.rtcp_packet_type_counter_observer = stats_proxy; - configuration.paced_sender = transport->packet_sender(); - configuration.transport_sequence_number_allocator = - transport->packet_router(); - configuration.send_bitrate_observer = stats_proxy; - configuration.send_frame_count_observer = stats_proxy; - configuration.send_side_delay_observer = stats_proxy; - configuration.send_packet_observer = send_delay_stats; - configuration.event_log = event_log; - configuration.retransmission_rate_limiter = retransmission_rate_limiter; - configuration.overhead_observer = overhead_observer; - configuration.keepalive_config = keepalive_config; - configuration.rtcp_interval_config.video_interval_ms = - config.rtcp.video_report_interval_ms; - configuration.rtcp_interval_config.audio_interval_ms = - config.rtcp.audio_report_interval_ms; - std::vector modules; - const std::vector& flexfec_protected_ssrcs = - config.rtp.flexfec.protected_media_ssrcs; - for (uint32_t ssrc : config.rtp.ssrcs) { - bool enable_flexfec = flexfec_sender != nullptr && - std::find(flexfec_protected_ssrcs.begin(), - flexfec_protected_ssrcs.end(), - ssrc) != flexfec_protected_ssrcs.end(); - configuration.flexfec_sender = enable_flexfec ? flexfec_sender : nullptr; - RtpRtcp* rtp_rtcp = RtpRtcp::CreateRtpRtcp(configuration); - rtp_rtcp->SetSendingStatus(false); - rtp_rtcp->SetSendingMediaStatus(false); - rtp_rtcp->SetRTCPStatus(RtcpMode::kCompound); - modules.push_back(rtp_rtcp); - } - return modules; -} - -// TODO(brandtr): Update this function when we support multistream protection. -std::unique_ptr MaybeCreateFlexfecSender( - const VideoSendStream::Config& config, - const std::map& suspended_ssrcs) { - if (config.rtp.flexfec.payload_type < 0) { - return nullptr; - } - RTC_DCHECK_GE(config.rtp.flexfec.payload_type, 0); - RTC_DCHECK_LE(config.rtp.flexfec.payload_type, 127); - if (config.rtp.flexfec.ssrc == 0) { - RTC_LOG(LS_WARNING) << "FlexFEC is enabled, but no FlexFEC SSRC given. " - "Therefore disabling FlexFEC."; - return nullptr; - } - if (config.rtp.flexfec.protected_media_ssrcs.empty()) { - RTC_LOG(LS_WARNING) - << "FlexFEC is enabled, but no protected media SSRC given. " - "Therefore disabling FlexFEC."; - return nullptr; - } - - if (config.rtp.flexfec.protected_media_ssrcs.size() > 1) { - RTC_LOG(LS_WARNING) - << "The supplied FlexfecConfig contained multiple protected " - "media streams, but our implementation currently only " - "supports protecting a single media stream. " - "To avoid confusion, disabling FlexFEC completely."; - return nullptr; - } - - const RtpState* rtp_state = nullptr; - auto it = suspended_ssrcs.find(config.rtp.flexfec.ssrc); - if (it != suspended_ssrcs.end()) { - rtp_state = &it->second; - } - - RTC_DCHECK_EQ(1U, config.rtp.flexfec.protected_media_ssrcs.size()); - return rtc::MakeUnique( - config.rtp.flexfec.payload_type, config.rtp.flexfec.ssrc, - config.rtp.flexfec.protected_media_ssrcs[0], config.rtp.mid, - config.rtp.extensions, RTPSender::FecExtensionSizes(), rtp_state, - Clock::GetRealTimeClock()); -} - -bool TransportSeqNumExtensionConfigured(const VideoSendStream::Config& config) { - const std::vector& extensions = config.rtp.extensions; - return std::find_if( - extensions.begin(), extensions.end(), [](const RtpExtension& ext) { - return ext.uri == RtpExtension::kTransportSequenceNumberUri; - }) != extensions.end(); -} - -const char kForcedFallbackFieldTrial[] = - "WebRTC-VP8-Forced-Fallback-Encoder-v2"; - -rtc::Optional GetFallbackMinBpsFromFieldTrial() { - if (!webrtc::field_trial::IsEnabled(kForcedFallbackFieldTrial)) - return rtc::nullopt; - - std::string group = - webrtc::field_trial::FindFullName(kForcedFallbackFieldTrial); - if (group.empty()) - return rtc::nullopt; - - int min_pixels; - int max_pixels; - int min_bps; - if (sscanf(group.c_str(), "Enabled-%d,%d,%d", &min_pixels, &max_pixels, - &min_bps) != 3) { - return rtc::nullopt; - } - - if (min_bps <= 0) - return rtc::nullopt; - - return min_bps; -} - -int GetEncoderMinBitrateBps() { - const int kDefaultEncoderMinBitrateBps = 30000; - return GetFallbackMinBpsFromFieldTrial().value_or( - kDefaultEncoderMinBitrateBps); -} - -bool PayloadTypeSupportsSkippingFecPackets(const std::string& payload_name) { - const VideoCodecType codecType = PayloadStringToCodecType(payload_name); - if (codecType == kVideoCodecVP8 || codecType == kVideoCodecVP9) { - return true; - } - return false; -} - -int CalculateMaxPadBitrateBps(std::vector streams, - int min_transmit_bitrate_bps, - bool pad_to_min_bitrate) { - int pad_up_to_bitrate_bps = 0; - // Calculate max padding bitrate for a multi layer codec. - if (streams.size() > 1) { - // Pad to min bitrate of the highest layer. - pad_up_to_bitrate_bps = streams[streams.size() - 1].min_bitrate_bps; - // Add target_bitrate_bps of the lower layers. - for (size_t i = 0; i < streams.size() - 1; ++i) - pad_up_to_bitrate_bps += streams[i].target_bitrate_bps; - } else if (pad_to_min_bitrate) { - pad_up_to_bitrate_bps = streams[0].min_bitrate_bps; - } - - pad_up_to_bitrate_bps = - std::max(pad_up_to_bitrate_bps, min_transmit_bitrate_bps); - - return pad_up_to_bitrate_bps; -} - -uint32_t CalculateOverheadRateBps(int packets_per_second, - size_t overhead_bytes_per_packet, - uint32_t max_overhead_bps) { - uint32_t overhead_bps = - static_cast(8 * overhead_bytes_per_packet * packets_per_second); - return std::min(overhead_bps, max_overhead_bps); -} - -int CalculatePacketRate(uint32_t bitrate_bps, size_t packet_size_bytes) { - size_t packet_size_bits = 8 * packet_size_bytes; - // Ceil for int value of bitrate_bps / packet_size_bits. - return static_cast((bitrate_bps + packet_size_bits - 1) / - packet_size_bits); -} - size_t CalculateMaxHeaderSize(const VideoSendStream::Config::Rtp& config) { size_t header_size = kRtpHeaderSize; size_t extensions_size = 0; @@ -274,221 +54,6 @@ size_t CalculateMaxHeaderSize(const VideoSendStream::Config::Rtp& config) { namespace internal { -// VideoSendStreamImpl implements internal::VideoSendStream. -// It is created and destroyed on |worker_queue|. The intent is to decrease the -// need for locking and to ensure methods are called in sequence. -// Public methods except |DeliverRtcp| must be called on |worker_queue|. -// DeliverRtcp is called on the libjingle worker thread or a network thread. -// An encoder may deliver frames through the EncodedImageCallback on an -// arbitrary thread. -class VideoSendStreamImpl : public webrtc::BitrateAllocatorObserver, - public webrtc::OverheadObserver, - public webrtc::VCMProtectionCallback, - public VideoStreamEncoder::EncoderSink, - public VideoBitrateAllocationObserver, - public webrtc::PacketFeedbackObserver { - public: - VideoSendStreamImpl( - SendStatisticsProxy* stats_proxy, - rtc::TaskQueue* worker_queue, - CallStats* call_stats, - RtpTransportControllerSendInterface* transport, - BitrateAllocator* bitrate_allocator, - SendDelayStats* send_delay_stats, - VideoStreamEncoder* video_stream_encoder, - RtcEventLog* event_log, - const VideoSendStream::Config* config, - int initial_encoder_max_bitrate, - double initial_encoder_bitrate_priority, - std::map suspended_ssrcs, - std::map suspended_payload_states, - VideoEncoderConfig::ContentType content_type, - std::unique_ptr fec_controller, - RateLimiter* retransmission_limiter); - ~VideoSendStreamImpl() override; - - // RegisterProcessThread register |module_process_thread| with those objects - // that use it. Registration has to happen on the thread were - // |module_process_thread| was created (libjingle's worker thread). - // TODO(perkj): Replace the use of |module_process_thread| with a TaskQueue, - // maybe |worker_queue|. - void RegisterProcessThread(ProcessThread* module_process_thread); - void DeRegisterProcessThread(); - - void SignalNetworkState(NetworkState state); - bool DeliverRtcp(const uint8_t* packet, size_t length); - void UpdateActiveSimulcastLayers(const std::vector active_layers); - void Start(); - void Stop(); - - VideoSendStream::RtpStateMap GetRtpStates() const; - VideoSendStream::RtpPayloadStateMap GetRtpPayloadStates() const; - - void EnableEncodedFrameRecording(const std::vector& files, - size_t byte_limit); - - void SetTransportOverhead(size_t transport_overhead_per_packet); - - rtc::Optional configured_pacing_factor_; - - // From PacketFeedbackObserver. - void OnPacketAdded(uint32_t ssrc, uint16_t seq_num) override; - void OnPacketFeedbackVector( - const std::vector& packet_feedback_vector) override; - - private: - class CheckEncoderActivityTask; - - // Implements BitrateAllocatorObserver. - uint32_t OnBitrateUpdated(uint32_t bitrate_bps, - uint8_t fraction_loss, - int64_t rtt, - int64_t probing_interval_ms) override; - - // Implements webrtc::VCMProtectionCallback. - int ProtectionRequest(const FecProtectionParams* delta_params, - const FecProtectionParams* key_params, - uint32_t* sent_video_rate_bps, - uint32_t* sent_nack_rate_bps, - uint32_t* sent_fec_rate_bps) override; - - // Implements OverheadObserver. - void OnOverheadChanged(size_t overhead_bytes_per_packet) override; - - void OnEncoderConfigurationChanged(std::vector streams, - int min_transmit_bitrate_bps) override; - - // Implements EncodedImageCallback. The implementation routes encoded frames - // to the |payload_router_| and |config.pre_encode_callback| if set. - // Called on an arbitrary encoder callback thread. - EncodedImageCallback::Result OnEncodedImage( - const EncodedImage& encoded_image, - const CodecSpecificInfo* codec_specific_info, - const RTPFragmentationHeader* fragmentation) override; - - // Implements VideoBitrateAllocationObserver. - void OnBitrateAllocationUpdated(const BitrateAllocation& allocation) override; - - // Starts monitoring and sends a keyframe. - void StartupVideoSendStream(); - // Removes the bitrate observer, stops monitoring and notifies the video - // encoder of the bitrate update. - void StopVideoSendStream(); - - void ConfigureProtection(); - void ConfigureSsrcs(); - void SignalEncoderTimedOut(); - void SignalEncoderActive(); - - const bool send_side_bwe_with_overhead_; - - SendStatisticsProxy* const stats_proxy_; - const VideoSendStream::Config* const config_; - std::map suspended_ssrcs_; - - std::unique_ptr fec_controller_; - ProcessThread* module_process_thread_; - rtc::ThreadChecker module_process_thread_checker_; - rtc::TaskQueue* const worker_queue_; - - rtc::CriticalSection encoder_activity_crit_sect_; - CheckEncoderActivityTask* check_encoder_activity_task_ - RTC_GUARDED_BY(encoder_activity_crit_sect_); - - CallStats* const call_stats_; - RtpTransportControllerSendInterface* const transport_; - BitrateAllocator* const bitrate_allocator_; - - // TODO(brandtr): Move ownership to PayloadRouter. - std::unique_ptr flexfec_sender_; - - rtc::CriticalSection ivf_writers_crit_; - std::unique_ptr - file_writers_[kMaxSimulcastStreams] RTC_GUARDED_BY(ivf_writers_crit_); - - int max_padding_bitrate_; - int encoder_min_bitrate_bps_; - uint32_t encoder_max_bitrate_bps_; - uint32_t encoder_target_rate_bps_; - double encoder_bitrate_priority_; - bool has_packet_feedback_; - - VideoStreamEncoder* const video_stream_encoder_; - EncoderRtcpFeedback encoder_feedback_; - - RtcpBandwidthObserver* const bandwidth_observer_; - // RtpRtcp modules, declared here as they use other members on construction. - const std::vector rtp_rtcp_modules_; - PayloadRouter payload_router_; - - // |weak_ptr_| to our self. This is used since we can not call - // |weak_ptr_factory_.GetWeakPtr| from multiple sequences but it is ok to copy - // an existing WeakPtr. - rtc::WeakPtr weak_ptr_; - // |weak_ptr_factory_| must be declared last to make sure all WeakPtr's are - // invalidated before any other members are destroyed. - rtc::WeakPtrFactory weak_ptr_factory_; - - rtc::CriticalSection overhead_bytes_per_packet_crit_; - size_t overhead_bytes_per_packet_ - RTC_GUARDED_BY(overhead_bytes_per_packet_crit_); - size_t transport_overhead_bytes_per_packet_; - - std::unordered_set feedback_packet_seq_num_set_; - std::vector loss_mask_vector_; -}; - -// CheckEncoderActivityTask is used for tracking when the encoder last produced -// and encoded video frame. If the encoder has not produced anything the last -// kEncoderTimeOutMs we also want to stop sending padding. -class VideoSendStreamImpl::CheckEncoderActivityTask : public rtc::QueuedTask { - public: - static const int kEncoderTimeOutMs = 2000; - explicit CheckEncoderActivityTask( - const rtc::WeakPtr& send_stream) - : activity_(0), send_stream_(std::move(send_stream)), timed_out_(false) {} - - void Stop() { - RTC_CHECK(task_checker_.CalledSequentially()); - send_stream_.reset(); - } - - void UpdateEncoderActivity() { - // UpdateEncoderActivity is called from VideoSendStreamImpl::Encoded on - // whatever thread the real encoder implementation run on. In the case of - // hardware encoders, there might be several encoders - // running in parallel on different threads. - rtc::AtomicOps::ReleaseStore(&activity_, 1); - } - - private: - bool Run() override { - RTC_CHECK(task_checker_.CalledSequentially()); - if (!send_stream_) - return true; - if (!rtc::AtomicOps::AcquireLoad(&activity_)) { - if (!timed_out_) { - send_stream_->SignalEncoderTimedOut(); - } - timed_out_ = true; - } else if (timed_out_) { - send_stream_->SignalEncoderActive(); - timed_out_ = false; - } - rtc::AtomicOps::ReleaseStore(&activity_, 0); - - rtc::TaskQueue::Current()->PostDelayedTask( - std::unique_ptr(this), kEncoderTimeOutMs); - // Return false to prevent this task from being deleted. Ownership has been - // transferred to the task queue when PostDelayedTask was called. - return false; - } - volatile int activity_; - - rtc::SequencedTaskChecker task_checker_; - rtc::WeakPtr send_stream_; - bool timed_out_; -}; VideoSendStream::VideoSendStream( int num_cpu_cores, @@ -660,756 +225,5 @@ void VideoSendStream::EnableEncodedFrameRecording( send_stream_->EnableEncodedFrameRecording(files, byte_limit); } -VideoSendStreamImpl::VideoSendStreamImpl( - SendStatisticsProxy* stats_proxy, - rtc::TaskQueue* worker_queue, - CallStats* call_stats, - RtpTransportControllerSendInterface* transport, - BitrateAllocator* bitrate_allocator, - SendDelayStats* send_delay_stats, - VideoStreamEncoder* video_stream_encoder, - RtcEventLog* event_log, - const VideoSendStream::Config* config, - int initial_encoder_max_bitrate, - double initial_encoder_bitrate_priority, - std::map suspended_ssrcs, - std::map suspended_payload_states, - VideoEncoderConfig::ContentType content_type, - std::unique_ptr fec_controller, - RateLimiter* retransmission_limiter) - : send_side_bwe_with_overhead_( - webrtc::field_trial::IsEnabled("WebRTC-SendSideBwe-WithOverhead")), - stats_proxy_(stats_proxy), - config_(config), - suspended_ssrcs_(std::move(suspended_ssrcs)), - fec_controller_(std::move(fec_controller)), - module_process_thread_(nullptr), - worker_queue_(worker_queue), - check_encoder_activity_task_(nullptr), - call_stats_(call_stats), - transport_(transport), - bitrate_allocator_(bitrate_allocator), - flexfec_sender_(MaybeCreateFlexfecSender(*config_, suspended_ssrcs_)), - max_padding_bitrate_(0), - encoder_min_bitrate_bps_(0), - encoder_target_rate_bps_(0), - encoder_bitrate_priority_(initial_encoder_bitrate_priority), - has_packet_feedback_(false), - video_stream_encoder_(video_stream_encoder), - encoder_feedback_(Clock::GetRealTimeClock(), - config_->rtp.ssrcs, - video_stream_encoder), - bandwidth_observer_(transport->GetBandwidthObserver()), - rtp_rtcp_modules_(CreateRtpRtcpModules(*config_, - &encoder_feedback_, - bandwidth_observer_, - transport, - call_stats, - flexfec_sender_.get(), - stats_proxy_, - send_delay_stats, - event_log, - retransmission_limiter, - this, - transport->keepalive_config())), - payload_router_(rtp_rtcp_modules_, - config_->rtp.ssrcs, - config_->rtp.payload_type, - suspended_payload_states), - weak_ptr_factory_(this), - overhead_bytes_per_packet_(0), - transport_overhead_bytes_per_packet_(0) { - RTC_DCHECK_RUN_ON(worker_queue_); - RTC_LOG(LS_INFO) << "VideoSendStreamInternal: " << config_->ToString(); - weak_ptr_ = weak_ptr_factory_.GetWeakPtr(); - module_process_thread_checker_.DetachFromThread(); - - RTC_DCHECK(!config_->rtp.ssrcs.empty()); - RTC_DCHECK(call_stats_); - RTC_DCHECK(transport_); - RTC_DCHECK_NE(initial_encoder_max_bitrate, 0); - - if (initial_encoder_max_bitrate > 0) { - encoder_max_bitrate_bps_ = - rtc::dchecked_cast(initial_encoder_max_bitrate); - } else { - // TODO(srte): Make sure max bitrate is not set to negative values. We don't - // have any way to handle unset values in downstream code, such as the - // bitrate allocator. Previously -1 was implicitly casted to UINT32_MAX, a - // behaviour that is not safe. Converting to 10 Mbps should be safe for - // reasonable use cases as it allows adding the max of multiple streams - // without wrappping around. - const int kFallbackMaxBitrateBps = 10000000; - RTC_DLOG(LS_ERROR) << "ERROR: Initial encoder max bitrate = " - << initial_encoder_max_bitrate << " which is <= 0!"; - RTC_DLOG(LS_INFO) << "Using default encoder max bitrate = 10 Mbps"; - encoder_max_bitrate_bps_ = kFallbackMaxBitrateBps; - } - - RTC_CHECK(AlrExperimentSettings::MaxOneFieldTrialEnabled()); - // If send-side BWE is enabled, check if we should apply updated probing and - // pacing settings. - if (TransportSeqNumExtensionConfigured(*config_)) { - has_packet_feedback_ = true; - - rtc::Optional alr_settings; - if (content_type == VideoEncoderConfig::ContentType::kScreen) { - alr_settings = AlrExperimentSettings::CreateFromFieldTrial( - AlrExperimentSettings::kScreenshareProbingBweExperimentName); - } else { - alr_settings = AlrExperimentSettings::CreateFromFieldTrial( - AlrExperimentSettings::kStrictPacingAndProbingExperimentName); - } - if (alr_settings) { - transport->EnablePeriodicAlrProbing(true); - transport->SetPacingFactor(alr_settings->pacing_factor); - configured_pacing_factor_ = alr_settings->pacing_factor; - transport->SetQueueTimeLimit(alr_settings->max_paced_queue_time); - } else { - transport->EnablePeriodicAlrProbing(false); - transport->SetPacingFactor(PacedSender::kDefaultPaceMultiplier); - configured_pacing_factor_ = PacedSender::kDefaultPaceMultiplier; - transport->SetQueueTimeLimit(PacedSender::kMaxQueueLengthMs); - } - } - - if (config_->periodic_alr_bandwidth_probing) { - transport->EnablePeriodicAlrProbing(true); - } - - // RTP/RTCP initialization. - - // We add the highest spatial layer first to ensure it'll be prioritized - // when sending padding, with the hope that the packet rate will be smaller, - // and that it's more important to protect than the lower layers. - for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_) { - constexpr bool remb_candidate = true; - transport->packet_router()->AddSendRtpModule(rtp_rtcp, remb_candidate); - } - - for (size_t i = 0; i < config_->rtp.extensions.size(); ++i) { - const std::string& extension = config_->rtp.extensions[i].uri; - int id = config_->rtp.extensions[i].id; - // One-byte-extension local identifiers are in the range 1-14 inclusive. - RTC_DCHECK_GE(id, 1); - RTC_DCHECK_LE(id, 14); - RTC_DCHECK(RtpExtension::IsSupportedForVideo(extension)); - for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_) { - RTC_CHECK_EQ(0, rtp_rtcp->RegisterSendRtpHeaderExtension( - StringToRtpExtensionType(extension), id)); - } - } - - ConfigureProtection(); - ConfigureSsrcs(); - - if (!config_->rtp.mid.empty()) { - for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_) { - rtp_rtcp->SetMid(config_->rtp.mid); - } - } - - // TODO(pbos): Should we set CNAME on all RTP modules? - rtp_rtcp_modules_.front()->SetCNAME(config_->rtp.c_name.c_str()); - - for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_) { - rtp_rtcp->RegisterRtcpStatisticsCallback(stats_proxy_); - rtp_rtcp->RegisterSendChannelRtpStatisticsCallback(stats_proxy_); - rtp_rtcp->SetMaxRtpPacketSize(config_->rtp.max_packet_size); - rtp_rtcp->RegisterVideoSendPayload(config_->rtp.payload_type, - config_->rtp.payload_name.c_str()); - } - - fec_controller_->SetProtectionCallback(this); - // Signal congestion controller this object is ready for OnPacket* callbacks. - if (fec_controller_->UseLossVectorMask()) { - transport_->RegisterPacketFeedbackObserver(this); - } - - RTC_DCHECK(config_->encoder_settings.encoder); - RTC_DCHECK_GE(config_->rtp.payload_type, 0); - RTC_DCHECK_LE(config_->rtp.payload_type, 127); - - video_stream_encoder_->SetStartBitrate( - bitrate_allocator_->GetStartBitrate(this)); - - // Only request rotation at the source when we positively know that the remote - // side doesn't support the rotation extension. This allows us to prepare the - // encoder in the expectation that rotation is supported - which is the common - // case. - bool rotation_applied = - std::find_if(config_->rtp.extensions.begin(), - config_->rtp.extensions.end(), - [](const RtpExtension& extension) { - return extension.uri == RtpExtension::kVideoRotationUri; - }) == config_->rtp.extensions.end(); - - video_stream_encoder_->SetSink(this, rotation_applied); -} - -void VideoSendStreamImpl::RegisterProcessThread( - ProcessThread* module_process_thread) { - RTC_DCHECK_RUN_ON(&module_process_thread_checker_); - RTC_DCHECK(!module_process_thread_); - module_process_thread_ = module_process_thread; - - for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_) - module_process_thread_->RegisterModule(rtp_rtcp, RTC_FROM_HERE); -} - -void VideoSendStreamImpl::DeRegisterProcessThread() { - RTC_DCHECK_RUN_ON(&module_process_thread_checker_); - for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_) - module_process_thread_->DeRegisterModule(rtp_rtcp); -} - -VideoSendStreamImpl::~VideoSendStreamImpl() { - RTC_DCHECK_RUN_ON(worker_queue_); - RTC_DCHECK(!payload_router_.IsActive()) - << "VideoSendStreamImpl::Stop not called"; - RTC_LOG(LS_INFO) << "~VideoSendStreamInternal: " << config_->ToString(); - if (fec_controller_->UseLossVectorMask()) { - transport_->DeRegisterPacketFeedbackObserver(this); - } - for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_) { - transport_->packet_router()->RemoveSendRtpModule(rtp_rtcp); - delete rtp_rtcp; - } -} - -bool VideoSendStreamImpl::DeliverRtcp(const uint8_t* packet, size_t length) { - // Runs on a network thread. - RTC_DCHECK(!worker_queue_->IsCurrent()); - for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_) - rtp_rtcp->IncomingRtcpPacket(packet, length); - return true; -} - -void VideoSendStreamImpl::UpdateActiveSimulcastLayers( - const std::vector active_layers) { - RTC_DCHECK_RUN_ON(worker_queue_); - RTC_DCHECK_EQ(rtp_rtcp_modules_.size(), active_layers.size()); - RTC_LOG(LS_INFO) << "VideoSendStream::UpdateActiveSimulcastLayers"; - bool previously_active = payload_router_.IsActive(); - payload_router_.SetActiveModules(active_layers); - if (!payload_router_.IsActive() && previously_active) { - // Payload router switched from active to inactive. - StopVideoSendStream(); - } else if (payload_router_.IsActive() && !previously_active) { - // Payload router switched from inactive to active. - StartupVideoSendStream(); - } -} - -void VideoSendStreamImpl::Start() { - RTC_DCHECK_RUN_ON(worker_queue_); - RTC_LOG(LS_INFO) << "VideoSendStream::Start"; - if (payload_router_.IsActive()) - return; - TRACE_EVENT_INSTANT0("webrtc", "VideoSendStream::Start"); - payload_router_.SetActive(true); - StartupVideoSendStream(); -} - -void VideoSendStreamImpl::StartupVideoSendStream() { - RTC_DCHECK_RUN_ON(worker_queue_); - bitrate_allocator_->AddObserver( - this, encoder_min_bitrate_bps_, encoder_max_bitrate_bps_, - max_padding_bitrate_, !config_->suspend_below_min_bitrate, - config_->track_id, encoder_bitrate_priority_, has_packet_feedback_); - // Start monitoring encoder activity. - { - rtc::CritScope lock(&encoder_activity_crit_sect_); - RTC_DCHECK(!check_encoder_activity_task_); - check_encoder_activity_task_ = new CheckEncoderActivityTask(weak_ptr_); - worker_queue_->PostDelayedTask( - std::unique_ptr(check_encoder_activity_task_), - CheckEncoderActivityTask::kEncoderTimeOutMs); - } - - video_stream_encoder_->SendKeyFrame(); -} - -void VideoSendStreamImpl::Stop() { - RTC_DCHECK_RUN_ON(worker_queue_); - RTC_LOG(LS_INFO) << "VideoSendStream::Stop"; - if (!payload_router_.IsActive()) - return; - TRACE_EVENT_INSTANT0("webrtc", "VideoSendStream::Stop"); - payload_router_.SetActive(false); - StopVideoSendStream(); -} - -void VideoSendStreamImpl::StopVideoSendStream() { - bitrate_allocator_->RemoveObserver(this); - { - rtc::CritScope lock(&encoder_activity_crit_sect_); - check_encoder_activity_task_->Stop(); - check_encoder_activity_task_ = nullptr; - } - video_stream_encoder_->OnBitrateUpdated(0, 0, 0); - stats_proxy_->OnSetEncoderTargetRate(0); -} - -void VideoSendStreamImpl::SignalEncoderTimedOut() { - RTC_DCHECK_RUN_ON(worker_queue_); - // If the encoder has not produced anything the last kEncoderTimeOutMs and it - // is supposed to, deregister as BitrateAllocatorObserver. This can happen - // if a camera stops producing frames. - if (encoder_target_rate_bps_ > 0) { - RTC_LOG(LS_INFO) << "SignalEncoderTimedOut, Encoder timed out."; - bitrate_allocator_->RemoveObserver(this); - } -} - -void VideoSendStreamImpl::OnBitrateAllocationUpdated( - const BitrateAllocation& allocation) { - payload_router_.OnBitrateAllocationUpdated(allocation); -} - -void VideoSendStreamImpl::SignalEncoderActive() { - RTC_DCHECK_RUN_ON(worker_queue_); - RTC_LOG(LS_INFO) << "SignalEncoderActive, Encoder is active."; - bitrate_allocator_->AddObserver( - this, encoder_min_bitrate_bps_, encoder_max_bitrate_bps_, - max_padding_bitrate_, !config_->suspend_below_min_bitrate, - config_->track_id, encoder_bitrate_priority_, has_packet_feedback_); -} - -void VideoSendStreamImpl::OnEncoderConfigurationChanged( - std::vector streams, - int min_transmit_bitrate_bps) { - if (!worker_queue_->IsCurrent()) { - rtc::WeakPtr send_stream = weak_ptr_; - worker_queue_->PostTask([send_stream, streams, min_transmit_bitrate_bps]() { - if (send_stream) - send_stream->OnEncoderConfigurationChanged(std::move(streams), - min_transmit_bitrate_bps); - }); - return; - } - RTC_DCHECK_GE(config_->rtp.ssrcs.size(), streams.size()); - TRACE_EVENT0("webrtc", "VideoSendStream::OnEncoderConfigurationChanged"); - RTC_DCHECK_GE(config_->rtp.ssrcs.size(), streams.size()); - RTC_DCHECK_RUN_ON(worker_queue_); - - encoder_min_bitrate_bps_ = - std::max(streams[0].min_bitrate_bps, GetEncoderMinBitrateBps()); - encoder_max_bitrate_bps_ = 0; - double stream_bitrate_priority_sum = 0; - for (const auto& stream : streams) { - // We don't want to allocate more bitrate than needed to inactive streams. - encoder_max_bitrate_bps_ += stream.active ? stream.max_bitrate_bps : 0; - if (stream.bitrate_priority) { - RTC_DCHECK_GT(*stream.bitrate_priority, 0); - stream_bitrate_priority_sum += *stream.bitrate_priority; - } - } - RTC_DCHECK_GT(stream_bitrate_priority_sum, 0); - encoder_bitrate_priority_ = stream_bitrate_priority_sum; - encoder_max_bitrate_bps_ = - std::max(static_cast(encoder_min_bitrate_bps_), - encoder_max_bitrate_bps_); - max_padding_bitrate_ = CalculateMaxPadBitrateBps( - streams, min_transmit_bitrate_bps, config_->suspend_below_min_bitrate); - - // Clear stats for disabled layers. - for (size_t i = streams.size(); i < config_->rtp.ssrcs.size(); ++i) { - stats_proxy_->OnInactiveSsrc(config_->rtp.ssrcs[i]); - } - - const size_t num_temporal_layers = - streams.back().num_temporal_layers.value_or(1); - fec_controller_->SetEncodingData(streams[0].width, streams[0].height, - num_temporal_layers, - config_->rtp.max_packet_size); - - if (payload_router_.IsActive()) { - // The send stream is started already. Update the allocator with new bitrate - // limits. - bitrate_allocator_->AddObserver( - this, encoder_min_bitrate_bps_, encoder_max_bitrate_bps_, - max_padding_bitrate_, !config_->suspend_below_min_bitrate, - config_->track_id, encoder_bitrate_priority_, has_packet_feedback_); - } -} - -EncodedImageCallback::Result VideoSendStreamImpl::OnEncodedImage( - const EncodedImage& encoded_image, - const CodecSpecificInfo* codec_specific_info, - const RTPFragmentationHeader* fragmentation) { - // Encoded is called on whatever thread the real encoder implementation run - // on. In the case of hardware encoders, there might be several encoders - // running in parallel on different threads. - size_t simulcast_idx = 0; - if (codec_specific_info->codecType == kVideoCodecVP8) { - simulcast_idx = codec_specific_info->codecSpecific.VP8.simulcastIdx; - } - if (config_->post_encode_callback) { - config_->post_encode_callback->EncodedFrameCallback(EncodedFrame( - encoded_image._buffer, encoded_image._length, encoded_image._frameType, - simulcast_idx, encoded_image._timeStamp)); - } - { - rtc::CritScope lock(&encoder_activity_crit_sect_); - if (check_encoder_activity_task_) - check_encoder_activity_task_->UpdateEncoderActivity(); - } - - fec_controller_->UpdateWithEncodedData(encoded_image._length, - encoded_image._frameType); - EncodedImageCallback::Result result = payload_router_.OnEncodedImage( - encoded_image, codec_specific_info, fragmentation); - - RTC_DCHECK(codec_specific_info); - - int layer = codec_specific_info->codecType == kVideoCodecVP8 - ? codec_specific_info->codecSpecific.VP8.simulcastIdx - : 0; - { - rtc::CritScope lock(&ivf_writers_crit_); - if (file_writers_[layer].get()) { - bool ok = file_writers_[layer]->WriteFrame( - encoded_image, codec_specific_info->codecType); - RTC_DCHECK(ok); - } - } - - return result; -} - -void VideoSendStreamImpl::ConfigureProtection() { - RTC_DCHECK_RUN_ON(worker_queue_); - - // Consistency of FlexFEC parameters is checked in MaybeCreateFlexfecSender. - const bool flexfec_enabled = (flexfec_sender_ != nullptr); - - // Consistency of NACK and RED+ULPFEC parameters is checked in this function. - const bool nack_enabled = config_->rtp.nack.rtp_history_ms > 0; - int red_payload_type = config_->rtp.ulpfec.red_payload_type; - int ulpfec_payload_type = config_->rtp.ulpfec.ulpfec_payload_type; - - // Shorthands. - auto IsRedEnabled = [&]() { return red_payload_type >= 0; }; - auto DisableRed = [&]() { red_payload_type = -1; }; - auto IsUlpfecEnabled = [&]() { return ulpfec_payload_type >= 0; }; - auto DisableUlpfec = [&]() { ulpfec_payload_type = -1; }; - - if (webrtc::field_trial::IsEnabled("WebRTC-DisableUlpFecExperiment")) { - RTC_LOG(LS_INFO) << "Experiment to disable sending ULPFEC is enabled."; - DisableUlpfec(); - } - - // If enabled, FlexFEC takes priority over RED+ULPFEC. - if (flexfec_enabled) { - // We can safely disable RED here, because if the remote supports FlexFEC, - // we know that it has a receiver without the RED/RTX workaround. - // See http://crbug.com/webrtc/6650 for more information. - if (IsRedEnabled()) { - RTC_LOG(LS_INFO) << "Both FlexFEC and RED are configured. Disabling RED."; - DisableRed(); - } - if (IsUlpfecEnabled()) { - RTC_LOG(LS_INFO) - << "Both FlexFEC and ULPFEC are configured. Disabling ULPFEC."; - DisableUlpfec(); - } - } - - // Payload types without picture ID cannot determine that a stream is complete - // without retransmitting FEC, so using ULPFEC + NACK for H.264 (for instance) - // is a waste of bandwidth since FEC packets still have to be transmitted. - // Note that this is not the case with FlexFEC. - if (nack_enabled && IsUlpfecEnabled() && - !PayloadTypeSupportsSkippingFecPackets(config_->rtp.payload_name)) { - RTC_LOG(LS_WARNING) - << "Transmitting payload type without picture ID using " - "NACK+ULPFEC is a waste of bandwidth since ULPFEC packets " - "also have to be retransmitted. Disabling ULPFEC."; - DisableUlpfec(); - } - - // Verify payload types. - // - // Due to how old receivers work, we need to always send RED if it has been - // negotiated. This is a remnant of an old RED/RTX workaround, see - // https://codereview.webrtc.org/2469093003. - // TODO(brandtr): This change went into M56, so we can remove it in ~M59. - // At that time, we can disable RED whenever ULPFEC is disabled, as there is - // no point in using RED without ULPFEC. - if (IsRedEnabled()) { - RTC_DCHECK_GE(red_payload_type, 0); - RTC_DCHECK_LE(red_payload_type, 127); - } - if (IsUlpfecEnabled()) { - RTC_DCHECK_GE(ulpfec_payload_type, 0); - RTC_DCHECK_LE(ulpfec_payload_type, 127); - if (!IsRedEnabled()) { - RTC_LOG(LS_WARNING) - << "ULPFEC is enabled but RED is disabled. Disabling ULPFEC."; - DisableUlpfec(); - } - } - - for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_) { - // Set NACK. - rtp_rtcp->SetStorePacketsStatus( - true, - kMinSendSidePacketHistorySize); - // Set RED/ULPFEC information. - rtp_rtcp->SetUlpfecConfig(red_payload_type, ulpfec_payload_type); - } - - // Currently, both ULPFEC and FlexFEC use the same FEC rate calculation logic, - // so enable that logic if either of those FEC schemes are enabled. - fec_controller_->SetProtectionMethod(flexfec_enabled || IsUlpfecEnabled(), - nack_enabled); -} - -void VideoSendStreamImpl::ConfigureSsrcs() { - RTC_DCHECK_RUN_ON(worker_queue_); - // Configure regular SSRCs. - for (size_t i = 0; i < config_->rtp.ssrcs.size(); ++i) { - uint32_t ssrc = config_->rtp.ssrcs[i]; - RtpRtcp* const rtp_rtcp = rtp_rtcp_modules_[i]; - rtp_rtcp->SetSSRC(ssrc); - - // Restore RTP state if previous existed. - VideoSendStream::RtpStateMap::iterator it = suspended_ssrcs_.find(ssrc); - if (it != suspended_ssrcs_.end()) - rtp_rtcp->SetRtpState(it->second); - } - - // Set up RTX if available. - if (config_->rtp.rtx.ssrcs.empty()) - return; - - // Configure RTX SSRCs. - RTC_DCHECK_EQ(config_->rtp.rtx.ssrcs.size(), config_->rtp.ssrcs.size()); - for (size_t i = 0; i < config_->rtp.rtx.ssrcs.size(); ++i) { - uint32_t ssrc = config_->rtp.rtx.ssrcs[i]; - RtpRtcp* const rtp_rtcp = rtp_rtcp_modules_[i]; - rtp_rtcp->SetRtxSsrc(ssrc); - VideoSendStream::RtpStateMap::iterator it = suspended_ssrcs_.find(ssrc); - if (it != suspended_ssrcs_.end()) - rtp_rtcp->SetRtxState(it->second); - } - - // Configure RTX payload types. - RTC_DCHECK_GE(config_->rtp.rtx.payload_type, 0); - for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_) { - rtp_rtcp->SetRtxSendPayloadType(config_->rtp.rtx.payload_type, - config_->rtp.payload_type); - rtp_rtcp->SetRtxSendStatus(kRtxRetransmitted | kRtxRedundantPayloads); - } - if (config_->rtp.ulpfec.red_payload_type != -1 && - config_->rtp.ulpfec.red_rtx_payload_type != -1) { - for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_) { - rtp_rtcp->SetRtxSendPayloadType(config_->rtp.ulpfec.red_rtx_payload_type, - config_->rtp.ulpfec.red_payload_type); - } - } -} - -std::map VideoSendStreamImpl::GetRtpStates() const { - RTC_DCHECK_RUN_ON(worker_queue_); - std::map rtp_states; - - for (size_t i = 0; i < config_->rtp.ssrcs.size(); ++i) { - uint32_t ssrc = config_->rtp.ssrcs[i]; - RTC_DCHECK_EQ(ssrc, rtp_rtcp_modules_[i]->SSRC()); - rtp_states[ssrc] = rtp_rtcp_modules_[i]->GetRtpState(); - } - - for (size_t i = 0; i < config_->rtp.rtx.ssrcs.size(); ++i) { - uint32_t ssrc = config_->rtp.rtx.ssrcs[i]; - rtp_states[ssrc] = rtp_rtcp_modules_[i]->GetRtxState(); - } - - if (flexfec_sender_) { - uint32_t ssrc = config_->rtp.flexfec.ssrc; - rtp_states[ssrc] = flexfec_sender_->GetRtpState(); - } - - return rtp_states; -} - -std::map VideoSendStreamImpl::GetRtpPayloadStates() - const { - RTC_DCHECK_RUN_ON(worker_queue_); - return payload_router_.GetRtpPayloadStates(); -} - -void VideoSendStreamImpl::SignalNetworkState(NetworkState state) { - RTC_DCHECK_RUN_ON(worker_queue_); - for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_) { - rtp_rtcp->SetRTCPStatus(state == kNetworkUp ? config_->rtp.rtcp_mode - : RtcpMode::kOff); - } -} - -uint32_t VideoSendStreamImpl::OnBitrateUpdated(uint32_t bitrate_bps, - uint8_t fraction_loss, - int64_t rtt, - int64_t probing_interval_ms) { - RTC_DCHECK_RUN_ON(worker_queue_); - RTC_DCHECK(payload_router_.IsActive()) - << "VideoSendStream::Start has not been called."; - - // Substract overhead from bitrate. - rtc::CritScope lock(&overhead_bytes_per_packet_crit_); - uint32_t payload_bitrate_bps = bitrate_bps; - if (send_side_bwe_with_overhead_) { - payload_bitrate_bps -= CalculateOverheadRateBps( - CalculatePacketRate(bitrate_bps, - config_->rtp.max_packet_size + - transport_overhead_bytes_per_packet_), - overhead_bytes_per_packet_ + transport_overhead_bytes_per_packet_, - bitrate_bps); - } - - // Get the encoder target rate. It is the estimated network rate - - // protection overhead. - encoder_target_rate_bps_ = fec_controller_->UpdateFecRates( - payload_bitrate_bps, stats_proxy_->GetSendFrameRate(), fraction_loss, - loss_mask_vector_, rtt); - loss_mask_vector_.clear(); - - uint32_t encoder_overhead_rate_bps = - send_side_bwe_with_overhead_ - ? CalculateOverheadRateBps( - CalculatePacketRate(encoder_target_rate_bps_, - config_->rtp.max_packet_size + - transport_overhead_bytes_per_packet_ - - overhead_bytes_per_packet_), - overhead_bytes_per_packet_ + - transport_overhead_bytes_per_packet_, - bitrate_bps - encoder_target_rate_bps_) - : 0; - - // When the field trial "WebRTC-SendSideBwe-WithOverhead" is enabled - // protection_bitrate includes overhead. - uint32_t protection_bitrate = - bitrate_bps - (encoder_target_rate_bps_ + encoder_overhead_rate_bps); - - encoder_target_rate_bps_ = - std::min(encoder_max_bitrate_bps_, encoder_target_rate_bps_); - video_stream_encoder_->OnBitrateUpdated(encoder_target_rate_bps_, - fraction_loss, rtt); - stats_proxy_->OnSetEncoderTargetRate(encoder_target_rate_bps_); - return protection_bitrate; -} - -void VideoSendStreamImpl::EnableEncodedFrameRecording( - const std::vector& files, - size_t byte_limit) { - { - rtc::CritScope lock(&ivf_writers_crit_); - for (unsigned int i = 0; i < kMaxSimulcastStreams; ++i) { - if (i < files.size()) { - file_writers_[i] = IvfFileWriter::Wrap(rtc::File(files[i]), byte_limit); - } else { - file_writers_[i].reset(); - } - } - } - - if (!files.empty()) { - // Make a keyframe appear as early as possible in the logs, to give actually - // decodable output. - video_stream_encoder_->SendKeyFrame(); - } -} - -int VideoSendStreamImpl::ProtectionRequest( - const FecProtectionParams* delta_params, - const FecProtectionParams* key_params, - uint32_t* sent_video_rate_bps, - uint32_t* sent_nack_rate_bps, - uint32_t* sent_fec_rate_bps) { - RTC_DCHECK_RUN_ON(worker_queue_); - *sent_video_rate_bps = 0; - *sent_nack_rate_bps = 0; - *sent_fec_rate_bps = 0; - for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_) { - uint32_t not_used = 0; - uint32_t module_video_rate = 0; - uint32_t module_fec_rate = 0; - uint32_t module_nack_rate = 0; - rtp_rtcp->SetFecParameters(*delta_params, *key_params); - rtp_rtcp->BitrateSent(¬_used, &module_video_rate, &module_fec_rate, - &module_nack_rate); - *sent_video_rate_bps += module_video_rate; - *sent_nack_rate_bps += module_nack_rate; - *sent_fec_rate_bps += module_fec_rate; - } - return 0; -} - -void VideoSendStreamImpl::OnOverheadChanged(size_t overhead_bytes_per_packet) { - rtc::CritScope lock(&overhead_bytes_per_packet_crit_); - overhead_bytes_per_packet_ = overhead_bytes_per_packet; -} - -void VideoSendStreamImpl::SetTransportOverhead( - size_t transport_overhead_bytes_per_packet) { - if (transport_overhead_bytes_per_packet >= static_cast(kPathMTU)) { - RTC_LOG(LS_ERROR) << "Transport overhead exceeds size of ethernet frame"; - return; - } - - transport_overhead_bytes_per_packet_ = transport_overhead_bytes_per_packet; - - size_t rtp_packet_size = - std::min(config_->rtp.max_packet_size, - kPathMTU - transport_overhead_bytes_per_packet_); - - for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_) { - rtp_rtcp->SetMaxRtpPacketSize(rtp_packet_size); - } -} - -void VideoSendStreamImpl::OnPacketAdded(uint32_t ssrc, uint16_t seq_num) { - if (!worker_queue_->IsCurrent()) { - auto ptr = weak_ptr_; - worker_queue_->PostTask([=] { - if (!ptr.get()) - return; - ptr->OnPacketAdded(ssrc, seq_num); - }); - return; - } - const auto ssrcs = config_->rtp.ssrcs; - if (std::find(ssrcs.begin(), ssrcs.end(), ssrc) != ssrcs.end()) { - feedback_packet_seq_num_set_.insert(seq_num); - if (feedback_packet_seq_num_set_.size() > kSendSideSeqNumSetMaxSize) { - RTC_LOG(LS_WARNING) << "Feedback packet sequence number set exceed it's " - "max size', will get reset."; - feedback_packet_seq_num_set_.clear(); - } - } -} - -void VideoSendStreamImpl::OnPacketFeedbackVector( - const std::vector& packet_feedback_vector) { - if (!worker_queue_->IsCurrent()) { - auto ptr = weak_ptr_; - worker_queue_->PostTask([=] { - if (!ptr.get()) - return; - ptr->OnPacketFeedbackVector(packet_feedback_vector); - }); - return; - } - // Lost feedbacks are not considered to be lost packets. - for (const PacketFeedback& packet : packet_feedback_vector) { - if (auto it = feedback_packet_seq_num_set_.find(packet.sequence_number) != - feedback_packet_seq_num_set_.end()) { - const bool lost = packet.arrival_time_ms == PacketFeedback::kNotReceived; - loss_mask_vector_.push_back(lost); - feedback_packet_seq_num_set_.erase(it); - } - } -} } // namespace internal } // namespace webrtc diff --git a/video/video_send_stream.h b/video/video_send_stream.h index a790a0b0f7..b65129e6cf 100644 --- a/video/video_send_stream.h +++ b/video/video_send_stream.h @@ -23,7 +23,6 @@ #include "rtc_base/criticalsection.h" #include "rtc_base/event.h" #include "rtc_base/task_queue.h" -#include "video/encoder_rtcp_feedback.h" #include "video/send_delay_stats.h" #include "video/payload_router.h" #include "video/send_statistics_proxy.h" diff --git a/video/video_send_stream_impl.cc b/video/video_send_stream_impl.cc new file mode 100644 index 0000000000..3cfdee2ccb --- /dev/null +++ b/video/video_send_stream_impl.cc @@ -0,0 +1,1031 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "video/video_send_stream_impl.h" + +#include +#include +#include + +#include "call/rtp_transport_controller_send_interface.h" +#include "modules/pacing/packet_router.h" +#include "modules/rtp_rtcp/include/rtp_rtcp.h" +#include "modules/rtp_rtcp/source/rtp_sender.h" +#include "rtc_base/checks.h" +#include "rtc_base/experiments/alr_experiment.h" +#include "rtc_base/file.h" +#include "rtc_base/location.h" +#include "rtc_base/logging.h" +#include "rtc_base/numerics/safe_conversions.h" +#include "rtc_base/trace_event.h" +#include "system_wrappers/include/field_trial.h" + +namespace webrtc { +namespace internal { +namespace { +static const int kMinSendSidePacketHistorySize = 600; + +// Assume an average video stream has around 3 packets per frame (1 mbps / 30 +// fps / 1400B) A sequence number set with size 5500 will be able to store +// packet sequence number for at least last 60 seconds. +static const int kSendSideSeqNumSetMaxSize = 5500; + +// We don't do MTU discovery, so assume that we have the standard ethernet MTU. +const size_t kPathMTU = 1500; + +std::vector CreateRtpRtcpModules( + const VideoSendStream::Config& config, + RtcpIntraFrameObserver* intra_frame_callback, + RtcpBandwidthObserver* bandwidth_callback, + RtpTransportControllerSendInterface* transport, + RtcpRttStats* rtt_stats, + FlexfecSender* flexfec_sender, + SendStatisticsProxy* stats_proxy, + SendDelayStats* send_delay_stats, + RtcEventLog* event_log, + RateLimiter* retransmission_rate_limiter, + OverheadObserver* overhead_observer, + RtpKeepAliveConfig keepalive_config) { + RTC_DCHECK_GT(config.rtp.ssrcs.size(), 0); + RtpRtcp::Configuration configuration; + configuration.audio = false; + configuration.receiver_only = false; + configuration.outgoing_transport = config.send_transport; + configuration.intra_frame_callback = intra_frame_callback; + configuration.bandwidth_callback = bandwidth_callback; + configuration.transport_feedback_callback = + transport->transport_feedback_observer(); + configuration.rtt_stats = rtt_stats; + configuration.rtcp_packet_type_counter_observer = stats_proxy; + configuration.paced_sender = transport->packet_sender(); + configuration.transport_sequence_number_allocator = + transport->packet_router(); + configuration.send_bitrate_observer = stats_proxy; + configuration.send_frame_count_observer = stats_proxy; + configuration.send_side_delay_observer = stats_proxy; + configuration.send_packet_observer = send_delay_stats; + configuration.event_log = event_log; + configuration.retransmission_rate_limiter = retransmission_rate_limiter; + configuration.overhead_observer = overhead_observer; + configuration.keepalive_config = keepalive_config; + configuration.rtcp_interval_config.video_interval_ms = + config.rtcp.video_report_interval_ms; + configuration.rtcp_interval_config.audio_interval_ms = + config.rtcp.audio_report_interval_ms; + std::vector modules; + const std::vector& flexfec_protected_ssrcs = + config.rtp.flexfec.protected_media_ssrcs; + for (uint32_t ssrc : config.rtp.ssrcs) { + bool enable_flexfec = flexfec_sender != nullptr && + std::find(flexfec_protected_ssrcs.begin(), + flexfec_protected_ssrcs.end(), + ssrc) != flexfec_protected_ssrcs.end(); + configuration.flexfec_sender = enable_flexfec ? flexfec_sender : nullptr; + RtpRtcp* rtp_rtcp = RtpRtcp::CreateRtpRtcp(configuration); + rtp_rtcp->SetSendingStatus(false); + rtp_rtcp->SetSendingMediaStatus(false); + rtp_rtcp->SetRTCPStatus(RtcpMode::kCompound); + modules.push_back(rtp_rtcp); + } + return modules; +} + +// TODO(brandtr): Update this function when we support multistream protection. +std::unique_ptr MaybeCreateFlexfecSender( + const VideoSendStream::Config& config, + const std::map& suspended_ssrcs) { + if (config.rtp.flexfec.payload_type < 0) { + return nullptr; + } + RTC_DCHECK_GE(config.rtp.flexfec.payload_type, 0); + RTC_DCHECK_LE(config.rtp.flexfec.payload_type, 127); + if (config.rtp.flexfec.ssrc == 0) { + RTC_LOG(LS_WARNING) << "FlexFEC is enabled, but no FlexFEC SSRC given. " + "Therefore disabling FlexFEC."; + return nullptr; + } + if (config.rtp.flexfec.protected_media_ssrcs.empty()) { + RTC_LOG(LS_WARNING) + << "FlexFEC is enabled, but no protected media SSRC given. " + "Therefore disabling FlexFEC."; + return nullptr; + } + + if (config.rtp.flexfec.protected_media_ssrcs.size() > 1) { + RTC_LOG(LS_WARNING) + << "The supplied FlexfecConfig contained multiple protected " + "media streams, but our implementation currently only " + "supports protecting a single media stream. " + "To avoid confusion, disabling FlexFEC completely."; + return nullptr; + } + + const RtpState* rtp_state = nullptr; + auto it = suspended_ssrcs.find(config.rtp.flexfec.ssrc); + if (it != suspended_ssrcs.end()) { + rtp_state = &it->second; + } + + RTC_DCHECK_EQ(1U, config.rtp.flexfec.protected_media_ssrcs.size()); + return rtc::MakeUnique( + config.rtp.flexfec.payload_type, config.rtp.flexfec.ssrc, + config.rtp.flexfec.protected_media_ssrcs[0], config.rtp.mid, + config.rtp.extensions, RTPSender::FecExtensionSizes(), rtp_state, + Clock::GetRealTimeClock()); +} + +bool TransportSeqNumExtensionConfigured(const VideoSendStream::Config& config) { + const std::vector& extensions = config.rtp.extensions; + return std::find_if( + extensions.begin(), extensions.end(), [](const RtpExtension& ext) { + return ext.uri == RtpExtension::kTransportSequenceNumberUri; + }) != extensions.end(); +} + +const char kForcedFallbackFieldTrial[] = + "WebRTC-VP8-Forced-Fallback-Encoder-v2"; + +rtc::Optional GetFallbackMinBpsFromFieldTrial() { + if (!webrtc::field_trial::IsEnabled(kForcedFallbackFieldTrial)) + return rtc::nullopt; + + std::string group = + webrtc::field_trial::FindFullName(kForcedFallbackFieldTrial); + if (group.empty()) + return rtc::nullopt; + + int min_pixels; + int max_pixels; + int min_bps; + if (sscanf(group.c_str(), "Enabled-%d,%d,%d", &min_pixels, &max_pixels, + &min_bps) != 3) { + return rtc::nullopt; + } + + if (min_bps <= 0) + return rtc::nullopt; + + return min_bps; +} + +int GetEncoderMinBitrateBps() { + const int kDefaultEncoderMinBitrateBps = 30000; + return GetFallbackMinBpsFromFieldTrial().value_or( + kDefaultEncoderMinBitrateBps); +} + +bool PayloadTypeSupportsSkippingFecPackets(const std::string& payload_name) { + const VideoCodecType codecType = PayloadStringToCodecType(payload_name); + if (codecType == kVideoCodecVP8 || codecType == kVideoCodecVP9) { + return true; + } + return false; +} + +int CalculateMaxPadBitrateBps(std::vector streams, + int min_transmit_bitrate_bps, + bool pad_to_min_bitrate) { + int pad_up_to_bitrate_bps = 0; + // Calculate max padding bitrate for a multi layer codec. + if (streams.size() > 1) { + // Pad to min bitrate of the highest layer. + pad_up_to_bitrate_bps = streams[streams.size() - 1].min_bitrate_bps; + // Add target_bitrate_bps of the lower layers. + for (size_t i = 0; i < streams.size() - 1; ++i) + pad_up_to_bitrate_bps += streams[i].target_bitrate_bps; + } else if (pad_to_min_bitrate) { + pad_up_to_bitrate_bps = streams[0].min_bitrate_bps; + } + + pad_up_to_bitrate_bps = + std::max(pad_up_to_bitrate_bps, min_transmit_bitrate_bps); + + return pad_up_to_bitrate_bps; +} + +uint32_t CalculateOverheadRateBps(int packets_per_second, + size_t overhead_bytes_per_packet, + uint32_t max_overhead_bps) { + uint32_t overhead_bps = + static_cast(8 * overhead_bytes_per_packet * packets_per_second); + return std::min(overhead_bps, max_overhead_bps); +} + +int CalculatePacketRate(uint32_t bitrate_bps, size_t packet_size_bytes) { + size_t packet_size_bits = 8 * packet_size_bytes; + // Ceil for int value of bitrate_bps / packet_size_bits. + return static_cast((bitrate_bps + packet_size_bits - 1) / + packet_size_bits); +} + +} // namespace + +// CheckEncoderActivityTask is used for tracking when the encoder last produced +// and encoded video frame. If the encoder has not produced anything the last +// kEncoderTimeOutMs we also want to stop sending padding. +class VideoSendStreamImpl::CheckEncoderActivityTask : public rtc::QueuedTask { + public: + static const int kEncoderTimeOutMs = 2000; + explicit CheckEncoderActivityTask( + const rtc::WeakPtr& send_stream) + : activity_(0), send_stream_(std::move(send_stream)), timed_out_(false) {} + + void Stop() { + RTC_CHECK(task_checker_.CalledSequentially()); + send_stream_.reset(); + } + + void UpdateEncoderActivity() { + // UpdateEncoderActivity is called from VideoSendStreamImpl::Encoded on + // whatever thread the real encoder implementation run on. In the case of + // hardware encoders, there might be several encoders + // running in parallel on different threads. + rtc::AtomicOps::ReleaseStore(&activity_, 1); + } + + private: + bool Run() override { + RTC_CHECK(task_checker_.CalledSequentially()); + if (!send_stream_) + return true; + if (!rtc::AtomicOps::AcquireLoad(&activity_)) { + if (!timed_out_) { + send_stream_->SignalEncoderTimedOut(); + } + timed_out_ = true; + } else if (timed_out_) { + send_stream_->SignalEncoderActive(); + timed_out_ = false; + } + rtc::AtomicOps::ReleaseStore(&activity_, 0); + + rtc::TaskQueue::Current()->PostDelayedTask( + std::unique_ptr(this), kEncoderTimeOutMs); + // Return false to prevent this task from being deleted. Ownership has been + // transferred to the task queue when PostDelayedTask was called. + return false; + } + volatile int activity_; + + rtc::SequencedTaskChecker task_checker_; + rtc::WeakPtr send_stream_; + bool timed_out_; +}; + +VideoSendStreamImpl::VideoSendStreamImpl( + SendStatisticsProxy* stats_proxy, + rtc::TaskQueue* worker_queue, + CallStats* call_stats, + RtpTransportControllerSendInterface* transport, + BitrateAllocator* bitrate_allocator, + SendDelayStats* send_delay_stats, + VideoStreamEncoder* video_stream_encoder, + RtcEventLog* event_log, + const VideoSendStream::Config* config, + int initial_encoder_max_bitrate, + double initial_encoder_bitrate_priority, + std::map suspended_ssrcs, + std::map suspended_payload_states, + VideoEncoderConfig::ContentType content_type, + std::unique_ptr fec_controller, + RateLimiter* retransmission_limiter) + : send_side_bwe_with_overhead_( + webrtc::field_trial::IsEnabled("WebRTC-SendSideBwe-WithOverhead")), + stats_proxy_(stats_proxy), + config_(config), + suspended_ssrcs_(std::move(suspended_ssrcs)), + fec_controller_(std::move(fec_controller)), + module_process_thread_(nullptr), + worker_queue_(worker_queue), + check_encoder_activity_task_(nullptr), + call_stats_(call_stats), + transport_(transport), + bitrate_allocator_(bitrate_allocator), + flexfec_sender_(MaybeCreateFlexfecSender(*config_, suspended_ssrcs_)), + max_padding_bitrate_(0), + encoder_min_bitrate_bps_(0), + encoder_target_rate_bps_(0), + encoder_bitrate_priority_(initial_encoder_bitrate_priority), + has_packet_feedback_(false), + video_stream_encoder_(video_stream_encoder), + encoder_feedback_(Clock::GetRealTimeClock(), + config_->rtp.ssrcs, + video_stream_encoder), + bandwidth_observer_(transport->GetBandwidthObserver()), + rtp_rtcp_modules_(CreateRtpRtcpModules(*config_, + &encoder_feedback_, + bandwidth_observer_, + transport, + call_stats, + flexfec_sender_.get(), + stats_proxy_, + send_delay_stats, + event_log, + retransmission_limiter, + this, + transport->keepalive_config())), + payload_router_(rtp_rtcp_modules_, + config_->rtp.ssrcs, + config_->rtp.payload_type, + suspended_payload_states), + weak_ptr_factory_(this), + overhead_bytes_per_packet_(0), + transport_overhead_bytes_per_packet_(0) { + RTC_DCHECK_RUN_ON(worker_queue_); + RTC_LOG(LS_INFO) << "VideoSendStreamInternal: " << config_->ToString(); + weak_ptr_ = weak_ptr_factory_.GetWeakPtr(); + module_process_thread_checker_.DetachFromThread(); + + RTC_DCHECK(!config_->rtp.ssrcs.empty()); + RTC_DCHECK(call_stats_); + RTC_DCHECK(transport_); + RTC_DCHECK_NE(initial_encoder_max_bitrate, 0); + + if (initial_encoder_max_bitrate > 0) { + encoder_max_bitrate_bps_ = + rtc::dchecked_cast(initial_encoder_max_bitrate); + } else { + // TODO(srte): Make sure max bitrate is not set to negative values. We don't + // have any way to handle unset values in downstream code, such as the + // bitrate allocator. Previously -1 was implicitly casted to UINT32_MAX, a + // behaviour that is not safe. Converting to 10 Mbps should be safe for + // reasonable use cases as it allows adding the max of multiple streams + // without wrappping around. + const int kFallbackMaxBitrateBps = 10000000; + RTC_DLOG(LS_ERROR) << "ERROR: Initial encoder max bitrate = " + << initial_encoder_max_bitrate << " which is <= 0!"; + RTC_DLOG(LS_INFO) << "Using default encoder max bitrate = 10 Mbps"; + encoder_max_bitrate_bps_ = kFallbackMaxBitrateBps; + } + + RTC_CHECK(AlrExperimentSettings::MaxOneFieldTrialEnabled()); + // If send-side BWE is enabled, check if we should apply updated probing and + // pacing settings. + if (TransportSeqNumExtensionConfigured(*config_)) { + has_packet_feedback_ = true; + + rtc::Optional alr_settings; + if (content_type == VideoEncoderConfig::ContentType::kScreen) { + alr_settings = AlrExperimentSettings::CreateFromFieldTrial( + AlrExperimentSettings::kScreenshareProbingBweExperimentName); + } else { + alr_settings = AlrExperimentSettings::CreateFromFieldTrial( + AlrExperimentSettings::kStrictPacingAndProbingExperimentName); + } + if (alr_settings) { + transport->EnablePeriodicAlrProbing(true); + transport->SetPacingFactor(alr_settings->pacing_factor); + configured_pacing_factor_ = alr_settings->pacing_factor; + transport->SetQueueTimeLimit(alr_settings->max_paced_queue_time); + } else { + transport->EnablePeriodicAlrProbing(false); + transport->SetPacingFactor(PacedSender::kDefaultPaceMultiplier); + configured_pacing_factor_ = PacedSender::kDefaultPaceMultiplier; + transport->SetQueueTimeLimit(PacedSender::kMaxQueueLengthMs); + } + } + + if (config_->periodic_alr_bandwidth_probing) { + transport->EnablePeriodicAlrProbing(true); + } + + // RTP/RTCP initialization. + + // We add the highest spatial layer first to ensure it'll be prioritized + // when sending padding, with the hope that the packet rate will be smaller, + // and that it's more important to protect than the lower layers. + for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_) { + constexpr bool remb_candidate = true; + transport->packet_router()->AddSendRtpModule(rtp_rtcp, remb_candidate); + } + + for (size_t i = 0; i < config_->rtp.extensions.size(); ++i) { + const std::string& extension = config_->rtp.extensions[i].uri; + int id = config_->rtp.extensions[i].id; + // One-byte-extension local identifiers are in the range 1-14 inclusive. + RTC_DCHECK_GE(id, 1); + RTC_DCHECK_LE(id, 14); + RTC_DCHECK(RtpExtension::IsSupportedForVideo(extension)); + for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_) { + RTC_CHECK_EQ(0, rtp_rtcp->RegisterSendRtpHeaderExtension( + StringToRtpExtensionType(extension), id)); + } + } + + ConfigureProtection(); + ConfigureSsrcs(); + + if (!config_->rtp.mid.empty()) { + for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_) { + rtp_rtcp->SetMid(config_->rtp.mid); + } + } + + // TODO(pbos): Should we set CNAME on all RTP modules? + rtp_rtcp_modules_.front()->SetCNAME(config_->rtp.c_name.c_str()); + + for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_) { + rtp_rtcp->RegisterRtcpStatisticsCallback(stats_proxy_); + rtp_rtcp->RegisterSendChannelRtpStatisticsCallback(stats_proxy_); + rtp_rtcp->SetMaxRtpPacketSize(config_->rtp.max_packet_size); + rtp_rtcp->RegisterVideoSendPayload(config_->rtp.payload_type, + config_->rtp.payload_name.c_str()); + } + + fec_controller_->SetProtectionCallback(this); + // Signal congestion controller this object is ready for OnPacket* callbacks. + if (fec_controller_->UseLossVectorMask()) { + transport_->RegisterPacketFeedbackObserver(this); + } + + RTC_DCHECK(config_->encoder_settings.encoder); + RTC_DCHECK_GE(config_->rtp.payload_type, 0); + RTC_DCHECK_LE(config_->rtp.payload_type, 127); + + video_stream_encoder_->SetStartBitrate( + bitrate_allocator_->GetStartBitrate(this)); + + // Only request rotation at the source when we positively know that the remote + // side doesn't support the rotation extension. This allows us to prepare the + // encoder in the expectation that rotation is supported - which is the common + // case. + bool rotation_applied = + std::find_if(config_->rtp.extensions.begin(), + config_->rtp.extensions.end(), + [](const RtpExtension& extension) { + return extension.uri == RtpExtension::kVideoRotationUri; + }) == config_->rtp.extensions.end(); + + video_stream_encoder_->SetSink(this, rotation_applied); +} + +void VideoSendStreamImpl::RegisterProcessThread( + ProcessThread* module_process_thread) { + RTC_DCHECK_RUN_ON(&module_process_thread_checker_); + RTC_DCHECK(!module_process_thread_); + module_process_thread_ = module_process_thread; + + for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_) + module_process_thread_->RegisterModule(rtp_rtcp, RTC_FROM_HERE); +} + +void VideoSendStreamImpl::DeRegisterProcessThread() { + RTC_DCHECK_RUN_ON(&module_process_thread_checker_); + for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_) + module_process_thread_->DeRegisterModule(rtp_rtcp); +} + +VideoSendStreamImpl::~VideoSendStreamImpl() { + RTC_DCHECK_RUN_ON(worker_queue_); + RTC_DCHECK(!payload_router_.IsActive()) + << "VideoSendStreamImpl::Stop not called"; + RTC_LOG(LS_INFO) << "~VideoSendStreamInternal: " << config_->ToString(); + if (fec_controller_->UseLossVectorMask()) { + transport_->DeRegisterPacketFeedbackObserver(this); + } + for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_) { + transport_->packet_router()->RemoveSendRtpModule(rtp_rtcp); + delete rtp_rtcp; + } +} + +bool VideoSendStreamImpl::DeliverRtcp(const uint8_t* packet, size_t length) { + // Runs on a network thread. + RTC_DCHECK(!worker_queue_->IsCurrent()); + for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_) + rtp_rtcp->IncomingRtcpPacket(packet, length); + return true; +} + +void VideoSendStreamImpl::UpdateActiveSimulcastLayers( + const std::vector active_layers) { + RTC_DCHECK_RUN_ON(worker_queue_); + RTC_DCHECK_EQ(rtp_rtcp_modules_.size(), active_layers.size()); + RTC_LOG(LS_INFO) << "VideoSendStream::UpdateActiveSimulcastLayers"; + bool previously_active = payload_router_.IsActive(); + payload_router_.SetActiveModules(active_layers); + if (!payload_router_.IsActive() && previously_active) { + // Payload router switched from active to inactive. + StopVideoSendStream(); + } else if (payload_router_.IsActive() && !previously_active) { + // Payload router switched from inactive to active. + StartupVideoSendStream(); + } +} + +void VideoSendStreamImpl::Start() { + RTC_DCHECK_RUN_ON(worker_queue_); + RTC_LOG(LS_INFO) << "VideoSendStream::Start"; + if (payload_router_.IsActive()) + return; + TRACE_EVENT_INSTANT0("webrtc", "VideoSendStream::Start"); + payload_router_.SetActive(true); + StartupVideoSendStream(); +} + +void VideoSendStreamImpl::StartupVideoSendStream() { + RTC_DCHECK_RUN_ON(worker_queue_); + bitrate_allocator_->AddObserver( + this, encoder_min_bitrate_bps_, encoder_max_bitrate_bps_, + max_padding_bitrate_, !config_->suspend_below_min_bitrate, + config_->track_id, encoder_bitrate_priority_, has_packet_feedback_); + // Start monitoring encoder activity. + { + rtc::CritScope lock(&encoder_activity_crit_sect_); + RTC_DCHECK(!check_encoder_activity_task_); + check_encoder_activity_task_ = new CheckEncoderActivityTask(weak_ptr_); + worker_queue_->PostDelayedTask( + std::unique_ptr(check_encoder_activity_task_), + CheckEncoderActivityTask::kEncoderTimeOutMs); + } + + video_stream_encoder_->SendKeyFrame(); +} + +void VideoSendStreamImpl::Stop() { + RTC_DCHECK_RUN_ON(worker_queue_); + RTC_LOG(LS_INFO) << "VideoSendStream::Stop"; + if (!payload_router_.IsActive()) + return; + TRACE_EVENT_INSTANT0("webrtc", "VideoSendStream::Stop"); + payload_router_.SetActive(false); + StopVideoSendStream(); +} + +void VideoSendStreamImpl::StopVideoSendStream() { + bitrate_allocator_->RemoveObserver(this); + { + rtc::CritScope lock(&encoder_activity_crit_sect_); + check_encoder_activity_task_->Stop(); + check_encoder_activity_task_ = nullptr; + } + video_stream_encoder_->OnBitrateUpdated(0, 0, 0); + stats_proxy_->OnSetEncoderTargetRate(0); +} + +void VideoSendStreamImpl::SignalEncoderTimedOut() { + RTC_DCHECK_RUN_ON(worker_queue_); + // If the encoder has not produced anything the last kEncoderTimeOutMs and it + // is supposed to, deregister as BitrateAllocatorObserver. This can happen + // if a camera stops producing frames. + if (encoder_target_rate_bps_ > 0) { + RTC_LOG(LS_INFO) << "SignalEncoderTimedOut, Encoder timed out."; + bitrate_allocator_->RemoveObserver(this); + } +} + +void VideoSendStreamImpl::OnBitrateAllocationUpdated( + const BitrateAllocation& allocation) { + payload_router_.OnBitrateAllocationUpdated(allocation); +} + +void VideoSendStreamImpl::SignalEncoderActive() { + RTC_DCHECK_RUN_ON(worker_queue_); + RTC_LOG(LS_INFO) << "SignalEncoderActive, Encoder is active."; + bitrate_allocator_->AddObserver( + this, encoder_min_bitrate_bps_, encoder_max_bitrate_bps_, + max_padding_bitrate_, !config_->suspend_below_min_bitrate, + config_->track_id, encoder_bitrate_priority_, has_packet_feedback_); +} + +void VideoSendStreamImpl::OnEncoderConfigurationChanged( + std::vector streams, + int min_transmit_bitrate_bps) { + if (!worker_queue_->IsCurrent()) { + rtc::WeakPtr send_stream = weak_ptr_; + worker_queue_->PostTask([send_stream, streams, min_transmit_bitrate_bps]() { + if (send_stream) + send_stream->OnEncoderConfigurationChanged(std::move(streams), + min_transmit_bitrate_bps); + }); + return; + } + RTC_DCHECK_GE(config_->rtp.ssrcs.size(), streams.size()); + TRACE_EVENT0("webrtc", "VideoSendStream::OnEncoderConfigurationChanged"); + RTC_DCHECK_GE(config_->rtp.ssrcs.size(), streams.size()); + RTC_DCHECK_RUN_ON(worker_queue_); + + encoder_min_bitrate_bps_ = + std::max(streams[0].min_bitrate_bps, GetEncoderMinBitrateBps()); + encoder_max_bitrate_bps_ = 0; + double stream_bitrate_priority_sum = 0; + for (const auto& stream : streams) { + // We don't want to allocate more bitrate than needed to inactive streams. + encoder_max_bitrate_bps_ += stream.active ? stream.max_bitrate_bps : 0; + if (stream.bitrate_priority) { + RTC_DCHECK_GT(*stream.bitrate_priority, 0); + stream_bitrate_priority_sum += *stream.bitrate_priority; + } + } + RTC_DCHECK_GT(stream_bitrate_priority_sum, 0); + encoder_bitrate_priority_ = stream_bitrate_priority_sum; + encoder_max_bitrate_bps_ = + std::max(static_cast(encoder_min_bitrate_bps_), + encoder_max_bitrate_bps_); + max_padding_bitrate_ = CalculateMaxPadBitrateBps( + streams, min_transmit_bitrate_bps, config_->suspend_below_min_bitrate); + + // Clear stats for disabled layers. + for (size_t i = streams.size(); i < config_->rtp.ssrcs.size(); ++i) { + stats_proxy_->OnInactiveSsrc(config_->rtp.ssrcs[i]); + } + + const size_t num_temporal_layers = + streams.back().num_temporal_layers.value_or(1); + fec_controller_->SetEncodingData(streams[0].width, streams[0].height, + num_temporal_layers, + config_->rtp.max_packet_size); + + if (payload_router_.IsActive()) { + // The send stream is started already. Update the allocator with new bitrate + // limits. + bitrate_allocator_->AddObserver( + this, encoder_min_bitrate_bps_, encoder_max_bitrate_bps_, + max_padding_bitrate_, !config_->suspend_below_min_bitrate, + config_->track_id, encoder_bitrate_priority_, has_packet_feedback_); + } +} + +EncodedImageCallback::Result VideoSendStreamImpl::OnEncodedImage( + const EncodedImage& encoded_image, + const CodecSpecificInfo* codec_specific_info, + const RTPFragmentationHeader* fragmentation) { + // Encoded is called on whatever thread the real encoder implementation run + // on. In the case of hardware encoders, there might be several encoders + // running in parallel on different threads. + size_t simulcast_idx = 0; + if (codec_specific_info->codecType == kVideoCodecVP8) { + simulcast_idx = codec_specific_info->codecSpecific.VP8.simulcastIdx; + } + if (config_->post_encode_callback) { + config_->post_encode_callback->EncodedFrameCallback(EncodedFrame( + encoded_image._buffer, encoded_image._length, encoded_image._frameType, + simulcast_idx, encoded_image._timeStamp)); + } + { + rtc::CritScope lock(&encoder_activity_crit_sect_); + if (check_encoder_activity_task_) + check_encoder_activity_task_->UpdateEncoderActivity(); + } + + fec_controller_->UpdateWithEncodedData(encoded_image._length, + encoded_image._frameType); + EncodedImageCallback::Result result = payload_router_.OnEncodedImage( + encoded_image, codec_specific_info, fragmentation); + + RTC_DCHECK(codec_specific_info); + + int layer = codec_specific_info->codecType == kVideoCodecVP8 + ? codec_specific_info->codecSpecific.VP8.simulcastIdx + : 0; + { + rtc::CritScope lock(&ivf_writers_crit_); + if (file_writers_[layer].get()) { + bool ok = file_writers_[layer]->WriteFrame( + encoded_image, codec_specific_info->codecType); + RTC_DCHECK(ok); + } + } + + return result; +} + +void VideoSendStreamImpl::ConfigureProtection() { + RTC_DCHECK_RUN_ON(worker_queue_); + + // Consistency of FlexFEC parameters is checked in MaybeCreateFlexfecSender. + const bool flexfec_enabled = (flexfec_sender_ != nullptr); + + // Consistency of NACK and RED+ULPFEC parameters is checked in this function. + const bool nack_enabled = config_->rtp.nack.rtp_history_ms > 0; + int red_payload_type = config_->rtp.ulpfec.red_payload_type; + int ulpfec_payload_type = config_->rtp.ulpfec.ulpfec_payload_type; + + // Shorthands. + auto IsRedEnabled = [&]() { return red_payload_type >= 0; }; + auto DisableRed = [&]() { red_payload_type = -1; }; + auto IsUlpfecEnabled = [&]() { return ulpfec_payload_type >= 0; }; + auto DisableUlpfec = [&]() { ulpfec_payload_type = -1; }; + + if (webrtc::field_trial::IsEnabled("WebRTC-DisableUlpFecExperiment")) { + RTC_LOG(LS_INFO) << "Experiment to disable sending ULPFEC is enabled."; + DisableUlpfec(); + } + + // If enabled, FlexFEC takes priority over RED+ULPFEC. + if (flexfec_enabled) { + // We can safely disable RED here, because if the remote supports FlexFEC, + // we know that it has a receiver without the RED/RTX workaround. + // See http://crbug.com/webrtc/6650 for more information. + if (IsRedEnabled()) { + RTC_LOG(LS_INFO) << "Both FlexFEC and RED are configured. Disabling RED."; + DisableRed(); + } + if (IsUlpfecEnabled()) { + RTC_LOG(LS_INFO) + << "Both FlexFEC and ULPFEC are configured. Disabling ULPFEC."; + DisableUlpfec(); + } + } + + // Payload types without picture ID cannot determine that a stream is complete + // without retransmitting FEC, so using ULPFEC + NACK for H.264 (for instance) + // is a waste of bandwidth since FEC packets still have to be transmitted. + // Note that this is not the case with FlexFEC. + if (nack_enabled && IsUlpfecEnabled() && + !PayloadTypeSupportsSkippingFecPackets(config_->rtp.payload_name)) { + RTC_LOG(LS_WARNING) + << "Transmitting payload type without picture ID using " + "NACK+ULPFEC is a waste of bandwidth since ULPFEC packets " + "also have to be retransmitted. Disabling ULPFEC."; + DisableUlpfec(); + } + + // Verify payload types. + // + // Due to how old receivers work, we need to always send RED if it has been + // negotiated. This is a remnant of an old RED/RTX workaround, see + // https://codereview.webrtc.org/2469093003. + // TODO(brandtr): This change went into M56, so we can remove it in ~M59. + // At that time, we can disable RED whenever ULPFEC is disabled, as there is + // no point in using RED without ULPFEC. + if (IsRedEnabled()) { + RTC_DCHECK_GE(red_payload_type, 0); + RTC_DCHECK_LE(red_payload_type, 127); + } + if (IsUlpfecEnabled()) { + RTC_DCHECK_GE(ulpfec_payload_type, 0); + RTC_DCHECK_LE(ulpfec_payload_type, 127); + if (!IsRedEnabled()) { + RTC_LOG(LS_WARNING) + << "ULPFEC is enabled but RED is disabled. Disabling ULPFEC."; + DisableUlpfec(); + } + } + + for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_) { + // Set NACK. + rtp_rtcp->SetStorePacketsStatus(true, kMinSendSidePacketHistorySize); + // Set RED/ULPFEC information. + rtp_rtcp->SetUlpfecConfig(red_payload_type, ulpfec_payload_type); + } + + // Currently, both ULPFEC and FlexFEC use the same FEC rate calculation logic, + // so enable that logic if either of those FEC schemes are enabled. + fec_controller_->SetProtectionMethod(flexfec_enabled || IsUlpfecEnabled(), + nack_enabled); +} + +void VideoSendStreamImpl::ConfigureSsrcs() { + RTC_DCHECK_RUN_ON(worker_queue_); + // Configure regular SSRCs. + for (size_t i = 0; i < config_->rtp.ssrcs.size(); ++i) { + uint32_t ssrc = config_->rtp.ssrcs[i]; + RtpRtcp* const rtp_rtcp = rtp_rtcp_modules_[i]; + rtp_rtcp->SetSSRC(ssrc); + + // Restore RTP state if previous existed. + VideoSendStream::RtpStateMap::iterator it = suspended_ssrcs_.find(ssrc); + if (it != suspended_ssrcs_.end()) + rtp_rtcp->SetRtpState(it->second); + } + + // Set up RTX if available. + if (config_->rtp.rtx.ssrcs.empty()) + return; + + // Configure RTX SSRCs. + RTC_DCHECK_EQ(config_->rtp.rtx.ssrcs.size(), config_->rtp.ssrcs.size()); + for (size_t i = 0; i < config_->rtp.rtx.ssrcs.size(); ++i) { + uint32_t ssrc = config_->rtp.rtx.ssrcs[i]; + RtpRtcp* const rtp_rtcp = rtp_rtcp_modules_[i]; + rtp_rtcp->SetRtxSsrc(ssrc); + VideoSendStream::RtpStateMap::iterator it = suspended_ssrcs_.find(ssrc); + if (it != suspended_ssrcs_.end()) + rtp_rtcp->SetRtxState(it->second); + } + + // Configure RTX payload types. + RTC_DCHECK_GE(config_->rtp.rtx.payload_type, 0); + for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_) { + rtp_rtcp->SetRtxSendPayloadType(config_->rtp.rtx.payload_type, + config_->rtp.payload_type); + rtp_rtcp->SetRtxSendStatus(kRtxRetransmitted | kRtxRedundantPayloads); + } + if (config_->rtp.ulpfec.red_payload_type != -1 && + config_->rtp.ulpfec.red_rtx_payload_type != -1) { + for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_) { + rtp_rtcp->SetRtxSendPayloadType(config_->rtp.ulpfec.red_rtx_payload_type, + config_->rtp.ulpfec.red_payload_type); + } + } +} + +std::map VideoSendStreamImpl::GetRtpStates() const { + RTC_DCHECK_RUN_ON(worker_queue_); + std::map rtp_states; + + for (size_t i = 0; i < config_->rtp.ssrcs.size(); ++i) { + uint32_t ssrc = config_->rtp.ssrcs[i]; + RTC_DCHECK_EQ(ssrc, rtp_rtcp_modules_[i]->SSRC()); + rtp_states[ssrc] = rtp_rtcp_modules_[i]->GetRtpState(); + } + + for (size_t i = 0; i < config_->rtp.rtx.ssrcs.size(); ++i) { + uint32_t ssrc = config_->rtp.rtx.ssrcs[i]; + rtp_states[ssrc] = rtp_rtcp_modules_[i]->GetRtxState(); + } + + if (flexfec_sender_) { + uint32_t ssrc = config_->rtp.flexfec.ssrc; + rtp_states[ssrc] = flexfec_sender_->GetRtpState(); + } + + return rtp_states; +} + +std::map VideoSendStreamImpl::GetRtpPayloadStates() + const { + RTC_DCHECK_RUN_ON(worker_queue_); + return payload_router_.GetRtpPayloadStates(); +} + +void VideoSendStreamImpl::SignalNetworkState(NetworkState state) { + RTC_DCHECK_RUN_ON(worker_queue_); + for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_) { + rtp_rtcp->SetRTCPStatus(state == kNetworkUp ? config_->rtp.rtcp_mode + : RtcpMode::kOff); + } +} + +uint32_t VideoSendStreamImpl::OnBitrateUpdated(uint32_t bitrate_bps, + uint8_t fraction_loss, + int64_t rtt, + int64_t probing_interval_ms) { + RTC_DCHECK_RUN_ON(worker_queue_); + RTC_DCHECK(payload_router_.IsActive()) + << "VideoSendStream::Start has not been called."; + + // Substract overhead from bitrate. + rtc::CritScope lock(&overhead_bytes_per_packet_crit_); + uint32_t payload_bitrate_bps = bitrate_bps; + if (send_side_bwe_with_overhead_) { + payload_bitrate_bps -= CalculateOverheadRateBps( + CalculatePacketRate(bitrate_bps, + config_->rtp.max_packet_size + + transport_overhead_bytes_per_packet_), + overhead_bytes_per_packet_ + transport_overhead_bytes_per_packet_, + bitrate_bps); + } + + // Get the encoder target rate. It is the estimated network rate - + // protection overhead. + encoder_target_rate_bps_ = fec_controller_->UpdateFecRates( + payload_bitrate_bps, stats_proxy_->GetSendFrameRate(), fraction_loss, + loss_mask_vector_, rtt); + loss_mask_vector_.clear(); + + uint32_t encoder_overhead_rate_bps = + send_side_bwe_with_overhead_ + ? CalculateOverheadRateBps( + CalculatePacketRate(encoder_target_rate_bps_, + config_->rtp.max_packet_size + + transport_overhead_bytes_per_packet_ - + overhead_bytes_per_packet_), + overhead_bytes_per_packet_ + + transport_overhead_bytes_per_packet_, + bitrate_bps - encoder_target_rate_bps_) + : 0; + + // When the field trial "WebRTC-SendSideBwe-WithOverhead" is enabled + // protection_bitrate includes overhead. + uint32_t protection_bitrate = + bitrate_bps - (encoder_target_rate_bps_ + encoder_overhead_rate_bps); + + encoder_target_rate_bps_ = + std::min(encoder_max_bitrate_bps_, encoder_target_rate_bps_); + video_stream_encoder_->OnBitrateUpdated(encoder_target_rate_bps_, + fraction_loss, rtt); + stats_proxy_->OnSetEncoderTargetRate(encoder_target_rate_bps_); + return protection_bitrate; +} + +void VideoSendStreamImpl::EnableEncodedFrameRecording( + const std::vector& files, + size_t byte_limit) { + { + rtc::CritScope lock(&ivf_writers_crit_); + for (unsigned int i = 0; i < kMaxSimulcastStreams; ++i) { + if (i < files.size()) { + file_writers_[i] = IvfFileWriter::Wrap(rtc::File(files[i]), byte_limit); + } else { + file_writers_[i].reset(); + } + } + } + + if (!files.empty()) { + // Make a keyframe appear as early as possible in the logs, to give actually + // decodable output. + video_stream_encoder_->SendKeyFrame(); + } +} + +int VideoSendStreamImpl::ProtectionRequest( + const FecProtectionParams* delta_params, + const FecProtectionParams* key_params, + uint32_t* sent_video_rate_bps, + uint32_t* sent_nack_rate_bps, + uint32_t* sent_fec_rate_bps) { + RTC_DCHECK_RUN_ON(worker_queue_); + *sent_video_rate_bps = 0; + *sent_nack_rate_bps = 0; + *sent_fec_rate_bps = 0; + for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_) { + uint32_t not_used = 0; + uint32_t module_video_rate = 0; + uint32_t module_fec_rate = 0; + uint32_t module_nack_rate = 0; + rtp_rtcp->SetFecParameters(*delta_params, *key_params); + rtp_rtcp->BitrateSent(¬_used, &module_video_rate, &module_fec_rate, + &module_nack_rate); + *sent_video_rate_bps += module_video_rate; + *sent_nack_rate_bps += module_nack_rate; + *sent_fec_rate_bps += module_fec_rate; + } + return 0; +} + +void VideoSendStreamImpl::OnOverheadChanged(size_t overhead_bytes_per_packet) { + rtc::CritScope lock(&overhead_bytes_per_packet_crit_); + overhead_bytes_per_packet_ = overhead_bytes_per_packet; +} + +void VideoSendStreamImpl::SetTransportOverhead( + size_t transport_overhead_bytes_per_packet) { + if (transport_overhead_bytes_per_packet >= static_cast(kPathMTU)) { + RTC_LOG(LS_ERROR) << "Transport overhead exceeds size of ethernet frame"; + return; + } + + transport_overhead_bytes_per_packet_ = transport_overhead_bytes_per_packet; + + size_t rtp_packet_size = + std::min(config_->rtp.max_packet_size, + kPathMTU - transport_overhead_bytes_per_packet_); + + for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_) { + rtp_rtcp->SetMaxRtpPacketSize(rtp_packet_size); + } +} + +void VideoSendStreamImpl::OnPacketAdded(uint32_t ssrc, uint16_t seq_num) { + if (!worker_queue_->IsCurrent()) { + auto ptr = weak_ptr_; + worker_queue_->PostTask([=] { + if (!ptr.get()) + return; + ptr->OnPacketAdded(ssrc, seq_num); + }); + return; + } + const auto ssrcs = config_->rtp.ssrcs; + if (std::find(ssrcs.begin(), ssrcs.end(), ssrc) != ssrcs.end()) { + feedback_packet_seq_num_set_.insert(seq_num); + if (feedback_packet_seq_num_set_.size() > kSendSideSeqNumSetMaxSize) { + RTC_LOG(LS_WARNING) << "Feedback packet sequence number set exceed it's " + "max size', will get reset."; + feedback_packet_seq_num_set_.clear(); + } + } +} + +void VideoSendStreamImpl::OnPacketFeedbackVector( + const std::vector& packet_feedback_vector) { + if (!worker_queue_->IsCurrent()) { + auto ptr = weak_ptr_; + worker_queue_->PostTask([=] { + if (!ptr.get()) + return; + ptr->OnPacketFeedbackVector(packet_feedback_vector); + }); + return; + } + // Lost feedbacks are not considered to be lost packets. + for (const PacketFeedback& packet : packet_feedback_vector) { + if (auto it = feedback_packet_seq_num_set_.find(packet.sequence_number) != + feedback_packet_seq_num_set_.end()) { + const bool lost = packet.arrival_time_ms == PacketFeedback::kNotReceived; + loss_mask_vector_.push_back(lost); + feedback_packet_seq_num_set_.erase(it); + } + } +} +} // namespace internal +} // namespace webrtc diff --git a/video/video_send_stream_impl.h b/video/video_send_stream_impl.h new file mode 100644 index 0000000000..8a52141bb5 --- /dev/null +++ b/video/video_send_stream_impl.h @@ -0,0 +1,201 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef VIDEO_VIDEO_SEND_STREAM_IMPL_H_ +#define VIDEO_VIDEO_SEND_STREAM_IMPL_H_ + +#include +#include +#include +#include + +#include "call/bitrate_allocator.h" +#include "common_types.h" // NOLINT(build/include) +#include "common_video/include/video_bitrate_allocator.h" +#include "modules/rtp_rtcp/include/flexfec_sender.h" +#include "modules/utility/include/process_thread.h" +#include "modules/video_coding/utility/ivf_file_writer.h" +#include "rtc_base/weak_ptr.h" +#include "video/call_stats.h" +#include "video/encoder_rtcp_feedback.h" +#include "video/payload_router.h" +#include "video/send_delay_stats.h" +#include "video/send_statistics_proxy.h" +#include "video/video_send_stream.h" +#include "video/video_stream_encoder.h" + +namespace webrtc { +namespace internal { + +// VideoSendStreamImpl implements internal::VideoSendStream. +// It is created and destroyed on |worker_queue|. The intent is to decrease the +// need for locking and to ensure methods are called in sequence. +// Public methods except |DeliverRtcp| must be called on |worker_queue|. +// DeliverRtcp is called on the libjingle worker thread or a network thread. +// An encoder may deliver frames through the EncodedImageCallback on an +// arbitrary thread. +class VideoSendStreamImpl : public webrtc::BitrateAllocatorObserver, + public webrtc::OverheadObserver, + public webrtc::VCMProtectionCallback, + public VideoStreamEncoder::EncoderSink, + public VideoBitrateAllocationObserver, + public webrtc::PacketFeedbackObserver { + public: + VideoSendStreamImpl( + SendStatisticsProxy* stats_proxy, + rtc::TaskQueue* worker_queue, + CallStats* call_stats, + RtpTransportControllerSendInterface* transport, + BitrateAllocator* bitrate_allocator, + SendDelayStats* send_delay_stats, + VideoStreamEncoder* video_stream_encoder, + RtcEventLog* event_log, + const VideoSendStream::Config* config, + int initial_encoder_max_bitrate, + double initial_encoder_bitrate_priority, + std::map suspended_ssrcs, + std::map suspended_payload_states, + VideoEncoderConfig::ContentType content_type, + std::unique_ptr fec_controller, + RateLimiter* retransmission_limiter); + ~VideoSendStreamImpl() override; + + // RegisterProcessThread register |module_process_thread| with those objects + // that use it. Registration has to happen on the thread were + // |module_process_thread| was created (libjingle's worker thread). + // TODO(perkj): Replace the use of |module_process_thread| with a TaskQueue, + // maybe |worker_queue|. + void RegisterProcessThread(ProcessThread* module_process_thread); + void DeRegisterProcessThread(); + + void SignalNetworkState(NetworkState state); + bool DeliverRtcp(const uint8_t* packet, size_t length); + void UpdateActiveSimulcastLayers(const std::vector active_layers); + void Start(); + void Stop(); + + VideoSendStream::RtpStateMap GetRtpStates() const; + VideoSendStream::RtpPayloadStateMap GetRtpPayloadStates() const; + + void EnableEncodedFrameRecording(const std::vector& files, + size_t byte_limit); + + void SetTransportOverhead(size_t transport_overhead_per_packet); + + rtc::Optional configured_pacing_factor_; + + // From PacketFeedbackObserver. + void OnPacketAdded(uint32_t ssrc, uint16_t seq_num) override; + void OnPacketFeedbackVector( + const std::vector& packet_feedback_vector) override; + + private: + class CheckEncoderActivityTask; + + // Implements BitrateAllocatorObserver. + uint32_t OnBitrateUpdated(uint32_t bitrate_bps, + uint8_t fraction_loss, + int64_t rtt, + int64_t probing_interval_ms) override; + + // Implements webrtc::VCMProtectionCallback. + int ProtectionRequest(const FecProtectionParams* delta_params, + const FecProtectionParams* key_params, + uint32_t* sent_video_rate_bps, + uint32_t* sent_nack_rate_bps, + uint32_t* sent_fec_rate_bps) override; + + // Implements OverheadObserver. + void OnOverheadChanged(size_t overhead_bytes_per_packet) override; + + void OnEncoderConfigurationChanged(std::vector streams, + int min_transmit_bitrate_bps) override; + + // Implements EncodedImageCallback. The implementation routes encoded frames + // to the |payload_router_| and |config.pre_encode_callback| if set. + // Called on an arbitrary encoder callback thread. + EncodedImageCallback::Result OnEncodedImage( + const EncodedImage& encoded_image, + const CodecSpecificInfo* codec_specific_info, + const RTPFragmentationHeader* fragmentation) override; + + // Implements VideoBitrateAllocationObserver. + void OnBitrateAllocationUpdated(const BitrateAllocation& allocation) override; + + // Starts monitoring and sends a keyframe. + void StartupVideoSendStream(); + // Removes the bitrate observer, stops monitoring and notifies the video + // encoder of the bitrate update. + void StopVideoSendStream(); + + void ConfigureProtection(); + void ConfigureSsrcs(); + void SignalEncoderTimedOut(); + void SignalEncoderActive(); + + const bool send_side_bwe_with_overhead_; + + SendStatisticsProxy* const stats_proxy_; + const VideoSendStream::Config* const config_; + std::map suspended_ssrcs_; + + std::unique_ptr fec_controller_; + ProcessThread* module_process_thread_; + rtc::ThreadChecker module_process_thread_checker_; + rtc::TaskQueue* const worker_queue_; + + rtc::CriticalSection encoder_activity_crit_sect_; + CheckEncoderActivityTask* check_encoder_activity_task_ + RTC_GUARDED_BY(encoder_activity_crit_sect_); + + CallStats* const call_stats_; + RtpTransportControllerSendInterface* const transport_; + BitrateAllocator* const bitrate_allocator_; + + // TODO(brandtr): Move ownership to PayloadRouter. + std::unique_ptr flexfec_sender_; + + rtc::CriticalSection ivf_writers_crit_; + std::unique_ptr + file_writers_[kMaxSimulcastStreams] RTC_GUARDED_BY(ivf_writers_crit_); + + int max_padding_bitrate_; + int encoder_min_bitrate_bps_; + uint32_t encoder_max_bitrate_bps_; + uint32_t encoder_target_rate_bps_; + double encoder_bitrate_priority_; + bool has_packet_feedback_; + + VideoStreamEncoder* const video_stream_encoder_; + EncoderRtcpFeedback encoder_feedback_; + + RtcpBandwidthObserver* const bandwidth_observer_; + // RtpRtcp modules, declared here as they use other members on construction. + const std::vector rtp_rtcp_modules_; + PayloadRouter payload_router_; + + // |weak_ptr_| to our self. This is used since we can not call + // |weak_ptr_factory_.GetWeakPtr| from multiple sequences but it is ok to copy + // an existing WeakPtr. + rtc::WeakPtr weak_ptr_; + // |weak_ptr_factory_| must be declared last to make sure all WeakPtr's are + // invalidated before any other members are destroyed. + rtc::WeakPtrFactory weak_ptr_factory_; + + rtc::CriticalSection overhead_bytes_per_packet_crit_; + size_t overhead_bytes_per_packet_ + RTC_GUARDED_BY(overhead_bytes_per_packet_crit_); + size_t transport_overhead_bytes_per_packet_; + + std::unordered_set feedback_packet_seq_num_set_; + std::vector loss_mask_vector_; +}; +} // namespace internal +} // namespace webrtc +#endif // VIDEO_VIDEO_SEND_STREAM_IMPL_H_ -- cgit v1.2.3