diff options
Diffstat (limited to 'webrtc/modules/video_coding/media_optimization.cc')
| -rw-r--r-- | webrtc/modules/video_coding/media_optimization.cc | 633 |
1 files changed, 633 insertions, 0 deletions
diff --git a/webrtc/modules/video_coding/media_optimization.cc b/webrtc/modules/video_coding/media_optimization.cc new file mode 100644 index 0000000000..a234a06f9b --- /dev/null +++ b/webrtc/modules/video_coding/media_optimization.cc @@ -0,0 +1,633 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "webrtc/modules/video_coding/media_optimization.h" + +#include "webrtc/base/logging.h" +#include "webrtc/modules/video_coding/content_metrics_processing.h" +#include "webrtc/modules/video_coding/qm_select.h" +#include "webrtc/modules/video_coding/utility/frame_dropper.h" +#include "webrtc/system_wrappers/include/clock.h" + +namespace webrtc { +namespace media_optimization { +namespace { +void UpdateProtectionCallback( + VCMProtectionMethod* selected_method, + uint32_t* video_rate_bps, + uint32_t* nack_overhead_rate_bps, + uint32_t* fec_overhead_rate_bps, + VCMProtectionCallback* video_protection_callback) { + FecProtectionParams delta_fec_params; + FecProtectionParams key_fec_params; + // Get the FEC code rate for Key frames (set to 0 when NA). + key_fec_params.fec_rate = selected_method->RequiredProtectionFactorK(); + + // Get the FEC code rate for Delta frames (set to 0 when NA). + delta_fec_params.fec_rate = selected_method->RequiredProtectionFactorD(); + + // Get the FEC-UEP protection status for Key frames: UEP on/off. + key_fec_params.use_uep_protection = selected_method->RequiredUepProtectionK(); + + // Get the FEC-UEP protection status for Delta frames: UEP on/off. + delta_fec_params.use_uep_protection = + selected_method->RequiredUepProtectionD(); + + // The RTP module currently requires the same |max_fec_frames| for both + // key and delta frames. + delta_fec_params.max_fec_frames = selected_method->MaxFramesFec(); + key_fec_params.max_fec_frames = selected_method->MaxFramesFec(); + + // Set the FEC packet mask type. |kFecMaskBursty| is more effective for + // consecutive losses and little/no packet re-ordering. As we currently + // do not have feedback data on the degree of correlated losses and packet + // re-ordering, we keep default setting to |kFecMaskRandom| for now. + delta_fec_params.fec_mask_type = kFecMaskRandom; + key_fec_params.fec_mask_type = kFecMaskRandom; + + // TODO(Marco): Pass FEC protection values per layer. + video_protection_callback->ProtectionRequest( + &delta_fec_params, &key_fec_params, video_rate_bps, + nack_overhead_rate_bps, fec_overhead_rate_bps); +} +} // namespace + +struct MediaOptimization::EncodedFrameSample { + EncodedFrameSample(size_t size_bytes, + uint32_t timestamp, + int64_t time_complete_ms) + : size_bytes(size_bytes), + timestamp(timestamp), + time_complete_ms(time_complete_ms) {} + + size_t size_bytes; + uint32_t timestamp; + int64_t time_complete_ms; +}; + +MediaOptimization::MediaOptimization(Clock* clock) + : crit_sect_(CriticalSectionWrapper::CreateCriticalSection()), + clock_(clock), + max_bit_rate_(0), + send_codec_type_(kVideoCodecUnknown), + codec_width_(0), + codec_height_(0), + user_frame_rate_(0), + frame_dropper_(new FrameDropper), + loss_prot_logic_( + new VCMLossProtectionLogic(clock_->TimeInMilliseconds())), + fraction_lost_(0), + send_statistics_zero_encode_(0), + max_payload_size_(1460), + video_target_bitrate_(0), + incoming_frame_rate_(0), + enable_qm_(false), + encoded_frame_samples_(), + avg_sent_bit_rate_bps_(0), + avg_sent_framerate_(0), + key_frame_cnt_(0), + delta_frame_cnt_(0), + content_(new VCMContentMetricsProcessing()), + qm_resolution_(new VCMQmResolution()), + last_qm_update_time_(0), + last_change_time_(0), + num_layers_(0), + suspension_enabled_(false), + video_suspended_(false), + suspension_threshold_bps_(0), + suspension_window_bps_(0) { + memset(send_statistics_, 0, sizeof(send_statistics_)); + memset(incoming_frame_times_, -1, sizeof(incoming_frame_times_)); +} + +MediaOptimization::~MediaOptimization(void) { + loss_prot_logic_->Release(); +} + +void MediaOptimization::Reset() { + CriticalSectionScoped lock(crit_sect_.get()); + SetEncodingDataInternal(kVideoCodecUnknown, 0, 0, 0, 0, 0, 0, + max_payload_size_); + memset(incoming_frame_times_, -1, sizeof(incoming_frame_times_)); + incoming_frame_rate_ = 0.0; + frame_dropper_->Reset(); + loss_prot_logic_->Reset(clock_->TimeInMilliseconds()); + frame_dropper_->SetRates(0, 0); + content_->Reset(); + qm_resolution_->Reset(); + loss_prot_logic_->UpdateFrameRate(incoming_frame_rate_); + loss_prot_logic_->Reset(clock_->TimeInMilliseconds()); + send_statistics_zero_encode_ = 0; + video_target_bitrate_ = 0; + codec_width_ = 0; + codec_height_ = 0; + user_frame_rate_ = 0; + key_frame_cnt_ = 0; + delta_frame_cnt_ = 0; + last_qm_update_time_ = 0; + last_change_time_ = 0; + encoded_frame_samples_.clear(); + avg_sent_bit_rate_bps_ = 0; + num_layers_ = 1; +} + +void MediaOptimization::SetEncodingData(VideoCodecType send_codec_type, + int32_t max_bit_rate, + uint32_t target_bitrate, + uint16_t width, + uint16_t height, + uint32_t frame_rate, + int num_layers, + int32_t mtu) { + CriticalSectionScoped lock(crit_sect_.get()); + SetEncodingDataInternal(send_codec_type, max_bit_rate, frame_rate, + target_bitrate, width, height, num_layers, mtu); +} + +void MediaOptimization::SetEncodingDataInternal(VideoCodecType send_codec_type, + int32_t max_bit_rate, + uint32_t frame_rate, + uint32_t target_bitrate, + uint16_t width, + uint16_t height, + int num_layers, + int32_t mtu) { + // Everything codec specific should be reset here since this means the codec + // has changed. If native dimension values have changed, then either user + // initiated change, or QM initiated change. Will be able to determine only + // after the processing of the first frame. + last_change_time_ = clock_->TimeInMilliseconds(); + content_->Reset(); + content_->UpdateFrameRate(frame_rate); + + max_bit_rate_ = max_bit_rate; + send_codec_type_ = send_codec_type; + video_target_bitrate_ = target_bitrate; + float target_bitrate_kbps = static_cast<float>(target_bitrate) / 1000.0f; + loss_prot_logic_->UpdateBitRate(target_bitrate_kbps); + loss_prot_logic_->UpdateFrameRate(static_cast<float>(frame_rate)); + loss_prot_logic_->UpdateFrameSize(width, height); + loss_prot_logic_->UpdateNumLayers(num_layers); + frame_dropper_->Reset(); + frame_dropper_->SetRates(target_bitrate_kbps, static_cast<float>(frame_rate)); + user_frame_rate_ = static_cast<float>(frame_rate); + codec_width_ = width; + codec_height_ = height; + num_layers_ = (num_layers <= 1) ? 1 : num_layers; // Can also be zero. + max_payload_size_ = mtu; + qm_resolution_->Initialize(target_bitrate_kbps, user_frame_rate_, + codec_width_, codec_height_, num_layers_); +} + +uint32_t MediaOptimization::SetTargetRates( + uint32_t target_bitrate, + uint8_t fraction_lost, + int64_t round_trip_time_ms, + VCMProtectionCallback* protection_callback, + VCMQMSettingsCallback* qmsettings_callback) { + CriticalSectionScoped lock(crit_sect_.get()); + VCMProtectionMethod* selected_method = loss_prot_logic_->SelectedMethod(); + float target_bitrate_kbps = static_cast<float>(target_bitrate) / 1000.0f; + loss_prot_logic_->UpdateBitRate(target_bitrate_kbps); + loss_prot_logic_->UpdateRtt(round_trip_time_ms); + + // Get frame rate for encoder: this is the actual/sent frame rate. + float actual_frame_rate = SentFrameRateInternal(); + + // Sanity check. + if (actual_frame_rate < 1.0) { + actual_frame_rate = 1.0; + } + + // Update frame rate for the loss protection logic class: frame rate should + // be the actual/sent rate. + loss_prot_logic_->UpdateFrameRate(actual_frame_rate); + + fraction_lost_ = fraction_lost; + + // Returns the filtered packet loss, used for the protection setting. + // The filtered loss may be the received loss (no filter), or some + // filtered value (average or max window filter). + // Use max window filter for now. + FilterPacketLossMode filter_mode = kMaxFilter; + uint8_t packet_loss_enc = loss_prot_logic_->FilteredLoss( + clock_->TimeInMilliseconds(), filter_mode, fraction_lost); + + // For now use the filtered loss for computing the robustness settings. + loss_prot_logic_->UpdateFilteredLossPr(packet_loss_enc); + + // Rate cost of the protection methods. + float protection_overhead_rate = 0.0f; + + // Update protection settings, when applicable. + float sent_video_rate_kbps = 0.0f; + if (loss_prot_logic_->SelectedType() != kNone) { + // Update protection method with content metrics. + selected_method->UpdateContentMetrics(content_->ShortTermAvgData()); + + // Update method will compute the robustness settings for the given + // protection method and the overhead cost + // the protection method is set by the user via SetVideoProtection. + loss_prot_logic_->UpdateMethod(); + + // Update protection callback with protection settings. + uint32_t sent_video_rate_bps = 0; + uint32_t sent_nack_rate_bps = 0; + uint32_t sent_fec_rate_bps = 0; + // Get the bit cost of protection method, based on the amount of + // overhead data actually transmitted (including headers) the last + // second. + if (protection_callback) { + UpdateProtectionCallback(selected_method, &sent_video_rate_bps, + &sent_nack_rate_bps, &sent_fec_rate_bps, + protection_callback); + } + uint32_t sent_total_rate_bps = + sent_video_rate_bps + sent_nack_rate_bps + sent_fec_rate_bps; + // Estimate the overhead costs of the next second as staying the same + // wrt the source bitrate. + if (sent_total_rate_bps > 0) { + protection_overhead_rate = + static_cast<float>(sent_nack_rate_bps + sent_fec_rate_bps) / + sent_total_rate_bps; + } + // Cap the overhead estimate to 50%. + if (protection_overhead_rate > 0.5) + protection_overhead_rate = 0.5; + + // Get the effective packet loss for encoder ER when applicable. Should be + // passed to encoder via fraction_lost. + packet_loss_enc = selected_method->RequiredPacketLossER(); + sent_video_rate_kbps = static_cast<float>(sent_video_rate_bps) / 1000.0f; + } + + // Source coding rate: total rate - protection overhead. + video_target_bitrate_ = target_bitrate * (1.0 - protection_overhead_rate); + + // Cap target video bitrate to codec maximum. + if (max_bit_rate_ > 0 && video_target_bitrate_ > max_bit_rate_) { + video_target_bitrate_ = max_bit_rate_; + } + + // Update encoding rates following protection settings. + float target_video_bitrate_kbps = + static_cast<float>(video_target_bitrate_) / 1000.0f; + frame_dropper_->SetRates(target_video_bitrate_kbps, incoming_frame_rate_); + + if (enable_qm_ && qmsettings_callback) { + // Update QM with rates. + qm_resolution_->UpdateRates(target_video_bitrate_kbps, sent_video_rate_kbps, + incoming_frame_rate_, fraction_lost_); + // Check for QM selection. + bool select_qm = CheckStatusForQMchange(); + if (select_qm) { + SelectQuality(qmsettings_callback); + } + // Reset the short-term averaged content data. + content_->ResetShortTermAvgData(); + } + + CheckSuspendConditions(); + + return video_target_bitrate_; +} + +void MediaOptimization::SetProtectionMethod(VCMProtectionMethodEnum method) { + CriticalSectionScoped lock(crit_sect_.get()); + loss_prot_logic_->SetMethod(method); +} + +uint32_t MediaOptimization::InputFrameRate() { + CriticalSectionScoped lock(crit_sect_.get()); + return InputFrameRateInternal(); +} + +uint32_t MediaOptimization::InputFrameRateInternal() { + ProcessIncomingFrameRate(clock_->TimeInMilliseconds()); + return uint32_t(incoming_frame_rate_ + 0.5f); +} + +uint32_t MediaOptimization::SentFrameRate() { + CriticalSectionScoped lock(crit_sect_.get()); + return SentFrameRateInternal(); +} + +uint32_t MediaOptimization::SentFrameRateInternal() { + PurgeOldFrameSamples(clock_->TimeInMilliseconds()); + UpdateSentFramerate(); + return avg_sent_framerate_; +} + +uint32_t MediaOptimization::SentBitRate() { + CriticalSectionScoped lock(crit_sect_.get()); + const int64_t now_ms = clock_->TimeInMilliseconds(); + PurgeOldFrameSamples(now_ms); + UpdateSentBitrate(now_ms); + return avg_sent_bit_rate_bps_; +} + +int32_t MediaOptimization::UpdateWithEncodedData( + const EncodedImage& encoded_image) { + size_t encoded_length = encoded_image._length; + uint32_t timestamp = encoded_image._timeStamp; + CriticalSectionScoped lock(crit_sect_.get()); + const int64_t now_ms = clock_->TimeInMilliseconds(); + PurgeOldFrameSamples(now_ms); + if (encoded_frame_samples_.size() > 0 && + encoded_frame_samples_.back().timestamp == timestamp) { + // Frames having the same timestamp are generated from the same input + // frame. We don't want to double count them, but only increment the + // size_bytes. + encoded_frame_samples_.back().size_bytes += encoded_length; + encoded_frame_samples_.back().time_complete_ms = now_ms; + } else { + encoded_frame_samples_.push_back( + EncodedFrameSample(encoded_length, timestamp, now_ms)); + } + UpdateSentBitrate(now_ms); + UpdateSentFramerate(); + if (encoded_length > 0) { + const bool delta_frame = encoded_image._frameType != kVideoFrameKey; + + frame_dropper_->Fill(encoded_length, delta_frame); + if (max_payload_size_ > 0 && encoded_length > 0) { + const float min_packets_per_frame = + encoded_length / static_cast<float>(max_payload_size_); + if (delta_frame) { + loss_prot_logic_->UpdatePacketsPerFrame(min_packets_per_frame, + clock_->TimeInMilliseconds()); + } else { + loss_prot_logic_->UpdatePacketsPerFrameKey( + min_packets_per_frame, clock_->TimeInMilliseconds()); + } + + if (enable_qm_) { + // Update quality select with encoded length. + qm_resolution_->UpdateEncodedSize(encoded_length); + } + } + if (!delta_frame && encoded_length > 0) { + loss_prot_logic_->UpdateKeyFrameSize(static_cast<float>(encoded_length)); + } + + // Updating counters. + if (delta_frame) { + delta_frame_cnt_++; + } else { + key_frame_cnt_++; + } + } + + return VCM_OK; +} + +void MediaOptimization::EnableQM(bool enable) { + CriticalSectionScoped lock(crit_sect_.get()); + enable_qm_ = enable; +} + +void MediaOptimization::EnableFrameDropper(bool enable) { + CriticalSectionScoped lock(crit_sect_.get()); + frame_dropper_->Enable(enable); +} + +void MediaOptimization::SuspendBelowMinBitrate(int threshold_bps, + int window_bps) { + CriticalSectionScoped lock(crit_sect_.get()); + assert(threshold_bps > 0 && window_bps >= 0); + suspension_threshold_bps_ = threshold_bps; + suspension_window_bps_ = window_bps; + suspension_enabled_ = true; + video_suspended_ = false; +} + +bool MediaOptimization::IsVideoSuspended() const { + CriticalSectionScoped lock(crit_sect_.get()); + return video_suspended_; +} + +bool MediaOptimization::DropFrame() { + CriticalSectionScoped lock(crit_sect_.get()); + UpdateIncomingFrameRate(); + // Leak appropriate number of bytes. + frame_dropper_->Leak((uint32_t)(InputFrameRateInternal() + 0.5f)); + if (video_suspended_) { + return true; // Drop all frames when muted. + } + return frame_dropper_->DropFrame(); +} + +void MediaOptimization::UpdateContentData( + const VideoContentMetrics* content_metrics) { + CriticalSectionScoped lock(crit_sect_.get()); + // Updating content metrics. + if (content_metrics == NULL) { + // Disable QM if metrics are NULL. + enable_qm_ = false; + qm_resolution_->Reset(); + } else { + content_->UpdateContentData(content_metrics); + } +} + +void MediaOptimization::UpdateIncomingFrameRate() { + int64_t now = clock_->TimeInMilliseconds(); + if (incoming_frame_times_[0] == 0) { + // No shifting if this is the first time. + } else { + // Shift all times one step. + for (int32_t i = (kFrameCountHistorySize - 2); i >= 0; i--) { + incoming_frame_times_[i + 1] = incoming_frame_times_[i]; + } + } + incoming_frame_times_[0] = now; + ProcessIncomingFrameRate(now); +} + +int32_t MediaOptimization::SelectQuality( + VCMQMSettingsCallback* video_qmsettings_callback) { + // Reset quantities for QM select. + qm_resolution_->ResetQM(); + + // Update QM will long-term averaged content metrics. + qm_resolution_->UpdateContent(content_->LongTermAvgData()); + + // Select quality mode. + VCMResolutionScale* qm = NULL; + int32_t ret = qm_resolution_->SelectResolution(&qm); + if (ret < 0) { + return ret; + } + + // Check for updates to spatial/temporal modes. + QMUpdate(qm, video_qmsettings_callback); + + // Reset all the rate and related frame counters quantities. + qm_resolution_->ResetRates(); + + // Reset counters. + last_qm_update_time_ = clock_->TimeInMilliseconds(); + + // Reset content metrics. + content_->Reset(); + + return VCM_OK; +} + +void MediaOptimization::PurgeOldFrameSamples(int64_t now_ms) { + while (!encoded_frame_samples_.empty()) { + if (now_ms - encoded_frame_samples_.front().time_complete_ms > + kBitrateAverageWinMs) { + encoded_frame_samples_.pop_front(); + } else { + break; + } + } +} + +void MediaOptimization::UpdateSentBitrate(int64_t now_ms) { + if (encoded_frame_samples_.empty()) { + avg_sent_bit_rate_bps_ = 0; + return; + } + size_t framesize_sum = 0; + for (FrameSampleList::iterator it = encoded_frame_samples_.begin(); + it != encoded_frame_samples_.end(); ++it) { + framesize_sum += it->size_bytes; + } + float denom = static_cast<float>( + now_ms - encoded_frame_samples_.front().time_complete_ms); + if (denom >= 1.0f) { + avg_sent_bit_rate_bps_ = + static_cast<uint32_t>(framesize_sum * 8.0f * 1000.0f / denom + 0.5f); + } else { + avg_sent_bit_rate_bps_ = framesize_sum * 8; + } +} + +void MediaOptimization::UpdateSentFramerate() { + if (encoded_frame_samples_.size() <= 1) { + avg_sent_framerate_ = encoded_frame_samples_.size(); + return; + } + int denom = encoded_frame_samples_.back().timestamp - + encoded_frame_samples_.front().timestamp; + if (denom > 0) { + avg_sent_framerate_ = + (90000 * (encoded_frame_samples_.size() - 1) + denom / 2) / denom; + } else { + avg_sent_framerate_ = encoded_frame_samples_.size(); + } +} + +bool MediaOptimization::QMUpdate( + VCMResolutionScale* qm, + VCMQMSettingsCallback* video_qmsettings_callback) { + // Check for no change. + if (!qm->change_resolution_spatial && !qm->change_resolution_temporal) { + return false; + } + + // Check for change in frame rate. + if (qm->change_resolution_temporal) { + incoming_frame_rate_ = qm->frame_rate; + // Reset frame rate estimate. + memset(incoming_frame_times_, -1, sizeof(incoming_frame_times_)); + } + + // Check for change in frame size. + if (qm->change_resolution_spatial) { + codec_width_ = qm->codec_width; + codec_height_ = qm->codec_height; + } + + LOG(LS_INFO) << "Media optimizer requests the video resolution to be changed " + "to " + << qm->codec_width << "x" << qm->codec_height << "@" + << qm->frame_rate; + + // Update VPM with new target frame rate and frame size. + // Note: use |qm->frame_rate| instead of |_incoming_frame_rate| for updating + // target frame rate in VPM frame dropper. The quantity |_incoming_frame_rate| + // will vary/fluctuate, and since we don't want to change the state of the + // VPM frame dropper, unless a temporal action was selected, we use the + // quantity |qm->frame_rate| for updating. + video_qmsettings_callback->SetVideoQMSettings(qm->frame_rate, codec_width_, + codec_height_); + content_->UpdateFrameRate(qm->frame_rate); + qm_resolution_->UpdateCodecParameters(qm->frame_rate, codec_width_, + codec_height_); + return true; +} + +// Check timing constraints and look for significant change in: +// (1) scene content, +// (2) target bit rate. +bool MediaOptimization::CheckStatusForQMchange() { + bool status = true; + + // Check that we do not call QMSelect too often, and that we waited some time + // (to sample the metrics) from the event last_change_time + // last_change_time is the time where user changed the size/rate/frame rate + // (via SetEncodingData). + int64_t now = clock_->TimeInMilliseconds(); + if ((now - last_qm_update_time_) < kQmMinIntervalMs || + (now - last_change_time_) < kQmMinIntervalMs) { + status = false; + } + + return status; +} + +// Allowing VCM to keep track of incoming frame rate. +void MediaOptimization::ProcessIncomingFrameRate(int64_t now) { + int32_t num = 0; + int32_t nr_of_frames = 0; + for (num = 1; num < (kFrameCountHistorySize - 1); ++num) { + if (incoming_frame_times_[num] <= 0 || + // don't use data older than 2 s + now - incoming_frame_times_[num] > kFrameHistoryWinMs) { + break; + } else { + nr_of_frames++; + } + } + if (num > 1) { + const int64_t diff = + incoming_frame_times_[0] - incoming_frame_times_[num - 1]; + incoming_frame_rate_ = 0.0; // No frame rate estimate available. + if (diff > 0) { + incoming_frame_rate_ = nr_of_frames * 1000.0f / static_cast<float>(diff); + } + } +} + +void MediaOptimization::CheckSuspendConditions() { + // Check conditions for SuspendBelowMinBitrate. |video_target_bitrate_| is in + // bps. + if (suspension_enabled_) { + if (!video_suspended_) { + // Check if we just went below the threshold. + if (video_target_bitrate_ < suspension_threshold_bps_) { + video_suspended_ = true; + } + } else { + // Video is already suspended. Check if we just went over the threshold + // with a margin. + if (video_target_bitrate_ > + suspension_threshold_bps_ + suspension_window_bps_) { + video_suspended_ = false; + } + } + } +} + +} // namespace media_optimization +} // namespace webrtc |