diff options
Diffstat (limited to 'webrtc/modules/video_coding/main/source/media_optimization.cc')
| -rw-r--r-- | webrtc/modules/video_coding/main/source/media_optimization.cc | 648 |
1 files changed, 0 insertions, 648 deletions
diff --git a/webrtc/modules/video_coding/main/source/media_optimization.cc b/webrtc/modules/video_coding/main/source/media_optimization.cc deleted file mode 100644 index cc73d3803d..0000000000 --- a/webrtc/modules/video_coding/main/source/media_optimization.cc +++ /dev/null @@ -1,648 +0,0 @@ -/* - * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "webrtc/modules/video_coding/main/source/media_optimization.h" - -#include "webrtc/modules/video_coding/main/source/content_metrics_processing.h" -#include "webrtc/modules/video_coding/main/source/qm_select.h" -#include "webrtc/modules/video_coding/utility/include/frame_dropper.h" -#include "webrtc/system_wrappers/include/clock.h" -#include "webrtc/system_wrappers/include/logging.h" - -namespace webrtc { -namespace media_optimization { -namespace { -void UpdateProtectionCallback( - VCMProtectionMethod* selected_method, - uint32_t* video_rate_bps, - uint32_t* nack_overhead_rate_bps, - uint32_t* fec_overhead_rate_bps, - VCMProtectionCallback* video_protection_callback) { - FecProtectionParams delta_fec_params; - FecProtectionParams key_fec_params; - // Get the FEC code rate for Key frames (set to 0 when NA). - key_fec_params.fec_rate = selected_method->RequiredProtectionFactorK(); - - // Get the FEC code rate for Delta frames (set to 0 when NA). - delta_fec_params.fec_rate = selected_method->RequiredProtectionFactorD(); - - // Get the FEC-UEP protection status for Key frames: UEP on/off. - key_fec_params.use_uep_protection = selected_method->RequiredUepProtectionK(); - - // Get the FEC-UEP protection status for Delta frames: UEP on/off. - delta_fec_params.use_uep_protection = - selected_method->RequiredUepProtectionD(); - - // The RTP module currently requires the same |max_fec_frames| for both - // key and delta frames. - delta_fec_params.max_fec_frames = selected_method->MaxFramesFec(); - key_fec_params.max_fec_frames = selected_method->MaxFramesFec(); - - // Set the FEC packet mask type. |kFecMaskBursty| is more effective for - // consecutive losses and little/no packet re-ordering. As we currently - // do not have feedback data on the degree of correlated losses and packet - // re-ordering, we keep default setting to |kFecMaskRandom| for now. - delta_fec_params.fec_mask_type = kFecMaskRandom; - key_fec_params.fec_mask_type = kFecMaskRandom; - - // TODO(Marco): Pass FEC protection values per layer. - video_protection_callback->ProtectionRequest(&delta_fec_params, - &key_fec_params, - video_rate_bps, - nack_overhead_rate_bps, - fec_overhead_rate_bps); -} -} // namespace - -struct MediaOptimization::EncodedFrameSample { - EncodedFrameSample(size_t size_bytes, - uint32_t timestamp, - int64_t time_complete_ms) - : size_bytes(size_bytes), - timestamp(timestamp), - time_complete_ms(time_complete_ms) {} - - size_t size_bytes; - uint32_t timestamp; - int64_t time_complete_ms; -}; - -MediaOptimization::MediaOptimization(Clock* clock) - : crit_sect_(CriticalSectionWrapper::CreateCriticalSection()), - clock_(clock), - max_bit_rate_(0), - send_codec_type_(kVideoCodecUnknown), - codec_width_(0), - codec_height_(0), - user_frame_rate_(0), - frame_dropper_(new FrameDropper), - loss_prot_logic_( - new VCMLossProtectionLogic(clock_->TimeInMilliseconds())), - fraction_lost_(0), - send_statistics_zero_encode_(0), - max_payload_size_(1460), - video_target_bitrate_(0), - incoming_frame_rate_(0), - enable_qm_(false), - encoded_frame_samples_(), - avg_sent_bit_rate_bps_(0), - avg_sent_framerate_(0), - key_frame_cnt_(0), - delta_frame_cnt_(0), - content_(new VCMContentMetricsProcessing()), - qm_resolution_(new VCMQmResolution()), - last_qm_update_time_(0), - last_change_time_(0), - num_layers_(0), - suspension_enabled_(false), - video_suspended_(false), - suspension_threshold_bps_(0), - suspension_window_bps_(0) { - memset(send_statistics_, 0, sizeof(send_statistics_)); - memset(incoming_frame_times_, -1, sizeof(incoming_frame_times_)); -} - -MediaOptimization::~MediaOptimization(void) { - loss_prot_logic_->Release(); -} - -void MediaOptimization::Reset() { - CriticalSectionScoped lock(crit_sect_.get()); - SetEncodingDataInternal( - kVideoCodecUnknown, 0, 0, 0, 0, 0, 0, max_payload_size_); - memset(incoming_frame_times_, -1, sizeof(incoming_frame_times_)); - incoming_frame_rate_ = 0.0; - frame_dropper_->Reset(); - loss_prot_logic_->Reset(clock_->TimeInMilliseconds()); - frame_dropper_->SetRates(0, 0); - content_->Reset(); - qm_resolution_->Reset(); - loss_prot_logic_->UpdateFrameRate(incoming_frame_rate_); - loss_prot_logic_->Reset(clock_->TimeInMilliseconds()); - send_statistics_zero_encode_ = 0; - video_target_bitrate_ = 0; - codec_width_ = 0; - codec_height_ = 0; - user_frame_rate_ = 0; - key_frame_cnt_ = 0; - delta_frame_cnt_ = 0; - last_qm_update_time_ = 0; - last_change_time_ = 0; - encoded_frame_samples_.clear(); - avg_sent_bit_rate_bps_ = 0; - num_layers_ = 1; -} - -void MediaOptimization::SetEncodingData(VideoCodecType send_codec_type, - int32_t max_bit_rate, - uint32_t target_bitrate, - uint16_t width, - uint16_t height, - uint32_t frame_rate, - int num_layers, - int32_t mtu) { - CriticalSectionScoped lock(crit_sect_.get()); - SetEncodingDataInternal(send_codec_type, - max_bit_rate, - frame_rate, - target_bitrate, - width, - height, - num_layers, - mtu); -} - -void MediaOptimization::SetEncodingDataInternal(VideoCodecType send_codec_type, - int32_t max_bit_rate, - uint32_t frame_rate, - uint32_t target_bitrate, - uint16_t width, - uint16_t height, - int num_layers, - int32_t mtu) { - // Everything codec specific should be reset here since this means the codec - // has changed. If native dimension values have changed, then either user - // initiated change, or QM initiated change. Will be able to determine only - // after the processing of the first frame. - last_change_time_ = clock_->TimeInMilliseconds(); - content_->Reset(); - content_->UpdateFrameRate(frame_rate); - - max_bit_rate_ = max_bit_rate; - send_codec_type_ = send_codec_type; - video_target_bitrate_ = target_bitrate; - float target_bitrate_kbps = static_cast<float>(target_bitrate) / 1000.0f; - loss_prot_logic_->UpdateBitRate(target_bitrate_kbps); - loss_prot_logic_->UpdateFrameRate(static_cast<float>(frame_rate)); - loss_prot_logic_->UpdateFrameSize(width, height); - loss_prot_logic_->UpdateNumLayers(num_layers); - frame_dropper_->Reset(); - frame_dropper_->SetRates(target_bitrate_kbps, static_cast<float>(frame_rate)); - user_frame_rate_ = static_cast<float>(frame_rate); - codec_width_ = width; - codec_height_ = height; - num_layers_ = (num_layers <= 1) ? 1 : num_layers; // Can also be zero. - max_payload_size_ = mtu; - qm_resolution_->Initialize(target_bitrate_kbps, - user_frame_rate_, - codec_width_, - codec_height_, - num_layers_); -} - -uint32_t MediaOptimization::SetTargetRates( - uint32_t target_bitrate, - uint8_t fraction_lost, - int64_t round_trip_time_ms, - VCMProtectionCallback* protection_callback, - VCMQMSettingsCallback* qmsettings_callback) { - CriticalSectionScoped lock(crit_sect_.get()); - VCMProtectionMethod* selected_method = loss_prot_logic_->SelectedMethod(); - float target_bitrate_kbps = static_cast<float>(target_bitrate) / 1000.0f; - loss_prot_logic_->UpdateBitRate(target_bitrate_kbps); - loss_prot_logic_->UpdateRtt(round_trip_time_ms); - - // Get frame rate for encoder: this is the actual/sent frame rate. - float actual_frame_rate = SentFrameRateInternal(); - - // Sanity check. - if (actual_frame_rate < 1.0) { - actual_frame_rate = 1.0; - } - - // Update frame rate for the loss protection logic class: frame rate should - // be the actual/sent rate. - loss_prot_logic_->UpdateFrameRate(actual_frame_rate); - - fraction_lost_ = fraction_lost; - - // Returns the filtered packet loss, used for the protection setting. - // The filtered loss may be the received loss (no filter), or some - // filtered value (average or max window filter). - // Use max window filter for now. - FilterPacketLossMode filter_mode = kMaxFilter; - uint8_t packet_loss_enc = loss_prot_logic_->FilteredLoss( - clock_->TimeInMilliseconds(), filter_mode, fraction_lost); - - // For now use the filtered loss for computing the robustness settings. - loss_prot_logic_->UpdateFilteredLossPr(packet_loss_enc); - - // Rate cost of the protection methods. - float protection_overhead_rate = 0.0f; - - // Update protection settings, when applicable. - float sent_video_rate_kbps = 0.0f; - if (loss_prot_logic_->SelectedType() != kNone) { - // Update protection method with content metrics. - selected_method->UpdateContentMetrics(content_->ShortTermAvgData()); - - // Update method will compute the robustness settings for the given - // protection method and the overhead cost - // the protection method is set by the user via SetVideoProtection. - loss_prot_logic_->UpdateMethod(); - - // Update protection callback with protection settings. - uint32_t sent_video_rate_bps = 0; - uint32_t sent_nack_rate_bps = 0; - uint32_t sent_fec_rate_bps = 0; - // Get the bit cost of protection method, based on the amount of - // overhead data actually transmitted (including headers) the last - // second. - if (protection_callback) { - UpdateProtectionCallback(selected_method, - &sent_video_rate_bps, - &sent_nack_rate_bps, - &sent_fec_rate_bps, - protection_callback); - } - uint32_t sent_total_rate_bps = - sent_video_rate_bps + sent_nack_rate_bps + sent_fec_rate_bps; - // Estimate the overhead costs of the next second as staying the same - // wrt the source bitrate. - if (sent_total_rate_bps > 0) { - protection_overhead_rate = - static_cast<float>(sent_nack_rate_bps + sent_fec_rate_bps) / - sent_total_rate_bps; - } - // Cap the overhead estimate to 50%. - if (protection_overhead_rate > 0.5) - protection_overhead_rate = 0.5; - - // Get the effective packet loss for encoder ER when applicable. Should be - // passed to encoder via fraction_lost. - packet_loss_enc = selected_method->RequiredPacketLossER(); - sent_video_rate_kbps = static_cast<float>(sent_video_rate_bps) / 1000.0f; - } - - // Source coding rate: total rate - protection overhead. - video_target_bitrate_ = target_bitrate * (1.0 - protection_overhead_rate); - - // Cap target video bitrate to codec maximum. - if (max_bit_rate_ > 0 && video_target_bitrate_ > max_bit_rate_) { - video_target_bitrate_ = max_bit_rate_; - } - - // Update encoding rates following protection settings. - float target_video_bitrate_kbps = - static_cast<float>(video_target_bitrate_) / 1000.0f; - frame_dropper_->SetRates(target_video_bitrate_kbps, incoming_frame_rate_); - - if (enable_qm_ && qmsettings_callback) { - // Update QM with rates. - qm_resolution_->UpdateRates(target_video_bitrate_kbps, - sent_video_rate_kbps, - incoming_frame_rate_, - fraction_lost_); - // Check for QM selection. - bool select_qm = CheckStatusForQMchange(); - if (select_qm) { - SelectQuality(qmsettings_callback); - } - // Reset the short-term averaged content data. - content_->ResetShortTermAvgData(); - } - - CheckSuspendConditions(); - - return video_target_bitrate_; -} - -void MediaOptimization::SetProtectionMethod(VCMProtectionMethodEnum method) { - CriticalSectionScoped lock(crit_sect_.get()); - loss_prot_logic_->SetMethod(method); -} - -uint32_t MediaOptimization::InputFrameRate() { - CriticalSectionScoped lock(crit_sect_.get()); - return InputFrameRateInternal(); -} - -uint32_t MediaOptimization::InputFrameRateInternal() { - ProcessIncomingFrameRate(clock_->TimeInMilliseconds()); - return uint32_t(incoming_frame_rate_ + 0.5f); -} - -uint32_t MediaOptimization::SentFrameRate() { - CriticalSectionScoped lock(crit_sect_.get()); - return SentFrameRateInternal(); -} - -uint32_t MediaOptimization::SentFrameRateInternal() { - PurgeOldFrameSamples(clock_->TimeInMilliseconds()); - UpdateSentFramerate(); - return avg_sent_framerate_; -} - -uint32_t MediaOptimization::SentBitRate() { - CriticalSectionScoped lock(crit_sect_.get()); - const int64_t now_ms = clock_->TimeInMilliseconds(); - PurgeOldFrameSamples(now_ms); - UpdateSentBitrate(now_ms); - return avg_sent_bit_rate_bps_; -} - -int32_t MediaOptimization::UpdateWithEncodedData( - const EncodedImage& encoded_image) { - size_t encoded_length = encoded_image._length; - uint32_t timestamp = encoded_image._timeStamp; - CriticalSectionScoped lock(crit_sect_.get()); - const int64_t now_ms = clock_->TimeInMilliseconds(); - PurgeOldFrameSamples(now_ms); - if (encoded_frame_samples_.size() > 0 && - encoded_frame_samples_.back().timestamp == timestamp) { - // Frames having the same timestamp are generated from the same input - // frame. We don't want to double count them, but only increment the - // size_bytes. - encoded_frame_samples_.back().size_bytes += encoded_length; - encoded_frame_samples_.back().time_complete_ms = now_ms; - } else { - encoded_frame_samples_.push_back( - EncodedFrameSample(encoded_length, timestamp, now_ms)); - } - UpdateSentBitrate(now_ms); - UpdateSentFramerate(); - if (encoded_length > 0) { - const bool delta_frame = encoded_image._frameType != kVideoFrameKey; - - frame_dropper_->Fill(encoded_length, delta_frame); - if (max_payload_size_ > 0 && encoded_length > 0) { - const float min_packets_per_frame = - encoded_length / static_cast<float>(max_payload_size_); - if (delta_frame) { - loss_prot_logic_->UpdatePacketsPerFrame(min_packets_per_frame, - clock_->TimeInMilliseconds()); - } else { - loss_prot_logic_->UpdatePacketsPerFrameKey( - min_packets_per_frame, clock_->TimeInMilliseconds()); - } - - if (enable_qm_) { - // Update quality select with encoded length. - qm_resolution_->UpdateEncodedSize(encoded_length); - } - } - if (!delta_frame && encoded_length > 0) { - loss_prot_logic_->UpdateKeyFrameSize(static_cast<float>(encoded_length)); - } - - // Updating counters. - if (delta_frame) { - delta_frame_cnt_++; - } else { - key_frame_cnt_++; - } - } - - return VCM_OK; -} - -void MediaOptimization::EnableQM(bool enable) { - CriticalSectionScoped lock(crit_sect_.get()); - enable_qm_ = enable; -} - -void MediaOptimization::EnableFrameDropper(bool enable) { - CriticalSectionScoped lock(crit_sect_.get()); - frame_dropper_->Enable(enable); -} - -void MediaOptimization::SuspendBelowMinBitrate(int threshold_bps, - int window_bps) { - CriticalSectionScoped lock(crit_sect_.get()); - assert(threshold_bps > 0 && window_bps >= 0); - suspension_threshold_bps_ = threshold_bps; - suspension_window_bps_ = window_bps; - suspension_enabled_ = true; - video_suspended_ = false; -} - -bool MediaOptimization::IsVideoSuspended() const { - CriticalSectionScoped lock(crit_sect_.get()); - return video_suspended_; -} - -bool MediaOptimization::DropFrame() { - CriticalSectionScoped lock(crit_sect_.get()); - UpdateIncomingFrameRate(); - // Leak appropriate number of bytes. - frame_dropper_->Leak((uint32_t)(InputFrameRateInternal() + 0.5f)); - if (video_suspended_) { - return true; // Drop all frames when muted. - } - return frame_dropper_->DropFrame(); -} - -void MediaOptimization::UpdateContentData( - const VideoContentMetrics* content_metrics) { - CriticalSectionScoped lock(crit_sect_.get()); - // Updating content metrics. - if (content_metrics == NULL) { - // Disable QM if metrics are NULL. - enable_qm_ = false; - qm_resolution_->Reset(); - } else { - content_->UpdateContentData(content_metrics); - } -} - -void MediaOptimization::UpdateIncomingFrameRate() { - int64_t now = clock_->TimeInMilliseconds(); - if (incoming_frame_times_[0] == 0) { - // No shifting if this is the first time. - } else { - // Shift all times one step. - for (int32_t i = (kFrameCountHistorySize - 2); i >= 0; i--) { - incoming_frame_times_[i + 1] = incoming_frame_times_[i]; - } - } - incoming_frame_times_[0] = now; - ProcessIncomingFrameRate(now); -} - -int32_t MediaOptimization::SelectQuality( - VCMQMSettingsCallback* video_qmsettings_callback) { - // Reset quantities for QM select. - qm_resolution_->ResetQM(); - - // Update QM will long-term averaged content metrics. - qm_resolution_->UpdateContent(content_->LongTermAvgData()); - - // Select quality mode. - VCMResolutionScale* qm = NULL; - int32_t ret = qm_resolution_->SelectResolution(&qm); - if (ret < 0) { - return ret; - } - - // Check for updates to spatial/temporal modes. - QMUpdate(qm, video_qmsettings_callback); - - // Reset all the rate and related frame counters quantities. - qm_resolution_->ResetRates(); - - // Reset counters. - last_qm_update_time_ = clock_->TimeInMilliseconds(); - - // Reset content metrics. - content_->Reset(); - - return VCM_OK; -} - -void MediaOptimization::PurgeOldFrameSamples(int64_t now_ms) { - while (!encoded_frame_samples_.empty()) { - if (now_ms - encoded_frame_samples_.front().time_complete_ms > - kBitrateAverageWinMs) { - encoded_frame_samples_.pop_front(); - } else { - break; - } - } -} - -void MediaOptimization::UpdateSentBitrate(int64_t now_ms) { - if (encoded_frame_samples_.empty()) { - avg_sent_bit_rate_bps_ = 0; - return; - } - size_t framesize_sum = 0; - for (FrameSampleList::iterator it = encoded_frame_samples_.begin(); - it != encoded_frame_samples_.end(); - ++it) { - framesize_sum += it->size_bytes; - } - float denom = static_cast<float>( - now_ms - encoded_frame_samples_.front().time_complete_ms); - if (denom >= 1.0f) { - avg_sent_bit_rate_bps_ = - static_cast<uint32_t>(framesize_sum * 8.0f * 1000.0f / denom + 0.5f); - } else { - avg_sent_bit_rate_bps_ = framesize_sum * 8; - } -} - -void MediaOptimization::UpdateSentFramerate() { - if (encoded_frame_samples_.size() <= 1) { - avg_sent_framerate_ = encoded_frame_samples_.size(); - return; - } - int denom = encoded_frame_samples_.back().timestamp - - encoded_frame_samples_.front().timestamp; - if (denom > 0) { - avg_sent_framerate_ = - (90000 * (encoded_frame_samples_.size() - 1) + denom / 2) / denom; - } else { - avg_sent_framerate_ = encoded_frame_samples_.size(); - } -} - -bool MediaOptimization::QMUpdate( - VCMResolutionScale* qm, - VCMQMSettingsCallback* video_qmsettings_callback) { - // Check for no change. - if (!qm->change_resolution_spatial && !qm->change_resolution_temporal) { - return false; - } - - // Check for change in frame rate. - if (qm->change_resolution_temporal) { - incoming_frame_rate_ = qm->frame_rate; - // Reset frame rate estimate. - memset(incoming_frame_times_, -1, sizeof(incoming_frame_times_)); - } - - // Check for change in frame size. - if (qm->change_resolution_spatial) { - codec_width_ = qm->codec_width; - codec_height_ = qm->codec_height; - } - - LOG(LS_INFO) << "Media optimizer requests the video resolution to be changed " - "to " << qm->codec_width << "x" << qm->codec_height << "@" - << qm->frame_rate; - - // Update VPM with new target frame rate and frame size. - // Note: use |qm->frame_rate| instead of |_incoming_frame_rate| for updating - // target frame rate in VPM frame dropper. The quantity |_incoming_frame_rate| - // will vary/fluctuate, and since we don't want to change the state of the - // VPM frame dropper, unless a temporal action was selected, we use the - // quantity |qm->frame_rate| for updating. - video_qmsettings_callback->SetVideoQMSettings( - qm->frame_rate, codec_width_, codec_height_); - content_->UpdateFrameRate(qm->frame_rate); - qm_resolution_->UpdateCodecParameters( - qm->frame_rate, codec_width_, codec_height_); - return true; -} - -// Check timing constraints and look for significant change in: -// (1) scene content, -// (2) target bit rate. -bool MediaOptimization::CheckStatusForQMchange() { - bool status = true; - - // Check that we do not call QMSelect too often, and that we waited some time - // (to sample the metrics) from the event last_change_time - // last_change_time is the time where user changed the size/rate/frame rate - // (via SetEncodingData). - int64_t now = clock_->TimeInMilliseconds(); - if ((now - last_qm_update_time_) < kQmMinIntervalMs || - (now - last_change_time_) < kQmMinIntervalMs) { - status = false; - } - - return status; -} - -// Allowing VCM to keep track of incoming frame rate. -void MediaOptimization::ProcessIncomingFrameRate(int64_t now) { - int32_t num = 0; - int32_t nr_of_frames = 0; - for (num = 1; num < (kFrameCountHistorySize - 1); ++num) { - if (incoming_frame_times_[num] <= 0 || - // don't use data older than 2 s - now - incoming_frame_times_[num] > kFrameHistoryWinMs) { - break; - } else { - nr_of_frames++; - } - } - if (num > 1) { - const int64_t diff = - incoming_frame_times_[0] - incoming_frame_times_[num - 1]; - incoming_frame_rate_ = 0.0; // No frame rate estimate available. - if (diff > 0) { - incoming_frame_rate_ = nr_of_frames * 1000.0f / static_cast<float>(diff); - } - } -} - -void MediaOptimization::CheckSuspendConditions() { - // Check conditions for SuspendBelowMinBitrate. |video_target_bitrate_| is in - // bps. - if (suspension_enabled_) { - if (!video_suspended_) { - // Check if we just went below the threshold. - if (video_target_bitrate_ < suspension_threshold_bps_) { - video_suspended_ = true; - } - } else { - // Video is already suspended. Check if we just went over the threshold - // with a margin. - if (video_target_bitrate_ > - suspension_threshold_bps_ + suspension_window_bps_) { - video_suspended_ = false; - } - } - } -} - -} // namespace media_optimization -} // namespace webrtc |