diff options
Diffstat (limited to 'call/adaptation/video_stream_adapter.cc')
| -rw-r--r-- | call/adaptation/video_stream_adapter.cc | 874 |
1 files changed, 512 insertions, 362 deletions
diff --git a/call/adaptation/video_stream_adapter.cc b/call/adaptation/video_stream_adapter.cc index b224e3e4d2..ec80a13a08 100644 --- a/call/adaptation/video_stream_adapter.cc +++ b/call/adaptation/video_stream_adapter.cc @@ -15,11 +15,17 @@ #include <utility> #include "absl/types/optional.h" +#include "absl/types/variant.h" +#include "api/video/video_adaptation_counters.h" #include "api/video/video_adaptation_reason.h" #include "api/video_codecs/video_encoder.h" +#include "call/adaptation/video_source_restrictions.h" +#include "call/adaptation/video_stream_input_state.h" +#include "rtc_base/checks.h" #include "rtc_base/constructor_magic.h" #include "rtc_base/logging.h" #include "rtc_base/numerics/safe_conversions.h" +#include "rtc_base/synchronization/sequence_checker.h" namespace webrtc { @@ -27,13 +33,6 @@ const int kMinFrameRateFps = 2; namespace { -// Generate suggested higher and lower frame rates and resolutions, to be -// applied to the VideoSourceRestrictor. These are used in "maintain-resolution" -// and "maintain-framerate". The "balanced" degradation preference also makes -// use of BalancedDegradationPreference when generating suggestions. The -// VideoSourceRestrictor decidedes whether or not a proposed adaptation is -// valid. - // For frame rate, the steps we take are 2/3 (down) and 3/2 (up). int GetLowerFrameRateThan(int fps) { RTC_DCHECK(fps != std::numeric_limits<int>::max()); @@ -59,8 +58,60 @@ int GetLowerResolutionThan(int pixel_count) { return (pixel_count * 3) / 5; } +int GetIncreasedMaxPixelsWanted(int target_pixels) { + if (target_pixels == std::numeric_limits<int>::max()) + return std::numeric_limits<int>::max(); + // When we decrease resolution, we go down to at most 3/5 of current pixels. + // Thus to increase resolution, we need 3/5 to get back to where we started. + // When going up, the desired max_pixels_per_frame() has to be significantly + // higher than the target because the source's native resolutions might not + // match the target. We pick 12/5 of the target. + // + // (This value was historically 4 times the old target, which is (3/5)*4 of + // the new target - or 12/5 - assuming the target is adjusted according to + // the above steps.) + RTC_DCHECK(target_pixels != std::numeric_limits<int>::max()); + return (target_pixels * 12) / 5; +} + +bool CanDecreaseResolutionTo(int target_pixels, + const VideoStreamInputState& input_state, + const VideoSourceRestrictions& restrictions) { + int max_pixels_per_frame = + rtc::dchecked_cast<int>(restrictions.max_pixels_per_frame().value_or( + std::numeric_limits<int>::max())); + return target_pixels < max_pixels_per_frame && + target_pixels >= input_state.min_pixels_per_frame(); +} + +bool CanIncreaseResolutionTo(int target_pixels, + const VideoSourceRestrictions& restrictions) { + int max_pixels_wanted = GetIncreasedMaxPixelsWanted(target_pixels); + int max_pixels_per_frame = + rtc::dchecked_cast<int>(restrictions.max_pixels_per_frame().value_or( + std::numeric_limits<int>::max())); + return max_pixels_wanted > max_pixels_per_frame; +} + +bool CanDecreaseFrameRateTo(int max_frame_rate, + const VideoSourceRestrictions& restrictions) { + const int fps_wanted = std::max(kMinFrameRateFps, max_frame_rate); + return fps_wanted < + rtc::dchecked_cast<int>(restrictions.max_frame_rate().value_or( + std::numeric_limits<int>::max())); +} + +bool CanIncreaseFrameRateTo(int max_frame_rate, + const VideoSourceRestrictions& restrictions) { + return max_frame_rate > + rtc::dchecked_cast<int>(restrictions.max_frame_rate().value_or( + std::numeric_limits<int>::max())); +} + } // namespace +VideoSourceRestrictionsListener::~VideoSourceRestrictionsListener() = default; + VideoSourceRestrictions FilterRestrictionsByDegradationPreference( VideoSourceRestrictions source_restrictions, DegradationPreference degradation_preference) { @@ -82,28 +133,6 @@ VideoSourceRestrictions FilterRestrictionsByDegradationPreference( return source_restrictions; } -VideoAdaptationCounters FilterVideoAdaptationCountersByDegradationPreference( - VideoAdaptationCounters counters, - DegradationPreference degradation_preference) { - switch (degradation_preference) { - case DegradationPreference::BALANCED: - break; - case DegradationPreference::MAINTAIN_FRAMERATE: - counters.fps_adaptations = 0; - break; - case DegradationPreference::MAINTAIN_RESOLUTION: - counters.resolution_adaptations = 0; - break; - case DegradationPreference::DISABLED: - counters.resolution_adaptations = 0; - counters.fps_adaptations = 0; - break; - default: - RTC_NOTREACHED(); - } - return counters; -} - // TODO(hbos): Use absl::optional<> instead? int GetHigherResolutionThan(int pixel_count) { return pixel_count != std::numeric_limits<int>::max() @@ -111,38 +140,44 @@ int GetHigherResolutionThan(int pixel_count) { : std::numeric_limits<int>::max(); } -Adaptation::Step::Step(StepType type, int target) - : type(type), target(target) {} - -Adaptation::Adaptation(int validation_id, Step step) - : validation_id_(validation_id), - status_(Status::kValid), - step_(std::move(step)), - min_pixel_limit_reached_(false) {} +// static +const char* Adaptation::StatusToString(Adaptation::Status status) { + switch (status) { + case Adaptation::Status::kValid: + return "kValid"; + case Adaptation::Status::kLimitReached: + return "kLimitReached"; + case Adaptation::Status::kAwaitingPreviousAdaptation: + return "kAwaitingPreviousAdaptation"; + case Status::kInsufficientInput: + return "kInsufficientInput"; + case Status::kAdaptationDisabled: + return "kAdaptationDisabled"; + case Status::kRejectedByConstraint: + return "kRejectedByConstraint"; + } +} Adaptation::Adaptation(int validation_id, - Step step, + VideoSourceRestrictions restrictions, + VideoAdaptationCounters counters, + VideoStreamInputState input_state, bool min_pixel_limit_reached) : validation_id_(validation_id), status_(Status::kValid), - step_(std::move(step)), - min_pixel_limit_reached_(min_pixel_limit_reached) {} - -Adaptation::Adaptation(int validation_id, Status invalid_status) - : validation_id_(validation_id), - status_(invalid_status), - step_(absl::nullopt), - min_pixel_limit_reached_(false) { - RTC_DCHECK_NE(status_, Status::kValid); -} + min_pixel_limit_reached_(min_pixel_limit_reached), + input_state_(std::move(input_state)), + restrictions_(std::move(restrictions)), + counters_(std::move(counters)) {} Adaptation::Adaptation(int validation_id, Status invalid_status, + VideoStreamInputState input_state, bool min_pixel_limit_reached) : validation_id_(validation_id), status_(invalid_status), - step_(absl::nullopt), - min_pixel_limit_reached_(min_pixel_limit_reached) { + min_pixel_limit_reached_(min_pixel_limit_reached), + input_state_(std::move(input_state)) { RTC_DCHECK_NE(status_, Status::kValid); } @@ -154,398 +189,513 @@ bool Adaptation::min_pixel_limit_reached() const { return min_pixel_limit_reached_; } -const Adaptation::Step& Adaptation::step() const { - RTC_DCHECK_EQ(status_, Status::kValid); - return step_.value(); +const VideoStreamInputState& Adaptation::input_state() const { + return input_state_; } -// VideoSourceRestrictor is responsible for keeping track of current -// VideoSourceRestrictions. -class VideoStreamAdapter::VideoSourceRestrictor { - public: - VideoSourceRestrictor() {} - - VideoSourceRestrictions source_restrictions() const { - return source_restrictions_; - } - const VideoAdaptationCounters& adaptation_counters() const { - return adaptations_; - } - void ClearRestrictions() { - source_restrictions_ = VideoSourceRestrictions(); - adaptations_ = VideoAdaptationCounters(); - } - - void set_min_pixels_per_frame(int min_pixels_per_frame) { - min_pixels_per_frame_ = min_pixels_per_frame; - } - - int min_pixels_per_frame() const { return min_pixels_per_frame_; } - - bool CanDecreaseResolutionTo(int target_pixels) { - int max_pixels_per_frame = rtc::dchecked_cast<int>( - source_restrictions_.max_pixels_per_frame().value_or( - std::numeric_limits<int>::max())); - return target_pixels < max_pixels_per_frame && - target_pixels >= min_pixels_per_frame_; - } - - bool CanIncreaseResolutionTo(int target_pixels) { - int max_pixels_wanted = GetIncreasedMaxPixelsWanted(target_pixels); - int max_pixels_per_frame = rtc::dchecked_cast<int>( - source_restrictions_.max_pixels_per_frame().value_or( - std::numeric_limits<int>::max())); - return max_pixels_wanted > max_pixels_per_frame; - } - - bool CanDecreaseFrameRateTo(int max_frame_rate) { - const int fps_wanted = std::max(kMinFrameRateFps, max_frame_rate); - return fps_wanted < rtc::dchecked_cast<int>( - source_restrictions_.max_frame_rate().value_or( - std::numeric_limits<int>::max())); - } - - bool CanIncreaseFrameRateTo(int max_frame_rate) { - return max_frame_rate > rtc::dchecked_cast<int>( - source_restrictions_.max_frame_rate().value_or( - std::numeric_limits<int>::max())); - } - - void ApplyAdaptationStep(const Adaptation::Step& step, - DegradationPreference degradation_preference) { - switch (step.type) { - case Adaptation::StepType::kIncreaseResolution: - IncreaseResolutionTo(step.target); - break; - case Adaptation::StepType::kDecreaseResolution: - DecreaseResolutionTo(step.target); - break; - case Adaptation::StepType::kIncreaseFrameRate: - IncreaseFrameRateTo(step.target); - // TODO(https://crbug.com/webrtc/11222): Don't adapt in two steps. - // GetAdaptationUp() should tell us the correct value, but BALANCED - // logic in DecrementFramerate() makes it hard to predict whether this - // will be the last step. Remove the dependency on - // adaptation_counters(). - if (degradation_preference == DegradationPreference::BALANCED && - adaptation_counters().fps_adaptations == 0 && - step.target != std::numeric_limits<int>::max()) { - RTC_LOG(LS_INFO) << "Removing framerate down-scaling setting."; - IncreaseFrameRateTo(std::numeric_limits<int>::max()); - } - break; - case Adaptation::StepType::kDecreaseFrameRate: - DecreaseFrameRateTo(step.target); - break; - } - } - - private: - static int GetIncreasedMaxPixelsWanted(int target_pixels) { - if (target_pixels == std::numeric_limits<int>::max()) - return std::numeric_limits<int>::max(); - // When we decrease resolution, we go down to at most 3/5 of current pixels. - // Thus to increase resolution, we need 3/5 to get back to where we started. - // When going up, the desired max_pixels_per_frame() has to be significantly - // higher than the target because the source's native resolutions might not - // match the target. We pick 12/5 of the target. - // - // (This value was historically 4 times the old target, which is (3/5)*4 of - // the new target - or 12/5 - assuming the target is adjusted according to - // the above steps.) - RTC_DCHECK(target_pixels != std::numeric_limits<int>::max()); - return (target_pixels * 12) / 5; - } - - void DecreaseResolutionTo(int target_pixels) { - RTC_DCHECK(CanDecreaseResolutionTo(target_pixels)); - RTC_LOG(LS_INFO) << "Scaling down resolution, max pixels: " - << target_pixels; - source_restrictions_.set_max_pixels_per_frame( - target_pixels != std::numeric_limits<int>::max() - ? absl::optional<size_t>(target_pixels) - : absl::nullopt); - source_restrictions_.set_target_pixels_per_frame(absl::nullopt); - ++adaptations_.resolution_adaptations; - } - - void IncreaseResolutionTo(int target_pixels) { - RTC_DCHECK(CanIncreaseResolutionTo(target_pixels)); - int max_pixels_wanted = GetIncreasedMaxPixelsWanted(target_pixels); - RTC_LOG(LS_INFO) << "Scaling up resolution, max pixels: " - << max_pixels_wanted; - source_restrictions_.set_max_pixels_per_frame( - max_pixels_wanted != std::numeric_limits<int>::max() - ? absl::optional<size_t>(max_pixels_wanted) - : absl::nullopt); - source_restrictions_.set_target_pixels_per_frame( - max_pixels_wanted != std::numeric_limits<int>::max() - ? absl::optional<size_t>(target_pixels) - : absl::nullopt); - --adaptations_.resolution_adaptations; - RTC_DCHECK_GE(adaptations_.resolution_adaptations, 0); - } - - void DecreaseFrameRateTo(int max_frame_rate) { - RTC_DCHECK(CanDecreaseFrameRateTo(max_frame_rate)); - max_frame_rate = std::max(kMinFrameRateFps, max_frame_rate); - RTC_LOG(LS_INFO) << "Scaling down framerate: " << max_frame_rate; - source_restrictions_.set_max_frame_rate( - max_frame_rate != std::numeric_limits<int>::max() - ? absl::optional<double>(max_frame_rate) - : absl::nullopt); - ++adaptations_.fps_adaptations; - } - - void IncreaseFrameRateTo(int max_frame_rate) { - RTC_DCHECK(CanIncreaseFrameRateTo(max_frame_rate)); - RTC_LOG(LS_INFO) << "Scaling up framerate: " << max_frame_rate; - source_restrictions_.set_max_frame_rate( - max_frame_rate != std::numeric_limits<int>::max() - ? absl::optional<double>(max_frame_rate) - : absl::nullopt); - --adaptations_.fps_adaptations; - RTC_DCHECK_GE(adaptations_.fps_adaptations, 0); - } - - // Needed by CanDecreaseResolutionTo(). - int min_pixels_per_frame_ = 0; - // Current State. - VideoSourceRestrictions source_restrictions_; - VideoAdaptationCounters adaptations_; -}; +const VideoSourceRestrictions& Adaptation::restrictions() const { + return restrictions_; +} -// static -VideoStreamAdapter::AdaptationRequest::Mode -VideoStreamAdapter::AdaptationRequest::GetModeFromAdaptationAction( - Adaptation::StepType step_type) { - switch (step_type) { - case Adaptation::StepType::kIncreaseResolution: - return AdaptationRequest::Mode::kAdaptUp; - case Adaptation::StepType::kDecreaseResolution: - return AdaptationRequest::Mode::kAdaptDown; - case Adaptation::StepType::kIncreaseFrameRate: - return AdaptationRequest::Mode::kAdaptUp; - case Adaptation::StepType::kDecreaseFrameRate: - return AdaptationRequest::Mode::kAdaptDown; - } +const VideoAdaptationCounters& Adaptation::counters() const { + return counters_; } -VideoStreamAdapter::VideoStreamAdapter() - : source_restrictor_(std::make_unique<VideoSourceRestrictor>()), +VideoStreamAdapter::VideoStreamAdapter( + VideoStreamInputStateProvider* input_state_provider) + : input_state_provider_(input_state_provider), balanced_settings_(), adaptation_validation_id_(0), degradation_preference_(DegradationPreference::DISABLED), - input_state_(), - last_adaptation_request_(absl::nullopt) {} + awaiting_frame_size_change_(absl::nullopt), + last_video_source_restrictions_() { + sequence_checker_.Detach(); +} -VideoStreamAdapter::~VideoStreamAdapter() {} +VideoStreamAdapter::~VideoStreamAdapter() { + RTC_DCHECK(adaptation_listeners_.empty()) + << "There are listener(s) attached to a VideoStreamAdapter being " + "destroyed."; + RTC_DCHECK(adaptation_constraints_.empty()) + << "There are constaint(s) attached to a VideoStreamAdapter being " + "destroyed."; +} VideoSourceRestrictions VideoStreamAdapter::source_restrictions() const { - return source_restrictor_->source_restrictions(); + RTC_DCHECK_RUN_ON(&sequence_checker_); + return current_restrictions_.restrictions; } const VideoAdaptationCounters& VideoStreamAdapter::adaptation_counters() const { - return source_restrictor_->adaptation_counters(); -} - -const BalancedDegradationSettings& VideoStreamAdapter::balanced_settings() - const { - return balanced_settings_; + RTC_DCHECK_RUN_ON(&sequence_checker_); + return current_restrictions_.counters; } void VideoStreamAdapter::ClearRestrictions() { + RTC_DCHECK_RUN_ON(&sequence_checker_); // Invalidate any previously returned Adaptation. + RTC_LOG(INFO) << "Resetting restrictions"; ++adaptation_validation_id_; - source_restrictor_->ClearRestrictions(); - last_adaptation_request_.reset(); + current_restrictions_ = {VideoSourceRestrictions(), + VideoAdaptationCounters()}; + awaiting_frame_size_change_ = absl::nullopt; + BroadcastVideoRestrictionsUpdate(input_state_provider_->InputState(), + nullptr); +} + +void VideoStreamAdapter::AddRestrictionsListener( + VideoSourceRestrictionsListener* restrictions_listener) { + RTC_DCHECK_RUN_ON(&sequence_checker_); + RTC_DCHECK(std::find(restrictions_listeners_.begin(), + restrictions_listeners_.end(), + restrictions_listener) == restrictions_listeners_.end()); + restrictions_listeners_.push_back(restrictions_listener); +} + +void VideoStreamAdapter::RemoveRestrictionsListener( + VideoSourceRestrictionsListener* restrictions_listener) { + RTC_DCHECK_RUN_ON(&sequence_checker_); + auto it = std::find(restrictions_listeners_.begin(), + restrictions_listeners_.end(), restrictions_listener); + RTC_DCHECK(it != restrictions_listeners_.end()); + restrictions_listeners_.erase(it); +} + +void VideoStreamAdapter::AddAdaptationListener( + AdaptationListener* adaptation_listener) { + RTC_DCHECK_RUN_ON(&sequence_checker_); + RTC_DCHECK(std::find(adaptation_listeners_.begin(), + adaptation_listeners_.end(), + adaptation_listener) == adaptation_listeners_.end()); + adaptation_listeners_.push_back(adaptation_listener); +} + +void VideoStreamAdapter::RemoveAdaptationListener( + AdaptationListener* adaptation_listener) { + RTC_DCHECK_RUN_ON(&sequence_checker_); + auto it = std::find(adaptation_listeners_.begin(), + adaptation_listeners_.end(), adaptation_listener); + RTC_DCHECK(it != adaptation_listeners_.end()); + adaptation_listeners_.erase(it); +} + +void VideoStreamAdapter::AddAdaptationConstraint( + AdaptationConstraint* adaptation_constraint) { + RTC_DCHECK_RUN_ON(&sequence_checker_); + RTC_DCHECK(std::find(adaptation_constraints_.begin(), + adaptation_constraints_.end(), + adaptation_constraint) == adaptation_constraints_.end()); + adaptation_constraints_.push_back(adaptation_constraint); +} + +void VideoStreamAdapter::RemoveAdaptationConstraint( + AdaptationConstraint* adaptation_constraint) { + RTC_DCHECK_RUN_ON(&sequence_checker_); + auto it = std::find(adaptation_constraints_.begin(), + adaptation_constraints_.end(), adaptation_constraint); + RTC_DCHECK(it != adaptation_constraints_.end()); + adaptation_constraints_.erase(it); } void VideoStreamAdapter::SetDegradationPreference( DegradationPreference degradation_preference) { + RTC_DCHECK_RUN_ON(&sequence_checker_); if (degradation_preference_ == degradation_preference) return; // Invalidate any previously returned Adaptation. ++adaptation_validation_id_; - if (degradation_preference == DegradationPreference::BALANCED || - degradation_preference_ == DegradationPreference::BALANCED) { + bool balanced_switch = + degradation_preference == DegradationPreference::BALANCED || + degradation_preference_ == DegradationPreference::BALANCED; + degradation_preference_ = degradation_preference; + if (balanced_switch) { + // ClearRestrictions() calls BroadcastVideoRestrictionsUpdate(nullptr). ClearRestrictions(); + } else { + BroadcastVideoRestrictionsUpdate(input_state_provider_->InputState(), + nullptr); } - degradation_preference_ = degradation_preference; } -void VideoStreamAdapter::SetInput(VideoStreamInputState input_state) { - // Invalidate any previously returned Adaptation. +struct VideoStreamAdapter::RestrictionsOrStateVisitor { + Adaptation operator()(const RestrictionsWithCounters& r) const { + return Adaptation(adaptation_validation_id, r.restrictions, r.counters, + input_state, min_pixel_limit_reached()); + } + Adaptation operator()(const Adaptation::Status& status) const { + RTC_DCHECK_NE(status, Adaptation::Status::kValid); + return Adaptation(adaptation_validation_id, status, input_state, + min_pixel_limit_reached()); + } + bool min_pixel_limit_reached() const { + return input_state.frame_size_pixels().has_value() && + GetLowerResolutionThan(input_state.frame_size_pixels().value()) < + input_state.min_pixels_per_frame(); + } + + const int adaptation_validation_id; + const VideoStreamInputState& input_state; +}; + +Adaptation VideoStreamAdapter::RestrictionsOrStateToAdaptation( + VideoStreamAdapter::RestrictionsOrState step_or_state, + const VideoStreamInputState& input_state) const { + RTC_DCHECK(!step_or_state.valueless_by_exception()); + return absl::visit( + RestrictionsOrStateVisitor{adaptation_validation_id_, input_state}, + step_or_state); +} + +Adaptation VideoStreamAdapter::GetAdaptationUp( + const VideoStreamInputState& input_state, + rtc::scoped_refptr<Resource> resource) const { + RestrictionsOrState step = GetAdaptationUpStep(input_state); + // If an adaptation proposed, check with the constraints that it is ok. + if (absl::holds_alternative<RestrictionsWithCounters>(step)) { + RestrictionsWithCounters restrictions = + absl::get<RestrictionsWithCounters>(step); + for (const auto* constraint : adaptation_constraints_) { + if (!constraint->IsAdaptationUpAllowed( + input_state, current_restrictions_.restrictions, + restrictions.restrictions, resource)) { + RTC_LOG(INFO) << "Not adapting up because constraint \"" + << constraint->Name() << "\" disallowed it"; + step = Adaptation::Status::kRejectedByConstraint; + } + } + } + return RestrictionsOrStateToAdaptation(step, input_state); +} + +Adaptation VideoStreamAdapter::GetAdaptationUp( + rtc::scoped_refptr<Resource> resource) { + RTC_DCHECK_RUN_ON(&sequence_checker_); + RTC_DCHECK(resource); + VideoStreamInputState input_state = input_state_provider_->InputState(); ++adaptation_validation_id_; - input_state_ = input_state; - source_restrictor_->set_min_pixels_per_frame( - input_state_.min_pixels_per_frame()); + Adaptation adaptation = GetAdaptationUp(input_state, resource); + return adaptation; } -Adaptation VideoStreamAdapter::GetAdaptationUp() const { - RTC_DCHECK_NE(degradation_preference_, DegradationPreference::DISABLED); - RTC_DCHECK(input_state_.HasInputFrameSizeAndFramesPerSecond()); +VideoStreamAdapter::RestrictionsOrState VideoStreamAdapter::GetAdaptationUpStep( + const VideoStreamInputState& input_state) const { + if (!HasSufficientInputForAdaptation(input_state)) { + return Adaptation::Status::kInsufficientInput; + } // Don't adapt if we're awaiting a previous adaptation to have an effect. - bool last_adaptation_was_up = - last_adaptation_request_ && - last_adaptation_request_->mode_ == AdaptationRequest::Mode::kAdaptUp; - if (last_adaptation_was_up && + if (awaiting_frame_size_change_ && + awaiting_frame_size_change_->pixels_increased && degradation_preference_ == DegradationPreference::MAINTAIN_FRAMERATE && - input_state_.frame_size_pixels().value() <= - last_adaptation_request_->input_pixel_count_) { - return Adaptation(adaptation_validation_id_, - Adaptation::Status::kAwaitingPreviousAdaptation); + input_state.frame_size_pixels().value() <= + awaiting_frame_size_change_->frame_size_pixels) { + return Adaptation::Status::kAwaitingPreviousAdaptation; } // Maybe propose targets based on degradation preference. switch (degradation_preference_) { case DegradationPreference::BALANCED: { // Attempt to increase target frame rate. - int target_fps = - balanced_settings_.MaxFps(input_state_.video_codec_type(), - input_state_.frame_size_pixels().value()); - if (source_restrictor_->CanIncreaseFrameRateTo(target_fps)) { - return Adaptation( - adaptation_validation_id_, - Adaptation::Step(Adaptation::StepType::kIncreaseFrameRate, - target_fps)); + RestrictionsOrState increase_frame_rate = + IncreaseFramerate(input_state, current_restrictions_); + if (absl::holds_alternative<RestrictionsWithCounters>( + increase_frame_rate)) { + return increase_frame_rate; } - // Scale up resolution. + // else, increase resolution. ABSL_FALLTHROUGH_INTENDED; } case DegradationPreference::MAINTAIN_FRAMERATE: { // Attempt to increase pixel count. - int target_pixels = input_state_.frame_size_pixels().value(); - if (source_restrictor_->adaptation_counters().resolution_adaptations == - 1) { - RTC_LOG(LS_INFO) << "Removing resolution down-scaling setting."; - target_pixels = std::numeric_limits<int>::max(); - } - target_pixels = GetHigherResolutionThan(target_pixels); - if (!source_restrictor_->CanIncreaseResolutionTo(target_pixels)) { - return Adaptation(adaptation_validation_id_, - Adaptation::Status::kLimitReached); - } - return Adaptation( - adaptation_validation_id_, - Adaptation::Step(Adaptation::StepType::kIncreaseResolution, - target_pixels)); + return IncreaseResolution(input_state, current_restrictions_); } case DegradationPreference::MAINTAIN_RESOLUTION: { // Scale up framerate. - int target_fps = input_state_.frames_per_second(); - if (source_restrictor_->adaptation_counters().fps_adaptations == 1) { - RTC_LOG(LS_INFO) << "Removing framerate down-scaling setting."; - target_fps = std::numeric_limits<int>::max(); - } - target_fps = GetHigherFrameRateThan(target_fps); - if (!source_restrictor_->CanIncreaseFrameRateTo(target_fps)) { - return Adaptation(adaptation_validation_id_, - Adaptation::Status::kLimitReached); - } - return Adaptation( - adaptation_validation_id_, - Adaptation::Step(Adaptation::StepType::kIncreaseFrameRate, - target_fps)); + return IncreaseFramerate(input_state, current_restrictions_); } case DegradationPreference::DISABLED: - RTC_NOTREACHED(); - return Adaptation(adaptation_validation_id_, - Adaptation::Status::kLimitReached); + return Adaptation::Status::kAdaptationDisabled; } } -Adaptation VideoStreamAdapter::GetAdaptationDown() const { - RTC_DCHECK_NE(degradation_preference_, DegradationPreference::DISABLED); - RTC_DCHECK(input_state_.HasInputFrameSizeAndFramesPerSecond()); - // Don't adapt adaptation is disabled. - bool last_adaptation_was_down = - last_adaptation_request_ && - last_adaptation_request_->mode_ == AdaptationRequest::Mode::kAdaptDown; - // Don't adapt if we're awaiting a previous adaptation to have an effect. - if (last_adaptation_was_down && +Adaptation VideoStreamAdapter::GetAdaptationDown() { + RTC_DCHECK_RUN_ON(&sequence_checker_); + VideoStreamInputState input_state = input_state_provider_->InputState(); + ++adaptation_validation_id_; + return RestrictionsOrStateToAdaptation(GetAdaptationDownStep(input_state), + input_state); +} + +VideoStreamAdapter::RestrictionsOrState +VideoStreamAdapter::GetAdaptationDownStep( + const VideoStreamInputState& input_state) const { + if (!HasSufficientInputForAdaptation(input_state)) { + return Adaptation::Status::kInsufficientInput; + } + // Don't adapt if we're awaiting a previous adaptation to have an effect or + // if we switched degradation preference. + if (awaiting_frame_size_change_ && + !awaiting_frame_size_change_->pixels_increased && degradation_preference_ == DegradationPreference::MAINTAIN_FRAMERATE && - input_state_.frame_size_pixels().value() >= - last_adaptation_request_->input_pixel_count_) { - return Adaptation(adaptation_validation_id_, - Adaptation::Status::kAwaitingPreviousAdaptation); + input_state.frame_size_pixels().value() >= + awaiting_frame_size_change_->frame_size_pixels) { + return Adaptation::Status::kAwaitingPreviousAdaptation; } - // Maybe propose targets based on degradation preference. switch (degradation_preference_) { case DegradationPreference::BALANCED: { // Try scale down framerate, if lower. - int target_fps = - balanced_settings_.MinFps(input_state_.video_codec_type(), - input_state_.frame_size_pixels().value()); - if (source_restrictor_->CanDecreaseFrameRateTo(target_fps)) { - return Adaptation( - adaptation_validation_id_, - Adaptation::Step(Adaptation::StepType::kDecreaseFrameRate, - target_fps)); + RestrictionsOrState decrease_frame_rate = + DecreaseFramerate(input_state, current_restrictions_); + if (absl::holds_alternative<RestrictionsWithCounters>( + decrease_frame_rate)) { + return decrease_frame_rate; } - // Scale down resolution. + // else, decrease resolution. ABSL_FALLTHROUGH_INTENDED; } case DegradationPreference::MAINTAIN_FRAMERATE: { - // Scale down resolution. - int target_pixels = - GetLowerResolutionThan(input_state_.frame_size_pixels().value()); - bool min_pixel_limit_reached = - target_pixels < source_restrictor_->min_pixels_per_frame(); - if (!source_restrictor_->CanDecreaseResolutionTo(target_pixels)) { - return Adaptation(adaptation_validation_id_, - Adaptation::Status::kLimitReached, - min_pixel_limit_reached); - } - return Adaptation( - adaptation_validation_id_, - Adaptation::Step(Adaptation::StepType::kDecreaseResolution, - target_pixels), - min_pixel_limit_reached); + return DecreaseResolution(input_state, current_restrictions_); } case DegradationPreference::MAINTAIN_RESOLUTION: { - int target_fps = GetLowerFrameRateThan(input_state_.frames_per_second()); - if (!source_restrictor_->CanDecreaseFrameRateTo(target_fps)) { - return Adaptation(adaptation_validation_id_, - Adaptation::Status::kLimitReached); - } - return Adaptation( - adaptation_validation_id_, - Adaptation::Step(Adaptation::StepType::kDecreaseFrameRate, - target_fps)); + return DecreaseFramerate(input_state, current_restrictions_); + } + case DegradationPreference::DISABLED: + return Adaptation::Status::kAdaptationDisabled; + } +} + +VideoStreamAdapter::RestrictionsOrState VideoStreamAdapter::DecreaseResolution( + const VideoStreamInputState& input_state, + const RestrictionsWithCounters& current_restrictions) { + int target_pixels = + GetLowerResolutionThan(input_state.frame_size_pixels().value()); + if (!CanDecreaseResolutionTo(target_pixels, input_state, + current_restrictions.restrictions)) { + return Adaptation::Status::kLimitReached; + } + RestrictionsWithCounters new_restrictions = current_restrictions; + RTC_LOG(LS_INFO) << "Scaling down resolution, max pixels: " << target_pixels; + new_restrictions.restrictions.set_max_pixels_per_frame( + target_pixels != std::numeric_limits<int>::max() + ? absl::optional<size_t>(target_pixels) + : absl::nullopt); + new_restrictions.restrictions.set_target_pixels_per_frame(absl::nullopt); + ++new_restrictions.counters.resolution_adaptations; + return new_restrictions; +} + +VideoStreamAdapter::RestrictionsOrState VideoStreamAdapter::DecreaseFramerate( + const VideoStreamInputState& input_state, + const RestrictionsWithCounters& current_restrictions) const { + int max_frame_rate; + if (degradation_preference_ == DegradationPreference::MAINTAIN_RESOLUTION) { + max_frame_rate = GetLowerFrameRateThan(input_state.frames_per_second()); + } else if (degradation_preference_ == DegradationPreference::BALANCED) { + max_frame_rate = + balanced_settings_.MinFps(input_state.video_codec_type(), + input_state.frame_size_pixels().value()); + } else { + RTC_NOTREACHED(); + max_frame_rate = GetLowerFrameRateThan(input_state.frames_per_second()); + } + if (!CanDecreaseFrameRateTo(max_frame_rate, + current_restrictions.restrictions)) { + return Adaptation::Status::kLimitReached; + } + RestrictionsWithCounters new_restrictions = current_restrictions; + max_frame_rate = std::max(kMinFrameRateFps, max_frame_rate); + RTC_LOG(LS_INFO) << "Scaling down framerate: " << max_frame_rate; + new_restrictions.restrictions.set_max_frame_rate( + max_frame_rate != std::numeric_limits<int>::max() + ? absl::optional<double>(max_frame_rate) + : absl::nullopt); + ++new_restrictions.counters.fps_adaptations; + return new_restrictions; +} + +VideoStreamAdapter::RestrictionsOrState VideoStreamAdapter::IncreaseResolution( + const VideoStreamInputState& input_state, + const RestrictionsWithCounters& current_restrictions) { + int target_pixels = input_state.frame_size_pixels().value(); + if (current_restrictions.counters.resolution_adaptations == 1) { + RTC_LOG(LS_INFO) << "Removing resolution down-scaling setting."; + target_pixels = std::numeric_limits<int>::max(); + } + target_pixels = GetHigherResolutionThan(target_pixels); + if (!CanIncreaseResolutionTo(target_pixels, + current_restrictions.restrictions)) { + return Adaptation::Status::kLimitReached; + } + int max_pixels_wanted = GetIncreasedMaxPixelsWanted(target_pixels); + RestrictionsWithCounters new_restrictions = current_restrictions; + RTC_LOG(LS_INFO) << "Scaling up resolution, max pixels: " + << max_pixels_wanted; + new_restrictions.restrictions.set_max_pixels_per_frame( + max_pixels_wanted != std::numeric_limits<int>::max() + ? absl::optional<size_t>(max_pixels_wanted) + : absl::nullopt); + new_restrictions.restrictions.set_target_pixels_per_frame( + max_pixels_wanted != std::numeric_limits<int>::max() + ? absl::optional<size_t>(target_pixels) + : absl::nullopt); + --new_restrictions.counters.resolution_adaptations; + RTC_DCHECK_GE(new_restrictions.counters.resolution_adaptations, 0); + return new_restrictions; +} + +VideoStreamAdapter::RestrictionsOrState VideoStreamAdapter::IncreaseFramerate( + const VideoStreamInputState& input_state, + const RestrictionsWithCounters& current_restrictions) const { + int max_frame_rate; + if (degradation_preference_ == DegradationPreference::MAINTAIN_RESOLUTION) { + max_frame_rate = GetHigherFrameRateThan(input_state.frames_per_second()); + } else if (degradation_preference_ == DegradationPreference::BALANCED) { + max_frame_rate = + balanced_settings_.MaxFps(input_state.video_codec_type(), + input_state.frame_size_pixels().value()); + // In BALANCED, the max_frame_rate must be checked before proceeding. This + // is because the MaxFps might be the current Fps and so the balanced + // settings may want to scale up the resolution.= + if (!CanIncreaseFrameRateTo(max_frame_rate, + current_restrictions.restrictions)) { + return Adaptation::Status::kLimitReached; } + } else { + RTC_NOTREACHED(); + max_frame_rate = GetHigherFrameRateThan(input_state.frames_per_second()); + } + if (current_restrictions.counters.fps_adaptations == 1) { + RTC_LOG(LS_INFO) << "Removing framerate down-scaling setting."; + max_frame_rate = std::numeric_limits<int>::max(); + } + if (!CanIncreaseFrameRateTo(max_frame_rate, + current_restrictions.restrictions)) { + return Adaptation::Status::kLimitReached; + } + RTC_LOG(LS_INFO) << "Scaling up framerate: " << max_frame_rate; + RestrictionsWithCounters new_restrictions = current_restrictions; + new_restrictions.restrictions.set_max_frame_rate( + max_frame_rate != std::numeric_limits<int>::max() + ? absl::optional<double>(max_frame_rate) + : absl::nullopt); + --new_restrictions.counters.fps_adaptations; + RTC_DCHECK_GE(new_restrictions.counters.fps_adaptations, 0); + return new_restrictions; +} + +Adaptation VideoStreamAdapter::GetAdaptDownResolution() { + RTC_DCHECK_RUN_ON(&sequence_checker_); + VideoStreamInputState input_state = input_state_provider_->InputState(); + switch (degradation_preference_) { case DegradationPreference::DISABLED: + return RestrictionsOrStateToAdaptation( + Adaptation::Status::kAdaptationDisabled, input_state); + case DegradationPreference::MAINTAIN_RESOLUTION: + return RestrictionsOrStateToAdaptation(Adaptation::Status::kLimitReached, + input_state); + case DegradationPreference::MAINTAIN_FRAMERATE: + return GetAdaptationDown(); + case DegradationPreference::BALANCED: { + return RestrictionsOrStateToAdaptation( + GetAdaptDownResolutionStepForBalanced(input_state), input_state); + } + default: RTC_NOTREACHED(); - return Adaptation(adaptation_validation_id_, - Adaptation::Status::kLimitReached); } } -VideoSourceRestrictions VideoStreamAdapter::PeekNextRestrictions( - const Adaptation& adaptation) const { - RTC_DCHECK_EQ(adaptation.validation_id_, adaptation_validation_id_); - if (adaptation.status() != Adaptation::Status::kValid) - return source_restrictor_->source_restrictions(); - VideoSourceRestrictor restrictor_copy = *source_restrictor_; - restrictor_copy.ApplyAdaptationStep(adaptation.step(), - degradation_preference_); - return restrictor_copy.source_restrictions(); +VideoStreamAdapter::RestrictionsOrState +VideoStreamAdapter::GetAdaptDownResolutionStepForBalanced( + const VideoStreamInputState& input_state) const { + // Adapt twice if the first adaptation did not decrease resolution. + auto first_step = GetAdaptationDownStep(input_state); + if (!absl::holds_alternative<RestrictionsWithCounters>(first_step)) { + return first_step; + } + auto first_restrictions = absl::get<RestrictionsWithCounters>(first_step); + if (first_restrictions.counters.resolution_adaptations > + current_restrictions_.counters.resolution_adaptations) { + return first_step; + } + // We didn't decrease resolution so force it; amend a resolution resuction + // to the existing framerate reduction in |first_restrictions|. + auto second_step = DecreaseResolution(input_state, first_restrictions); + if (absl::holds_alternative<RestrictionsWithCounters>(second_step)) { + return second_step; + } + // If the second step was not successful then settle for the first one. + return first_step; } -void VideoStreamAdapter::ApplyAdaptation(const Adaptation& adaptation) { +void VideoStreamAdapter::ApplyAdaptation( + const Adaptation& adaptation, + rtc::scoped_refptr<Resource> resource) { + RTC_DCHECK_RUN_ON(&sequence_checker_); RTC_DCHECK_EQ(adaptation.validation_id_, adaptation_validation_id_); if (adaptation.status() != Adaptation::Status::kValid) return; // Remember the input pixels and fps of this adaptation. Used to avoid // adapting again before this adaptation has had an effect. - last_adaptation_request_.emplace(AdaptationRequest{ - input_state_.frame_size_pixels().value(), - input_state_.frames_per_second(), - AdaptationRequest::GetModeFromAdaptationAction(adaptation.step().type)}); - // Adapt! - source_restrictor_->ApplyAdaptationStep(adaptation.step(), - degradation_preference_); + if (DidIncreaseResolution(current_restrictions_.restrictions, + adaptation.restrictions())) { + awaiting_frame_size_change_.emplace( + true, adaptation.input_state().frame_size_pixels().value()); + } else if (DidDecreaseResolution(current_restrictions_.restrictions, + adaptation.restrictions())) { + awaiting_frame_size_change_.emplace( + false, adaptation.input_state().frame_size_pixels().value()); + } else { + awaiting_frame_size_change_ = absl::nullopt; + } + current_restrictions_ = {adaptation.restrictions(), adaptation.counters()}; + BroadcastVideoRestrictionsUpdate(adaptation.input_state(), resource); +} + +Adaptation VideoStreamAdapter::GetAdaptationTo( + const VideoAdaptationCounters& counters, + const VideoSourceRestrictions& restrictions) { + // Adapts up/down from the current levels so counters are equal. + RTC_DCHECK_RUN_ON(&sequence_checker_); + VideoStreamInputState input_state = input_state_provider_->InputState(); + return Adaptation(adaptation_validation_id_, restrictions, counters, + input_state, false); +} + +void VideoStreamAdapter::BroadcastVideoRestrictionsUpdate( + const VideoStreamInputState& input_state, + const rtc::scoped_refptr<Resource>& resource) { + RTC_DCHECK_RUN_ON(&sequence_checker_); + VideoSourceRestrictions filtered = FilterRestrictionsByDegradationPreference( + source_restrictions(), degradation_preference_); + if (last_filtered_restrictions_ == filtered) { + return; + } + for (auto* restrictions_listener : restrictions_listeners_) { + restrictions_listener->OnVideoSourceRestrictionsUpdated( + filtered, current_restrictions_.counters, resource, + source_restrictions()); + } + for (auto* adaptation_listener : adaptation_listeners_) { + adaptation_listener->OnAdaptationApplied( + input_state, last_video_source_restrictions_, + current_restrictions_.restrictions, resource); + } + last_video_source_restrictions_ = current_restrictions_.restrictions; + last_filtered_restrictions_ = filtered; } +bool VideoStreamAdapter::HasSufficientInputForAdaptation( + const VideoStreamInputState& input_state) const { + return input_state.HasInputFrameSizeAndFramesPerSecond() && + (degradation_preference_ != + DegradationPreference::MAINTAIN_RESOLUTION || + input_state.frames_per_second() >= kMinFrameRateFps); +} + +VideoStreamAdapter::AwaitingFrameSizeChange::AwaitingFrameSizeChange( + bool pixels_increased, + int frame_size_pixels) + : pixels_increased(pixels_increased), + frame_size_pixels(frame_size_pixels) {} + } // namespace webrtc |