From b2074333b14325c183a3bc66cb7e1af881e67663 Mon Sep 17 00:00:00 2001 From: Abraham Corea Diaz Date: Fri, 16 Jul 2021 20:22:50 +0000 Subject: Enable VP9 codec in standalone sender and receiver This patch adds support for the VP9 video codec by modifying the former streaming VP8 encoder to a general streaming VPX encoder that handles both VP8 and VP9. It also adds a command line argument to the sender to allow choosing which codec to use. Change-Id: Ia254df071c37dddcbe8ea136aaaee47bc8db80a2 Reviewed-on: https://chromium-review.googlesource.com/c/openscreen/+/3028642 Reviewed-by: Jordan Bayles Reviewed-by: mark a. foltz Commit-Queue: mark a. foltz --- cast/standalone_receiver/mirroring_application.cc | 2 + cast/standalone_sender/BUILD.gn | 4 +- cast/standalone_sender/connection_settings.h | 4 + cast/standalone_sender/looping_file_cast_agent.cc | 13 +- cast/standalone_sender/looping_file_sender.cc | 7 +- cast/standalone_sender/looping_file_sender.h | 4 +- cast/standalone_sender/main.cc | 24 +- cast/standalone_sender/streaming_vp8_encoder.cc | 491 --------------------- cast/standalone_sender/streaming_vp8_encoder.h | 302 ------------- cast/standalone_sender/streaming_vpx_encoder.cc | 507 ++++++++++++++++++++++ cast/standalone_sender/streaming_vpx_encoder.h | 307 +++++++++++++ cast/streaming/receiver_message.cc | 22 +- cast/streaming/receiver_message.h | 3 +- cast/streaming/receiver_session.cc | 2 + 14 files changed, 872 insertions(+), 820 deletions(-) delete mode 100644 cast/standalone_sender/streaming_vp8_encoder.cc delete mode 100644 cast/standalone_sender/streaming_vp8_encoder.h create mode 100644 cast/standalone_sender/streaming_vpx_encoder.cc create mode 100644 cast/standalone_sender/streaming_vpx_encoder.h diff --git a/cast/standalone_receiver/mirroring_application.cc b/cast/standalone_receiver/mirroring_application.cc index 83921d76..683fad5d 100644 --- a/cast/standalone_receiver/mirroring_application.cc +++ b/cast/standalone_receiver/mirroring_application.cc @@ -57,6 +57,8 @@ bool MirroringApplication::Launch(const std::string& app_id, std::make_unique(task_runner_, this); ReceiverSession::Preferences preferences; + preferences.video_codecs.insert(preferences.video_codecs.end(), + {VideoCodec::kVp9, VideoCodec::kAv1}); preferences.remoting = std::make_unique(); current_session_ = diff --git a/cast/standalone_sender/BUILD.gn b/cast/standalone_sender/BUILD.gn index 780e0609..b00e769d 100644 --- a/cast/standalone_sender/BUILD.gn +++ b/cast/standalone_sender/BUILD.gn @@ -55,8 +55,8 @@ if (!build_with_chromium) { "simulated_capturer.h", "streaming_opus_encoder.cc", "streaming_opus_encoder.h", - "streaming_vp8_encoder.cc", - "streaming_vp8_encoder.h", + "streaming_vpx_encoder.cc", + "streaming_vpx_encoder.h", ] include_dirs += ffmpeg_include_dirs + libopus_include_dirs + libvpx_include_dirs diff --git a/cast/standalone_sender/connection_settings.h b/cast/standalone_sender/connection_settings.h index f42842c3..4c7e4849 100644 --- a/cast/standalone_sender/connection_settings.h +++ b/cast/standalone_sender/connection_settings.h @@ -7,6 +7,7 @@ #include +#include "cast/streaming/constants.h" #include "platform/base/interface_info.h" namespace openscreen { @@ -40,6 +41,9 @@ struct ConnectionSettings { // Whether we should loop the video when it is completed. bool should_loop_video = true; + + // The codec to use for encoding negotiated video streams. + VideoCodec codec; }; } // namespace cast diff --git a/cast/standalone_sender/looping_file_cast_agent.cc b/cast/standalone_sender/looping_file_cast_agent.cc index bfb970bf..4ee2231a 100644 --- a/cast/standalone_sender/looping_file_cast_agent.cc +++ b/cast/standalone_sender/looping_file_cast_agent.cc @@ -260,10 +260,11 @@ void LoopingFileCastAgent::CreateAndStartSession() { AudioCaptureConfig audio_config; // Opus does best at 192kbps, so we cap that here. audio_config.bit_rate = 192 * 1000; - VideoCaptureConfig video_config; - // The video config is allowed to use whatever is left over after audio. - video_config.max_bit_rate = - connection_settings_->max_bitrate - audio_config.bit_rate; + VideoCaptureConfig video_config = { + .codec = connection_settings_->codec, + // The video config is allowed to use whatever is left over after audio. + .max_bit_rate = + connection_settings_->max_bitrate - audio_config.bit_rate}; // Use default display resolution of 1080P. video_config.resolutions.emplace_back(Resolution{1920, 1080}); @@ -271,8 +272,8 @@ void LoopingFileCastAgent::CreateAndStartSession() { Error negotiation_error; if (connection_settings_->use_remoting) { remoting_sender_ = std::make_unique( - current_session_->rpc_messenger(), AudioCodec::kOpus, VideoCodec::kVp8, - [this]() { OnRemotingReceiverReady(); }); + current_session_->rpc_messenger(), AudioCodec::kOpus, + connection_settings_->codec, [this]() { OnRemotingReceiverReady(); }); negotiation_error = current_session_->NegotiateRemoting(audio_config, video_config); diff --git a/cast/standalone_sender/looping_file_sender.cc b/cast/standalone_sender/looping_file_sender.cc index 1392c413..9f6137dc 100644 --- a/cast/standalone_sender/looping_file_sender.cc +++ b/cast/standalone_sender/looping_file_sender.cc @@ -23,7 +23,7 @@ LoopingFileSender::LoopingFileSender(Environment* environment, audio_encoder_(senders.audio_sender->config().channels, StreamingOpusEncoder::kDefaultCastAudioFramesPerSecond, senders.audio_sender), - video_encoder_(StreamingVp8Encoder::Parameters{}, + video_encoder_(StreamingVpxEncoder::Parameters{.codec = settings.codec}, env_->task_runner(), senders.video_sender), next_task_(env_->now_function(), env_->task_runner()), @@ -32,7 +32,8 @@ LoopingFileSender::LoopingFileSender(Environment* environment, // to a different value that means we offered a codec that we do not // support, which is a developer error. OSP_CHECK(senders.audio_config.codec == AudioCodec::kOpus); - OSP_CHECK(senders.video_config.codec == VideoCodec::kVp8); + OSP_CHECK(senders.video_config.codec == VideoCodec::kVp8 || + senders.video_config.codec == VideoCodec::kVp9); OSP_LOG_INFO << "Max allowed media bitrate (audio + video) will be " << settings_.max_bitrate; bandwidth_being_utilized_ = settings_.max_bitrate / 2; @@ -117,7 +118,7 @@ void LoopingFileSender::OnVideoFrame(const AVFrame& av_frame, Clock::time_point capture_time) { TRACE_DEFAULT_SCOPED(TraceCategory::kStandaloneSender); latest_frame_time_ = std::max(capture_time, latest_frame_time_); - StreamingVp8Encoder::VideoFrame frame{}; + StreamingVpxEncoder::VideoFrame frame{}; frame.width = av_frame.width - av_frame.crop_left - av_frame.crop_right; frame.height = av_frame.height - av_frame.crop_top - av_frame.crop_bottom; frame.yuv_planes[0] = av_frame.data[0] + av_frame.crop_left + diff --git a/cast/standalone_sender/looping_file_sender.h b/cast/standalone_sender/looping_file_sender.h index ef283dc3..56a8aa45 100644 --- a/cast/standalone_sender/looping_file_sender.h +++ b/cast/standalone_sender/looping_file_sender.h @@ -12,7 +12,7 @@ #include "cast/standalone_sender/constants.h" #include "cast/standalone_sender/simulated_capturer.h" #include "cast/standalone_sender/streaming_opus_encoder.h" -#include "cast/standalone_sender/streaming_vp8_encoder.h" +#include "cast/standalone_sender/streaming_vpx_encoder.h" #include "cast/streaming/sender_session.h" namespace openscreen { @@ -73,7 +73,7 @@ class LoopingFileSender final : public SimulatedAudioCapturer::Client, int bandwidth_being_utilized_; StreamingOpusEncoder audio_encoder_; - StreamingVp8Encoder video_encoder_; + StreamingVpxEncoder video_encoder_; int num_capturers_running_ = 0; Clock::time_point capture_start_time_{}; diff --git a/cast/standalone_sender/main.cc b/cast/standalone_sender/main.cc index 228c00b1..22a9b6fa 100644 --- a/cast/standalone_sender/main.cc +++ b/cast/standalone_sender/main.cc @@ -81,6 +81,9 @@ usage: %s addr[:port] media_file -v, --verbose: Enable verbose logging. -h, --help: Show this help message. + + -c, --codec: Specifies the video codec to be used. Can be one of: + vp8, vp9, av1. Defaults to vp8 if not specified. )"; std::cerr << StringPrintf(kTemplate, argv0, argv0, kDefaultCastPort, @@ -122,6 +125,7 @@ int StandaloneSenderMain(int argc, char* argv[]) { {"tracing", no_argument, nullptr, 't'}, {"verbose", no_argument, nullptr, 'v'}, {"help", no_argument, nullptr, 'h'}, + {"codec", required_argument, nullptr, 'c'}, {nullptr, 0, nullptr, 0} }; @@ -131,9 +135,10 @@ int StandaloneSenderMain(int argc, char* argv[]) { bool use_android_rtp_hack = false; bool use_remoting = false; bool is_verbose = false; + VideoCodec codec = VideoCodec::kVp8; std::unique_ptr trace_logger; int ch = -1; - while ((ch = getopt_long(argc, argv, "m:nd:artvh", kArgumentOptions, + while ((ch = getopt_long(argc, argv, "m:nd:artvhc:", kArgumentOptions, nullptr)) != -1) { switch (ch) { case 'm': @@ -168,6 +173,20 @@ int StandaloneSenderMain(int argc, char* argv[]) { case 'h': LogUsage(argv[0]); return 1; + case 'c': + auto specified_codec = StringToVideoCodec(optarg); + if (specified_codec.is_value() && + (specified_codec.value() == VideoCodec::kVp8 || + specified_codec.value() == VideoCodec::kVp9 || + specified_codec.value() == VideoCodec::kAv1)) { + codec = specified_codec.value(); + } else { + OSP_LOG_ERROR << "Invalid --codec specified: " << optarg + << " is not one of: vp8, vp9, av1."; + LogUsage(argv[0]); + return 1; + } + break; } } @@ -228,7 +247,8 @@ int StandaloneSenderMain(int argc, char* argv[]) { .should_include_video = true, .use_android_rtp_hack = use_android_rtp_hack, .use_remoting = use_remoting, - .should_loop_video = should_loop_video}); + .should_loop_video = should_loop_video, + .codec = codec}); }); // Run the event loop until SIGINT (e.g., CTRL-C at the console) or diff --git a/cast/standalone_sender/streaming_vp8_encoder.cc b/cast/standalone_sender/streaming_vp8_encoder.cc deleted file mode 100644 index 7cc10e35..00000000 --- a/cast/standalone_sender/streaming_vp8_encoder.cc +++ /dev/null @@ -1,491 +0,0 @@ -// Copyright 2020 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include "cast/standalone_sender/streaming_vp8_encoder.h" - -#include -#include -#include - -#include -#include -#include - -#include "cast/streaming/encoded_frame.h" -#include "cast/streaming/environment.h" -#include "cast/streaming/sender.h" -#include "util/chrono_helpers.h" -#include "util/osp_logging.h" -#include "util/saturate_cast.h" - -namespace openscreen { -namespace cast { - -// TODO(https://crbug.com/openscreen/123): Fix the declarations and then remove -// this: -using openscreen::operator<<; // For std::chrono::duration pretty-printing. - -namespace { - -constexpr int kBytesPerKilobyte = 1024; - -// Lower and upper bounds to the frame duration passed to vpx_codec_encode(), to -// ensure sanity. Note that the upper-bound is especially important in cases -// where the video paused for some lengthy amount of time. -constexpr Clock::duration kMinFrameDuration = milliseconds(1); -constexpr Clock::duration kMaxFrameDuration = milliseconds(125); - -// Highest/lowest allowed encoding speed set to the encoder. The valid range is -// [4, 16], but experiments show that with speed higher than 12, the saving of -// the encoding time is not worth the dropping of the quality. And, with speed -// lower than 6, the increasing amount of quality is not worth the increasing -// amount of encoding time. -constexpr int kHighestEncodingSpeed = 12; -constexpr int kLowestEncodingSpeed = 6; - -// This is the equivalent change in encoding speed per one quantizer step. -constexpr double kEquivalentEncodingSpeedStepPerQuantizerStep = 1 / 20.0; - -} // namespace - -StreamingVp8Encoder::StreamingVp8Encoder(const Parameters& params, - TaskRunner* task_runner, - Sender* sender) - : params_(params), - main_task_runner_(task_runner), - sender_(sender), - ideal_speed_setting_(kHighestEncodingSpeed), - encode_thread_([this] { ProcessWorkUnitsUntilTimeToQuit(); }) { - OSP_DCHECK_LE(1, params_.num_encode_threads); - OSP_DCHECK_LE(kMinQuantizer, params_.min_quantizer); - OSP_DCHECK_LE(params_.min_quantizer, params_.max_cpu_saver_quantizer); - OSP_DCHECK_LE(params_.max_cpu_saver_quantizer, params_.max_quantizer); - OSP_DCHECK_LE(params_.max_quantizer, kMaxQuantizer); - OSP_DCHECK_LT(0.0, params_.max_time_utilization); - OSP_DCHECK_LE(params_.max_time_utilization, 1.0); - OSP_DCHECK(main_task_runner_); - OSP_DCHECK(sender_); - - const auto result = - vpx_codec_enc_config_default(vpx_codec_vp8_cx(), &config_, 0); - OSP_CHECK_EQ(result, VPX_CODEC_OK); - - // This is set to non-zero in ConfigureForNewFrameSize() later, to flag that - // the encoder has been initialized. - config_.g_threads = 0; - - // Set the timebase to match that of openscreen::Clock::duration. - config_.g_timebase.num = Clock::duration::period::num; - config_.g_timebase.den = Clock::duration::period::den; - - // |g_pass| and |g_lag_in_frames| must be "one pass" and zero, respectively, - // because of the way the libvpx API is used. - config_.g_pass = VPX_RC_ONE_PASS; - config_.g_lag_in_frames = 0; - - // Rate control settings. - config_.rc_dropframe_thresh = 0; // The encoder may not drop any frames. - config_.rc_resize_allowed = 0; - config_.rc_end_usage = VPX_CBR; - config_.rc_target_bitrate = target_bitrate_ / kBytesPerKilobyte; - config_.rc_min_quantizer = params_.min_quantizer; - config_.rc_max_quantizer = params_.max_quantizer; - - // The reasons for the values chosen here (rc_*shoot_pct and rc_buf_*_sz) are - // lost in history. They were brought-over from the legacy Chrome Cast - // Streaming Sender implemenation. - config_.rc_undershoot_pct = 100; - config_.rc_overshoot_pct = 15; - config_.rc_buf_initial_sz = 500; - config_.rc_buf_optimal_sz = 600; - config_.rc_buf_sz = 1000; - - config_.kf_mode = VPX_KF_DISABLED; -} - -StreamingVp8Encoder::~StreamingVp8Encoder() { - { - std::unique_lock lock(mutex_); - target_bitrate_ = 0; - cv_.notify_one(); - } - encode_thread_.join(); -} - -int StreamingVp8Encoder::GetTargetBitrate() const { - // Note: No need to lock the |mutex_| since this method should be called on - // the same thread as SetTargetBitrate(). - return target_bitrate_; -} - -void StreamingVp8Encoder::SetTargetBitrate(int new_bitrate) { - // Ensure that, when bps is converted to kbps downstream, that the encoder - // bitrate will not be zero. - new_bitrate = std::max(new_bitrate, kBytesPerKilobyte); - - std::unique_lock lock(mutex_); - // Only assign the new target bitrate if |target_bitrate_| has not yet been - // used to signal the |encode_thread_| to end. - if (target_bitrate_ > 0) { - target_bitrate_ = new_bitrate; - } -} - -void StreamingVp8Encoder::EncodeAndSend( - const VideoFrame& frame, - Clock::time_point reference_time, - std::function stats_callback) { - WorkUnit work_unit; - - // TODO(jophba): The |VideoFrame| struct should provide the media timestamp, - // instead of this code inferring it from the reference timestamps, since: 1) - // the video capturer's clock may tick at a different rate than the system - // clock; and 2) to reduce jitter. - if (start_time_ == Clock::time_point::min()) { - start_time_ = reference_time; - work_unit.rtp_timestamp = RtpTimeTicks(); - } else { - work_unit.rtp_timestamp = RtpTimeTicks::FromTimeSinceOrigin( - reference_time - start_time_, sender_->rtp_timebase()); - if (work_unit.rtp_timestamp <= last_enqueued_rtp_timestamp_) { - OSP_LOG_WARN << "VIDEO[" << sender_->ssrc() - << "] Dropping: RTP timestamp is not monotonically " - "increasing from last frame."; - return; - } - } - if (sender_->GetInFlightMediaDuration(work_unit.rtp_timestamp) > - sender_->GetMaxInFlightMediaDuration()) { - OSP_LOG_WARN << "VIDEO[" << sender_->ssrc() - << "] Dropping: In-flight media duration would be too high."; - return; - } - - Clock::duration frame_duration = frame.duration; - if (frame_duration <= Clock::duration::zero()) { - // The caller did not provide the frame duration in |frame|. - if (reference_time == start_time_) { - // Use the max for the first frame so libvpx will spend extra effort on - // its quality. - frame_duration = kMaxFrameDuration; - } else { - // Use the actual amount of time between the current and previous frame as - // a prediction for the next frame's duration. - frame_duration = - (work_unit.rtp_timestamp - last_enqueued_rtp_timestamp_) - .ToDuration(sender_->rtp_timebase()); - } - } - work_unit.duration = - std::max(std::min(frame_duration, kMaxFrameDuration), kMinFrameDuration); - - last_enqueued_rtp_timestamp_ = work_unit.rtp_timestamp; - - work_unit.image = CloneAsVpxImage(frame); - work_unit.reference_time = reference_time; - work_unit.stats_callback = std::move(stats_callback); - const bool force_key_frame = sender_->NeedsKeyFrame(); - { - std::unique_lock lock(mutex_); - needs_key_frame_ |= force_key_frame; - encode_queue_.push(std::move(work_unit)); - cv_.notify_one(); - } -} - -void StreamingVp8Encoder::DestroyEncoder() { - OSP_DCHECK_EQ(std::this_thread::get_id(), encode_thread_.get_id()); - - if (is_encoder_initialized()) { - vpx_codec_destroy(&encoder_); - // Flag that the encoder is not initialized. See header comments for - // is_encoder_initialized(). - config_.g_threads = 0; - } -} - -void StreamingVp8Encoder::ProcessWorkUnitsUntilTimeToQuit() { - OSP_DCHECK_EQ(std::this_thread::get_id(), encode_thread_.get_id()); - - for (;;) { - WorkUnitWithResults work_unit{}; - bool force_key_frame; - int target_bitrate; - { - std::unique_lock lock(mutex_); - if (target_bitrate_ <= 0) { - break; // Time to end this thread. - } - if (encode_queue_.empty()) { - cv_.wait(lock); - if (encode_queue_.empty()) { - continue; - } - } - static_cast(work_unit) = std::move(encode_queue_.front()); - encode_queue_.pop(); - force_key_frame = needs_key_frame_; - needs_key_frame_ = false; - target_bitrate = target_bitrate_; - } - - // Clock::now() is being called directly, instead of using a - // dependency-injected "now function," since actual wall time is being - // measured. - const Clock::time_point encode_start_time = Clock::now(); - PrepareEncoder(work_unit.image->d_w, work_unit.image->d_h, target_bitrate); - EncodeFrame(force_key_frame, &work_unit); - ComputeFrameEncodeStats(Clock::now() - encode_start_time, target_bitrate, - &work_unit); - UpdateSpeedSettingForNextFrame(work_unit.stats); - - main_task_runner_->PostTask( - [this, results = std::move(work_unit)]() mutable { - SendEncodedFrame(std::move(results)); - }); - } - - DestroyEncoder(); -} - -void StreamingVp8Encoder::PrepareEncoder(int width, - int height, - int target_bitrate) { - OSP_DCHECK_EQ(std::this_thread::get_id(), encode_thread_.get_id()); - - const int target_kbps = target_bitrate / kBytesPerKilobyte; - - // Translate the |ideal_speed_setting_| into the VP8E_SET_CPUUSED setting and - // the minimum quantizer to use. - int speed; - int min_quantizer; - if (ideal_speed_setting_ > kHighestEncodingSpeed) { - speed = kHighestEncodingSpeed; - const double remainder = ideal_speed_setting_ - speed; - min_quantizer = rounded_saturate_cast( - remainder / kEquivalentEncodingSpeedStepPerQuantizerStep + - params_.min_quantizer); - min_quantizer = std::min(min_quantizer, params_.max_cpu_saver_quantizer); - } else { - speed = std::max(rounded_saturate_cast(ideal_speed_setting_), - kLowestEncodingSpeed); - min_quantizer = params_.min_quantizer; - } - - if (static_cast(config_.g_w) != width || - static_cast(config_.g_h) != height) { - DestroyEncoder(); - } - - if (!is_encoder_initialized()) { - config_.g_threads = params_.num_encode_threads; - config_.g_w = width; - config_.g_h = height; - config_.rc_target_bitrate = target_kbps; - config_.rc_min_quantizer = min_quantizer; - - encoder_ = {}; - const vpx_codec_flags_t flags = 0; - const auto init_result = - vpx_codec_enc_init(&encoder_, vpx_codec_vp8_cx(), &config_, flags); - OSP_CHECK_EQ(init_result, VPX_CODEC_OK); - - // Raise the threshold for considering macroblocks as static. The default is - // zero, so this setting makes the encoder less sensitive to motion. This - // lowers the probability of needing to utilize more CPU to search for - // motion vectors. - const auto ctl_result = - vpx_codec_control(&encoder_, VP8E_SET_STATIC_THRESHOLD, 1); - OSP_CHECK_EQ(ctl_result, VPX_CODEC_OK); - - // Ensure the speed will be set (below). - current_speed_setting_ = ~speed; - } else if (static_cast(config_.rc_target_bitrate) != target_kbps || - static_cast(config_.rc_min_quantizer) != min_quantizer) { - config_.rc_target_bitrate = target_kbps; - config_.rc_min_quantizer = min_quantizer; - const auto update_config_result = - vpx_codec_enc_config_set(&encoder_, &config_); - OSP_CHECK_EQ(update_config_result, VPX_CODEC_OK); - } - - if (current_speed_setting_ != speed) { - // Pass the |speed| as a negative value to turn off VP8's automatic speed - // selection logic and force the exact setting. - const auto ctl_result = - vpx_codec_control(&encoder_, VP8E_SET_CPUUSED, -speed); - OSP_CHECK_EQ(ctl_result, VPX_CODEC_OK); - current_speed_setting_ = speed; - } -} - -void StreamingVp8Encoder::EncodeFrame(bool force_key_frame, - WorkUnitWithResults* work_unit) { - OSP_DCHECK_EQ(std::this_thread::get_id(), encode_thread_.get_id()); - - // The presentation timestamp argument here is fixed to zero to force the - // encoder to base its single-frame bandwidth calculations entirely on - // |frame_duration| and the target bitrate setting. - const vpx_codec_pts_t pts = 0; - const vpx_enc_frame_flags_t flags = force_key_frame ? VPX_EFLAG_FORCE_KF : 0; - const auto encode_result = - vpx_codec_encode(&encoder_, work_unit->image.get(), pts, - work_unit->duration.count(), flags, VPX_DL_REALTIME); - OSP_CHECK_EQ(encode_result, VPX_CODEC_OK); - - const vpx_codec_cx_pkt_t* pkt; - for (vpx_codec_iter_t iter = nullptr;;) { - pkt = vpx_codec_get_cx_data(&encoder_, &iter); - // vpx_codec_get_cx_data() returns null once the "iteration" is complete. - // However, that point should never be reached because a - // VPX_CODEC_CX_FRAME_PKT must be encountered before that. - OSP_CHECK(pkt); - if (pkt->kind == VPX_CODEC_CX_FRAME_PKT) { - break; - } - } - - // A copy of the payload data is being made here. That's okay since it has to - // be copied at some point anyway, to be passed back to the main thread. - auto* const begin = static_cast(pkt->data.frame.buf); - auto* const end = begin + pkt->data.frame.sz; - work_unit->payload.assign(begin, end); - work_unit->is_key_frame = !!(pkt->data.frame.flags & VPX_FRAME_IS_KEY); -} - -void StreamingVp8Encoder::ComputeFrameEncodeStats( - Clock::duration encode_wall_time, - int target_bitrate, - WorkUnitWithResults* work_unit) { - OSP_DCHECK_EQ(std::this_thread::get_id(), encode_thread_.get_id()); - - Stats& stats = work_unit->stats; - - // Note: stats.frame_id is set later, in SendEncodedFrame(). - stats.rtp_timestamp = work_unit->rtp_timestamp; - stats.encode_wall_time = encode_wall_time; - stats.frame_duration = work_unit->duration; - stats.encoded_size = work_unit->payload.size(); - - constexpr double kBytesPerBit = 1.0 / CHAR_BIT; - constexpr double kSecondsPerClockTick = - 1.0 / Clock::to_duration(seconds(1)).count(); - const double target_bytes_per_clock_tick = - target_bitrate * (kBytesPerBit * kSecondsPerClockTick); - stats.target_size = target_bytes_per_clock_tick * work_unit->duration.count(); - - // The quantizer the encoder used. This is the result of the VP8 encoder - // taking a guess at what quantizer value would produce an encoded frame size - // as close to the target as possible. - const auto get_quantizer_result = vpx_codec_control( - &encoder_, VP8E_GET_LAST_QUANTIZER_64, &stats.quantizer); - OSP_CHECK_EQ(get_quantizer_result, VPX_CODEC_OK); - - // Now that the frame has been encoded and the number of bytes is known, the - // perfect quantizer value (i.e., the one that should have been used) can be - // determined. - stats.perfect_quantizer = stats.quantizer * stats.space_utilization(); -} - -void StreamingVp8Encoder::UpdateSpeedSettingForNextFrame(const Stats& stats) { - OSP_DCHECK_EQ(std::this_thread::get_id(), encode_thread_.get_id()); - - // Combine the speed setting that was used to encode the last frame, and the - // quantizer the encoder chose into a single speed metric. - const double speed = current_speed_setting_ + - kEquivalentEncodingSpeedStepPerQuantizerStep * - std::max(0, stats.quantizer - params_.min_quantizer); - - // Like |Stats::perfect_quantizer|, this computes a "hindsight" speed setting - // for the last frame, one that may have potentially allowed for a - // better-quality quantizer choice by the encoder, while also keeping CPU - // utilization within budget. - const double perfect_speed = - speed * stats.time_utilization() / params_.max_time_utilization; - - // Update the ideal speed setting, to be used for the next frame. An - // exponentially-decaying weighted average is used here to smooth-out noise. - // The weight is based on the duration of the frame that was encoded. - constexpr Clock::duration kDecayHalfLife = milliseconds(120); - const double ticks = stats.frame_duration.count(); - const double weight = ticks / (ticks + kDecayHalfLife.count()); - ideal_speed_setting_ = - weight * perfect_speed + (1.0 - weight) * ideal_speed_setting_; - OSP_DCHECK(std::isfinite(ideal_speed_setting_)); -} - -void StreamingVp8Encoder::SendEncodedFrame(WorkUnitWithResults results) { - OSP_DCHECK(main_task_runner_->IsRunningOnTaskRunner()); - - EncodedFrame frame; - frame.frame_id = sender_->GetNextFrameId(); - if (results.is_key_frame) { - frame.dependency = EncodedFrame::KEY_FRAME; - frame.referenced_frame_id = frame.frame_id; - } else { - frame.dependency = EncodedFrame::DEPENDS_ON_ANOTHER; - frame.referenced_frame_id = frame.frame_id - 1; - } - frame.rtp_timestamp = results.rtp_timestamp; - frame.reference_time = results.reference_time; - frame.data = absl::Span(results.payload); - - if (sender_->EnqueueFrame(frame) != Sender::OK) { - // Since the frame will not be sent, the encoder's frame dependency chain - // has been broken. Force a key frame for the next frame. - std::unique_lock lock(mutex_); - needs_key_frame_ = true; - } - - if (results.stats_callback) { - results.stats.frame_id = frame.frame_id; - results.stats_callback(results.stats); - } -} - -namespace { -void CopyPlane(const uint8_t* src, - int src_stride, - int num_rows, - uint8_t* dst, - int dst_stride) { - if (src_stride == dst_stride) { - memcpy(dst, src, src_stride * num_rows); - return; - } - const int bytes_per_row = std::min(src_stride, dst_stride); - while (--num_rows >= 0) { - memcpy(dst, src, bytes_per_row); - dst += dst_stride; - src += src_stride; - } -} -} // namespace - -// static -StreamingVp8Encoder::VpxImageUniquePtr StreamingVp8Encoder::CloneAsVpxImage( - const VideoFrame& frame) { - OSP_DCHECK_GE(frame.width, 0); - OSP_DCHECK_GE(frame.height, 0); - OSP_DCHECK_GE(frame.yuv_strides[0], 0); - OSP_DCHECK_GE(frame.yuv_strides[1], 0); - OSP_DCHECK_GE(frame.yuv_strides[2], 0); - - constexpr int kAlignment = 32; - VpxImageUniquePtr image(vpx_img_alloc(nullptr, VPX_IMG_FMT_I420, frame.width, - frame.height, kAlignment)); - OSP_CHECK(image); - - CopyPlane(frame.yuv_planes[0], frame.yuv_strides[0], frame.height, - image->planes[VPX_PLANE_Y], image->stride[VPX_PLANE_Y]); - CopyPlane(frame.yuv_planes[1], frame.yuv_strides[1], (frame.height + 1) / 2, - image->planes[VPX_PLANE_U], image->stride[VPX_PLANE_U]); - CopyPlane(frame.yuv_planes[2], frame.yuv_strides[2], (frame.height + 1) / 2, - image->planes[VPX_PLANE_V], image->stride[VPX_PLANE_V]); - - return image; -} - -} // namespace cast -} // namespace openscreen diff --git a/cast/standalone_sender/streaming_vp8_encoder.h b/cast/standalone_sender/streaming_vp8_encoder.h deleted file mode 100644 index c5d52248..00000000 --- a/cast/standalone_sender/streaming_vp8_encoder.h +++ /dev/null @@ -1,302 +0,0 @@ -// Copyright 2020 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#ifndef CAST_STANDALONE_SENDER_STREAMING_VP8_ENCODER_H_ -#define CAST_STANDALONE_SENDER_STREAMING_VP8_ENCODER_H_ - -#include -#include - -#include -#include // NOLINT -#include -#include -#include -#include -#include -#include - -#include "absl/base/thread_annotations.h" -#include "cast/streaming/frame_id.h" -#include "cast/streaming/rtp_time.h" -#include "platform/api/task_runner.h" -#include "platform/api/time.h" - -namespace openscreen { - -class TaskRunner; - -namespace cast { - -class Sender; - -// Uses libvpx to encode VP8 video and streams it to a Sender. Includes -// extensive logic for fine-tuning the encoder parameters in real-time, to -// provide the best quality results given external, uncontrollable factors: -// CPU/network availability, and the complexity of the video frame content. -// -// Internally, a separate encode thread is created and used to prevent blocking -// the main thread while frames are being encoded. All public API methods are -// assumed to be called on the same sequence/thread as the main TaskRunner -// (injected via the constructor). -// -// Usage: -// -// 1. EncodeAndSend() is used to queue-up video frames for encoding and sending, -// which will be done on a best-effort basis. -// -// 2. The client is expected to call SetTargetBitrate() frequently based on its -// own bandwidth estimates and congestion control logic. In addition, a client -// may provide a callback for each frame's encode statistics, which can be used -// to further optimize the user experience. For example, the stats can be used -// as a signal to reduce the data volume (i.e., resolution and/or frame rate) -// coming from the video capture source. -class StreamingVp8Encoder { - public: - // Configurable parameters passed to the StreamingVp8Encoder constructor. - struct Parameters { - // Number of threads to parallelize frame encoding. This should be set based - // on the number of CPU cores available for encoding, but no more than 8. - int num_encode_threads = - std::min(std::max(std::thread::hardware_concurrency(), 1), 8); - - // Best-quality quantizer (lower is better quality). Range: [0,63] - int min_quantizer = 4; - - // Worst-quality quantizer (lower is better quality). Range: [0,63] - int max_quantizer = 63; - - // Worst-quality quantizer to use when the CPU is extremely constrained. - // Range: [min_quantizer,max_quantizer] - int max_cpu_saver_quantizer = 25; - - // Maximum amount of wall-time a frame's encode can take, relative to the - // frame's duration, before the CPU-saver logic is activated. The default - // (70%) is appropriate for systems with four or more cores, but should be - // reduced (e.g., 50%) for systems with fewer than three cores. - // - // Example: For 30 FPS (continuous) video, the frame duration is ~33.3ms, - // and a value of 0.5 here would mean that the CPU-saver logic starts - // sacrificing quality when frame encodes start taking longer than ~16.7ms. - double max_time_utilization = 0.7; - }; - - // Represents an input VideoFrame, passed to EncodeAndSend(). - struct VideoFrame { - // Image width and height. - int width; - int height; - - // I420 format image pointers and row strides (the number of bytes between - // the start of successive rows). The pointers only need to remain valid - // until the EncodeAndSend() call returns. - const uint8_t* yuv_planes[3]; - int yuv_strides[3]; - - // How long this frame will be held before the next frame will be displayed, - // or zero if unknown. The frame duration is passed to the VP8 codec, - // affecting a number of important behaviors, including: per-frame - // bandwidth, CPU time spent encoding, temporal quality trade-offs, and - // key/golden/alt-ref frame generation intervals. - Clock::duration duration; - }; - - // Performance statistics for a single frame's encode. - // - // For full details on how to use these stats in an end-to-end system, see: - // https://www.chromium.org/developers/design-documents/ - // auto-throttled-screen-capture-and-mirroring - // and https://source.chromium.org/chromium/chromium/src/+/master: - // media/cast/sender/performance_metrics_overlay.h - struct Stats { - // The Cast Streaming ID that was assigned to the frame. - FrameId frame_id; - - // The RTP timestamp of the frame. - RtpTimeTicks rtp_timestamp; - - // How long the frame took to encode. This is wall time, not CPU time or - // some other load metric. - Clock::duration encode_wall_time; - - // The frame's predicted duration; or, the actual duration if it was - // provided in the VideoFrame. - Clock::duration frame_duration; - - // The encoded frame's size in bytes. - int encoded_size; - - // The average size of an encoded frame in bytes, having this - // |frame_duration| and current target bitrate. - double target_size; - - // The actual quantizer the VP8 encoder used, in the range [0,63]. - int quantizer; - - // The "hindsight" quantizer value that would have produced the best quality - // encoding of the frame at the current target bitrate. The nominal range is - // [0.0,63.0]. If it is larger than 63.0, then it was impossible for VP8 to - // encode the frame within the current target bitrate (e.g., too much - // "entropy" in the image, or too low a target bitrate). - double perfect_quantizer; - - // Utilization feedback metrics. The nominal range for each of these is - // [0.0,1.0] where 1.0 means "the entire budget available for the frame was - // exhausted." Going above 1.0 is okay for one or a few frames, since it's - // the average over many frames that matters before the system is considered - // "redlining." - // - // The max of these three provides an overall utilization control signal. - // The usual approach is for upstream control logic to increase/decrease the - // data volume (e.g., video resolution and/or frame rate) to maintain a good - // target point. - double time_utilization() const { - return static_cast(encode_wall_time.count()) / - frame_duration.count(); - } - double space_utilization() const { return encoded_size / target_size; } - double entropy_utilization() const { - return perfect_quantizer / kMaxQuantizer; - } - }; - - StreamingVp8Encoder(const Parameters& params, - TaskRunner* task_runner, - Sender* sender); - - ~StreamingVp8Encoder(); - - // Get/Set the target bitrate. This may be changed at any time, as frequently - // as desired, and it will take effect internally as soon as possible. - int GetTargetBitrate() const; - void SetTargetBitrate(int new_bitrate); - - // Encode |frame| using the VP8 encoder, assemble an EncodedFrame, and enqueue - // into the Sender. The frame may be dropped if too many frames are in-flight. - // If provided, the |stats_callback| is run after the frame is enqueued in the - // Sender (via the main TaskRunner). - void EncodeAndSend(const VideoFrame& frame, - Clock::time_point reference_time, - std::function stats_callback); - - static constexpr int kMinQuantizer = 0; - static constexpr int kMaxQuantizer = 63; - - private: - // Syntactic convenience to wrap the vpx_image_t alloc/free API in a smart - // pointer. - struct VpxImageDeleter { - void operator()(vpx_image_t* ptr) const { vpx_img_free(ptr); } - }; - using VpxImageUniquePtr = std::unique_ptr; - - // Represents the state of one frame encode. This is created in - // EncodeAndSend(), and passed to the encode thread via the |encode_queue_|. - struct WorkUnit { - VpxImageUniquePtr image; - Clock::duration duration; - Clock::time_point reference_time; - RtpTimeTicks rtp_timestamp; - std::function stats_callback; - }; - - // Same as WorkUnit, but with additional fields to carry the encode results. - struct WorkUnitWithResults : public WorkUnit { - std::vector payload; - bool is_key_frame; - Stats stats; - }; - - bool is_encoder_initialized() const { return config_.g_threads != 0; } - - // Destroys the VP8 encoder context if it has been initialized. - void DestroyEncoder(); - - // The procedure for the |encode_thread_| that loops, processing work units - // from the |encode_queue_| by calling Encode() until it's time to end the - // thread. - void ProcessWorkUnitsUntilTimeToQuit(); - - // If the |encoder_| is live, attempt reconfiguration to allow it to encode - // frames at a new frame size, target bitrate, or "CPU encoding speed." If - // reconfiguration is not possible, destroy the existing instance and - // re-create a new |encoder_| instance. - void PrepareEncoder(int width, int height, int target_bitrate); - - // Wraps the complex libvpx vpx_codec_encode() call using inputs from - // |work_unit| and populating results there. - void EncodeFrame(bool force_key_frame, WorkUnitWithResults* work_unit); - - // Computes and populates |work_unit.stats| after the last call to - // EncodeFrame(). - void ComputeFrameEncodeStats(Clock::duration encode_wall_time, - int target_bitrate, - WorkUnitWithResults* work_unit); - - // Updates the |ideal_speed_setting_|, to take effect with the next frame - // encode, based on the given performance |stats|. - void UpdateSpeedSettingForNextFrame(const Stats& stats); - - // Assembles and enqueues an EncodedFrame with the Sender on the main thread. - void SendEncodedFrame(WorkUnitWithResults results); - - // Allocates a vpx_image_t and copies the content from |frame| to it. - static VpxImageUniquePtr CloneAsVpxImage(const VideoFrame& frame); - - const Parameters params_; - TaskRunner* const main_task_runner_; - Sender* const sender_; - - // The reference time of the first frame passed to EncodeAndSend(). - Clock::time_point start_time_ = Clock::time_point::min(); - - // The RTP timestamp of the last frame that was pushed into the - // |encode_queue_| by EncodeAndSend(). This is used to check whether - // timestamps are monotonically increasing. - RtpTimeTicks last_enqueued_rtp_timestamp_; - - // Guards a few members shared by both the main and encode threads. - std::mutex mutex_; - - // Used by the encode thread to sleep until more work is available. - std::condition_variable cv_ ABSL_GUARDED_BY(mutex_); - - // These encode parameters not passed in the WorkUnit struct because it is - // desirable for them to be applied as soon as possible, with the very next - // WorkUnit popped from the |encode_queue_| on the encode thread, and not to - // wait until some later WorkUnit is processed. - bool needs_key_frame_ ABSL_GUARDED_BY(mutex_) = true; - int target_bitrate_ ABSL_GUARDED_BY(mutex_) = 2 << 20; // Default: 2 Mbps. - - // The queue of frame encodes. The size of this queue is implicitly bounded by - // EncodeAndSend(), where it checks for the total in-flight media duration and - // maybe drops a frame. - std::queue encode_queue_ ABSL_GUARDED_BY(mutex_); - - // Current VP8 encoder configuration. Most of the fields are unchanging, and - // are populated in the ctor; but thereafter, only the encode thread accesses - // this struct. - // - // The speed setting is controlled via a separate libvpx API (see members - // below). - vpx_codec_enc_cfg_t config_{}; - - // These represent the magnitude of the VP8 speed setting, where larger values - // (i.e., faster speed) request less CPU usage but will provide lower video - // quality. Only the encode thread accesses these. - double ideal_speed_setting_; // A time-weighted average, from measurements. - int current_speed_setting_; // Current |encoder_| speed setting. - - // libvpx VP8 encoder instance. Only the encode thread accesses this. - vpx_codec_ctx_t encoder_; - - // This member should be last in the class since the thread should not start - // until all above members have been initialized by the constructor. - std::thread encode_thread_; -}; - -} // namespace cast -} // namespace openscreen - -#endif // CAST_STANDALONE_SENDER_STREAMING_VP8_ENCODER_H_ diff --git a/cast/standalone_sender/streaming_vpx_encoder.cc b/cast/standalone_sender/streaming_vpx_encoder.cc new file mode 100644 index 00000000..1c9de314 --- /dev/null +++ b/cast/standalone_sender/streaming_vpx_encoder.cc @@ -0,0 +1,507 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "cast/standalone_sender/streaming_vpx_encoder.h" + +#include +#include +#include + +#include +#include +#include + +#include "cast/streaming/encoded_frame.h" +#include "cast/streaming/environment.h" +#include "cast/streaming/sender.h" +#include "util/chrono_helpers.h" +#include "util/osp_logging.h" +#include "util/saturate_cast.h" + +namespace openscreen { +namespace cast { + +// TODO(https://crbug.com/openscreen/123): Fix the declarations and then remove +// this: +using openscreen::operator<<; // For std::chrono::duration pretty-printing. + +namespace { + +constexpr int kBytesPerKilobyte = 1024; + +// Lower and upper bounds to the frame duration passed to vpx_codec_encode(), to +// ensure sanity. Note that the upper-bound is especially important in cases +// where the video paused for some lengthy amount of time. +constexpr Clock::duration kMinFrameDuration = milliseconds(1); +constexpr Clock::duration kMaxFrameDuration = milliseconds(125); + +// Highest/lowest allowed encoding speed set to the encoder. The valid range is +// [4, 16], but experiments show that with speed higher than 12, the saving of +// the encoding time is not worth the dropping of the quality. And, with speed +// lower than 6, the increasing amount of quality is not worth the increasing +// amount of encoding time. +constexpr int kHighestEncodingSpeed = 12; +constexpr int kLowestEncodingSpeed = 6; + +// This is the equivalent change in encoding speed per one quantizer step. +constexpr double kEquivalentEncodingSpeedStepPerQuantizerStep = 1 / 20.0; + +} // namespace + +StreamingVpxEncoder::StreamingVpxEncoder(const Parameters& params, + TaskRunner* task_runner, + Sender* sender) + : params_(params), + main_task_runner_(task_runner), + sender_(sender), + ideal_speed_setting_(kHighestEncodingSpeed), + encode_thread_([this] { ProcessWorkUnitsUntilTimeToQuit(); }) { + OSP_DCHECK_LE(1, params_.num_encode_threads); + OSP_DCHECK_LE(kMinQuantizer, params_.min_quantizer); + OSP_DCHECK_LE(params_.min_quantizer, params_.max_cpu_saver_quantizer); + OSP_DCHECK_LE(params_.max_cpu_saver_quantizer, params_.max_quantizer); + OSP_DCHECK_LE(params_.max_quantizer, kMaxQuantizer); + OSP_DCHECK_LT(0.0, params_.max_time_utilization); + OSP_DCHECK_LE(params_.max_time_utilization, 1.0); + OSP_DCHECK(main_task_runner_); + OSP_DCHECK(sender_); + + vpx_codec_iface_t* ctx; + if (params_.codec == VideoCodec::kVp9) { + ctx = vpx_codec_vp9_cx(); + } else { + OSP_DCHECK(params_.codec == VideoCodec::kVp8); + ctx = vpx_codec_vp8_cx(); + } + + const auto result = vpx_codec_enc_config_default(ctx, &config_, 0); + OSP_CHECK_EQ(result, VPX_CODEC_OK); + + // This is set to non-zero in ConfigureForNewFrameSize() later, to flag that + // the encoder has been initialized. + config_.g_threads = 0; + + // Set the timebase to match that of openscreen::Clock::duration. + config_.g_timebase.num = Clock::duration::period::num; + config_.g_timebase.den = Clock::duration::period::den; + + // |g_pass| and |g_lag_in_frames| must be "one pass" and zero, respectively, + // because of the way the libvpx API is used. + config_.g_pass = VPX_RC_ONE_PASS; + config_.g_lag_in_frames = 0; + + // Rate control settings. + config_.rc_dropframe_thresh = 0; // The encoder may not drop any frames. + config_.rc_resize_allowed = 0; + config_.rc_end_usage = VPX_CBR; + config_.rc_target_bitrate = target_bitrate_ / kBytesPerKilobyte; + config_.rc_min_quantizer = params_.min_quantizer; + config_.rc_max_quantizer = params_.max_quantizer; + + // The reasons for the values chosen here (rc_*shoot_pct and rc_buf_*_sz) are + // lost in history. They were brought-over from the legacy Chrome Cast + // Streaming Sender implemenation. + config_.rc_undershoot_pct = 100; + config_.rc_overshoot_pct = 15; + config_.rc_buf_initial_sz = 500; + config_.rc_buf_optimal_sz = 600; + config_.rc_buf_sz = 1000; + + config_.kf_mode = VPX_KF_DISABLED; +} + +StreamingVpxEncoder::~StreamingVpxEncoder() { + { + std::unique_lock lock(mutex_); + target_bitrate_ = 0; + cv_.notify_one(); + } + encode_thread_.join(); +} + +int StreamingVpxEncoder::GetTargetBitrate() const { + // Note: No need to lock the |mutex_| since this method should be called on + // the same thread as SetTargetBitrate(). + return target_bitrate_; +} + +void StreamingVpxEncoder::SetTargetBitrate(int new_bitrate) { + // Ensure that, when bps is converted to kbps downstream, that the encoder + // bitrate will not be zero. + new_bitrate = std::max(new_bitrate, kBytesPerKilobyte); + + std::unique_lock lock(mutex_); + // Only assign the new target bitrate if |target_bitrate_| has not yet been + // used to signal the |encode_thread_| to end. + if (target_bitrate_ > 0) { + target_bitrate_ = new_bitrate; + } +} + +void StreamingVpxEncoder::EncodeAndSend( + const VideoFrame& frame, + Clock::time_point reference_time, + std::function stats_callback) { + WorkUnit work_unit; + + // TODO(jophba): The |VideoFrame| struct should provide the media timestamp, + // instead of this code inferring it from the reference timestamps, since: 1) + // the video capturer's clock may tick at a different rate than the system + // clock; and 2) to reduce jitter. + if (start_time_ == Clock::time_point::min()) { + start_time_ = reference_time; + work_unit.rtp_timestamp = RtpTimeTicks(); + } else { + work_unit.rtp_timestamp = RtpTimeTicks::FromTimeSinceOrigin( + reference_time - start_time_, sender_->rtp_timebase()); + if (work_unit.rtp_timestamp <= last_enqueued_rtp_timestamp_) { + OSP_LOG_WARN << "VIDEO[" << sender_->ssrc() + << "] Dropping: RTP timestamp is not monotonically " + "increasing from last frame."; + return; + } + } + if (sender_->GetInFlightMediaDuration(work_unit.rtp_timestamp) > + sender_->GetMaxInFlightMediaDuration()) { + OSP_LOG_WARN << "VIDEO[" << sender_->ssrc() + << "] Dropping: In-flight media duration would be too high."; + return; + } + + Clock::duration frame_duration = frame.duration; + if (frame_duration <= Clock::duration::zero()) { + // The caller did not provide the frame duration in |frame|. + if (reference_time == start_time_) { + // Use the max for the first frame so libvpx will spend extra effort on + // its quality. + frame_duration = kMaxFrameDuration; + } else { + // Use the actual amount of time between the current and previous frame as + // a prediction for the next frame's duration. + frame_duration = + (work_unit.rtp_timestamp - last_enqueued_rtp_timestamp_) + .ToDuration(sender_->rtp_timebase()); + } + } + work_unit.duration = + std::max(std::min(frame_duration, kMaxFrameDuration), kMinFrameDuration); + + last_enqueued_rtp_timestamp_ = work_unit.rtp_timestamp; + + work_unit.image = CloneAsVpxImage(frame); + work_unit.reference_time = reference_time; + work_unit.stats_callback = std::move(stats_callback); + const bool force_key_frame = sender_->NeedsKeyFrame(); + { + std::unique_lock lock(mutex_); + needs_key_frame_ |= force_key_frame; + encode_queue_.push(std::move(work_unit)); + cv_.notify_one(); + } +} + +void StreamingVpxEncoder::DestroyEncoder() { + OSP_DCHECK_EQ(std::this_thread::get_id(), encode_thread_.get_id()); + + if (is_encoder_initialized()) { + vpx_codec_destroy(&encoder_); + // Flag that the encoder is not initialized. See header comments for + // is_encoder_initialized(). + config_.g_threads = 0; + } +} + +void StreamingVpxEncoder::ProcessWorkUnitsUntilTimeToQuit() { + OSP_DCHECK_EQ(std::this_thread::get_id(), encode_thread_.get_id()); + + for (;;) { + WorkUnitWithResults work_unit{}; + bool force_key_frame; + int target_bitrate; + { + std::unique_lock lock(mutex_); + if (target_bitrate_ <= 0) { + break; // Time to end this thread. + } + if (encode_queue_.empty()) { + cv_.wait(lock); + if (encode_queue_.empty()) { + continue; + } + } + static_cast(work_unit) = std::move(encode_queue_.front()); + encode_queue_.pop(); + force_key_frame = needs_key_frame_; + needs_key_frame_ = false; + target_bitrate = target_bitrate_; + } + + // Clock::now() is being called directly, instead of using a + // dependency-injected "now function," since actual wall time is being + // measured. + const Clock::time_point encode_start_time = Clock::now(); + PrepareEncoder(work_unit.image->d_w, work_unit.image->d_h, target_bitrate); + EncodeFrame(force_key_frame, &work_unit); + ComputeFrameEncodeStats(Clock::now() - encode_start_time, target_bitrate, + &work_unit); + UpdateSpeedSettingForNextFrame(work_unit.stats); + + main_task_runner_->PostTask( + [this, results = std::move(work_unit)]() mutable { + SendEncodedFrame(std::move(results)); + }); + } + + DestroyEncoder(); +} + +void StreamingVpxEncoder::PrepareEncoder(int width, + int height, + int target_bitrate) { + OSP_DCHECK_EQ(std::this_thread::get_id(), encode_thread_.get_id()); + + const int target_kbps = target_bitrate / kBytesPerKilobyte; + + // Translate the |ideal_speed_setting_| into the VP8E_SET_CPUUSED setting and + // the minimum quantizer to use. + int speed; + int min_quantizer; + if (ideal_speed_setting_ > kHighestEncodingSpeed) { + speed = kHighestEncodingSpeed; + const double remainder = ideal_speed_setting_ - speed; + min_quantizer = rounded_saturate_cast( + remainder / kEquivalentEncodingSpeedStepPerQuantizerStep + + params_.min_quantizer); + min_quantizer = std::min(min_quantizer, params_.max_cpu_saver_quantizer); + } else { + speed = std::max(rounded_saturate_cast(ideal_speed_setting_), + kLowestEncodingSpeed); + min_quantizer = params_.min_quantizer; + } + + if (static_cast(config_.g_w) != width || + static_cast(config_.g_h) != height) { + DestroyEncoder(); + } + + if (!is_encoder_initialized()) { + config_.g_threads = params_.num_encode_threads; + config_.g_w = width; + config_.g_h = height; + config_.rc_target_bitrate = target_kbps; + config_.rc_min_quantizer = min_quantizer; + + encoder_ = {}; + const vpx_codec_flags_t flags = 0; + + vpx_codec_iface_t* ctx; + if (params_.codec == VideoCodec::kVp9) { + ctx = vpx_codec_vp9_cx(); + } else { + OSP_DCHECK(params_.codec == VideoCodec::kVp8); + ctx = vpx_codec_vp8_cx(); + } + + const auto init_result = + vpx_codec_enc_init(&encoder_, ctx, &config_, flags); + OSP_CHECK_EQ(init_result, VPX_CODEC_OK); + + // Raise the threshold for considering macroblocks as static. The default is + // zero, so this setting makes the encoder less sensitive to motion. This + // lowers the probability of needing to utilize more CPU to search for + // motion vectors. + const auto ctl_result = + vpx_codec_control(&encoder_, VP8E_SET_STATIC_THRESHOLD, 1); + OSP_CHECK_EQ(ctl_result, VPX_CODEC_OK); + + // Ensure the speed will be set (below). + current_speed_setting_ = ~speed; + } else if (static_cast(config_.rc_target_bitrate) != target_kbps || + static_cast(config_.rc_min_quantizer) != min_quantizer) { + config_.rc_target_bitrate = target_kbps; + config_.rc_min_quantizer = min_quantizer; + const auto update_config_result = + vpx_codec_enc_config_set(&encoder_, &config_); + OSP_CHECK_EQ(update_config_result, VPX_CODEC_OK); + } + + if (current_speed_setting_ != speed) { + // Pass the |speed| as a negative value to turn off VP8/9's automatic speed + // selection logic and force the exact setting. + const auto ctl_result = + vpx_codec_control(&encoder_, VP8E_SET_CPUUSED, -speed); + OSP_CHECK_EQ(ctl_result, VPX_CODEC_OK); + current_speed_setting_ = speed; + } +} + +void StreamingVpxEncoder::EncodeFrame(bool force_key_frame, + WorkUnitWithResults* work_unit) { + OSP_DCHECK_EQ(std::this_thread::get_id(), encode_thread_.get_id()); + + // The presentation timestamp argument here is fixed to zero to force the + // encoder to base its single-frame bandwidth calculations entirely on + // |frame_duration| and the target bitrate setting. + const vpx_codec_pts_t pts = 0; + const vpx_enc_frame_flags_t flags = force_key_frame ? VPX_EFLAG_FORCE_KF : 0; + const auto encode_result = + vpx_codec_encode(&encoder_, work_unit->image.get(), pts, + work_unit->duration.count(), flags, VPX_DL_REALTIME); + OSP_CHECK_EQ(encode_result, VPX_CODEC_OK); + + const vpx_codec_cx_pkt_t* pkt; + for (vpx_codec_iter_t iter = nullptr;;) { + pkt = vpx_codec_get_cx_data(&encoder_, &iter); + // vpx_codec_get_cx_data() returns null once the "iteration" is complete. + // However, that point should never be reached because a + // VPX_CODEC_CX_FRAME_PKT must be encountered before that. + OSP_CHECK(pkt); + if (pkt->kind == VPX_CODEC_CX_FRAME_PKT) { + break; + } + } + + // A copy of the payload data is being made here. That's okay since it has to + // be copied at some point anyway, to be passed back to the main thread. + auto* const begin = static_cast(pkt->data.frame.buf); + auto* const end = begin + pkt->data.frame.sz; + work_unit->payload.assign(begin, end); + work_unit->is_key_frame = !!(pkt->data.frame.flags & VPX_FRAME_IS_KEY); +} + +void StreamingVpxEncoder::ComputeFrameEncodeStats( + Clock::duration encode_wall_time, + int target_bitrate, + WorkUnitWithResults* work_unit) { + OSP_DCHECK_EQ(std::this_thread::get_id(), encode_thread_.get_id()); + + Stats& stats = work_unit->stats; + + // Note: stats.frame_id is set later, in SendEncodedFrame(). + stats.rtp_timestamp = work_unit->rtp_timestamp; + stats.encode_wall_time = encode_wall_time; + stats.frame_duration = work_unit->duration; + stats.encoded_size = work_unit->payload.size(); + + constexpr double kBytesPerBit = 1.0 / CHAR_BIT; + constexpr double kSecondsPerClockTick = + 1.0 / Clock::to_duration(seconds(1)).count(); + const double target_bytes_per_clock_tick = + target_bitrate * (kBytesPerBit * kSecondsPerClockTick); + stats.target_size = target_bytes_per_clock_tick * work_unit->duration.count(); + + // The quantizer the encoder used. This is the result of the VP8/9 encoder + // taking a guess at what quantizer value would produce an encoded frame size + // as close to the target as possible. + const auto get_quantizer_result = vpx_codec_control( + &encoder_, VP8E_GET_LAST_QUANTIZER_64, &stats.quantizer); + OSP_CHECK_EQ(get_quantizer_result, VPX_CODEC_OK); + + // Now that the frame has been encoded and the number of bytes is known, the + // perfect quantizer value (i.e., the one that should have been used) can be + // determined. + stats.perfect_quantizer = stats.quantizer * stats.space_utilization(); +} + +void StreamingVpxEncoder::UpdateSpeedSettingForNextFrame(const Stats& stats) { + OSP_DCHECK_EQ(std::this_thread::get_id(), encode_thread_.get_id()); + + // Combine the speed setting that was used to encode the last frame, and the + // quantizer the encoder chose into a single speed metric. + const double speed = current_speed_setting_ + + kEquivalentEncodingSpeedStepPerQuantizerStep * + std::max(0, stats.quantizer - params_.min_quantizer); + + // Like |Stats::perfect_quantizer|, this computes a "hindsight" speed setting + // for the last frame, one that may have potentially allowed for a + // better-quality quantizer choice by the encoder, while also keeping CPU + // utilization within budget. + const double perfect_speed = + speed * stats.time_utilization() / params_.max_time_utilization; + + // Update the ideal speed setting, to be used for the next frame. An + // exponentially-decaying weighted average is used here to smooth-out noise. + // The weight is based on the duration of the frame that was encoded. + constexpr Clock::duration kDecayHalfLife = milliseconds(120); + const double ticks = stats.frame_duration.count(); + const double weight = ticks / (ticks + kDecayHalfLife.count()); + ideal_speed_setting_ = + weight * perfect_speed + (1.0 - weight) * ideal_speed_setting_; + OSP_DCHECK(std::isfinite(ideal_speed_setting_)); +} + +void StreamingVpxEncoder::SendEncodedFrame(WorkUnitWithResults results) { + OSP_DCHECK(main_task_runner_->IsRunningOnTaskRunner()); + + EncodedFrame frame; + frame.frame_id = sender_->GetNextFrameId(); + if (results.is_key_frame) { + frame.dependency = EncodedFrame::KEY_FRAME; + frame.referenced_frame_id = frame.frame_id; + } else { + frame.dependency = EncodedFrame::DEPENDS_ON_ANOTHER; + frame.referenced_frame_id = frame.frame_id - 1; + } + frame.rtp_timestamp = results.rtp_timestamp; + frame.reference_time = results.reference_time; + frame.data = absl::Span(results.payload); + + if (sender_->EnqueueFrame(frame) != Sender::OK) { + // Since the frame will not be sent, the encoder's frame dependency chain + // has been broken. Force a key frame for the next frame. + std::unique_lock lock(mutex_); + needs_key_frame_ = true; + } + + if (results.stats_callback) { + results.stats.frame_id = frame.frame_id; + results.stats_callback(results.stats); + } +} + +namespace { +void CopyPlane(const uint8_t* src, + int src_stride, + int num_rows, + uint8_t* dst, + int dst_stride) { + if (src_stride == dst_stride) { + memcpy(dst, src, src_stride * num_rows); + return; + } + const int bytes_per_row = std::min(src_stride, dst_stride); + while (--num_rows >= 0) { + memcpy(dst, src, bytes_per_row); + dst += dst_stride; + src += src_stride; + } +} +} // namespace + +// static +StreamingVpxEncoder::VpxImageUniquePtr StreamingVpxEncoder::CloneAsVpxImage( + const VideoFrame& frame) { + OSP_DCHECK_GE(frame.width, 0); + OSP_DCHECK_GE(frame.height, 0); + OSP_DCHECK_GE(frame.yuv_strides[0], 0); + OSP_DCHECK_GE(frame.yuv_strides[1], 0); + OSP_DCHECK_GE(frame.yuv_strides[2], 0); + + constexpr int kAlignment = 32; + VpxImageUniquePtr image(vpx_img_alloc(nullptr, VPX_IMG_FMT_I420, frame.width, + frame.height, kAlignment)); + OSP_CHECK(image); + + CopyPlane(frame.yuv_planes[0], frame.yuv_strides[0], frame.height, + image->planes[VPX_PLANE_Y], image->stride[VPX_PLANE_Y]); + CopyPlane(frame.yuv_planes[1], frame.yuv_strides[1], (frame.height + 1) / 2, + image->planes[VPX_PLANE_U], image->stride[VPX_PLANE_U]); + CopyPlane(frame.yuv_planes[2], frame.yuv_strides[2], (frame.height + 1) / 2, + image->planes[VPX_PLANE_V], image->stride[VPX_PLANE_V]); + + return image; +} + +} // namespace cast +} // namespace openscreen diff --git a/cast/standalone_sender/streaming_vpx_encoder.h b/cast/standalone_sender/streaming_vpx_encoder.h new file mode 100644 index 00000000..6935efdd --- /dev/null +++ b/cast/standalone_sender/streaming_vpx_encoder.h @@ -0,0 +1,307 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef CAST_STANDALONE_SENDER_STREAMING_VPX_ENCODER_H_ +#define CAST_STANDALONE_SENDER_STREAMING_VPX_ENCODER_H_ + +#include +#include + +#include +#include // NOLINT +#include +#include +#include +#include +#include +#include + +#include "absl/base/thread_annotations.h" +#include "cast/streaming/constants.h" +#include "cast/streaming/frame_id.h" +#include "cast/streaming/rtp_time.h" +#include "platform/api/task_runner.h" +#include "platform/api/time.h" + +namespace openscreen { + +class TaskRunner; + +namespace cast { + +class Sender; + +// Uses libvpx to encode VP8/9 video and streams it to a Sender. Includes +// extensive logic for fine-tuning the encoder parameters in real-time, to +// provide the best quality results given external, uncontrollable factors: +// CPU/network availability, and the complexity of the video frame content. +// +// Internally, a separate encode thread is created and used to prevent blocking +// the main thread while frames are being encoded. All public API methods are +// assumed to be called on the same sequence/thread as the main TaskRunner +// (injected via the constructor). +// +// Usage: +// +// 1. EncodeAndSend() is used to queue-up video frames for encoding and sending, +// which will be done on a best-effort basis. +// +// 2. The client is expected to call SetTargetBitrate() frequently based on its +// own bandwidth estimates and congestion control logic. In addition, a client +// may provide a callback for each frame's encode statistics, which can be used +// to further optimize the user experience. For example, the stats can be used +// as a signal to reduce the data volume (i.e., resolution and/or frame rate) +// coming from the video capture source. +class StreamingVpxEncoder { + public: + // Configurable parameters passed to the StreamingVpxEncoder constructor. + struct Parameters { + // Number of threads to parallelize frame encoding. This should be set based + // on the number of CPU cores available for encoding, but no more than 8. + int num_encode_threads = + std::min(std::max(std::thread::hardware_concurrency(), 1), 8); + + // Best-quality quantizer (lower is better quality). Range: [0,63] + int min_quantizer = 4; + + // Worst-quality quantizer (lower is better quality). Range: [0,63] + int max_quantizer = 63; + + // Worst-quality quantizer to use when the CPU is extremely constrained. + // Range: [min_quantizer,max_quantizer] + int max_cpu_saver_quantizer = 25; + + // Maximum amount of wall-time a frame's encode can take, relative to the + // frame's duration, before the CPU-saver logic is activated. The default + // (70%) is appropriate for systems with four or more cores, but should be + // reduced (e.g., 50%) for systems with fewer than three cores. + // + // Example: For 30 FPS (continuous) video, the frame duration is ~33.3ms, + // and a value of 0.5 here would mean that the CPU-saver logic starts + // sacrificing quality when frame encodes start taking longer than ~16.7ms. + double max_time_utilization = 0.7; + + // Determines which codec (VP8 or VP9) is to be used for encoding. Defaults + // to VP8. + VideoCodec codec = VideoCodec::kVp8; + }; + + // Represents an input VideoFrame, passed to EncodeAndSend(). + struct VideoFrame { + // Image width and height. + int width; + int height; + + // I420 format image pointers and row strides (the number of bytes between + // the start of successive rows). The pointers only need to remain valid + // until the EncodeAndSend() call returns. + const uint8_t* yuv_planes[3]; + int yuv_strides[3]; + + // How long this frame will be held before the next frame will be displayed, + // or zero if unknown. The frame duration is passed to the VP8/9 codec, + // affecting a number of important behaviors, including: per-frame + // bandwidth, CPU time spent encoding, temporal quality trade-offs, and + // key/golden/alt-ref frame generation intervals. + Clock::duration duration; + }; + + // Performance statistics for a single frame's encode. + // + // For full details on how to use these stats in an end-to-end system, see: + // https://www.chromium.org/developers/design-documents/ + // auto-throttled-screen-capture-and-mirroring + // and https://source.chromium.org/chromium/chromium/src/+/master: + // media/cast/sender/performance_metrics_overlay.h + struct Stats { + // The Cast Streaming ID that was assigned to the frame. + FrameId frame_id; + + // The RTP timestamp of the frame. + RtpTimeTicks rtp_timestamp; + + // How long the frame took to encode. This is wall time, not CPU time or + // some other load metric. + Clock::duration encode_wall_time; + + // The frame's predicted duration; or, the actual duration if it was + // provided in the VideoFrame. + Clock::duration frame_duration; + + // The encoded frame's size in bytes. + int encoded_size; + + // The average size of an encoded frame in bytes, having this + // |frame_duration| and current target bitrate. + double target_size; + + // The actual quantizer the VP8 encoder used, in the range [0,63]. + int quantizer; + + // The "hindsight" quantizer value that would have produced the best quality + // encoding of the frame at the current target bitrate. The nominal range is + // [0.0,63.0]. If it is larger than 63.0, then it was impossible for VP8 to + // encode the frame within the current target bitrate (e.g., too much + // "entropy" in the image, or too low a target bitrate). + double perfect_quantizer; + + // Utilization feedback metrics. The nominal range for each of these is + // [0.0,1.0] where 1.0 means "the entire budget available for the frame was + // exhausted." Going above 1.0 is okay for one or a few frames, since it's + // the average over many frames that matters before the system is considered + // "redlining." + // + // The max of these three provides an overall utilization control signal. + // The usual approach is for upstream control logic to increase/decrease the + // data volume (e.g., video resolution and/or frame rate) to maintain a good + // target point. + double time_utilization() const { + return static_cast(encode_wall_time.count()) / + frame_duration.count(); + } + double space_utilization() const { return encoded_size / target_size; } + double entropy_utilization() const { + return perfect_quantizer / kMaxQuantizer; + } + }; + + StreamingVpxEncoder(const Parameters& params, + TaskRunner* task_runner, + Sender* sender); + + ~StreamingVpxEncoder(); + + // Get/Set the target bitrate. This may be changed at any time, as frequently + // as desired, and it will take effect internally as soon as possible. + int GetTargetBitrate() const; + void SetTargetBitrate(int new_bitrate); + + // Encode |frame| using the VP8 encoder, assemble an EncodedFrame, and enqueue + // into the Sender. The frame may be dropped if too many frames are in-flight. + // If provided, the |stats_callback| is run after the frame is enqueued in the + // Sender (via the main TaskRunner). + void EncodeAndSend(const VideoFrame& frame, + Clock::time_point reference_time, + std::function stats_callback); + + static constexpr int kMinQuantizer = 0; + static constexpr int kMaxQuantizer = 63; + + private: + // Syntactic convenience to wrap the vpx_image_t alloc/free API in a smart + // pointer. + struct VpxImageDeleter { + void operator()(vpx_image_t* ptr) const { vpx_img_free(ptr); } + }; + using VpxImageUniquePtr = std::unique_ptr; + + // Represents the state of one frame encode. This is created in + // EncodeAndSend(), and passed to the encode thread via the |encode_queue_|. + struct WorkUnit { + VpxImageUniquePtr image; + Clock::duration duration; + Clock::time_point reference_time; + RtpTimeTicks rtp_timestamp; + std::function stats_callback; + }; + + // Same as WorkUnit, but with additional fields to carry the encode results. + struct WorkUnitWithResults : public WorkUnit { + std::vector payload; + bool is_key_frame; + Stats stats; + }; + + bool is_encoder_initialized() const { return config_.g_threads != 0; } + + // Destroys the VP8 encoder context if it has been initialized. + void DestroyEncoder(); + + // The procedure for the |encode_thread_| that loops, processing work units + // from the |encode_queue_| by calling Encode() until it's time to end the + // thread. + void ProcessWorkUnitsUntilTimeToQuit(); + + // If the |encoder_| is live, attempt reconfiguration to allow it to encode + // frames at a new frame size, target bitrate, or "CPU encoding speed." If + // reconfiguration is not possible, destroy the existing instance and + // re-create a new |encoder_| instance. + void PrepareEncoder(int width, int height, int target_bitrate); + + // Wraps the complex libvpx vpx_codec_encode() call using inputs from + // |work_unit| and populating results there. + void EncodeFrame(bool force_key_frame, WorkUnitWithResults* work_unit); + + // Computes and populates |work_unit.stats| after the last call to + // EncodeFrame(). + void ComputeFrameEncodeStats(Clock::duration encode_wall_time, + int target_bitrate, + WorkUnitWithResults* work_unit); + + // Updates the |ideal_speed_setting_|, to take effect with the next frame + // encode, based on the given performance |stats|. + void UpdateSpeedSettingForNextFrame(const Stats& stats); + + // Assembles and enqueues an EncodedFrame with the Sender on the main thread. + void SendEncodedFrame(WorkUnitWithResults results); + + // Allocates a vpx_image_t and copies the content from |frame| to it. + static VpxImageUniquePtr CloneAsVpxImage(const VideoFrame& frame); + + const Parameters params_; + TaskRunner* const main_task_runner_; + Sender* const sender_; + + // The reference time of the first frame passed to EncodeAndSend(). + Clock::time_point start_time_ = Clock::time_point::min(); + + // The RTP timestamp of the last frame that was pushed into the + // |encode_queue_| by EncodeAndSend(). This is used to check whether + // timestamps are monotonically increasing. + RtpTimeTicks last_enqueued_rtp_timestamp_; + + // Guards a few members shared by both the main and encode threads. + std::mutex mutex_; + + // Used by the encode thread to sleep until more work is available. + std::condition_variable cv_ ABSL_GUARDED_BY(mutex_); + + // These encode parameters not passed in the WorkUnit struct because it is + // desirable for them to be applied as soon as possible, with the very next + // WorkUnit popped from the |encode_queue_| on the encode thread, and not to + // wait until some later WorkUnit is processed. + bool needs_key_frame_ ABSL_GUARDED_BY(mutex_) = true; + int target_bitrate_ ABSL_GUARDED_BY(mutex_) = 2 << 20; // Default: 2 Mbps. + + // The queue of frame encodes. The size of this queue is implicitly bounded by + // EncodeAndSend(), where it checks for the total in-flight media duration and + // maybe drops a frame. + std::queue encode_queue_ ABSL_GUARDED_BY(mutex_); + + // Current VP8 encoder configuration. Most of the fields are unchanging, and + // are populated in the ctor; but thereafter, only the encode thread accesses + // this struct. + // + // The speed setting is controlled via a separate libvpx API (see members + // below). + vpx_codec_enc_cfg_t config_{}; + + // These represent the magnitude of the VP8 speed setting, where larger values + // (i.e., faster speed) request less CPU usage but will provide lower video + // quality. Only the encode thread accesses these. + double ideal_speed_setting_; // A time-weighted average, from measurements. + int current_speed_setting_; // Current |encoder_| speed setting. + + // libvpx VP8/9 encoder instance. Only the encode thread accesses this. + vpx_codec_ctx_t encoder_; + + // This member should be last in the class since the thread should not start + // until all above members have been initialized by the constructor. + std::thread encode_thread_; +}; + +} // namespace cast +} // namespace openscreen + +#endif // CAST_STANDALONE_SENDER_STREAMING_VPX_ENCODER_H_ diff --git a/cast/streaming/receiver_message.cc b/cast/streaming/receiver_message.cc index 4fb1a8a6..7f0999f0 100644 --- a/cast/streaming/receiver_message.cc +++ b/cast/streaming/receiver_message.cc @@ -27,17 +27,17 @@ EnumNameTable kMessageTypeNames{ {"CAPABILITIES_RESPONSE", ReceiverMessage::Type::kCapabilitiesResponse}, {"RPC", ReceiverMessage::Type::kRpc}}}; -EnumNameTable kMediaCapabilityNames{{ - {"audio", MediaCapability::kAudio}, - {"aac", MediaCapability::kAac}, - {"opus", MediaCapability::kOpus}, - {"video", MediaCapability::kVideo}, - {"4k", MediaCapability::k4k}, - {"h264", MediaCapability::kH264}, - {"vp8", MediaCapability::kVp8}, - {"vp9", MediaCapability::kVp9}, - {"hevc", MediaCapability::kHevc}, -}}; +EnumNameTable kMediaCapabilityNames{ + {{"audio", MediaCapability::kAudio}, + {"aac", MediaCapability::kAac}, + {"opus", MediaCapability::kOpus}, + {"video", MediaCapability::kVideo}, + {"4k", MediaCapability::k4k}, + {"h264", MediaCapability::kH264}, + {"vp8", MediaCapability::kVp8}, + {"vp9", MediaCapability::kVp9}, + {"hevc", MediaCapability::kHevc}, + {"av1", MediaCapability::kAv1}}}; ReceiverMessage::Type GetMessageType(const Json::Value& root) { std::string type; diff --git a/cast/streaming/receiver_message.h b/cast/streaming/receiver_message.h index e25a64bd..f4adbfb3 100644 --- a/cast/streaming/receiver_message.h +++ b/cast/streaming/receiver_message.h @@ -26,7 +26,8 @@ enum class MediaCapability { kH264, kVp8, kVp9, - kHevc + kHevc, + kAv1 }; struct ReceiverCapability { diff --git a/cast/streaming/receiver_session.cc b/cast/streaming/receiver_session.cc index b55784b2..bda6d984 100644 --- a/cast/streaming/receiver_session.cc +++ b/cast/streaming/receiver_session.cc @@ -68,6 +68,8 @@ MediaCapability ToCapability(VideoCodec codec) { return MediaCapability::kH264; case VideoCodec::kHevc: return MediaCapability::kHevc; + case VideoCodec::kAv1: + return MediaCapability::kAv1; default: OSP_DLOG_FATAL << "Invalid video codec: " << static_cast(codec); OSP_NOTREACHED(); -- cgit v1.2.3