diff options
Diffstat (limited to 'webrtc/modules/video_coding/codecs/vp9')
9 files changed, 851 insertions, 147 deletions
diff --git a/webrtc/modules/video_coding/codecs/vp9/include/vp9.h b/webrtc/modules/video_coding/codecs/vp9/include/vp9.h index cd77f72dcb..3bcbe46b3a 100644 --- a/webrtc/modules/video_coding/codecs/vp9/include/vp9.h +++ b/webrtc/modules/video_coding/codecs/vp9/include/vp9.h @@ -12,7 +12,7 @@ #ifndef WEBRTC_MODULES_VIDEO_CODING_CODECS_VP9_INCLUDE_VP9_H_ #define WEBRTC_MODULES_VIDEO_CODING_CODECS_VP9_INCLUDE_VP9_H_ -#include "webrtc/modules/video_coding/codecs/interface/video_codec_interface.h" +#include "webrtc/modules/video_coding/include/video_codec_interface.h" namespace webrtc { @@ -23,7 +23,6 @@ class VP9Encoder : public VideoEncoder { virtual ~VP9Encoder() {} }; - class VP9Decoder : public VideoDecoder { public: static VP9Decoder* Create(); diff --git a/webrtc/modules/video_coding/codecs/vp9/screenshare_layers.cc b/webrtc/modules/video_coding/codecs/vp9/screenshare_layers.cc new file mode 100644 index 0000000000..c7ed78a192 --- /dev/null +++ b/webrtc/modules/video_coding/codecs/vp9/screenshare_layers.cc @@ -0,0 +1,93 @@ +/* Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. +* +* Use of this source code is governed by a BSD-style license +* that can be found in the LICENSE file in the root of the source +* tree. An additional intellectual property rights grant can be found +* in the file PATENTS. All contributing project authors may +* be found in the AUTHORS file in the root of the source tree. +*/ + +#include <algorithm> +#include "webrtc/modules/video_coding/codecs/vp9/screenshare_layers.h" +#include "webrtc/base/checks.h" + +namespace webrtc { + +ScreenshareLayersVP9::ScreenshareLayersVP9(uint8_t num_layers) + : num_layers_(num_layers), + start_layer_(0), + last_timestamp_(0), + timestamp_initialized_(false) { + RTC_DCHECK_GT(num_layers, 0); + RTC_DCHECK_LE(num_layers, kMaxVp9NumberOfSpatialLayers); + memset(bits_used_, 0, sizeof(bits_used_)); + memset(threshold_kbps_, 0, sizeof(threshold_kbps_)); +} + +uint8_t ScreenshareLayersVP9::GetStartLayer() const { + return start_layer_; +} + +void ScreenshareLayersVP9::ConfigureBitrate(int threshold_kbps, + uint8_t layer_id) { + // The upper layer is always the layer we spill frames + // to when the bitrate becomes to high, therefore setting + // a max limit is not allowed. The top layer bitrate is + // never used either so configuring it makes no difference. + RTC_DCHECK_LT(layer_id, num_layers_ - 1); + threshold_kbps_[layer_id] = threshold_kbps; +} + +void ScreenshareLayersVP9::LayerFrameEncoded(unsigned int size_bytes, + uint8_t layer_id) { + RTC_DCHECK_LT(layer_id, num_layers_); + bits_used_[layer_id] += size_bytes * 8; +} + +VP9EncoderImpl::SuperFrameRefSettings +ScreenshareLayersVP9::GetSuperFrameSettings(uint32_t timestamp, + bool is_keyframe) { + VP9EncoderImpl::SuperFrameRefSettings settings; + if (!timestamp_initialized_) { + last_timestamp_ = timestamp; + timestamp_initialized_ = true; + } + float time_diff = (timestamp - last_timestamp_) / 90.f; + float total_bits_used = 0; + float total_threshold_kbps = 0; + start_layer_ = 0; + + // Up to (num_layers - 1) because we only have + // (num_layers - 1) thresholds to check. + for (int layer_id = 0; layer_id < num_layers_ - 1; ++layer_id) { + bits_used_[layer_id] = std::max( + 0.f, bits_used_[layer_id] - time_diff * threshold_kbps_[layer_id]); + total_bits_used += bits_used_[layer_id]; + total_threshold_kbps += threshold_kbps_[layer_id]; + + // If this is a keyframe then there should be no + // references to any previous frames. + if (!is_keyframe) { + settings.layer[layer_id].ref_buf1 = layer_id; + if (total_bits_used > total_threshold_kbps * 1000) + start_layer_ = layer_id + 1; + } + + settings.layer[layer_id].upd_buf = layer_id; + } + // Since the above loop does not iterate over the last layer + // the reference of the last layer has to be set after the loop, + // and if this is a keyframe there should be no references to + // any previous frames. + if (!is_keyframe) + settings.layer[num_layers_ - 1].ref_buf1 = num_layers_ - 1; + + settings.layer[num_layers_ - 1].upd_buf = num_layers_ - 1; + settings.is_keyframe = is_keyframe; + settings.start_layer = start_layer_; + settings.stop_layer = num_layers_ - 1; + last_timestamp_ = timestamp; + return settings; +} + +} // namespace webrtc diff --git a/webrtc/modules/video_coding/codecs/vp9/screenshare_layers.h b/webrtc/modules/video_coding/codecs/vp9/screenshare_layers.h new file mode 100644 index 0000000000..5a901ae359 --- /dev/null +++ b/webrtc/modules/video_coding/codecs/vp9/screenshare_layers.h @@ -0,0 +1,66 @@ +/* Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. +* +* Use of this source code is governed by a BSD-style license +* that can be found in the LICENSE file in the root of the source +* tree. An additional intellectual property rights grant can be found +* in the file PATENTS. All contributing project authors may +* be found in the AUTHORS file in the root of the source tree. +*/ + +#ifndef WEBRTC_MODULES_VIDEO_CODING_CODECS_VP9_SCREENSHARE_LAYERS_H_ +#define WEBRTC_MODULES_VIDEO_CODING_CODECS_VP9_SCREENSHARE_LAYERS_H_ + +#include "webrtc/modules/video_coding/codecs/vp9/vp9_impl.h" + +namespace webrtc { + +class ScreenshareLayersVP9 { + public: + explicit ScreenshareLayersVP9(uint8_t num_layers); + + // The target bitrate for layer with id layer_id. + void ConfigureBitrate(int threshold_kbps, uint8_t layer_id); + + // The current start layer. + uint8_t GetStartLayer() const; + + // Update the layer with the size of the layer frame. + void LayerFrameEncoded(unsigned int size_bytes, uint8_t layer_id); + + // Get the layer settings for the next superframe. + // + // In short, each time the GetSuperFrameSettings is called the + // bitrate of every layer is calculated and if the cummulative + // bitrate exceeds the configured cummulative bitrates + // (ConfigureBitrate to configure) up to and including that + // layer then the resulting encoding settings for the + // superframe will only encode layers above that layer. + VP9EncoderImpl::SuperFrameRefSettings GetSuperFrameSettings( + uint32_t timestamp, + bool is_keyframe); + + private: + // How many layers that are used. + uint8_t num_layers_; + + // The index of the first layer to encode. + uint8_t start_layer_; + + // Cummulative target kbps for the different layers. + float threshold_kbps_[kMaxVp9NumberOfSpatialLayers - 1]; + + // How many bits that has been used for a certain layer. Increased in + // FrameEncoded() by the size of the encoded frame and decreased in + // GetSuperFrameSettings() depending on the time between frames. + float bits_used_[kMaxVp9NumberOfSpatialLayers]; + + // Timestamp of last frame. + uint32_t last_timestamp_; + + // If the last_timestamp_ has been set. + bool timestamp_initialized_; +}; + +} // namespace webrtc + +#endif // WEBRTC_MODULES_VIDEO_CODING_CODECS_VP9_SCREENSHARE_LAYERS_H_ diff --git a/webrtc/modules/video_coding/codecs/vp9/screenshare_layers_unittest.cc b/webrtc/modules/video_coding/codecs/vp9/screenshare_layers_unittest.cc new file mode 100644 index 0000000000..5eb7b237ac --- /dev/null +++ b/webrtc/modules/video_coding/codecs/vp9/screenshare_layers_unittest.cc @@ -0,0 +1,323 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include <limits> + +#include "testing/gtest/include/gtest/gtest.h" +#include "vpx/vp8cx.h" +#include "webrtc/base/logging.h" +#include "webrtc/modules/video_coding/codecs/vp9/screenshare_layers.h" +#include "webrtc/modules/video_coding/codecs/vp9/vp9_impl.h" +#include "webrtc/system_wrappers/include/clock.h" + +namespace webrtc { + +typedef VP9EncoderImpl::SuperFrameRefSettings Settings; + +const uint32_t kTickFrequency = 90000; + +class ScreenshareLayerTestVP9 : public ::testing::Test { + protected: + ScreenshareLayerTestVP9() : clock_(0) {} + virtual ~ScreenshareLayerTestVP9() {} + + void InitScreenshareLayers(int layers) { + layers_.reset(new ScreenshareLayersVP9(layers)); + } + + void ConfigureBitrateForLayer(int kbps, uint8_t layer_id) { + layers_->ConfigureBitrate(kbps, layer_id); + } + + void AdvanceTime(int64_t milliseconds) { + clock_.AdvanceTimeMilliseconds(milliseconds); + } + + void AddKilobitsToLayer(int kilobits, uint8_t layer_id) { + layers_->LayerFrameEncoded(kilobits * 1000 / 8, layer_id); + } + + void EqualRefsForLayer(const Settings& actual, uint8_t layer_id) { + EXPECT_EQ(expected_.layer[layer_id].upd_buf, + actual.layer[layer_id].upd_buf); + EXPECT_EQ(expected_.layer[layer_id].ref_buf1, + actual.layer[layer_id].ref_buf1); + EXPECT_EQ(expected_.layer[layer_id].ref_buf2, + actual.layer[layer_id].ref_buf2); + EXPECT_EQ(expected_.layer[layer_id].ref_buf3, + actual.layer[layer_id].ref_buf3); + } + + void EqualRefs(const Settings& actual) { + for (unsigned int layer_id = 0; layer_id < kMaxVp9NumberOfSpatialLayers; + ++layer_id) { + EqualRefsForLayer(actual, layer_id); + } + } + + void EqualStartStopKeyframe(const Settings& actual) { + EXPECT_EQ(expected_.start_layer, actual.start_layer); + EXPECT_EQ(expected_.stop_layer, actual.stop_layer); + EXPECT_EQ(expected_.is_keyframe, actual.is_keyframe); + } + + // Check that the settings returned by GetSuperFrameSettings() is + // equal to the expected_ settings. + void EqualToExpected() { + uint32_t frame_timestamp_ = + clock_.TimeInMilliseconds() * (kTickFrequency / 1000); + Settings actual = + layers_->GetSuperFrameSettings(frame_timestamp_, expected_.is_keyframe); + EqualRefs(actual); + EqualStartStopKeyframe(actual); + } + + Settings expected_; + SimulatedClock clock_; + rtc::scoped_ptr<ScreenshareLayersVP9> layers_; +}; + +TEST_F(ScreenshareLayerTestVP9, NoRefsOnKeyFrame) { + const int kNumLayers = kMaxVp9NumberOfSpatialLayers; + InitScreenshareLayers(kNumLayers); + expected_.start_layer = 0; + expected_.stop_layer = kNumLayers - 1; + + for (int l = 0; l < kNumLayers; ++l) { + expected_.layer[l].upd_buf = l; + } + expected_.is_keyframe = true; + EqualToExpected(); + + for (int l = 0; l < kNumLayers; ++l) { + expected_.layer[l].ref_buf1 = l; + } + expected_.is_keyframe = false; + EqualToExpected(); +} + +// Test if it is possible to send at a high bitrate (over the threshold) +// after a longer period of low bitrate. This should not be possible. +TEST_F(ScreenshareLayerTestVP9, DontAccumelateAvailableBitsOverTime) { + InitScreenshareLayers(2); + ConfigureBitrateForLayer(100, 0); + + expected_.layer[0].upd_buf = 0; + expected_.layer[0].ref_buf1 = 0; + expected_.layer[1].upd_buf = 1; + expected_.layer[1].ref_buf1 = 1; + expected_.start_layer = 0; + expected_.stop_layer = 1; + + // Send 10 frames at a low bitrate (50 kbps) + for (int i = 0; i < 10; ++i) { + AdvanceTime(200); + EqualToExpected(); + AddKilobitsToLayer(10, 0); + } + + AdvanceTime(200); + EqualToExpected(); + AddKilobitsToLayer(301, 0); + + // Send 10 frames at a high bitrate (200 kbps) + expected_.start_layer = 1; + for (int i = 0; i < 10; ++i) { + AdvanceTime(200); + EqualToExpected(); + AddKilobitsToLayer(40, 1); + } +} + +// Test if used bits are accumelated over layers, as they should; +TEST_F(ScreenshareLayerTestVP9, AccumelateUsedBitsOverLayers) { + const int kNumLayers = kMaxVp9NumberOfSpatialLayers; + InitScreenshareLayers(kNumLayers); + for (int l = 0; l < kNumLayers - 1; ++l) + ConfigureBitrateForLayer(100, l); + for (int l = 0; l < kNumLayers; ++l) { + expected_.layer[l].upd_buf = l; + expected_.layer[l].ref_buf1 = l; + } + + expected_.start_layer = 0; + expected_.stop_layer = kNumLayers - 1; + EqualToExpected(); + + for (int layer = 0; layer < kNumLayers - 1; ++layer) { + expected_.start_layer = layer; + EqualToExpected(); + AddKilobitsToLayer(101, layer); + } +} + +// General testing of the bitrate controller. +TEST_F(ScreenshareLayerTestVP9, 2LayerBitrate) { + InitScreenshareLayers(2); + ConfigureBitrateForLayer(100, 0); + + expected_.layer[0].upd_buf = 0; + expected_.layer[1].upd_buf = 1; + expected_.layer[0].ref_buf1 = -1; + expected_.layer[1].ref_buf1 = -1; + expected_.start_layer = 0; + expected_.stop_layer = 1; + + expected_.is_keyframe = true; + EqualToExpected(); + AddKilobitsToLayer(100, 0); + + expected_.layer[0].ref_buf1 = 0; + expected_.layer[1].ref_buf1 = 1; + expected_.is_keyframe = false; + AdvanceTime(199); + EqualToExpected(); + AddKilobitsToLayer(100, 0); + + expected_.start_layer = 1; + for (int frame = 0; frame < 3; ++frame) { + AdvanceTime(200); + EqualToExpected(); + AddKilobitsToLayer(100, 1); + } + + // Just before enough bits become available for L0 @0.999 seconds. + AdvanceTime(199); + EqualToExpected(); + AddKilobitsToLayer(100, 1); + + // Just after enough bits become available for L0 @1.0001 seconds. + expected_.start_layer = 0; + AdvanceTime(2); + EqualToExpected(); + AddKilobitsToLayer(100, 0); + + // Keyframes always encode all layers, even if it is over budget. + expected_.layer[0].ref_buf1 = -1; + expected_.layer[1].ref_buf1 = -1; + expected_.is_keyframe = true; + AdvanceTime(499); + EqualToExpected(); + expected_.layer[0].ref_buf1 = 0; + expected_.layer[1].ref_buf1 = 1; + expected_.start_layer = 1; + expected_.is_keyframe = false; + EqualToExpected(); + AddKilobitsToLayer(100, 0); + + // 400 kb in L0 --> @3 second mark to fall below the threshold.. + // just before @2.999 seconds. + expected_.is_keyframe = false; + AdvanceTime(1499); + EqualToExpected(); + AddKilobitsToLayer(100, 1); + + // just after @3.001 seconds. + expected_.start_layer = 0; + AdvanceTime(2); + EqualToExpected(); + AddKilobitsToLayer(100, 0); +} + +// General testing of the bitrate controller. +TEST_F(ScreenshareLayerTestVP9, 3LayerBitrate) { + InitScreenshareLayers(3); + ConfigureBitrateForLayer(100, 0); + ConfigureBitrateForLayer(100, 1); + + for (int l = 0; l < 3; ++l) { + expected_.layer[l].upd_buf = l; + expected_.layer[l].ref_buf1 = l; + } + expected_.start_layer = 0; + expected_.stop_layer = 2; + + EqualToExpected(); + AddKilobitsToLayer(105, 0); + AddKilobitsToLayer(30, 1); + + AdvanceTime(199); + EqualToExpected(); + AddKilobitsToLayer(105, 0); + AddKilobitsToLayer(30, 1); + + expected_.start_layer = 1; + AdvanceTime(200); + EqualToExpected(); + AddKilobitsToLayer(130, 1); + + expected_.start_layer = 2; + AdvanceTime(200); + EqualToExpected(); + + // 400 kb in L1 --> @1.0 second mark to fall below threshold. + // 210 kb in L0 --> @1.1 second mark to fall below threshold. + // Just before L1 @0.999 seconds. + AdvanceTime(399); + EqualToExpected(); + + // Just after L1 @1.001 seconds. + expected_.start_layer = 1; + AdvanceTime(2); + EqualToExpected(); + + // Just before L0 @1.099 seconds. + AdvanceTime(99); + EqualToExpected(); + + // Just after L0 @1.101 seconds. + expected_.start_layer = 0; + AdvanceTime(2); + EqualToExpected(); + + // @1.1 seconds + AdvanceTime(99); + EqualToExpected(); + AddKilobitsToLayer(200, 1); + + expected_.is_keyframe = true; + for (int l = 0; l < 3; ++l) + expected_.layer[l].ref_buf1 = -1; + AdvanceTime(200); + EqualToExpected(); + + expected_.is_keyframe = false; + expected_.start_layer = 2; + for (int l = 0; l < 3; ++l) + expected_.layer[l].ref_buf1 = l; + AdvanceTime(200); + EqualToExpected(); +} + +// Test that the bitrate calculations are +// correct when the timestamp wrap. +TEST_F(ScreenshareLayerTestVP9, TimestampWrap) { + InitScreenshareLayers(2); + ConfigureBitrateForLayer(100, 0); + + expected_.layer[0].upd_buf = 0; + expected_.layer[0].ref_buf1 = 0; + expected_.layer[1].upd_buf = 1; + expected_.layer[1].ref_buf1 = 1; + expected_.start_layer = 0; + expected_.stop_layer = 1; + + // Advance time to just before the timestamp wraps. + AdvanceTime(std::numeric_limits<uint32_t>::max() / (kTickFrequency / 1000)); + EqualToExpected(); + AddKilobitsToLayer(200, 0); + + // Wrap + expected_.start_layer = 1; + AdvanceTime(1); + EqualToExpected(); +} + +} // namespace webrtc diff --git a/webrtc/modules/video_coding/codecs/vp9/vp9.gyp b/webrtc/modules/video_coding/codecs/vp9/vp9.gyp index 752521c5cb..8993d79bd7 100644 --- a/webrtc/modules/video_coding/codecs/vp9/vp9.gyp +++ b/webrtc/modules/video_coding/codecs/vp9/vp9.gyp @@ -14,30 +14,26 @@ { 'target_name': 'webrtc_vp9', 'type': 'static_library', - 'dependencies': [ - '<(webrtc_root)/common_video/common_video.gyp:common_video', - '<(webrtc_root)/modules/video_coding/utility/video_coding_utility.gyp:video_coding_utility', - '<(webrtc_root)/system_wrappers/system_wrappers.gyp:system_wrappers', - ], 'conditions': [ ['build_libvpx==1', { 'dependencies': [ '<(libvpx_dir)/libvpx.gyp:libvpx_new', ], }], - ['build_vp9==1', { - 'sources': [ - 'include/vp9.h', - 'vp9_frame_buffer_pool.cc', - 'vp9_frame_buffer_pool.h', - 'vp9_impl.cc', - 'vp9_impl.h', - ], - }, { - 'sources': [ - 'vp9_dummy_impl.cc', - ], - }], + ], + 'dependencies': [ + '<(webrtc_root)/common_video/common_video.gyp:common_video', + '<(webrtc_root)/modules/video_coding/utility/video_coding_utility.gyp:video_coding_utility', + '<(webrtc_root)/system_wrappers/system_wrappers.gyp:system_wrappers', + ], + 'sources': [ + 'include/vp9.h', + 'screenshare_layers.cc', + 'screenshare_layers.h', + 'vp9_frame_buffer_pool.cc', + 'vp9_frame_buffer_pool.h', + 'vp9_impl.cc', + 'vp9_impl.h', ], }, ], diff --git a/webrtc/modules/video_coding/codecs/vp9/vp9_dummy_impl.cc b/webrtc/modules/video_coding/codecs/vp9/vp9_dummy_impl.cc deleted file mode 100644 index 491ccbe79c..0000000000 --- a/webrtc/modules/video_coding/codecs/vp9/vp9_dummy_impl.cc +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - * - */ - -// This file contains an implementation of empty webrtc VP9 encoder/decoder -// factories so it is possible to build webrtc without linking with vp9. -#include "webrtc/modules/video_coding/codecs/vp9/vp9_impl.h" - -namespace webrtc { -VP9Encoder* VP9Encoder::Create() { return nullptr; } -VP9Decoder* VP9Decoder::Create() { return nullptr; } -} diff --git a/webrtc/modules/video_coding/codecs/vp9/vp9_frame_buffer_pool.cc b/webrtc/modules/video_coding/codecs/vp9/vp9_frame_buffer_pool.cc index bedbe68ca8..62c05d34fa 100644 --- a/webrtc/modules/video_coding/codecs/vp9/vp9_frame_buffer_pool.cc +++ b/webrtc/modules/video_coding/codecs/vp9/vp9_frame_buffer_pool.cc @@ -16,7 +16,7 @@ #include "vpx/vpx_frame_buffer.h" #include "webrtc/base/checks.h" -#include "webrtc/system_wrappers/include/logging.h" +#include "webrtc/base/logging.h" namespace webrtc { diff --git a/webrtc/modules/video_coding/codecs/vp9/vp9_impl.cc b/webrtc/modules/video_coding/codecs/vp9/vp9_impl.cc index 0ca7eeabe9..e554795519 100644 --- a/webrtc/modules/video_coding/codecs/vp9/vp9_impl.cc +++ b/webrtc/modules/video_coding/codecs/vp9/vp9_impl.cc @@ -21,36 +21,31 @@ #include "vpx/vp8cx.h" #include "vpx/vp8dx.h" -#include "webrtc/base/bind.h" #include "webrtc/base/checks.h" +#include "webrtc/base/keep_ref_until_done.h" +#include "webrtc/base/logging.h" #include "webrtc/base/trace_event.h" #include "webrtc/common.h" #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h" -#include "webrtc/modules/interface/module_common_types.h" -#include "webrtc/system_wrappers/include/logging.h" +#include "webrtc/modules/include/module_common_types.h" +#include "webrtc/modules/video_coding/codecs/vp9/screenshare_layers.h" #include "webrtc/system_wrappers/include/tick_util.h" -namespace { - -// VP9DecoderImpl::ReturnFrame helper function used with WrappedI420Buffer. -static void WrappedI420BufferNoLongerUsedCb( - webrtc::Vp9FrameBufferPool::Vp9FrameBuffer* img_buffer) { - img_buffer->Release(); -} - -} // anonymous namespace - namespace webrtc { // Only positive speeds, range for real-time coding currently is: 5 - 8. // Lower means slower/better quality, higher means fastest/lower quality. int GetCpuSpeed(int width, int height) { +#if defined(WEBRTC_ARCH_ARM) || defined(WEBRTC_ARCH_ARM64) + return 8; +#else // For smaller resolutions, use lower speed setting (get some coding gain at // the cost of increased encoding complexity). if (width * height <= 352 * 288) return 5; else return 7; +#endif } VP9Encoder* VP9Encoder::Create() { @@ -59,7 +54,7 @@ VP9Encoder* VP9Encoder::Create() { void VP9EncoderImpl::EncoderOutputCodedPacketCallback(vpx_codec_cx_pkt* pkt, void* user_data) { - VP9EncoderImpl* enc = (VP9EncoderImpl*)(user_data); + VP9EncoderImpl* enc = static_cast<VP9EncoderImpl*>(user_data); enc->GetEncodedLayerFrame(pkt); } @@ -76,9 +71,12 @@ VP9EncoderImpl::VP9EncoderImpl() raw_(NULL), input_image_(NULL), tl0_pic_idx_(0), - gof_idx_(0), + frames_since_kf_(0), num_temporal_layers_(0), - num_spatial_layers_(0) { + num_spatial_layers_(0), + frames_encoded_(0), + // Use two spatial when screensharing with flexible mode. + spatial_layer_(new ScreenshareLayersVP9(2)) { memset(&codec_, 0, sizeof(codec_)); uint32_t seed = static_cast<uint32_t>(TickTime::MillisecondTimestamp()); srand(seed); @@ -90,7 +88,7 @@ VP9EncoderImpl::~VP9EncoderImpl() { int VP9EncoderImpl::Release() { if (encoded_image_._buffer != NULL) { - delete [] encoded_image_._buffer; + delete[] encoded_image_._buffer; encoded_image_._buffer = NULL; } if (encoder_ != NULL) { @@ -112,42 +110,72 @@ int VP9EncoderImpl::Release() { return WEBRTC_VIDEO_CODEC_OK; } +bool VP9EncoderImpl::ExplicitlyConfiguredSpatialLayers() const { + // We check target_bitrate_bps of the 0th layer to see if the spatial layers + // (i.e. bitrates) were explicitly configured. + return num_spatial_layers_ > 1 && + codec_.spatialLayers[0].target_bitrate_bps > 0; +} + bool VP9EncoderImpl::SetSvcRates() { - float rate_ratio[VPX_MAX_LAYERS] = {0}; - float total = 0; uint8_t i = 0; - for (i = 0; i < num_spatial_layers_; ++i) { - if (svc_internal_.svc_params.scaling_factor_num[i] <= 0 || - svc_internal_.svc_params.scaling_factor_den[i] <= 0) { + if (ExplicitlyConfiguredSpatialLayers()) { + if (num_temporal_layers_ > 1) { + LOG(LS_ERROR) << "Multiple temporal layers when manually specifying " + "spatial layers not implemented yet!"; return false; } - rate_ratio[i] = static_cast<float>( - svc_internal_.svc_params.scaling_factor_num[i]) / - svc_internal_.svc_params.scaling_factor_den[i]; - total += rate_ratio[i]; - } - - for (i = 0; i < num_spatial_layers_; ++i) { - config_->ss_target_bitrate[i] = static_cast<unsigned int>( - config_->rc_target_bitrate * rate_ratio[i] / total); - if (num_temporal_layers_ == 1) { - config_->layer_target_bitrate[i] = config_->ss_target_bitrate[i]; - } else if (num_temporal_layers_ == 2) { - config_->layer_target_bitrate[i * num_temporal_layers_] = - config_->ss_target_bitrate[i] * 2 / 3; - config_->layer_target_bitrate[i * num_temporal_layers_ + 1] = - config_->ss_target_bitrate[i]; - } else if (num_temporal_layers_ == 3) { - config_->layer_target_bitrate[i * num_temporal_layers_] = - config_->ss_target_bitrate[i] / 2; - config_->layer_target_bitrate[i * num_temporal_layers_ + 1] = - config_->layer_target_bitrate[i * num_temporal_layers_] + - (config_->ss_target_bitrate[i] / 4); - config_->layer_target_bitrate[i * num_temporal_layers_ + 2] = - config_->ss_target_bitrate[i]; - } else { - return false; + int total_bitrate_bps = 0; + for (i = 0; i < num_spatial_layers_; ++i) + total_bitrate_bps += codec_.spatialLayers[i].target_bitrate_bps; + // If total bitrate differs now from what has been specified at the + // beginning, update the bitrates in the same ratio as before. + for (i = 0; i < num_spatial_layers_; ++i) { + config_->ss_target_bitrate[i] = config_->layer_target_bitrate[i] = + static_cast<int>(static_cast<int64_t>(config_->rc_target_bitrate) * + codec_.spatialLayers[i].target_bitrate_bps / + total_bitrate_bps); + } + } else { + float rate_ratio[VPX_MAX_LAYERS] = {0}; + float total = 0; + + for (i = 0; i < num_spatial_layers_; ++i) { + if (svc_internal_.svc_params.scaling_factor_num[i] <= 0 || + svc_internal_.svc_params.scaling_factor_den[i] <= 0) { + LOG(LS_ERROR) << "Scaling factors not specified!"; + return false; + } + rate_ratio[i] = + static_cast<float>(svc_internal_.svc_params.scaling_factor_num[i]) / + svc_internal_.svc_params.scaling_factor_den[i]; + total += rate_ratio[i]; + } + + for (i = 0; i < num_spatial_layers_; ++i) { + config_->ss_target_bitrate[i] = static_cast<unsigned int>( + config_->rc_target_bitrate * rate_ratio[i] / total); + if (num_temporal_layers_ == 1) { + config_->layer_target_bitrate[i] = config_->ss_target_bitrate[i]; + } else if (num_temporal_layers_ == 2) { + config_->layer_target_bitrate[i * num_temporal_layers_] = + config_->ss_target_bitrate[i] * 2 / 3; + config_->layer_target_bitrate[i * num_temporal_layers_ + 1] = + config_->ss_target_bitrate[i]; + } else if (num_temporal_layers_ == 3) { + config_->layer_target_bitrate[i * num_temporal_layers_] = + config_->ss_target_bitrate[i] / 2; + config_->layer_target_bitrate[i * num_temporal_layers_ + 1] = + config_->layer_target_bitrate[i * num_temporal_layers_] + + (config_->ss_target_bitrate[i] / 4); + config_->layer_target_bitrate[i * num_temporal_layers_ + 2] = + config_->ss_target_bitrate[i]; + } else { + LOG(LS_ERROR) << "Unsupported number of temporal layers: " + << num_temporal_layers_; + return false; + } } } @@ -178,6 +206,7 @@ int VP9EncoderImpl::SetRates(uint32_t new_bitrate_kbit, } config_->rc_target_bitrate = new_bitrate_kbit; codec_.maxFramerate = new_framerate; + spatial_layer_->ConfigureBitrate(new_bitrate_kbit, 0); if (!SetSvcRates()) { return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; @@ -216,6 +245,7 @@ int VP9EncoderImpl::InitEncode(const VideoCodec* inst, if (inst->codecSpecific.VP9.numberOfSpatialLayers > 2) { return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; } + int retVal = Release(); if (retVal < 0) { return retVal; @@ -237,10 +267,10 @@ int VP9EncoderImpl::InitEncode(const VideoCodec* inst, num_temporal_layers_ = 1; // Random start 16 bits is enough. - picture_id_ = static_cast<uint16_t>(rand()) & 0x7FFF; + picture_id_ = static_cast<uint16_t>(rand()) & 0x7FFF; // NOLINT // Allocate memory for encoded image if (encoded_image_._buffer != NULL) { - delete [] encoded_image_._buffer; + delete[] encoded_image_._buffer; } encoded_image_._size = CalcBufferSize(kI420, codec_.width, codec_.height); encoded_image_._buffer = new uint8_t[encoded_image_._size]; @@ -248,8 +278,8 @@ int VP9EncoderImpl::InitEncode(const VideoCodec* inst, // Creating a wrapper to the image - setting image data to NULL. Actual // pointer will be set in encode. Setting align to 1, as it is meaningless // (actual memory is not allocated). - raw_ = vpx_img_wrap(NULL, VPX_IMG_FMT_I420, codec_.width, codec_.height, - 1, NULL); + raw_ = vpx_img_wrap(NULL, VPX_IMG_FMT_I420, codec_.width, codec_.height, 1, + NULL); // Populate encoder configuration with default values. if (vpx_codec_enc_config_default(vpx_codec_vp9_cx(), config_, 0)) { return WEBRTC_VIDEO_CODEC_ERROR; @@ -264,8 +294,8 @@ int VP9EncoderImpl::InitEncode(const VideoCodec* inst, config_->g_lag_in_frames = 0; // 0- no frame lagging config_->g_threads = 1; // Rate control settings. - config_->rc_dropframe_thresh = inst->codecSpecific.VP9.frameDroppingOn ? - 30 : 0; + config_->rc_dropframe_thresh = + inst->codecSpecific.VP9.frameDroppingOn ? 30 : 0; config_->rc_end_usage = VPX_CBR; config_->g_pass = VPX_RC_ONE_PASS; config_->rc_min_quantizer = 2; @@ -277,24 +307,32 @@ int VP9EncoderImpl::InitEncode(const VideoCodec* inst, config_->rc_buf_sz = 1000; // Set the maximum target size of any key-frame. rc_max_intra_target_ = MaxIntraTarget(config_->rc_buf_optimal_sz); - if (inst->codecSpecific.VP9.keyFrameInterval > 0) { + if (inst->codecSpecific.VP9.keyFrameInterval > 0) { config_->kf_mode = VPX_KF_AUTO; config_->kf_max_dist = inst->codecSpecific.VP9.keyFrameInterval; + // Needs to be set (in svc mode) to get correct periodic key frame interval + // (will have no effect in non-svc). + config_->kf_min_dist = config_->kf_max_dist; } else { config_->kf_mode = VPX_KF_DISABLED; } - config_->rc_resize_allowed = inst->codecSpecific.VP9.automaticResizeOn ? - 1 : 0; + config_->rc_resize_allowed = + inst->codecSpecific.VP9.automaticResizeOn ? 1 : 0; // Determine number of threads based on the image size and #cores. - config_->g_threads = NumberOfThreads(config_->g_w, - config_->g_h, - number_of_cores); + config_->g_threads = + NumberOfThreads(config_->g_w, config_->g_h, number_of_cores); cpu_speed_ = GetCpuSpeed(config_->g_w, config_->g_h); // TODO(asapersson): Check configuration of temporal switch up and increase // pattern length. - if (num_temporal_layers_ == 1) { + is_flexible_mode_ = inst->codecSpecific.VP9.flexibleMode; + if (is_flexible_mode_) { + config_->temporal_layering_mode = VP9E_TEMPORAL_LAYERING_MODE_BYPASS; + config_->ts_number_layers = num_temporal_layers_; + if (codec_.mode == kScreensharing) + spatial_layer_->ConfigureBitrate(inst->startBitrate, 0); + } else if (num_temporal_layers_ == 1) { gof_.SetGofInfoVP9(kTemporalStructureMode1); config_->temporal_layering_mode = VP9E_TEMPORAL_LAYERING_MODE_NOLAYERING; config_->ts_number_layers = 1; @@ -326,7 +364,7 @@ int VP9EncoderImpl::InitEncode(const VideoCodec* inst, return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; } - tl0_pic_idx_ = static_cast<uint8_t>(rand()); + tl0_pic_idx_ = static_cast<uint8_t>(rand()); // NOLINT return InitAndSetControlSettings(inst); } @@ -347,16 +385,28 @@ int VP9EncoderImpl::NumberOfThreads(int width, } int VP9EncoderImpl::InitAndSetControlSettings(const VideoCodec* inst) { - config_->ss_number_layers = num_spatial_layers_; - - int scaling_factor_num = 256; - for (int i = num_spatial_layers_ - 1; i >= 0; --i) { + // Set QP-min/max per spatial and temporal layer. + int tot_num_layers = num_spatial_layers_ * num_temporal_layers_; + for (int i = 0; i < tot_num_layers; ++i) { svc_internal_.svc_params.max_quantizers[i] = config_->rc_max_quantizer; svc_internal_.svc_params.min_quantizers[i] = config_->rc_min_quantizer; - // 1:2 scaling in each dimension. - svc_internal_.svc_params.scaling_factor_num[i] = scaling_factor_num; - svc_internal_.svc_params.scaling_factor_den[i] = 256; - scaling_factor_num /= 2; + } + config_->ss_number_layers = num_spatial_layers_; + if (ExplicitlyConfiguredSpatialLayers()) { + for (int i = 0; i < num_spatial_layers_; ++i) { + const auto& layer = codec_.spatialLayers[i]; + svc_internal_.svc_params.scaling_factor_num[i] = layer.scaling_factor_num; + svc_internal_.svc_params.scaling_factor_den[i] = layer.scaling_factor_den; + } + } else { + int scaling_factor_num = 256; + for (int i = num_spatial_layers_ - 1; i >= 0; --i) { + // 1:2 scaling in each dimension. + svc_internal_.svc_params.scaling_factor_num[i] = scaling_factor_num; + svc_internal_.svc_params.scaling_factor_den[i] = 256; + if (codec_.mode != kScreensharing) + scaling_factor_num /= 2; + } } if (!SetSvcRates()) { @@ -381,8 +431,10 @@ int VP9EncoderImpl::InitAndSetControlSettings(const VideoCodec* inst) { } // Register callback for getting each spatial layer. vpx_codec_priv_output_cx_pkt_cb_pair_t cbp = { - VP9EncoderImpl::EncoderOutputCodedPacketCallback, (void*)(this)}; - vpx_codec_control(encoder_, VP9E_REGISTER_CX_CALLBACK, (void*)(&cbp)); + VP9EncoderImpl::EncoderOutputCodedPacketCallback, + reinterpret_cast<void*>(this)}; + vpx_codec_control(encoder_, VP9E_REGISTER_CX_CALLBACK, + reinterpret_cast<void*>(&cbp)); // Control function to set the number of column tiles in encoding a frame, in // log2 unit: e.g., 0 = 1 tile column, 1 = 2 tile columns, 2 = 4 tile columns. @@ -417,7 +469,7 @@ uint32_t VP9EncoderImpl::MaxIntraTarget(uint32_t optimal_buffer_size) { optimal_buffer_size * scale_par * codec_.maxFramerate / 10; // Don't go below 3 times the per frame bandwidth. const uint32_t min_intra_size = 300; - return (target_pct < min_intra_size) ? min_intra_size: target_pct; + return (target_pct < min_intra_size) ? min_intra_size : target_pct; } int VP9EncoderImpl::Encode(const VideoFrame& input_image, @@ -455,12 +507,35 @@ int VP9EncoderImpl::Encode(const VideoFrame& input_image, raw_->stride[VPX_PLANE_U] = input_image.stride(kUPlane); raw_->stride[VPX_PLANE_V] = input_image.stride(kVPlane); - int flags = 0; + vpx_enc_frame_flags_t flags = 0; bool send_keyframe = (frame_type == kVideoFrameKey); if (send_keyframe) { // Key frame request from caller. flags = VPX_EFLAG_FORCE_KF; } + + if (is_flexible_mode_) { + SuperFrameRefSettings settings; + + // These structs are copied when calling vpx_codec_control, + // therefore it is ok for them to go out of scope. + vpx_svc_ref_frame_config enc_layer_conf; + vpx_svc_layer_id layer_id; + + if (codec_.mode == kRealtimeVideo) { + // Real time video not yet implemented in flexible mode. + RTC_NOTREACHED(); + } else { + settings = spatial_layer_->GetSuperFrameSettings(input_image.timestamp(), + send_keyframe); + } + enc_layer_conf = GenerateRefsAndFlags(settings); + layer_id.temporal_layer_id = 0; + layer_id.spatial_layer_id = settings.start_layer; + vpx_codec_control(encoder_, VP9E_SET_SVC_LAYER_ID, &layer_id); + vpx_codec_control(encoder_, VP9E_SET_SVC_REF_FRAME_CONFIG, &enc_layer_conf); + } + assert(codec_.maxFramerate > 0); uint32_t duration = 90000 / codec_.maxFramerate; if (vpx_codec_encode(encoder_, raw_, timestamp_, duration, flags, @@ -473,12 +548,12 @@ int VP9EncoderImpl::Encode(const VideoFrame& input_image, } void VP9EncoderImpl::PopulateCodecSpecific(CodecSpecificInfo* codec_specific, - const vpx_codec_cx_pkt& pkt, - uint32_t timestamp) { + const vpx_codec_cx_pkt& pkt, + uint32_t timestamp) { assert(codec_specific != NULL); codec_specific->codecType = kVideoCodecVP9; - CodecSpecificInfoVP9 *vp9_info = &(codec_specific->codecSpecific.VP9); - // TODO(asapersson): Set correct values. + CodecSpecificInfoVP9* vp9_info = &(codec_specific->codecSpecific.VP9); + // TODO(asapersson): Set correct value. vp9_info->inter_pic_predicted = (pkt.data.frame.flags & VPX_FRAME_IS_KEY) ? false : true; vp9_info->flexible_mode = codec_.codecSpecific.VP9.flexibleMode; @@ -486,9 +561,6 @@ void VP9EncoderImpl::PopulateCodecSpecific(CodecSpecificInfo* codec_specific, !codec_.codecSpecific.VP9.flexibleMode) ? true : false; - if (pkt.data.frame.flags & VPX_FRAME_IS_KEY) { - gof_idx_ = 0; - } vpx_svc_layer_id_t layer_id = {0}; vpx_codec_control(encoder_, VP9E_GET_SVC_LAYER_ID, &layer_id); @@ -511,25 +583,31 @@ void VP9EncoderImpl::PopulateCodecSpecific(CodecSpecificInfo* codec_specific, vp9_info->ss_data_available = false; } - if (vp9_info->flexible_mode) { - vp9_info->gof_idx = kNoGofIdx; + // TODO(asapersson): this info has to be obtained from the encoder. + vp9_info->temporal_up_switch = false; + + bool is_first_frame = false; + if (is_flexible_mode_) { + is_first_frame = + layer_id.spatial_layer_id == spatial_layer_->GetStartLayer(); } else { - vp9_info->gof_idx = - static_cast<uint8_t>(gof_idx_++ % gof_.num_frames_in_gof); + is_first_frame = layer_id.spatial_layer_id == 0; } - // TODO(asapersson): this info has to be obtained from the encoder. - vp9_info->temporal_up_switch = true; - - if (layer_id.spatial_layer_id == 0) { + if (is_first_frame) { picture_id_ = (picture_id_ + 1) & 0x7FFF; // TODO(asapersson): this info has to be obtained from the encoder. vp9_info->inter_layer_predicted = false; + ++frames_since_kf_; } else { // TODO(asapersson): this info has to be obtained from the encoder. vp9_info->inter_layer_predicted = true; } + if (pkt.data.frame.flags & VPX_FRAME_IS_KEY) { + frames_since_kf_ = 0; + } + vp9_info->picture_id = picture_id_; if (!vp9_info->flexible_mode) { @@ -542,6 +620,20 @@ void VP9EncoderImpl::PopulateCodecSpecific(CodecSpecificInfo* codec_specific, // Always populate this, so that the packetizer can properly set the marker // bit. vp9_info->num_spatial_layers = num_spatial_layers_; + + vp9_info->num_ref_pics = 0; + if (vp9_info->flexible_mode) { + vp9_info->gof_idx = kNoGofIdx; + vp9_info->num_ref_pics = num_ref_pics_[layer_id.spatial_layer_id]; + for (int i = 0; i < num_ref_pics_[layer_id.spatial_layer_id]; ++i) { + vp9_info->p_diff[i] = p_diff_[layer_id.spatial_layer_id][i]; + } + } else { + vp9_info->gof_idx = + static_cast<uint8_t>(frames_since_kf_ % gof_.num_frames_in_gof); + vp9_info->temporal_up_switch = gof_.temporal_up_switch[vp9_info->gof_idx]; + } + if (vp9_info->ss_data_available) { vp9_info->spatial_layer_resolution_present = true; for (size_t i = 0; i < vp9_info->num_spatial_layers; ++i) { @@ -577,6 +669,14 @@ int VP9EncoderImpl::GetEncodedLayerFrame(const vpx_codec_cx_pkt* pkt) { frag_info.fragmentationPlType[part_idx] = 0; frag_info.fragmentationTimeDiff[part_idx] = 0; encoded_image_._length += static_cast<uint32_t>(pkt->data.frame.sz); + + vpx_svc_layer_id_t layer_id = {0}; + vpx_codec_control(encoder_, VP9E_GET_SVC_LAYER_ID, &layer_id); + if (is_flexible_mode_ && codec_.mode == kScreensharing) + spatial_layer_->LayerFrameEncoded( + static_cast<unsigned int>(encoded_image_._length), + layer_id.spatial_layer_id); + assert(encoded_image_._length <= encoded_image_._size); // End of frame. @@ -598,6 +698,108 @@ int VP9EncoderImpl::GetEncodedLayerFrame(const vpx_codec_cx_pkt* pkt) { return WEBRTC_VIDEO_CODEC_OK; } +vpx_svc_ref_frame_config VP9EncoderImpl::GenerateRefsAndFlags( + const SuperFrameRefSettings& settings) { + static const vpx_enc_frame_flags_t kAllFlags = + VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_REF_LAST | + VP8_EFLAG_NO_UPD_LAST | VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_GF; + vpx_svc_ref_frame_config sf_conf = {}; + if (settings.is_keyframe) { + // Used later on to make sure we don't make any invalid references. + memset(buffer_updated_at_frame_, -1, sizeof(buffer_updated_at_frame_)); + for (int layer = settings.start_layer; layer <= settings.stop_layer; + ++layer) { + num_ref_pics_[layer] = 0; + buffer_updated_at_frame_[settings.layer[layer].upd_buf] = frames_encoded_; + // When encoding a keyframe only the alt_fb_idx is used + // to specify which layer ends up in which buffer. + sf_conf.alt_fb_idx[layer] = settings.layer[layer].upd_buf; + } + } else { + for (int layer_idx = settings.start_layer; layer_idx <= settings.stop_layer; + ++layer_idx) { + vpx_enc_frame_flags_t layer_flags = kAllFlags; + num_ref_pics_[layer_idx] = 0; + int8_t refs[3] = {settings.layer[layer_idx].ref_buf1, + settings.layer[layer_idx].ref_buf2, + settings.layer[layer_idx].ref_buf3}; + + for (unsigned int ref_idx = 0; ref_idx < kMaxVp9RefPics; ++ref_idx) { + if (refs[ref_idx] == -1) + continue; + + RTC_DCHECK_GE(refs[ref_idx], 0); + RTC_DCHECK_LE(refs[ref_idx], 7); + // Easier to remove flags from all flags rather than having to + // build the flags from 0. + switch (num_ref_pics_[layer_idx]) { + case 0: { + sf_conf.lst_fb_idx[layer_idx] = refs[ref_idx]; + layer_flags &= ~VP8_EFLAG_NO_REF_LAST; + break; + } + case 1: { + sf_conf.gld_fb_idx[layer_idx] = refs[ref_idx]; + layer_flags &= ~VP8_EFLAG_NO_REF_GF; + break; + } + case 2: { + sf_conf.alt_fb_idx[layer_idx] = refs[ref_idx]; + layer_flags &= ~VP8_EFLAG_NO_REF_ARF; + break; + } + } + // Make sure we don't reference a buffer that hasn't been + // used at all or hasn't been used since a keyframe. + RTC_DCHECK_NE(buffer_updated_at_frame_[refs[ref_idx]], -1); + + p_diff_[layer_idx][num_ref_pics_[layer_idx]] = + frames_encoded_ - buffer_updated_at_frame_[refs[ref_idx]]; + num_ref_pics_[layer_idx]++; + } + + bool upd_buf_same_as_a_ref = false; + if (settings.layer[layer_idx].upd_buf != -1) { + for (unsigned int ref_idx = 0; ref_idx < kMaxVp9RefPics; ++ref_idx) { + if (settings.layer[layer_idx].upd_buf == refs[ref_idx]) { + switch (ref_idx) { + case 0: { + layer_flags &= ~VP8_EFLAG_NO_UPD_LAST; + break; + } + case 1: { + layer_flags &= ~VP8_EFLAG_NO_UPD_GF; + break; + } + case 2: { + layer_flags &= ~VP8_EFLAG_NO_UPD_ARF; + break; + } + } + upd_buf_same_as_a_ref = true; + break; + } + } + if (!upd_buf_same_as_a_ref) { + // If we have three references and a buffer is specified to be + // updated, then that buffer must be the same as one of the + // three references. + RTC_CHECK_LT(num_ref_pics_[layer_idx], kMaxVp9RefPics); + + sf_conf.alt_fb_idx[layer_idx] = settings.layer[layer_idx].upd_buf; + layer_flags ^= VP8_EFLAG_NO_UPD_ARF; + } + + int updated_buffer = settings.layer[layer_idx].upd_buf; + buffer_updated_at_frame_[updated_buffer] = frames_encoded_; + sf_conf.frame_flags[layer_idx] = layer_flags; + } + } + } + ++frames_encoded_; + return sf_conf; +} + int VP9EncoderImpl::SetChannelParameters(uint32_t packet_loss, int64_t rtt) { return WEBRTC_VIDEO_CODEC_OK; } @@ -608,6 +810,10 @@ int VP9EncoderImpl::RegisterEncodeCompleteCallback( return WEBRTC_VIDEO_CODEC_OK; } +const char* VP9EncoderImpl::ImplementationName() const { + return "libvpx"; +} + VP9Decoder* VP9Decoder::Create() { return new VP9DecoderImpl(); } @@ -652,7 +858,7 @@ int VP9DecoderImpl::InitDecode(const VideoCodec* inst, int number_of_cores) { if (decoder_ == NULL) { decoder_ = new vpx_codec_ctx_t; } - vpx_codec_dec_cfg_t cfg; + vpx_codec_dec_cfg_t cfg; // Setting number of threads to a constant value (1) cfg.threads = 1; cfg.h = cfg.w = 0; // set after decode @@ -705,10 +911,8 @@ int VP9DecoderImpl::Decode(const EncodedImage& input_image, } // During decode libvpx may get and release buffers from |frame_buffer_pool_|. // In practice libvpx keeps a few (~3-4) buffers alive at a time. - if (vpx_codec_decode(decoder_, - buffer, - static_cast<unsigned int>(input_image._length), - 0, + if (vpx_codec_decode(decoder_, buffer, + static_cast<unsigned int>(input_image._length), 0, VPX_DL_REALTIME)) { return WEBRTC_VIDEO_CODEC_ERROR; } @@ -730,24 +934,22 @@ int VP9DecoderImpl::ReturnFrame(const vpx_image_t* img, uint32_t timestamp) { } // This buffer contains all of |img|'s image data, a reference counted - // Vp9FrameBuffer. Performing AddRef/Release ensures it is not released and - // recycled during use (libvpx is done with the buffers after a few + // Vp9FrameBuffer. (libvpx is done with the buffers after a few // vpx_codec_decode calls or vpx_codec_destroy). Vp9FrameBufferPool::Vp9FrameBuffer* img_buffer = static_cast<Vp9FrameBufferPool::Vp9FrameBuffer*>(img->fb_priv); - img_buffer->AddRef(); // The buffer can be used directly by the VideoFrame (without copy) by // using a WrappedI420Buffer. rtc::scoped_refptr<WrappedI420Buffer> img_wrapped_buffer( new rtc::RefCountedObject<webrtc::WrappedI420Buffer>( - img->d_w, img->d_h, - img->planes[VPX_PLANE_Y], img->stride[VPX_PLANE_Y], - img->planes[VPX_PLANE_U], img->stride[VPX_PLANE_U], - img->planes[VPX_PLANE_V], img->stride[VPX_PLANE_V], + img->d_w, img->d_h, img->planes[VPX_PLANE_Y], + img->stride[VPX_PLANE_Y], img->planes[VPX_PLANE_U], + img->stride[VPX_PLANE_U], img->planes[VPX_PLANE_V], + img->stride[VPX_PLANE_V], // WrappedI420Buffer's mechanism for allowing the release of its frame // buffer is through a callback function. This is where we should // release |img_buffer|. - rtc::Bind(&WrappedI420BufferNoLongerUsedCb, img_buffer))); + rtc::KeepRefUntilDone(img_buffer))); VideoFrame decoded_image; decoded_image.set_video_frame_buffer(img_wrapped_buffer); @@ -781,4 +983,9 @@ int VP9DecoderImpl::Release() { inited_ = false; return WEBRTC_VIDEO_CODEC_OK; } + +const char* VP9DecoderImpl::ImplementationName() const { + return "libvpx"; +} + } // namespace webrtc diff --git a/webrtc/modules/video_coding/codecs/vp9/vp9_impl.h b/webrtc/modules/video_coding/codecs/vp9/vp9_impl.h index f9c123079e..bfa4540304 100644 --- a/webrtc/modules/video_coding/codecs/vp9/vp9_impl.h +++ b/webrtc/modules/video_coding/codecs/vp9/vp9_impl.h @@ -9,8 +9,10 @@ * */ -#ifndef WEBRTC_MODULES_VIDEO_CODING_CODECS_VP9_IMPL_H_ -#define WEBRTC_MODULES_VIDEO_CODING_CODECS_VP9_IMPL_H_ +#ifndef WEBRTC_MODULES_VIDEO_CODING_CODECS_VP9_VP9_IMPL_H_ +#define WEBRTC_MODULES_VIDEO_CODING_CODECS_VP9_VP9_IMPL_H_ + +#include <vector> #include "webrtc/modules/video_coding/codecs/vp9/include/vp9.h" #include "webrtc/modules/video_coding/codecs/vp9/vp9_frame_buffer_pool.h" @@ -21,6 +23,8 @@ namespace webrtc { +class ScreenshareLayersVP9; + class VP9EncoderImpl : public VP9Encoder { public: VP9EncoderImpl(); @@ -45,6 +49,22 @@ class VP9EncoderImpl : public VP9Encoder { void OnDroppedFrame() override {} + const char* ImplementationName() const override; + + struct LayerFrameRefSettings { + int8_t upd_buf = -1; // -1 - no update, 0..7 - update buffer 0..7 + int8_t ref_buf1 = -1; // -1 - no reference, 0..7 - reference buffer 0..7 + int8_t ref_buf2 = -1; // -1 - no reference, 0..7 - reference buffer 0..7 + int8_t ref_buf3 = -1; // -1 - no reference, 0..7 - reference buffer 0..7 + }; + + struct SuperFrameRefSettings { + LayerFrameRefSettings layer[kMaxVp9NumberOfSpatialLayers]; + uint8_t start_layer = 0; // The first spatial layer to be encoded. + uint8_t stop_layer = 0; // The last spatial layer to be encoded. + bool is_keyframe = false; + }; + private: // Determine number of encoder threads to use. int NumberOfThreads(int width, int height, int number_of_cores); @@ -56,8 +76,18 @@ class VP9EncoderImpl : public VP9Encoder { const vpx_codec_cx_pkt& pkt, uint32_t timestamp); + bool ExplicitlyConfiguredSpatialLayers() const; bool SetSvcRates(); + // Used for flexible mode to set the flags and buffer references used + // by the encoder. Also calculates the references used by the RTP + // packetizer. + // + // Has to be called for every frame (keyframes included) to update the + // state used to calculate references. + vpx_svc_ref_frame_config GenerateRefsAndFlags( + const SuperFrameRefSettings& settings); + virtual int GetEncodedLayerFrame(const vpx_codec_cx_pkt* pkt); // Callback function for outputting packets per spatial layer. @@ -88,11 +118,18 @@ class VP9EncoderImpl : public VP9Encoder { GofInfoVP9 gof_; // Contains each frame's temporal information for // non-flexible mode. uint8_t tl0_pic_idx_; // Only used in non-flexible mode. - size_t gof_idx_; // Only used in non-flexible mode. + size_t frames_since_kf_; uint8_t num_temporal_layers_; uint8_t num_spatial_layers_; -}; + // Used for flexible mode. + bool is_flexible_mode_; + int64_t buffer_updated_at_frame_[kNumVp9Buffers]; + int64_t frames_encoded_; + uint8_t num_ref_pics_[kMaxVp9NumberOfSpatialLayers]; + uint8_t p_diff_[kMaxVp9NumberOfSpatialLayers][kMaxVp9RefPics]; + rtc::scoped_ptr<ScreenshareLayersVP9> spatial_layer_; +}; class VP9DecoderImpl : public VP9Decoder { public: @@ -114,6 +151,8 @@ class VP9DecoderImpl : public VP9Decoder { int Reset() override; + const char* ImplementationName() const override; + private: int ReturnFrame(const vpx_image_t* img, uint32_t timeStamp); @@ -127,4 +166,4 @@ class VP9DecoderImpl : public VP9Decoder { }; } // namespace webrtc -#endif // WEBRTC_MODULES_VIDEO_CODING_CODECS_VP9_IMPL_H_ +#endif // WEBRTC_MODULES_VIDEO_CODING_CODECS_VP9_VP9_IMPL_H_ |