Lazy conversion to i420 buffer in SimulcastEncoderAdapter

This CL makes native the preferred buffer type, and if any of the
encoders in the adapter do not support native, the input frame will
be converted to I420 and used as input to encoders that require it,
along with any needed downscaling.

Bug: chromium:1052352
Change-Id: Iff43d6dc56793037a6164729b9e6c69797b6f8cd
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/168609
Reviewed-by: Florent Castelli <orphis@webrtc.org>
Commit-Queue: Erik Språng <sprang@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#30565}

2 files changed, 76 insertions, 16 deletions

diff --git a/media/engine/simulcast_encoder_adapter.cc b/media/engine/simulcast_encoder_adapter.cc
index 03768de1cd..9ca6afe5f2 100644
--- a/media/engine/simulcast_encoder_adapter.cc
+++ b/media/engine/simulcast_encoder_adapter.cc
@@ -377,6 +377,8 @@ int SimulcastEncoderAdapter::Encode(
     }
   }
 
+  // Temporary thay may hold the result of texture to i420 buffer conversion.
+  rtc::scoped_refptr<I420BufferInterface> src_buffer;
   int src_width = input_image.width();
   int src_height = input_image.height();
   for (size_t stream_idx = 0; stream_idx < streaminfos_.size(); ++stream_idx) {
@@ -420,18 +422,23 @@ int SimulcastEncoderAdapter::Encode(
     // TODO(perkj): ensure that works going forward, and figure out how this
     // affects webrtc:5683.
     if ((dst_width == src_width && dst_height == src_height) ||
-        input_image.video_frame_buffer()->type() ==
-            VideoFrameBuffer::Type::kNative) {
+        (input_image.video_frame_buffer()->type() ==
+             VideoFrameBuffer::Type::kNative &&
+         streaminfos_[stream_idx]
+             .encoder->GetEncoderInfo()
+             .supports_native_handle)) {
       int ret = streaminfos_[stream_idx].encoder->Encode(input_image,
                                                          &stream_frame_types);
       if (ret != WEBRTC_VIDEO_CODEC_OK) {
         return ret;
       }
     } else {
+      if (src_buffer == nullptr) {
+        src_buffer = input_image.video_frame_buffer()->ToI420();
+      }
       rtc::scoped_refptr<I420Buffer> dst_buffer =
           I420Buffer::Create(dst_width, dst_height);
-      rtc::scoped_refptr<I420BufferInterface> src_buffer =
-          input_image.video_frame_buffer()->ToI420();
+
       dst_buffer->ScaleFrom(*src_buffer);
 
       // UpdateRect is not propagated to lower simulcast layers currently.
@@ -660,8 +667,8 @@ VideoEncoder::EncoderInfo SimulcastEncoderAdapter::GetEncoderInfo() const {
       encoder_info.implementation_name += ", ";
       encoder_info.implementation_name += encoder_impl_info.implementation_name;
 
-      // Native handle supported only if all encoders supports it.
-      encoder_info.supports_native_handle &=
+      // Native handle supported if any encoder supports it.
+      encoder_info.supports_native_handle |=
           encoder_impl_info.supports_native_handle;
 
       // Trusted rate controller only if all encoders have it.
diff --git a/media/engine/simulcast_encoder_adapter_unittest.cc b/media/engine/simulcast_encoder_adapter_unittest.cc
index 04ea92e2d8..14c1cd4010 100644
--- a/media/engine/simulcast_encoder_adapter_unittest.cc
+++ b/media/engine/simulcast_encoder_adapter_unittest.cc
@@ -932,33 +932,39 @@ TEST_F(TestSimulcastEncoderAdapterFake,
   ASSERT_EQ(3u, helper_->factory()->encoders().size());
   for (MockVideoEncoder* encoder : helper_->factory()->encoders())
     encoder->set_supports_native_handle(true);
-  // If one encoder doesn't support it, then overall support is disabled.
+  // As long as one encoder supports native handle, it's enabled.
   helper_->factory()->encoders()[0]->set_supports_native_handle(false);
-  EXPECT_FALSE(adapter_->GetEncoderInfo().supports_native_handle);
-  // Once all do, then the adapter claims support.
-  helper_->factory()->encoders()[0]->set_supports_native_handle(true);
-  EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings));
   EXPECT_TRUE(adapter_->GetEncoderInfo().supports_native_handle);
+  // Once none do, then the adapter claims no support.
+  helper_->factory()->encoders()[1]->set_supports_native_handle(false);
+  helper_->factory()->encoders()[2]->set_supports_native_handle(false);
+  EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings));
+  EXPECT_FALSE(adapter_->GetEncoderInfo().supports_native_handle);
 }
 
 // TODO(nisse): Reuse definition in webrtc/test/fake_texture_handle.h.
-class FakeNativeBufferNoI420 : public VideoFrameBuffer {
+class FakeNativeBufferI420 : public VideoFrameBuffer {
  public:
-  FakeNativeBufferNoI420(int width, int height)
-      : width_(width), height_(height) {}
+  FakeNativeBufferI420(int width, int height, bool allow_to_i420)
+      : width_(width), height_(height), allow_to_i420_(allow_to_i420) {}
 
   Type type() const override { return Type::kNative; }
   int width() const override { return width_; }
   int height() const override { return height_; }
 
   rtc::scoped_refptr<I420BufferInterface> ToI420() override {
-    RTC_NOTREACHED();
+    if (allow_to_i420_) {
+      return I420Buffer::Create(width_, height_);
+    } else {
+      RTC_NOTREACHED();
+    }
     return nullptr;
   }
 
  private:
   const int width_;
   const int height_;
+  const bool allow_to_i420_;
 };
 
 TEST_F(TestSimulcastEncoderAdapterFake,
@@ -978,7 +984,8 @@ TEST_F(TestSimulcastEncoderAdapterFake,
   EXPECT_TRUE(adapter_->GetEncoderInfo().supports_native_handle);
 
   rtc::scoped_refptr<VideoFrameBuffer> buffer(
-      new rtc::RefCountedObject<FakeNativeBufferNoI420>(1280, 720));
+      new rtc::RefCountedObject<FakeNativeBufferI420>(1280, 720,
+                                                      /*allow_to_i420=*/false));
   VideoFrame input_frame = VideoFrame::Builder()
                                .set_video_frame_buffer(buffer)
                                .set_timestamp_rtp(100)
@@ -993,6 +1000,52 @@ TEST_F(TestSimulcastEncoderAdapterFake,
   EXPECT_EQ(0, adapter_->Encode(input_frame, &frame_types));
 }
 
+TEST_F(TestSimulcastEncoderAdapterFake, NativeHandleForwardingOnlyIfSupported) {
+  SimulcastTestFixtureImpl::DefaultSettings(
+      &codec_, static_cast<const int*>(kTestTemporalLayerProfile),
+      kVideoCodecVP8);
+  codec_.numberOfSimulcastStreams = 3;
+  // High start bitrate, so all streams are enabled.
+  codec_.startBitrate = 3000;
+  EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings));
+  adapter_->RegisterEncodeCompleteCallback(this);
+  ASSERT_EQ(3u, helper_->factory()->encoders().size());
+
+  // QVGA encoders has fallen back to software.
+  auto& encoders = helper_->factory()->encoders();
+  encoders[0]->set_supports_native_handle(false);
+  encoders[1]->set_supports_native_handle(true);
+  encoders[2]->set_supports_native_handle(true);
+
+  EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings));
+  EXPECT_TRUE(adapter_->GetEncoderInfo().supports_native_handle);
+
+  rtc::scoped_refptr<VideoFrameBuffer> buffer(
+      new rtc::RefCountedObject<FakeNativeBufferI420>(1280, 720,
+                                                      /*allow_to_i420=*/true));
+  VideoFrame input_frame = VideoFrame::Builder()
+                               .set_video_frame_buffer(buffer)
+                               .set_timestamp_rtp(100)
+                               .set_timestamp_ms(1000)
+                               .set_rotation(kVideoRotation_180)
+                               .build();
+  // Expect calls with the given video frame verbatim, since it's a texture
+  // frame and can't otherwise be modified/resized, but only on the two
+  // streams supporting it...
+  EXPECT_CALL(*encoders[1], Encode(::testing::Ref(input_frame), _)).Times(1);
+  EXPECT_CALL(*encoders[2], Encode(::testing::Ref(input_frame), _)).Times(1);
+  // ...the lowest one gets a software buffer.
+  EXPECT_CALL(*encoders[0], Encode)
+      .WillOnce([&](const VideoFrame& frame,
+                    const std::vector<VideoFrameType>* frame_types) {
+        EXPECT_EQ(frame.video_frame_buffer()->type(),
+                  VideoFrameBuffer::Type::kI420);
+        return 0;
+      });
+  std::vector<VideoFrameType> frame_types(3, VideoFrameType::kVideoFrameKey);
+  EXPECT_EQ(0, adapter_->Encode(input_frame, &frame_types));
+}
+
 TEST_F(TestSimulcastEncoderAdapterFake, TestFailureReturnCodesFromEncodeCalls) {
   SimulcastTestFixtureImpl::DefaultSettings(
       &codec_, static_cast<const int*>(kTestTemporalLayerProfile),