Delay creation of decoders until they are needed

Before this CL, WebRTC created a decoder for each negotiated codec
profile. This quickly consumed all available HW decoder resources
on some platforms. This CL adds a field trial,
WebRTC-PreStreamDecoders, that makes it possible to set how many
decoders that should be created up front, from 0 to ALL. If the
field trial is set to 1, we only create a decoder for the
preferred codec. The other decoders are only created when they are
needed (i.e., if we receive the corresponding payload type).

Bug: webrtc:12462
Change-Id: I087571b540f6796d32d34923f9c7f8e89b0959c5
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/208284
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Commit-Queue: Johannes Kron <kron@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#33300}

7 files changed, 211 insertions, 47 deletions

diff --git a/modules/video_coding/decoder_database.cc b/modules/video_coding/decoder_database.cc
index 594ca86553..6aa332eb88 100644
--- a/modules/video_coding/decoder_database.cc
+++ b/modules/video_coding/decoder_database.cc
@@ -56,7 +56,6 @@ bool VCMDecoderDataBase::DeregisterExternalDecoder(uint8_t payload_type) {
     // Release it if it was registered and in use.
     ptr_decoder_.reset();
   }
-  DeregisterReceiveCodec(payload_type);
   delete it->second;
   dec_external_map_.erase(it);
   return true;
@@ -73,6 +72,12 @@ void VCMDecoderDataBase::RegisterExternalDecoder(VideoDecoder* external_decoder,
   dec_external_map_[payload_type] = ext_decoder;
 }
 
+bool VCMDecoderDataBase::IsExternalDecoderRegistered(
+    uint8_t payload_type) const {
+  return payload_type == current_payload_type_ ||
+         FindExternalDecoderItem(payload_type);
+}
+
 bool VCMDecoderDataBase::RegisterReceiveCodec(uint8_t payload_type,
                                               const VideoCodec* receive_codec,
                                               int number_of_cores) {
diff --git a/modules/video_coding/decoder_database.h b/modules/video_coding/decoder_database.h
index f7c5d70338..81c68e4138 100644
--- a/modules/video_coding/decoder_database.h
+++ b/modules/video_coding/decoder_database.h
@@ -44,6 +44,7 @@ class VCMDecoderDataBase {
   bool DeregisterExternalDecoder(uint8_t payload_type);
   void RegisterExternalDecoder(VideoDecoder* external_decoder,
                                uint8_t payload_type);
+  bool IsExternalDecoderRegistered(uint8_t payload_type) const;
 
   bool RegisterReceiveCodec(uint8_t payload_type,
                             const VideoCodec* receive_codec,
diff --git a/modules/video_coding/video_receiver2.cc b/modules/video_coding/video_receiver2.cc
index 6b3cb63679..b893b954bc 100644
--- a/modules/video_coding/video_receiver2.cc
+++ b/modules/video_coding/video_receiver2.cc
@@ -33,18 +33,18 @@ VideoReceiver2::VideoReceiver2(Clock* clock, VCMTiming* timing)
       timing_(timing),
       decodedFrameCallback_(timing_, clock_),
       codecDataBase_() {
-  decoder_thread_checker_.Detach();
+  decoder_sequence_checker_.Detach();
 }
 
 VideoReceiver2::~VideoReceiver2() {
-  RTC_DCHECK_RUN_ON(&construction_thread_checker_);
+  RTC_DCHECK_RUN_ON(&construction_sequence_checker_);
 }
 
 // Register a receive callback. Will be called whenever there is a new frame
 // ready for rendering.
 int32_t VideoReceiver2::RegisterReceiveCallback(
     VCMReceiveCallback* receiveCallback) {
-  RTC_DCHECK_RUN_ON(&construction_thread_checker_);
+  RTC_DCHECK_RUN_ON(&construction_sequence_checker_);
   RTC_DCHECK(!IsDecoderThreadRunning());
   // This value is set before the decoder thread starts and unset after
   // the decoder thread has been stopped.
@@ -52,20 +52,35 @@ int32_t VideoReceiver2::RegisterReceiveCallback(
   return VCM_OK;
 }
 
-// Register an externally defined decoder object.
+// Register an externally defined decoder object. This may be called on either
+// the construction sequence or the decoder sequence to allow for lazy creation
+// of video decoders. If called on the decoder sequence |externalDecoder| cannot
+// be a nullptr. It's the responsibility of the caller to make sure that the
+// access from the two sequences are mutually exclusive.
 void VideoReceiver2::RegisterExternalDecoder(VideoDecoder* externalDecoder,
                                              uint8_t payloadType) {
-  RTC_DCHECK_RUN_ON(&construction_thread_checker_);
-  RTC_DCHECK(!IsDecoderThreadRunning());
+  if (IsDecoderThreadRunning()) {
+    RTC_DCHECK_RUN_ON(&decoder_sequence_checker_);
+    // Don't allow deregistering decoders on the decoder thread.
+    RTC_DCHECK(externalDecoder != nullptr);
+  } else {
+    RTC_DCHECK_RUN_ON(&construction_sequence_checker_);
+  }
+
   if (externalDecoder == nullptr) {
-    RTC_CHECK(codecDataBase_.DeregisterExternalDecoder(payloadType));
+    codecDataBase_.DeregisterExternalDecoder(payloadType);
     return;
   }
   codecDataBase_.RegisterExternalDecoder(externalDecoder, payloadType);
 }
 
+bool VideoReceiver2::IsExternalDecoderRegistered(uint8_t payloadType) const {
+  RTC_DCHECK_RUN_ON(&decoder_sequence_checker_);
+  return codecDataBase_.IsExternalDecoderRegistered(payloadType);
+}
+
 void VideoReceiver2::DecoderThreadStarting() {
-  RTC_DCHECK_RUN_ON(&construction_thread_checker_);
+  RTC_DCHECK_RUN_ON(&construction_sequence_checker_);
   RTC_DCHECK(!IsDecoderThreadRunning());
 #if RTC_DCHECK_IS_ON
   decoder_thread_is_running_ = true;
@@ -73,17 +88,17 @@ void VideoReceiver2::DecoderThreadStarting() {
 }
 
 void VideoReceiver2::DecoderThreadStopped() {
-  RTC_DCHECK_RUN_ON(&construction_thread_checker_);
+  RTC_DCHECK_RUN_ON(&construction_sequence_checker_);
   RTC_DCHECK(IsDecoderThreadRunning());
 #if RTC_DCHECK_IS_ON
   decoder_thread_is_running_ = false;
-  decoder_thread_checker_.Detach();
+  decoder_sequence_checker_.Detach();
 #endif
 }
 
 // Must be called from inside the receive side critical section.
 int32_t VideoReceiver2::Decode(const VCMEncodedFrame* frame) {
-  RTC_DCHECK_RUN_ON(&decoder_thread_checker_);
+  RTC_DCHECK_RUN_ON(&decoder_sequence_checker_);
   TRACE_EVENT0("webrtc", "VideoReceiver2::Decode");
   // Change decoder if payload type has changed
   VCMGenericDecoder* decoder =
@@ -98,7 +113,7 @@ int32_t VideoReceiver2::Decode(const VCMEncodedFrame* frame) {
 int32_t VideoReceiver2::RegisterReceiveCodec(uint8_t payload_type,
                                              const VideoCodec* receiveCodec,
                                              int32_t numberOfCores) {
-  RTC_DCHECK_RUN_ON(&construction_thread_checker_);
+  RTC_DCHECK_RUN_ON(&construction_sequence_checker_);
   RTC_DCHECK(!IsDecoderThreadRunning());
   if (receiveCodec == nullptr) {
     return VCM_PARAMETER_ERROR;
diff --git a/modules/video_coding/video_receiver2.h b/modules/video_coding/video_receiver2.h
index 6d354b1013..0c3fe1a257 100644
--- a/modules/video_coding/video_receiver2.h
+++ b/modules/video_coding/video_receiver2.h
@@ -36,6 +36,7 @@ class VideoReceiver2 {
 
   void RegisterExternalDecoder(VideoDecoder* externalDecoder,
                                uint8_t payloadType);
+  bool IsExternalDecoderRegistered(uint8_t payloadType) const;
   int32_t RegisterReceiveCallback(VCMReceiveCallback* receiveCallback);
 
   int32_t Decode(const webrtc::VCMEncodedFrame* frame);
@@ -54,8 +55,8 @@ class VideoReceiver2 {
   // In builds where DCHECKs aren't enabled, it will return true.
   bool IsDecoderThreadRunning();
 
-  SequenceChecker construction_thread_checker_;
-  SequenceChecker decoder_thread_checker_;
+  SequenceChecker construction_sequence_checker_;
+  SequenceChecker decoder_sequence_checker_;
   Clock* const clock_;
   VCMTiming* timing_;
   VCMDecodedFrameCallback decodedFrameCallback_;
diff --git a/video/video_receive_stream2.cc b/video/video_receive_stream2.cc
index 3ecbdeecaa..ff31c6613a 100644
--- a/video/video_receive_stream2.cc
+++ b/video/video_receive_stream2.cc
@@ -66,6 +66,8 @@ constexpr int kMaxBaseMinimumDelayMs = 10000;
 
 constexpr int kMaxWaitForFrameMs = 3000;
 
+constexpr int kDefaultMaximumPreStreamDecoders = 100;
+
 // Concrete instance of RecordableEncodedFrame wrapping needed content
 // from video_coding::EncodedFrame.
 class WebRtcRecordableEncodedFrame : public RecordableEncodedFrame {
@@ -234,6 +236,7 @@ VideoReceiveStream2::VideoReceiveStream2(
       low_latency_renderer_enabled_("enabled", true),
       low_latency_renderer_include_predecode_buffer_("include_predecode_buffer",
                                                      true),
+      maximum_pre_stream_decoders_("max", kDefaultMaximumPreStreamDecoders),
       decode_queue_(task_queue_factory_->CreateTaskQueue(
           "DecodingQueue",
           TaskQueueFactory::Priority::HIGH)) {
@@ -278,6 +281,11 @@ VideoReceiveStream2::VideoReceiveStream2(
   ParseFieldTrial({&low_latency_renderer_enabled_,
                    &low_latency_renderer_include_predecode_buffer_},
                   field_trial::FindFullName("WebRTC-LowLatencyRenderer"));
+  ParseFieldTrial(
+      {
+          &maximum_pre_stream_decoders_,
+      },
+      field_trial::FindFullName("WebRTC-PreStreamDecoders"));
 }
 
 VideoReceiveStream2::~VideoReceiveStream2() {
@@ -325,41 +333,16 @@ void VideoReceiveStream2::Start() {
     renderer = this;
   }
 
+  int decoders_count = 0;
   for (const Decoder& decoder : config_.decoders) {
-    std::unique_ptr<VideoDecoder> video_decoder =
-        config_.decoder_factory->LegacyCreateVideoDecoder(decoder.video_format,
-                                                          config_.stream_id);
-    // If we still have no valid decoder, we have to create a "Null" decoder
-    // that ignores all calls. The reason we can get into this state is that the
-    // old decoder factory interface doesn't have a way to query supported
-    // codecs.
-    if (!video_decoder) {
-      video_decoder = std::make_unique<NullVideoDecoder>();
-    }
-
-    std::string decoded_output_file =
-        field_trial::FindFullName("WebRTC-DecoderDataDumpDirectory");
-    // Because '/' can't be used inside a field trial parameter, we use ';'
-    // instead.
-    // This is only relevant to WebRTC-DecoderDataDumpDirectory
-    // field trial. ';' is chosen arbitrary. Even though it's a legal character
-    // in some file systems, we can sacrifice ability to use it in the path to
-    // dumped video, since it's developers-only feature for debugging.
-    absl::c_replace(decoded_output_file, ';', '/');
-    if (!decoded_output_file.empty()) {
-      char filename_buffer[256];
-      rtc::SimpleStringBuilder ssb(filename_buffer);
-      ssb << decoded_output_file << "/webrtc_receive_stream_"
-          << this->config_.rtp.remote_ssrc << "-" << rtc::TimeMicros()
-          << ".ivf";
-      video_decoder = CreateFrameDumpingDecoderWrapper(
-          std::move(video_decoder), FileWrapper::OpenWriteOnly(ssb.str()));
+    // Create up to maximum_pre_stream_decoders_ up front, wait the the other
+    // decoders until they are requested (i.e., we receive the corresponding
+    // payload).
+    if (decoders_count < maximum_pre_stream_decoders_) {
+      CreateAndRegisterExternalDecoder(decoder);
+      ++decoders_count;
     }
 
-    video_decoders_.push_back(std::move(video_decoder));
-
-    video_receiver_.RegisterExternalDecoder(video_decoders_.back().get(),
-                                            decoder.payload_type);
     VideoCodec codec = CreateDecoderVideoCodec(decoder);
 
     const bool raw_payload =
@@ -429,6 +412,41 @@ void VideoReceiveStream2::Stop() {
   transport_adapter_.Disable();
 }
 
+void VideoReceiveStream2::CreateAndRegisterExternalDecoder(
+    const Decoder& decoder) {
+  std::unique_ptr<VideoDecoder> video_decoder =
+      config_.decoder_factory->CreateVideoDecoder(decoder.video_format);
+  // If we still have no valid decoder, we have to create a "Null" decoder
+  // that ignores all calls. The reason we can get into this state is that the
+  // old decoder factory interface doesn't have a way to query supported
+  // codecs.
+  if (!video_decoder) {
+    video_decoder = std::make_unique<NullVideoDecoder>();
+  }
+
+  std::string decoded_output_file =
+      field_trial::FindFullName("WebRTC-DecoderDataDumpDirectory");
+  // Because '/' can't be used inside a field trial parameter, we use ';'
+  // instead.
+  // This is only relevant to WebRTC-DecoderDataDumpDirectory
+  // field trial. ';' is chosen arbitrary. Even though it's a legal character
+  // in some file systems, we can sacrifice ability to use it in the path to
+  // dumped video, since it's developers-only feature for debugging.
+  absl::c_replace(decoded_output_file, ';', '/');
+  if (!decoded_output_file.empty()) {
+    char filename_buffer[256];
+    rtc::SimpleStringBuilder ssb(filename_buffer);
+    ssb << decoded_output_file << "/webrtc_receive_stream_"
+        << this->config_.rtp.remote_ssrc << "-" << rtc::TimeMicros() << ".ivf";
+    video_decoder = CreateFrameDumpingDecoderWrapper(
+        std::move(video_decoder), FileWrapper::OpenWriteOnly(ssb.str()));
+  }
+
+  video_decoders_.push_back(std::move(video_decoder));
+  video_receiver_.RegisterExternalDecoder(video_decoders_.back().get(),
+                                          decoder.payload_type);
+}
+
 VideoReceiveStream::Stats VideoReceiveStream2::GetStats() const {
   RTC_DCHECK_RUN_ON(&worker_sequence_checker_);
   VideoReceiveStream2::Stats stats = stats_proxy_.GetStats();
@@ -661,6 +679,16 @@ void VideoReceiveStream2::HandleEncodedFrame(
   const bool keyframe_request_is_due =
       now_ms >= (last_keyframe_request_ms_ + max_wait_for_keyframe_ms_);
 
+  if (!video_receiver_.IsExternalDecoderRegistered(frame->PayloadType())) {
+    // Look for the decoder with this payload type.
+    for (const Decoder& decoder : config_.decoders) {
+      if (decoder.payload_type == frame->PayloadType()) {
+        CreateAndRegisterExternalDecoder(decoder);
+        break;
+      }
+    }
+  }
+
   int decode_result = video_receiver_.Decode(frame.get());
   if (decode_result == WEBRTC_VIDEO_CODEC_OK ||
       decode_result == WEBRTC_VIDEO_CODEC_OK_REQUEST_KEYFRAME) {
diff --git a/video/video_receive_stream2.h b/video/video_receive_stream2.h
index 5cc074b207..33855e7f4a 100644
--- a/video/video_receive_stream2.h
+++ b/video/video_receive_stream2.h
@@ -155,6 +155,7 @@ class VideoReceiveStream2 : public webrtc::VideoReceiveStream,
   void GenerateKeyFrame() override;
 
  private:
+  void CreateAndRegisterExternalDecoder(const Decoder& decoder);
   int64_t GetMaxWaitMs() const RTC_RUN_ON(decode_queue_);
   void StartNextDecode() RTC_RUN_ON(decode_queue_);
   void HandleEncodedFrame(std::unique_ptr<video_coding::EncodedFrame> frame)
@@ -265,6 +266,11 @@ class VideoReceiveStream2 : public webrtc::VideoReceiveStream,
   // queue.
   FieldTrialParameter<bool> low_latency_renderer_include_predecode_buffer_;
 
+  // Set by the field trial WebRTC-PreStreamDecoders. The parameter |max|
+  // determines the maximum number of decoders that are created up front before
+  // any video frame has been received.
+  FieldTrialParameter<int> maximum_pre_stream_decoders_;
+
   // Defined last so they are destroyed before all other members.
   rtc::TaskQueue decode_queue_;
 
diff --git a/video/video_receive_stream2_unittest.cc b/video/video_receive_stream2_unittest.cc
index 34588813c3..b59d167958 100644
--- a/video/video_receive_stream2_unittest.cc
+++ b/video/video_receive_stream2_unittest.cc
@@ -76,6 +76,14 @@ class MockVideoDecoder : public VideoDecoder {
   const char* ImplementationName() const { return "MockVideoDecoder"; }
 };
 
+class MockVideoDecoderFactory : public VideoDecoderFactory {
+ public:
+  MOCK_CONST_METHOD0(GetSupportedFormats, std::vector<SdpVideoFormat>());
+
+  MOCK_METHOD1(CreateVideoDecoder,
+               std::unique_ptr<VideoDecoder>(const SdpVideoFormat& format));
+};
+
 class FrameObjectFake : public video_coding::EncodedFrame {
  public:
   void SetPayloadType(uint8_t payload_type) { _payloadType = payload_type; }
@@ -111,6 +119,7 @@ class VideoReceiveStream2Test : public ::testing::Test {
     h264_decoder.video_format = SdpVideoFormat("H264");
     h264_decoder.video_format.parameters.insert(
         {"sprop-parameter-sets", "Z0IACpZTBYmI,aMljiA=="});
+    config_.decoders.clear();
     config_.decoders.push_back(h264_decoder);
 
     clock_ = Clock::GetRealTimeClock();
@@ -593,4 +602,103 @@ TEST_F(VideoReceiveStream2TestWithSimulatedClock,
   loop_.Run();
 }
 
+class VideoReceiveStream2TestWithLazyDecoderCreation : public ::testing::Test {
+ public:
+  VideoReceiveStream2TestWithLazyDecoderCreation()
+      : process_thread_(ProcessThread::Create("TestThread")),
+        task_queue_factory_(CreateDefaultTaskQueueFactory()),
+        config_(&mock_transport_),
+        call_stats_(Clock::GetRealTimeClock(), loop_.task_queue()) {}
+
+  void SetUp() {
+    webrtc::test::ScopedFieldTrials field_trials(
+        "WebRTC-PreStreamDecoders/max:0/");
+    constexpr int kDefaultNumCpuCores = 2;
+    config_.rtp.remote_ssrc = 1111;
+    config_.rtp.local_ssrc = 2222;
+    config_.renderer = &fake_renderer_;
+    config_.decoder_factory = &mock_h264_decoder_factory_;
+    VideoReceiveStream::Decoder h264_decoder;
+    h264_decoder.payload_type = 99;
+    h264_decoder.video_format = SdpVideoFormat("H264");
+    h264_decoder.video_format.parameters.insert(
+        {"sprop-parameter-sets", "Z0IACpZTBYmI,aMljiA=="});
+    config_.decoders.clear();
+    config_.decoders.push_back(h264_decoder);
+
+    clock_ = Clock::GetRealTimeClock();
+    timing_ = new VCMTiming(clock_);
+
+    video_receive_stream_ =
+        std::make_unique<webrtc::internal::VideoReceiveStream2>(
+            task_queue_factory_.get(), loop_.task_queue(),
+            &rtp_stream_receiver_controller_, kDefaultNumCpuCores,
+            &packet_router_, config_.Copy(), process_thread_.get(),
+            &call_stats_, clock_, timing_);
+  }
+
+ protected:
+  test::RunLoop loop_;
+  std::unique_ptr<ProcessThread> process_thread_;
+  const std::unique_ptr<TaskQueueFactory> task_queue_factory_;
+  VideoReceiveStream::Config config_;
+  internal::CallStats call_stats_;
+  MockVideoDecoder mock_h264_video_decoder_;
+  MockVideoDecoderFactory mock_h264_decoder_factory_;
+  cricket::FakeVideoRenderer fake_renderer_;
+  MockTransport mock_transport_;
+  PacketRouter packet_router_;
+  RtpStreamReceiverController rtp_stream_receiver_controller_;
+  std::unique_ptr<webrtc::internal::VideoReceiveStream2> video_receive_stream_;
+  Clock* clock_;
+  VCMTiming* timing_;
+};
+
+TEST_F(VideoReceiveStream2TestWithLazyDecoderCreation, LazyDecoderCreation) {
+  constexpr uint8_t idr_nalu[] = {0x05, 0xFF, 0xFF, 0xFF};
+  RtpPacketToSend rtppacket(nullptr);
+  uint8_t* payload = rtppacket.AllocatePayload(sizeof(idr_nalu));
+  memcpy(payload, idr_nalu, sizeof(idr_nalu));
+  rtppacket.SetMarker(true);
+  rtppacket.SetSsrc(1111);
+  rtppacket.SetPayloadType(99);
+  rtppacket.SetSequenceNumber(1);
+  rtppacket.SetTimestamp(0);
+
+  // No decoder is created here.
+  EXPECT_CALL(mock_h264_decoder_factory_, CreateVideoDecoder(_)).Times(0);
+  video_receive_stream_->Start();
+
+  EXPECT_CALL(mock_h264_decoder_factory_, CreateVideoDecoder(_))
+      .WillOnce(Invoke([this](const SdpVideoFormat& format) {
+        test::VideoDecoderProxyFactory h264_decoder_factory(
+            &mock_h264_video_decoder_);
+        return h264_decoder_factory.CreateVideoDecoder(format);
+      }));
+  rtc::Event init_decode_event_;
+  EXPECT_CALL(mock_h264_video_decoder_, InitDecode(_, _))
+      .WillOnce(Invoke([&init_decode_event_](const VideoCodec* config,
+                                             int32_t number_of_cores) {
+        init_decode_event_.Set();
+        return 0;
+      }));
+  EXPECT_CALL(mock_h264_video_decoder_, RegisterDecodeCompleteCallback(_));
+  EXPECT_CALL(mock_h264_video_decoder_, Decode(_, false, _));
+  RtpPacketReceived parsed_packet;
+  ASSERT_TRUE(parsed_packet.Parse(rtppacket.data(), rtppacket.size()));
+  rtp_stream_receiver_controller_.OnRtpPacket(parsed_packet);
+  EXPECT_CALL(mock_h264_video_decoder_, Release());
+
+  // Make sure the decoder thread had a chance to run.
+  init_decode_event_.Wait(kDefaultTimeOutMs);
+}
+
+TEST_F(VideoReceiveStream2TestWithLazyDecoderCreation,
+       DeregisterDecoderThatsNotCreated) {
+  // No decoder is created here.
+  EXPECT_CALL(mock_h264_decoder_factory_, CreateVideoDecoder(_)).Times(0);
+  video_receive_stream_->Start();
+  video_receive_stream_->Stop();
+}
+
 }  // namespace webrtc