Add VideoCodec::PreferDecodeLate

The purpose is so that a decoder (Android) that only have a limited number of output buffers can make sure that decoding is done just before the frame is needed.

Removed unused iSupportsRenderTiming and the settings structs since it was not used.
Added VCMReceiver::FrameForDecoding unit test for the case when PreferDecodeLate is set.

Note that this does not change the current behaviour. We actually currently always decode frames late. This cl is to make sure the behaviour is kept for Android, if the default behaviour is changed.

Review URL: https://codereview.webrtc.org/1428293003

Cr-Commit-Position: refs/heads/master@{#10974}

13 files changed, 99 insertions, 64 deletions

diff --git a/webrtc/modules/video_coding/codec_database.cc b/webrtc/modules/video_coding/codec_database.cc
index be7f5ca376..20a1143272 100644
--- a/webrtc/modules/video_coding/codec_database.cc
+++ b/webrtc/modules/video_coding/codec_database.cc
@@ -85,12 +85,9 @@ VCMDecoderMapItem::VCMDecoderMapItem(VideoCodec* settings,
 
 VCMExtDecoderMapItem::VCMExtDecoderMapItem(
     VideoDecoder* external_decoder_instance,
-    uint8_t payload_type,
-    bool internal_render_timing)
+    uint8_t payload_type)
     : payload_type(payload_type),
-      external_decoder_instance(external_decoder_instance),
-      internal_render_timing(internal_render_timing) {
-}
+      external_decoder_instance(external_decoder_instance) {}
 
 VCMCodecDataBase::VCMCodecDataBase(
     VideoEncoderRateObserver* encoder_rate_observer,
@@ -419,13 +416,11 @@ bool VCMCodecDataBase::DeregisterExternalDecoder(uint8_t payload_type) {
 
 // Add the external encoder object to the list of external decoders.
 // Won't be registered as a receive codec until RegisterReceiveCodec is called.
-void VCMCodecDataBase::RegisterExternalDecoder(
-    VideoDecoder* external_decoder,
-    uint8_t payload_type,
-    bool internal_render_timing) {
+void VCMCodecDataBase::RegisterExternalDecoder(VideoDecoder* external_decoder,
+                                               uint8_t payload_type) {
   // Check if payload value already exists, if so  - erase old and insert new.
-  VCMExtDecoderMapItem* ext_decoder = new VCMExtDecoderMapItem(
-      external_decoder, payload_type, internal_render_timing);
+  VCMExtDecoderMapItem* ext_decoder =
+      new VCMExtDecoderMapItem(external_decoder, payload_type);
   DeregisterExternalDecoder(payload_type);
   dec_external_map_[payload_type] = ext_decoder;
 }
@@ -524,12 +519,10 @@ void VCMCodecDataBase::ReleaseDecoder(VCMGenericDecoder* decoder) const {
   }
 }
 
-bool VCMCodecDataBase::SupportsRenderScheduling() const {
-  const VCMExtDecoderMapItem* ext_item = FindExternalDecoderItem(
-      receive_codec_.plType);
-  if (!ext_item)
+bool VCMCodecDataBase::PrefersLateDecoding() const {
+  if (!ptr_decoder_)
     return true;
-  return ext_item->internal_render_timing;
+  return ptr_decoder_->PrefersLateDecoding();
 }
 
 bool VCMCodecDataBase::MatchesCurrentResolution(int width, int height) const {
diff --git a/webrtc/modules/video_coding/codec_database.h b/webrtc/modules/video_coding/codec_database.h
index c39474f6a2..62ec30a46e 100644
--- a/webrtc/modules/video_coding/codec_database.h
+++ b/webrtc/modules/video_coding/codec_database.h
@@ -36,12 +36,10 @@ struct VCMDecoderMapItem {
 struct VCMExtDecoderMapItem {
  public:
   VCMExtDecoderMapItem(VideoDecoder* external_decoder_instance,
-                       uint8_t payload_type,
-                       bool internal_render_timing);
+                       uint8_t payload_type);
 
   uint8_t payload_type;
   VideoDecoder* external_decoder_instance;
-  bool internal_render_timing;
 };
 
 class VCMCodecDataBase {
@@ -90,12 +88,8 @@ class VCMCodecDataBase {
   bool DeregisterExternalDecoder(uint8_t payload_type);
 
   // Registers an external decoder object to the payload type |payload_type|.
-  // |internal_render_timing| is set to true if the |external_decoder| has
-  // built in rendering which is able to obey the render timestamps of the
-  // encoded frames.
   void RegisterExternalDecoder(VideoDecoder* external_decoder,
-                               uint8_t payload_type,
-                               bool internal_render_timing);
+                               uint8_t payload_type);
 
   bool DecoderRegistered() const;
 
@@ -124,10 +118,9 @@ class VCMCodecDataBase {
   // deep copies returned by CreateDecoderCopy().
   void ReleaseDecoder(VCMGenericDecoder* decoder) const;
 
-  // Returns true if the currently active decoder supports render scheduling,
-  // that is, it is able to render frames according to the render timestamp of
-  // the encoded frames.
-  bool SupportsRenderScheduling() const;
+  // Returns true if the currently active decoder prefer to decode frames late.
+  // That means that frames must be decoded near the render times stamp.
+  bool PrefersLateDecoding() const;
 
   bool MatchesCurrentResolution(int width, int height) const;
 
diff --git a/webrtc/modules/video_coding/generic_decoder.cc b/webrtc/modules/video_coding/generic_decoder.cc
index 08282f9e6c..dc78ec21f2 100644
--- a/webrtc/modules/video_coding/generic_decoder.cc
+++ b/webrtc/modules/video_coding/generic_decoder.cc
@@ -198,4 +198,8 @@ bool VCMGenericDecoder::External() const {
   return _isExternal;
 }
 
+bool VCMGenericDecoder::PrefersLateDecoding() const {
+  return _decoder->PrefersLateDecoding();
+}
+
 }  // namespace
diff --git a/webrtc/modules/video_coding/generic_decoder.h b/webrtc/modules/video_coding/generic_decoder.h
index c633519853..40a9845c94 100644
--- a/webrtc/modules/video_coding/generic_decoder.h
+++ b/webrtc/modules/video_coding/generic_decoder.h
@@ -97,6 +97,7 @@ public:
     int32_t RegisterDecodeCompleteCallback(VCMDecodedFrameCallback* callback);
 
     bool External() const;
+    bool PrefersLateDecoding() const;
 
 private:
     VCMDecodedFrameCallback*    _callback;
diff --git a/webrtc/modules/video_coding/include/video_coding.h b/webrtc/modules/video_coding/include/video_coding.h
index 5f5961c71e..880180d5b9 100644
--- a/webrtc/modules/video_coding/include/video_coding.h
+++ b/webrtc/modules/video_coding/include/video_coding.h
@@ -333,11 +333,9 @@ public:
     //      - externalDecoder        : The external decoder/renderer object.
     //      - payloadType            : The payload type which this decoder should be
     //                                 registered to.
-    //      - internalRenderTiming   : True if the internal renderer (if any) of the decoder
-    //                                 object can make sure to render at a given time in ms.
+    //
     virtual void RegisterExternalDecoder(VideoDecoder* externalDecoder,
-                                         uint8_t payloadType,
-                                         bool internalRenderTiming) = 0;
+                                         uint8_t payloadType) = 0;
 
     // Register a receive callback. Will be called whenever there is a new frame ready
     // for rendering.
diff --git a/webrtc/modules/video_coding/receiver.cc b/webrtc/modules/video_coding/receiver.cc
index 5314b30767..fe05b86825 100644
--- a/webrtc/modules/video_coding/receiver.cc
+++ b/webrtc/modules/video_coding/receiver.cc
@@ -96,7 +96,7 @@ void VCMReceiver::TriggerDecoderShutdown() {
 
 VCMEncodedFrame* VCMReceiver::FrameForDecoding(uint16_t max_wait_time_ms,
                                                int64_t& next_render_time_ms,
-                                               bool render_timing) {
+                                               bool prefer_late_decoding) {
   const int64_t start_time_ms = clock_->TimeInMilliseconds();
   uint32_t frame_timestamp = 0;
   // Exhaust wait time to get a complete frame for decoding.
@@ -140,7 +140,7 @@ VCMEncodedFrame* VCMReceiver::FrameForDecoding(uint16_t max_wait_time_ms,
     return NULL;
   }
 
-  if (!render_timing) {
+  if (prefer_late_decoding) {
     // Decode frame as close as possible to the render timestamp.
     const int32_t available_wait_time = max_wait_time_ms -
         static_cast<int32_t>(clock_->TimeInMilliseconds() - start_time_ms);
diff --git a/webrtc/modules/video_coding/receiver.h b/webrtc/modules/video_coding/receiver.h
index bc2e026fef..a30d16c8c6 100644
--- a/webrtc/modules/video_coding/receiver.h
+++ b/webrtc/modules/video_coding/receiver.h
@@ -47,7 +47,7 @@ class VCMReceiver {
                        uint16_t frame_height);
   VCMEncodedFrame* FrameForDecoding(uint16_t max_wait_time_ms,
                                     int64_t& next_render_time_ms,
-                                    bool render_timing = true);
+                                    bool prefer_late_decoding);
   void ReleaseFrame(VCMEncodedFrame* frame);
   void ReceiveStatistics(uint32_t* bitrate, uint32_t* framerate);
   uint32_t DiscardedPackets() const;
diff --git a/webrtc/modules/video_coding/receiver_unittest.cc b/webrtc/modules/video_coding/receiver_unittest.cc
index 82345c8d59..d51b004a9b 100644
--- a/webrtc/modules/video_coding/receiver_unittest.cc
+++ b/webrtc/modules/video_coding/receiver_unittest.cc
@@ -14,6 +14,7 @@
 
 #include "testing/gtest/include/gtest/gtest.h"
 #include "webrtc/base/checks.h"
+#include "webrtc/modules/video_coding/encoded_frame.h"
 #include "webrtc/modules/video_coding/packet.h"
 #include "webrtc/modules/video_coding/receiver.h"
 #include "webrtc/modules/video_coding/test/stream_generator.h"
@@ -141,7 +142,8 @@ TEST_F(TestVCMReceiver, RenderBufferSize_NoKeyFrame) {
     EXPECT_GE(InsertFrame(kVideoFrameDelta, true), kNoError);
   }
   int64_t next_render_time_ms = 0;
-  VCMEncodedFrame* frame = receiver_.FrameForDecoding(10, next_render_time_ms);
+  VCMEncodedFrame* frame =
+      receiver_.FrameForDecoding(10, next_render_time_ms, false);
   EXPECT_TRUE(frame == NULL);
   receiver_.ReleaseFrame(frame);
   EXPECT_GE(InsertFrame(kVideoFrameDelta, false), kNoError);
@@ -523,4 +525,60 @@ TEST_F(VCMReceiverTimingTest, FrameForDecoding) {
   }
 }
 
+// Test whether VCMReceiver::FrameForDecoding handles parameter
+// |prefer_late_decoding| and |max_wait_time_ms| correctly:
+// 1. The function execution should never take more than |max_wait_time_ms|.
+// 2. If the function exit before now + |max_wait_time_ms|, a frame must be
+//    returned and the end time must be equal to the render timestamp - delay
+//    for decoding and rendering.
+TEST_F(VCMReceiverTimingTest, FrameForDecodingPreferLateDecoding) {
+  const size_t kNumFrames = 100;
+  const int kFramePeriod = 40;
+
+  int64_t arrive_timestamps[kNumFrames];
+  int64_t render_timestamps[kNumFrames];
+  int64_t next_render_time;
+
+  int render_delay_ms;
+  int max_decode_ms;
+  int dummy;
+  timing_.GetTimings(&dummy, &max_decode_ms, &dummy, &dummy, &dummy, &dummy,
+                     &render_delay_ms);
+
+  // Construct test samples.
+  // render_timestamps are the timestamps stored in the Frame;
+  // arrive_timestamps controls when the Frame packet got received.
+  for (size_t i = 0; i < kNumFrames; i++) {
+    // Preset frame rate to 25Hz.
+    // But we add a reasonable deviation to arrive_timestamps to mimic Internet
+    // fluctuation.
+    arrive_timestamps[i] =
+        (i + 1) * kFramePeriod + (i % 10) * ((i % 2) ? 1 : -1);
+    render_timestamps[i] = (i + 1) * kFramePeriod;
+  }
+
+  clock_.SetFrames(arrive_timestamps, render_timestamps, kNumFrames);
+
+  // Record how many frames we finally get out of the receiver.
+  size_t num_frames_return = 0;
+  const int64_t kMaxWaitTime = 30;
+  bool prefer_late_decoding = true;
+  while (num_frames_return < kNumFrames) {
+    int64_t start_time = clock_.TimeInMilliseconds();
+
+    VCMEncodedFrame* frame =
+        receiver_.FrameForDecoding(kMaxWaitTime, next_render_time,
+                                   prefer_late_decoding);
+    int64_t end_time = clock_.TimeInMilliseconds();
+    if (frame) {
+      EXPECT_EQ(frame->RenderTimeMs() - max_decode_ms - render_delay_ms,
+                end_time);
+      receiver_.ReleaseFrame(frame);
+      ++num_frames_return;
+    } else {
+      EXPECT_EQ(kMaxWaitTime, end_time - start_time);
+    }
+  }
+}
+
 }  // namespace webrtc
diff --git a/webrtc/modules/video_coding/video_coding_impl.cc b/webrtc/modules/video_coding/video_coding_impl.cc
index c9ed6f4461..64cc090a70 100644
--- a/webrtc/modules/video_coding/video_coding_impl.cc
+++ b/webrtc/modules/video_coding/video_coding_impl.cc
@@ -197,10 +197,8 @@ class VideoCodingModuleImpl : public VideoCodingModule {
   }
 
   void RegisterExternalDecoder(VideoDecoder* externalDecoder,
-                               uint8_t payloadType,
-                               bool internalRenderTiming) override {
-    receiver_.RegisterExternalDecoder(externalDecoder, payloadType,
-                                      internalRenderTiming);
+                               uint8_t payloadType) override {
+    receiver_.RegisterExternalDecoder(externalDecoder, payloadType);
   }
 
   int32_t RegisterReceiveCallback(
diff --git a/webrtc/modules/video_coding/video_coding_impl.h b/webrtc/modules/video_coding/video_coding_impl.h
index cfa0f92b73..2134e4d533 100644
--- a/webrtc/modules/video_coding/video_coding_impl.h
+++ b/webrtc/modules/video_coding/video_coding_impl.h
@@ -151,8 +151,7 @@ class VideoReceiver {
                                bool requireKeyFrame);
 
   void RegisterExternalDecoder(VideoDecoder* externalDecoder,
-                               uint8_t payloadType,
-                               bool internalRenderTiming);
+                               uint8_t payloadType);
   int32_t RegisterReceiveCallback(VCMReceiveCallback* receiveCallback);
   int32_t RegisterReceiveStatisticsCallback(
       VCMReceiveStatisticsCallback* receiveStats);
diff --git a/webrtc/modules/video_coding/video_coding_robustness_unittest.cc b/webrtc/modules/video_coding/video_coding_robustness_unittest.cc
index d5a179dccb..4111109e60 100644
--- a/webrtc/modules/video_coding/video_coding_robustness_unittest.cc
+++ b/webrtc/modules/video_coding/video_coding_robustness_unittest.cc
@@ -44,7 +44,7 @@ class VCMRobustnessTest : public ::testing::Test {
     ASSERT_EQ(0, vcm_->RegisterPacketRequestCallback(&request_callback_));
     ASSERT_EQ(VCM_OK, vcm_->Codec(kVideoCodecVP8, &video_codec_));
     ASSERT_EQ(VCM_OK, vcm_->RegisterReceiveCodec(&video_codec_, 1));
-    vcm_->RegisterExternalDecoder(&decoder_, video_codec_.plType, true);
+    vcm_->RegisterExternalDecoder(&decoder_, video_codec_.plType);
   }
 
   virtual void TearDown() {
@@ -204,7 +204,7 @@ TEST_F(VCMRobustnessTest, TestModeNoneWithErrors) {
   InsertPacket(0, 0, true, false, kVideoFrameKey);
   InsertPacket(0, 1, false, false, kVideoFrameKey);
   InsertPacket(0, 2, false, true, kVideoFrameKey);
-  EXPECT_EQ(VCM_OK, vcm_->Decode(0));  // Decode timestamp 0.
+  EXPECT_EQ(VCM_OK, vcm_->Decode(33));  // Decode timestamp 0.
   EXPECT_EQ(VCM_OK, vcm_->Process());  // Expect no NACK list.
 
   clock_->AdvanceTimeMilliseconds(33);
@@ -222,7 +222,7 @@ TEST_F(VCMRobustnessTest, TestModeNoneWithErrors) {
   EXPECT_EQ(VCM_OK, vcm_->Process());  // Expect no NACK list.
 
   clock_->AdvanceTimeMilliseconds(10);
-  EXPECT_EQ(VCM_OK, vcm_->Decode(0));  // Decode timestamp 6000 complete.
+  EXPECT_EQ(VCM_OK, vcm_->Decode(23));  // Decode timestamp 6000 complete.
   EXPECT_EQ(VCM_OK, vcm_->Process());  // Expect no NACK list.
 
   clock_->AdvanceTimeMilliseconds(23);
@@ -231,6 +231,6 @@ TEST_F(VCMRobustnessTest, TestModeNoneWithErrors) {
   InsertPacket(9000, 9, true, false, kVideoFrameDelta);
   InsertPacket(9000, 10, false, false, kVideoFrameDelta);
   InsertPacket(9000, 11, false, true, kVideoFrameDelta);
-  EXPECT_EQ(VCM_OK, vcm_->Decode(0));  // Decode timestamp 9000 complete.
+  EXPECT_EQ(VCM_OK, vcm_->Decode(33));  // Decode timestamp 9000 complete.
 }
 }  // namespace webrtc
diff --git a/webrtc/modules/video_coding/video_receiver.cc b/webrtc/modules/video_coding/video_receiver.cc
index c347baa7b8..f7ac4bc862 100644
--- a/webrtc/modules/video_coding/video_receiver.cc
+++ b/webrtc/modules/video_coding/video_receiver.cc
@@ -235,11 +235,9 @@ int32_t VideoReceiver::RegisterDecoderTimingCallback(
   return VCM_OK;
 }
 
-// Register an externally defined decoder/render object.
-// Can be a decoder only or a decoder coupled with a renderer.
+// Register an externally defined decoder object.
 void VideoReceiver::RegisterExternalDecoder(VideoDecoder* externalDecoder,
-                                               uint8_t payloadType,
-                                               bool internalRenderTiming) {
+                                            uint8_t payloadType) {
   CriticalSectionScoped cs(_receiveCritSect);
   if (externalDecoder == NULL) {
     // Make sure the VCM updates the decoder next time it decodes.
@@ -247,8 +245,7 @@ void VideoReceiver::RegisterExternalDecoder(VideoDecoder* externalDecoder,
     RTC_CHECK(_codecDataBase.DeregisterExternalDecoder(payloadType));
     return;
   }
-  _codecDataBase.RegisterExternalDecoder(externalDecoder, payloadType,
-                                         internalRenderTiming);
+  _codecDataBase.RegisterExternalDecoder(externalDecoder, payloadType);
 }
 
 // Register a frame type request callback.
@@ -281,14 +278,14 @@ void VideoReceiver::TriggerDecoderShutdown() {
 // Should be called as often as possible to get the most out of the decoder.
 int32_t VideoReceiver::Decode(uint16_t maxWaitTimeMs) {
   int64_t nextRenderTimeMs;
-  bool supports_render_scheduling;
+  bool prefer_late_decoding = false;
   {
     CriticalSectionScoped cs(_receiveCritSect);
-    supports_render_scheduling = _codecDataBase.SupportsRenderScheduling();
+    prefer_late_decoding = _codecDataBase.PrefersLateDecoding();
   }
 
   VCMEncodedFrame* frame = _receiver.FrameForDecoding(
-      maxWaitTimeMs, nextRenderTimeMs, supports_render_scheduling);
+      maxWaitTimeMs, nextRenderTimeMs, prefer_late_decoding);
 
   if (frame == NULL) {
     return VCM_FRAME_NOT_READY;
@@ -369,13 +366,7 @@ int32_t VideoReceiver::Decode(const VCMEncodedFrame& frame) {
                           "type",
                           frame.FrameType());
   // Change decoder if payload type has changed
-  const bool renderTimingBefore = _codecDataBase.SupportsRenderScheduling();
   _decoder = _codecDataBase.GetDecoder(frame, &_decodedFrameCallback);
-  if (renderTimingBefore != _codecDataBase.SupportsRenderScheduling()) {
-    // Make sure we reset the decode time estimate since it will
-    // be zero for codecs without render timing.
-    _timing.ResetDecodeTime();
-  }
   if (_decoder == NULL) {
     return VCM_NO_CODEC_REGISTERED;
   }
diff --git a/webrtc/modules/video_coding/video_receiver_unittest.cc b/webrtc/modules/video_coding/video_receiver_unittest.cc
index 049f6007b4..8fd46ecc80 100644
--- a/webrtc/modules/video_coding/video_receiver_unittest.cc
+++ b/webrtc/modules/video_coding/video_receiver_unittest.cc
@@ -34,7 +34,7 @@ class TestVideoReceiver : public ::testing::Test {
 
   virtual void SetUp() {
     receiver_.reset(new VideoReceiver(&clock_, &event_factory_));
-    receiver_->RegisterExternalDecoder(&decoder_, kUnusedPayloadType, true);
+    receiver_->RegisterExternalDecoder(&decoder_, kUnusedPayloadType);
     const size_t kMaxNackListSize = 250;
     const int kMaxPacketAgeToNack = 450;
     receiver_->SetNackSettings(kMaxNackListSize, kMaxPacketAgeToNack, 0);
@@ -55,7 +55,7 @@ class TestVideoReceiver : public ::testing::Test {
     }
     EXPECT_EQ(0, receiver_->Process());
     EXPECT_CALL(decoder_, Decode(_, _, _, _, _)).Times(0);
-    EXPECT_EQ(VCM_FRAME_NOT_READY, receiver_->Decode(0));
+    EXPECT_EQ(VCM_FRAME_NOT_READY, receiver_->Decode(100));
   }
 
   void InsertAndVerifyDecodableFrame(const uint8_t* payload,
@@ -67,7 +67,7 @@ class TestVideoReceiver : public ::testing::Test {
     EXPECT_CALL(packet_request_callback_, ResendPackets(_, _)).Times(0);
     EXPECT_EQ(0, receiver_->Process());
     EXPECT_CALL(decoder_, Decode(_, _, _, _, _)).Times(1);
-    EXPECT_EQ(0, receiver_->Decode(0));
+    EXPECT_EQ(0, receiver_->Decode(100));
   }
 
   SimulatedClock clock_;