VideoStreamDecoderImpl implementation, part 4.

In this CL the DecodedImageCallback functions are implemented.

Bug: webrtc:8909
Change-Id: I27ba4525702a6b372697f92c6c97a52ed5bed3c6
Reviewed-on: https://webrtc-review.googlesource.com/67162
Reviewed-by: Stefan Holmer <stefan@webrtc.org>
Reviewed-by: Niels Moller <nisse@webrtc.org>
Commit-Queue: Philip Eliasson <philipel@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#22961}

2 files changed, 89 insertions, 14 deletions

diff --git a/video/video_stream_decoder_impl.cc b/video/video_stream_decoder_impl.cc
index 7b4a1bab49..2d7fa1753f 100644
--- a/video/video_stream_decoder_impl.cc
+++ b/video/video_stream_decoder_impl.cc
@@ -34,8 +34,8 @@ VideoStreamDecoderImpl::VideoStreamDecoderImpl(
                     &jitter_estimator_,
                     &timing_,
                     nullptr),
-      next_start_time_index_(0) {
-  decode_start_time_.fill({-1, 0});
+      next_frame_timestamps_index_(0) {
+  frame_timestamps_.fill({-1, -1, -1});
   decode_thread_.Start();
 }
 
@@ -186,16 +186,21 @@ VideoStreamDecoderImpl::DecodeResult VideoStreamDecoderImpl::DecodeNextFrame(
     }
 
     int64_t decode_start_time_ms = rtc::TimeMillis();
-    uint32_t frame_timestamp = frame->timestamp;
+    int64_t timestamp = frame->timestamp;
+    int64_t render_time_us = frame->RenderTimeMs() * 1000;
     bookkeeping_queue_.PostTask(
-        [this, decode_start_time_ms, frame_timestamp]() {
+        [this, decode_start_time_ms, timestamp, render_time_us]() {
           RTC_DCHECK_RUN_ON(&bookkeeping_queue_);
           // Saving decode start time this way wont work if we decode spatial
           // layers sequentially.
-          decode_start_time_[next_start_time_index_] = {frame_timestamp,
-                                                        decode_start_time_ms};
-          next_start_time_index_ =
-              Add<kDecodeTimeMemory>(next_start_time_index_, 1);
+          FrameTimestamps* frame_timestamps =
+              &frame_timestamps_[next_frame_timestamps_index_];
+          frame_timestamps->timestamp = timestamp;
+          frame_timestamps->decode_start_time_ms = decode_start_time_ms;
+          frame_timestamps->render_time_us = render_time_us;
+
+          next_frame_timestamps_index_ =
+              Add<kFrameTimestampsMemory>(next_frame_timestamps_index_, 1);
         });
 
     int32_t decode_result =
@@ -211,4 +216,67 @@ VideoStreamDecoderImpl::DecodeResult VideoStreamDecoderImpl::DecodeNextFrame(
   return kNoFrame;
 }
 
+VideoStreamDecoderImpl::FrameTimestamps*
+VideoStreamDecoderImpl::GetFrameTimestamps(int64_t timestamp) {
+  RTC_DCHECK_RUN_ON(&bookkeeping_queue_);
+
+  int start_time_index = next_frame_timestamps_index_;
+  for (int i = 0; i < kFrameTimestampsMemory; ++i) {
+    start_time_index = Subtract<kFrameTimestampsMemory>(start_time_index, 1);
+
+    if (frame_timestamps_[start_time_index].timestamp == timestamp)
+      return &frame_timestamps_[start_time_index];
+  }
+
+  return nullptr;
+}
+
+// VideoDecoder::DecodedImageCallback
+int32_t VideoStreamDecoderImpl::Decoded(VideoFrame& decoded_image) {
+  Decoded(decoded_image, rtc::nullopt, rtc::nullopt);
+  return WEBRTC_VIDEO_CODEC_OK;
+}
+
+// VideoDecoder::DecodedImageCallback
+int32_t VideoStreamDecoderImpl::Decoded(VideoFrame& decoded_image,
+                                        int64_t decode_time_ms) {
+  Decoded(decoded_image, decode_time_ms, rtc::nullopt);
+  return WEBRTC_VIDEO_CODEC_OK;
+}
+
+// VideoDecoder::DecodedImageCallback
+void VideoStreamDecoderImpl::Decoded(VideoFrame& decoded_image,
+                                     rtc::Optional<int32_t> decode_time_ms,
+                                     rtc::Optional<uint8_t> qp) {
+  int64_t decode_stop_time_ms = rtc::TimeMillis();
+
+  bookkeeping_queue_.PostTask([this, decode_stop_time_ms, decoded_image,
+                               decode_time_ms, qp]() {
+    RTC_DCHECK_RUN_ON(&bookkeeping_queue_);
+
+    FrameTimestamps* frame_timestamps =
+        GetFrameTimestamps(decoded_image.timestamp());
+    if (!frame_timestamps) {
+      RTC_LOG(LS_ERROR) << "No frame information found for frame with timestamp"
+                        << decoded_image.timestamp();
+      return;
+    }
+
+    rtc::Optional<int> casted_qp;
+    if (qp)
+      casted_qp.emplace(*qp);
+
+    rtc::Optional<int> casted_decode_time_ms(decode_time_ms.value_or(
+        decode_stop_time_ms - frame_timestamps->decode_start_time_ms));
+
+    timing_.StopDecodeTimer(0, *casted_decode_time_ms, decode_stop_time_ms,
+                            frame_timestamps->render_time_us / 1000);
+
+    callbacks_->OnDecodedFrame(
+        VideoFrame(decoded_image.video_frame_buffer(), decoded_image.rotation(),
+                   frame_timestamps->render_time_us),
+        casted_decode_time_ms, casted_qp);
+  });
+}
+
 }  // namespace webrtc
diff --git a/video/video_stream_decoder_impl.h b/video/video_stream_decoder_impl.h
index 42c1b946d2..d4de7b6ef2 100644
--- a/video/video_stream_decoder_impl.h
+++ b/video/video_stream_decoder_impl.h
@@ -48,10 +48,18 @@ class VideoStreamDecoderImpl : public VideoStreamDecoder,
     kShutdown,
   };
 
+  struct FrameTimestamps {
+    int64_t timestamp;
+    int64_t decode_start_time_ms;
+    int64_t render_time_us;
+  };
+
   VideoDecoder* GetDecoder(int payload_type);
   static void DecodeLoop(void* ptr);
   DecodeResult DecodeNextFrame(int max_wait_time_ms, bool keyframe_required);
 
+  FrameTimestamps* GetFrameTimestamps(int64_t timestamp);
+
   // Implements DecodedImageCallback interface
   int32_t Decoded(VideoFrame& decodedImage) override;
   int32_t Decoded(VideoFrame& decodedImage, int64_t decode_time_ms) override;
@@ -78,13 +86,12 @@ class VideoStreamDecoderImpl : public VideoStreamDecoder,
   rtc::Optional<int> current_payload_type_;
   std::unique_ptr<VideoDecoder> decoder_;
 
-  // Keep track of the |decode_start_time_| of the last |kDecodeTimeMemory|
-  // number of frames. The |decode_start_time_| array contain
-  // <frame timestamp> --> <decode start time> pairs.
-  static constexpr int kDecodeTimeMemory = 8;
-  std::array<std::pair<int64_t, int64_t>, kDecodeTimeMemory> decode_start_time_
+  // Some decoders are pipelined so it is not sufficient to save frame info
+  // for the last frame only.
+  static constexpr int kFrameTimestampsMemory = 8;
+  std::array<FrameTimestamps, kFrameTimestampsMemory> frame_timestamps_
       RTC_GUARDED_BY(bookkeeping_queue_);
-  int next_start_time_index_ RTC_GUARDED_BY(bookkeeping_queue_);
+  int next_frame_timestamps_index_ RTC_GUARDED_BY(bookkeeping_queue_);
 };
 
 }  // namespace webrtc