Add GetMetadata() to TransformableVideoFrameInterface API.

Define VideoHeaderMetadata, containing a subset of the metadata in RTP
video header, and expose it the TransformableVideoFrameInterface, to
enable web application to compute additional data according to their own
logic, and eventually remove GetAdditionalData() from the interface.

Bug: chromium:1069295
Change-Id: Id85b494a72cfd8bdd4c0614844b9f0ffae98c956
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/174822
Commit-Queue: Marina Ciocea <marinaciocea@webrtc.org>
Reviewed-by: Danil Chapovalov <danilchap@webrtc.org>
Reviewed-by: Magnus Flodman <mflodman@webrtc.org>
Reviewed-by: Harald Alvestrand <hta@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#31265}

6 files changed, 240 insertions, 1 deletions

diff --git a/api/BUILD.gn b/api/BUILD.gn
index c5629c0fcd..ebd22c6b43 100644
--- a/api/BUILD.gn
+++ b/api/BUILD.gn
@@ -221,6 +221,7 @@ rtc_source_set("frame_transformer_interface") {
     ":scoped_refptr",
     "../rtc_base:refcount",
     "video:encoded_frame",
+    "video:video_frame_metadata",
   ]
 }
 
diff --git a/api/frame_transformer_interface.h b/api/frame_transformer_interface.h
index e712b3c190..753c7f4bb9 100644
--- a/api/frame_transformer_interface.h
+++ b/api/frame_transformer_interface.h
@@ -16,6 +16,7 @@
 
 #include "api/scoped_refptr.h"
 #include "api/video/encoded_frame.h"
+#include "api/video/video_frame_metadata.h"
 #include "rtc_base/ref_count.h"
 
 namespace webrtc {
@@ -48,6 +49,15 @@ class TransformableVideoFrameInterface : public TransformableFrameInterface {
   // TODO(bugs.webrtc.org/11380) remove from interface once
   // webrtc::RtpDescriptorAuthentication is exposed in api/.
   virtual std::vector<uint8_t> GetAdditionalData() const = 0;
+
+  // TODO(bugs.webrtc.org/11380) make pure virtual after implementating it
+  // downstream.
+  virtual const VideoFrameMetadata& GetMetadata() const { return metadata_; }
+
+ private:
+  // TODO(bugs.webrtc.org/11380) remove from interface once GetRtpVideoHeader is
+  // pure virtual.
+  VideoFrameMetadata metadata_;
 };
 
 // Extends the TransformableFrameInterface to expose audio-specific information.
diff --git a/api/video/BUILD.gn b/api/video/BUILD.gn
index 290c2f2abb..7f9b034192 100644
--- a/api/video/BUILD.gn
+++ b/api/video/BUILD.gn
@@ -264,6 +264,21 @@ rtc_source_set("video_stream_encoder") {
   ]
 }
 
+rtc_source_set("video_frame_metadata") {
+  visibility = [ "*" ]
+  sources = [
+    "video_frame_metadata.cc",
+    "video_frame_metadata.h",
+  ]
+  deps = [
+    "..:array_view",
+    "../../modules/rtp_rtcp:rtp_video_header",
+    "../transport/rtp:dependency_descriptor",
+    "//third_party/abseil-cpp/absl/container:inlined_vector",
+    "//third_party/abseil-cpp/absl/types:optional",
+  ]
+}
+
 rtc_library("video_stream_encoder_create") {
   visibility = [ "*" ]
   sources = [
@@ -306,9 +321,14 @@ rtc_library("builtin_video_bitrate_allocator_factory") {
 if (rtc_include_tests) {
   rtc_library("video_unittests") {
     testonly = true
-    sources = [ "video_stream_decoder_create_unittest.cc" ]
+    sources = [
+      "video_frame_metadata_unittest.cc",
+      "video_stream_decoder_create_unittest.cc",
+    ]
     deps = [
+      ":video_frame_metadata",
       ":video_stream_decoder_create",
+      "../../modules/rtp_rtcp:rtp_video_header",
       "../../test:test_support",
       "../task_queue:default_task_queue_factory",
       "../video_codecs:builtin_video_decoder_factory",
diff --git a/api/video/video_frame_metadata.cc b/api/video/video_frame_metadata.cc
new file mode 100644
index 0000000000..df82875eb9
--- /dev/null
+++ b/api/video/video_frame_metadata.cc
@@ -0,0 +1,28 @@
+/*
+ *  Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "api/video/video_frame_metadata.h"
+
+#include "modules/rtp_rtcp/source/rtp_video_header.h"
+
+namespace webrtc {
+
+VideoFrameMetadata::VideoFrameMetadata(const RTPVideoHeader& header)
+    : width_(header.width), height_(header.height) {
+  if (header.generic) {
+    frame_id_ = header.generic->frame_id;
+    spatial_index_ = header.generic->spatial_index;
+    temporal_index_ = header.generic->temporal_index;
+    frame_dependencies_ = header.generic->dependencies;
+    decode_target_indications_ = header.generic->decode_target_indications;
+  }
+}
+
+}  // namespace webrtc
diff --git a/api/video/video_frame_metadata.h b/api/video/video_frame_metadata.h
new file mode 100644
index 0000000000..ce9b2a1318
--- /dev/null
+++ b/api/video/video_frame_metadata.h
@@ -0,0 +1,60 @@
+/*
+ *  Copyright 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef API_VIDEO_VIDEO_FRAME_METADATA_H_
+#define API_VIDEO_VIDEO_FRAME_METADATA_H_
+
+#include <cstdint>
+
+#include "absl/container/inlined_vector.h"
+#include "absl/types/optional.h"
+#include "api/array_view.h"
+#include "api/transport/rtp/dependency_descriptor.h"
+
+namespace webrtc {
+
+struct RTPVideoHeader;
+
+// A subset of metadata from the RTP video header, exposed in insertable streams
+// API.
+class VideoFrameMetadata {
+ public:
+  explicit VideoFrameMetadata(const RTPVideoHeader& header);
+  VideoFrameMetadata() = default;
+  VideoFrameMetadata(const VideoFrameMetadata&) = default;
+  VideoFrameMetadata& operator=(const VideoFrameMetadata&) = default;
+
+  uint16_t GetWidth() const { return width_; }
+  uint16_t GetHeight() const { return height_; }
+  absl::optional<int64_t> GetFrameId() const { return frame_id_; }
+  int GetSpatialIndex() const { return spatial_index_; }
+  int GetTemporalIndex() const { return temporal_index_; }
+
+  rtc::ArrayView<const int64_t> GetFrameDependencies() const {
+    return frame_dependencies_;
+  }
+
+  rtc::ArrayView<const DecodeTargetIndication> GetDecodeTargetIndications()
+      const {
+    return decode_target_indications_;
+  }
+
+ private:
+  int16_t width_;
+  int16_t height_;
+  absl::optional<int64_t> frame_id_;
+  int spatial_index_ = 0;
+  int temporal_index_ = 0;
+  absl::InlinedVector<int64_t, 5> frame_dependencies_;
+  absl::InlinedVector<DecodeTargetIndication, 10> decode_target_indications_;
+};
+}  // namespace webrtc
+
+#endif  // API_VIDEO_VIDEO_FRAME_METADATA_H_
diff --git a/api/video/video_frame_metadata_unittest.cc b/api/video/video_frame_metadata_unittest.cc
new file mode 100644
index 0000000000..7a808e1ea9
--- /dev/null
+++ b/api/video/video_frame_metadata_unittest.cc
@@ -0,0 +1,120 @@
+/*
+ *  Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "api/video/video_frame_metadata.h"
+
+#include "modules/rtp_rtcp/source/rtp_video_header.h"
+#include "test/gmock.h"
+#include "test/gtest.h"
+
+namespace webrtc {
+namespace {
+
+using ::testing::ElementsAre;
+using ::testing::IsEmpty;
+
+TEST(VideoFrameMetadata, GetWidthReturnsCorrectValue) {
+  RTPVideoHeader video_header;
+  video_header.width = 1280u;
+  VideoFrameMetadata metadata(video_header);
+  EXPECT_EQ(metadata.GetWidth(), video_header.width);
+}
+
+TEST(VideoFrameMetadata, GetHeightReturnsCorrectValue) {
+  RTPVideoHeader video_header;
+  video_header.height = 720u;
+  VideoFrameMetadata metadata(video_header);
+  EXPECT_EQ(metadata.GetHeight(), video_header.height);
+}
+
+TEST(VideoFrameMetadata, GetFrameIdReturnsCorrectValue) {
+  RTPVideoHeader video_header;
+  RTPVideoHeader::GenericDescriptorInfo& generic =
+      video_header.generic.emplace();
+  generic.frame_id = 10;
+  VideoFrameMetadata metadata(video_header);
+  EXPECT_EQ(metadata.GetFrameId().value(), 10);
+}
+
+TEST(VideoFrameMetadata, HasNoFrameIdForHeaderWithoutGeneric) {
+  RTPVideoHeader video_header;
+  VideoFrameMetadata metadata(video_header);
+  ASSERT_FALSE(video_header.generic);
+  EXPECT_EQ(metadata.GetFrameId(), absl::nullopt);
+}
+
+TEST(VideoFrameMetadata, GetSpatialIndexReturnsCorrectValue) {
+  RTPVideoHeader video_header;
+  RTPVideoHeader::GenericDescriptorInfo& generic =
+      video_header.generic.emplace();
+  generic.spatial_index = 2;
+  VideoFrameMetadata metadata(video_header);
+  EXPECT_EQ(metadata.GetSpatialIndex(), 2);
+}
+
+TEST(VideoFrameMetadata, SpatialIndexIsZeroForHeaderWithoutGeneric) {
+  RTPVideoHeader video_header;
+  VideoFrameMetadata metadata(video_header);
+  ASSERT_FALSE(video_header.generic);
+  EXPECT_EQ(metadata.GetSpatialIndex(), 0);
+}
+
+TEST(VideoFrameMetadata, GetTemporalIndexReturnsCorrectValue) {
+  RTPVideoHeader video_header;
+  RTPVideoHeader::GenericDescriptorInfo& generic =
+      video_header.generic.emplace();
+  generic.temporal_index = 3;
+  VideoFrameMetadata metadata(video_header);
+  EXPECT_EQ(metadata.GetTemporalIndex(), 3);
+}
+
+TEST(VideoFrameMetadata, TemporalIndexIsZeroForHeaderWithoutGeneric) {
+  RTPVideoHeader video_header;
+  VideoFrameMetadata metadata(video_header);
+  ASSERT_FALSE(video_header.generic);
+  EXPECT_EQ(metadata.GetTemporalIndex(), 0);
+}
+
+TEST(VideoFrameMetadata, GetFrameDependenciesReturnsCorrectValue) {
+  RTPVideoHeader video_header;
+  RTPVideoHeader::GenericDescriptorInfo& generic =
+      video_header.generic.emplace();
+  generic.dependencies = {5, 6, 7};
+  VideoFrameMetadata metadata(video_header);
+  EXPECT_THAT(metadata.GetFrameDependencies(), ElementsAre(5, 6, 7));
+}
+
+TEST(VideoFrameMetadata, FrameDependencyVectorIsEmptyForHeaderWithoutGeneric) {
+  RTPVideoHeader video_header;
+  VideoFrameMetadata metadata(video_header);
+  ASSERT_FALSE(video_header.generic);
+  EXPECT_THAT(metadata.GetFrameDependencies(), IsEmpty());
+}
+
+TEST(VideoFrameMetadata, GetDecodeTargetIndicationsReturnsCorrectValue) {
+  RTPVideoHeader video_header;
+  RTPVideoHeader::GenericDescriptorInfo& generic =
+      video_header.generic.emplace();
+  generic.decode_target_indications = {DecodeTargetIndication::kSwitch};
+  VideoFrameMetadata metadata(video_header);
+  EXPECT_THAT(metadata.GetDecodeTargetIndications(),
+              ElementsAre(DecodeTargetIndication::kSwitch));
+}
+
+TEST(VideoFrameMetadata,
+     DecodeTargetIndicationsVectorIsEmptyForHeaderWithoutGeneric) {
+  RTPVideoHeader video_header;
+  VideoFrameMetadata metadata(video_header);
+  ASSERT_FALSE(video_header.generic);
+  EXPECT_THAT(metadata.GetDecodeTargetIndications(), IsEmpty());
+}
+
+}  // namespace
+}  // namespace webrtc