add RTCRtpSender.generateKeyFrame

defined in
  https://w3c.github.io/webrtc-encoded-transform/#rtcrtpsender-extension

Note: this does not implement the "rid(s)" parameter which will be done in a future CL.

VP8 still synchronizes keyframes on all layers even when asked for ones on individual layers while H264 (when implemented as three different encoders in SimulcastEncoderAdapter) can actually utilize this.

This does not change the behavior when receiving a RTCP PLI for a particular layer.

BUG=chromium:1354101

Change-Id: Ic8b14d155242e32c9aeafa55fe6652f346ac76b8
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/274169
Reviewed-by: Harald Alvestrand <hta@webrtc.org>
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Commit-Queue: Philipp Hancke <phancke@microsoft.com>
Cr-Commit-Position: refs/heads/main@{#38472}

15 files changed, 80 insertions, 15 deletions

diff --git a/api/rtp_sender_interface.h b/api/rtp_sender_interface.h
index 500bd252b8..6fc658f9ee 100644
--- a/api/rtp_sender_interface.h
+++ b/api/rtp_sender_interface.h
@@ -104,6 +104,9 @@ class RTC_EXPORT RtpSenderInterface : public rtc::RefCountInterface {
       std::unique_ptr<VideoEncoderFactory::EncoderSelectorInterface>
           encoder_selector) = 0;
 
+  // TODO(crbug.com/1354101): make pure virtual again after Chrome roll.
+  virtual RTCError GenerateKeyFrame() { return RTCError::OK(); }
+
  protected:
   ~RtpSenderInterface() override = default;
 };
diff --git a/call/video_send_stream.h b/call/video_send_stream.h
index 3a3ccce0bf..be6aa4a614 100644
--- a/call/video_send_stream.h
+++ b/call/video_send_stream.h
@@ -252,6 +252,8 @@ class VideoSendStream {
 
   virtual Stats GetStats() = 0;
 
+  virtual void GenerateKeyFrame() = 0;
+
  protected:
   virtual ~VideoSendStream() {}
 };
diff --git a/media/base/fake_media_engine.cc b/media/base/fake_media_engine.cc
index 7692efe468..60f158eb2c 100644
--- a/media/base/fake_media_engine.cc
+++ b/media/base/fake_media_engine.cc
@@ -427,7 +427,8 @@ void FakeVideoMediaChannel::SetRecordableEncodedFrameCallback(
 void FakeVideoMediaChannel::ClearRecordableEncodedFrameCallback(uint32_t ssrc) {
 }
 
-void FakeVideoMediaChannel::GenerateKeyFrame(uint32_t ssrc) {}
+void FakeVideoMediaChannel::RequestRecvKeyFrame(uint32_t ssrc) {}
+void FakeVideoMediaChannel::GenerateSendKeyFrame(uint32_t ssrc) {}
 
 FakeVoiceEngine::FakeVoiceEngine() : fail_create_channel_(false) {
   // Add a fake audio codec. Note that the name must not be "" as there are
diff --git a/media/base/fake_media_engine.h b/media/base/fake_media_engine.h
index 55c85d7c64..b98a2da950 100644
--- a/media/base/fake_media_engine.h
+++ b/media/base/fake_media_engine.h
@@ -462,7 +462,8 @@ class FakeVideoMediaChannel : public RtpHelper<VideoMediaChannel> {
       std::function<void(const webrtc::RecordableEncodedFrame&)> callback)
       override;
   void ClearRecordableEncodedFrameCallback(uint32_t ssrc) override;
-  void GenerateKeyFrame(uint32_t ssrc) override;
+  void RequestRecvKeyFrame(uint32_t ssrc) override;
+  void GenerateSendKeyFrame(uint32_t ssrc) override;
 
  private:
   bool SetRecvCodecs(const std::vector<VideoCodec>& codecs);
diff --git a/media/base/media_channel.h b/media/base/media_channel.h
index 924e8621ab..ef90484f8c 100644
--- a/media/base/media_channel.h
+++ b/media/base/media_channel.h
@@ -921,8 +921,11 @@ class VideoMediaChannel : public MediaChannel, public Delayable {
       std::function<void(const webrtc::RecordableEncodedFrame&)> callback) = 0;
   // Clear recordable encoded frame callback for `ssrc`
   virtual void ClearRecordableEncodedFrameCallback(uint32_t ssrc) = 0;
-  // Cause generation of a keyframe for `ssrc`
-  virtual void GenerateKeyFrame(uint32_t ssrc) = 0;
+  // Request generation of a keyframe for `ssrc` on a receiving channel via
+  // RTCP feedback.
+  virtual void RequestRecvKeyFrame(uint32_t ssrc) = 0;
+  // Cause generation of a keyframe for `ssrc` on a sending channel.
+  virtual void GenerateSendKeyFrame(uint32_t ssrc) = 0;
 
   virtual std::vector<webrtc::RtpSource> GetSources(uint32_t ssrc) const = 0;
 };
diff --git a/media/engine/fake_webrtc_call.h b/media/engine/fake_webrtc_call.h
index 311e35a7a9..65ee0d5b17 100644
--- a/media/engine/fake_webrtc_call.h
+++ b/media/engine/fake_webrtc_call.h
@@ -194,6 +194,7 @@ class FakeVideoSendStream final
   rtc::VideoSourceInterface<webrtc::VideoFrame>* source() const {
     return source_;
   }
+  void GenerateKeyFrame() override {}
 
  private:
   // rtc::VideoSinkInterface<VideoFrame> implementation.
diff --git a/media/engine/webrtc_video_engine.cc b/media/engine/webrtc_video_engine.cc
index 80c226790f..714cb674ce 100644
--- a/media/engine/webrtc_video_engine.cc
+++ b/media/engine/webrtc_video_engine.cc
@@ -2827,6 +2827,16 @@ void WebRtcVideoChannel::WebRtcVideoSendStream::RecreateWebRtcStream() {
   }
 }
 
+void WebRtcVideoChannel::WebRtcVideoSendStream::GenerateKeyFrame() {
+  RTC_DCHECK_RUN_ON(&thread_checker_);
+  if (stream_ != NULL) {
+    stream_->GenerateKeyFrame();
+  } else {
+    RTC_LOG(LS_WARNING)
+        << "Absent send stream; ignoring request to generate keyframe.";
+  }
+}
+
 WebRtcVideoChannel::WebRtcVideoReceiveStream::WebRtcVideoReceiveStream(
     WebRtcVideoChannel* channel,
     webrtc::Call* call,
@@ -3551,11 +3561,11 @@ void WebRtcVideoChannel::ClearRecordableEncodedFrameCallback(uint32_t ssrc) {
   }
 }
 
-void WebRtcVideoChannel::GenerateKeyFrame(uint32_t ssrc) {
+void WebRtcVideoChannel::RequestRecvKeyFrame(uint32_t ssrc) {
   RTC_DCHECK_RUN_ON(&thread_checker_);
   WebRtcVideoReceiveStream* stream = FindReceiveStream(ssrc);
   if (stream) {
-    stream->GenerateKeyFrame();
+    return stream->GenerateKeyFrame();
   } else {
     RTC_LOG(LS_ERROR)
         << "Absent receive stream; ignoring key frame generation for ssrc "
@@ -3563,6 +3573,18 @@ void WebRtcVideoChannel::GenerateKeyFrame(uint32_t ssrc) {
   }
 }
 
+void WebRtcVideoChannel::GenerateSendKeyFrame(uint32_t ssrc) {
+  RTC_DCHECK_RUN_ON(&thread_checker_);
+  auto it = send_streams_.find(ssrc);
+  if (it != send_streams_.end()) {
+    it->second->GenerateKeyFrame();
+  } else {
+    RTC_LOG(LS_ERROR)
+        << "Absent send stream; ignoring key frame generation for ssrc "
+        << ssrc;
+  }
+}
+
 void WebRtcVideoChannel::SetEncoderToPacketizerFrameTransformer(
     uint32_t ssrc,
     rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer) {
diff --git a/media/engine/webrtc_video_engine.h b/media/engine/webrtc_video_engine.h
index d87a612e72..a0150a8589 100644
--- a/media/engine/webrtc_video_engine.h
+++ b/media/engine/webrtc_video_engine.h
@@ -248,7 +248,8 @@ class WebRtcVideoChannel : public VideoMediaChannel,
       std::function<void(const webrtc::RecordableEncodedFrame&)> callback)
       override;
   void ClearRecordableEncodedFrameCallback(uint32_t ssrc) override;
-  void GenerateKeyFrame(uint32_t ssrc) override;
+  void RequestRecvKeyFrame(uint32_t ssrc) override;
+  void GenerateSendKeyFrame(uint32_t ssrc) override;
 
   void SetEncoderToPacketizerFrameTransformer(
       uint32_t ssrc,
@@ -390,6 +391,7 @@ class WebRtcVideoChannel : public VideoMediaChannel,
     void SetEncoderToPacketizerFrameTransformer(
         rtc::scoped_refptr<webrtc::FrameTransformerInterface>
             frame_transformer);
+    void GenerateKeyFrame();
 
    private:
     // Parameters needed to reconstruct the underlying stream.
diff --git a/pc/rtp_sender.cc b/pc/rtp_sender.cc
index 753ac2f130..98e86b3a51 100644
--- a/pc/rtp_sender.cc
+++ b/pc/rtp_sender.cc
@@ -596,6 +596,13 @@ rtc::scoped_refptr<DtmfSenderInterface> AudioRtpSender::GetDtmfSender() const {
   return dtmf_sender_proxy_;
 }
 
+RTCError AudioRtpSender::GenerateKeyFrame() {
+  RTC_DCHECK_RUN_ON(signaling_thread_);
+  RTC_DLOG(LS_ERROR) << "Tried to get generate a key frame for audio.";
+  return RTCError(RTCErrorType::UNSUPPORTED_OPERATION,
+                  "Generating key frames for audio is not supported.");
+}
+
 void AudioRtpSender::SetSend() {
   RTC_DCHECK_RUN_ON(signaling_thread_);
   RTC_DCHECK(!stopped_);
@@ -686,6 +693,18 @@ rtc::scoped_refptr<DtmfSenderInterface> VideoRtpSender::GetDtmfSender() const {
   return nullptr;
 }
 
+RTCError VideoRtpSender::GenerateKeyFrame() {
+  RTC_DCHECK_RUN_ON(signaling_thread_);
+  if (video_media_channel() && ssrc_ && !stopped_) {
+    worker_thread_->PostTask(
+        [&] { video_media_channel()->GenerateSendKeyFrame(ssrc_); });
+  } else {
+    RTC_LOG(LS_WARNING) << "Tried to generate key frame for sender that is "
+                           "stopped or has no media channel.";
+  }
+  return RTCError::OK();
+}
+
 void VideoRtpSender::SetSend() {
   RTC_DCHECK_RUN_ON(signaling_thread_);
   RTC_DCHECK(!stopped_);
diff --git a/pc/rtp_sender.h b/pc/rtp_sender.h
index e3cd12fed4..2cfa08dc07 100644
--- a/pc/rtp_sender.h
+++ b/pc/rtp_sender.h
@@ -351,6 +351,7 @@ class AudioRtpSender : public DtmfProviderInterface, public RtpSenderBase {
   }
 
   rtc::scoped_refptr<DtmfSenderInterface> GetDtmfSender() const override;
+  RTCError GenerateKeyFrame() override;
 
  protected:
   AudioRtpSender(rtc::Thread* worker_thread,
@@ -410,6 +411,7 @@ class VideoRtpSender : public RtpSenderBase {
   }
 
   rtc::scoped_refptr<DtmfSenderInterface> GetDtmfSender() const override;
+  RTCError GenerateKeyFrame() override;
 
   RTCError CheckSVCParameters(const RtpParameters& parameters) override;
 
diff --git a/pc/rtp_sender_proxy.h b/pc/rtp_sender_proxy.h
index 140b5ff97e..376fd29d24 100644
--- a/pc/rtp_sender_proxy.h
+++ b/pc/rtp_sender_proxy.h
@@ -48,6 +48,7 @@ PROXY_METHOD1(void,
 PROXY_METHOD1(void,
               SetEncoderSelector,
               std::unique_ptr<VideoEncoderFactory::EncoderSelectorInterface>)
+PROXY_METHOD0(RTCError, GenerateKeyFrame)
 END_PROXY_MAP(RtpSender)
 
 }  // namespace webrtc
diff --git a/pc/video_rtp_receiver.cc b/pc/video_rtp_receiver.cc
index e01a33fd31..098ffde7cd 100644
--- a/pc/video_rtp_receiver.cc
+++ b/pc/video_rtp_receiver.cc
@@ -279,7 +279,7 @@ void VideoRtpReceiver::SetMediaChannel_w(cricket::MediaChannel* media_channel) {
   if (media_channel_) {
     if (saved_generate_keyframe_) {
       // TODO(bugs.webrtc.org/8694): Stop using 0 to mean unsignalled SSRC
-      media_channel_->GenerateKeyFrame(ssrc_.value_or(0));
+      media_channel_->RequestRecvKeyFrame(ssrc_.value_or(0));
       saved_generate_keyframe_ = false;
     }
     if (encoded_sink_enabled) {
@@ -331,7 +331,7 @@ void VideoRtpReceiver::OnGenerateKeyFrame() {
     return;
   }
   // TODO(bugs.webrtc.org/8694): Stop using 0 to mean unsignalled SSRC
-  media_channel_->GenerateKeyFrame(ssrc_.value_or(0));
+  media_channel_->RequestRecvKeyFrame(ssrc_.value_or(0));
   // We need to remember to request generation of a new key frame if the media
   // channel changes, because there's no feedback whether the keyframe
   // generation has completed on the channel.
diff --git a/pc/video_rtp_receiver_unittest.cc b/pc/video_rtp_receiver_unittest.cc
index 401987960c..c2664590ff 100644
--- a/pc/video_rtp_receiver_unittest.cc
+++ b/pc/video_rtp_receiver_unittest.cc
@@ -49,7 +49,8 @@ class VideoRtpReceiverTest : public testing::Test {
                 ClearRecordableEncodedFrameCallback,
                 (uint32_t),
                 (override));
-    MOCK_METHOD(void, GenerateKeyFrame, (uint32_t), (override));
+    MOCK_METHOD(void, RequestRecvKeyFrame, (uint32_t), (override));
+    MOCK_METHOD(void, GenerateSendKeyFrame, (uint32_t), (override));
   };
 
   class MockVideoSink : public rtc::VideoSinkInterface<RecordableEncodedFrame> {
@@ -96,7 +97,7 @@ TEST_F(VideoRtpReceiverTest, SupportsEncodedOutput) {
 }
 
 TEST_F(VideoRtpReceiverTest, GeneratesKeyFrame) {
-  EXPECT_CALL(channel_, GenerateKeyFrame(0));
+  EXPECT_CALL(channel_, RequestRecvKeyFrame(0));
   Source()->GenerateKeyFrame();
 }
 
@@ -105,17 +106,17 @@ TEST_F(VideoRtpReceiverTest,
   // A channel switch without previous call to GenerateKeyFrame shouldn't
   // cause a call to happen on the new channel.
   MockVideoMediaChannel channel2(nullptr, cricket::VideoOptions());
-  EXPECT_CALL(channel_, GenerateKeyFrame).Times(0);
-  EXPECT_CALL(channel2, GenerateKeyFrame).Times(0);
+  EXPECT_CALL(channel_, RequestRecvKeyFrame).Times(0);
+  EXPECT_CALL(channel2, RequestRecvKeyFrame).Times(0);
   SetMediaChannel(&channel2);
   Mock::VerifyAndClearExpectations(&channel2);
 
   // Generate a key frame. When we switch channel next time, we will have to
   // re-generate it as we don't know if it was eventually received
-  EXPECT_CALL(channel2, GenerateKeyFrame).Times(1);
+  EXPECT_CALL(channel2, RequestRecvKeyFrame).Times(1);
   Source()->GenerateKeyFrame();
   MockVideoMediaChannel channel3(nullptr, cricket::VideoOptions());
-  EXPECT_CALL(channel3, GenerateKeyFrame);
+  EXPECT_CALL(channel3, RequestRecvKeyFrame);
   SetMediaChannel(&channel3);
 
   // Switching to a new channel should now not cause calls to GenerateKeyFrame.
diff --git a/video/video_send_stream.cc b/video/video_send_stream.cc
index b2599987b8..f245332753 100644
--- a/video/video_send_stream.cc
+++ b/video/video_send_stream.cc
@@ -343,5 +343,11 @@ void VideoSendStream::DeliverRtcp(const uint8_t* packet, size_t length) {
   send_stream_.DeliverRtcp(packet, length);
 }
 
+void VideoSendStream::GenerateKeyFrame() {
+  if (video_stream_encoder_) {
+    video_stream_encoder_->SendKeyFrame();
+  }
+}
+
 }  // namespace internal
 }  // namespace webrtc
diff --git a/video/video_send_stream.h b/video/video_send_stream.h
index 5b4323d329..a7763731b7 100644
--- a/video/video_send_stream.h
+++ b/video/video_send_stream.h
@@ -93,6 +93,7 @@ class VideoSendStream : public webrtc::VideoSendStream {
 
   void StopPermanentlyAndGetRtpStates(RtpStateMap* rtp_state_map,
                                       RtpPayloadStateMap* payload_state_map);
+  void GenerateKeyFrame() override;
 
  private:
   friend class test::VideoSendStreamPeer;