diff options
author | Marina Ciocea <marinaciocea@webrtc.org> | 2020-03-31 22:41:30 +0200 |
---|---|---|
committer | Commit Bot <commit-bot@chromium.org> | 2020-03-31 21:59:26 +0000 |
commit | 65674d83e1f59c2233a26987916caed81cee62f5 (patch) | |
tree | 83049dcb7f252fc4a81c9fa853ec4e6fccd4a741 /audio | |
parent | 21c80320caa5739b33d17ecc60ee4899e204b66e (diff) | |
download | webrtc-65674d83e1f59c2233a26987916caed81cee62f5.tar.gz |
Transform encoded frames in ChannelSend.
This change is part of the implementation of the Insertable Streams Web
API: https://github.com/alvestrand/webrtc-media-streams/blob/master/explainer.md
Design doc for WebRTC library changes:
http://doc/1eiLkjNUkRy2FssCPLUp6eH08BZuXXoHfbbBP1ZN7EVk
Bug: webrtc:11380
Change-Id: I75444283ddb7f8db742687b497bf532c6dda47be
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/171871
Commit-Queue: Marina Ciocea <marinaciocea@webrtc.org>
Reviewed-by: Per Ã…hgren <peah@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#30952}
Diffstat (limited to 'audio')
-rw-r--r-- | audio/BUILD.gn | 2 | ||||
-rw-r--r-- | audio/channel_send.cc | 55 | ||||
-rw-r--r-- | audio/channel_send_frame_transformer_delegate.cc | 116 | ||||
-rw-r--r-- | audio/channel_send_frame_transformer_delegate.h | 80 |
4 files changed, 248 insertions, 5 deletions
diff --git a/audio/BUILD.gn b/audio/BUILD.gn index 7da3c79677..cc52664312 100644 --- a/audio/BUILD.gn +++ b/audio/BUILD.gn @@ -28,6 +28,8 @@ rtc_library("audio") { "channel_receive.h", "channel_send.cc", "channel_send.h", + "channel_send_frame_transformer_delegate.cc", + "channel_send_frame_transformer_delegate.h", "conversion.h", "null_audio_poller.cc", "null_audio_poller.h", diff --git a/audio/channel_send.cc b/audio/channel_send.cc index d8ac39c8e2..21f36aaa17 100644 --- a/audio/channel_send.cc +++ b/audio/channel_send.cc @@ -21,6 +21,7 @@ #include "api/call/transport.h" #include "api/crypto/frame_encryptor_interface.h" #include "api/rtc_event_log/rtc_event_log.h" +#include "audio/channel_send_frame_transformer_delegate.h" #include "audio/utility/audio_frame_operations.h" #include "call/rtp_transport_controller_send_interface.h" #include "logging/rtc_event_log/events/rtc_event_audio_playout.h" @@ -170,6 +171,9 @@ class ChannelSend : public ChannelSendInterface, void OnReceivedRtt(int64_t rtt_ms); + void InitFrameTransformerDelegate( + rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer); + // Thread checkers document and lock usage of some methods on voe::Channel to // specific threads we know about. The goal is to eventually split up // voe::Channel into parts with single-threaded semantics, and thereby reduce @@ -224,9 +228,11 @@ class ChannelSend : public ChannelSendInterface, // E2EE Frame Encryption Options const webrtc::CryptoOptions crypto_options_; - // Frame transformer used by insertable streams to transform encoded frames. - rtc::scoped_refptr<FrameTransformerInterface> frame_transformer_ - RTC_GUARDED_BY(encoder_queue_); + // Delegates calls to a frame transformer to transform audio, and + // receives callbacks with the transformed frames; delegates calls to + // ChannelSend::SendRtpAudio to send the transformed audio. + rtc::scoped_refptr<ChannelSendFrameTransformerDelegate> + frame_transformer_delegate_ RTC_GUARDED_BY(encoder_queue_); rtc::CriticalSection bitrate_crit_section_; int configured_bitrate_bps_ RTC_GUARDED_BY(bitrate_crit_section_) = 0; @@ -379,6 +385,14 @@ int32_t ChannelSend::SendData(AudioFrameType frameType, int64_t absolute_capture_timestamp_ms) { RTC_DCHECK_RUN_ON(&encoder_queue_); rtc::ArrayView<const uint8_t> payload(payloadData, payloadSize); + if (frame_transformer_delegate_) { + // Asynchronously transform the payload before sending it. After the payload + // is transformed, the delegate will call SendRtpAudio to send it. + frame_transformer_delegate_->Transform( + frameType, payloadType, rtp_timestamp, payloadData, payloadSize, + absolute_capture_timestamp_ms, _rtpRtcpModule->SSRC()); + return 0; + } return SendRtpAudio(frameType, payloadType, rtp_timestamp, payload, absolute_capture_timestamp_ms); } @@ -491,7 +505,6 @@ ChannelSend::ChannelSend( new RateLimiter(clock, kMaxRetransmissionWindowMs)), frame_encryptor_(frame_encryptor), crypto_options_(crypto_options), - frame_transformer_(std::move(frame_transformer)), encoder_queue_(task_queue_factory->CreateTaskQueue( "AudioEncoder", TaskQueueFactory::Priority::NORMAL)) { @@ -532,11 +545,17 @@ ChannelSend::ChannelSend( int error = audio_coding_->RegisterTransportCallback(this); RTC_DCHECK_EQ(0, error); + if (frame_transformer) + InitFrameTransformerDelegate(std::move(frame_transformer)); } ChannelSend::~ChannelSend() { RTC_DCHECK(construction_thread_.IsCurrent()); + // Resets the delegate's callback to ChannelSend::SendRtpAudio. + if (frame_transformer_delegate_) + frame_transformer_delegate_->Reset(); + StopSend(); int error = audio_coding_->RegisterTransportCallback(NULL); RTC_DCHECK_EQ(0, error); @@ -915,10 +934,13 @@ void ChannelSend::SetFrameEncryptor( void ChannelSend::SetEncoderToPacketizerFrameTransformer( rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer) { RTC_DCHECK_RUN_ON(&worker_thread_checker_); + if (!frame_transformer) + return; + encoder_queue_.PostTask( [this, frame_transformer = std::move(frame_transformer)]() mutable { RTC_DCHECK_RUN_ON(&encoder_queue_); - frame_transformer_ = std::move(frame_transformer); + InitFrameTransformerDelegate(std::move(frame_transformer)); }); } @@ -928,6 +950,29 @@ void ChannelSend::OnReceivedRtt(int64_t rtt_ms) { [rtt_ms](AudioEncoder* encoder) { encoder->OnReceivedRtt(rtt_ms); }); } +void ChannelSend::InitFrameTransformerDelegate( + rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer) { + RTC_DCHECK_RUN_ON(&encoder_queue_); + RTC_DCHECK(frame_transformer); + RTC_DCHECK(!frame_transformer_delegate_); + + // Pass a callback to ChannelSend::SendRtpAudio, to be called by the delegate + // to send the transformed audio. + ChannelSendFrameTransformerDelegate::SendFrameCallback send_audio_callback = + [this](AudioFrameType frameType, uint8_t payloadType, + uint32_t rtp_timestamp, rtc::ArrayView<const uint8_t> payload, + int64_t absolute_capture_timestamp_ms) { + RTC_DCHECK_RUN_ON(&encoder_queue_); + return SendRtpAudio(frameType, payloadType, rtp_timestamp, payload, + absolute_capture_timestamp_ms); + }; + frame_transformer_delegate_ = + new rtc::RefCountedObject<ChannelSendFrameTransformerDelegate>( + std::move(send_audio_callback), std::move(frame_transformer), + &encoder_queue_); + frame_transformer_delegate_->Init(); +} + } // namespace std::unique_ptr<ChannelSendInterface> CreateChannelSend( diff --git a/audio/channel_send_frame_transformer_delegate.cc b/audio/channel_send_frame_transformer_delegate.cc new file mode 100644 index 0000000000..53df6b0ff0 --- /dev/null +++ b/audio/channel_send_frame_transformer_delegate.cc @@ -0,0 +1,116 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "audio/channel_send_frame_transformer_delegate.h" + +#include <utility> + +namespace webrtc { +namespace { + +class TransformableAudioFrame : public TransformableFrameInterface { + public: + TransformableAudioFrame(AudioFrameType frame_type, + uint8_t payload_type, + uint32_t rtp_timestamp, + const uint8_t* payload_data, + size_t payload_size, + int64_t absolute_capture_timestamp_ms, + uint32_t ssrc) + : frame_type_(frame_type), + payload_type_(payload_type), + rtp_timestamp_(rtp_timestamp), + payload_(payload_data, payload_size), + absolute_capture_timestamp_ms_(absolute_capture_timestamp_ms), + ssrc_(ssrc) {} + ~TransformableAudioFrame() override = default; + rtc::ArrayView<const uint8_t> GetData() const override { return payload_; } + void SetData(rtc::ArrayView<const uint8_t> data) override { + payload_.SetData(data.data(), data.size()); + } + uint32_t GetTimestamp() const override { return rtp_timestamp_; } + uint32_t GetSsrc() const override { return ssrc_; } + + AudioFrameType GetFrameType() const { return frame_type_; } + uint8_t GetPayloadType() const { return payload_type_; } + int64_t GetAbsoluteCaptureTimestampMs() const { + return absolute_capture_timestamp_ms_; + } + + private: + AudioFrameType frame_type_; + uint8_t payload_type_; + uint32_t rtp_timestamp_; + rtc::Buffer payload_; + int64_t absolute_capture_timestamp_ms_; + uint32_t ssrc_; +}; +} // namespace + +ChannelSendFrameTransformerDelegate::ChannelSendFrameTransformerDelegate( + SendFrameCallback send_frame_callback, + rtc::scoped_refptr<FrameTransformerInterface> frame_transformer, + rtc::TaskQueue* encoder_queue) + : send_frame_callback_(send_frame_callback), + frame_transformer_(std::move(frame_transformer)), + encoder_queue_(encoder_queue) {} + +void ChannelSendFrameTransformerDelegate::Init() { + frame_transformer_->RegisterTransformedFrameCallback( + rtc::scoped_refptr<TransformedFrameCallback>(this)); +} + +void ChannelSendFrameTransformerDelegate::Reset() { + frame_transformer_->UnregisterTransformedFrameCallback(); + frame_transformer_ = nullptr; + + rtc::CritScope lock(&send_lock_); + send_frame_callback_ = SendFrameCallback(); +} + +void ChannelSendFrameTransformerDelegate::Transform( + AudioFrameType frame_type, + uint8_t payload_type, + uint32_t rtp_timestamp, + const uint8_t* payload_data, + size_t payload_size, + int64_t absolute_capture_timestamp_ms, + uint32_t ssrc) { + frame_transformer_->Transform(std::make_unique<TransformableAudioFrame>( + frame_type, payload_type, rtp_timestamp, payload_data, payload_size, + absolute_capture_timestamp_ms, ssrc)); +} + +void ChannelSendFrameTransformerDelegate::OnTransformedFrame( + std::unique_ptr<TransformableFrameInterface> frame) { + rtc::CritScope lock(&send_lock_); + if (!send_frame_callback_) + return; + rtc::scoped_refptr<ChannelSendFrameTransformerDelegate> delegate = this; + encoder_queue_->PostTask( + [delegate = std::move(delegate), frame = std::move(frame)]() mutable { + delegate->SendFrame(std::move(frame)); + }); +} + +void ChannelSendFrameTransformerDelegate::SendFrame( + std::unique_ptr<TransformableFrameInterface> frame) const { + rtc::CritScope lock(&send_lock_); + RTC_DCHECK_RUN_ON(encoder_queue_); + if (!send_frame_callback_) + return; + auto* transformed_frame = static_cast<TransformableAudioFrame*>(frame.get()); + send_frame_callback_( + transformed_frame->GetFrameType(), transformed_frame->GetPayloadType(), + transformed_frame->GetTimestamp(), transformed_frame->GetData(), + transformed_frame->GetAbsoluteCaptureTimestampMs()); +} + +} // namespace webrtc diff --git a/audio/channel_send_frame_transformer_delegate.h b/audio/channel_send_frame_transformer_delegate.h new file mode 100644 index 0000000000..f2655e7c51 --- /dev/null +++ b/audio/channel_send_frame_transformer_delegate.h @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef AUDIO_CHANNEL_SEND_FRAME_TRANSFORMER_DELEGATE_H_ +#define AUDIO_CHANNEL_SEND_FRAME_TRANSFORMER_DELEGATE_H_ + +#include <memory> + +#include "api/frame_transformer_interface.h" +#include "modules/audio_coding/include/audio_coding_module_typedefs.h" +#include "rtc_base/buffer.h" +#include "rtc_base/critical_section.h" +#include "rtc_base/synchronization/sequence_checker.h" +#include "rtc_base/task_queue.h" + +namespace webrtc { + +// Delegates calls to FrameTransformerInterface to transform frames, and to +// ChannelSend to send the transformed frames using |send_frame_callback_| on +// the |encoder_queue_|. +// OnTransformedFrame() can be called from any thread, the delegate ensures +// thread-safe access to the ChannelSend callback. +class ChannelSendFrameTransformerDelegate : public TransformedFrameCallback { + public: + using SendFrameCallback = + std::function<int32_t(AudioFrameType frameType, + uint8_t payloadType, + uint32_t rtp_timestamp, + rtc::ArrayView<const uint8_t> payload, + int64_t absolute_capture_timestamp_ms)>; + ChannelSendFrameTransformerDelegate( + SendFrameCallback send_frame_callback, + rtc::scoped_refptr<FrameTransformerInterface> frame_transformer, + rtc::TaskQueue* encoder_queue); + + // Registers |this| as callback for |frame_transformer_|, to get the + // transformed frames. + void Init(); + + // Unregisters and releases the |frame_transformer_| reference, and resets + // |send_frame_callback_| under lock. Called from ChannelSend destructor to + // prevent running the callback on a dangling channel. + void Reset(); + + // Delegates the call to FrameTransformerInterface::TransformFrame, to + // transform the frame asynchronously. + void Transform(AudioFrameType frame_type, + uint8_t payload_type, + uint32_t rtp_timestamp, + const uint8_t* payload_data, + size_t payload_size, + int64_t absolute_capture_timestamp_ms, + uint32_t ssrc); + + // Implements TransformedFrameCallback. Can be called on any thread. + void OnTransformedFrame( + std::unique_ptr<TransformableFrameInterface> frame) override; + + // Delegates the call to ChannelSend::SendRtpAudio on the |encoder_queue_|, + // by calling |send_audio_callback_|. + void SendFrame(std::unique_ptr<TransformableFrameInterface> frame) const; + + protected: + ~ChannelSendFrameTransformerDelegate() override = default; + + private: + rtc::CriticalSection send_lock_; + SendFrameCallback send_frame_callback_ RTC_GUARDED_BY(send_lock_); + rtc::scoped_refptr<FrameTransformerInterface> frame_transformer_; + rtc::TaskQueue* encoder_queue_ RTC_GUARDED_BY(send_lock_); +}; +} // namespace webrtc +#endif // AUDIO_CHANNEL_SEND_FRAME_TRANSFORMER_DELEGATE_H_ |