diff options
author | Torne (Richard Coles) <torne@google.com> | 2014-06-09 12:00:27 +0100 |
---|---|---|
committer | Torne (Richard Coles) <torne@google.com> | 2014-06-09 12:00:27 +0100 |
commit | 46d4c2bc3267f3f028f39e7e311b0f89aba2e4fd (patch) | |
tree | ed52337c337f5fd1db77873d9ff980ca3e334b35 /media/cast | |
parent | 7ef4c70daab901f557268ad466f62cd2f7896916 (diff) | |
download | chromium_org-46d4c2bc3267f3f028f39e7e311b0f89aba2e4fd.tar.gz |
Merge from Chromium at DEPS revision 275586
This commit was generated by merge_to_master.py.
Change-Id: Ief3a0ffd810858bfddbe0ec5931e3ee90d53f78c
Diffstat (limited to 'media/cast')
56 files changed, 1413 insertions, 2435 deletions
diff --git a/media/cast/audio_receiver/audio_receiver.h b/media/cast/audio_receiver/audio_receiver.h deleted file mode 100644 index 87c5147b50..0000000000 --- a/media/cast/audio_receiver/audio_receiver.h +++ /dev/null @@ -1,210 +0,0 @@ -// Copyright 2013 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#ifndef MEDIA_CAST_AUDIO_RECEIVER_AUDIO_RECEIVER_H_ -#define MEDIA_CAST_AUDIO_RECEIVER_AUDIO_RECEIVER_H_ - -#include "base/basictypes.h" -#include "base/callback.h" -#include "base/macros.h" -#include "base/memory/ref_counted.h" -#include "base/memory/scoped_ptr.h" -#include "base/memory/weak_ptr.h" -#include "base/threading/non_thread_safe.h" -#include "base/time/tick_clock.h" -#include "base/time/time.h" -#include "media/cast/base/clock_drift_smoother.h" -#include "media/cast/cast_config.h" -#include "media/cast/cast_environment.h" -#include "media/cast/cast_receiver.h" -#include "media/cast/framer/framer.h" -#include "media/cast/rtcp/receiver_rtcp_event_subscriber.h" -#include "media/cast/rtcp/rtcp.h" -#include "media/cast/rtp_receiver/rtp_receiver.h" -#include "media/cast/rtp_receiver/rtp_receiver_defines.h" -#include "media/cast/transport/utility/transport_encryption_handler.h" - -namespace media { -namespace cast { - -class AudioDecoder; - -// AudioReceiver receives packets out-of-order while clients make requests for -// complete frames in-order. (A frame consists of one or more packets.) -// -// AudioReceiver also includes logic for computing the playout time for each -// frame, accounting for a constant targeted playout delay. The purpose of the -// playout delay is to provide a fixed window of time between the capture event -// on the sender and the playout on the receiver. This is important because -// each step of the pipeline (i.e., encode frame, then transmit/retransmit from -// the sender, then receive and re-order packets on the receiver, then decode -// frame) can vary in duration and is typically very hard to predict. -// -// Two types of frames can be requested: 1) A frame of decoded audio data; or 2) -// a frame of still-encoded audio data, to be passed into an external audio -// decoder. Each request for a frame includes a callback which AudioReceiver -// guarantees will be called at some point in the future unless the -// AudioReceiver is destroyed. Clients should generally limit the number of -// outstanding requests (perhaps to just one or two). -// -// This class is not thread safe. Should only be called from the Main cast -// thread. -class AudioReceiver : public RtpReceiver, - public RtpPayloadFeedback, - public base::NonThreadSafe, - public base::SupportsWeakPtr<AudioReceiver> { - public: - AudioReceiver(scoped_refptr<CastEnvironment> cast_environment, - const FrameReceiverConfig& audio_config, - transport::PacedPacketSender* const packet_sender); - - virtual ~AudioReceiver(); - - // Request a decoded audio frame. The audio signal data returned in the - // callback will have the sampling rate and number of channels as requested in - // the configuration that was passed to the ctor. - // - // The given |callback| is guaranteed to be run at some point in the future, - // even if to respond with NULL at shutdown time. - void GetRawAudioFrame(const AudioFrameDecodedCallback& callback); - - // Request an encoded audio frame. - // - // The given |callback| is guaranteed to be run at some point in the future, - // even if to respond with NULL at shutdown time. - void GetEncodedAudioFrame(const FrameEncodedCallback& callback); - - // Deliver another packet, possibly a duplicate, and possibly out-of-order. - void IncomingPacket(scoped_ptr<Packet> packet); - - protected: - friend class AudioReceiverTest; // Invokes OnReceivedPayloadData(). - - virtual void OnReceivedPayloadData(const uint8* payload_data, - size_t payload_size, - const RtpCastHeader& rtp_header) OVERRIDE; - - // RtpPayloadFeedback implementation. - virtual void CastFeedback(const RtcpCastMessage& cast_message) OVERRIDE; - - private: - // Processes ready-to-consume packets from |framer_|, decrypting each packet's - // payload data, and then running the enqueued callbacks in order (one for - // each packet). This method may post a delayed task to re-invoke itself in - // the future to wait for missing/incomplete frames. - void EmitAvailableEncodedFrames(); - - // Clears the |is_waiting_for_consecutive_frame_| flag and invokes - // EmitAvailableEncodedFrames(). - void EmitAvailableEncodedFramesAfterWaiting(); - - // Feeds an EncodedFrame into |audio_decoder_|. GetRawAudioFrame() uses this - // as a callback for GetEncodedAudioFrame(). - void DecodeEncodedAudioFrame( - const AudioFrameDecodedCallback& callback, - scoped_ptr<transport::EncodedFrame> encoded_frame); - - // Computes the playout time for a frame with the given |rtp_timestamp|. - // Because lip-sync info is refreshed regularly, calling this method with the - // same argument may return different results. - base::TimeTicks GetPlayoutTime(uint32 rtp_timestamp) const; - - // Schedule the next RTCP report. - void ScheduleNextRtcpReport(); - - // Actually send the next RTCP report. - void SendNextRtcpReport(); - - // Schedule timing for the next cast message. - void ScheduleNextCastMessage(); - - // Actually send the next cast message. - void SendNextCastMessage(); - - // Receives an AudioBus from |audio_decoder_|, logs the event, and passes the - // data on by running the given |callback|. This method is static to ensure - // it can be called after an AudioReceiver instance is destroyed. - // DecodeEncodedAudioFrame() uses this as a callback for - // AudioDecoder::DecodeFrame(). - static void EmitRawAudioFrame( - const scoped_refptr<CastEnvironment>& cast_environment, - const AudioFrameDecodedCallback& callback, - uint32 frame_id, - uint32 rtp_timestamp, - const base::TimeTicks& playout_time, - scoped_ptr<AudioBus> audio_bus, - bool is_continuous); - - const scoped_refptr<CastEnvironment> cast_environment_; - - // Subscribes to raw events. - // Processes raw audio events to be sent over to the cast sender via RTCP. - ReceiverRtcpEventSubscriber event_subscriber_; - - // Configured audio codec. - const transport::AudioCodec codec_; - - // RTP timebase: The number of RTP units advanced per one second. For audio, - // this is the sampling rate. - const int frequency_; - - // The total amount of time between a frame's capture/recording on the sender - // and its playback on the receiver (i.e., shown to a user). This is fixed as - // a value large enough to give the system sufficient time to encode, - // transmit/retransmit, receive, decode, and render; given its run-time - // environment (sender/receiver hardware performance, network conditions, - // etc.). - const base::TimeDelta target_playout_delay_; - - // Hack: This is used in logic that determines whether to skip frames. - const base::TimeDelta expected_frame_duration_; - - // Set to false initially, then set to true after scheduling the periodic - // sending of reports back to the sender. Reports are first scheduled just - // after receiving a first packet (since the first packet identifies the - // sender for the remainder of the session). - bool reports_are_scheduled_; - - // Assembles packets into frames, providing this receiver with complete, - // decodable EncodedFrames. - Framer framer_; - - // Decodes frames into raw audio for playback. - scoped_ptr<AudioDecoder> audio_decoder_; - - // Manages sending/receiving of RTCP packets, including sender/receiver - // reports. - Rtcp rtcp_; - - // Decrypts encrypted frames. - transport::TransportEncryptionHandler decryptor_; - - // Outstanding callbacks to run to deliver on client requests for frames. - std::list<FrameEncodedCallback> frame_request_queue_; - - // True while there's an outstanding task to re-invoke - // EmitAvailableEncodedFrames(). - bool is_waiting_for_consecutive_frame_; - - // This mapping allows us to log AUDIO_ACK_SENT as a frame event. In addition - // it allows the event to be transmitted via RTCP. - RtpTimestamp frame_id_to_rtp_timestamp_[256]; - - // Lip-sync values used to compute the playout time of each frame from its RTP - // timestamp. These are updated each time the first packet of a frame is - // received. - RtpTimestamp lip_sync_rtp_timestamp_; - base::TimeTicks lip_sync_reference_time_; - ClockDriftSmoother lip_sync_drift_; - - // NOTE: Weak pointers must be invalidated before all other member variables. - base::WeakPtrFactory<AudioReceiver> weak_factory_; - - DISALLOW_COPY_AND_ASSIGN(AudioReceiver); -}; - -} // namespace cast -} // namespace media - -#endif // MEDIA_CAST_AUDIO_RECEIVER_AUDIO_RECEIVER_H_ diff --git a/media/cast/audio_receiver/audio_receiver_unittest.cc b/media/cast/audio_receiver/audio_receiver_unittest.cc deleted file mode 100644 index e53c1b9310..0000000000 --- a/media/cast/audio_receiver/audio_receiver_unittest.cc +++ /dev/null @@ -1,261 +0,0 @@ -// Copyright 2013 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include <deque> -#include <utility> - -#include "base/bind.h" -#include "base/memory/ref_counted.h" -#include "base/memory/scoped_ptr.h" -#include "base/test/simple_test_tick_clock.h" -#include "media/cast/audio_receiver/audio_receiver.h" -#include "media/cast/cast_defines.h" -#include "media/cast/cast_environment.h" -#include "media/cast/logging/simple_event_subscriber.h" -#include "media/cast/rtcp/test_rtcp_packet_builder.h" -#include "media/cast/test/fake_single_thread_task_runner.h" -#include "media/cast/test/utility/default_config.h" -#include "media/cast/transport/pacing/mock_paced_packet_sender.h" -#include "testing/gmock/include/gmock/gmock.h" - -using ::testing::_; - -namespace media { -namespace cast { - -namespace { - -const uint32 kFirstFrameId = 1234; -const int kPlayoutDelayMillis = 300; - -class FakeAudioClient { - public: - FakeAudioClient() : num_called_(0) {} - virtual ~FakeAudioClient() {} - - void AddExpectedResult(uint32 expected_frame_id, - const base::TimeTicks& expected_playout_time) { - expected_results_.push_back( - std::make_pair(expected_frame_id, expected_playout_time)); - } - - void DeliverEncodedAudioFrame( - scoped_ptr<transport::EncodedFrame> audio_frame) { - SCOPED_TRACE(::testing::Message() << "num_called_ is " << num_called_); - ASSERT_FALSE(!audio_frame) - << "If at shutdown: There were unsatisfied requests enqueued."; - ASSERT_FALSE(expected_results_.empty()); - EXPECT_EQ(expected_results_.front().first, audio_frame->frame_id); - EXPECT_EQ(expected_results_.front().second, audio_frame->reference_time); - expected_results_.pop_front(); - num_called_++; - } - - int number_times_called() const { return num_called_; } - - private: - std::deque<std::pair<uint32, base::TimeTicks> > expected_results_; - int num_called_; - - DISALLOW_COPY_AND_ASSIGN(FakeAudioClient); -}; - -} // namespace - -class AudioReceiverTest : public ::testing::Test { - protected: - AudioReceiverTest() { - // Configure the audio receiver to use PCM16. - audio_config_ = GetDefaultAudioReceiverConfig(); - audio_config_.rtp_max_delay_ms = kPlayoutDelayMillis; - audio_config_.frequency = 16000; - audio_config_.channels = 1; - audio_config_.codec.audio = transport::kPcm16; - testing_clock_ = new base::SimpleTestTickClock(); - testing_clock_->Advance(base::TimeTicks::Now() - base::TimeTicks()); - start_time_ = testing_clock_->NowTicks(); - task_runner_ = new test::FakeSingleThreadTaskRunner(testing_clock_); - - cast_environment_ = new CastEnvironment( - scoped_ptr<base::TickClock>(testing_clock_).Pass(), - task_runner_, - task_runner_, - task_runner_); - - receiver_.reset(new AudioReceiver(cast_environment_, audio_config_, - &mock_transport_)); - } - - virtual ~AudioReceiverTest() {} - - virtual void SetUp() { - payload_.assign(kMaxIpPacketSize, 0); - rtp_header_.is_key_frame = true; - rtp_header_.frame_id = kFirstFrameId; - rtp_header_.packet_id = 0; - rtp_header_.max_packet_id = 0; - rtp_header_.reference_frame_id = rtp_header_.frame_id; - rtp_header_.rtp_timestamp = 0; - } - - void FeedOneFrameIntoReceiver() { - receiver_->OnReceivedPayloadData( - payload_.data(), payload_.size(), rtp_header_); - } - - void FeedLipSyncInfoIntoReceiver() { - const base::TimeTicks now = testing_clock_->NowTicks(); - const int64 rtp_timestamp = (now - start_time_) * - audio_config_.frequency / base::TimeDelta::FromSeconds(1); - CHECK_LE(0, rtp_timestamp); - uint32 ntp_seconds; - uint32 ntp_fraction; - ConvertTimeTicksToNtp(now, &ntp_seconds, &ntp_fraction); - TestRtcpPacketBuilder rtcp_packet; - rtcp_packet.AddSrWithNtp(audio_config_.incoming_ssrc, - ntp_seconds, ntp_fraction, - static_cast<uint32>(rtp_timestamp)); - receiver_->IncomingPacket(rtcp_packet.GetPacket().Pass()); - } - - FrameReceiverConfig audio_config_; - std::vector<uint8> payload_; - RtpCastHeader rtp_header_; - base::SimpleTestTickClock* testing_clock_; // Owned by CastEnvironment. - base::TimeTicks start_time_; - transport::MockPacedPacketSender mock_transport_; - scoped_refptr<test::FakeSingleThreadTaskRunner> task_runner_; - scoped_refptr<CastEnvironment> cast_environment_; - FakeAudioClient fake_audio_client_; - - // Important for the AudioReceiver to be declared last, since its dependencies - // must remain alive until after its destruction. - scoped_ptr<AudioReceiver> receiver_; -}; - -TEST_F(AudioReceiverTest, ReceivesOneFrame) { - SimpleEventSubscriber event_subscriber; - cast_environment_->Logging()->AddRawEventSubscriber(&event_subscriber); - - EXPECT_CALL(mock_transport_, SendRtcpPacket(_, _)) - .WillRepeatedly(testing::Return(true)); - - FeedLipSyncInfoIntoReceiver(); - task_runner_->RunTasks(); - - // Enqueue a request for an audio frame. - receiver_->GetEncodedAudioFrame( - base::Bind(&FakeAudioClient::DeliverEncodedAudioFrame, - base::Unretained(&fake_audio_client_))); - - // The request should not be satisfied since no packets have been received. - task_runner_->RunTasks(); - EXPECT_EQ(0, fake_audio_client_.number_times_called()); - - // Deliver one audio frame to the receiver and expect to get one frame back. - const base::TimeDelta target_playout_delay = - base::TimeDelta::FromMilliseconds(kPlayoutDelayMillis); - fake_audio_client_.AddExpectedResult( - kFirstFrameId, testing_clock_->NowTicks() + target_playout_delay); - FeedOneFrameIntoReceiver(); - task_runner_->RunTasks(); - EXPECT_EQ(1, fake_audio_client_.number_times_called()); - - std::vector<FrameEvent> frame_events; - event_subscriber.GetFrameEventsAndReset(&frame_events); - - ASSERT_TRUE(!frame_events.empty()); - EXPECT_EQ(FRAME_ACK_SENT, frame_events.begin()->type); - EXPECT_EQ(AUDIO_EVENT, frame_events.begin()->media_type); - EXPECT_EQ(rtp_header_.frame_id, frame_events.begin()->frame_id); - EXPECT_EQ(rtp_header_.rtp_timestamp, frame_events.begin()->rtp_timestamp); - - cast_environment_->Logging()->RemoveRawEventSubscriber(&event_subscriber); -} - -TEST_F(AudioReceiverTest, ReceivesFramesSkippingWhenAppropriate) { - EXPECT_CALL(mock_transport_, SendRtcpPacket(_, _)) - .WillRepeatedly(testing::Return(true)); - - const uint32 rtp_advance_per_frame = - audio_config_.frequency / audio_config_.max_frame_rate; - const base::TimeDelta time_advance_per_frame = - base::TimeDelta::FromSeconds(1) / audio_config_.max_frame_rate; - - FeedLipSyncInfoIntoReceiver(); - task_runner_->RunTasks(); - const base::TimeTicks first_frame_capture_time = testing_clock_->NowTicks(); - - // Enqueue a request for an audio frame. - const FrameEncodedCallback frame_encoded_callback = - base::Bind(&FakeAudioClient::DeliverEncodedAudioFrame, - base::Unretained(&fake_audio_client_)); - receiver_->GetEncodedAudioFrame(frame_encoded_callback); - task_runner_->RunTasks(); - EXPECT_EQ(0, fake_audio_client_.number_times_called()); - - // Receive one audio frame and expect to see the first request satisfied. - const base::TimeDelta target_playout_delay = - base::TimeDelta::FromMilliseconds(kPlayoutDelayMillis); - fake_audio_client_.AddExpectedResult( - kFirstFrameId, first_frame_capture_time + target_playout_delay); - rtp_header_.rtp_timestamp = 0; - FeedOneFrameIntoReceiver(); - task_runner_->RunTasks(); - EXPECT_EQ(1, fake_audio_client_.number_times_called()); - - // Enqueue a second request for an audio frame, but it should not be - // fulfilled yet. - receiver_->GetEncodedAudioFrame(frame_encoded_callback); - task_runner_->RunTasks(); - EXPECT_EQ(1, fake_audio_client_.number_times_called()); - - // Receive one audio frame out-of-order: Make sure that we are not continuous - // and that the RTP timestamp represents a time in the future. - rtp_header_.is_key_frame = false; - rtp_header_.frame_id = kFirstFrameId + 2; - rtp_header_.reference_frame_id = 0; - rtp_header_.rtp_timestamp += 2 * rtp_advance_per_frame; - fake_audio_client_.AddExpectedResult( - kFirstFrameId + 2, - first_frame_capture_time + 2 * time_advance_per_frame + - target_playout_delay); - FeedOneFrameIntoReceiver(); - - // Frame 2 should not come out at this point in time. - task_runner_->RunTasks(); - EXPECT_EQ(1, fake_audio_client_.number_times_called()); - - // Enqueue a third request for an audio frame. - receiver_->GetEncodedAudioFrame(frame_encoded_callback); - task_runner_->RunTasks(); - EXPECT_EQ(1, fake_audio_client_.number_times_called()); - - // Now, advance time forward such that the receiver is convinced it should - // skip Frame 2. Frame 3 is emitted (to satisfy the second request) because a - // decision was made to skip over the no-show Frame 2. - testing_clock_->Advance(2 * time_advance_per_frame + target_playout_delay); - task_runner_->RunTasks(); - EXPECT_EQ(2, fake_audio_client_.number_times_called()); - - // Receive Frame 4 and expect it to fulfill the third request immediately. - rtp_header_.frame_id = kFirstFrameId + 3; - rtp_header_.reference_frame_id = rtp_header_.frame_id - 1; - rtp_header_.rtp_timestamp += rtp_advance_per_frame; - fake_audio_client_.AddExpectedResult( - kFirstFrameId + 3, first_frame_capture_time + 3 * time_advance_per_frame + - target_playout_delay); - FeedOneFrameIntoReceiver(); - task_runner_->RunTasks(); - EXPECT_EQ(3, fake_audio_client_.number_times_called()); - - // Move forward to the playout time of an unreceived Frame 5. Expect no - // additional frames were emitted. - testing_clock_->Advance(3 * time_advance_per_frame); - task_runner_->RunTasks(); - EXPECT_EQ(3, fake_audio_client_.number_times_called()); -} - -} // namespace cast -} // namespace media diff --git a/media/cast/audio_sender/audio_sender.cc b/media/cast/audio_sender/audio_sender.cc index 513665a78e..f5183e957d 100644 --- a/media/cast/audio_sender/audio_sender.cc +++ b/media/cast/audio_sender/audio_sender.cc @@ -33,9 +33,9 @@ AudioSender::AudioSender(scoped_refptr<CastEnvironment> cast_environment, audio_config.rtp_config.ssrc, audio_config.incoming_feedback_ssrc, audio_config.rtcp_c_name, - true), + AUDIO_EVENT), num_aggressive_rtcp_reports_sent_(0), - cast_initialization_cb_(STATUS_AUDIO_UNINITIALIZED), + cast_initialization_status_(STATUS_AUDIO_UNINITIALIZED), weak_factory_(this) { rtcp_.SetCastReceiverEventHistorySize(kReceiverRtcpEventHistorySize); if (!audio_config.use_external_encoder) { @@ -44,7 +44,10 @@ AudioSender::AudioSender(scoped_refptr<CastEnvironment> cast_environment, audio_config, base::Bind(&AudioSender::SendEncodedAudioFrame, weak_factory_.GetWeakPtr()))); - cast_initialization_cb_ = audio_encoder_->InitializationResult(); + cast_initialization_status_ = audio_encoder_->InitializationResult(); + } else { + NOTREACHED(); // No support for external audio encoding. + cast_initialization_status_ = STATUS_AUDIO_INITIALIZED; } media::cast::transport::CastTransportAudioConfig transport_config; @@ -55,6 +58,8 @@ AudioSender::AudioSender(scoped_refptr<CastEnvironment> cast_environment, transport_config.rtp.max_outstanding_frames = audio_config.rtp_config.max_delay_ms / 100 + 1; transport_sender_->InitializeAudio(transport_config); + + memset(frame_id_to_rtp_timestamp_, 0, sizeof(frame_id_to_rtp_timestamp_)); } AudioSender::~AudioSender() {} @@ -62,6 +67,10 @@ AudioSender::~AudioSender() {} void AudioSender::InsertAudio(scoped_ptr<AudioBus> audio_bus, const base::TimeTicks& recorded_time) { DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); + if (cast_initialization_status_ != STATUS_AUDIO_INITIALIZED) { + NOTREACHED(); + return; + } DCHECK(audio_encoder_.get()) << "Invalid internal state"; audio_encoder_->InsertAudio(audio_bus.Pass(), recorded_time); } @@ -69,6 +78,7 @@ void AudioSender::InsertAudio(scoped_ptr<AudioBus> audio_bus, void AudioSender::SendEncodedAudioFrame( scoped_ptr<transport::EncodedFrame> audio_frame) { DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); + DCHECK(!audio_frame->reference_time.is_null()); rtp_timestamp_helper_.StoreLatestTime(audio_frame->reference_time, audio_frame->rtp_timestamp); @@ -88,6 +98,8 @@ void AudioSender::SendEncodedAudioFrame( SendRtcpReport(is_last_aggressive_report); } + frame_id_to_rtp_timestamp_[audio_frame->frame_id & 0xff] = + audio_frame->rtp_timestamp; transport_sender_->InsertCodedAudioFrame(*audio_frame); } @@ -152,8 +164,12 @@ void AudioSender::OnReceivedCastFeedback(const RtcpCastMessage& cast_feedback) { if (!cast_feedback.missing_frames_and_packets_.empty()) { ResendPackets(cast_feedback.missing_frames_and_packets_); } - VLOG(2) << "Received audio ACK " - << static_cast<int>(cast_feedback.ack_frame_id_); + uint32 acked_frame_id = static_cast<uint32>(cast_feedback.ack_frame_id_); + VLOG(2) << "Received audio ACK: " << acked_frame_id; + cast_environment_->Logging()->InsertFrameEvent( + cast_environment_->Clock()->NowTicks(), + FRAME_ACK_RECEIVED, AUDIO_EVENT, + frame_id_to_rtp_timestamp_[acked_frame_id & 0xff], acked_frame_id); } } // namespace cast diff --git a/media/cast/audio_sender/audio_sender.h b/media/cast/audio_sender/audio_sender.h index 8911320b3f..03de633bef 100644 --- a/media/cast/audio_sender/audio_sender.h +++ b/media/cast/audio_sender/audio_sender.h @@ -16,7 +16,6 @@ #include "media/cast/cast_config.h" #include "media/cast/rtcp/rtcp.h" #include "media/cast/rtp_timestamp_helper.h" -#include "media/cast/transport/rtp_sender/rtp_sender.h" namespace media { namespace cast { @@ -36,18 +35,17 @@ class AudioSender : public RtcpSenderFeedback, virtual ~AudioSender(); CastInitializationStatus InitializationResult() const { - return cast_initialization_cb_; + return cast_initialization_status_; } + // Note: It is invalid to call this method if InitializationResult() returns + // anything but STATUS_AUDIO_INITIALIZED. void InsertAudio(scoped_ptr<AudioBus> audio_bus, const base::TimeTicks& recorded_time); // Only called from the main cast thread. void IncomingRtcpPacket(scoped_ptr<Packet> packet); - protected: - void SendEncodedAudioFrame(scoped_ptr<transport::EncodedFrame> audio_frame); - private: void ResendPackets( const MissingFramesAndPacketsMap& missing_frames_and_packets); @@ -55,6 +53,9 @@ class AudioSender : public RtcpSenderFeedback, void ScheduleNextRtcpReport(); void SendRtcpReport(bool schedule_future_reports); + // Called by the |audio_encoder_| with the next EncodedFrame to send. + void SendEncodedAudioFrame(scoped_ptr<transport::EncodedFrame> audio_frame); + virtual void OnReceivedCastFeedback(const RtcpCastMessage& cast_feedback) OVERRIDE; @@ -64,7 +65,13 @@ class AudioSender : public RtcpSenderFeedback, RtpTimestampHelper rtp_timestamp_helper_; Rtcp rtcp_; int num_aggressive_rtcp_reports_sent_; - CastInitializationStatus cast_initialization_cb_; + + // If this sender is ready for use, this is STATUS_AUDIO_INITIALIZED. + CastInitializationStatus cast_initialization_status_; + + // Used to map the lower 8 bits of the frame id to a RTP timestamp. This is + // good enough as we only use it for logging. + RtpTimestamp frame_id_to_rtp_timestamp_[256]; // NOTE: Weak pointers must be invalidated before all other member variables. base::WeakPtrFactory<AudioSender> weak_factory_; diff --git a/media/cast/cast.gyp b/media/cast/cast.gyp index 4f0c2dd4a9..5de8796079 100644 --- a/media/cast/cast.gyp +++ b/media/cast/cast.gyp @@ -99,13 +99,7 @@ '<(DEPTH)/ui/gfx/gfx.gyp:gfx_geometry', ], 'sources': [ - 'audio_receiver/audio_decoder.h', - 'audio_receiver/audio_decoder.cc', - 'audio_receiver/audio_receiver.h', - 'audio_receiver/audio_receiver.cc', 'cast_receiver.h', - 'cast_receiver_impl.cc', - 'cast_receiver_impl.h', 'framer/cast_message_builder.cc', 'framer/cast_message_builder.h', 'framer/frame_buffer.cc', @@ -114,18 +108,20 @@ 'framer/frame_id_map.h', 'framer/framer.cc', 'framer/framer.h', + 'receiver/audio_decoder.cc', + 'receiver/audio_decoder.h', + 'receiver/cast_receiver_impl.cc', + 'receiver/cast_receiver_impl.h', + 'receiver/frame_receiver.cc', + 'receiver/frame_receiver.h', + 'receiver/video_decoder.cc', + 'receiver/video_decoder.h', 'rtp_receiver/receiver_stats.cc', 'rtp_receiver/receiver_stats.h', - 'rtp_receiver/rtp_receiver.cc', - 'rtp_receiver/rtp_receiver.h', 'rtp_receiver/rtp_receiver_defines.cc', 'rtp_receiver/rtp_receiver_defines.h', 'rtp_receiver/rtp_parser/rtp_parser.cc', 'rtp_receiver/rtp_parser/rtp_parser.h', - 'video_receiver/video_decoder.h', - 'video_receiver/video_decoder.cc', - 'video_receiver/video_receiver.h', - 'video_receiver/video_receiver.cc', ], # source }, { @@ -221,10 +217,6 @@ 'transport/rtp_sender/rtp_sender.h', 'transport/transport/udp_transport.cc', 'transport/transport/udp_transport.h', - 'transport/transport_audio_sender.cc', - 'transport/transport_audio_sender.h', - 'transport/transport_video_sender.cc', - 'transport/transport_video_sender.h', 'transport/utility/transport_encryption_handler.cc', 'transport/utility/transport_encryption_handler.h', ], # source diff --git a/media/cast/cast_defines.h b/media/cast/cast_defines.h index afb50e0156..bf6df39120 100644 --- a/media/cast/cast_defines.h +++ b/media/cast/cast_defines.h @@ -22,11 +22,13 @@ namespace cast { const int64 kDontShowTimeoutMs = 33; const float kDefaultCongestionControlBackOff = 0.875f; const uint32 kVideoFrequency = 90000; -const int64 kSkippedFramesCheckPeriodkMs = 10000; const uint32 kStartFrameId = UINT32_C(0xffffffff); -// Number of skipped frames threshold in fps (as configured) per period above. -const int kSkippedFramesThreshold = 3; +// This is an important system-wide constant. This limits how much history the +// implementation must retain in order to process the acknowledgements of past +// frames. +const int kMaxUnackedFrames = 255; + const size_t kMaxIpPacketSize = 1500; const int kStartRttMs = 20; const int64 kCastMessageUpdateIntervalMs = 33; diff --git a/media/cast/cast_receiver.h b/media/cast/cast_receiver.h index 2d83dd263f..a9d3edeb78 100644 --- a/media/cast/cast_receiver.h +++ b/media/cast/cast_receiver.h @@ -45,27 +45,8 @@ typedef base::Callback<void(const scoped_refptr<media::VideoFrame>& video_frame, // dropped (i.e., frame_id should be incrementing by one each time). Note: A // NULL pointer can be returned on error. typedef base::Callback<void(scoped_ptr<transport::EncodedFrame>)> - FrameEncodedCallback; + ReceiveEncodedFrameCallback; -// This Class is thread safe. -class FrameReceiver : public base::RefCountedThreadSafe<FrameReceiver> { - public: - virtual void GetRawAudioFrame(const AudioFrameDecodedCallback& callback) = 0; - - virtual void GetCodedAudioFrame(const FrameEncodedCallback& callback) = 0; - - virtual void GetRawVideoFrame(const VideoFrameDecodedCallback& callback) = 0; - - virtual void GetEncodedVideoFrame(const FrameEncodedCallback& callback) = 0; - - protected: - virtual ~FrameReceiver() {} - - private: - friend class base::RefCountedThreadSafe<FrameReceiver>; -}; - -// This Class is thread safe. class CastReceiver { public: static scoped_ptr<CastReceiver> Create( @@ -75,13 +56,28 @@ class CastReceiver { transport::PacketSender* const packet_sender); // All received RTP and RTCP packets for the call should be sent to this - // PacketReceiver. Can be called from any function. + // PacketReceiver. Can be called from any thread. // TODO(hubbe): Replace with: // virtual void ReceivePacket(scoped_ptr<Packet> packet) = 0; virtual transport::PacketReceiverCallback packet_receiver() = 0; - // Polling interface to get audio and video frames from the CastReceiver. - virtual scoped_refptr<FrameReceiver> frame_receiver() = 0; + // Polling interface to get audio and video frames from the CastReceiver. The + // the RequestDecodedXXXXXFrame() methods utilize internal software-based + // decoding, while the RequestEncodedXXXXXFrame() methods provides + // still-encoded frames for use with external/hardware decoders. + // + // In all cases, the given |callback| is guaranteed to be run at some point in + // the future, except for those requests still enqueued at destruction time. + // + // These methods should all be called on the CastEnvironment's MAIN thread. + virtual void RequestDecodedAudioFrame( + const AudioFrameDecodedCallback& callback) = 0; + virtual void RequestEncodedAudioFrame( + const ReceiveEncodedFrameCallback& callback) = 0; + virtual void RequestDecodedVideoFrame( + const VideoFrameDecodedCallback& callback) = 0; + virtual void RequestEncodedVideoFrame( + const ReceiveEncodedFrameCallback& callback) = 0; virtual ~CastReceiver() {} }; diff --git a/media/cast/cast_receiver_impl.cc b/media/cast/cast_receiver_impl.cc deleted file mode 100644 index 661cbbe293..0000000000 --- a/media/cast/cast_receiver_impl.cc +++ /dev/null @@ -1,150 +0,0 @@ -// Copyright 2013 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include "media/cast/cast_receiver_impl.h" - -#include "base/bind.h" -#include "base/callback.h" -#include "base/logging.h" -#include "base/message_loop/message_loop.h" - -namespace media { -namespace cast { - -// The video and audio receivers should only be called from the main thread. -// LocalFrameReciever posts tasks to the main thread, making the cast interface -// thread safe. -class LocalFrameReceiver : public FrameReceiver { - public: - LocalFrameReceiver(scoped_refptr<CastEnvironment> cast_environment, - AudioReceiver* audio_receiver, - VideoReceiver* video_receiver) - : cast_environment_(cast_environment), - audio_receiver_(audio_receiver), - video_receiver_(video_receiver) {} - - virtual void GetRawVideoFrame(const VideoFrameDecodedCallback& callback) - OVERRIDE { - cast_environment_->PostTask(CastEnvironment::MAIN, - FROM_HERE, - base::Bind(&VideoReceiver::GetRawVideoFrame, - video_receiver_->AsWeakPtr(), - callback)); - } - - virtual void GetEncodedVideoFrame(const FrameEncodedCallback& callback) - OVERRIDE { - cast_environment_->PostTask(CastEnvironment::MAIN, - FROM_HERE, - base::Bind(&VideoReceiver::GetEncodedVideoFrame, - video_receiver_->AsWeakPtr(), - callback)); - } - - virtual void GetRawAudioFrame(const AudioFrameDecodedCallback& callback) - OVERRIDE { - cast_environment_->PostTask(CastEnvironment::MAIN, - FROM_HERE, - base::Bind(&AudioReceiver::GetRawAudioFrame, - audio_receiver_->AsWeakPtr(), - callback)); - } - - virtual void GetCodedAudioFrame(const FrameEncodedCallback& callback) - OVERRIDE { - cast_environment_->PostTask(CastEnvironment::MAIN, - FROM_HERE, - base::Bind(&AudioReceiver::GetEncodedAudioFrame, - audio_receiver_->AsWeakPtr(), - callback)); - } - - protected: - virtual ~LocalFrameReceiver() {} - - private: - friend class base::RefCountedThreadSafe<LocalFrameReceiver>; - - scoped_refptr<CastEnvironment> cast_environment_; - AudioReceiver* audio_receiver_; - VideoReceiver* video_receiver_; -}; - -scoped_ptr<CastReceiver> CastReceiver::Create( - scoped_refptr<CastEnvironment> cast_environment, - const FrameReceiverConfig& audio_config, - const FrameReceiverConfig& video_config, - transport::PacketSender* const packet_sender) { - return scoped_ptr<CastReceiver>(new CastReceiverImpl( - cast_environment, audio_config, video_config, packet_sender)); -} - -CastReceiverImpl::CastReceiverImpl( - scoped_refptr<CastEnvironment> cast_environment, - const FrameReceiverConfig& audio_config, - const FrameReceiverConfig& video_config, - transport::PacketSender* const packet_sender) - : pacer_(cast_environment->Clock(), - cast_environment->Logging(), - packet_sender, - cast_environment->GetTaskRunner(CastEnvironment::MAIN)), - audio_receiver_(cast_environment, audio_config, &pacer_), - video_receiver_(cast_environment, - video_config, - &pacer_), - frame_receiver_(new LocalFrameReceiver(cast_environment, - &audio_receiver_, - &video_receiver_)), - cast_environment_(cast_environment), - ssrc_of_audio_sender_(audio_config.incoming_ssrc), - ssrc_of_video_sender_(video_config.incoming_ssrc) {} - -CastReceiverImpl::~CastReceiverImpl() {} - -// The video and audio receivers should only be called from the main thread. -void CastReceiverImpl::ReceivedPacket(scoped_ptr<Packet> packet) { - const uint8_t* data = &packet->front(); - size_t length = packet->size(); - if (length < kMinLengthOfRtcp) { - VLOG(1) << "Received a packet which is too short " << length; - return; - } - uint32 ssrc_of_sender; - if (!Rtcp::IsRtcpPacket(data, length)) { - if (length < kMinLengthOfRtp) { - VLOG(1) << "Received a RTP packet which is too short " << length; - return; - } - ssrc_of_sender = RtpReceiver::GetSsrcOfSender(data, length); - } else { - ssrc_of_sender = Rtcp::GetSsrcOfSender(data, length); - } - if (ssrc_of_sender == ssrc_of_audio_sender_) { - cast_environment_->PostTask(CastEnvironment::MAIN, - FROM_HERE, - base::Bind(&AudioReceiver::IncomingPacket, - audio_receiver_.AsWeakPtr(), - base::Passed(&packet))); - } else if (ssrc_of_sender == ssrc_of_video_sender_) { - cast_environment_->PostTask(CastEnvironment::MAIN, - FROM_HERE, - base::Bind(&VideoReceiver::IncomingPacket, - video_receiver_.AsWeakPtr(), - base::Passed(&packet))); - } else { - VLOG(1) << "Received a packet with a non matching sender SSRC " - << ssrc_of_sender; - } -} - -transport::PacketReceiverCallback CastReceiverImpl::packet_receiver() { - return base::Bind(&CastReceiverImpl::ReceivedPacket, base::Unretained(this)); -} - -scoped_refptr<FrameReceiver> CastReceiverImpl::frame_receiver() { - return frame_receiver_; -} - -} // namespace cast -} // namespace media diff --git a/media/cast/cast_receiver_impl.h b/media/cast/cast_receiver_impl.h deleted file mode 100644 index ae7d50e1ae..0000000000 --- a/media/cast/cast_receiver_impl.h +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright 2013 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#ifndef MEDIA_CAST_CAST_RECEIVER_IMPL_H_ -#define MEDIA_CAST_CAST_RECEIVER_IMPL_H_ - -#include "base/memory/ref_counted.h" -#include "base/memory/scoped_ptr.h" -#include "media/cast/audio_receiver/audio_receiver.h" -#include "media/cast/cast_config.h" -#include "media/cast/cast_environment.h" -#include "media/cast/cast_receiver.h" -#include "media/cast/transport/pacing/paced_sender.h" -#include "media/cast/video_receiver/video_receiver.h" - -namespace media { -namespace cast { -// This calls is a pure owner class that group all required receive objects -// together such as pacer, packet receiver, frame receiver, audio and video -// receivers. -class CastReceiverImpl : public CastReceiver { - public: - CastReceiverImpl(scoped_refptr<CastEnvironment> cast_environment, - const FrameReceiverConfig& audio_config, - const FrameReceiverConfig& video_config, - transport::PacketSender* const packet_sender); - - virtual ~CastReceiverImpl(); - - // All received RTP and RTCP packets for the call should be sent to this - // PacketReceiver; - virtual transport::PacketReceiverCallback packet_receiver() OVERRIDE; - - // Interface to get audio and video frames from the CastReceiver. - virtual scoped_refptr<FrameReceiver> frame_receiver() OVERRIDE; - - private: - void ReceivedPacket(scoped_ptr<Packet> packet); - - transport::PacedSender pacer_; - AudioReceiver audio_receiver_; - VideoReceiver video_receiver_; - scoped_refptr<FrameReceiver> frame_receiver_; - scoped_refptr<CastEnvironment> cast_environment_; - const uint32 ssrc_of_audio_sender_; - const uint32 ssrc_of_video_sender_; - - DISALLOW_COPY_AND_ASSIGN(CastReceiverImpl); -}; - -} // namespace cast -} // namespace media - -#endif // MEDIA_CAST_CAST_RECEIVER_IMPL_ diff --git a/media/cast/cast_sender_impl.cc b/media/cast/cast_sender_impl.cc index 2bcad36cce..361e4d8dc1 100644 --- a/media/cast/cast_sender_impl.cc +++ b/media/cast/cast_sender_impl.cc @@ -97,11 +97,12 @@ void CastSenderImpl::InitializeAudio( CHECK(audio_config.use_external_encoder || cast_environment_->HasAudioThread()); + VLOG(1) << "CastSenderImpl@" << this << "::InitializeAudio()"; + audio_sender_.reset( new AudioSender(cast_environment_, audio_config, transport_sender_)); - CastInitializationStatus status = audio_sender_->InitializationResult(); - + const CastInitializationStatus status = audio_sender_->InitializationResult(); if (status == STATUS_AUDIO_INITIALIZED) { ssrc_of_audio_sender_ = audio_config.incoming_feedback_ssrc; audio_frame_input_ = @@ -118,22 +119,26 @@ void CastSenderImpl::InitializeVideo( DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); CHECK(video_config.use_external_encoder || cast_environment_->HasVideoThread()); - VLOG(1) << "CastSender::ctor"; + + VLOG(1) << "CastSenderImpl@" << this << "::InitializeVideo()"; video_sender_.reset(new VideoSender(cast_environment_, video_config, create_vea_cb, create_video_encode_mem_cb, - cast_initialization_cb, transport_sender_)); - ssrc_of_video_sender_ = video_config.incoming_feedback_ssrc; - video_frame_input_ = - new LocalVideoFrameInput(cast_environment_, video_sender_->AsWeakPtr()); + const CastInitializationStatus status = video_sender_->InitializationResult(); + if (status == STATUS_VIDEO_INITIALIZED) { + ssrc_of_video_sender_ = video_config.incoming_feedback_ssrc; + video_frame_input_ = + new LocalVideoFrameInput(cast_environment_, video_sender_->AsWeakPtr()); + } + cast_initialization_cb.Run(status); } CastSenderImpl::~CastSenderImpl() { - VLOG(1) << "CastSender::dtor"; + VLOG(1) << "CastSenderImpl@" << this << "::~CastSenderImpl()"; } // ReceivedPacket handle the incoming packets to the cast sender @@ -166,8 +171,8 @@ void CastSenderImpl::ReceivedPacket(scoped_ptr<Packet> packet) { size_t length = packet->size(); const uint8_t* data = &packet->front(); if (!Rtcp::IsRtcpPacket(data, length)) { - // We should have no incoming RTP packets. - VLOG(1) << "Unexpectedly received a RTP packet in the cast sender"; + VLOG(1) << "CastSenderImpl@" << this << "::ReceivedPacket() -- " + << "Received an invalid (non-RTCP?) packet in the cast sender."; return; } uint32 ssrc_of_sender = Rtcp::GetSsrcOfSender(data, length); @@ -192,7 +197,8 @@ void CastSenderImpl::ReceivedPacket(scoped_ptr<Packet> packet) { video_sender_->AsWeakPtr(), base::Passed(&packet))); } else { - VLOG(1) << "Received a RTCP packet with a non matching sender SSRC " + VLOG(1) << "CastSenderImpl@" << this << "::ReceivedPacket() -- " + << "Received a RTCP packet with a non matching sender SSRC " << ssrc_of_sender; } } diff --git a/media/cast/cast_testing.gypi b/media/cast/cast_testing.gypi index 666b5823fe..ff6bb57e43 100644 --- a/media/cast/cast_testing.gypi +++ b/media/cast/cast_testing.gypi @@ -67,8 +67,6 @@ ], 'sources': [ '<(DEPTH)/media/base/run_all_unittests.cc', - 'audio_receiver/audio_decoder_unittest.cc', - 'audio_receiver/audio_receiver_unittest.cc', 'audio_sender/audio_encoder_unittest.cc', 'audio_sender/audio_sender_unittest.cc', 'congestion_control/congestion_control_unittest.cc', @@ -82,6 +80,9 @@ 'logging/receiver_time_offset_estimator_impl_unittest.cc', 'logging/simple_event_subscriber_unittest.cc', 'logging/stats_event_subscriber_unittest.cc', + 'receiver/audio_decoder_unittest.cc', + 'receiver/frame_receiver_unittest.cc', + 'receiver/video_decoder_unittest.cc', 'rtcp/mock_rtcp_receiver_feedback.cc', 'rtcp/mock_rtcp_receiver_feedback.h', 'rtcp/mock_rtcp_sender_feedback.cc', @@ -117,8 +118,6 @@ 'transport/rtp_sender/rtp_packetizer/test/rtp_header_parser.cc', 'transport/rtp_sender/rtp_packetizer/test/rtp_header_parser.h', 'transport/transport/udp_transport_unittest.cc', - 'video_receiver/video_decoder_unittest.cc', - 'video_receiver/video_receiver_unittest.cc', 'video_sender/external_video_encoder_unittest.cc', 'video_sender/video_encoder_impl_unittest.cc', 'video_sender/video_sender_unittest.cc', diff --git a/media/cast/logging/proto/raw_events.proto b/media/cast/logging/proto/raw_events.proto index e94aed3868..08bf53d19a 100644 --- a/media/cast/logging/proto/raw_events.proto +++ b/media/cast/logging/proto/raw_events.proto @@ -68,6 +68,14 @@ enum EventType { PACKET_RECEIVED = 38; } +// Contains information independent of the stream that describes the system +// setup, e.g. OS and hardware info. +message GeneralDescription { + optional string product = 1; + optional string product_version = 2; + optional string os = 3; +} + // Each log will contain one |LogMetadata|. message LogMetadata { // |true| if the events are related to audio. |false| if they are related to @@ -90,9 +98,11 @@ message LogMetadata { // to a real time and date. optional int64 reference_timestamp_ms_at_unix_epoch = 5; - // Extra data to attach to the log, e.g. system info or - // experiment tags, in key-value JSON string format. + // Extra data to attach to the log, e.g. experiment tags, + // in key-value JSON string format. The data is supplied by the application. optional string extra_data = 6; + + optional GeneralDescription general_description = 7; } message AggregatedFrameEvent { diff --git a/media/cast/audio_receiver/audio_decoder.cc b/media/cast/receiver/audio_decoder.cc index e4e9a1453e..a4d1896835 100644 --- a/media/cast/audio_receiver/audio_decoder.cc +++ b/media/cast/receiver/audio_decoder.cc @@ -1,8 +1,8 @@ -// Copyright 2013 The Chromium Authors. All rights reserved. +// Copyright 2014 The Chromium Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#include "media/cast/audio_receiver/audio_decoder.h" +#include "media/cast/receiver/audio_decoder.h" #include "base/bind.h" #include "base/bind_helpers.h" @@ -200,18 +200,16 @@ class AudioDecoder::Pcm16Impl : public AudioDecoder::ImplBase { AudioDecoder::AudioDecoder( const scoped_refptr<CastEnvironment>& cast_environment, - const FrameReceiverConfig& audio_config) + int channels, + int sampling_rate, + transport::AudioCodec codec) : cast_environment_(cast_environment) { - switch (audio_config.codec.audio) { + switch (codec) { case transport::kOpus: - impl_ = new OpusImpl(cast_environment, - audio_config.channels, - audio_config.frequency); + impl_ = new OpusImpl(cast_environment, channels, sampling_rate); break; case transport::kPcm16: - impl_ = new Pcm16Impl(cast_environment, - audio_config.channels, - audio_config.frequency); + impl_ = new Pcm16Impl(cast_environment, channels, sampling_rate); break; default: NOTREACHED() << "Unknown or unspecified codec."; diff --git a/media/cast/audio_receiver/audio_decoder.h b/media/cast/receiver/audio_decoder.h index a8e264dc94..c66735e4e6 100644 --- a/media/cast/audio_receiver/audio_decoder.h +++ b/media/cast/receiver/audio_decoder.h @@ -1,9 +1,9 @@ -// Copyright 2013 The Chromium Authors. All rights reserved. +// Copyright 2014 The Chromium Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#ifndef MEDIA_CAST_AUDIO_RECEIVER_AUDIO_DECODER_H_ -#define MEDIA_CAST_AUDIO_RECEIVER_AUDIO_DECODER_H_ +#ifndef MEDIA_CAST_RECEIVER_AUDIO_DECODER_H_ +#define MEDIA_CAST_RECEIVER_AUDIO_DECODER_H_ #include "base/callback.h" #include "base/memory/ref_counted.h" @@ -27,7 +27,9 @@ class AudioDecoder { bool is_continuous)> DecodeFrameCallback; AudioDecoder(const scoped_refptr<CastEnvironment>& cast_environment, - const FrameReceiverConfig& audio_config); + int channels, + int sampling_rate, + transport::AudioCodec codec); virtual ~AudioDecoder(); // Returns STATUS_AUDIO_INITIALIZED if the decoder was successfully @@ -59,4 +61,4 @@ class AudioDecoder { } // namespace cast } // namespace media -#endif // MEDIA_CAST_AUDIO_RECEIVER_AUDIO_DECODER_H_ +#endif // MEDIA_CAST_RECEIVER_AUDIO_DECODER_H_ diff --git a/media/cast/audio_receiver/audio_decoder_unittest.cc b/media/cast/receiver/audio_decoder_unittest.cc index 11973ab9a0..6985a69423 100644 --- a/media/cast/audio_receiver/audio_decoder_unittest.cc +++ b/media/cast/receiver/audio_decoder_unittest.cc @@ -1,4 +1,4 @@ -// Copyright 2013 The Chromium Authors. All rights reserved. +// Copyright 2014 The Chromium Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. @@ -8,10 +8,9 @@ #include "base/synchronization/lock.h" #include "base/sys_byteorder.h" #include "base/time/time.h" -#include "media/cast/audio_receiver/audio_decoder.h" #include "media/cast/cast_config.h" +#include "media/cast/receiver/audio_decoder.h" #include "media/cast/test/utility/audio_utility.h" -#include "media/cast/test/utility/default_config.h" #include "media/cast/test/utility/standalone_cast_environment.h" #include "testing/gtest/include/gtest/gtest.h" #include "third_party/opus/src/include/opus.h" @@ -38,11 +37,10 @@ class AudioDecoderTest : public ::testing::TestWithParam<TestScenario> { protected: virtual void SetUp() OVERRIDE { - FrameReceiverConfig decoder_config = GetDefaultAudioReceiverConfig(); - decoder_config.frequency = GetParam().sampling_rate; - decoder_config.channels = GetParam().num_channels; - decoder_config.codec.audio = GetParam().codec; - audio_decoder_.reset(new AudioDecoder(cast_environment_, decoder_config)); + audio_decoder_.reset(new AudioDecoder(cast_environment_, + GetParam().num_channels, + GetParam().sampling_rate, + GetParam().codec)); CHECK_EQ(STATUS_AUDIO_INITIALIZED, audio_decoder_->InitializationResult()); audio_bus_factory_.reset( diff --git a/media/cast/receiver/cast_receiver_impl.cc b/media/cast/receiver/cast_receiver_impl.cc new file mode 100644 index 0000000000..7cff354c14 --- /dev/null +++ b/media/cast/receiver/cast_receiver_impl.cc @@ -0,0 +1,232 @@ +// Copyright 2014 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "media/cast/receiver/cast_receiver_impl.h" + +#include "base/bind.h" +#include "base/bind_helpers.h" +#include "base/callback.h" +#include "base/debug/trace_event.h" +#include "base/logging.h" +#include "base/message_loop/message_loop.h" +#include "media/cast/receiver/audio_decoder.h" +#include "media/cast/receiver/video_decoder.h" + +namespace media { +namespace cast { + +scoped_ptr<CastReceiver> CastReceiver::Create( + scoped_refptr<CastEnvironment> cast_environment, + const FrameReceiverConfig& audio_config, + const FrameReceiverConfig& video_config, + transport::PacketSender* const packet_sender) { + return scoped_ptr<CastReceiver>(new CastReceiverImpl( + cast_environment, audio_config, video_config, packet_sender)); +} + +CastReceiverImpl::CastReceiverImpl( + scoped_refptr<CastEnvironment> cast_environment, + const FrameReceiverConfig& audio_config, + const FrameReceiverConfig& video_config, + transport::PacketSender* const packet_sender) + : cast_environment_(cast_environment), + pacer_(cast_environment->Clock(), + cast_environment->Logging(), + packet_sender, + cast_environment->GetTaskRunner(CastEnvironment::MAIN)), + audio_receiver_(cast_environment, audio_config, AUDIO_EVENT, &pacer_), + video_receiver_(cast_environment, video_config, VIDEO_EVENT, &pacer_), + ssrc_of_audio_sender_(audio_config.incoming_ssrc), + ssrc_of_video_sender_(video_config.incoming_ssrc), + num_audio_channels_(audio_config.channels), + audio_sampling_rate_(audio_config.frequency), + audio_codec_(audio_config.codec.audio), + video_codec_(video_config.codec.video) {} + +CastReceiverImpl::~CastReceiverImpl() {} + +void CastReceiverImpl::DispatchReceivedPacket(scoped_ptr<Packet> packet) { + const uint8_t* const data = &packet->front(); + const size_t length = packet->size(); + + uint32 ssrc_of_sender; + if (Rtcp::IsRtcpPacket(data, length)) { + ssrc_of_sender = Rtcp::GetSsrcOfSender(data, length); + } else if (!FrameReceiver::ParseSenderSsrc(data, length, &ssrc_of_sender)) { + VLOG(1) << "Invalid RTP packet."; + return; + } + + base::WeakPtr<FrameReceiver> target; + if (ssrc_of_sender == ssrc_of_video_sender_) { + target = video_receiver_.AsWeakPtr(); + } else if (ssrc_of_sender == ssrc_of_audio_sender_) { + target = audio_receiver_.AsWeakPtr(); + } else { + VLOG(1) << "Dropping packet with a non matching sender SSRC: " + << ssrc_of_sender; + return; + } + cast_environment_->PostTask( + CastEnvironment::MAIN, + FROM_HERE, + base::Bind(base::IgnoreResult(&FrameReceiver::ProcessPacket), + target, + base::Passed(&packet))); +} + +transport::PacketReceiverCallback CastReceiverImpl::packet_receiver() { + return base::Bind(&CastReceiverImpl::DispatchReceivedPacket, + // TODO(miu): This code structure is dangerous, since the + // callback could be stored and then invoked after + // destruction of |this|. + base::Unretained(this)); +} + +void CastReceiverImpl::RequestDecodedAudioFrame( + const AudioFrameDecodedCallback& callback) { + DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); + DCHECK(!callback.is_null()); + audio_receiver_.RequestEncodedFrame(base::Bind( + &CastReceiverImpl::DecodeEncodedAudioFrame, + // Note: Use of Unretained is safe since this Closure is guaranteed to be + // invoked or discarded by |audio_receiver_| before destruction of |this|. + base::Unretained(this), + callback)); +} + +void CastReceiverImpl::RequestEncodedAudioFrame( + const ReceiveEncodedFrameCallback& callback) { + DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); + audio_receiver_.RequestEncodedFrame(callback); +} + +void CastReceiverImpl::RequestDecodedVideoFrame( + const VideoFrameDecodedCallback& callback) { + DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); + DCHECK(!callback.is_null()); + video_receiver_.RequestEncodedFrame(base::Bind( + &CastReceiverImpl::DecodeEncodedVideoFrame, + // Note: Use of Unretained is safe since this Closure is guaranteed to be + // invoked or discarded by |video_receiver_| before destruction of |this|. + base::Unretained(this), + callback)); +} + +void CastReceiverImpl::RequestEncodedVideoFrame( + const ReceiveEncodedFrameCallback& callback) { + DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); + video_receiver_.RequestEncodedFrame(callback); +} + +void CastReceiverImpl::DecodeEncodedAudioFrame( + const AudioFrameDecodedCallback& callback, + scoped_ptr<transport::EncodedFrame> encoded_frame) { + DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); + if (!encoded_frame) { + callback.Run(make_scoped_ptr<AudioBus>(NULL), base::TimeTicks(), false); + return; + } + + if (!audio_decoder_) { + audio_decoder_.reset(new AudioDecoder(cast_environment_, + num_audio_channels_, + audio_sampling_rate_, + audio_codec_)); + } + const uint32 frame_id = encoded_frame->frame_id; + const uint32 rtp_timestamp = encoded_frame->rtp_timestamp; + const base::TimeTicks playout_time = encoded_frame->reference_time; + audio_decoder_->DecodeFrame( + encoded_frame.Pass(), + base::Bind(&CastReceiverImpl::EmitDecodedAudioFrame, + cast_environment_, + callback, + frame_id, + rtp_timestamp, + playout_time)); +} + +void CastReceiverImpl::DecodeEncodedVideoFrame( + const VideoFrameDecodedCallback& callback, + scoped_ptr<transport::EncodedFrame> encoded_frame) { + DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); + if (!encoded_frame) { + callback.Run( + make_scoped_refptr<VideoFrame>(NULL), base::TimeTicks(), false); + return; + } + + // Used by chrome/browser/extension/api/cast_streaming/performance_test.cc + TRACE_EVENT_INSTANT2( + "cast_perf_test", "PullEncodedVideoFrame", + TRACE_EVENT_SCOPE_THREAD, + "rtp_timestamp", encoded_frame->rtp_timestamp, + "render_time", encoded_frame->reference_time.ToInternalValue()); + + if (!video_decoder_) + video_decoder_.reset(new VideoDecoder(cast_environment_, video_codec_)); + const uint32 frame_id = encoded_frame->frame_id; + const uint32 rtp_timestamp = encoded_frame->rtp_timestamp; + const base::TimeTicks playout_time = encoded_frame->reference_time; + video_decoder_->DecodeFrame( + encoded_frame.Pass(), + base::Bind(&CastReceiverImpl::EmitDecodedVideoFrame, + cast_environment_, + callback, + frame_id, + rtp_timestamp, + playout_time)); +} + +// static +void CastReceiverImpl::EmitDecodedAudioFrame( + const scoped_refptr<CastEnvironment>& cast_environment, + const AudioFrameDecodedCallback& callback, + uint32 frame_id, + uint32 rtp_timestamp, + const base::TimeTicks& playout_time, + scoped_ptr<AudioBus> audio_bus, + bool is_continuous) { + DCHECK(cast_environment->CurrentlyOn(CastEnvironment::MAIN)); + if (audio_bus.get()) { + const base::TimeTicks now = cast_environment->Clock()->NowTicks(); + cast_environment->Logging()->InsertFrameEvent( + now, FRAME_DECODED, AUDIO_EVENT, rtp_timestamp, frame_id); + cast_environment->Logging()->InsertFrameEventWithDelay( + now, FRAME_PLAYOUT, AUDIO_EVENT, rtp_timestamp, frame_id, + playout_time - now); + } + callback.Run(audio_bus.Pass(), playout_time, is_continuous); +} + +// static +void CastReceiverImpl::EmitDecodedVideoFrame( + const scoped_refptr<CastEnvironment>& cast_environment, + const VideoFrameDecodedCallback& callback, + uint32 frame_id, + uint32 rtp_timestamp, + const base::TimeTicks& playout_time, + const scoped_refptr<VideoFrame>& video_frame, + bool is_continuous) { + DCHECK(cast_environment->CurrentlyOn(CastEnvironment::MAIN)); + if (video_frame) { + const base::TimeTicks now = cast_environment->Clock()->NowTicks(); + cast_environment->Logging()->InsertFrameEvent( + now, FRAME_DECODED, VIDEO_EVENT, rtp_timestamp, frame_id); + cast_environment->Logging()->InsertFrameEventWithDelay( + now, FRAME_PLAYOUT, VIDEO_EVENT, rtp_timestamp, frame_id, + playout_time - now); + + // Used by chrome/browser/extension/api/cast_streaming/performance_test.cc + TRACE_EVENT_INSTANT1( + "cast_perf_test", "FrameDecoded", + TRACE_EVENT_SCOPE_THREAD, + "rtp_timestamp", rtp_timestamp); + } + callback.Run(video_frame, playout_time, is_continuous); +} + +} // namespace cast +} // namespace media diff --git a/media/cast/receiver/cast_receiver_impl.h b/media/cast/receiver/cast_receiver_impl.h new file mode 100644 index 0000000000..c0dd5f38d1 --- /dev/null +++ b/media/cast/receiver/cast_receiver_impl.h @@ -0,0 +1,122 @@ +// Copyright 2014 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef MEDIA_CAST_RECEIVER_CAST_RECEIVER_IMPL_H_ +#define MEDIA_CAST_RECEIVER_CAST_RECEIVER_IMPL_H_ + +#include "base/memory/ref_counted.h" +#include "base/memory/scoped_ptr.h" +#include "media/cast/cast_config.h" +#include "media/cast/cast_environment.h" +#include "media/cast/cast_receiver.h" +#include "media/cast/receiver/frame_receiver.h" +#include "media/cast/transport/pacing/paced_sender.h" + +namespace media { +namespace cast { + +class AudioDecoder; +class VideoDecoder; + +// This is a pure owner class that groups all required receiver-related objects +// together, such as the paced packet sender, audio/video RTP frame receivers, +// and software decoders (created on-demand). +class CastReceiverImpl : public CastReceiver { + public: + CastReceiverImpl(scoped_refptr<CastEnvironment> cast_environment, + const FrameReceiverConfig& audio_config, + const FrameReceiverConfig& video_config, + transport::PacketSender* const packet_sender); + + virtual ~CastReceiverImpl(); + + // CastReceiver implementation. + virtual transport::PacketReceiverCallback packet_receiver() OVERRIDE; + virtual void RequestDecodedAudioFrame( + const AudioFrameDecodedCallback& callback) OVERRIDE; + virtual void RequestEncodedAudioFrame( + const ReceiveEncodedFrameCallback& callback) OVERRIDE; + virtual void RequestDecodedVideoFrame( + const VideoFrameDecodedCallback& callback) OVERRIDE; + virtual void RequestEncodedVideoFrame( + const ReceiveEncodedFrameCallback& callback) OVERRIDE; + + private: + // Forwards |packet| to a specific RTP frame receiver, or drops it if SSRC + // does not map to one of the receivers. + void DispatchReceivedPacket(scoped_ptr<Packet> packet); + + // Feeds an EncodedFrame into |audio_decoder_|. RequestDecodedAudioFrame() + // uses this as a callback for RequestEncodedAudioFrame(). + void DecodeEncodedAudioFrame( + const AudioFrameDecodedCallback& callback, + scoped_ptr<transport::EncodedFrame> encoded_frame); + + // Feeds an EncodedFrame into |video_decoder_|. RequestDecodedVideoFrame() + // uses this as a callback for RequestEncodedVideoFrame(). + void DecodeEncodedVideoFrame( + const VideoFrameDecodedCallback& callback, + scoped_ptr<transport::EncodedFrame> encoded_frame); + + // Receives an AudioBus from |audio_decoder_|, logs the event, and passes the + // data on by running the given |callback|. This method is static to ensure + // it can be called after a CastReceiverImpl instance is destroyed. + // DecodeEncodedAudioFrame() uses this as a callback for + // AudioDecoder::DecodeFrame(). + static void EmitDecodedAudioFrame( + const scoped_refptr<CastEnvironment>& cast_environment, + const AudioFrameDecodedCallback& callback, + uint32 frame_id, + uint32 rtp_timestamp, + const base::TimeTicks& playout_time, + scoped_ptr<AudioBus> audio_bus, + bool is_continuous); + + // Receives a VideoFrame from |video_decoder_|, logs the event, and passes the + // data on by running the given |callback|. This method is static to ensure + // it can be called after a CastReceiverImpl instance is destroyed. + // DecodeEncodedVideoFrame() uses this as a callback for + // VideoDecoder::DecodeFrame(). + static void EmitDecodedVideoFrame( + const scoped_refptr<CastEnvironment>& cast_environment, + const VideoFrameDecodedCallback& callback, + uint32 frame_id, + uint32 rtp_timestamp, + const base::TimeTicks& playout_time, + const scoped_refptr<VideoFrame>& video_frame, + bool is_continuous); + + const scoped_refptr<CastEnvironment> cast_environment_; + transport::PacedSender pacer_; + FrameReceiver audio_receiver_; + FrameReceiver video_receiver_; + + // Used by DispatchReceivedPacket() to direct packets to the appropriate frame + // receiver. + const uint32 ssrc_of_audio_sender_; + const uint32 ssrc_of_video_sender_; + + // Parameters for the decoders that are created on-demand. The values here + // might be nonsense if the client of CastReceiverImpl never intends to use + // the internal software-based decoders. + const int num_audio_channels_; + const int audio_sampling_rate_; + const transport::AudioCodec audio_codec_; + const transport::VideoCodec video_codec_; + + // Created on-demand to decode frames from |audio_receiver_| into AudioBuses + // for playback. + scoped_ptr<AudioDecoder> audio_decoder_; + + // Created on-demand to decode frames from |video_receiver_| into VideoFrame + // images for playback. + scoped_ptr<VideoDecoder> video_decoder_; + + DISALLOW_COPY_AND_ASSIGN(CastReceiverImpl); +}; + +} // namespace cast +} // namespace media + +#endif // MEDIA_CAST_RECEIVER_CAST_RECEIVER_IMPL_ diff --git a/media/cast/audio_receiver/audio_receiver.cc b/media/cast/receiver/frame_receiver.cc index 1f47827ec6..e189cc99a7 100644 --- a/media/cast/audio_receiver/audio_receiver.cc +++ b/media/cast/receiver/frame_receiver.cc @@ -1,16 +1,16 @@ -// Copyright 2013 The Chromium Authors. All rights reserved. +// Copyright 2014 The Chromium Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#include "media/cast/audio_receiver/audio_receiver.h" +#include "media/cast/receiver/frame_receiver.h" #include <algorithm> +#include "base/big_endian.h" #include "base/bind.h" #include "base/logging.h" #include "base/message_loop/message_loop.h" -#include "media/cast/audio_receiver/audio_decoder.h" -#include "media/cast/transport/cast_transport_defines.h" +#include "media/cast/cast_environment.h" namespace { const int kMinSchedulingDelayMs = 1; @@ -19,56 +19,103 @@ const int kMinSchedulingDelayMs = 1; namespace media { namespace cast { -AudioReceiver::AudioReceiver(scoped_refptr<CastEnvironment> cast_environment, - const FrameReceiverConfig& audio_config, - transport::PacedPacketSender* const packet_sender) - : RtpReceiver(cast_environment->Clock(), &audio_config, NULL), - cast_environment_(cast_environment), - event_subscriber_(kReceiverRtcpEventHistorySize, AUDIO_EVENT), - codec_(audio_config.codec.audio), - frequency_(audio_config.frequency), +FrameReceiver::FrameReceiver( + const scoped_refptr<CastEnvironment>& cast_environment, + const FrameReceiverConfig& config, + EventMediaType event_media_type, + transport::PacedPacketSender* const packet_sender) + : cast_environment_(cast_environment), + packet_parser_(config.incoming_ssrc, config.rtp_payload_type), + stats_(cast_environment->Clock()), + event_media_type_(event_media_type), + event_subscriber_(kReceiverRtcpEventHistorySize, event_media_type), + rtp_timebase_(config.frequency), target_playout_delay_( - base::TimeDelta::FromMilliseconds(audio_config.rtp_max_delay_ms)), + base::TimeDelta::FromMilliseconds(config.rtp_max_delay_ms)), expected_frame_duration_( - base::TimeDelta::FromSeconds(1) / audio_config.max_frame_rate), + base::TimeDelta::FromSeconds(1) / config.max_frame_rate), reports_are_scheduled_(false), framer_(cast_environment->Clock(), this, - audio_config.incoming_ssrc, + config.incoming_ssrc, true, - audio_config.rtp_max_delay_ms * audio_config.max_frame_rate / - 1000), - rtcp_(cast_environment, + config.rtp_max_delay_ms * config.max_frame_rate / 1000), + rtcp_(cast_environment_, NULL, NULL, packet_sender, - GetStatistics(), - audio_config.rtcp_mode, - base::TimeDelta::FromMilliseconds(audio_config.rtcp_interval), - audio_config.feedback_ssrc, - audio_config.incoming_ssrc, - audio_config.rtcp_c_name, - true), + &stats_, + config.rtcp_mode, + base::TimeDelta::FromMilliseconds(config.rtcp_interval), + config.feedback_ssrc, + config.incoming_ssrc, + config.rtcp_c_name, + event_media_type), is_waiting_for_consecutive_frame_(false), lip_sync_drift_(ClockDriftSmoother::GetDefaultTimeConstant()), weak_factory_(this) { - DCHECK_GT(audio_config.rtp_max_delay_ms, 0); - DCHECK_GT(audio_config.max_frame_rate, 0); - audio_decoder_.reset(new AudioDecoder(cast_environment, audio_config)); - decryptor_.Initialize(audio_config.aes_key, audio_config.aes_iv_mask); + DCHECK_GT(config.rtp_max_delay_ms, 0); + DCHECK_GT(config.max_frame_rate, 0); + decryptor_.Initialize(config.aes_key, config.aes_iv_mask); rtcp_.SetTargetDelay(target_playout_delay_); cast_environment_->Logging()->AddRawEventSubscriber(&event_subscriber_); memset(frame_id_to_rtp_timestamp_, 0, sizeof(frame_id_to_rtp_timestamp_)); } -AudioReceiver::~AudioReceiver() { +FrameReceiver::~FrameReceiver() { DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); cast_environment_->Logging()->RemoveRawEventSubscriber(&event_subscriber_); } -void AudioReceiver::OnReceivedPayloadData(const uint8* payload_data, - size_t payload_size, - const RtpCastHeader& rtp_header) { +void FrameReceiver::RequestEncodedFrame( + const ReceiveEncodedFrameCallback& callback) { + DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); + frame_request_queue_.push_back(callback); + EmitAvailableEncodedFrames(); +} + +bool FrameReceiver::ProcessPacket(scoped_ptr<Packet> packet) { + DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); + + if (Rtcp::IsRtcpPacket(&packet->front(), packet->size())) { + rtcp_.IncomingRtcpPacket(&packet->front(), packet->size()); + } else { + RtpCastHeader rtp_header; + const uint8* payload_data; + size_t payload_size; + if (!packet_parser_.ParsePacket(&packet->front(), + packet->size(), + &rtp_header, + &payload_data, + &payload_size)) { + return false; + } + + ProcessParsedPacket(rtp_header, payload_data, payload_size); + stats_.UpdateStatistics(rtp_header); + } + + if (!reports_are_scheduled_) { + ScheduleNextRtcpReport(); + ScheduleNextCastMessage(); + reports_are_scheduled_ = true; + } + + return true; +} + +// static +bool FrameReceiver::ParseSenderSsrc(const uint8* packet, + size_t length, + uint32* ssrc) { + base::BigEndianReader big_endian_reader( + reinterpret_cast<const char*>(packet), length); + return big_endian_reader.Skip(8) && big_endian_reader.ReadU32(ssrc); +} + +void FrameReceiver::ProcessParsedPacket(const RtpCastHeader& rtp_header, + const uint8* payload_data, + size_t payload_size) { DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); const base::TimeTicks now = cast_environment_->Clock()->NowTicks(); @@ -76,7 +123,7 @@ void AudioReceiver::OnReceivedPayloadData(const uint8* payload_data, frame_id_to_rtp_timestamp_[rtp_header.frame_id & 0xff] = rtp_header.rtp_timestamp; cast_environment_->Logging()->InsertPacketEvent( - now, PACKET_RECEIVED, AUDIO_EVENT, rtp_header.rtp_timestamp, + now, PACKET_RECEIVED, event_media_type_, rtp_header.rtp_timestamp, rtp_header.frame_id, rtp_header.packet_id, rtp_header.max_packet_id, payload_size); @@ -111,81 +158,35 @@ void AudioReceiver::OnReceivedPayloadData(const uint8* payload_data, } else { lip_sync_reference_time_ += RtpDeltaToTimeDelta( static_cast<int32>(fresh_sync_rtp - lip_sync_rtp_timestamp_), - frequency_); + rtp_timebase_); } lip_sync_rtp_timestamp_ = fresh_sync_rtp; lip_sync_drift_.Update( now, fresh_sync_reference - lip_sync_reference_time_); } - // Frame not complete; wait for more packets. - if (!complete) - return; - - EmitAvailableEncodedFrames(); -} - -void AudioReceiver::GetRawAudioFrame( - const AudioFrameDecodedCallback& callback) { - DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); - DCHECK(!callback.is_null()); - DCHECK(audio_decoder_.get()); - GetEncodedAudioFrame(base::Bind( - &AudioReceiver::DecodeEncodedAudioFrame, - // Note: Use of Unretained is safe since this Closure is guaranteed to be - // invoked before destruction of |this|. - base::Unretained(this), - callback)); + // Another frame is complete from a non-duplicate packet. Attempt to emit + // more frames to satisfy enqueued requests. + if (complete) + EmitAvailableEncodedFrames(); } -void AudioReceiver::DecodeEncodedAudioFrame( - const AudioFrameDecodedCallback& callback, - scoped_ptr<transport::EncodedFrame> encoded_frame) { +void FrameReceiver::CastFeedback(const RtcpCastMessage& cast_message) { DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); - if (!encoded_frame) { - callback.Run(make_scoped_ptr<AudioBus>(NULL), base::TimeTicks(), false); - return; - } - const uint32 frame_id = encoded_frame->frame_id; - const uint32 rtp_timestamp = encoded_frame->rtp_timestamp; - const base::TimeTicks playout_time = encoded_frame->reference_time; - audio_decoder_->DecodeFrame(encoded_frame.Pass(), - base::Bind(&AudioReceiver::EmitRawAudioFrame, - cast_environment_, - callback, - frame_id, - rtp_timestamp, - playout_time)); -} -// static -void AudioReceiver::EmitRawAudioFrame( - const scoped_refptr<CastEnvironment>& cast_environment, - const AudioFrameDecodedCallback& callback, - uint32 frame_id, - uint32 rtp_timestamp, - const base::TimeTicks& playout_time, - scoped_ptr<AudioBus> audio_bus, - bool is_continuous) { - DCHECK(cast_environment->CurrentlyOn(CastEnvironment::MAIN)); - if (audio_bus.get()) { - const base::TimeTicks now = cast_environment->Clock()->NowTicks(); - cast_environment->Logging()->InsertFrameEvent( - now, FRAME_DECODED, AUDIO_EVENT, rtp_timestamp, frame_id); - cast_environment->Logging()->InsertFrameEventWithDelay( - now, FRAME_PLAYOUT, AUDIO_EVENT, rtp_timestamp, frame_id, - playout_time - now); - } - callback.Run(audio_bus.Pass(), playout_time, is_continuous); -} + base::TimeTicks now = cast_environment_->Clock()->NowTicks(); + RtpTimestamp rtp_timestamp = + frame_id_to_rtp_timestamp_[cast_message.ack_frame_id_ & 0xff]; + cast_environment_->Logging()->InsertFrameEvent( + now, FRAME_ACK_SENT, event_media_type_, + rtp_timestamp, cast_message.ack_frame_id_); -void AudioReceiver::GetEncodedAudioFrame(const FrameEncodedCallback& callback) { - DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); - frame_request_queue_.push_back(callback); - EmitAvailableEncodedFrames(); + ReceiverRtcpEventSubscriber::RtcpEventMultiMap rtcp_events; + event_subscriber_.GetRtcpEventsAndReset(&rtcp_events); + rtcp_.SendRtcpFromRtpReceiver(&cast_message, &rtcp_events); } -void AudioReceiver::EmitAvailableEncodedFrames() { +void FrameReceiver::EmitAvailableEncodedFrames() { DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); while (!frame_request_queue_.empty()) { @@ -199,8 +200,8 @@ void AudioReceiver::EmitAvailableEncodedFrames() { if (!framer_.GetEncodedFrame(encoded_frame.get(), &is_consecutively_next_frame, &have_multiple_complete_frames)) { - VLOG(1) << "Wait for more audio packets to produce a completed frame."; - return; // OnReceivedPayloadData() will invoke this method in the future. + VLOG(1) << "Wait for more packets to produce a completed frame."; + return; // ProcessParsedPacket() will invoke this method in the future. } const base::TimeTicks now = cast_environment_->Clock()->NowTicks(); @@ -229,7 +230,7 @@ void AudioReceiver::EmitAvailableEncodedFrames() { cast_environment_->PostDelayedTask( CastEnvironment::MAIN, FROM_HERE, - base::Bind(&AudioReceiver::EmitAvailableEncodedFramesAfterWaiting, + base::Bind(&FrameReceiver::EmitAvailableEncodedFramesAfterWaiting, weak_factory_.GetWeakPtr()), playout_time - now); } @@ -239,16 +240,15 @@ void AudioReceiver::EmitAvailableEncodedFrames() { // Decrypt the payload data in the frame, if crypto is being used. if (decryptor_.initialized()) { - std::string decrypted_audio_data; + std::string decrypted_data; if (!decryptor_.Decrypt(encoded_frame->frame_id, encoded_frame->data, - &decrypted_audio_data)) { - // Decryption failed. Give up on this frame, releasing it from the - // jitter buffer. + &decrypted_data)) { + // Decryption failed. Give up on this frame. framer_.ReleaseFrame(encoded_frame->frame_id); continue; } - encoded_frame->data.swap(decrypted_audio_data); + encoded_frame->data.swap(decrypted_data); } // At this point, we have a decrypted EncodedFrame ready to be emitted. @@ -262,95 +262,64 @@ void AudioReceiver::EmitAvailableEncodedFrames() { } } -void AudioReceiver::EmitAvailableEncodedFramesAfterWaiting() { +void FrameReceiver::EmitAvailableEncodedFramesAfterWaiting() { DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); DCHECK(is_waiting_for_consecutive_frame_); is_waiting_for_consecutive_frame_ = false; EmitAvailableEncodedFrames(); } -base::TimeTicks AudioReceiver::GetPlayoutTime(uint32 rtp_timestamp) const { +base::TimeTicks FrameReceiver::GetPlayoutTime(uint32 rtp_timestamp) const { return lip_sync_reference_time_ + lip_sync_drift_.Current() + RtpDeltaToTimeDelta( static_cast<int32>(rtp_timestamp - lip_sync_rtp_timestamp_), - frequency_) + + rtp_timebase_) + target_playout_delay_; } -void AudioReceiver::IncomingPacket(scoped_ptr<Packet> packet) { +void FrameReceiver::ScheduleNextCastMessage() { DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); - if (Rtcp::IsRtcpPacket(&packet->front(), packet->size())) { - rtcp_.IncomingRtcpPacket(&packet->front(), packet->size()); - } else { - ReceivedPacket(&packet->front(), packet->size()); - } - if (!reports_are_scheduled_) { - ScheduleNextRtcpReport(); - ScheduleNextCastMessage(); - reports_are_scheduled_ = true; - } -} - -void AudioReceiver::CastFeedback(const RtcpCastMessage& cast_message) { - DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); - base::TimeTicks now = cast_environment_->Clock()->NowTicks(); - RtpTimestamp rtp_timestamp = - frame_id_to_rtp_timestamp_[cast_message.ack_frame_id_ & 0xff]; - cast_environment_->Logging()->InsertFrameEvent( - now, FRAME_ACK_SENT, AUDIO_EVENT, rtp_timestamp, - cast_message.ack_frame_id_); - - ReceiverRtcpEventSubscriber::RtcpEventMultiMap rtcp_events; - event_subscriber_.GetRtcpEventsAndReset(&rtcp_events); - rtcp_.SendRtcpFromRtpReceiver(&cast_message, &rtcp_events); -} - -void AudioReceiver::ScheduleNextRtcpReport() { - DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); - base::TimeDelta time_to_send = rtcp_.TimeToSendNextRtcpReport() - - cast_environment_->Clock()->NowTicks(); - + base::TimeTicks send_time; + framer_.TimeToSendNextCastMessage(&send_time); + base::TimeDelta time_to_send = + send_time - cast_environment_->Clock()->NowTicks(); time_to_send = std::max( time_to_send, base::TimeDelta::FromMilliseconds(kMinSchedulingDelayMs)); - cast_environment_->PostDelayedTask( CastEnvironment::MAIN, FROM_HERE, - base::Bind(&AudioReceiver::SendNextRtcpReport, + base::Bind(&FrameReceiver::SendNextCastMessage, weak_factory_.GetWeakPtr()), time_to_send); } -void AudioReceiver::SendNextRtcpReport() { +void FrameReceiver::SendNextCastMessage() { DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); - // TODO(pwestin): add logging. - rtcp_.SendRtcpFromRtpReceiver(NULL, NULL); - ScheduleNextRtcpReport(); + framer_.SendCastMessage(); // Will only send a message if it is time. + ScheduleNextCastMessage(); } -// Cast messages should be sent within a maximum interval. Schedule a call -// if not triggered elsewhere, e.g. by the cast message_builder. -void AudioReceiver::ScheduleNextCastMessage() { +void FrameReceiver::ScheduleNextRtcpReport() { DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); - base::TimeTicks send_time; - framer_.TimeToSendNextCastMessage(&send_time); - base::TimeDelta time_to_send = - send_time - cast_environment_->Clock()->NowTicks(); - time_to_send = std::max( - time_to_send, base::TimeDelta::FromMilliseconds(kMinSchedulingDelayMs)); + base::TimeDelta time_to_next = rtcp_.TimeToSendNextRtcpReport() - + cast_environment_->Clock()->NowTicks(); + + time_to_next = std::max( + time_to_next, base::TimeDelta::FromMilliseconds(kMinSchedulingDelayMs)); + cast_environment_->PostDelayedTask( CastEnvironment::MAIN, FROM_HERE, - base::Bind(&AudioReceiver::SendNextCastMessage, + base::Bind(&FrameReceiver::SendNextRtcpReport, weak_factory_.GetWeakPtr()), - time_to_send); + time_to_next); } -void AudioReceiver::SendNextCastMessage() { +void FrameReceiver::SendNextRtcpReport() { DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); - framer_.SendCastMessage(); // Will only send a message if it is time. - ScheduleNextCastMessage(); + rtcp_.SendRtcpFromRtpReceiver(NULL, NULL); + ScheduleNextRtcpReport(); } } // namespace cast diff --git a/media/cast/video_receiver/video_receiver.h b/media/cast/receiver/frame_receiver.h index 14a4bfa5c9..ac14ab1e0f 100644 --- a/media/cast/video_receiver/video_receiver.h +++ b/media/cast/receiver/frame_receiver.h @@ -1,41 +1,35 @@ -// Copyright 2013 The Chromium Authors. All rights reserved. +// Copyright 2014 The Chromium Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#ifndef MEDIA_CAST_VIDEO_RECEIVER_VIDEO_RECEIVER_H_ -#define MEDIA_CAST_VIDEO_RECEIVER_VIDEO_RECEIVER_H_ +#ifndef MEDIA_CAST_RECEIVER_FRAME_RECEIVER_H_ +#define MEDIA_CAST_RECEIVER_FRAME_RECEIVER_H_ -#include "base/basictypes.h" -#include "base/callback.h" #include "base/memory/ref_counted.h" #include "base/memory/scoped_ptr.h" #include "base/memory/weak_ptr.h" -#include "base/threading/non_thread_safe.h" -#include "base/time/tick_clock.h" #include "base/time/time.h" #include "media/cast/base/clock_drift_smoother.h" #include "media/cast/cast_config.h" -#include "media/cast/cast_environment.h" #include "media/cast/cast_receiver.h" #include "media/cast/framer/framer.h" +#include "media/cast/logging/logging_defines.h" #include "media/cast/rtcp/receiver_rtcp_event_subscriber.h" #include "media/cast/rtcp/rtcp.h" -#include "media/cast/rtp_receiver/rtp_receiver.h" +#include "media/cast/rtp_receiver/receiver_stats.h" +#include "media/cast/rtp_receiver/rtp_parser/rtp_parser.h" #include "media/cast/rtp_receiver/rtp_receiver_defines.h" #include "media/cast/transport/utility/transport_encryption_handler.h" namespace media { - -class VideoFrame; - namespace cast { -class VideoDecoder; +class CastEnvironment; -// VideoReceiver receives packets out-of-order while clients make requests for +// FrameReceiver receives packets out-of-order while clients make requests for // complete frames in-order. (A frame consists of one or more packets.) // -// VideoReceiver also includes logic for computing the playout time for each +// FrameReceiver also includes logic for computing the playout time for each // frame, accounting for a constant targeted playout delay. The purpose of the // playout delay is to provide a fixed window of time between the capture event // on the sender and the playout on the receiver. This is important because @@ -43,47 +37,43 @@ class VideoDecoder; // the sender, then receive and re-order packets on the receiver, then decode // frame) can vary in duration and is typically very hard to predict. // -// Two types of frames can be requested: 1) A frame of decoded video data; or 2) -// a frame of still-encoded video data, to be passed into an external video -// decoder. Each request for a frame includes a callback which VideoReceiver -// guarantees will be called at some point in the future unless the -// VideoReceiver is destroyed. Clients should generally limit the number of -// outstanding requests (perhaps to just one or two). +// Each request for a frame includes a callback which FrameReceiver guarantees +// will be called at some point in the future unless the FrameReceiver is +// destroyed. Clients should generally limit the number of outstanding requests +// (perhaps to just one or two). // // This class is not thread safe. Should only be called from the Main cast // thread. -class VideoReceiver : public RtpReceiver, - public RtpPayloadFeedback, - public base::NonThreadSafe, - public base::SupportsWeakPtr<VideoReceiver> { +class FrameReceiver : public RtpPayloadFeedback, + public base::SupportsWeakPtr<FrameReceiver> { public: - VideoReceiver(scoped_refptr<CastEnvironment> cast_environment, - const FrameReceiverConfig& video_config, + FrameReceiver(const scoped_refptr<CastEnvironment>& cast_environment, + const FrameReceiverConfig& config, + EventMediaType event_media_type, transport::PacedPacketSender* const packet_sender); - virtual ~VideoReceiver(); + virtual ~FrameReceiver(); - // Request a decoded video frame. + // Request an encoded frame. // // The given |callback| is guaranteed to be run at some point in the future, - // even if to respond with NULL at shutdown time. - void GetRawVideoFrame(const VideoFrameDecodedCallback& callback); + // except for those requests still enqueued at destruction time. + void RequestEncodedFrame(const ReceiveEncodedFrameCallback& callback); - // Request an encoded video frame. - // - // The given |callback| is guaranteed to be run at some point in the future, - // even if to respond with NULL at shutdown time. - void GetEncodedVideoFrame(const FrameEncodedCallback& callback); + // Called to deliver another packet, possibly a duplicate, and possibly + // out-of-order. Returns true if the parsing of the packet succeeded. + bool ProcessPacket(scoped_ptr<Packet> packet); - // Deliver another packet, possibly a duplicate, and possibly out-of-order. - void IncomingPacket(scoped_ptr<Packet> packet); + // TODO(miu): This is the wrong place for this, but the (de)serialization + // implementation needs to be consolidated first. + static bool ParseSenderSsrc(const uint8* packet, size_t length, uint32* ssrc); protected: - friend class VideoReceiverTest; // Invokes OnReceivedPayloadData(). + friend class FrameReceiverTest; // Invokes ProcessParsedPacket(). - virtual void OnReceivedPayloadData(const uint8* payload_data, - size_t payload_size, - const RtpCastHeader& rtp_header) OVERRIDE; + void ProcessParsedPacket(const RtpCastHeader& rtp_header, + const uint8* payload_data, + size_t payload_size); // RtpPayloadFeedback implementation. virtual void CastFeedback(const RtcpCastMessage& cast_message) OVERRIDE; @@ -99,12 +89,6 @@ class VideoReceiver : public RtpReceiver, // EmitAvailableEncodedFrames(). void EmitAvailableEncodedFramesAfterWaiting(); - // Feeds an EncodedFrame into |video_decoder_|. GetRawVideoFrame() uses this - // as a callback for GetEncodedVideoFrame(). - void DecodeEncodedVideoFrame( - const VideoFrameDecodedCallback& callback, - scoped_ptr<transport::EncodedFrame> encoded_frame); - // Computes the playout time for a frame with the given |rtp_timestamp|. // Because lip-sync info is refreshed regularly, calling this method with the // same argument may return different results. @@ -122,28 +106,23 @@ class VideoReceiver : public RtpReceiver, // Actually send the next RTCP report. void SendNextRtcpReport(); - // Receives a VideoFrame from |video_decoder_|, logs the event, and passes the - // data on by running the given |callback|. This method is static to ensure - // it can be called after a VideoReceiver instance is destroyed. - // DecodeEncodedVideoFrame() uses this as a callback for - // VideoDecoder::DecodeFrame(). - static void EmitRawVideoFrame( - const scoped_refptr<CastEnvironment>& cast_environment, - const VideoFrameDecodedCallback& callback, - uint32 frame_id, - uint32 rtp_timestamp, - const base::TimeTicks& playout_time, - const scoped_refptr<VideoFrame>& video_frame, - bool is_continuous); - const scoped_refptr<CastEnvironment> cast_environment_; + // Deserializes a packet into a RtpHeader + payload bytes. + RtpParser packet_parser_; + + // Accumulates packet statistics, including packet loss, counts, and jitter. + ReceiverStats stats_; + + // Partitions logged events by the type of media passing through. + EventMediaType event_media_type_; + // Subscribes to raw events. - // Processes raw audio events to be sent over to the cast sender via RTCP. + // Processes raw events to be sent over to the cast sender via RTCP. ReceiverRtcpEventSubscriber event_subscriber_; - // Configured video codec. - const transport::VideoCodec codec_; + // RTP timebase: The number of RTP units advanced per one second. + const int rtp_timebase_; // The total amount of time between a frame's capture/recording on the sender // and its playback on the receiver (i.e., shown to a user). This is fixed as @@ -154,6 +133,8 @@ class VideoReceiver : public RtpReceiver, const base::TimeDelta target_playout_delay_; // Hack: This is used in logic that determines whether to skip frames. + // TODO(miu): Revisit this. Logic needs to also account for expected decode + // time. const base::TimeDelta expected_frame_duration_; // Set to false initially, then set to true after scheduling the periodic @@ -166,9 +147,6 @@ class VideoReceiver : public RtpReceiver, // decodable EncodedFrames. Framer framer_; - // Decodes frames into media::VideoFrame images for playback. - scoped_ptr<VideoDecoder> video_decoder_; - // Manages sending/receiving of RTCP packets, including sender/receiver // reports. Rtcp rtcp_; @@ -177,7 +155,7 @@ class VideoReceiver : public RtpReceiver, transport::TransportEncryptionHandler decryptor_; // Outstanding callbacks to run to deliver on client requests for frames. - std::list<FrameEncodedCallback> frame_request_queue_; + std::list<ReceiveEncodedFrameCallback> frame_request_queue_; // True while there's an outstanding task to re-invoke // EmitAvailableEncodedFrames(). @@ -195,12 +173,12 @@ class VideoReceiver : public RtpReceiver, ClockDriftSmoother lip_sync_drift_; // NOTE: Weak pointers must be invalidated before all other member variables. - base::WeakPtrFactory<VideoReceiver> weak_factory_; + base::WeakPtrFactory<FrameReceiver> weak_factory_; - DISALLOW_COPY_AND_ASSIGN(VideoReceiver); + DISALLOW_COPY_AND_ASSIGN(FrameReceiver); }; } // namespace cast } // namespace media -#endif // MEDIA_CAST_VIDEO_RECEIVER_VIDEO_RECEIVER_H_ +#endif // MEDIA_CAST_RECEIVER_FRAME_RECEIVER_H_ diff --git a/media/cast/receiver/frame_receiver_unittest.cc b/media/cast/receiver/frame_receiver_unittest.cc new file mode 100644 index 0000000000..4d8273e132 --- /dev/null +++ b/media/cast/receiver/frame_receiver_unittest.cc @@ -0,0 +1,419 @@ +// Copyright 2014 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include <deque> +#include <utility> + +#include "base/bind.h" +#include "base/memory/ref_counted.h" +#include "base/memory/scoped_ptr.h" +#include "base/test/simple_test_tick_clock.h" +#include "media/cast/cast_defines.h" +#include "media/cast/cast_environment.h" +#include "media/cast/logging/simple_event_subscriber.h" +#include "media/cast/receiver/frame_receiver.h" +#include "media/cast/rtcp/test_rtcp_packet_builder.h" +#include "media/cast/test/fake_single_thread_task_runner.h" +#include "media/cast/test/utility/default_config.h" +#include "media/cast/transport/pacing/mock_paced_packet_sender.h" +#include "testing/gmock/include/gmock/gmock.h" + +using ::testing::_; + +namespace media { +namespace cast { + +namespace { + +const int kPacketSize = 1500; +const uint32 kFirstFrameId = 1234; +const int kPlayoutDelayMillis = 100; + +class FakeFrameClient { + public: + FakeFrameClient() : num_called_(0) {} + virtual ~FakeFrameClient() {} + + void AddExpectedResult(uint32 expected_frame_id, + const base::TimeTicks& expected_playout_time) { + expected_results_.push_back( + std::make_pair(expected_frame_id, expected_playout_time)); + } + + void DeliverEncodedFrame(scoped_ptr<transport::EncodedFrame> frame) { + SCOPED_TRACE(::testing::Message() << "num_called_ is " << num_called_); + ASSERT_FALSE(!frame) + << "If at shutdown: There were unsatisfied requests enqueued."; + ASSERT_FALSE(expected_results_.empty()); + EXPECT_EQ(expected_results_.front().first, frame->frame_id); + EXPECT_EQ(expected_results_.front().second, frame->reference_time); + expected_results_.pop_front(); + ++num_called_; + } + + int number_times_called() const { return num_called_; } + + private: + std::deque<std::pair<uint32, base::TimeTicks> > expected_results_; + int num_called_; + + DISALLOW_COPY_AND_ASSIGN(FakeFrameClient); +}; +} // namespace + +class FrameReceiverTest : public ::testing::Test { + protected: + FrameReceiverTest() { + testing_clock_ = new base::SimpleTestTickClock(); + testing_clock_->Advance(base::TimeTicks::Now() - base::TimeTicks()); + start_time_ = testing_clock_->NowTicks(); + task_runner_ = new test::FakeSingleThreadTaskRunner(testing_clock_); + + cast_environment_ = + new CastEnvironment(scoped_ptr<base::TickClock>(testing_clock_).Pass(), + task_runner_, + task_runner_, + task_runner_); + } + + virtual ~FrameReceiverTest() {} + + virtual void SetUp() { + payload_.assign(kPacketSize, 0); + + // Always start with a key frame. + rtp_header_.is_key_frame = true; + rtp_header_.frame_id = kFirstFrameId; + rtp_header_.packet_id = 0; + rtp_header_.max_packet_id = 0; + rtp_header_.reference_frame_id = rtp_header_.frame_id; + rtp_header_.rtp_timestamp = 0; + } + + void CreateFrameReceiverOfAudio() { + config_ = GetDefaultAudioReceiverConfig(); + config_.rtp_max_delay_ms = kPlayoutDelayMillis; + + receiver_.reset(new FrameReceiver( + cast_environment_, config_, AUDIO_EVENT, &mock_transport_)); + } + + void CreateFrameReceiverOfVideo() { + config_ = GetDefaultVideoReceiverConfig(); + config_.rtp_max_delay_ms = kPlayoutDelayMillis; + // Note: Frame rate must divide 1000 without remainder so the test code + // doesn't have to account for rounding errors. + config_.max_frame_rate = 25; + + receiver_.reset(new FrameReceiver( + cast_environment_, config_, VIDEO_EVENT, &mock_transport_)); + } + + void FeedOneFrameIntoReceiver() { + // Note: For testing purposes, a frame consists of only a single packet. + receiver_->ProcessParsedPacket( + rtp_header_, payload_.data(), payload_.size()); + } + + void FeedLipSyncInfoIntoReceiver() { + const base::TimeTicks now = testing_clock_->NowTicks(); + const int64 rtp_timestamp = (now - start_time_) * + config_.frequency / base::TimeDelta::FromSeconds(1); + CHECK_LE(0, rtp_timestamp); + uint32 ntp_seconds; + uint32 ntp_fraction; + ConvertTimeTicksToNtp(now, &ntp_seconds, &ntp_fraction); + TestRtcpPacketBuilder rtcp_packet; + rtcp_packet.AddSrWithNtp(config_.incoming_ssrc, + ntp_seconds, ntp_fraction, + static_cast<uint32>(rtp_timestamp)); + ASSERT_TRUE(receiver_->ProcessPacket(rtcp_packet.GetPacket().Pass())); + } + + FrameReceiverConfig config_; + std::vector<uint8> payload_; + RtpCastHeader rtp_header_; + base::SimpleTestTickClock* testing_clock_; // Owned by CastEnvironment. + base::TimeTicks start_time_; + transport::MockPacedPacketSender mock_transport_; + scoped_refptr<test::FakeSingleThreadTaskRunner> task_runner_; + scoped_refptr<CastEnvironment> cast_environment_; + FakeFrameClient frame_client_; + + // Important for the FrameReceiver to be declared last, since its dependencies + // must remain alive until after its destruction. + scoped_ptr<FrameReceiver> receiver_; + + DISALLOW_COPY_AND_ASSIGN(FrameReceiverTest); +}; + +TEST_F(FrameReceiverTest, RejectsUnparsablePackets) { + CreateFrameReceiverOfVideo(); + + SimpleEventSubscriber event_subscriber; + cast_environment_->Logging()->AddRawEventSubscriber(&event_subscriber); + + const bool success = receiver_->ProcessPacket( + scoped_ptr<Packet>(new Packet(kPacketSize, 0xff)).Pass()); + EXPECT_FALSE(success); + + // Confirm no log events. + std::vector<FrameEvent> frame_events; + event_subscriber.GetFrameEventsAndReset(&frame_events); + EXPECT_TRUE(frame_events.empty()); + cast_environment_->Logging()->RemoveRawEventSubscriber(&event_subscriber); +} + +TEST_F(FrameReceiverTest, ReceivesOneFrame) { + CreateFrameReceiverOfAudio(); + + SimpleEventSubscriber event_subscriber; + cast_environment_->Logging()->AddRawEventSubscriber(&event_subscriber); + + EXPECT_CALL(mock_transport_, SendRtcpPacket(_, _)) + .WillRepeatedly(testing::Return(true)); + + FeedLipSyncInfoIntoReceiver(); + task_runner_->RunTasks(); + + // Enqueue a request for a frame. + receiver_->RequestEncodedFrame( + base::Bind(&FakeFrameClient::DeliverEncodedFrame, + base::Unretained(&frame_client_))); + + // The request should not be satisfied since no packets have been received. + task_runner_->RunTasks(); + EXPECT_EQ(0, frame_client_.number_times_called()); + + // Deliver one frame to the receiver and expect to get one frame back. + const base::TimeDelta target_playout_delay = + base::TimeDelta::FromMilliseconds(kPlayoutDelayMillis); + frame_client_.AddExpectedResult( + kFirstFrameId, testing_clock_->NowTicks() + target_playout_delay); + FeedOneFrameIntoReceiver(); + task_runner_->RunTasks(); + EXPECT_EQ(1, frame_client_.number_times_called()); + + // Was the frame logged? + std::vector<FrameEvent> frame_events; + event_subscriber.GetFrameEventsAndReset(&frame_events); + ASSERT_TRUE(!frame_events.empty()); + EXPECT_EQ(FRAME_ACK_SENT, frame_events.begin()->type); + EXPECT_EQ(AUDIO_EVENT, frame_events.begin()->media_type); + EXPECT_EQ(rtp_header_.frame_id, frame_events.begin()->frame_id); + EXPECT_EQ(rtp_header_.rtp_timestamp, frame_events.begin()->rtp_timestamp); + cast_environment_->Logging()->RemoveRawEventSubscriber(&event_subscriber); +} + +TEST_F(FrameReceiverTest, ReceivesFramesSkippingWhenAppropriate) { + CreateFrameReceiverOfAudio(); + + SimpleEventSubscriber event_subscriber; + cast_environment_->Logging()->AddRawEventSubscriber(&event_subscriber); + + EXPECT_CALL(mock_transport_, SendRtcpPacket(_, _)) + .WillRepeatedly(testing::Return(true)); + + const uint32 rtp_advance_per_frame = + config_.frequency / config_.max_frame_rate; + const base::TimeDelta time_advance_per_frame = + base::TimeDelta::FromSeconds(1) / config_.max_frame_rate; + + // Feed and process lip sync in receiver. + FeedLipSyncInfoIntoReceiver(); + task_runner_->RunTasks(); + const base::TimeTicks first_frame_capture_time = testing_clock_->NowTicks(); + + // Enqueue a request for a frame. + const ReceiveEncodedFrameCallback frame_encoded_callback = + base::Bind(&FakeFrameClient::DeliverEncodedFrame, + base::Unretained(&frame_client_)); + receiver_->RequestEncodedFrame(frame_encoded_callback); + task_runner_->RunTasks(); + EXPECT_EQ(0, frame_client_.number_times_called()); + + // Receive one frame and expect to see the first request satisfied. + const base::TimeDelta target_playout_delay = + base::TimeDelta::FromMilliseconds(kPlayoutDelayMillis); + frame_client_.AddExpectedResult( + kFirstFrameId, first_frame_capture_time + target_playout_delay); + rtp_header_.rtp_timestamp = 0; + FeedOneFrameIntoReceiver(); // Frame 1 + task_runner_->RunTasks(); + EXPECT_EQ(1, frame_client_.number_times_called()); + + // Enqueue a second request for a frame, but it should not be fulfilled yet. + receiver_->RequestEncodedFrame(frame_encoded_callback); + task_runner_->RunTasks(); + EXPECT_EQ(1, frame_client_.number_times_called()); + + // Receive one frame out-of-order: Make sure that we are not continuous and + // that the RTP timestamp represents a time in the future. + rtp_header_.frame_id = kFirstFrameId + 2; // "Frame 3" + rtp_header_.reference_frame_id = rtp_header_.frame_id; + rtp_header_.rtp_timestamp += 2 * rtp_advance_per_frame; + frame_client_.AddExpectedResult( + kFirstFrameId + 2, + first_frame_capture_time + 2 * time_advance_per_frame + + target_playout_delay); + FeedOneFrameIntoReceiver(); // Frame 3 + + // Frame 2 should not come out at this point in time. + task_runner_->RunTasks(); + EXPECT_EQ(1, frame_client_.number_times_called()); + + // Enqueue a third request for a frame. + receiver_->RequestEncodedFrame(frame_encoded_callback); + task_runner_->RunTasks(); + EXPECT_EQ(1, frame_client_.number_times_called()); + + // Now, advance time forward such that the receiver is convinced it should + // skip Frame 2. Frame 3 is emitted (to satisfy the second request) because a + // decision was made to skip over the no-show Frame 2. + testing_clock_->Advance(2 * time_advance_per_frame + target_playout_delay); + task_runner_->RunTasks(); + EXPECT_EQ(2, frame_client_.number_times_called()); + + // Receive Frame 4 and expect it to fulfill the third request immediately. + rtp_header_.frame_id = kFirstFrameId + 3; // "Frame 4" + rtp_header_.reference_frame_id = rtp_header_.frame_id; + rtp_header_.rtp_timestamp += rtp_advance_per_frame; + frame_client_.AddExpectedResult( + kFirstFrameId + 3, first_frame_capture_time + 3 * time_advance_per_frame + + target_playout_delay); + FeedOneFrameIntoReceiver(); // Frame 4 + task_runner_->RunTasks(); + EXPECT_EQ(3, frame_client_.number_times_called()); + + // Move forward to the playout time of an unreceived Frame 5. Expect no + // additional frames were emitted. + testing_clock_->Advance(3 * time_advance_per_frame); + task_runner_->RunTasks(); + EXPECT_EQ(3, frame_client_.number_times_called()); + + // Were only non-skipped frames logged? + std::vector<FrameEvent> frame_events; + event_subscriber.GetFrameEventsAndReset(&frame_events); + ASSERT_TRUE(!frame_events.empty()); + for (size_t i = 0; i < frame_events.size(); ++i) { + EXPECT_EQ(FRAME_ACK_SENT, frame_events[i].type); + EXPECT_EQ(AUDIO_EVENT, frame_events[i].media_type); + EXPECT_LE(kFirstFrameId, frame_events[i].frame_id); + EXPECT_GE(kFirstFrameId + 4, frame_events[i].frame_id); + const int frame_offset = frame_events[i].frame_id - kFirstFrameId; + EXPECT_NE(frame_offset, 1); // Frame 2 never received. + EXPECT_EQ(frame_offset * rtp_advance_per_frame, + frame_events[i].rtp_timestamp); + } + cast_environment_->Logging()->RemoveRawEventSubscriber(&event_subscriber); +} + +TEST_F(FrameReceiverTest, ReceivesFramesRefusingToSkipAny) { + CreateFrameReceiverOfVideo(); + + SimpleEventSubscriber event_subscriber; + cast_environment_->Logging()->AddRawEventSubscriber(&event_subscriber); + + EXPECT_CALL(mock_transport_, SendRtcpPacket(_, _)) + .WillRepeatedly(testing::Return(true)); + + const uint32 rtp_advance_per_frame = + config_.frequency / config_.max_frame_rate; + const base::TimeDelta time_advance_per_frame = + base::TimeDelta::FromSeconds(1) / config_.max_frame_rate; + + // Feed and process lip sync in receiver. + FeedLipSyncInfoIntoReceiver(); + task_runner_->RunTasks(); + const base::TimeTicks first_frame_capture_time = testing_clock_->NowTicks(); + + // Enqueue a request for a frame. + const ReceiveEncodedFrameCallback frame_encoded_callback = + base::Bind(&FakeFrameClient::DeliverEncodedFrame, + base::Unretained(&frame_client_)); + receiver_->RequestEncodedFrame(frame_encoded_callback); + task_runner_->RunTasks(); + EXPECT_EQ(0, frame_client_.number_times_called()); + + // Receive one frame and expect to see the first request satisfied. + const base::TimeDelta target_playout_delay = + base::TimeDelta::FromMilliseconds(kPlayoutDelayMillis); + frame_client_.AddExpectedResult( + kFirstFrameId, first_frame_capture_time + target_playout_delay); + rtp_header_.rtp_timestamp = 0; + FeedOneFrameIntoReceiver(); // Frame 1 + task_runner_->RunTasks(); + EXPECT_EQ(1, frame_client_.number_times_called()); + + // Enqueue a second request for a frame, but it should not be fulfilled yet. + receiver_->RequestEncodedFrame(frame_encoded_callback); + task_runner_->RunTasks(); + EXPECT_EQ(1, frame_client_.number_times_called()); + + // Receive one frame out-of-order: Make sure that we are not continuous and + // that the RTP timestamp represents a time in the future. + rtp_header_.is_key_frame = false; + rtp_header_.frame_id = kFirstFrameId + 2; // "Frame 3" + rtp_header_.reference_frame_id = kFirstFrameId + 1; // "Frame 2" + rtp_header_.rtp_timestamp += 2 * rtp_advance_per_frame; + FeedOneFrameIntoReceiver(); // Frame 3 + + // Frame 2 should not come out at this point in time. + task_runner_->RunTasks(); + EXPECT_EQ(1, frame_client_.number_times_called()); + + // Enqueue a third request for a frame. + receiver_->RequestEncodedFrame(frame_encoded_callback); + task_runner_->RunTasks(); + EXPECT_EQ(1, frame_client_.number_times_called()); + + // Now, advance time forward such that Frame 2 is now too late for playback. + // Regardless, the receiver must NOT emit Frame 3 yet because it is not + // allowed to skip frames when dependencies are not satisfied. In other + // words, Frame 3 is not decodable without Frame 2. + testing_clock_->Advance(2 * time_advance_per_frame + target_playout_delay); + task_runner_->RunTasks(); + EXPECT_EQ(1, frame_client_.number_times_called()); + + // Now receive Frame 2 and expect both the second and third requests to be + // fulfilled immediately. + frame_client_.AddExpectedResult( + kFirstFrameId + 1, // "Frame 2" + first_frame_capture_time + 1 * time_advance_per_frame + + target_playout_delay); + frame_client_.AddExpectedResult( + kFirstFrameId + 2, // "Frame 3" + first_frame_capture_time + 2 * time_advance_per_frame + + target_playout_delay); + --rtp_header_.frame_id; // "Frame 2" + --rtp_header_.reference_frame_id; // "Frame 1" + rtp_header_.rtp_timestamp -= rtp_advance_per_frame; + FeedOneFrameIntoReceiver(); // Frame 2 + task_runner_->RunTasks(); + EXPECT_EQ(3, frame_client_.number_times_called()); + + // Move forward to the playout time of an unreceived Frame 5. Expect no + // additional frames were emitted. + testing_clock_->Advance(3 * time_advance_per_frame); + task_runner_->RunTasks(); + EXPECT_EQ(3, frame_client_.number_times_called()); + + // Sanity-check logging results. + std::vector<FrameEvent> frame_events; + event_subscriber.GetFrameEventsAndReset(&frame_events); + ASSERT_TRUE(!frame_events.empty()); + for (size_t i = 0; i < frame_events.size(); ++i) { + EXPECT_EQ(FRAME_ACK_SENT, frame_events[i].type); + EXPECT_EQ(VIDEO_EVENT, frame_events[i].media_type); + EXPECT_LE(kFirstFrameId, frame_events[i].frame_id); + EXPECT_GE(kFirstFrameId + 3, frame_events[i].frame_id); + const int frame_offset = frame_events[i].frame_id - kFirstFrameId; + EXPECT_EQ(frame_offset * rtp_advance_per_frame, + frame_events[i].rtp_timestamp); + } + cast_environment_->Logging()->RemoveRawEventSubscriber(&event_subscriber); +} + +} // namespace cast +} // namespace media diff --git a/media/cast/video_receiver/video_decoder.cc b/media/cast/receiver/video_decoder.cc index c2d256268f..21f49d89a5 100644 --- a/media/cast/video_receiver/video_decoder.cc +++ b/media/cast/receiver/video_decoder.cc @@ -1,8 +1,8 @@ -// Copyright 2013 The Chromium Authors. All rights reserved. +// Copyright 2014 The Chromium Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#include "media/cast/video_receiver/video_decoder.h" +#include "media/cast/receiver/video_decoder.h" #include "base/bind.h" #include "base/bind_helpers.h" @@ -209,9 +209,9 @@ class VideoDecoder::FakeImpl : public VideoDecoder::ImplBase { VideoDecoder::VideoDecoder( const scoped_refptr<CastEnvironment>& cast_environment, - const FrameReceiverConfig& video_config) + transport::VideoCodec codec) : cast_environment_(cast_environment) { - switch (video_config.codec.video) { + switch (codec) { #ifndef OFFICIAL_BUILD case transport::kFakeSoftwareVideo: impl_ = new FakeImpl(cast_environment); diff --git a/media/cast/video_receiver/video_decoder.h b/media/cast/receiver/video_decoder.h index ea40173004..66dc36bb2a 100644 --- a/media/cast/video_receiver/video_decoder.h +++ b/media/cast/receiver/video_decoder.h @@ -1,9 +1,9 @@ -// Copyright 2013 The Chromium Authors. All rights reserved. +// Copyright 2014 The Chromium Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#ifndef MEDIA_CAST_VIDEO_RECEIVER_VIDEO_DECODER_H_ -#define MEDIA_CAST_VIDEO_RECEIVER_VIDEO_DECODER_H_ +#ifndef MEDIA_CAST_RECEIVER_VIDEO_DECODER_H_ +#define MEDIA_CAST_RECEIVER_VIDEO_DECODER_H_ #include "base/callback.h" #include "base/memory/ref_counted.h" @@ -28,7 +28,7 @@ class VideoDecoder { bool is_continuous)> DecodeFrameCallback; VideoDecoder(const scoped_refptr<CastEnvironment>& cast_environment, - const FrameReceiverConfig& video_config); + transport::VideoCodec codec); virtual ~VideoDecoder(); // Returns STATUS_VIDEO_INITIALIZED if the decoder was successfully @@ -60,4 +60,4 @@ class VideoDecoder { } // namespace cast } // namespace media -#endif // MEDIA_CAST_VIDEO_RECEIVER_VIDEO_DECODER_H_ +#endif // MEDIA_CAST_RECEIVER_VIDEO_DECODER_H_ diff --git a/media/cast/video_receiver/video_decoder_unittest.cc b/media/cast/receiver/video_decoder_unittest.cc index 2ca28b070c..1d16534b96 100644 --- a/media/cast/video_receiver/video_decoder_unittest.cc +++ b/media/cast/receiver/video_decoder_unittest.cc @@ -1,4 +1,4 @@ -// Copyright 2013 The Chromium Authors. All rights reserved. +// Copyright 2014 The Chromium Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. @@ -10,10 +10,9 @@ #include "base/synchronization/lock.h" #include "base/time/time.h" #include "media/cast/cast_config.h" -#include "media/cast/test/utility/default_config.h" +#include "media/cast/receiver/video_decoder.h" #include "media/cast/test/utility/standalone_cast_environment.h" #include "media/cast/test/utility/video_utility.h" -#include "media/cast/video_receiver/video_decoder.h" #include "media/cast/video_sender/codecs/vp8/vp8_encoder.h" #include "testing/gtest/include/gtest/gtest.h" @@ -48,9 +47,7 @@ class VideoDecoderTest protected: virtual void SetUp() OVERRIDE { - FrameReceiverConfig decoder_config = GetDefaultVideoReceiverConfig(); - decoder_config.codec.video = GetParam(); - video_decoder_.reset(new VideoDecoder(cast_environment_, decoder_config)); + video_decoder_.reset(new VideoDecoder(cast_environment_, GetParam())); CHECK_EQ(STATUS_VIDEO_INITIALIZED, video_decoder_->InitializationResult()); next_frame_timestamp_ = base::TimeDelta(); diff --git a/media/cast/rtcp/rtcp.cc b/media/cast/rtcp/rtcp.cc index 3aa936b135..480b2ac399 100644 --- a/media/cast/rtcp/rtcp.cc +++ b/media/cast/rtcp/rtcp.cc @@ -79,7 +79,8 @@ Rtcp::Rtcp(scoped_refptr<CastEnvironment> cast_environment, transport::PacedPacketSender* paced_packet_sender, RtpReceiverStatistics* rtp_receiver_statistics, RtcpMode rtcp_mode, const base::TimeDelta& rtcp_interval, uint32 local_ssrc, - uint32 remote_ssrc, const std::string& c_name, bool is_audio) + uint32 remote_ssrc, const std::string& c_name, + EventMediaType event_media_type) : cast_environment_(cast_environment), transport_sender_(transport_sender), rtcp_interval_(rtcp_interval), @@ -87,6 +88,7 @@ Rtcp::Rtcp(scoped_refptr<CastEnvironment> cast_environment, local_ssrc_(local_ssrc), remote_ssrc_(remote_ssrc), c_name_(c_name), + event_media_type_(event_media_type), rtp_receiver_statistics_(rtp_receiver_statistics), rtt_feedback_(new LocalRtcpRttFeedback(this)), receiver_feedback_(new LocalRtcpReceiverFeedback(this, cast_environment)), @@ -97,8 +99,7 @@ Rtcp::Rtcp(scoped_refptr<CastEnvironment> cast_environment, lip_sync_rtp_timestamp_(0), lip_sync_ntp_timestamp_(0), min_rtt_(base::TimeDelta::FromMilliseconds(kMaxRttMs)), - number_of_rtt_in_avg_(0), - is_audio_(is_audio) { + number_of_rtt_in_avg_(0) { rtcp_receiver_.reset(new RtcpReceiver(cast_environment, sender_feedback, receiver_feedback_.get(), rtt_feedback_.get(), local_ssrc)); @@ -394,7 +395,6 @@ void Rtcp::UpdateNextTimeToSendRtcp() { void Rtcp::OnReceivedReceiverLog(const RtcpReceiverLogMessage& receiver_log) { // Add received log messages into our log system. RtcpReceiverLogMessage::const_iterator it = receiver_log.begin(); - EventMediaType media_type = is_audio_ ? AUDIO_EVENT : VIDEO_EVENT; for (; it != receiver_log.end(); ++it) { uint32 rtp_timestamp = it->rtp_timestamp_; @@ -405,18 +405,18 @@ void Rtcp::OnReceivedReceiverLog(const RtcpReceiverLogMessage& receiver_log) { case PACKET_RECEIVED: cast_environment_->Logging()->InsertPacketEvent( event_it->event_timestamp, event_it->type, - media_type, rtp_timestamp, + event_media_type_, rtp_timestamp, kFrameIdUnknown, event_it->packet_id, 0, 0); break; case FRAME_ACK_SENT: case FRAME_DECODED: cast_environment_->Logging()->InsertFrameEvent( - event_it->event_timestamp, event_it->type, media_type, + event_it->event_timestamp, event_it->type, event_media_type_, rtp_timestamp, kFrameIdUnknown); break; case FRAME_PLAYOUT: cast_environment_->Logging()->InsertFrameEventWithDelay( - event_it->event_timestamp, event_it->type, media_type, + event_it->event_timestamp, event_it->type, event_media_type_, rtp_timestamp, kFrameIdUnknown, event_it->delay_delta); break; default: diff --git a/media/cast/rtcp/rtcp.h b/media/cast/rtcp/rtcp.h index ff81bb90e0..9d0184f903 100644 --- a/media/cast/rtcp/rtcp.h +++ b/media/cast/rtcp/rtcp.h @@ -68,7 +68,7 @@ class Rtcp { uint32 local_ssrc, uint32 remote_ssrc, const std::string& c_name, - bool is_audio); + EventMediaType event_media_type); virtual ~Rtcp(); @@ -156,6 +156,7 @@ class Rtcp { const uint32 local_ssrc_; const uint32 remote_ssrc_; const std::string c_name_; + const EventMediaType event_media_type_; // Not owned by this class. RtpReceiverStatistics* const rtp_receiver_statistics_; @@ -195,7 +196,6 @@ class Rtcp { int number_of_rtt_in_avg_; double avg_rtt_ms_; uint16 target_delay_ms_; - bool is_audio_; DISALLOW_COPY_AND_ASSIGN(Rtcp); }; diff --git a/media/cast/rtcp/rtcp_unittest.cc b/media/cast/rtcp/rtcp_unittest.cc index 3cae9b9fea..d5bf312c3e 100644 --- a/media/cast/rtcp/rtcp_unittest.cc +++ b/media/cast/rtcp/rtcp_unittest.cc @@ -144,7 +144,7 @@ class RtcpPeer : public Rtcp { local_ssrc, remote_ssrc, c_name, - true) {} + AUDIO_EVENT) {} using Rtcp::OnReceivedNtp; using Rtcp::OnReceivedLipSyncInfo; @@ -219,7 +219,7 @@ TEST_F(RtcpTest, TimeToSend) { kSenderSsrc, kReceiverSsrc, kCName, - true); + AUDIO_EVENT); receiver_to_sender_.set_rtcp_receiver(&rtcp); EXPECT_LE(start_time, rtcp.TimeToSendNextRtcpReport()); EXPECT_GE( @@ -241,7 +241,7 @@ TEST_F(RtcpTest, BasicSenderReport) { kSenderSsrc, kReceiverSsrc, kCName, - true); + AUDIO_EVENT); sender_to_receiver_.set_rtcp_receiver(&rtcp); rtcp.SendRtcpFromRtpSender(base::TimeTicks(), 0); } @@ -257,7 +257,7 @@ TEST_F(RtcpTest, BasicReceiverReport) { kSenderSsrc, kReceiverSsrc, kCName, - true); + AUDIO_EVENT); receiver_to_sender_.set_rtcp_receiver(&rtcp); rtcp.SendRtcpFromRtpReceiver(NULL, NULL); } @@ -276,7 +276,7 @@ TEST_F(RtcpTest, BasicCast) { kSenderSsrc, kSenderSsrc, kCName, - true); + AUDIO_EVENT); receiver_to_sender_.set_rtcp_receiver(&rtcp); RtcpCastMessage cast_message(kSenderSsrc); cast_message.ack_frame_id_ = kAckFrameId; @@ -303,7 +303,7 @@ TEST_F(RtcpTest, RttReducedSizeRtcp) { kReceiverSsrc, kSenderSsrc, kCName, - true); + AUDIO_EVENT); // Media sender. Rtcp rtcp_sender(cast_environment_, @@ -316,7 +316,7 @@ TEST_F(RtcpTest, RttReducedSizeRtcp) { kSenderSsrc, kReceiverSsrc, kCName, - true); + AUDIO_EVENT); sender_to_receiver_.set_rtcp_receiver(&rtcp_receiver); receiver_to_sender_.set_rtcp_receiver(&rtcp_sender); @@ -359,7 +359,7 @@ TEST_F(RtcpTest, Rtt) { kReceiverSsrc, kSenderSsrc, kCName, - true); + AUDIO_EVENT); // Media sender. Rtcp rtcp_sender(cast_environment_, @@ -372,7 +372,7 @@ TEST_F(RtcpTest, Rtt) { kSenderSsrc, kReceiverSsrc, kCName, - true); + AUDIO_EVENT); receiver_to_sender_.set_rtcp_receiver(&rtcp_sender); sender_to_receiver_.set_rtcp_receiver(&rtcp_receiver); @@ -452,7 +452,7 @@ TEST_F(RtcpTest, RttWithPacketLoss) { kReceiverSsrc, kSenderSsrc, kCName, - true); + AUDIO_EVENT); // Media sender. Rtcp rtcp_sender(cast_environment_, @@ -465,7 +465,7 @@ TEST_F(RtcpTest, RttWithPacketLoss) { kSenderSsrc, kReceiverSsrc, kCName, - true); + AUDIO_EVENT); receiver_to_sender_.set_rtcp_receiver(&rtcp_sender); sender_to_receiver_.set_rtcp_receiver(&rtcp_receiver); diff --git a/media/cast/rtp_receiver/rtp_parser/rtp_parser.cc b/media/cast/rtp_receiver/rtp_parser/rtp_parser.cc index d75696a861..f44e82dac2 100644 --- a/media/cast/rtp_receiver/rtp_parser/rtp_parser.cc +++ b/media/cast/rtp_receiver/rtp_parser/rtp_parser.cc @@ -7,7 +7,6 @@ #include "base/big_endian.h" #include "base/logging.h" #include "media/cast/cast_defines.h" -#include "media/cast/rtp_receiver/rtp_receiver.h" namespace media { namespace cast { diff --git a/media/cast/rtp_receiver/rtp_parser/rtp_parser_unittest.cc b/media/cast/rtp_receiver/rtp_parser/rtp_parser_unittest.cc index 4e3560a9c7..47c79139ff 100644 --- a/media/cast/rtp_receiver/rtp_parser/rtp_parser_unittest.cc +++ b/media/cast/rtp_receiver/rtp_parser/rtp_parser_unittest.cc @@ -6,7 +6,6 @@ #include "base/rand_util.h" #include "media/cast/rtp_receiver/rtp_parser/rtp_parser.h" #include "media/cast/rtp_receiver/rtp_parser/test/rtp_packet_builder.h" -#include "media/cast/rtp_receiver/rtp_receiver.h" #include "media/cast/rtp_receiver/rtp_receiver_defines.h" #include "testing/gtest/include/gtest/gtest.h" diff --git a/media/cast/rtp_receiver/rtp_receiver.cc b/media/cast/rtp_receiver/rtp_receiver.cc deleted file mode 100644 index f7ff50b375..0000000000 --- a/media/cast/rtp_receiver/rtp_receiver.cc +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright 2013 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include "media/cast/rtp_receiver/rtp_receiver.h" - -#include "base/big_endian.h" -#include "base/logging.h" -#include "media/cast/rtp_receiver/receiver_stats.h" -#include "media/cast/rtp_receiver/rtp_parser/rtp_parser.h" -#include "media/cast/rtp_receiver/rtp_receiver_defines.h" - -namespace media { -namespace cast { - -RtpReceiver::RtpReceiver(base::TickClock* clock, - const FrameReceiverConfig* audio_config, - const FrameReceiverConfig* video_config) : - packet_parser_(audio_config ? audio_config->incoming_ssrc : - (video_config ? video_config->incoming_ssrc : 0), - audio_config ? audio_config->rtp_payload_type : - (video_config ? video_config->rtp_payload_type : 0)), - stats_(clock) { - DCHECK(audio_config || video_config); -} - -RtpReceiver::~RtpReceiver() {} - -// static -uint32 RtpReceiver::GetSsrcOfSender(const uint8* rtcp_buffer, size_t length) { - DCHECK_GE(length, kMinLengthOfRtp) << "Invalid RTP packet"; - uint32 ssrc_of_sender; - base::BigEndianReader big_endian_reader( - reinterpret_cast<const char*>(rtcp_buffer), length); - big_endian_reader.Skip(8); // Skip header - big_endian_reader.ReadU32(&ssrc_of_sender); - return ssrc_of_sender; -} - -bool RtpReceiver::ReceivedPacket(const uint8* packet, size_t length) { - RtpCastHeader rtp_header; - const uint8* payload_data; - size_t payload_size; - if (!packet_parser_.ParsePacket( - packet, length, &rtp_header, &payload_data, &payload_size)) { - return false; - } - - OnReceivedPayloadData(payload_data, payload_size, rtp_header); - stats_.UpdateStatistics(rtp_header); - return true; -} - -} // namespace cast -} // namespace media diff --git a/media/cast/rtp_receiver/rtp_receiver.h b/media/cast/rtp_receiver/rtp_receiver.h deleted file mode 100644 index 35c7c933a2..0000000000 --- a/media/cast/rtp_receiver/rtp_receiver.h +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright 2013 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -// Interface to the rtp receiver. - -#ifndef MEDIA_CAST_RTP_RECEIVER_RTP_RECEIVER_H_ -#define MEDIA_CAST_RTP_RECEIVER_RTP_RECEIVER_H_ - -#include "base/memory/scoped_ptr.h" -#include "media/cast/cast_config.h" -#include "media/cast/rtcp/rtcp.h" -#include "media/cast/rtp_receiver/receiver_stats.h" -#include "media/cast/rtp_receiver/rtp_parser/rtp_parser.h" -#include "media/cast/rtp_receiver/rtp_receiver_defines.h" - -namespace media { -namespace cast { - -// TODO(miu): This is a good candidate to contain common functionality that's -// identical in both AudioReceiver and VideoReceiver. -class RtpReceiver { - public: - RtpReceiver(base::TickClock* clock, - const FrameReceiverConfig* audio_config, - const FrameReceiverConfig* video_config); - virtual ~RtpReceiver(); - - static uint32 GetSsrcOfSender(const uint8* rtcp_buffer, size_t length); - - bool ReceivedPacket(const uint8* packet, size_t length); - - RtpReceiverStatistics* GetStatistics() { - return &stats_; - } - - protected: - // Subclasses implement this to consume and process deserialized packets. - virtual void OnReceivedPayloadData(const uint8* payload_data, - size_t payload_size, - const RtpCastHeader& rtp_header) = 0; - - private: - RtpParser packet_parser_; - ReceiverStats stats_; - - DISALLOW_COPY_AND_ASSIGN(RtpReceiver); -}; - -} // namespace cast -} // namespace media - -#endif // MEDIA_CAST_RTP_RECEIVER_RTP_RECEIVER_H_ diff --git a/media/cast/test/end2end_unittest.cc b/media/cast/test/end2end_unittest.cc index 119067c7b2..4a0d820214 100644 --- a/media/cast/test/end2end_unittest.cc +++ b/media/cast/test/end2end_unittest.cc @@ -566,7 +566,7 @@ class End2EndTest : public ::testing::Test { void RequestAudioFrames(int count, bool with_check) { for (int i = 0; i < count; ++i) { - frame_receiver_->GetRawAudioFrame( + cast_receiver_->RequestDecodedAudioFrame( base::Bind(with_check ? &TestReceiverAudioCallback::CheckAudioFrame : &TestReceiverAudioCallback::IgnoreAudioFrame, test_receiver_audio_callback_)); @@ -611,8 +611,6 @@ class End2EndTest : public ::testing::Test { audio_frame_input_ = cast_sender_->audio_frame_input(); video_frame_input_ = cast_sender_->video_frame_input(); - frame_receiver_ = cast_receiver_->frame_receiver(); - audio_bus_factory_.reset( new TestAudioBusFactory(audio_sender_config_.channels, audio_sender_config_.frequency, @@ -684,7 +682,7 @@ class End2EndTest : public ::testing::Test { video_ticks_.push_back(std::make_pair( testing_clock_receiver_->NowTicks(), playout_time)); - frame_receiver_->GetRawVideoFrame( + cast_receiver_->RequestDecodedVideoFrame( base::Bind(&End2EndTest::BasicPlayerGotVideoFrame, base::Unretained(this))); } @@ -701,16 +699,16 @@ class End2EndTest : public ::testing::Test { audio_ticks_.push_back(std::make_pair( testing_clock_receiver_->NowTicks(), playout_time)); - frame_receiver_->GetRawAudioFrame( + cast_receiver_->RequestDecodedAudioFrame( base::Bind(&End2EndTest::BasicPlayerGotAudioFrame, base::Unretained(this))); } void StartBasicPlayer() { - frame_receiver_->GetRawVideoFrame( + cast_receiver_->RequestDecodedVideoFrame( base::Bind(&End2EndTest::BasicPlayerGotVideoFrame, base::Unretained(this))); - frame_receiver_->GetRawAudioFrame( + cast_receiver_->RequestDecodedAudioFrame( base::Bind(&End2EndTest::BasicPlayerGotAudioFrame, base::Unretained(this))); } @@ -768,7 +766,6 @@ class End2EndTest : public ::testing::Test { scoped_ptr<CastSender> cast_sender_; scoped_refptr<AudioFrameInput> audio_frame_input_; scoped_refptr<VideoFrameInput> video_frame_input_; - scoped_refptr<FrameReceiver> frame_receiver_; scoped_refptr<TestReceiverAudioCallback> test_receiver_audio_callback_; scoped_refptr<TestReceiverVideoCallback> test_receiver_video_callback_; @@ -820,7 +817,7 @@ TEST_F(End2EndTest, LoopNoLossPcm16) { RequestAudioFrames(num_audio_frames, true); num_audio_frames_requested += num_audio_frames; - frame_receiver_->GetRawVideoFrame( + cast_receiver_->RequestDecodedVideoFrame( base::Bind(&TestReceiverVideoCallback::CheckVideoFrame, test_receiver_video_callback_)); @@ -846,7 +843,7 @@ TEST_F(End2EndTest, LoopNoLossPcm16ExternalDecoder) { for (int i = 0; i < kNumIterations; ++i) { FeedAudioFrames(1, true); RunTasks(kAudioFrameDurationMs); - frame_receiver_->GetCodedAudioFrame( + cast_receiver_->RequestEncodedAudioFrame( base::Bind(&TestReceiverAudioCallback::CheckCodedAudioFrame, test_receiver_audio_callback_)); } @@ -954,7 +951,7 @@ TEST_F(End2EndTest, DISABLED_StartSenderBeforeReceiver) { RequestAudioFrames(num_audio_frames, true); num_audio_frames_requested += num_audio_frames; - frame_receiver_->GetRawVideoFrame( + cast_receiver_->RequestDecodedVideoFrame( base::Bind(&TestReceiverVideoCallback::CheckVideoFrame, test_receiver_video_callback_)); @@ -990,7 +987,7 @@ TEST_F(End2EndTest, DISABLED_GlitchWith3Buffers) { video_sender_config_.height, capture_time + base::TimeDelta::FromMilliseconds(kTargetPlayoutDelayMs), true); - frame_receiver_->GetRawVideoFrame( + cast_receiver_->RequestDecodedVideoFrame( base::Bind(&TestReceiverVideoCallback::CheckVideoFrame, test_receiver_video_callback_)); RunTasks(kFrameTimerMs); @@ -1023,7 +1020,7 @@ TEST_F(End2EndTest, DISABLED_GlitchWith3Buffers) { capture_time + base::TimeDelta::FromMilliseconds(kTargetPlayoutDelayMs), true); - frame_receiver_->GetRawVideoFrame( + cast_receiver_->RequestDecodedVideoFrame( base::Bind(&TestReceiverVideoCallback::CheckVideoFrame, test_receiver_video_callback_)); @@ -1059,7 +1056,7 @@ TEST_F(End2EndTest, DISABLED_DropEveryOtherFrame3Buffers) { // GetRawVideoFrame will not return the frame until we are close in // time before we should render the frame. - frame_receiver_->GetRawVideoFrame( + cast_receiver_->RequestDecodedVideoFrame( base::Bind(&TestReceiverVideoCallback::CheckVideoFrame, test_receiver_video_callback_)); } @@ -1100,7 +1097,7 @@ TEST_F(End2EndTest, CryptoVideo) { RunTasks(kFrameTimerMs); - frame_receiver_->GetRawVideoFrame( + cast_receiver_->RequestDecodedVideoFrame( base::Bind(&TestReceiverVideoCallback::CheckVideoFrame, test_receiver_video_callback_)); } @@ -1156,7 +1153,7 @@ TEST_F(End2EndTest, VideoLogging) { SendVideoFrame(video_start, capture_time); RunTasks(kFrameTimerMs); - frame_receiver_->GetRawVideoFrame( + cast_receiver_->RequestDecodedVideoFrame( base::Bind(&TestReceiverVideoCallback::CheckVideoFrame, test_receiver_video_callback_)); @@ -1332,7 +1329,10 @@ TEST_F(End2EndTest, AudioLogging) { map_it->second.counter[FRAME_DECODED]; EXPECT_GT(map_it->second.counter[FRAME_ACK_SENT], 0); + EXPECT_GT(map_it->second.counter[FRAME_ACK_RECEIVED], 0); expected_event_count_for_frame += map_it->second.counter[FRAME_ACK_SENT]; + expected_event_count_for_frame += + map_it->second.counter[FRAME_ACK_RECEIVED]; // Verify that there were no other events logged with respect to this frame. // (i.e. Total event count = expected event count) diff --git a/media/cast/test/utility/in_process_receiver.cc b/media/cast/test/utility/in_process_receiver.cc index ba190cec1f..cfcc1fcb46 100644 --- a/media/cast/test/utility/in_process_receiver.cc +++ b/media/cast/test/utility/in_process_receiver.cc @@ -114,14 +114,14 @@ void InProcessReceiver::GotVideoFrame( void InProcessReceiver::PullNextAudioFrame() { DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); - cast_receiver_->frame_receiver()->GetRawAudioFrame( + cast_receiver_->RequestDecodedAudioFrame( base::Bind(&InProcessReceiver::GotAudioFrame, weak_factory_.GetWeakPtr())); } void InProcessReceiver::PullNextVideoFrame() { DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); - cast_receiver_->frame_receiver()->GetRawVideoFrame(base::Bind( + cast_receiver_->RequestDecodedVideoFrame(base::Bind( &InProcessReceiver::GotVideoFrame, weak_factory_.GetWeakPtr())); } diff --git a/media/cast/transport/cast_transport_config.cc b/media/cast/transport/cast_transport_config.cc index 2c40a19d0a..16e9034713 100644 --- a/media/cast/transport/cast_transport_config.cc +++ b/media/cast/transport/cast_transport_config.cc @@ -41,6 +41,15 @@ EncodedFrame::EncodedFrame() EncodedFrame::~EncodedFrame() {} +void EncodedFrame::CopyMetadataTo(EncodedFrame* dest) const { + DCHECK(dest); + dest->dependency = this->dependency; + dest->frame_id = this->frame_id; + dest->referenced_frame_id = this->referenced_frame_id; + dest->rtp_timestamp = this->rtp_timestamp; + dest->reference_time = this->reference_time; +} + RtcpSenderInfo::RtcpSenderInfo() : ntp_seconds(0), ntp_fraction(0), diff --git a/media/cast/transport/cast_transport_config.h b/media/cast/transport/cast_transport_config.h index 683587080b..96b771acb9 100644 --- a/media/cast/transport/cast_transport_config.h +++ b/media/cast/transport/cast_transport_config.h @@ -108,6 +108,10 @@ struct EncodedFrame { return reinterpret_cast<uint8*>(string_as_array(&data)); } + // Copies all data members except |data| to |dest|. + // Does not modify |dest->data|. + void CopyMetadataTo(EncodedFrame* dest) const; + // This frame's dependency relationship with respect to other frames. Dependency dependency; diff --git a/media/cast/transport/cast_transport_sender.h b/media/cast/transport/cast_transport_sender.h index a69b7418f9..5c1eb04b41 100644 --- a/media/cast/transport/cast_transport_sender.h +++ b/media/cast/transport/cast_transport_sender.h @@ -2,9 +2,9 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -// This is the main interface for the cast transport sender. The cast sender -// handles the cast pipeline from encoded frames (both audio and video), to -// encryption, packetization and transport. +// This is the main interface for the cast transport sender. It accepts encoded +// frames (both audio and video), encrypts their encoded data, packetizes them +// and feeds them into a transport (e.g., UDP). // Construction of the Cast Sender and the Cast Transport Sender should be done // in the following order: diff --git a/media/cast/transport/cast_transport_sender_impl.cc b/media/cast/transport/cast_transport_sender_impl.cc index 8f07832d1c..06877b08ba 100644 --- a/media/cast/transport/cast_transport_sender_impl.cc +++ b/media/cast/transport/cast_transport_sender_impl.cc @@ -56,6 +56,7 @@ CastTransportSenderImpl::CastTransportSenderImpl( transport_task_runner), rtcp_builder_(&pacer_), raw_events_callback_(raw_events_callback) { + DCHECK(clock_); if (!raw_events_callback_.is_null()) { DCHECK(raw_events_callback_interval > base::TimeDelta()); event_subscriber_.reset(new SimpleEventSubscriber); @@ -74,24 +75,42 @@ CastTransportSenderImpl::~CastTransportSenderImpl() { void CastTransportSenderImpl::InitializeAudio( const CastTransportAudioConfig& config) { - pacer_.RegisterAudioSsrc(config.rtp.config.ssrc); - audio_sender_.reset(new TransportAudioSender( - config, clock_, transport_task_runner_, &pacer_)); - if (audio_sender_->initialized()) + LOG_IF(WARNING, config.rtp.config.aes_key.empty() || + config.rtp.config.aes_iv_mask.empty()) + << "Unsafe to send audio with encryption DISABLED."; + if (!audio_encryptor_.Initialize(config.rtp.config.aes_key, + config.rtp.config.aes_iv_mask)) { + status_callback_.Run(TRANSPORT_AUDIO_UNINITIALIZED); + return; + } + audio_sender_.reset(new RtpSender(clock_, transport_task_runner_, &pacer_)); + if (audio_sender_->InitializeAudio(config)) { + pacer_.RegisterAudioSsrc(config.rtp.config.ssrc); status_callback_.Run(TRANSPORT_AUDIO_INITIALIZED); - else + } else { + audio_sender_.reset(); status_callback_.Run(TRANSPORT_AUDIO_UNINITIALIZED); + } } void CastTransportSenderImpl::InitializeVideo( const CastTransportVideoConfig& config) { - pacer_.RegisterVideoSsrc(config.rtp.config.ssrc); - video_sender_.reset(new TransportVideoSender( - config, clock_, transport_task_runner_, &pacer_)); - if (video_sender_->initialized()) + LOG_IF(WARNING, config.rtp.config.aes_key.empty() || + config.rtp.config.aes_iv_mask.empty()) + << "Unsafe to send video with encryption DISABLED."; + if (!video_encryptor_.Initialize(config.rtp.config.aes_key, + config.rtp.config.aes_iv_mask)) { + status_callback_.Run(TRANSPORT_VIDEO_UNINITIALIZED); + return; + } + video_sender_.reset(new RtpSender(clock_, transport_task_runner_, &pacer_)); + if (video_sender_->InitializeVideo(config)) { + pacer_.RegisterVideoSsrc(config.rtp.config.ssrc); status_callback_.Run(TRANSPORT_VIDEO_INITIALIZED); - else + } else { + video_sender_.reset(); status_callback_.Run(TRANSPORT_VIDEO_UNINITIALIZED); + } } void CastTransportSenderImpl::SetPacketReceiver( @@ -99,16 +118,35 @@ void CastTransportSenderImpl::SetPacketReceiver( transport_->StartReceiving(packet_receiver); } +namespace { +void EncryptAndSendFrame(const EncodedFrame& frame, + TransportEncryptionHandler* encryptor, + RtpSender* sender) { + if (encryptor->initialized()) { + EncodedFrame encrypted_frame; + frame.CopyMetadataTo(&encrypted_frame); + if (encryptor->Encrypt(frame.frame_id, frame.data, &encrypted_frame.data)) { + sender->SendFrame(encrypted_frame); + } else { + LOG(ERROR) << "Encryption failed. Not sending frame with ID " + << frame.frame_id; + } + } else { + sender->SendFrame(frame); + } +} +} // namespace + void CastTransportSenderImpl::InsertCodedAudioFrame( const EncodedFrame& audio_frame) { DCHECK(audio_sender_) << "Audio sender uninitialized"; - audio_sender_->SendFrame(audio_frame); + EncryptAndSendFrame(audio_frame, &audio_encryptor_, audio_sender_.get()); } void CastTransportSenderImpl::InsertCodedVideoFrame( const EncodedFrame& video_frame) { DCHECK(video_sender_) << "Video sender uninitialized"; - video_sender_->SendFrame(video_frame); + EncryptAndSendFrame(video_frame, &video_encryptor_, video_sender_.get()); } void CastTransportSenderImpl::SendRtcpFromRtpSender( diff --git a/media/cast/transport/cast_transport_sender_impl.h b/media/cast/transport/cast_transport_sender_impl.h index 91b03c52be..0f440fb507 100644 --- a/media/cast/transport/cast_transport_sender_impl.h +++ b/media/cast/transport/cast_transport_sender_impl.h @@ -17,8 +17,8 @@ #include "media/cast/transport/cast_transport_sender.h" #include "media/cast/transport/pacing/paced_sender.h" #include "media/cast/transport/rtcp/rtcp_builder.h" -#include "media/cast/transport/transport_audio_sender.h" -#include "media/cast/transport/transport_video_sender.h" +#include "media/cast/transport/rtp_sender/rtp_sender.h" +#include "media/cast/transport/utility/transport_encryption_handler.h" namespace media { namespace cast { @@ -82,8 +82,15 @@ class CastTransportSenderImpl : public CastTransportSender { LoggingImpl logging_; PacedSender pacer_; RtcpBuilder rtcp_builder_; - scoped_ptr<TransportAudioSender> audio_sender_; - scoped_ptr<TransportVideoSender> video_sender_; + scoped_ptr<RtpSender> audio_sender_; + scoped_ptr<RtpSender> video_sender_; + + // Encrypts data in EncodedFrames before they are sent. Note that it's + // important for the encryption to happen here, in code that would execute in + // the main browser process, for security reasons. This helps to mitigate + // the damage that could be caused by a compromised renderer process. + TransportEncryptionHandler audio_encryptor_; + TransportEncryptionHandler video_encryptor_; // This is non-null iff |raw_events_callback_| is non-null. scoped_ptr<SimpleEventSubscriber> event_subscriber_; diff --git a/media/cast/transport/transport_audio_sender.cc b/media/cast/transport/transport_audio_sender.cc deleted file mode 100644 index 5d06c8b075..0000000000 --- a/media/cast/transport/transport_audio_sender.cc +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright 2014 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include "media/cast/transport/transport_audio_sender.h" - -#include "base/bind.h" -#include "base/logging.h" -#include "base/message_loop/message_loop.h" -#include "media/cast/transport/rtp_sender/rtp_sender.h" - -namespace media { -namespace cast { -namespace transport { - -TransportAudioSender::TransportAudioSender( - const CastTransportAudioConfig& config, - base::TickClock* clock, - const scoped_refptr<base::SingleThreadTaskRunner>& transport_task_runner, - PacedSender* const paced_packet_sender) - : rtp_sender_(clock, transport_task_runner, paced_packet_sender), - encryptor_() { - initialized_ = rtp_sender_.InitializeAudio(config) && - encryptor_.Initialize(config.rtp.config.aes_key, - config.rtp.config.aes_iv_mask); -} - -TransportAudioSender::~TransportAudioSender() {} - -void TransportAudioSender::SendFrame(const EncodedFrame& audio_frame) { - if (!initialized_) { - return; - } - if (encryptor_.initialized()) { - EncodedFrame encrypted_frame; - if (!EncryptAudioFrame(audio_frame, &encrypted_frame)) { - NOTREACHED(); - return; - } - rtp_sender_.SendFrame(encrypted_frame); - } else { - rtp_sender_.SendFrame(audio_frame); - } -} - -bool TransportAudioSender::EncryptAudioFrame( - const EncodedFrame& audio_frame, EncodedFrame* encrypted_frame) { - if (!initialized_) { - return false; - } - if (!encryptor_.Encrypt( - audio_frame.frame_id, audio_frame.data, &encrypted_frame->data)) - return false; - - encrypted_frame->dependency = audio_frame.dependency; - encrypted_frame->frame_id = audio_frame.frame_id; - encrypted_frame->referenced_frame_id = audio_frame.referenced_frame_id; - encrypted_frame->rtp_timestamp = audio_frame.rtp_timestamp; - encrypted_frame->reference_time = audio_frame.reference_time; - return true; -} - -void TransportAudioSender::ResendPackets( - const MissingFramesAndPacketsMap& missing_frames_and_packets) { - if (!initialized_) { - return; - } - rtp_sender_.ResendPackets(missing_frames_and_packets); -} - -} // namespace transport -} // namespace cast -} // namespace media diff --git a/media/cast/transport/transport_audio_sender.h b/media/cast/transport/transport_audio_sender.h deleted file mode 100644 index 84780d0c63..0000000000 --- a/media/cast/transport/transport_audio_sender.h +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright 2014 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#ifndef MEDIA_CAST_TRANSPORT_TRANSPORT_AUDIO_SENDER_H_ -#define MEDIA_CAST_TRANSPORT_TRANSPORT_AUDIO_SENDER_H_ - -#include "base/callback.h" -#include "base/memory/ref_counted.h" -#include "base/memory/scoped_ptr.h" -#include "base/threading/non_thread_safe.h" -#include "media/cast/transport/rtp_sender/rtp_sender.h" -#include "media/cast/transport/utility/transport_encryption_handler.h" - -namespace media { -namespace cast { - -namespace transport { - -class PacedSender; - -// It's only called from the main cast transport thread. -class TransportAudioSender : public base::NonThreadSafe { - public: - TransportAudioSender( - const CastTransportAudioConfig& config, - base::TickClock* clock, - const scoped_refptr<base::SingleThreadTaskRunner>& transport_task_runner, - PacedSender* const paced_packet_sender); - - virtual ~TransportAudioSender(); - - // Handles the encoded audio frames to be processed. - // Frames will be encrypted, packetized and transmitted to the network. - void SendFrame(const EncodedFrame& audio_frame); - - // Retransmision request. - void ResendPackets( - const MissingFramesAndPacketsMap& missing_frames_and_packets); - - size_t send_packet_count() const { return rtp_sender_.send_packet_count(); } - size_t send_octet_count() const { return rtp_sender_.send_octet_count(); } - uint32 ssrc() const { return rtp_sender_.ssrc(); } - bool initialized() const { return initialized_; } - - private: - friend class LocalRtcpAudioSenderFeedback; - - // Caller must allocate the destination |encrypted_frame|. The data member - // will be resized to hold the encrypted size. - bool EncryptAudioFrame(const EncodedFrame& audio_frame, - EncodedFrame* encrypted_frame); - - RtpSender rtp_sender_; - TransportEncryptionHandler encryptor_; - bool initialized_; - - DISALLOW_IMPLICIT_CONSTRUCTORS(TransportAudioSender); -}; - -} // namespace transport -} // namespace cast -} // namespace media - -#endif // MEDIA_CAST_TRANSPORT_TRANSPORT_AUDIO_SENDER_H_ diff --git a/media/cast/transport/transport_video_sender.cc b/media/cast/transport/transport_video_sender.cc deleted file mode 100644 index 1add29b8cb..0000000000 --- a/media/cast/transport/transport_video_sender.cc +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright 2014 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include "media/cast/transport/transport_video_sender.h" - -#include "base/bind.h" -#include "base/logging.h" -#include "base/message_loop/message_loop.h" -#include "media/base/video_frame.h" -#include "media/cast/transport/pacing/paced_sender.h" - -namespace media { -namespace cast { -namespace transport { - -TransportVideoSender::TransportVideoSender( - const CastTransportVideoConfig& config, - base::TickClock* clock, - const scoped_refptr<base::SingleThreadTaskRunner>& transport_task_runner, - PacedSender* const paced_packet_sender) - : rtp_sender_(clock, transport_task_runner, paced_packet_sender) { - initialized_ = rtp_sender_.InitializeVideo(config) && - encryptor_.Initialize(config.rtp.config.aes_key, - config.rtp.config.aes_iv_mask); -} - -TransportVideoSender::~TransportVideoSender() {} - -void TransportVideoSender::SendFrame(const EncodedFrame& video_frame) { - if (!initialized_) { - return; - } - if (encryptor_.initialized()) { - EncodedFrame encrypted_frame; - if (!EncryptVideoFrame(video_frame, &encrypted_frame)) { - NOTREACHED(); - return; - } - rtp_sender_.SendFrame(encrypted_frame); - } else { - rtp_sender_.SendFrame(video_frame); - } -} - -bool TransportVideoSender::EncryptVideoFrame( - const EncodedFrame& video_frame, EncodedFrame* encrypted_frame) { - if (!initialized_) { - return false; - } - if (!encryptor_.Encrypt( - video_frame.frame_id, video_frame.data, &(encrypted_frame->data))) - return false; - - encrypted_frame->dependency = video_frame.dependency; - encrypted_frame->frame_id = video_frame.frame_id; - encrypted_frame->referenced_frame_id = video_frame.referenced_frame_id; - encrypted_frame->rtp_timestamp = video_frame.rtp_timestamp; - encrypted_frame->reference_time = video_frame.reference_time; - return true; -} - -void TransportVideoSender::ResendPackets( - const MissingFramesAndPacketsMap& missing_frames_and_packets) { - if (!initialized_) { - return; - } - rtp_sender_.ResendPackets(missing_frames_and_packets); -} - -} // namespace transport -} // namespace cast -} // namespace media diff --git a/media/cast/transport/transport_video_sender.h b/media/cast/transport/transport_video_sender.h deleted file mode 100644 index 3025cec19b..0000000000 --- a/media/cast/transport/transport_video_sender.h +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright 2014 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#ifndef MEDIA_CAST_TRANSPORT_TRANSPORT_VIDEO_SENDER_H_ -#define MEDIA_CAST_TRANSPORT_TRANSPORT_VIDEO_SENDER_H_ - -#include "base/callback.h" -#include "base/memory/ref_counted.h" -#include "base/memory/scoped_ptr.h" -#include "base/threading/non_thread_safe.h" -#include "base/time/tick_clock.h" -#include "media/cast/transport/cast_transport_config.h" -#include "media/cast/transport/rtp_sender/rtp_sender.h" -#include "media/cast/transport/utility/transport_encryption_handler.h" - -namespace media { -class VideoFrame; - -namespace cast { - -namespace transport { -class PacedSender; - -// Not thread safe. Only called from the main cast transport thread. -// This class owns all objects related to sending coded video, objects that -// encrypt, create RTP packets and send to network. -class TransportVideoSender : public base::NonThreadSafe { - public: - TransportVideoSender( - const CastTransportVideoConfig& config, - base::TickClock* clock, - const scoped_refptr<base::SingleThreadTaskRunner>& transport_task_runner, - PacedSender* const paced_packet_sender); - - virtual ~TransportVideoSender(); - - // Handles the encoded video frames to be processed. - // Frames will be encrypted, packetized and transmitted to the network. - void SendFrame(const EncodedFrame& video_frame); - - // Retransmision request. - void ResendPackets( - const MissingFramesAndPacketsMap& missing_frames_and_packets); - - size_t send_packet_count() const { return rtp_sender_.send_packet_count(); } - size_t send_octet_count() const { return rtp_sender_.send_octet_count(); } - uint32 ssrc() const { return rtp_sender_.ssrc(); } - bool initialized() const { return initialized_; } - - private: - // Caller must allocate the destination |encrypted_video_frame| the data - // member will be resized to hold the encrypted size. - bool EncryptVideoFrame(const EncodedFrame& encoded_frame, - EncodedFrame* encrypted_video_frame); - - const base::TimeDelta rtp_max_delay_; - TransportEncryptionHandler encryptor_; - RtpSender rtp_sender_; - bool initialized_; - - DISALLOW_IMPLICIT_CONSTRUCTORS(TransportVideoSender); -}; - -} // namespace transport -} // namespace cast -} // namespace media - -#endif // MEDIA_CAST_TRANSPORT_TRANSPORT_VIDEO_SENDER_H_ diff --git a/media/cast/transport/utility/transport_encryption_handler.h b/media/cast/transport/utility/transport_encryption_handler.h index dd740be207..06d0e3f34d 100644 --- a/media/cast/transport/utility/transport_encryption_handler.h +++ b/media/cast/transport/utility/transport_encryption_handler.h @@ -37,6 +37,9 @@ class TransportEncryptionHandler : public base::NonThreadSafe { const base::StringPiece& ciphertext, std::string* plaintext); + // TODO(miu): This naming is very misleading. It should be called + // is_activated() since Initialize() without keys (i.e., cypto is disabled) + // may have succeeded. bool initialized() const { return initialized_; } private: diff --git a/media/cast/video_receiver/video_receiver.cc b/media/cast/video_receiver/video_receiver.cc deleted file mode 100644 index d9000de88e..0000000000 --- a/media/cast/video_receiver/video_receiver.cc +++ /dev/null @@ -1,380 +0,0 @@ -// Copyright 2013 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include "media/cast/video_receiver/video_receiver.h" - -#include <algorithm> - -#include "base/bind.h" -#include "base/debug/trace_event.h" -#include "base/logging.h" -#include "base/message_loop/message_loop.h" -#include "media/base/video_frame.h" -#include "media/cast/logging/logging_defines.h" -#include "media/cast/transport/cast_transport_defines.h" -#include "media/cast/video_receiver/video_decoder.h" - -namespace { -const int kMinSchedulingDelayMs = 1; -} // namespace - -namespace media { -namespace cast { - -VideoReceiver::VideoReceiver(scoped_refptr<CastEnvironment> cast_environment, - const FrameReceiverConfig& video_config, - transport::PacedPacketSender* const packet_sender) - : RtpReceiver(cast_environment->Clock(), NULL, &video_config), - cast_environment_(cast_environment), - event_subscriber_(kReceiverRtcpEventHistorySize, VIDEO_EVENT), - codec_(video_config.codec.video), - target_playout_delay_( - base::TimeDelta::FromMilliseconds(video_config.rtp_max_delay_ms)), - expected_frame_duration_( - base::TimeDelta::FromSeconds(1) / video_config.max_frame_rate), - reports_are_scheduled_(false), - framer_(cast_environment->Clock(), - this, - video_config.incoming_ssrc, - true, - video_config.rtp_max_delay_ms * video_config.max_frame_rate / - 1000), - rtcp_(cast_environment_, - NULL, - NULL, - packet_sender, - GetStatistics(), - video_config.rtcp_mode, - base::TimeDelta::FromMilliseconds(video_config.rtcp_interval), - video_config.feedback_ssrc, - video_config.incoming_ssrc, - video_config.rtcp_c_name, - false), - is_waiting_for_consecutive_frame_(false), - lip_sync_drift_(ClockDriftSmoother::GetDefaultTimeConstant()), - weak_factory_(this) { - DCHECK_GT(video_config.rtp_max_delay_ms, 0); - DCHECK_GT(video_config.max_frame_rate, 0); - video_decoder_.reset(new VideoDecoder(cast_environment, video_config)); - decryptor_.Initialize(video_config.aes_key, video_config.aes_iv_mask); - rtcp_.SetTargetDelay(target_playout_delay_); - cast_environment_->Logging()->AddRawEventSubscriber(&event_subscriber_); - memset(frame_id_to_rtp_timestamp_, 0, sizeof(frame_id_to_rtp_timestamp_)); -} - -VideoReceiver::~VideoReceiver() { - DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); - cast_environment_->Logging()->RemoveRawEventSubscriber(&event_subscriber_); -} - -void VideoReceiver::GetRawVideoFrame( - const VideoFrameDecodedCallback& callback) { - DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); - DCHECK(!callback.is_null()); - DCHECK(video_decoder_.get()); - GetEncodedVideoFrame(base::Bind( - &VideoReceiver::DecodeEncodedVideoFrame, - // Note: Use of Unretained is safe since this Closure is guaranteed to be - // invoked before destruction of |this|. - base::Unretained(this), - callback)); -} - -void VideoReceiver::DecodeEncodedVideoFrame( - const VideoFrameDecodedCallback& callback, - scoped_ptr<transport::EncodedFrame> encoded_frame) { - DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); - if (!encoded_frame) { - callback.Run( - make_scoped_refptr<VideoFrame>(NULL), base::TimeTicks(), false); - return; - } - const uint32 frame_id = encoded_frame->frame_id; - const uint32 rtp_timestamp = encoded_frame->rtp_timestamp; - const base::TimeTicks playout_time = encoded_frame->reference_time; - video_decoder_->DecodeFrame(encoded_frame.Pass(), - base::Bind(&VideoReceiver::EmitRawVideoFrame, - cast_environment_, - callback, - frame_id, - rtp_timestamp, - playout_time)); -} - -// static -void VideoReceiver::EmitRawVideoFrame( - const scoped_refptr<CastEnvironment>& cast_environment, - const VideoFrameDecodedCallback& callback, - uint32 frame_id, - uint32 rtp_timestamp, - const base::TimeTicks& playout_time, - const scoped_refptr<VideoFrame>& video_frame, - bool is_continuous) { - DCHECK(cast_environment->CurrentlyOn(CastEnvironment::MAIN)); - if (video_frame) { - const base::TimeTicks now = cast_environment->Clock()->NowTicks(); - cast_environment->Logging()->InsertFrameEvent( - now, FRAME_DECODED, VIDEO_EVENT, rtp_timestamp, frame_id); - cast_environment->Logging()->InsertFrameEventWithDelay( - now, FRAME_PLAYOUT, VIDEO_EVENT, rtp_timestamp, frame_id, - playout_time - now); - // Used by chrome/browser/extension/api/cast_streaming/performance_test.cc - TRACE_EVENT_INSTANT1( - "cast_perf_test", "FrameDecoded", - TRACE_EVENT_SCOPE_THREAD, - "rtp_timestamp", rtp_timestamp); - } - callback.Run(video_frame, playout_time, is_continuous); -} - -void VideoReceiver::GetEncodedVideoFrame(const FrameEncodedCallback& callback) { - DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); - frame_request_queue_.push_back(callback); - EmitAvailableEncodedFrames(); -} - -void VideoReceiver::EmitAvailableEncodedFrames() { - DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); - - while (!frame_request_queue_.empty()) { - // Attempt to peek at the next completed frame from the |framer_|. - // TODO(miu): We should only be peeking at the metadata, and not copying the - // payload yet! Or, at least, peek using a StringPiece instead of a copy. - scoped_ptr<transport::EncodedFrame> encoded_frame( - new transport::EncodedFrame()); - bool is_consecutively_next_frame = false; - bool have_multiple_complete_frames = false; - - if (!framer_.GetEncodedFrame(encoded_frame.get(), - &is_consecutively_next_frame, - &have_multiple_complete_frames)) { - VLOG(1) << "Wait for more video packets to produce a completed frame."; - return; // OnReceivedPayloadData() will invoke this method in the future. - } - - const base::TimeTicks now = cast_environment_->Clock()->NowTicks(); - const base::TimeTicks playout_time = - GetPlayoutTime(encoded_frame->rtp_timestamp); - - // If we have multiple decodable frames, and the current frame is - // too old, then skip it and decode the next frame instead. - if (have_multiple_complete_frames && now > playout_time) { - framer_.ReleaseFrame(encoded_frame->frame_id); - continue; - } - - // If |framer_| has a frame ready that is out of sequence, examine the - // playout time to determine whether it's acceptable to continue, thereby - // skipping one or more frames. Skip if the missing frame wouldn't complete - // playing before the start of playback of the available frame. - if (!is_consecutively_next_frame) { - // TODO(miu): Also account for expected decode time here? - const base::TimeTicks earliest_possible_end_time_of_missing_frame = - now + expected_frame_duration_; - if (earliest_possible_end_time_of_missing_frame < playout_time) { - VLOG(1) << "Wait for next consecutive frame instead of skipping."; - if (!is_waiting_for_consecutive_frame_) { - is_waiting_for_consecutive_frame_ = true; - cast_environment_->PostDelayedTask( - CastEnvironment::MAIN, - FROM_HERE, - base::Bind(&VideoReceiver::EmitAvailableEncodedFramesAfterWaiting, - weak_factory_.GetWeakPtr()), - playout_time - now); - } - return; - } - } - - // Decrypt the payload data in the frame, if crypto is being used. - if (decryptor_.initialized()) { - std::string decrypted_video_data; - if (!decryptor_.Decrypt(encoded_frame->frame_id, - encoded_frame->data, - &decrypted_video_data)) { - // Decryption failed. Give up on this frame, releasing it from the - // jitter buffer. - framer_.ReleaseFrame(encoded_frame->frame_id); - continue; - } - encoded_frame->data.swap(decrypted_video_data); - } - - // At this point, we have a decrypted EncodedFrame ready to be emitted. - encoded_frame->reference_time = playout_time; - framer_.ReleaseFrame(encoded_frame->frame_id); - // Used by chrome/browser/extension/api/cast_streaming/performance_test.cc - TRACE_EVENT_INSTANT2( - "cast_perf_test", "PullEncodedVideoFrame", - TRACE_EVENT_SCOPE_THREAD, - "rtp_timestamp", encoded_frame->rtp_timestamp, - // TODO(miu): Need to find an alternative to using ToInternalValue(): - "render_time", playout_time.ToInternalValue()); - cast_environment_->PostTask(CastEnvironment::MAIN, - FROM_HERE, - base::Bind(frame_request_queue_.front(), - base::Passed(&encoded_frame))); - frame_request_queue_.pop_front(); - } -} - -void VideoReceiver::EmitAvailableEncodedFramesAfterWaiting() { - DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); - DCHECK(is_waiting_for_consecutive_frame_); - is_waiting_for_consecutive_frame_ = false; - EmitAvailableEncodedFrames(); -} - -base::TimeTicks VideoReceiver::GetPlayoutTime(uint32 rtp_timestamp) const { - return lip_sync_reference_time_ + - lip_sync_drift_.Current() + - RtpDeltaToTimeDelta( - static_cast<int32>(rtp_timestamp - lip_sync_rtp_timestamp_), - kVideoFrequency) + - target_playout_delay_; -} - -void VideoReceiver::IncomingPacket(scoped_ptr<Packet> packet) { - DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); - if (Rtcp::IsRtcpPacket(&packet->front(), packet->size())) { - rtcp_.IncomingRtcpPacket(&packet->front(), packet->size()); - } else { - ReceivedPacket(&packet->front(), packet->size()); - } - if (!reports_are_scheduled_) { - ScheduleNextRtcpReport(); - ScheduleNextCastMessage(); - reports_are_scheduled_ = true; - } -} - -void VideoReceiver::OnReceivedPayloadData(const uint8* payload_data, - size_t payload_size, - const RtpCastHeader& rtp_header) { - DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); - - const base::TimeTicks now = cast_environment_->Clock()->NowTicks(); - - frame_id_to_rtp_timestamp_[rtp_header.frame_id & 0xff] = - rtp_header.rtp_timestamp; - cast_environment_->Logging()->InsertPacketEvent( - now, - PACKET_RECEIVED, - VIDEO_EVENT, - rtp_header.rtp_timestamp, - rtp_header.frame_id, - rtp_header.packet_id, - rtp_header.max_packet_id, - payload_size); - - bool duplicate = false; - const bool complete = - framer_.InsertPacket(payload_data, payload_size, rtp_header, &duplicate); - - // Duplicate packets are ignored. - if (duplicate) - return; - - // Update lip-sync values upon receiving the first packet of each frame, or if - // they have never been set yet. - if (rtp_header.packet_id == 0 || lip_sync_reference_time_.is_null()) { - RtpTimestamp fresh_sync_rtp; - base::TimeTicks fresh_sync_reference; - if (!rtcp_.GetLatestLipSyncTimes(&fresh_sync_rtp, &fresh_sync_reference)) { - // HACK: The sender should have provided Sender Reports before the first - // frame was sent. However, the spec does not currently require this. - // Therefore, when the data is missing, the local clock is used to - // generate reference timestamps. - VLOG(2) << "Lip sync info missing. Falling-back to local clock."; - fresh_sync_rtp = rtp_header.rtp_timestamp; - fresh_sync_reference = now; - } - // |lip_sync_reference_time_| is always incremented according to the time - // delta computed from the difference in RTP timestamps. Then, - // |lip_sync_drift_| accounts for clock drift and also smoothes-out any - // sudden/discontinuous shifts in the series of reference time values. - if (lip_sync_reference_time_.is_null()) { - lip_sync_reference_time_ = fresh_sync_reference; - } else { - lip_sync_reference_time_ += RtpDeltaToTimeDelta( - static_cast<int32>(fresh_sync_rtp - lip_sync_rtp_timestamp_), - kVideoFrequency); - } - lip_sync_rtp_timestamp_ = fresh_sync_rtp; - lip_sync_drift_.Update( - now, fresh_sync_reference - lip_sync_reference_time_); - } - - // Video frame not complete; wait for more packets. - if (!complete) - return; - - EmitAvailableEncodedFrames(); -} - -// Send a cast feedback message. Actual message created in the framer (cast -// message builder). -void VideoReceiver::CastFeedback(const RtcpCastMessage& cast_message) { - DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); - - base::TimeTicks now = cast_environment_->Clock()->NowTicks(); - RtpTimestamp rtp_timestamp = - frame_id_to_rtp_timestamp_[cast_message.ack_frame_id_ & 0xff]; - cast_environment_->Logging()->InsertFrameEvent( - now, FRAME_ACK_SENT, VIDEO_EVENT, - rtp_timestamp, cast_message.ack_frame_id_); - - ReceiverRtcpEventSubscriber::RtcpEventMultiMap rtcp_events; - event_subscriber_.GetRtcpEventsAndReset(&rtcp_events); - rtcp_.SendRtcpFromRtpReceiver(&cast_message, &rtcp_events); -} - -// Cast messages should be sent within a maximum interval. Schedule a call -// if not triggered elsewhere, e.g. by the cast message_builder. -void VideoReceiver::ScheduleNextCastMessage() { - DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); - base::TimeTicks send_time; - framer_.TimeToSendNextCastMessage(&send_time); - base::TimeDelta time_to_send = - send_time - cast_environment_->Clock()->NowTicks(); - time_to_send = std::max( - time_to_send, base::TimeDelta::FromMilliseconds(kMinSchedulingDelayMs)); - cast_environment_->PostDelayedTask( - CastEnvironment::MAIN, - FROM_HERE, - base::Bind(&VideoReceiver::SendNextCastMessage, - weak_factory_.GetWeakPtr()), - time_to_send); -} - -void VideoReceiver::SendNextCastMessage() { - DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); - framer_.SendCastMessage(); // Will only send a message if it is time. - ScheduleNextCastMessage(); -} - -void VideoReceiver::ScheduleNextRtcpReport() { - DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); - base::TimeDelta time_to_next = rtcp_.TimeToSendNextRtcpReport() - - cast_environment_->Clock()->NowTicks(); - - time_to_next = std::max( - time_to_next, base::TimeDelta::FromMilliseconds(kMinSchedulingDelayMs)); - - cast_environment_->PostDelayedTask( - CastEnvironment::MAIN, - FROM_HERE, - base::Bind(&VideoReceiver::SendNextRtcpReport, - weak_factory_.GetWeakPtr()), - time_to_next); -} - -void VideoReceiver::SendNextRtcpReport() { - DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); - rtcp_.SendRtcpFromRtpReceiver(NULL, NULL); - ScheduleNextRtcpReport(); -} - -} // namespace cast -} // namespace media diff --git a/media/cast/video_receiver/video_receiver_unittest.cc b/media/cast/video_receiver/video_receiver_unittest.cc deleted file mode 100644 index d158d7e325..0000000000 --- a/media/cast/video_receiver/video_receiver_unittest.cc +++ /dev/null @@ -1,266 +0,0 @@ -// Copyright 2013 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include <deque> -#include <utility> - -#include "base/bind.h" -#include "base/memory/ref_counted.h" -#include "base/memory/scoped_ptr.h" -#include "base/test/simple_test_tick_clock.h" -#include "media/cast/cast_defines.h" -#include "media/cast/cast_environment.h" -#include "media/cast/logging/simple_event_subscriber.h" -#include "media/cast/rtcp/test_rtcp_packet_builder.h" -#include "media/cast/test/fake_single_thread_task_runner.h" -#include "media/cast/test/utility/default_config.h" -#include "media/cast/transport/pacing/mock_paced_packet_sender.h" -#include "media/cast/video_receiver/video_receiver.h" -#include "testing/gmock/include/gmock/gmock.h" - -using ::testing::_; - -namespace media { -namespace cast { - -namespace { - -const int kPacketSize = 1500; -const uint32 kFirstFrameId = 1234; -const int kPlayoutDelayMillis = 100; - -class FakeVideoClient { - public: - FakeVideoClient() : num_called_(0) {} - virtual ~FakeVideoClient() {} - - void AddExpectedResult(uint32 expected_frame_id, - const base::TimeTicks& expected_playout_time) { - expected_results_.push_back( - std::make_pair(expected_frame_id, expected_playout_time)); - } - - void DeliverEncodedVideoFrame( - scoped_ptr<transport::EncodedFrame> video_frame) { - SCOPED_TRACE(::testing::Message() << "num_called_ is " << num_called_); - ASSERT_FALSE(!video_frame) - << "If at shutdown: There were unsatisfied requests enqueued."; - ASSERT_FALSE(expected_results_.empty()); - EXPECT_EQ(expected_results_.front().first, video_frame->frame_id); - EXPECT_EQ(expected_results_.front().second, video_frame->reference_time); - expected_results_.pop_front(); - ++num_called_; - } - - int number_times_called() const { return num_called_; } - - private: - std::deque<std::pair<uint32, base::TimeTicks> > expected_results_; - int num_called_; - - DISALLOW_COPY_AND_ASSIGN(FakeVideoClient); -}; -} // namespace - -class VideoReceiverTest : public ::testing::Test { - protected: - VideoReceiverTest() { - config_ = GetDefaultVideoReceiverConfig(); - config_.rtp_max_delay_ms = kPlayoutDelayMillis; - // Note: Frame rate must divide 1000 without remainder so the test code - // doesn't have to account for rounding errors. - config_.max_frame_rate = 25; - config_.codec.video = transport::kVp8; // Frame skipping not allowed. - testing_clock_ = new base::SimpleTestTickClock(); - testing_clock_->Advance(base::TimeTicks::Now() - base::TimeTicks()); - start_time_ = testing_clock_->NowTicks(); - task_runner_ = new test::FakeSingleThreadTaskRunner(testing_clock_); - - cast_environment_ = - new CastEnvironment(scoped_ptr<base::TickClock>(testing_clock_).Pass(), - task_runner_, - task_runner_, - task_runner_); - - receiver_.reset(new VideoReceiver( - cast_environment_, config_, &mock_transport_)); - } - - virtual ~VideoReceiverTest() {} - - virtual void SetUp() { - payload_.assign(kPacketSize, 0); - - // Always start with a key frame. - rtp_header_.is_key_frame = true; - rtp_header_.frame_id = kFirstFrameId; - rtp_header_.reference_frame_id = rtp_header_.frame_id; - rtp_header_.packet_id = 0; - rtp_header_.max_packet_id = 0; - } - - void FeedOneFrameIntoReceiver() { - receiver_->OnReceivedPayloadData( - payload_.data(), payload_.size(), rtp_header_); - } - - void FeedLipSyncInfoIntoReceiver() { - const base::TimeTicks now = testing_clock_->NowTicks(); - const int64 rtp_timestamp = (now - start_time_) * - kVideoFrequency / base::TimeDelta::FromSeconds(1); - CHECK_LE(0, rtp_timestamp); - uint32 ntp_seconds; - uint32 ntp_fraction; - ConvertTimeTicksToNtp(now, &ntp_seconds, &ntp_fraction); - TestRtcpPacketBuilder rtcp_packet; - rtcp_packet.AddSrWithNtp(config_.incoming_ssrc, - ntp_seconds, ntp_fraction, - static_cast<uint32>(rtp_timestamp)); - receiver_->IncomingPacket(rtcp_packet.GetPacket().Pass()); - } - - FrameReceiverConfig config_; - std::vector<uint8> payload_; - RtpCastHeader rtp_header_; - base::SimpleTestTickClock* testing_clock_; // Owned by CastEnvironment. - base::TimeTicks start_time_; - transport::MockPacedPacketSender mock_transport_; - scoped_refptr<test::FakeSingleThreadTaskRunner> task_runner_; - scoped_refptr<CastEnvironment> cast_environment_; - FakeVideoClient fake_video_client_; - - // Important for the VideoReceiver to be declared last, since its dependencies - // must remain alive until after its destruction. - scoped_ptr<VideoReceiver> receiver_; - - DISALLOW_COPY_AND_ASSIGN(VideoReceiverTest); -}; - -TEST_F(VideoReceiverTest, ReceivesOneFrame) { - SimpleEventSubscriber event_subscriber; - cast_environment_->Logging()->AddRawEventSubscriber(&event_subscriber); - - EXPECT_CALL(mock_transport_, SendRtcpPacket(_, _)) - .WillRepeatedly(testing::Return(true)); - - FeedLipSyncInfoIntoReceiver(); - task_runner_->RunTasks(); - - // Enqueue a request for a video frame. - receiver_->GetEncodedVideoFrame( - base::Bind(&FakeVideoClient::DeliverEncodedVideoFrame, - base::Unretained(&fake_video_client_))); - - // The request should not be satisfied since no packets have been received. - task_runner_->RunTasks(); - EXPECT_EQ(0, fake_video_client_.number_times_called()); - - // Deliver one video frame to the receiver and expect to get one frame back. - const base::TimeDelta target_playout_delay = - base::TimeDelta::FromMilliseconds(kPlayoutDelayMillis); - fake_video_client_.AddExpectedResult( - kFirstFrameId, testing_clock_->NowTicks() + target_playout_delay); - FeedOneFrameIntoReceiver(); - task_runner_->RunTasks(); - EXPECT_EQ(1, fake_video_client_.number_times_called()); - - std::vector<FrameEvent> frame_events; - event_subscriber.GetFrameEventsAndReset(&frame_events); - - ASSERT_TRUE(!frame_events.empty()); - EXPECT_EQ(FRAME_ACK_SENT, frame_events.begin()->type); - EXPECT_EQ(rtp_header_.frame_id, frame_events.begin()->frame_id); - EXPECT_EQ(rtp_header_.rtp_timestamp, frame_events.begin()->rtp_timestamp); - - cast_environment_->Logging()->RemoveRawEventSubscriber(&event_subscriber); -} - -TEST_F(VideoReceiverTest, ReceivesFramesRefusingToSkipAny) { - EXPECT_CALL(mock_transport_, SendRtcpPacket(_, _)) - .WillRepeatedly(testing::Return(true)); - - const uint32 rtp_advance_per_frame = - config_.frequency / config_.max_frame_rate; - const base::TimeDelta time_advance_per_frame = - base::TimeDelta::FromSeconds(1) / config_.max_frame_rate; - - // Feed and process lip sync in receiver. - FeedLipSyncInfoIntoReceiver(); - task_runner_->RunTasks(); - const base::TimeTicks first_frame_capture_time = testing_clock_->NowTicks(); - - // Enqueue a request for a video frame. - const FrameEncodedCallback frame_encoded_callback = - base::Bind(&FakeVideoClient::DeliverEncodedVideoFrame, - base::Unretained(&fake_video_client_)); - receiver_->GetEncodedVideoFrame(frame_encoded_callback); - task_runner_->RunTasks(); - EXPECT_EQ(0, fake_video_client_.number_times_called()); - - // Receive one video frame and expect to see the first request satisfied. - const base::TimeDelta target_playout_delay = - base::TimeDelta::FromMilliseconds(kPlayoutDelayMillis); - fake_video_client_.AddExpectedResult( - kFirstFrameId, first_frame_capture_time + target_playout_delay); - rtp_header_.rtp_timestamp = 0; - FeedOneFrameIntoReceiver(); - task_runner_->RunTasks(); - EXPECT_EQ(1, fake_video_client_.number_times_called()); - - // Enqueue a second request for a video frame, but it should not be - // fulfilled yet. - receiver_->GetEncodedVideoFrame(frame_encoded_callback); - task_runner_->RunTasks(); - EXPECT_EQ(1, fake_video_client_.number_times_called()); - - // Receive one video frame out-of-order: Make sure that we are not continuous - // and that the RTP timestamp represents a time in the future. - rtp_header_.is_key_frame = false; - rtp_header_.frame_id = kFirstFrameId + 2; // "Frame 3" - rtp_header_.reference_frame_id = kFirstFrameId + 1; // "Frame 2" - rtp_header_.rtp_timestamp += 2 * rtp_advance_per_frame; - FeedOneFrameIntoReceiver(); - - // Frame 2 should not come out at this point in time. - task_runner_->RunTasks(); - EXPECT_EQ(1, fake_video_client_.number_times_called()); - - // Enqueue a third request for a video frame. - receiver_->GetEncodedVideoFrame(frame_encoded_callback); - task_runner_->RunTasks(); - EXPECT_EQ(1, fake_video_client_.number_times_called()); - - // Now, advance time forward such that Frame 2 is now too late for playback. - // Regardless, the receiver must NOT emit Frame 3 yet because it is not - // allowed to skip frames for VP8. - testing_clock_->Advance(2 * time_advance_per_frame + target_playout_delay); - task_runner_->RunTasks(); - EXPECT_EQ(1, fake_video_client_.number_times_called()); - - // Now receive Frame 2 and expect both the second and third requests to be - // fulfilled immediately. - fake_video_client_.AddExpectedResult( - kFirstFrameId + 1, // "Frame 2" - first_frame_capture_time + 1 * time_advance_per_frame + - target_playout_delay); - fake_video_client_.AddExpectedResult( - kFirstFrameId + 2, // "Frame 3" - first_frame_capture_time + 2 * time_advance_per_frame + - target_playout_delay); - --rtp_header_.frame_id; // "Frame 2" - --rtp_header_.reference_frame_id; // "Frame 1" - rtp_header_.rtp_timestamp -= rtp_advance_per_frame; - FeedOneFrameIntoReceiver(); - task_runner_->RunTasks(); - EXPECT_EQ(3, fake_video_client_.number_times_called()); - - // Move forward to the playout time of an unreceived Frame 5. Expect no - // additional frames were emitted. - testing_clock_->Advance(3 * time_advance_per_frame); - task_runner_->RunTasks(); - EXPECT_EQ(3, fake_video_client_.number_times_called()); -} - -} // namespace cast -} // namespace media diff --git a/media/cast/video_sender/codecs/vp8/vp8_encoder.cc b/media/cast/video_sender/codecs/vp8/vp8_encoder.cc index 4905d3475e..c7374babd1 100644 --- a/media/cast/video_sender/codecs/vp8/vp8_encoder.cc +++ b/media/cast/video_sender/codecs/vp8/vp8_encoder.cc @@ -19,7 +19,7 @@ namespace cast { static const uint32 kMinIntra = 300; -static int ComputeMaxNumOfRepeatedBuffes(uint8 max_unacked_frames) { +static int ComputeMaxNumOfRepeatedBuffes(int max_unacked_frames) { if (max_unacked_frames > kNumberOfVp8VideoBuffers) return (max_unacked_frames - 1) / kNumberOfVp8VideoBuffers; @@ -27,7 +27,7 @@ static int ComputeMaxNumOfRepeatedBuffes(uint8 max_unacked_frames) { } Vp8Encoder::Vp8Encoder(const VideoSenderConfig& video_config, - uint8 max_unacked_frames) + int max_unacked_frames) : cast_config_(video_config), use_multiple_video_buffers_( cast_config_.max_number_of_video_buffers_used == diff --git a/media/cast/video_sender/codecs/vp8/vp8_encoder.h b/media/cast/video_sender/codecs/vp8/vp8_encoder.h index 82ef2c27e0..2421cf1511 100644 --- a/media/cast/video_sender/codecs/vp8/vp8_encoder.h +++ b/media/cast/video_sender/codecs/vp8/vp8_encoder.h @@ -27,7 +27,7 @@ const int kNumberOfVp8VideoBuffers = 3; class Vp8Encoder : public SoftwareVideoEncoder { public: - Vp8Encoder(const VideoSenderConfig& video_config, uint8 max_unacked_frames); + Vp8Encoder(const VideoSenderConfig& video_config, int max_unacked_frames); virtual ~Vp8Encoder(); diff --git a/media/cast/video_sender/external_video_encoder.cc b/media/cast/video_sender/external_video_encoder.cc index 1b31850c92..5978d8669a 100644 --- a/media/cast/video_sender/external_video_encoder.cc +++ b/media/cast/video_sender/external_video_encoder.cc @@ -336,8 +336,6 @@ ExternalVideoEncoder::ExternalVideoEncoder( cast_environment_(cast_environment), encoder_active_(false), key_frame_requested_(false), - skip_next_frame_(false), - skip_count_(0), weak_factory_(this) { DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); @@ -393,12 +391,6 @@ bool ExternalVideoEncoder::EncodeVideoFrame( if (!encoder_active_) return false; - if (skip_next_frame_) { - VLOG(1) << "Skip encoding frame"; - ++skip_count_; - return false; - } - encoder_task_runner_->PostTask( FROM_HERE, base::Bind(&LocalVideoEncodeAcceleratorClient::EncodeVideoFrame, @@ -421,12 +413,6 @@ void ExternalVideoEncoder::SetBitRate(int new_bit_rate) { new_bit_rate)); } -// Inform the encoder to not encode the next frame. -void ExternalVideoEncoder::SkipNextFrame(bool skip_next_frame) { - DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); - skip_next_frame_ = skip_next_frame; -} - // Inform the encoder to encode the next frame as a key frame. void ExternalVideoEncoder::GenerateKeyFrame() { DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); @@ -438,10 +424,5 @@ void ExternalVideoEncoder::LatestFrameIdToReference(uint32 /*frame_id*/) { // Do nothing not supported. } -int ExternalVideoEncoder::NumberOfSkippedFrames() const { - DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); - return skip_count_; -} - } // namespace cast } // namespace media diff --git a/media/cast/video_sender/external_video_encoder.h b/media/cast/video_sender/external_video_encoder.h index 0a7102627d..ee69b962ab 100644 --- a/media/cast/video_sender/external_video_encoder.h +++ b/media/cast/video_sender/external_video_encoder.h @@ -47,10 +47,8 @@ class ExternalVideoEncoder : public VideoEncoder { // The following functions are called from the main cast thread. virtual void SetBitRate(int new_bit_rate) OVERRIDE; - virtual void SkipNextFrame(bool skip_next_frame) OVERRIDE; virtual void GenerateKeyFrame() OVERRIDE; virtual void LatestFrameIdToReference(uint32 frame_id) OVERRIDE; - virtual int NumberOfSkippedFrames() const OVERRIDE; // Called when a VEA is created. void OnCreateVideoEncodeAccelerator( @@ -70,8 +68,6 @@ class ExternalVideoEncoder : public VideoEncoder { bool encoder_active_; bool key_frame_requested_; - bool skip_next_frame_; - int skip_count_; scoped_refptr<LocalVideoEncodeAcceleratorClient> video_accelerator_client_; scoped_refptr<base::SingleThreadTaskRunner> encoder_task_runner_; diff --git a/media/cast/video_sender/external_video_encoder_unittest.cc b/media/cast/video_sender/external_video_encoder_unittest.cc index 20c97562d1..853258ce30 100644 --- a/media/cast/video_sender/external_video_encoder_unittest.cc +++ b/media/cast/video_sender/external_video_encoder_unittest.cc @@ -164,41 +164,6 @@ TEST_F(ExternalVideoEncoderTest, EncodePattern30fpsRunningOutOfAck) { task_runner_->RunTasks(); } -TEST_F(ExternalVideoEncoderTest, SkipNextFrame) { - task_runner_->RunTasks(); // Run the initializer on the correct thread. - - VideoEncoder::FrameEncodedCallback frame_encoded_callback = - base::Bind(&TestVideoEncoderCallback::DeliverEncodedVideoFrame, - test_video_encoder_callback_.get()); - - base::TimeTicks capture_time; - capture_time += base::TimeDelta::FromMilliseconds(33); - test_video_encoder_callback_->SetExpectedResult(0, 0, capture_time); - EXPECT_TRUE(video_encoder_->EncodeVideoFrame( - video_frame_, capture_time, frame_encoded_callback)); - task_runner_->RunTasks(); - - video_encoder_->SkipNextFrame(true); - for (int i = 0; i < 2; ++i) { - capture_time += base::TimeDelta::FromMilliseconds(33); - EXPECT_FALSE(video_encoder_->EncodeVideoFrame( - video_frame_, capture_time, frame_encoded_callback)); - task_runner_->RunTasks(); - } - - video_encoder_->SkipNextFrame(false); - for (int i = 0; i < 2; ++i) { - capture_time += base::TimeDelta::FromMilliseconds(33); - test_video_encoder_callback_->SetExpectedResult(i + 1, i, capture_time); - EXPECT_TRUE(video_encoder_->EncodeVideoFrame( - video_frame_, capture_time, frame_encoded_callback)); - task_runner_->RunTasks(); - } - // We need to run the task to cleanup the GPU instance. - video_encoder_.reset(NULL); - task_runner_->RunTasks(); -} - TEST_F(ExternalVideoEncoderTest, StreamHeader) { task_runner_->RunTasks(); // Run the initializer on the correct thread. diff --git a/media/cast/video_sender/video_encoder.h b/media/cast/video_sender/video_encoder.h index 48d63ab948..c7b1049ce6 100644 --- a/media/cast/video_sender/video_encoder.h +++ b/media/cast/video_sender/video_encoder.h @@ -38,18 +38,11 @@ class VideoEncoder { // Inform the encoder about the new target bit rate. virtual void SetBitRate(int new_bit_rate) = 0; - // Inform the encoder to not encode the next frame. - // Note: this setting is sticky and should last until called with false. - virtual void SkipNextFrame(bool skip_next_frame) = 0; - // Inform the encoder to encode the next frame as a key frame. virtual void GenerateKeyFrame() = 0; // Inform the encoder to only reference frames older or equal to frame_id; virtual void LatestFrameIdToReference(uint32 frame_id) = 0; - - // Query the codec about how many frames it has skipped due to slow ACK. - virtual int NumberOfSkippedFrames() const = 0; }; } // namespace cast diff --git a/media/cast/video_sender/video_encoder_impl.cc b/media/cast/video_sender/video_encoder_impl.cc index 039813b129..b90ef0f07e 100644 --- a/media/cast/video_sender/video_encoder_impl.cc +++ b/media/cast/video_sender/video_encoder_impl.cc @@ -67,11 +67,9 @@ void EncodeVideoFrameOnEncoderThread( VideoEncoderImpl::VideoEncoderImpl( scoped_refptr<CastEnvironment> cast_environment, const VideoSenderConfig& video_config, - uint8 max_unacked_frames) + int max_unacked_frames) : video_config_(video_config), - cast_environment_(cast_environment), - skip_next_frame_(false), - skip_count_(0) { + cast_environment_(cast_environment) { if (video_config.codec == transport::kVp8) { encoder_.reset(new Vp8Encoder(video_config, max_unacked_frames)); cast_environment_->PostTask(CastEnvironment::VIDEO, @@ -108,11 +106,6 @@ bool VideoEncoderImpl::EncodeVideoFrame( const base::TimeTicks& capture_time, const FrameEncodedCallback& frame_encoded_callback) { DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); - if (skip_next_frame_) { - ++skip_count_; - return false; - } - cast_environment_->PostTask(CastEnvironment::VIDEO, FROM_HERE, base::Bind(&EncodeVideoFrameOnEncoderThread, @@ -132,11 +125,6 @@ void VideoEncoderImpl::SetBitRate(int new_bit_rate) { dynamic_config_.bit_rate = new_bit_rate; } -// Inform the encoder to not encode the next frame. -void VideoEncoderImpl::SkipNextFrame(bool skip_next_frame) { - skip_next_frame_ = skip_next_frame; -} - // Inform the encoder to encode the next frame as a key frame. void VideoEncoderImpl::GenerateKeyFrame() { dynamic_config_.key_frame_requested = true; @@ -147,7 +135,5 @@ void VideoEncoderImpl::LatestFrameIdToReference(uint32 frame_id) { dynamic_config_.latest_frame_id_to_reference = frame_id; } -int VideoEncoderImpl::NumberOfSkippedFrames() const { return skip_count_; } - } // namespace cast } // namespace media diff --git a/media/cast/video_sender/video_encoder_impl.h b/media/cast/video_sender/video_encoder_impl.h index 47265c1f0b..b34b440c93 100644 --- a/media/cast/video_sender/video_encoder_impl.h +++ b/media/cast/video_sender/video_encoder_impl.h @@ -31,7 +31,7 @@ class VideoEncoderImpl : public VideoEncoder { VideoEncoderImpl(scoped_refptr<CastEnvironment> cast_environment, const VideoSenderConfig& video_config, - uint8 max_unacked_frames); + int max_unacked_frames); virtual ~VideoEncoderImpl(); @@ -49,17 +49,13 @@ class VideoEncoderImpl : public VideoEncoder { // The following functions are called from the main cast thread. virtual void SetBitRate(int new_bit_rate) OVERRIDE; - virtual void SkipNextFrame(bool skip_next_frame) OVERRIDE; virtual void GenerateKeyFrame() OVERRIDE; virtual void LatestFrameIdToReference(uint32 frame_id) OVERRIDE; - virtual int NumberOfSkippedFrames() const OVERRIDE; private: const VideoSenderConfig video_config_; scoped_refptr<CastEnvironment> cast_environment_; CodecDynamicConfig dynamic_config_; - bool skip_next_frame_; - int skip_count_; // This member belongs to the video encoder thread. It must not be // dereferenced on the main thread. We manage the lifetime of this member diff --git a/media/cast/video_sender/video_encoder_impl_unittest.cc b/media/cast/video_sender/video_encoder_impl_unittest.cc index b1a5cb8cef..a60812304f 100644 --- a/media/cast/video_sender/video_encoder_impl_unittest.cc +++ b/media/cast/video_sender/video_encoder_impl_unittest.cc @@ -103,7 +103,7 @@ class VideoEncoderImplTest : public ::testing::Test { task_runner_->RunTasks(); } - void Configure(uint8 max_unacked_frames) { + void Configure(int max_unacked_frames) { video_encoder_.reset(new VideoEncoderImpl( cast_environment_, video_config_, max_unacked_frames)); } diff --git a/media/cast/video_sender/video_sender.cc b/media/cast/video_sender/video_sender.cc index 57b8ea7299..928360b6cc 100644 --- a/media/cast/video_sender/video_sender.cc +++ b/media/cast/video_sender/video_sender.cc @@ -4,8 +4,8 @@ #include "media/cast/video_sender/video_sender.h" +#include <algorithm> #include <cstring> -#include <list> #include "base/bind.h" #include "base/debug/trace_event.h" @@ -28,34 +28,43 @@ VideoSender::VideoSender( const VideoSenderConfig& video_config, const CreateVideoEncodeAcceleratorCallback& create_vea_cb, const CreateVideoEncodeMemoryCallback& create_video_encode_mem_cb, - const CastInitializationCallback& cast_initialization_cb, transport::CastTransportSender* const transport_sender) - : rtp_max_delay_(base::TimeDelta::FromMilliseconds( + : cast_environment_(cast_environment), + target_playout_delay_(base::TimeDelta::FromMilliseconds( video_config.rtp_config.max_delay_ms)), - max_frame_rate_(video_config.max_frame_rate), - cast_environment_(cast_environment), transport_sender_(transport_sender), + max_unacked_frames_(std::min( + kMaxUnackedFrames, + 1 + static_cast<int>( + target_playout_delay_ * video_config.max_frame_rate / + base::TimeDelta::FromSeconds(1)))), + rtcp_(cast_environment_, + this, + transport_sender_, + NULL, // paced sender. + NULL, + video_config.rtcp_mode, + base::TimeDelta::FromMilliseconds(video_config.rtcp_interval), + video_config.rtp_config.ssrc, + video_config.incoming_feedback_ssrc, + video_config.rtcp_c_name, + VIDEO_EVENT), rtp_timestamp_helper_(kVideoFrequency), num_aggressive_rtcp_reports_sent_(0), - last_acked_frame_id_(-1), - last_sent_frame_id_(-1), frames_in_encoder_(0), - duplicate_ack_(0), - last_skip_count_(0), + last_sent_frame_id_(0), + latest_acked_frame_id_(0), + duplicate_ack_counter_(0), current_requested_bitrate_(video_config.start_bitrate), congestion_control_(cast_environment->Clock(), video_config.congestion_control_back_off, video_config.max_bitrate, video_config.min_bitrate, video_config.start_bitrate), - initialized_(false), - active_session_(false), + cast_initialization_status_(STATUS_VIDEO_UNINITIALIZED), weak_factory_(this) { - max_unacked_frames_ = - 1 + static_cast<uint8>(video_config.rtp_config.max_delay_ms * - max_frame_rate_ / 1000); - VLOG(1) << "max_unacked_frames " << static_cast<int>(max_unacked_frames_); - DCHECK_GT(max_unacked_frames_, 0) << "Invalid argument"; + VLOG(1) << "max_unacked_frames " << max_unacked_frames_; + DCHECK_GT(max_unacked_frames_, 0); if (video_config.use_external_encoder) { video_encoder_.reset(new ExternalVideoEncoder(cast_environment, @@ -66,7 +75,7 @@ VideoSender::VideoSender( video_encoder_.reset(new VideoEncoderImpl( cast_environment, video_config, max_unacked_frames_)); } - + cast_initialization_status_ = STATUS_VIDEO_INITIALIZED; media::cast::transport::CastTransportVideoConfig transport_config; transport_config.codec = video_config.codec; @@ -74,26 +83,7 @@ VideoSender::VideoSender( transport_config.rtp.max_outstanding_frames = max_unacked_frames_ + 1; transport_sender_->InitializeVideo(transport_config); - rtcp_.reset( - new Rtcp(cast_environment_, - this, - transport_sender_, - NULL, // paced sender. - NULL, - video_config.rtcp_mode, - base::TimeDelta::FromMilliseconds(video_config.rtcp_interval), - video_config.rtp_config.ssrc, - video_config.incoming_feedback_ssrc, - video_config.rtcp_c_name, - false)); - rtcp_->SetCastReceiverEventHistorySize(kReceiverRtcpEventHistorySize); - - // TODO(pwestin): pass cast_initialization_cb to |video_encoder_| - // and remove this call. - cast_environment_->PostTask( - CastEnvironment::MAIN, - FROM_HERE, - base::Bind(cast_initialization_cb, STATUS_VIDEO_INITIALIZED)); + rtcp_.SetCastReceiverEventHistorySize(kReceiverRtcpEventHistorySize); memset(frame_id_to_rtp_timestamp_, 0, sizeof(frame_id_to_rtp_timestamp_)); } @@ -101,19 +91,14 @@ VideoSender::VideoSender( VideoSender::~VideoSender() { } -void VideoSender::InitializeTimers() { - DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); - if (!initialized_) { - initialized_ = true; - ScheduleNextResendCheck(); - ScheduleNextSkippedFramesCheck(); - } -} - void VideoSender::InsertRawVideoFrame( const scoped_refptr<media::VideoFrame>& video_frame, const base::TimeTicks& capture_time) { DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); + if (cast_initialization_status_ != STATUS_VIDEO_INITIALIZED) { + NOTREACHED(); + return; + } DCHECK(video_encoder_.get()) << "Invalid state"; RtpTimestamp rtp_timestamp = GetVideoRtpTimestamp(capture_time); @@ -133,33 +118,54 @@ void VideoSender::InsertRawVideoFrame( "timestamp", capture_time.ToInternalValue(), "rtp_timestamp", rtp_timestamp); + if (AreTooManyFramesInFlight()) { + VLOG(1) << "Dropping frame due to too many frames currently in-flight."; + return; + } + if (video_encoder_->EncodeVideoFrame( video_frame, capture_time, - base::Bind(&VideoSender::SendEncodedVideoFrameMainThread, + base::Bind(&VideoSender::SendEncodedVideoFrame, weak_factory_.GetWeakPtr(), current_requested_bitrate_))) { frames_in_encoder_++; - UpdateFramesInFlight(); + } else { + VLOG(1) << "Encoder rejected a frame. Skipping..."; } } -void VideoSender::SendEncodedVideoFrameMainThread( +void VideoSender::SendEncodedVideoFrame( int requested_bitrate_before_encode, scoped_ptr<transport::EncodedFrame> encoded_frame) { DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); - last_send_time_ = cast_environment_->Clock()->NowTicks(); - VLOG_IF(1, encoded_frame->dependency == transport::EncodedFrame::KEY) - << "Send encoded key frame; frame_id: " << encoded_frame->frame_id; DCHECK_GT(frames_in_encoder_, 0); frames_in_encoder_--; - uint32 frame_id = encoded_frame->frame_id; + + const uint32 frame_id = encoded_frame->frame_id; + + const bool is_first_frame_to_be_sent = last_send_time_.is_null(); + last_send_time_ = cast_environment_->Clock()->NowTicks(); + last_sent_frame_id_ = frame_id; + // If this is the first frame about to be sent, fake the value of + // |latest_acked_frame_id_| to indicate the receiver starts out all caught up. + // Also, schedule the periodic frame re-send checks. + if (is_first_frame_to_be_sent) { + latest_acked_frame_id_ = frame_id - 1; + ScheduleNextResendCheck(); + } + + VLOG_IF(1, encoded_frame->dependency == transport::EncodedFrame::KEY) + << "Send encoded key frame; frame_id: " << frame_id; + cast_environment_->Logging()->InsertEncodedFrameEvent( last_send_time_, FRAME_ENCODED, VIDEO_EVENT, encoded_frame->rtp_timestamp, frame_id, static_cast<int>(encoded_frame->data.size()), encoded_frame->dependency == transport::EncodedFrame::KEY, requested_bitrate_before_encode); + // Only use lowest 8 bits as key. + frame_id_to_rtp_timestamp_[frame_id & 0xff] = encoded_frame->rtp_timestamp; // Used by chrome/browser/extension/api/cast_streaming/performance_test.cc TRACE_EVENT_INSTANT1( @@ -167,10 +173,6 @@ void VideoSender::SendEncodedVideoFrameMainThread( TRACE_EVENT_SCOPE_THREAD, "rtp_timestamp", encoded_frame->rtp_timestamp); - // Only use lowest 8 bits as key. - frame_id_to_rtp_timestamp_[frame_id & 0xff] = encoded_frame->rtp_timestamp; - - last_sent_frame_id_ = static_cast<int>(encoded_frame->frame_id); DCHECK(!encoded_frame->reference_time.is_null()); rtp_timestamp_helper_.StoreLatestTime(encoded_frame->reference_time, encoded_frame->rtp_timestamp); @@ -191,18 +193,16 @@ void VideoSender::SendEncodedVideoFrameMainThread( } transport_sender_->InsertCodedVideoFrame(*encoded_frame); - UpdateFramesInFlight(); - InitializeTimers(); } void VideoSender::IncomingRtcpPacket(scoped_ptr<Packet> packet) { DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); - rtcp_->IncomingRtcpPacket(&packet->front(), packet->size()); + rtcp_.IncomingRtcpPacket(&packet->front(), packet->size()); } void VideoSender::ScheduleNextRtcpReport() { DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); - base::TimeDelta time_to_next = rtcp_->TimeToSendNextRtcpReport() - + base::TimeDelta time_to_next = rtcp_.TimeToSendNextRtcpReport() - cast_environment_->Clock()->NowTicks(); time_to_next = std::max( @@ -223,7 +223,7 @@ void VideoSender::SendRtcpReport(bool schedule_future_reports) { uint32 now_as_rtp_timestamp = 0; if (rtp_timestamp_helper_.GetCurrentTimeAsRtpTimestamp( now, &now_as_rtp_timestamp)) { - rtcp_->SendRtcpFromRtpSender(now, now_as_rtp_timestamp); + rtcp_.SendRtcpFromRtpSender(now, now_as_rtp_timestamp); } else { // |rtp_timestamp_helper_| should have stored a mapping by this point. NOTREACHED(); @@ -234,16 +234,12 @@ void VideoSender::SendRtcpReport(bool schedule_future_reports) { void VideoSender::ScheduleNextResendCheck() { DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); - base::TimeDelta time_to_next; - if (last_send_time_.is_null()) { - time_to_next = rtp_max_delay_; - } else { - time_to_next = last_send_time_ - cast_environment_->Clock()->NowTicks() + - rtp_max_delay_; - } + DCHECK(!last_send_time_.is_null()); + base::TimeDelta time_to_next = + last_send_time_ - cast_environment_->Clock()->NowTicks() + + target_playout_delay_; time_to_next = std::max( time_to_next, base::TimeDelta::FromMilliseconds(kMinSchedulingDelayMs)); - cast_environment_->PostDelayedTask( CastEnvironment::MAIN, FROM_HERE, @@ -253,71 +249,29 @@ void VideoSender::ScheduleNextResendCheck() { void VideoSender::ResendCheck() { DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); - if (!last_send_time_.is_null() && last_sent_frame_id_ != -1) { - base::TimeDelta time_since_last_send = - cast_environment_->Clock()->NowTicks() - last_send_time_; - if (time_since_last_send > rtp_max_delay_) { - if (!active_session_) { - // We have not received any acks, resend the first encoded frame (id 0), - // which must also be a key frame. - VLOG(1) << "ACK timeout resend first key frame"; - ResendFrame(0); - } else { - if (last_acked_frame_id_ == last_sent_frame_id_) { - // Last frame acked, no point in doing anything - } else { - DCHECK_LE(0, last_acked_frame_id_); - uint32 frame_id = static_cast<uint32>(last_acked_frame_id_ + 1); - VLOG(1) << "ACK timeout resend frame:" << static_cast<int>(frame_id); - ResendFrame(frame_id); - } - } + DCHECK(!last_send_time_.is_null()); + const base::TimeDelta time_since_last_send = + cast_environment_->Clock()->NowTicks() - last_send_time_; + if (time_since_last_send > target_playout_delay_) { + if (latest_acked_frame_id_ == last_sent_frame_id_) { + // Last frame acked, no point in doing anything + } else { + const uint32 kickstart_frame_id = latest_acked_frame_id_ + 1; + VLOG(1) << "ACK timeout, re-sending frame " << kickstart_frame_id; + ResendFrame(kickstart_frame_id); } } ScheduleNextResendCheck(); } -void VideoSender::ScheduleNextSkippedFramesCheck() { - DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); - base::TimeDelta time_to_next; - if (last_checked_skip_count_time_.is_null()) { - time_to_next = - base::TimeDelta::FromMilliseconds(kSkippedFramesCheckPeriodkMs); - } else { - time_to_next = - last_checked_skip_count_time_ - cast_environment_->Clock()->NowTicks() + - base::TimeDelta::FromMilliseconds(kSkippedFramesCheckPeriodkMs); - } - time_to_next = std::max( - time_to_next, base::TimeDelta::FromMilliseconds(kMinSchedulingDelayMs)); - - cast_environment_->PostDelayedTask( - CastEnvironment::MAIN, - FROM_HERE, - base::Bind(&VideoSender::SkippedFramesCheck, weak_factory_.GetWeakPtr()), - time_to_next); -} - -void VideoSender::SkippedFramesCheck() { - DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); - int skip_count = video_encoder_->NumberOfSkippedFrames(); - if (skip_count - last_skip_count_ > - kSkippedFramesThreshold * max_frame_rate_) { - // TODO(pwestin): Propagate this up to the application. - } - last_skip_count_ = skip_count; - last_checked_skip_count_time_ = cast_environment_->Clock()->NowTicks(); - ScheduleNextSkippedFramesCheck(); -} - void VideoSender::OnReceivedCastFeedback(const RtcpCastMessage& cast_feedback) { DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); + base::TimeDelta rtt; base::TimeDelta avg_rtt; base::TimeDelta min_rtt; base::TimeDelta max_rtt; - - if (rtcp_->Rtt(&rtt, &avg_rtt, &min_rtt, &max_rtt)) { + if (rtcp_.Rtt(&rtt, &avg_rtt, &min_rtt, &max_rtt)) { // Don't use a RTT lower than our average. rtt = std::max(rtt, avg_rtt); @@ -334,38 +288,33 @@ void VideoSender::OnReceivedCastFeedback(const RtcpCastMessage& cast_feedback) { // We have no measured value use default. rtt = base::TimeDelta::FromMilliseconds(kStartRttMs); } - if (cast_feedback.missing_frames_and_packets_.empty()) { - // No lost packets. - int resend_frame = -1; - if (last_sent_frame_id_ == -1) - return; + if (last_send_time_.is_null()) + return; // Cannot get an ACK without having first sent a frame. + + if (cast_feedback.missing_frames_and_packets_.empty()) { video_encoder_->LatestFrameIdToReference(cast_feedback.ack_frame_id_); - if (static_cast<uint32>(last_acked_frame_id_ + 1) == - cast_feedback.ack_frame_id_) { + if ((latest_acked_frame_id_ + 1) == cast_feedback.ack_frame_id_) { uint32 new_bitrate = 0; if (congestion_control_.OnAck(rtt, &new_bitrate)) { UpdateBitrate(new_bitrate); } } // We only count duplicate ACKs when we have sent newer frames. - if (static_cast<uint32>(last_acked_frame_id_) == - cast_feedback.ack_frame_id_ && - IsNewerFrameId(last_sent_frame_id_, last_acked_frame_id_)) { - duplicate_ack_++; + if (latest_acked_frame_id_ == cast_feedback.ack_frame_id_ && + latest_acked_frame_id_ != last_sent_frame_id_) { + duplicate_ack_counter_++; } else { - duplicate_ack_ = 0; + duplicate_ack_counter_ = 0; } - if (duplicate_ack_ >= 2 && duplicate_ack_ % 3 == 2) { + // TODO(miu): The values "2" and "3" should be derived from configuration. + if (duplicate_ack_counter_ >= 2 && duplicate_ack_counter_ % 3 == 2) { // Resend last ACK + 1 frame. - resend_frame = static_cast<uint32>(last_acked_frame_id_ + 1); - } - if (resend_frame != -1) { - DCHECK_LE(0, resend_frame); - VLOG(1) << "Received duplicate ACK for frame:" - << static_cast<int>(resend_frame); - ResendFrame(static_cast<uint32>(resend_frame)); + const uint32 frame_to_resend = latest_acked_frame_id_ + 1; + VLOG(1) << "Received duplicate ACK for frame " << latest_acked_frame_id_ + << ", will re-send frame " << frame_to_resend; + ResendFrame(frame_to_resend); } } else { transport_sender_->ResendPackets( @@ -375,57 +324,39 @@ void VideoSender::OnReceivedCastFeedback(const RtcpCastMessage& cast_feedback) { UpdateBitrate(new_bitrate); } } - ReceivedAck(cast_feedback.ack_frame_id_); -} - -void VideoSender::ReceivedAck(uint32 acked_frame_id) { - DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); - if (acked_frame_id == UINT32_C(0xFFFFFFFF)) { - // Receiver is sending a status message before any frames are ready to - // be acked. Ignore. - return; - } - last_acked_frame_id_ = static_cast<int>(acked_frame_id); - base::TimeTicks now = cast_environment_->Clock()->NowTicks(); RtpTimestamp rtp_timestamp = - frame_id_to_rtp_timestamp_[acked_frame_id & 0xff]; + frame_id_to_rtp_timestamp_[cast_feedback.ack_frame_id_ & 0xff]; cast_environment_->Logging()->InsertFrameEvent( - now, FRAME_ACK_RECEIVED, VIDEO_EVENT, rtp_timestamp, acked_frame_id); - - VLOG(2) << "ReceivedAck:" << static_cast<int>(acked_frame_id); - active_session_ = true; - DCHECK_NE(-1, last_acked_frame_id_); - UpdateFramesInFlight(); + cast_environment_->Clock()->NowTicks(), FRAME_ACK_RECEIVED, VIDEO_EVENT, + rtp_timestamp, cast_feedback.ack_frame_id_); + + const bool is_acked_out_of_order = + static_cast<int32>(cast_feedback.ack_frame_id_ - + latest_acked_frame_id_) < 0; + VLOG(2) << "Received ACK" << (is_acked_out_of_order ? " out-of-order" : "") + << " for frame " << cast_feedback.ack_frame_id_; + if (!is_acked_out_of_order) + latest_acked_frame_id_ = cast_feedback.ack_frame_id_; } -void VideoSender::UpdateFramesInFlight() { +bool VideoSender::AreTooManyFramesInFlight() const { DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); - if (last_sent_frame_id_ != -1) { - DCHECK_LE(0, last_sent_frame_id_); - int frames_in_flight = 0; - if (last_acked_frame_id_ != -1) { - DCHECK_LE(0, last_acked_frame_id_); - frames_in_flight = last_sent_frame_id_ - last_acked_frame_id_; - } else { - frames_in_flight = last_sent_frame_id_ + 1; - } - frames_in_flight += frames_in_encoder_; - VLOG(2) << frames_in_flight - << " Frames in flight; last sent: " << last_sent_frame_id_ - << " last acked:" << last_acked_frame_id_ - << " frames in encoder: " << frames_in_encoder_; - if (frames_in_flight >= max_unacked_frames_) { - video_encoder_->SkipNextFrame(true); - return; - } - DCHECK(frames_in_flight <= max_unacked_frames_); + int frames_in_flight = frames_in_encoder_; + if (!last_send_time_.is_null()) { + frames_in_flight += + static_cast<int32>(last_sent_frame_id_ - latest_acked_frame_id_); } - video_encoder_->SkipNextFrame(false); + VLOG(2) << frames_in_flight + << " frames in flight; last sent: " << last_sent_frame_id_ + << " latest acked: " << latest_acked_frame_id_ + << " frames in encoder: " << frames_in_encoder_; + return frames_in_flight >= max_unacked_frames_; } void VideoSender::ResendFrame(uint32 resend_frame_id) { DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); + DCHECK(!last_send_time_.is_null()); MissingFramesAndPacketsMap missing_frames_and_packets; PacketIdSet missing; missing_frames_and_packets.insert(std::make_pair(resend_frame_id, missing)); diff --git a/media/cast/video_sender/video_sender.h b/media/cast/video_sender/video_sender.h index 59792f7693..089680283b 100644 --- a/media/cast/video_sender/video_sender.h +++ b/media/cast/video_sender/video_sender.h @@ -20,9 +20,11 @@ #include "media/cast/rtp_timestamp_helper.h" namespace media { + class VideoFrame; namespace cast { + class LocalVideoEncoderCallback; class VideoEncoder; @@ -44,15 +46,20 @@ class VideoSender : public RtcpSenderFeedback, const VideoSenderConfig& video_config, const CreateVideoEncodeAcceleratorCallback& create_vea_cb, const CreateVideoEncodeMemoryCallback& create_video_encode_mem_cb, - const CastInitializationCallback& cast_initialization_cb, transport::CastTransportSender* const transport_sender); virtual ~VideoSender(); - // The video_frame must be valid until the closure callback is called. - // The closure callback is called from the video encoder thread as soon as - // the encoder is done with the frame; it does not mean that the encoded frame - // has been sent out. + CastInitializationStatus InitializationResult() const { + return cast_initialization_status_; + } + + // Note: It is not guaranteed that |video_frame| will actually be encoded and + // sent, if VideoSender detects too many frames in flight. Therefore, clients + // should be careful about the rate at which this method is called. + // + // Note: It is invalid to call this method if InitializationResult() returns + // anything but STATUS_VIDEO_INITIALIZED. void InsertRawVideoFrame(const scoped_refptr<media::VideoFrame>& video_frame, const base::TimeTicks& capture_time); @@ -65,63 +72,107 @@ class VideoSender : public RtcpSenderFeedback, OVERRIDE; private: - friend class LocalRtcpVideoSenderFeedback; - - // Schedule when we should send the next RTPC report, - // via a PostDelayedTask to the main cast thread. + // Schedule and execute periodic sending of RTCP report. void ScheduleNextRtcpReport(); void SendRtcpReport(bool schedule_future_reports); - // Schedule when we should check that we have received an acknowledgment, or a - // loss report from our remote peer. If we have not heard back from our remote - // peer we speculatively resend our oldest unacknowledged frame (the whole - // frame). Note for this to happen we need to lose all pending packets (in - // normal operation 3 full frames), hence this is the last resort to prevent - // us getting stuck after a long outage. + // Schedule and execute periodic checks for re-sending frames. If no + // acknowledgements have been received for "too long," VideoSender will + // speculatively re-send the frame just after |latest_acked_frame_id_| (the + // whole frame). This is a last resort tactic to prevent the session from + // getting stuck after a long outage. void ScheduleNextResendCheck(); void ResendCheck(); - // Monitor how many frames that are silently dropped by the video sender - // per time unit. - void ScheduleNextSkippedFramesCheck(); - void SkippedFramesCheck(); - + // Asks |transport_sender_| to resend all the packets for a particular frame. void ResendFrame(uint32 resend_frame_id); - void ReceivedAck(uint32 acked_frame_id); - void UpdateFramesInFlight(); - void SendEncodedVideoFrameMainThread( - int requested_bitrate_before_encode, - scoped_ptr<transport::EncodedFrame> encoded_frame); + // Returns true if there are too many frames in flight, as defined by the + // configured target playout delay plus simple logic. When this is true, + // InsertRawVideoFrame() will silenty drop frames instead of sending them to + // the video encoder. + bool AreTooManyFramesInFlight() const; - void InitializeTimers(); + // Called by the |video_encoder_| with the next EncodeFrame to send. + void SendEncodedVideoFrame(int requested_bitrate_before_encode, + scoped_ptr<transport::EncodedFrame> encoded_frame); void UpdateBitrate(int32 new_bitrate); - base::TimeDelta rtp_max_delay_; - const int max_frame_rate_; - - scoped_refptr<CastEnvironment> cast_environment_; + const scoped_refptr<CastEnvironment> cast_environment_; + + // The total amount of time between a frame's capture/recording on the sender + // and its playback on the receiver (i.e., shown to a user). This is fixed as + // a value large enough to give the system sufficient time to encode, + // transmit/retransmit, receive, decode, and render; given its run-time + // environment (sender/receiver hardware performance, network conditions, + // etc.). + const base::TimeDelta target_playout_delay_; + + // Sends encoded frames over the configured transport (e.g., UDP). In + // Chromium, this could be a proxy that first sends the frames from a renderer + // process to the browser process over IPC, with the browser process being + // responsible for "packetizing" the frames and pushing packets into the + // network layer. transport::CastTransportSender* const transport_sender_; - RtpTimestampHelper rtp_timestamp_helper_; + // Maximum number of outstanding frames before the encoding and sending of + // new frames shall halt. + const int max_unacked_frames_; + + // Encodes media::VideoFrame images into EncodedFrames. Per configuration, + // this will point to either the internal software-based encoder or a proxy to + // a hardware-based encoder. scoped_ptr<VideoEncoder> video_encoder_; - scoped_ptr<Rtcp> rtcp_; + + // Manages sending/receiving of RTCP packets, including sender/receiver + // reports. + Rtcp rtcp_; + + // Records lip-sync (i.e., mapping of RTP <--> NTP timestamps), and + // extrapolates this mapping to any other point in time. + RtpTimestampHelper rtp_timestamp_helper_; + + // Counts how many RTCP reports are being "aggressively" sent (i.e., one per + // frame) at the start of the session. Once a threshold is reached, RTCP + // reports are instead sent at the configured interval + random drift. int num_aggressive_rtcp_reports_sent_; - uint8 max_unacked_frames_; - int last_acked_frame_id_; - int last_sent_frame_id_; + + // The number of frames currently being processed in |video_encoder_|. int frames_in_encoder_; - int duplicate_ack_; + + // This is "null" until the first frame is sent. Thereafter, this tracks the + // last time any frame was sent or re-sent. base::TimeTicks last_send_time_; - base::TimeTicks last_checked_skip_count_time_; - int last_skip_count_; + + // The ID of the last frame sent. Logic throughout VideoSender assumes this + // can safely wrap-around. This member is invalid until + // |!last_send_time_.is_null()|. + uint32 last_sent_frame_id_; + + // The ID of the latest (not necessarily the last) frame that has been + // acknowledged. Logic throughout VideoSender assumes this can safely + // wrap-around. This member is invalid until |!last_send_time_.is_null()|. + uint32 latest_acked_frame_id_; + + // Counts the number of duplicate ACK that are being received. When this + // number reaches a threshold, the sender will take this as a sign that the + // receiver hasn't yet received the first packet of the next frame. In this + // case, VideoSender will trigger a re-send of the next frame. + int duplicate_ack_counter_; + + // Desired encoder bitrate (in bits per second). This is updated by querying + // |congestion_control_| as each ACK is received. int current_requested_bitrate_; + // When we get close to the max number of un-acked frames, we set lower // the bitrate drastically to ensure that we catch up. Without this we // risk getting stuck in a catch-up state forever. CongestionControl congestion_control_; + // If this sender is ready for use, this is STATUS_VIDEO_INITIALIZED. + CastInitializationStatus cast_initialization_status_; + // This is a "good enough" mapping for finding the RTP timestamp associated // with a video frame. The key is the lowest 8 bits of frame id (which is // what is sent via RTCP). This map is used for logging purposes. The only @@ -130,10 +181,6 @@ class VideoSender : public RtcpSenderFeedback, // should be pretty rare. RtpTimestamp frame_id_to_rtp_timestamp_[256]; - bool initialized_; - // Indicator for receiver acknowledgments. - bool active_session_; - // NOTE: Weak pointers must be invalidated before all other member variables. base::WeakPtrFactory<VideoSender> weak_factory_; diff --git a/media/cast/video_sender/video_sender_unittest.cc b/media/cast/video_sender/video_sender_unittest.cc index a7987a2a0c..faa3180a8c 100644 --- a/media/cast/video_sender/video_sender_unittest.cc +++ b/media/cast/video_sender/video_sender_unittest.cc @@ -90,13 +90,11 @@ class PeerVideoSender : public VideoSender { const VideoSenderConfig& video_config, const CreateVideoEncodeAcceleratorCallback& create_vea_cb, const CreateVideoEncodeMemoryCallback& create_video_encode_mem_cb, - const CastInitializationCallback& cast_initialization_cb, transport::CastTransportSender* const transport_sender) : VideoSender(cast_environment, video_config, create_vea_cb, create_video_encode_mem_cb, - cast_initialization_cb, transport_sender) {} using VideoSender::OnReceivedCastFeedback; }; @@ -164,8 +162,6 @@ class VideoSenderTest : public ::testing::Test { task_runner_, base::Passed(&fake_vea)), base::Bind(&CreateSharedMemory), - base::Bind(&VideoSenderTest::InitializationResult, - base::Unretained(this)), transport_sender_.get())); } else { video_sender_.reset( @@ -173,10 +169,9 @@ class VideoSenderTest : public ::testing::Test { video_config, CreateDefaultVideoEncodeAcceleratorCallback(), CreateDefaultVideoEncodeMemoryCallback(), - base::Bind(&VideoSenderTest::InitializationResult, - base::Unretained(this)), transport_sender_.get())); } + ASSERT_EQ(STATUS_VIDEO_INITIALIZED, video_sender_->InitializationResult()); } scoped_refptr<media::VideoFrame> GetNewVideoFrame() { @@ -196,10 +191,6 @@ class VideoSenderTest : public ::testing::Test { } } - void InitializationResult(CastInitializationStatus result) { - EXPECT_EQ(STATUS_VIDEO_INITIALIZED, result); - } - base::SimpleTestTickClock* testing_clock_; // Owned by CastEnvironment. TestPacketSender transport_; scoped_ptr<transport::CastTransportSenderImpl> transport_sender_; @@ -322,7 +313,7 @@ TEST_F(VideoSenderTest, LogAckReceivedEvent) { cast_environment_->Logging()->RemoveRawEventSubscriber(&event_subscriber); } -TEST_F(VideoSenderTest, StopSendingIntheAbsenceOfAck) { +TEST_F(VideoSenderTest, StopSendingInTheAbsenceOfAck) { InitEncoder(false); // Send a stream of frames and don't ACK; by default we shouldn't have more // than 4 frames in flight. |