From 6f8ee344b38f3e9e801deaa3d8acfb3119704fe9 Mon Sep 17 00:00:00 2001 From: Jordan Bayles Date: Mon, 26 Apr 2021 13:04:29 -0700 Subject: [Cast] Add capabilities handling to ReceiverSession This patch is the first in a series for finishing support for remoting in the ReceiverSession. In this patch, the public API for the session adds explicit remoting opt-in, and the session can now respond to capabilities. Bug: b/184186390 Change-Id: Ic1abcd32d240d9c64dec4b56fb01d6b3c74e9ddb Reviewed-on: https://chromium-review.googlesource.com/c/openscreen/+/2815610 Reviewed-by: mark a. foltz Reviewed-by: Ryan Keane Commit-Queue: Jordan Bayles --- cast/streaming/constants.h | 5 ++ cast/streaming/receiver_message.cc | 12 +++- cast/streaming/receiver_message.h | 2 + cast/streaming/receiver_session.cc | 91 ++++++++++++++++++++++++++++- cast/streaming/receiver_session.h | 81 ++++++++++++++++++++++--- cast/streaming/receiver_session_unittest.cc | 83 ++++++++++++++++++++++++-- cast/streaming/sender_session.cc | 2 - 7 files changed, 258 insertions(+), 18 deletions(-) (limited to 'cast') diff --git a/cast/streaming/constants.h b/cast/streaming/constants.h index bfcc2365..c640796d 100644 --- a/cast/streaming/constants.h +++ b/cast/streaming/constants.h @@ -96,6 +96,11 @@ constexpr int kDefaultAudioChannels = 2; // to provide any constraints. constexpr std::chrono::milliseconds kDefaultMaxDelayMs(1500); +// TODO(issuetracker.google.com/184189100): As part of updating remoting +// OFFER/ANSWER and capabilities exchange, remoting version should be updated +// to 3. +constexpr int kSupportedRemotingVersion = 2; + // Codecs known and understood by cast senders and receivers. Note: receivers // are required to implement the following codecs to be Cast V2 compliant: H264, // VP8, AAC, Opus. Senders have to implement at least one codec from this diff --git a/cast/streaming/receiver_message.cc b/cast/streaming/receiver_message.cc index 5f433362..ec8f9619 100644 --- a/cast/streaming/receiver_message.cc +++ b/cast/streaming/receiver_message.cc @@ -7,6 +7,7 @@ #include #include "absl/strings/ascii.h" +#include "absl/types/optional.h" #include "cast/streaming/message_fields.h" #include "json/reader.h" #include "json/writer.h" @@ -209,9 +210,14 @@ ErrorOr ReceiverMessage::ToJson() const { break; case ReceiverMessage::Type::kCapabilitiesResponse: - root[kResult] = kResultOk; - root[kCapabilitiesMessageBody] = - absl::get(body).ToJson(); + if (valid) { + root[kResult] = kResultOk; + root[kCapabilitiesMessageBody] = + absl::get(body).ToJson(); + } else { + root[kResult] = kResultError; + root[kErrorMessageBody] = absl::get(body).ToJson(); + } break; // NOTE: RPC messages do NOT have a result field. diff --git a/cast/streaming/receiver_message.h b/cast/streaming/receiver_message.h index 555016df..e25a64bd 100644 --- a/cast/streaming/receiver_message.h +++ b/cast/streaming/receiver_message.h @@ -47,6 +47,8 @@ struct ReceiverError { static ErrorOr Parse(const Json::Value& value); // Error code. + // TODO(issuetracker.google.com/184766188): Error codes should be well + // defined. int32_t code = -1; // Error description. diff --git a/cast/streaming/receiver_session.cc b/cast/streaming/receiver_session.cc index 46e1bac3..a9203734 100644 --- a/cast/streaming/receiver_session.cc +++ b/cast/streaming/receiver_session.cc @@ -42,6 +42,34 @@ std::unique_ptr SelectStream( return nullptr; } +MediaCapability ToCapability(AudioCodec codec) { + switch (codec) { + case AudioCodec::kAac: + return MediaCapability::kAac; + case AudioCodec::kOpus: + return MediaCapability::kOpus; + default: + OSP_DLOG_FATAL << "Invalid audio codec: " << static_cast(codec); + OSP_NOTREACHED(); + } +} + +MediaCapability ToCapability(VideoCodec codec) { + switch (codec) { + case VideoCodec::kVp8: + return MediaCapability::kVp8; + case VideoCodec::kVp9: + return MediaCapability::kVp9; + case VideoCodec::kH264: + return MediaCapability::kH264; + case VideoCodec::kHevc: + return MediaCapability::kHevc; + default: + OSP_DLOG_FATAL << "Invalid video codec: " << static_cast(codec); + OSP_NOTREACHED(); + } +} + } // namespace ReceiverSession::Client::~Client() = default; @@ -89,6 +117,10 @@ ReceiverSession::ReceiverSession(Client* const client, messager_.SetHandler( SenderMessage::Type::kOffer, [this](SenderMessage message) { OnOffer(std::move(message)); }); + messager_.SetHandler(SenderMessage::Type::kGetCapabilities, + [this](SenderMessage message) { + OnCapabilitiesRequest(std::move(message)); + }); environment_->SetSocketSubscriber(this); } @@ -140,7 +172,15 @@ void ReceiverSession::OnOffer(SenderMessage message) { auto properties = std::make_unique(); properties->sequence_number = message.sequence_number; + // TODO(issuetracker.google.com/184186390): ReceiverSession needs to support + // fielding remoting offers. const Offer& offer = absl::get(message.body); + if (offer.cast_mode == CastMode::kRemoting) { + SendErrorAnswerReply(message.sequence_number, + "Remoting support is not complete in libcast"); + return; + } + if (!offer.audio_streams.empty() && !preferences_.audio_codecs.empty()) { properties->selected_audio = SelectStream(preferences_.audio_codecs, offer.audio_streams); @@ -177,6 +217,32 @@ void ReceiverSession::OnOffer(SenderMessage message) { } } +void ReceiverSession::OnCapabilitiesRequest(SenderMessage message) { + if (message.sequence_number < 0) { + OSP_DLOG_WARN + << "Dropping offer with missing sequence number, can't respond"; + return; + } + + ReceiverMessage response{ + ReceiverMessage::Type::kCapabilitiesResponse, message.sequence_number, + true /* valid */ + }; + if (preferences_.remoting) { + response.body = CreateRemotingCapabilityV2(); + } else { + response.valid = false; + response.body = + ReceiverError{static_cast(Error::Code::kRemotingNotSupported), + "Remoting is not supported"}; + } + + const Error result = messager_.SendMessage(std::move(response)); + if (!result.ok()) { + client_->OnError(this, std::move(result)); + } +} + void ReceiverSession::InitializeSession(const SessionProperties& properties) { Answer answer = ConstructAnswer(properties); if (!answer.IsValid()) { @@ -307,7 +373,30 @@ Answer ReceiverSession::ConstructAnswer(const SessionProperties& properties) { } return Answer{environment_->GetBoundLocalEndpoint().port, std::move(stream_indexes), std::move(stream_ssrcs), - std::move(answer_constraints), std::move(display)}; + answer_constraints, std::move(display)}; +} + +ReceiverCapability ReceiverSession::CreateRemotingCapabilityV2() { + // If we don't support remoting, there is no reason to respond to + // capability requests--they are not used for mirroring. + OSP_DCHECK(preferences_.remoting); + ReceiverCapability capability; + capability.remoting_version = kSupportedRemotingVersion; + + for (const AudioCodec& codec : preferences_.audio_codecs) { + capability.media_capabilities.push_back(ToCapability(codec)); + } + for (const VideoCodec& codec : preferences_.video_codecs) { + capability.media_capabilities.push_back(ToCapability(codec)); + } + + if (preferences_.remoting->supports_chrome_audio_codecs) { + capability.media_capabilities.push_back(MediaCapability::kAudio); + } + if (preferences_.remoting->supports_4k) { + capability.media_capabilities.push_back(MediaCapability::k4k); + } + return capability; } void ReceiverSession::SendErrorAnswerReply(int sequence_number, diff --git a/cast/streaming/receiver_session.h b/cast/streaming/receiver_session.h index 51ba49c6..bb6e4493 100644 --- a/cast/streaming/receiver_session.h +++ b/cast/streaming/receiver_session.h @@ -26,6 +26,18 @@ namespace cast { class Environment; class Receiver; +// This class is responsible for listening for streaming (both mirroring and +// remoting) requests from Cast Sender devices, then negotiating capture +// constraints and instantiating audio and video Receiver objects. +// The owner of this session is expected to provide a client for +// updates, an environment for getting UDP socket information (as well as +// other OS dependencies), and a set of preferences to be used for +// negotiation. +// +// NOTE: In some cases, the session initialization may be pending waiting for +// the UDP socket to be ready. In this case, the receivers and the answer +// message will not be configured and sent until the UDP socket has finished +// binding. class ReceiverSession final : public Environment::SocketSubscriber { public: // Upon successful negotiation, a set of configured receivers is constructed @@ -143,6 +155,39 @@ class ReceiverSession final : public Environment::SocketSubscriber { std::chrono::milliseconds max_delay = kDefaultMaxDelayMs; }; + // This struct is used to provide preferences for setting up and running + // remoting streams. The kludgy properties are based on the current control + // protocol and allow remoting with current senders. Once libcast has + // been adopted in Chrome, new, cleaner APIs will be added here to replace + // these. + // + // TODO(issuetracker.google.com/184759616): Chrome should use libcast + // for mirroring and remoting. + // TODO(issuetracker.google.com/184429130): the mirroring control + // protocol needs to be updated to allow more discrete support. + struct RemotingPreferences { + // Current remoting senders take an "all or nothing" support for audio + // codec support. While Opus and AAC support is handled in our Preferences' + // |audio_codecs| property, support for the following codecs must be + // enabled or disabled all together: + // MP3 + // PCM, including Mu-Law, S16BE, S24BE, and ALAW variants + // Ogg Vorbis + // FLAC + // AMR, including narrow band (NB) and wide band (WB) variants + // GSM Mobile Station (MS) + // EAC3 (Dolby Digital Plus) + // ALAC (Apple Lossless) + // AC-3 (Dolby Digital) + // These properties are tied directly to what Chrome supports. See: + // https://source.chromium.org/chromium/chromium/src/+/master:media/base/audio_codecs.h + bool supports_chrome_audio_codecs = false; + + // Current remoting senders assume that the receiver supports 4K for all + // video codecs supplied in |video_codecs|, or none of them. + bool supports_4k = false; + }; + // Note: embedders are required to implement the following // codecs to be Cast V2 compliant: H264, VP8, AAC, Opus. struct Preferences { @@ -171,6 +216,11 @@ class ReceiverSession final : public Environment::SocketSubscriber { std::vector audio_limits; std::vector video_limits; std::unique_ptr display_description; + + // Libcast remoting support is opt-in: embedders wishing to field remoting + // offers may provide a set of remoting preferences, or leave nullptr for + // all remoting OFFERs to be rejected in favor of continuing mirroring. + std::unique_ptr remoting; }; ReceiverSession(Client* const client, @@ -190,9 +240,18 @@ class ReceiverSession final : public Environment::SocketSubscriber { void OnSocketInvalid(Error error) override; private: + // In some cases, such as waiting for the UDP socket to be bound, we + // may have a pending session that cannot start yet. This class provides + // all necessary info to instantiate a session. struct SessionProperties { + // The cast mode the OFFER was sent for. + CastMode mode; + + // The selected audio and video streams from the original OFFER message. std::unique_ptr selected_audio; std::unique_ptr selected_video; + + // The sequence number of the OFFER that produced these properties. int sequence_number; // To be valid either the audio or video must be selected, and we must @@ -202,6 +261,7 @@ class ReceiverSession final : public Environment::SocketSubscriber { // Specific message type handler methods. void OnOffer(SenderMessage message); + void OnCapabilitiesRequest(SenderMessage message); // Creates receivers and sends an appropriate Answer message using the // session properties. @@ -214,9 +274,13 @@ class ReceiverSession final : public Environment::SocketSubscriber { // video streams. NOTE: either audio or video may be null, but not both. ConfiguredReceivers SpawnReceivers(const SessionProperties& properties); - // Callers of this method should ensure at least one stream is non-null. + // Creates an ANSWER object. Assumes at least one stream is not nullptr. Answer ConstructAnswer(const SessionProperties& properties); + // Creates a ReceiverCapability version 2 object. This will be deprecated + // as part of https://issuetracker.google.com/184429130. + ReceiverCapability CreateRemotingCapabilityV2(); + // Handles resetting receivers and notifying the client. void ResetReceivers(Client::ReceiversDestroyingReason reason); @@ -226,18 +290,21 @@ class ReceiverSession final : public Environment::SocketSubscriber { Client* const client_; Environment* const environment_; const Preferences preferences_; + // The sender_id of this session. const std::string session_id_; - ReceiverSessionMessager messager_; - // In some cases, the session initialization may be pending waiting for the - // UDP socket to be ready. In this case, the receivers and the answer - // message will not be configured and sent until the UDP socket has finished - // binding. - std::unique_ptr pending_session_; + // The session messager used for the lifetime of this session. + ReceiverSessionMessager messager_; + // The packet router to be used for all Receivers spawned by this session. ReceiverPacketRouter packet_router_; + // Any session pending while the UDP socket is being bound. + std::unique_ptr pending_session_; + + // The negotiated receivers we own, clients are notified of destruction + // through |Client::OnReceiversDestroying|. std::unique_ptr current_audio_receiver_; std::unique_ptr current_video_receiver_; }; diff --git a/cast/streaming/receiver_session_unittest.cc b/cast/streaming/receiver_session_unittest.cc index 1ed169ec..0a7ad05f 100644 --- a/cast/streaming/receiver_session_unittest.cc +++ b/cast/streaming/receiver_session_unittest.cc @@ -239,6 +239,11 @@ constexpr char kInvalidTypeMessage[] = R"({ "seqNum": 1337 })"; +constexpr char kGetCapabilitiesMessage[] = R"({ + "seqNum": 820263770, + "type": "GET_CAPABILITIES" +})"; + class FakeClient : public ReceiverSession::Client { public: MOCK_METHOD(void, @@ -281,12 +286,17 @@ class ReceiverSessionTest : public ::testing::Test { return environment_; } - void SetUp() { + void SetUp() { SetUpWithPreferences(ReceiverSession::Preferences{}); } + + // Since preferences are constant throughout the life of a session, + // changing them requires configuring a new session. + void SetUpWithPreferences(ReceiverSession::Preferences preferences) { + session_.reset(); message_port_ = std::make_unique("sender-12345"); environment_ = MakeEnvironment(); - session_ = std::make_unique( - &client_, environment_.get(), message_port_.get(), - ReceiverSession::Preferences{}); + session_ = std::make_unique(&client_, environment_.get(), + message_port_.get(), + std::move(preferences)); } protected: @@ -393,7 +403,6 @@ TEST_F(ReceiverSessionTest, CanNegotiateWithCustomCodecPreferences) { TEST_F(ReceiverSessionTest, CanNegotiateWithLimits) { std::vector audio_limits = { {false, AudioCodec::kOpus, 48001, 2, 32001, 32002, milliseconds(3001)}}; - std::vector video_limits = { {true, VideoCodec::kVp9, @@ -679,5 +688,69 @@ TEST_F(ReceiverSessionTest, ReturnsErrorAnswerIfEnvironmentIsInvalidated) { EXPECT_EQ("error", message_body.value()["result"].asString()); } +TEST_F(ReceiverSessionTest, ReturnsErrorCapabilitiesIfRemotingDisabled) { + message_port_->ReceiveMessage(kGetCapabilitiesMessage); + const auto& messages = message_port_->posted_messages(); + ASSERT_EQ(1u, messages.size()); + + // We should have an error response. + auto message_body = json::Parse(messages[0]); + EXPECT_TRUE(message_body.is_value()); + EXPECT_EQ("CAPABILITIES_RESPONSE", message_body.value()["type"].asString()); + EXPECT_EQ("error", message_body.value()["result"].asString()); +} + +TEST_F(ReceiverSessionTest, ReturnsCapabilitiesWithRemotingDefaults) { + ReceiverSession::Preferences preferences; + preferences.remoting = + std::make_unique(); + + SetUpWithPreferences(std::move(preferences)); + message_port_->ReceiveMessage(kGetCapabilitiesMessage); + const auto& messages = message_port_->posted_messages(); + ASSERT_EQ(1u, messages.size()); + + // We should have an error response. + auto message_body = json::Parse(messages[0]); + EXPECT_TRUE(message_body.is_value()); + EXPECT_EQ("CAPABILITIES_RESPONSE", message_body.value()["type"].asString()); + EXPECT_EQ("ok", message_body.value()["result"].asString()); + const ReceiverCapability response = + ReceiverCapability::Parse(message_body.value()["capabilities"]).value(); + + EXPECT_THAT( + response.media_capabilities, + testing::ElementsAre(MediaCapability::kOpus, MediaCapability::kAac, + MediaCapability::kVp8, MediaCapability::kH264)); +} + +TEST_F(ReceiverSessionTest, ReturnsCapabilitiesWithRemotingPreferences) { + ReceiverSession::Preferences preferences; + preferences.video_codecs = {VideoCodec::kH264}; + preferences.remoting = + std::make_unique(); + preferences.remoting->supports_chrome_audio_codecs = true; + preferences.remoting->supports_4k = true; + + SetUpWithPreferences(std::move(preferences)); + message_port_->ReceiveMessage(kGetCapabilitiesMessage); + const auto& messages = message_port_->posted_messages(); + ASSERT_EQ(1u, messages.size()); + + // We should have an error response. + auto message_body = json::Parse(messages[0]); + EXPECT_TRUE(message_body.is_value()); + EXPECT_EQ("CAPABILITIES_RESPONSE", message_body.value()["type"].asString()); + EXPECT_EQ("ok", message_body.value()["result"].asString()); + const ReceiverCapability response = + ReceiverCapability::Parse(message_body.value()["capabilities"]).value(); + + EXPECT_THAT( + response.media_capabilities, + testing::ElementsAre(MediaCapability::kOpus, MediaCapability::kAac, + MediaCapability::kH264, MediaCapability::kAudio, + MediaCapability::k4k)); +} + } // namespace cast } // namespace openscreen diff --git a/cast/streaming/sender_session.cc b/cast/streaming/sender_session.cc index 6b1a9743..866afe5a 100644 --- a/cast/streaming/sender_session.cc +++ b/cast/streaming/sender_session.cc @@ -31,8 +31,6 @@ namespace cast { namespace { -constexpr int kSupportedRemotingVersion = 3; - AudioStream CreateStream(int index, const AudioCaptureConfig& config, bool use_android_rtp_hack) { -- cgit v1.2.3