/* * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source * tree. An additional intellectual property rights grant can be found * in the file PATENTS. All contributing project authors may * be found in the AUTHORS file in the root of the source tree. */ #ifndef WEBRTC_VOICE_ENGINE_CHANNEL_H_ #define WEBRTC_VOICE_ENGINE_CHANNEL_H_ #include "webrtc/audio/audio_sink.h" #include "webrtc/base/criticalsection.h" #include "webrtc/base/scoped_ptr.h" #include "webrtc/common_audio/resampler/include/push_resampler.h" #include "webrtc/common_types.h" #include "webrtc/modules/audio_coding/include/audio_coding_module.h" #include "webrtc/modules/audio_conference_mixer/include/audio_conference_mixer_defines.h" #include "webrtc/modules/audio_processing/rms_level.h" #include "webrtc/modules/rtp_rtcp/include/remote_ntp_time_estimator.h" #include "webrtc/modules/rtp_rtcp/include/rtp_header_parser.h" #include "webrtc/modules/rtp_rtcp/include/rtp_rtcp.h" #include "webrtc/modules/utility/include/file_player.h" #include "webrtc/modules/utility/include/file_recorder.h" #include "webrtc/voice_engine/dtmf_inband.h" #include "webrtc/voice_engine/dtmf_inband_queue.h" #include "webrtc/voice_engine/include/voe_audio_processing.h" #include "webrtc/voice_engine/include/voe_network.h" #include "webrtc/voice_engine/level_indicator.h" #include "webrtc/voice_engine/network_predictor.h" #include "webrtc/voice_engine/shared_data.h" #include "webrtc/voice_engine/voice_engine_defines.h" #ifdef WEBRTC_DTMF_DETECTION // TelephoneEventDetectionMethods, TelephoneEventObserver #include "webrtc/voice_engine/include/voe_dtmf.h" #endif namespace rtc { class TimestampWrapAroundHandler; } namespace webrtc { class AudioDeviceModule; class Config; class CriticalSectionWrapper; class FileWrapper; class PacketRouter; class ProcessThread; class ReceiveStatistics; class RemoteNtpTimeEstimator; class RtcEventLog; class RTPPayloadRegistry; class RtpReceiver; class RTPReceiverAudio; class RtpRtcp; class TelephoneEventHandler; class VoEMediaProcess; class VoERTPObserver; class VoiceEngineObserver; struct CallStatistics; struct ReportBlock; struct SenderInfo; namespace voe { class OutputMixer; class RtpPacketSenderProxy; class Statistics; class StatisticsProxy; class TransportFeedbackProxy; class TransmitMixer; class TransportSequenceNumberProxy; class VoERtcpObserver; // Helper class to simplify locking scheme for members that are accessed from // multiple threads. // Example: a member can be set on thread T1 and read by an internal audio // thread T2. Accessing the member via this class ensures that we are // safe and also avoid TSan v2 warnings. class ChannelState { public: struct State { State() : rx_apm_is_enabled(false), input_external_media(false), output_file_playing(false), input_file_playing(false), playing(false), sending(false), receiving(false) {} bool rx_apm_is_enabled; bool input_external_media; bool output_file_playing; bool input_file_playing; bool playing; bool sending; bool receiving; }; ChannelState() : lock_(CriticalSectionWrapper::CreateCriticalSection()) { } virtual ~ChannelState() {} void Reset() { CriticalSectionScoped lock(lock_.get()); state_ = State(); } State Get() const { CriticalSectionScoped lock(lock_.get()); return state_; } void SetRxApmIsEnabled(bool enable) { CriticalSectionScoped lock(lock_.get()); state_.rx_apm_is_enabled = enable; } void SetInputExternalMedia(bool enable) { CriticalSectionScoped lock(lock_.get()); state_.input_external_media = enable; } void SetOutputFilePlaying(bool enable) { CriticalSectionScoped lock(lock_.get()); state_.output_file_playing = enable; } void SetInputFilePlaying(bool enable) { CriticalSectionScoped lock(lock_.get()); state_.input_file_playing = enable; } void SetPlaying(bool enable) { CriticalSectionScoped lock(lock_.get()); state_.playing = enable; } void SetSending(bool enable) { CriticalSectionScoped lock(lock_.get()); state_.sending = enable; } void SetReceiving(bool enable) { CriticalSectionScoped lock(lock_.get()); state_.receiving = enable; } private: rtc::scoped_ptr lock_; State state_; }; class Channel: public RtpData, public RtpFeedback, public FileCallback, // receiving notification from file player & recorder public Transport, public RtpAudioFeedback, public AudioPacketizationCallback, // receive encoded packets from the ACM public ACMVADCallback, // receive voice activity from the ACM public MixerParticipant // supplies output mixer with audio frames { public: friend class VoERtcpObserver; enum {KNumSocketThreads = 1}; enum {KNumberOfSocketBuffers = 8}; virtual ~Channel(); static int32_t CreateChannel(Channel*& channel, int32_t channelId, uint32_t instanceId, RtcEventLog* const event_log, const Config& config); Channel(int32_t channelId, uint32_t instanceId, RtcEventLog* const event_log, const Config& config); int32_t Init(); int32_t SetEngineInformation( Statistics& engineStatistics, OutputMixer& outputMixer, TransmitMixer& transmitMixer, ProcessThread& moduleProcessThread, AudioDeviceModule& audioDeviceModule, VoiceEngineObserver* voiceEngineObserver, CriticalSectionWrapper* callbackCritSect); int32_t UpdateLocalTimeStamp(); void SetSink(rtc::scoped_ptr sink); // API methods // VoEBase int32_t StartPlayout(); int32_t StopPlayout(); int32_t StartSend(); int32_t StopSend(); int32_t StartReceiving(); int32_t StopReceiving(); int32_t RegisterVoiceEngineObserver(VoiceEngineObserver& observer); int32_t DeRegisterVoiceEngineObserver(); // VoECodec int32_t GetSendCodec(CodecInst& codec); int32_t GetRecCodec(CodecInst& codec); int32_t SetSendCodec(const CodecInst& codec); void SetBitRate(int bitrate_bps); int32_t SetVADStatus(bool enableVAD, ACMVADMode mode, bool disableDTX); int32_t GetVADStatus(bool& enabledVAD, ACMVADMode& mode, bool& disabledDTX); int32_t SetRecPayloadType(const CodecInst& codec); int32_t GetRecPayloadType(CodecInst& codec); int32_t SetSendCNPayloadType(int type, PayloadFrequencies frequency); int SetOpusMaxPlaybackRate(int frequency_hz); int SetOpusDtx(bool enable_dtx); // VoENetwork int32_t RegisterExternalTransport(Transport& transport); int32_t DeRegisterExternalTransport(); int32_t ReceivedRTPPacket(const int8_t* data, size_t length, const PacketTime& packet_time); int32_t ReceivedRTCPPacket(const int8_t* data, size_t length); // VoEFile int StartPlayingFileLocally(const char* fileName, bool loop, FileFormats format, int startPosition, float volumeScaling, int stopPosition, const CodecInst* codecInst); int StartPlayingFileLocally(InStream* stream, FileFormats format, int startPosition, float volumeScaling, int stopPosition, const CodecInst* codecInst); int StopPlayingFileLocally(); int IsPlayingFileLocally() const; int RegisterFilePlayingToMixer(); int StartPlayingFileAsMicrophone(const char* fileName, bool loop, FileFormats format, int startPosition, float volumeScaling, int stopPosition, const CodecInst* codecInst); int StartPlayingFileAsMicrophone(InStream* stream, FileFormats format, int startPosition, float volumeScaling, int stopPosition, const CodecInst* codecInst); int StopPlayingFileAsMicrophone(); int IsPlayingFileAsMicrophone() const; int StartRecordingPlayout(const char* fileName, const CodecInst* codecInst); int StartRecordingPlayout(OutStream* stream, const CodecInst* codecInst); int StopRecordingPlayout(); void SetMixWithMicStatus(bool mix); // VoEExternalMediaProcessing int RegisterExternalMediaProcessing(ProcessingTypes type, VoEMediaProcess& processObject); int DeRegisterExternalMediaProcessing(ProcessingTypes type); int SetExternalMixing(bool enabled); // VoEVolumeControl int GetSpeechOutputLevel(uint32_t& level) const; int GetSpeechOutputLevelFullRange(uint32_t& level) const; int SetMute(bool enable); bool Mute() const; int SetOutputVolumePan(float left, float right); int GetOutputVolumePan(float& left, float& right) const; int SetChannelOutputVolumeScaling(float scaling); int GetChannelOutputVolumeScaling(float& scaling) const; // VoENetEqStats int GetNetworkStatistics(NetworkStatistics& stats); void GetDecodingCallStatistics(AudioDecodingCallStats* stats) const; // VoEVideoSync bool GetDelayEstimate(int* jitter_buffer_delay_ms, int* playout_buffer_delay_ms) const; uint32_t GetDelayEstimate() const; int LeastRequiredDelayMs() const; int SetMinimumPlayoutDelay(int delayMs); int GetPlayoutTimestamp(unsigned int& timestamp); int SetInitTimestamp(unsigned int timestamp); int SetInitSequenceNumber(short sequenceNumber); // VoEVideoSyncExtended int GetRtpRtcp(RtpRtcp** rtpRtcpModule, RtpReceiver** rtp_receiver) const; // VoEDtmf int SendTelephoneEventOutband(unsigned char eventCode, int lengthMs, int attenuationDb, bool playDtmfEvent); int SendTelephoneEventInband(unsigned char eventCode, int lengthMs, int attenuationDb, bool playDtmfEvent); int SetSendTelephoneEventPayloadType(unsigned char type); int GetSendTelephoneEventPayloadType(unsigned char& type); // VoEAudioProcessingImpl int UpdateRxVadDetection(AudioFrame& audioFrame); int RegisterRxVadObserver(VoERxVadCallback &observer); int DeRegisterRxVadObserver(); int VoiceActivityIndicator(int &activity); #ifdef WEBRTC_VOICE_ENGINE_AGC int SetRxAgcStatus(bool enable, AgcModes mode); int GetRxAgcStatus(bool& enabled, AgcModes& mode); int SetRxAgcConfig(AgcConfig config); int GetRxAgcConfig(AgcConfig& config); #endif #ifdef WEBRTC_VOICE_ENGINE_NR int SetRxNsStatus(bool enable, NsModes mode); int GetRxNsStatus(bool& enabled, NsModes& mode); #endif // VoERTP_RTCP int SetLocalSSRC(unsigned int ssrc); int GetLocalSSRC(unsigned int& ssrc); int GetRemoteSSRC(unsigned int& ssrc); int SetSendAudioLevelIndicationStatus(bool enable, unsigned char id); int SetReceiveAudioLevelIndicationStatus(bool enable, unsigned char id); int SetSendAbsoluteSenderTimeStatus(bool enable, unsigned char id); int SetReceiveAbsoluteSenderTimeStatus(bool enable, unsigned char id); void EnableSendTransportSequenceNumber(int id); void SetCongestionControlObjects( RtpPacketSender* rtp_packet_sender, TransportFeedbackObserver* transport_feedback_observer, PacketRouter* packet_router); void SetRTCPStatus(bool enable); int GetRTCPStatus(bool& enabled); int SetRTCP_CNAME(const char cName[256]); int GetRemoteRTCP_CNAME(char cName[256]); int GetRemoteRTCPData(unsigned int& NTPHigh, unsigned int& NTPLow, unsigned int& timestamp, unsigned int& playoutTimestamp, unsigned int* jitter, unsigned short* fractionLost); int SendApplicationDefinedRTCPPacket(unsigned char subType, unsigned int name, const char* data, unsigned short dataLengthInBytes); int GetRTPStatistics(unsigned int& averageJitterMs, unsigned int& maxJitterMs, unsigned int& discardedPackets); int GetRemoteRTCPReportBlocks(std::vector* report_blocks); int GetRTPStatistics(CallStatistics& stats); int SetREDStatus(bool enable, int redPayloadtype); int GetREDStatus(bool& enabled, int& redPayloadtype); int SetCodecFECStatus(bool enable); bool GetCodecFECStatus(); void SetNACKStatus(bool enable, int maxNumberOfPackets); // From AudioPacketizationCallback in the ACM int32_t SendData(FrameType frameType, uint8_t payloadType, uint32_t timeStamp, const uint8_t* payloadData, size_t payloadSize, const RTPFragmentationHeader* fragmentation) override; // From ACMVADCallback in the ACM int32_t InFrameType(FrameType frame_type) override; int32_t OnRxVadDetected(int vadDecision); // From RtpData in the RTP/RTCP module int32_t OnReceivedPayloadData(const uint8_t* payloadData, size_t payloadSize, const WebRtcRTPHeader* rtpHeader) override; bool OnRecoveredPacket(const uint8_t* packet, size_t packet_length) override; // From RtpFeedback in the RTP/RTCP module int32_t OnInitializeDecoder(int8_t payloadType, const char payloadName[RTP_PAYLOAD_NAME_SIZE], int frequency, size_t channels, uint32_t rate) override; void OnIncomingSSRCChanged(uint32_t ssrc) override; void OnIncomingCSRCChanged(uint32_t CSRC, bool added) override; // From RtpAudioFeedback in the RTP/RTCP module void OnPlayTelephoneEvent(uint8_t event, uint16_t lengthMs, uint8_t volume) override; // From Transport (called by the RTP/RTCP module) bool SendRtp(const uint8_t* data, size_t len, const PacketOptions& packet_options) override; bool SendRtcp(const uint8_t* data, size_t len) override; // From MixerParticipant int32_t GetAudioFrame(int32_t id, AudioFrame* audioFrame) override; int32_t NeededFrequency(int32_t id) const override; // From FileCallback void PlayNotification(int32_t id, uint32_t durationMs) override; void RecordNotification(int32_t id, uint32_t durationMs) override; void PlayFileEnded(int32_t id) override; void RecordFileEnded(int32_t id) override; uint32_t InstanceId() const { return _instanceId; } int32_t ChannelId() const { return _channelId; } bool Playing() const { return channel_state_.Get().playing; } bool Sending() const { return channel_state_.Get().sending; } bool Receiving() const { return channel_state_.Get().receiving; } bool ExternalTransport() const { CriticalSectionScoped cs(&_callbackCritSect); return _externalTransport; } bool ExternalMixing() const { return _externalMixing; } RtpRtcp* RtpRtcpModulePtr() const { return _rtpRtcpModule.get(); } int8_t OutputEnergyLevel() const { return _outputAudioLevel.Level(); } uint32_t Demultiplex(const AudioFrame& audioFrame); // Demultiplex the data to the channel's |_audioFrame|. The difference // between this method and the overloaded method above is that |audio_data| // does not go through transmit_mixer and APM. void Demultiplex(const int16_t* audio_data, int sample_rate, size_t number_of_frames, size_t number_of_channels); uint32_t PrepareEncodeAndSend(int mixingFrequency); uint32_t EncodeAndSend(); // Associate to a send channel. // Used for obtaining RTT for a receive-only channel. void set_associate_send_channel(const ChannelOwner& channel) { assert(_channelId != channel.channel()->ChannelId()); CriticalSectionScoped lock(assoc_send_channel_lock_.get()); associate_send_channel_ = channel; } // Disassociate a send channel if it was associated. void DisassociateSendChannel(int channel_id); protected: void OnIncomingFractionLoss(int fraction_lost); private: bool ReceivePacket(const uint8_t* packet, size_t packet_length, const RTPHeader& header, bool in_order); bool HandleRtxPacket(const uint8_t* packet, size_t packet_length, const RTPHeader& header); bool IsPacketInOrder(const RTPHeader& header) const; bool IsPacketRetransmitted(const RTPHeader& header, bool in_order) const; int ResendPackets(const uint16_t* sequence_numbers, int length); int InsertInbandDtmfTone(); int32_t MixOrReplaceAudioWithFile(int mixingFrequency); int32_t MixAudioWithFile(AudioFrame& audioFrame, int mixingFrequency); void UpdatePlayoutTimestamp(bool rtcp); void UpdatePacketDelay(uint32_t timestamp, uint16_t sequenceNumber); void RegisterReceiveCodecsToRTPModule(); int SetRedPayloadType(int red_payload_type); int SetSendRtpHeaderExtension(bool enable, RTPExtensionType type, unsigned char id); int32_t GetPlayoutFrequency(); int64_t GetRTT(bool allow_associate_channel) const; CriticalSectionWrapper& _fileCritSect; CriticalSectionWrapper& _callbackCritSect; CriticalSectionWrapper& volume_settings_critsect_; uint32_t _instanceId; int32_t _channelId; ChannelState channel_state_; RtcEventLog* const event_log_; rtc::scoped_ptr rtp_header_parser_; rtc::scoped_ptr rtp_payload_registry_; rtc::scoped_ptr rtp_receive_statistics_; rtc::scoped_ptr statistics_proxy_; rtc::scoped_ptr rtp_receiver_; TelephoneEventHandler* telephone_event_handler_; rtc::scoped_ptr _rtpRtcpModule; rtc::scoped_ptr audio_coding_; rtc::scoped_ptr audio_sink_; AudioLevel _outputAudioLevel; bool _externalTransport; AudioFrame _audioFrame; // Downsamples to the codec rate if necessary. PushResampler input_resampler_; FilePlayer* _inputFilePlayerPtr; FilePlayer* _outputFilePlayerPtr; FileRecorder* _outputFileRecorderPtr; int _inputFilePlayerId; int _outputFilePlayerId; int _outputFileRecorderId; bool _outputFileRecording; DtmfInbandQueue _inbandDtmfQueue; DtmfInband _inbandDtmfGenerator; bool _outputExternalMedia; VoEMediaProcess* _inputExternalMediaCallbackPtr; VoEMediaProcess* _outputExternalMediaCallbackPtr; uint32_t _timeStamp; uint8_t _sendTelephoneEventPayloadType; RemoteNtpTimeEstimator ntp_estimator_ GUARDED_BY(ts_stats_lock_); // Timestamp of the audio pulled from NetEq. uint32_t jitter_buffer_playout_timestamp_; uint32_t playout_timestamp_rtp_ GUARDED_BY(video_sync_lock_); uint32_t playout_timestamp_rtcp_; uint32_t playout_delay_ms_ GUARDED_BY(video_sync_lock_); uint32_t _numberOfDiscardedPackets; uint16_t send_sequence_number_; uint8_t restored_packet_[kVoiceEngineMaxIpPacketSizeBytes]; rtc::scoped_ptr ts_stats_lock_; rtc::scoped_ptr rtp_ts_wraparound_handler_; // The rtp timestamp of the first played out audio frame. int64_t capture_start_rtp_time_stamp_; // The capture ntp time (in local timebase) of the first played out audio // frame. int64_t capture_start_ntp_time_ms_ GUARDED_BY(ts_stats_lock_); // uses Statistics* _engineStatisticsPtr; OutputMixer* _outputMixerPtr; TransmitMixer* _transmitMixerPtr; ProcessThread* _moduleProcessThreadPtr; AudioDeviceModule* _audioDeviceModulePtr; VoiceEngineObserver* _voiceEngineObserverPtr; // owned by base CriticalSectionWrapper* _callbackCritSectPtr; // owned by base Transport* _transportPtr; // WebRtc socket or external transport RMSLevel rms_level_; rtc::scoped_ptr rx_audioproc_; // far end AudioProcessing VoERxVadCallback* _rxVadObserverPtr; int32_t _oldVadDecision; int32_t _sendFrameType; // Send data is voice, 1-voice, 0-otherwise // VoEBase bool _externalMixing; bool _mixFileWithMicrophone; // VoEVolumeControl bool _mute; float _panLeft; float _panRight; float _outputGain; // VoEDtmf bool _playOutbandDtmfEvent; bool _playInbandDtmfEvent; // VoeRTP_RTCP uint32_t _lastLocalTimeStamp; int8_t _lastPayloadType; bool _includeAudioLevelIndication; // VoENetwork AudioFrame::SpeechType _outputSpeechType; // VoEVideoSync rtc::scoped_ptr video_sync_lock_; uint32_t _average_jitter_buffer_delay_us GUARDED_BY(video_sync_lock_); uint32_t _previousTimestamp; uint16_t _recPacketDelayMs GUARDED_BY(video_sync_lock_); // VoEAudioProcessing bool _RxVadDetection; bool _rxAgcIsEnabled; bool _rxNsIsEnabled; bool restored_packet_in_use_; // RtcpBandwidthObserver rtc::scoped_ptr rtcp_observer_; rtc::scoped_ptr network_predictor_; // An associated send channel. rtc::scoped_ptr assoc_send_channel_lock_; ChannelOwner associate_send_channel_ GUARDED_BY(assoc_send_channel_lock_); bool pacing_enabled_; PacketRouter* packet_router_ = nullptr; rtc::scoped_ptr feedback_observer_proxy_; rtc::scoped_ptr seq_num_allocator_proxy_; rtc::scoped_ptr rtp_packet_sender_proxy_; }; } // namespace voe } // namespace webrtc #endif // WEBRTC_VOICE_ENGINE_CHANNEL_H_