/* * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source * tree. An additional intellectual property rights grant can be found * in the file PATENTS. All contributing project authors may * be found in the AUTHORS file in the root of the source tree. */ #include "webrtc/modules/audio_conference_mixer/interface/audio_conference_mixer_defines.h" #include "webrtc/modules/audio_conference_mixer/source/audio_conference_mixer_impl.h" #include "webrtc/modules/audio_conference_mixer/source/audio_frame_manipulator.h" #include "webrtc/modules/audio_processing/include/audio_processing.h" #include "webrtc/modules/utility/interface/audio_frame_operations.h" #include "webrtc/system_wrappers/interface/critical_section_wrapper.h" #include "webrtc/system_wrappers/interface/trace.h" namespace webrtc { namespace { struct ParticipantFramePair { MixerParticipant* participant; AudioFrame* audioFrame; }; typedef std::list ParticipantFramePairList; // Mix |frame| into |mixed_frame|, with saturation protection and upmixing. // These effects are applied to |frame| itself prior to mixing. Assumes that // |mixed_frame| always has at least as many channels as |frame|. Supports // stereo at most. // // TODO(andrew): consider not modifying |frame| here. void MixFrames(AudioFrame* mixed_frame, AudioFrame* frame) { assert(mixed_frame->num_channels_ >= frame->num_channels_); // Divide by two to avoid saturation in the mixing. *frame >>= 1; if (mixed_frame->num_channels_ > frame->num_channels_) { // We only support mono-to-stereo. assert(mixed_frame->num_channels_ == 2 && frame->num_channels_ == 1); AudioFrameOperations::MonoToStereo(frame); } *mixed_frame += *frame; } // Return the max number of channels from a |list| composed of AudioFrames. int MaxNumChannels(const AudioFrameList* list) { int max_num_channels = 1; for (AudioFrameList::const_iterator iter = list->begin(); iter != list->end(); ++iter) { max_num_channels = std::max(max_num_channels, (*iter)->num_channels_); } return max_num_channels; } void SetParticipantStatistics(ParticipantStatistics* stats, const AudioFrame& frame) { stats->participant = frame.id_; stats->level = 0; // TODO(andrew): to what should this be set? } } // namespace MixerParticipant::MixerParticipant() : _mixHistory(new MixHistory()) { } MixerParticipant::~MixerParticipant() { delete _mixHistory; } int32_t MixerParticipant::IsMixed(bool& mixed) const { return _mixHistory->IsMixed(mixed); } MixHistory::MixHistory() : _isMixed(0) { } MixHistory::~MixHistory() { } int32_t MixHistory::IsMixed(bool& mixed) const { mixed = _isMixed; return 0; } int32_t MixHistory::WasMixed(bool& wasMixed) const { // Was mixed is the same as is mixed depending on perspective. This function // is for the perspective of AudioConferenceMixerImpl. return IsMixed(wasMixed); } int32_t MixHistory::SetIsMixed(const bool mixed) { _isMixed = mixed; return 0; } void MixHistory::ResetMixedStatus() { _isMixed = false; } AudioConferenceMixer* AudioConferenceMixer::Create(int id) { AudioConferenceMixerImpl* mixer = new AudioConferenceMixerImpl(id); if(!mixer->Init()) { delete mixer; return NULL; } return mixer; } AudioConferenceMixerImpl::AudioConferenceMixerImpl(int id) : _scratchParticipantsToMixAmount(0), _scratchMixedParticipants(), _scratchVadPositiveParticipantsAmount(0), _scratchVadPositiveParticipants(), _id(id), _minimumMixingFreq(kLowestPossible), _mixReceiver(NULL), _mixerStatusCallback(NULL), _amountOf10MsBetweenCallbacks(1), _amountOf10MsUntilNextCallback(0), _mixerStatusCb(false), _outputFrequency(kDefaultFrequency), _sampleSize(0), _audioFramePool(NULL), _participantList(), _additionalParticipantList(), _numMixedParticipants(0), _timeStamp(0), _timeScheduler(kProcessPeriodicityInMs), _mixedAudioLevel(), _processCalls(0) {} bool AudioConferenceMixerImpl::Init() { _crit.reset(CriticalSectionWrapper::CreateCriticalSection()); if (_crit.get() == NULL) return false; _cbCrit.reset(CriticalSectionWrapper::CreateCriticalSection()); if(_cbCrit.get() == NULL) return false; Config config; config.Set(new ExperimentalAgc(false)); _limiter.reset(AudioProcessing::Create(config)); if(!_limiter.get()) return false; MemoryPool::CreateMemoryPool(_audioFramePool, DEFAULT_AUDIO_FRAME_POOLSIZE); if(_audioFramePool == NULL) return false; if(SetOutputFrequency(kDefaultFrequency) == -1) return false; if(_limiter->gain_control()->set_mode(GainControl::kFixedDigital) != _limiter->kNoError) return false; // We smoothly limit the mixed frame to -7 dbFS. -6 would correspond to the // divide-by-2 but -7 is used instead to give a bit of headroom since the // AGC is not a hard limiter. if(_limiter->gain_control()->set_target_level_dbfs(7) != _limiter->kNoError) return false; if(_limiter->gain_control()->set_compression_gain_db(0) != _limiter->kNoError) return false; if(_limiter->gain_control()->enable_limiter(true) != _limiter->kNoError) return false; if(_limiter->gain_control()->Enable(true) != _limiter->kNoError) return false; return true; } AudioConferenceMixerImpl::~AudioConferenceMixerImpl() { MemoryPool::DeleteMemoryPool(_audioFramePool); assert(_audioFramePool == NULL); } int32_t AudioConferenceMixerImpl::ChangeUniqueId(const int32_t id) { _id = id; return 0; } // Process should be called every kProcessPeriodicityInMs ms int32_t AudioConferenceMixerImpl::TimeUntilNextProcess() { int32_t timeUntilNextProcess = 0; CriticalSectionScoped cs(_crit.get()); if(_timeScheduler.TimeToNextUpdate(timeUntilNextProcess) != 0) { WEBRTC_TRACE(kTraceError, kTraceAudioMixerServer, _id, "failed in TimeToNextUpdate() call"); // Sanity check assert(false); return -1; } return timeUntilNextProcess; } int32_t AudioConferenceMixerImpl::Process() { size_t remainingParticipantsAllowedToMix = kMaximumAmountOfMixedParticipants; { CriticalSectionScoped cs(_crit.get()); assert(_processCalls == 0); _processCalls++; // Let the scheduler know that we are running one iteration. _timeScheduler.UpdateScheduler(); } AudioFrameList mixList; AudioFrameList rampOutList; AudioFrameList additionalFramesList; std::map mixedParticipantsMap; { CriticalSectionScoped cs(_cbCrit.get()); int32_t lowFreq = GetLowestMixingFrequency(); // SILK can run in 12 kHz and 24 kHz. These frequencies are not // supported so use the closest higher frequency to not lose any // information. // TODO(henrike): this is probably more appropriate to do in // GetLowestMixingFrequency(). if (lowFreq == 12000) { lowFreq = 16000; } else if (lowFreq == 24000) { lowFreq = 32000; } if(lowFreq <= 0) { CriticalSectionScoped cs(_crit.get()); _processCalls--; return 0; } else { switch(lowFreq) { case 8000: if(OutputFrequency() != kNbInHz) { SetOutputFrequency(kNbInHz); } break; case 16000: if(OutputFrequency() != kWbInHz) { SetOutputFrequency(kWbInHz); } break; case 32000: if(OutputFrequency() != kSwbInHz) { SetOutputFrequency(kSwbInHz); } break; case 48000: if(OutputFrequency() != kFbInHz) { SetOutputFrequency(kFbInHz); } break; default: assert(false); CriticalSectionScoped cs(_crit.get()); _processCalls--; return -1; } } UpdateToMix(&mixList, &rampOutList, &mixedParticipantsMap, remainingParticipantsAllowedToMix); GetAdditionalAudio(&additionalFramesList); UpdateMixedStatus(mixedParticipantsMap); _scratchParticipantsToMixAmount = mixedParticipantsMap.size(); } // Get an AudioFrame for mixing from the memory pool. AudioFrame* mixedAudio = NULL; if(_audioFramePool->PopMemory(mixedAudio) == -1) { WEBRTC_TRACE(kTraceMemory, kTraceAudioMixerServer, _id, "failed PopMemory() call"); assert(false); return -1; } bool timeForMixerCallback = false; int retval = 0; int32_t audioLevel = 0; { CriticalSectionScoped cs(_crit.get()); // TODO(henrike): it might be better to decide the number of channels // with an API instead of dynamically. // Find the max channels over all mixing lists. const int num_mixed_channels = std::max(MaxNumChannels(&mixList), std::max(MaxNumChannels(&additionalFramesList), MaxNumChannels(&rampOutList))); mixedAudio->UpdateFrame(-1, _timeStamp, NULL, 0, _outputFrequency, AudioFrame::kNormalSpeech, AudioFrame::kVadPassive, num_mixed_channels); _timeStamp += _sampleSize; MixFromList(*mixedAudio, &mixList); MixAnonomouslyFromList(*mixedAudio, &additionalFramesList); MixAnonomouslyFromList(*mixedAudio, &rampOutList); if(mixedAudio->samples_per_channel_ == 0) { // Nothing was mixed, set the audio samples to silence. mixedAudio->samples_per_channel_ = _sampleSize; mixedAudio->Mute(); } else { // Only call the limiter if we have something to mix. if(!LimitMixedAudio(*mixedAudio)) retval = -1; } _mixedAudioLevel.ComputeLevel(mixedAudio->data_,_sampleSize); audioLevel = _mixedAudioLevel.GetLevel(); if(_mixerStatusCb) { _scratchVadPositiveParticipantsAmount = 0; UpdateVADPositiveParticipants(&mixList); if(_amountOf10MsUntilNextCallback-- == 0) { _amountOf10MsUntilNextCallback = _amountOf10MsBetweenCallbacks; timeForMixerCallback = true; } } } { CriticalSectionScoped cs(_cbCrit.get()); if(_mixReceiver != NULL) { const AudioFrame** dummy = NULL; _mixReceiver->NewMixedAudio( _id, *mixedAudio, dummy, 0); } if((_mixerStatusCallback != NULL) && timeForMixerCallback) { _mixerStatusCallback->MixedParticipants( _id, _scratchMixedParticipants, static_cast(_scratchParticipantsToMixAmount)); _mixerStatusCallback->VADPositiveParticipants( _id, _scratchVadPositiveParticipants, _scratchVadPositiveParticipantsAmount); _mixerStatusCallback->MixedAudioLevel(_id,audioLevel); } } // Reclaim all outstanding memory. _audioFramePool->PushMemory(mixedAudio); ClearAudioFrameList(&mixList); ClearAudioFrameList(&rampOutList); ClearAudioFrameList(&additionalFramesList); { CriticalSectionScoped cs(_crit.get()); _processCalls--; } return retval; } int32_t AudioConferenceMixerImpl::RegisterMixedStreamCallback( AudioMixerOutputReceiver& mixReceiver) { CriticalSectionScoped cs(_cbCrit.get()); if(_mixReceiver != NULL) { return -1; } _mixReceiver = &mixReceiver; return 0; } int32_t AudioConferenceMixerImpl::UnRegisterMixedStreamCallback() { CriticalSectionScoped cs(_cbCrit.get()); if(_mixReceiver == NULL) { return -1; } _mixReceiver = NULL; return 0; } int32_t AudioConferenceMixerImpl::SetOutputFrequency( const Frequency frequency) { CriticalSectionScoped cs(_crit.get()); _outputFrequency = frequency; _sampleSize = (_outputFrequency*kProcessPeriodicityInMs) / 1000; return 0; } AudioConferenceMixer::Frequency AudioConferenceMixerImpl::OutputFrequency() const { CriticalSectionScoped cs(_crit.get()); return _outputFrequency; } int32_t AudioConferenceMixerImpl::RegisterMixerStatusCallback( AudioMixerStatusReceiver& mixerStatusCallback, const uint32_t amountOf10MsBetweenCallbacks) { if(amountOf10MsBetweenCallbacks == 0) { WEBRTC_TRACE( kTraceWarning, kTraceAudioMixerServer, _id, "amountOf10MsBetweenCallbacks(%d) needs to be larger than 0"); return -1; } { CriticalSectionScoped cs(_cbCrit.get()); if(_mixerStatusCallback != NULL) { WEBRTC_TRACE(kTraceWarning, kTraceAudioMixerServer, _id, "Mixer status callback already registered"); return -1; } _mixerStatusCallback = &mixerStatusCallback; } { CriticalSectionScoped cs(_crit.get()); _amountOf10MsBetweenCallbacks = amountOf10MsBetweenCallbacks; _amountOf10MsUntilNextCallback = 0; _mixerStatusCb = true; } return 0; } int32_t AudioConferenceMixerImpl::UnRegisterMixerStatusCallback() { { CriticalSectionScoped cs(_crit.get()); if(!_mixerStatusCb) { WEBRTC_TRACE(kTraceWarning, kTraceAudioMixerServer, _id, "Mixer status callback not registered"); return -1; } _mixerStatusCb = false; } { CriticalSectionScoped cs(_cbCrit.get()); _mixerStatusCallback = NULL; } return 0; } int32_t AudioConferenceMixerImpl::SetMixabilityStatus( MixerParticipant& participant, bool mixable) { if (!mixable) { // Anonymous participants are in a separate list. Make sure that the // participant is in the _participantList if it is being mixed. SetAnonymousMixabilityStatus(participant, false); } size_t numMixedParticipants; { CriticalSectionScoped cs(_cbCrit.get()); const bool isMixed = IsParticipantInList(participant, &_participantList); // API must be called with a new state. if(!(mixable ^ isMixed)) { WEBRTC_TRACE(kTraceWarning, kTraceAudioMixerServer, _id, "Mixable is aready %s", isMixed ? "ON" : "off"); return -1; } bool success = false; if(mixable) { success = AddParticipantToList(participant, &_participantList); } else { success = RemoveParticipantFromList(participant, &_participantList); } if(!success) { WEBRTC_TRACE(kTraceError, kTraceAudioMixerServer, _id, "failed to %s participant", mixable ? "add" : "remove"); assert(false); return -1; } size_t numMixedNonAnonymous = _participantList.size(); if (numMixedNonAnonymous > kMaximumAmountOfMixedParticipants) { numMixedNonAnonymous = kMaximumAmountOfMixedParticipants; } numMixedParticipants = numMixedNonAnonymous + _additionalParticipantList.size(); } // A MixerParticipant was added or removed. Make sure the scratch // buffer is updated if necessary. // Note: The scratch buffer may only be updated in Process(). CriticalSectionScoped cs(_crit.get()); _numMixedParticipants = numMixedParticipants; return 0; } int32_t AudioConferenceMixerImpl::MixabilityStatus( MixerParticipant& participant, bool& mixable) { CriticalSectionScoped cs(_cbCrit.get()); mixable = IsParticipantInList(participant, &_participantList); return 0; } int32_t AudioConferenceMixerImpl::SetAnonymousMixabilityStatus( MixerParticipant& participant, const bool anonymous) { CriticalSectionScoped cs(_cbCrit.get()); if(IsParticipantInList(participant, &_additionalParticipantList)) { if(anonymous) { return 0; } if(!RemoveParticipantFromList(participant, &_additionalParticipantList)) { WEBRTC_TRACE(kTraceError, kTraceAudioMixerServer, _id, "unable to remove participant from anonymous list"); assert(false); return -1; } return AddParticipantToList(participant, &_participantList) ? 0 : -1; } if(!anonymous) { return 0; } const bool mixable = RemoveParticipantFromList(participant, &_participantList); if(!mixable) { WEBRTC_TRACE( kTraceWarning, kTraceAudioMixerServer, _id, "participant must be registered before turning it into anonymous"); // Setting anonymous status is only possible if MixerParticipant is // already registered. return -1; } return AddParticipantToList(participant, &_additionalParticipantList) ? 0 : -1; } int32_t AudioConferenceMixerImpl::AnonymousMixabilityStatus( MixerParticipant& participant, bool& mixable) { CriticalSectionScoped cs(_cbCrit.get()); mixable = IsParticipantInList(participant, &_additionalParticipantList); return 0; } int32_t AudioConferenceMixerImpl::SetMinimumMixingFrequency( Frequency freq) { // Make sure that only allowed sampling frequencies are used. Use closest // higher sampling frequency to avoid losing information. if (static_cast(freq) == 12000) { freq = kWbInHz; } else if (static_cast(freq) == 24000) { freq = kSwbInHz; } if((freq == kNbInHz) || (freq == kWbInHz) || (freq == kSwbInHz) || (freq == kLowestPossible)) { _minimumMixingFreq=freq; return 0; } else { WEBRTC_TRACE(kTraceError, kTraceAudioMixerServer, _id, "SetMinimumMixingFrequency incorrect frequency: %i",freq); assert(false); return -1; } } // Check all AudioFrames that are to be mixed. The highest sampling frequency // found is the lowest that can be used without losing information. int32_t AudioConferenceMixerImpl::GetLowestMixingFrequency() { const int participantListFrequency = GetLowestMixingFrequencyFromList(&_participantList); const int anonymousListFrequency = GetLowestMixingFrequencyFromList(&_additionalParticipantList); const int highestFreq = (participantListFrequency > anonymousListFrequency) ? participantListFrequency : anonymousListFrequency; // Check if the user specified a lowest mixing frequency. if(_minimumMixingFreq != kLowestPossible) { if(_minimumMixingFreq > highestFreq) { return _minimumMixingFreq; } } return highestFreq; } int32_t AudioConferenceMixerImpl::GetLowestMixingFrequencyFromList( MixerParticipantList* mixList) { int32_t highestFreq = 8000; for (MixerParticipantList::iterator iter = mixList->begin(); iter != mixList->end(); ++iter) { const int32_t neededFrequency = (*iter)->NeededFrequency(_id); if(neededFrequency > highestFreq) { highestFreq = neededFrequency; } } return highestFreq; } void AudioConferenceMixerImpl::UpdateToMix( AudioFrameList* mixList, AudioFrameList* rampOutList, std::map* mixParticipantList, size_t& maxAudioFrameCounter) { WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, _id, "UpdateToMix(mixList,rampOutList,mixParticipantList,%d)", maxAudioFrameCounter); const size_t mixListStartSize = mixList->size(); AudioFrameList activeList; // Struct needed by the passive lists to keep track of which AudioFrame // belongs to which MixerParticipant. ParticipantFramePairList passiveWasNotMixedList; ParticipantFramePairList passiveWasMixedList; for (MixerParticipantList::iterator participant = _participantList.begin(); participant != _participantList.end(); ++participant) { // Stop keeping track of passive participants if there are already // enough participants available (they wont be mixed anyway). bool mustAddToPassiveList = (maxAudioFrameCounter > (activeList.size() + passiveWasMixedList.size() + passiveWasNotMixedList.size())); bool wasMixed = false; (*participant)->_mixHistory->WasMixed(wasMixed); AudioFrame* audioFrame = NULL; if(_audioFramePool->PopMemory(audioFrame) == -1) { WEBRTC_TRACE(kTraceMemory, kTraceAudioMixerServer, _id, "failed PopMemory() call"); assert(false); return; } audioFrame->sample_rate_hz_ = _outputFrequency; if((*participant)->GetAudioFrame(_id,*audioFrame) != 0) { WEBRTC_TRACE(kTraceWarning, kTraceAudioMixerServer, _id, "failed to GetAudioFrame() from participant"); _audioFramePool->PushMemory(audioFrame); continue; } if (_participantList.size() != 1) { // TODO(wu): Issue 3390, add support for multiple participants case. audioFrame->ntp_time_ms_ = -1; } // TODO(henrike): this assert triggers in some test cases where SRTP is // used which prevents NetEQ from making a VAD. Temporarily disable this // assert until the problem is fixed on a higher level. // assert(audioFrame->vad_activity_ != AudioFrame::kVadUnknown); if (audioFrame->vad_activity_ == AudioFrame::kVadUnknown) { WEBRTC_TRACE(kTraceWarning, kTraceAudioMixerServer, _id, "invalid VAD state from participant"); } if(audioFrame->vad_activity_ == AudioFrame::kVadActive) { if(!wasMixed) { RampIn(*audioFrame); } if(activeList.size() >= maxAudioFrameCounter) { // There are already more active participants than should be // mixed. Only keep the ones with the highest energy. AudioFrameList::iterator replaceItem; CalculateEnergy(*audioFrame); uint32_t lowestEnergy = audioFrame->energy_; bool found_replace_item = false; for (AudioFrameList::iterator iter = activeList.begin(); iter != activeList.end(); ++iter) { CalculateEnergy(**iter); if((*iter)->energy_ < lowestEnergy) { replaceItem = iter; lowestEnergy = (*iter)->energy_; found_replace_item = true; } } if(found_replace_item) { AudioFrame* replaceFrame = *replaceItem; bool replaceWasMixed = false; std::map::iterator it = mixParticipantList->find(replaceFrame->id_); // When a frame is pushed to |activeList| it is also pushed // to mixParticipantList with the frame's id. This means // that the Find call above should never fail. assert(it != mixParticipantList->end()); it->second->_mixHistory->WasMixed(replaceWasMixed); mixParticipantList->erase(replaceFrame->id_); activeList.erase(replaceItem); activeList.push_front(audioFrame); (*mixParticipantList)[audioFrame->id_] = *participant; assert(mixParticipantList->size() <= kMaximumAmountOfMixedParticipants); if (replaceWasMixed) { RampOut(*replaceFrame); rampOutList->push_back(replaceFrame); assert(rampOutList->size() <= kMaximumAmountOfMixedParticipants); } else { _audioFramePool->PushMemory(replaceFrame); } } else { if(wasMixed) { RampOut(*audioFrame); rampOutList->push_back(audioFrame); assert(rampOutList->size() <= kMaximumAmountOfMixedParticipants); } else { _audioFramePool->PushMemory(audioFrame); } } } else { activeList.push_front(audioFrame); (*mixParticipantList)[audioFrame->id_] = *participant; assert(mixParticipantList->size() <= kMaximumAmountOfMixedParticipants); } } else { if(wasMixed) { ParticipantFramePair* pair = new ParticipantFramePair; pair->audioFrame = audioFrame; pair->participant = *participant; passiveWasMixedList.push_back(pair); } else if(mustAddToPassiveList) { RampIn(*audioFrame); ParticipantFramePair* pair = new ParticipantFramePair; pair->audioFrame = audioFrame; pair->participant = *participant; passiveWasNotMixedList.push_back(pair); } else { _audioFramePool->PushMemory(audioFrame); } } } assert(activeList.size() <= maxAudioFrameCounter); // At this point it is known which participants should be mixed. Transfer // this information to this functions output parameters. for (AudioFrameList::iterator iter = activeList.begin(); iter != activeList.end(); ++iter) { mixList->push_back(*iter); } activeList.clear(); // Always mix a constant number of AudioFrames. If there aren't enough // active participants mix passive ones. Starting with those that was mixed // last iteration. for (ParticipantFramePairList::iterator iter = passiveWasMixedList.begin(); iter != passiveWasMixedList.end(); ++iter) { if(mixList->size() < maxAudioFrameCounter + mixListStartSize) { mixList->push_back((*iter)->audioFrame); (*mixParticipantList)[(*iter)->audioFrame->id_] = (*iter)->participant; assert(mixParticipantList->size() <= kMaximumAmountOfMixedParticipants); } else { _audioFramePool->PushMemory((*iter)->audioFrame); } delete *iter; } // And finally the ones that have not been mixed for a while. for (ParticipantFramePairList::iterator iter = passiveWasNotMixedList.begin(); iter != passiveWasNotMixedList.end(); ++iter) { if(mixList->size() < maxAudioFrameCounter + mixListStartSize) { mixList->push_back((*iter)->audioFrame); (*mixParticipantList)[(*iter)->audioFrame->id_] = (*iter)->participant; assert(mixParticipantList->size() <= kMaximumAmountOfMixedParticipants); } else { _audioFramePool->PushMemory((*iter)->audioFrame); } delete *iter; } assert(maxAudioFrameCounter + mixListStartSize >= mixList->size()); maxAudioFrameCounter += mixListStartSize - mixList->size(); } void AudioConferenceMixerImpl::GetAdditionalAudio( AudioFrameList* additionalFramesList) { WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, _id, "GetAdditionalAudio(additionalFramesList)"); // The GetAudioFrame() callback may result in the participant being removed // from additionalParticipantList_. If that happens it will invalidate any // iterators. Create a copy of the participants list such that the list of // participants can be traversed safely. MixerParticipantList additionalParticipantList; additionalParticipantList.insert(additionalParticipantList.begin(), _additionalParticipantList.begin(), _additionalParticipantList.end()); for (MixerParticipantList::iterator participant = additionalParticipantList.begin(); participant != additionalParticipantList.end(); ++participant) { AudioFrame* audioFrame = NULL; if(_audioFramePool->PopMemory(audioFrame) == -1) { WEBRTC_TRACE(kTraceMemory, kTraceAudioMixerServer, _id, "failed PopMemory() call"); assert(false); return; } audioFrame->sample_rate_hz_ = _outputFrequency; if((*participant)->GetAudioFrame(_id, *audioFrame) != 0) { WEBRTC_TRACE(kTraceWarning, kTraceAudioMixerServer, _id, "failed to GetAudioFrame() from participant"); _audioFramePool->PushMemory(audioFrame); continue; } if(audioFrame->samples_per_channel_ == 0) { // Empty frame. Don't use it. _audioFramePool->PushMemory(audioFrame); continue; } additionalFramesList->push_back(audioFrame); } } void AudioConferenceMixerImpl::UpdateMixedStatus( std::map& mixedParticipantsMap) { WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, _id, "UpdateMixedStatus(mixedParticipantsMap)"); assert(mixedParticipantsMap.size() <= kMaximumAmountOfMixedParticipants); // Loop through all participants. If they are in the mix map they // were mixed. for (MixerParticipantList::iterator participant = _participantList.begin(); participant != _participantList.end(); ++participant) { bool isMixed = false; for (std::map::iterator it = mixedParticipantsMap.begin(); it != mixedParticipantsMap.end(); ++it) { if (it->second == *participant) { isMixed = true; break; } } (*participant)->_mixHistory->SetIsMixed(isMixed); } } void AudioConferenceMixerImpl::ClearAudioFrameList( AudioFrameList* audioFrameList) { WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, _id, "ClearAudioFrameList(audioFrameList)"); for (AudioFrameList::iterator iter = audioFrameList->begin(); iter != audioFrameList->end(); ++iter) { _audioFramePool->PushMemory(*iter); } audioFrameList->clear(); } void AudioConferenceMixerImpl::UpdateVADPositiveParticipants( AudioFrameList* mixList) { WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, _id, "UpdateVADPositiveParticipants(mixList)"); for (AudioFrameList::iterator iter = mixList->begin(); iter != mixList->end(); ++iter) { CalculateEnergy(**iter); if((*iter)->vad_activity_ == AudioFrame::kVadActive) { _scratchVadPositiveParticipants[ _scratchVadPositiveParticipantsAmount].participant = (*iter)->id_; // TODO(andrew): to what should this be set? _scratchVadPositiveParticipants[ _scratchVadPositiveParticipantsAmount].level = 0; _scratchVadPositiveParticipantsAmount++; } } } bool AudioConferenceMixerImpl::IsParticipantInList( MixerParticipant& participant, MixerParticipantList* participantList) const { WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, _id, "IsParticipantInList(participant,participantList)"); for (MixerParticipantList::const_iterator iter = participantList->begin(); iter != participantList->end(); ++iter) { if(&participant == *iter) { return true; } } return false; } bool AudioConferenceMixerImpl::AddParticipantToList( MixerParticipant& participant, MixerParticipantList* participantList) { WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, _id, "AddParticipantToList(participant, participantList)"); participantList->push_back(&participant); // Make sure that the mixed status is correct for new MixerParticipant. participant._mixHistory->ResetMixedStatus(); return true; } bool AudioConferenceMixerImpl::RemoveParticipantFromList( MixerParticipant& participant, MixerParticipantList* participantList) { WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, _id, "RemoveParticipantFromList(participant, participantList)"); for (MixerParticipantList::iterator iter = participantList->begin(); iter != participantList->end(); ++iter) { if(*iter == &participant) { participantList->erase(iter); // Participant is no longer mixed, reset to default. participant._mixHistory->ResetMixedStatus(); return true; } } return false; } int32_t AudioConferenceMixerImpl::MixFromList( AudioFrame& mixedAudio, const AudioFrameList* audioFrameList) { WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, _id, "MixFromList(mixedAudio, audioFrameList)"); if(audioFrameList->empty()) return 0; uint32_t position = 0; if(_numMixedParticipants == 1) { // No mixing required here; skip the saturation protection. AudioFrame* audioFrame = audioFrameList->front(); mixedAudio.CopyFrom(*audioFrame); SetParticipantStatistics(&_scratchMixedParticipants[position], *audioFrame); return 0; } if (audioFrameList->size() == 1) { mixedAudio.timestamp_ = audioFrameList->front()->timestamp_; mixedAudio.elapsed_time_ms_ = audioFrameList->front()->elapsed_time_ms_; } else { // TODO(wu): Issue 3390. // Audio frame timestamp is only supported in one channel case. mixedAudio.timestamp_ = 0; mixedAudio.elapsed_time_ms_ = -1; } for (AudioFrameList::const_iterator iter = audioFrameList->begin(); iter != audioFrameList->end(); ++iter) { if(position >= kMaximumAmountOfMixedParticipants) { WEBRTC_TRACE( kTraceMemory, kTraceAudioMixerServer, _id, "Trying to mix more than max amount of mixed participants:%d!", kMaximumAmountOfMixedParticipants); // Assert and avoid crash assert(false); position = 0; } MixFrames(&mixedAudio, (*iter)); SetParticipantStatistics(&_scratchMixedParticipants[position], **iter); position++; } return 0; } // TODO(andrew): consolidate this function with MixFromList. int32_t AudioConferenceMixerImpl::MixAnonomouslyFromList( AudioFrame& mixedAudio, const AudioFrameList* audioFrameList) { WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, _id, "MixAnonomouslyFromList(mixedAudio, audioFrameList)"); if(audioFrameList->empty()) return 0; if(_numMixedParticipants == 1) { // No mixing required here; skip the saturation protection. AudioFrame* audioFrame = audioFrameList->front(); mixedAudio.CopyFrom(*audioFrame); return 0; } for (AudioFrameList::const_iterator iter = audioFrameList->begin(); iter != audioFrameList->end(); ++iter) { MixFrames(&mixedAudio, *iter); } return 0; } bool AudioConferenceMixerImpl::LimitMixedAudio(AudioFrame& mixedAudio) { if(_numMixedParticipants == 1) { return true; } // Smoothly limit the mixed frame. const int error = _limiter->ProcessStream(&mixedAudio); // And now we can safely restore the level. This procedure results in // some loss of resolution, deemed acceptable. // // It's possible to apply the gain in the AGC (with a target level of 0 dbFS // and compression gain of 6 dB). However, in the transition frame when this // is enabled (moving from one to two participants) it has the potential to // create discontinuities in the mixed frame. // // Instead we double the frame (with addition since left-shifting a // negative value is undefined). mixedAudio += mixedAudio; if(error != _limiter->kNoError) { WEBRTC_TRACE(kTraceError, kTraceAudioMixerServer, _id, "Error from AudioProcessing: %d", error); assert(false); return false; } return true; } } // namespace webrtc