/* * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source * tree. An additional intellectual property rights grant can be found * in the file PATENTS. All contributing project authors may * be found in the AUTHORS file in the root of the source tree. */ #include "webrtc/video_engine/stream_synchronization.h" #include #include #include #include #include "webrtc/system_wrappers/interface/logging.h" namespace webrtc { static const int kMaxChangeMs = 80; static const int kMaxDeltaDelayMs = 10000; static const int kFilterLength = 4; // Minimum difference between audio and video to warrant a change. static const int kMinDeltaMs = 30; struct ViESyncDelay { ViESyncDelay() { extra_video_delay_ms = 0; last_video_delay_ms = 0; extra_audio_delay_ms = 0; last_audio_delay_ms = 0; network_delay = 120; } int extra_video_delay_ms; int last_video_delay_ms; int extra_audio_delay_ms; int last_audio_delay_ms; int network_delay; }; StreamSynchronization::StreamSynchronization(int audio_channel_id, int video_channel_id) : channel_delay_(new ViESyncDelay), audio_channel_id_(audio_channel_id), video_channel_id_(video_channel_id), base_target_delay_ms_(0), avg_diff_ms_(0) {} StreamSynchronization::~StreamSynchronization() { delete channel_delay_; } bool StreamSynchronization::ComputeRelativeDelay( const Measurements& audio_measurement, const Measurements& video_measurement, int* relative_delay_ms) { assert(relative_delay_ms); if (audio_measurement.rtcp.size() < 2 || video_measurement.rtcp.size() < 2) { // We need two RTCP SR reports per stream to do synchronization. return false; } int64_t audio_last_capture_time_ms; if (!RtpToNtpMs(audio_measurement.latest_timestamp, audio_measurement.rtcp, &audio_last_capture_time_ms)) { return false; } int64_t video_last_capture_time_ms; if (!RtpToNtpMs(video_measurement.latest_timestamp, video_measurement.rtcp, &video_last_capture_time_ms)) { return false; } if (video_last_capture_time_ms < 0) { return false; } // Positive diff means that video_measurement is behind audio_measurement. *relative_delay_ms = video_measurement.latest_receive_time_ms - audio_measurement.latest_receive_time_ms - (video_last_capture_time_ms - audio_last_capture_time_ms); if (*relative_delay_ms > kMaxDeltaDelayMs || *relative_delay_ms < -kMaxDeltaDelayMs) { return false; } return true; } bool StreamSynchronization::ComputeDelays(int relative_delay_ms, int current_audio_delay_ms, int* total_audio_delay_target_ms, int* total_video_delay_target_ms) { assert(total_audio_delay_target_ms && total_video_delay_target_ms); int current_video_delay_ms = *total_video_delay_target_ms; LOG(LS_VERBOSE) << "Audio delay: " << current_audio_delay_ms << ", network delay diff: " << channel_delay_->network_delay << " current diff: " << relative_delay_ms << " for channel " << audio_channel_id_; // Calculate the difference between the lowest possible video delay and // the current audio delay. int current_diff_ms = current_video_delay_ms - current_audio_delay_ms + relative_delay_ms; avg_diff_ms_ = ((kFilterLength - 1) * avg_diff_ms_ + current_diff_ms) / kFilterLength; if (abs(avg_diff_ms_) < kMinDeltaMs) { // Don't adjust if the diff is within our margin. return false; } // Make sure we don't move too fast. int diff_ms = avg_diff_ms_ / 2; diff_ms = std::min(diff_ms, kMaxChangeMs); diff_ms = std::max(diff_ms, -kMaxChangeMs); // Reset the average after a move to prevent overshooting reaction. avg_diff_ms_ = 0; if (diff_ms > 0) { // The minimum video delay is longer than the current audio delay. // We need to decrease extra video delay, or add extra audio delay. if (channel_delay_->extra_video_delay_ms > base_target_delay_ms_) { // We have extra delay added to ViE. Reduce this delay before adding // extra delay to VoE. channel_delay_->extra_video_delay_ms -= diff_ms; channel_delay_->extra_audio_delay_ms = base_target_delay_ms_; } else { // channel_delay_->extra_video_delay_ms > 0 // We have no extra video delay to remove, increase the audio delay. channel_delay_->extra_audio_delay_ms += diff_ms; channel_delay_->extra_video_delay_ms = base_target_delay_ms_; } } else { // if (diff_ms > 0) // The video delay is lower than the current audio delay. // We need to decrease extra audio delay, or add extra video delay. if (channel_delay_->extra_audio_delay_ms > base_target_delay_ms_) { // We have extra delay in VoiceEngine. // Start with decreasing the voice delay. // Note: diff_ms is negative; add the negative difference. channel_delay_->extra_audio_delay_ms += diff_ms; channel_delay_->extra_video_delay_ms = base_target_delay_ms_; } else { // channel_delay_->extra_audio_delay_ms > base_target_delay_ms_ // We have no extra delay in VoiceEngine, increase the video delay. // Note: diff_ms is negative; subtract the negative difference. channel_delay_->extra_video_delay_ms -= diff_ms; // X - (-Y) = X + Y. channel_delay_->extra_audio_delay_ms = base_target_delay_ms_; } } // Make sure that video is never below our target. channel_delay_->extra_video_delay_ms = std::max( channel_delay_->extra_video_delay_ms, base_target_delay_ms_); int new_video_delay_ms; if (channel_delay_->extra_video_delay_ms > base_target_delay_ms_) { new_video_delay_ms = channel_delay_->extra_video_delay_ms; } else { // No change to the extra video delay. We are changing audio and we only // allow to change one at the time. new_video_delay_ms = channel_delay_->last_video_delay_ms; } // Make sure that we don't go below the extra video delay. new_video_delay_ms = std::max( new_video_delay_ms, channel_delay_->extra_video_delay_ms); // Verify we don't go above the maximum allowed video delay. new_video_delay_ms = std::min(new_video_delay_ms, base_target_delay_ms_ + kMaxDeltaDelayMs); int new_audio_delay_ms; if (channel_delay_->extra_audio_delay_ms > base_target_delay_ms_) { new_audio_delay_ms = channel_delay_->extra_audio_delay_ms; } else { // No change to the audio delay. We are changing video and we only // allow to change one at the time. new_audio_delay_ms = channel_delay_->last_audio_delay_ms; } // Make sure that we don't go below the extra audio delay. new_audio_delay_ms = std::max( new_audio_delay_ms, channel_delay_->extra_audio_delay_ms); // Verify we don't go above the maximum allowed audio delay. new_audio_delay_ms = std::min(new_audio_delay_ms, base_target_delay_ms_ + kMaxDeltaDelayMs); // Remember our last audio and video delays. channel_delay_->last_video_delay_ms = new_video_delay_ms; channel_delay_->last_audio_delay_ms = new_audio_delay_ms; LOG(LS_VERBOSE) << "Sync video delay " << new_video_delay_ms << " and audio delay " << channel_delay_->extra_audio_delay_ms << " for video channel " << video_channel_id_ << " for audio channel " << audio_channel_id_; // Return values. *total_video_delay_target_ms = new_video_delay_ms; *total_audio_delay_target_ms = new_audio_delay_ms; return true; } void StreamSynchronization::SetTargetBufferingDelay(int target_delay_ms) { // Initial extra delay for audio (accounting for existing extra delay). channel_delay_->extra_audio_delay_ms += target_delay_ms - base_target_delay_ms_; channel_delay_->last_audio_delay_ms += target_delay_ms - base_target_delay_ms_; // The video delay is compared to the last value (and how much we can update // is limited by that as well). channel_delay_->last_video_delay_ms += target_delay_ms - base_target_delay_ms_; channel_delay_->extra_video_delay_ms += target_delay_ms - base_target_delay_ms_; // Video is already delayed by the desired amount. base_target_delay_ms_ = target_delay_ms; } } // namespace webrtc