diff options
Diffstat (limited to 'webrtc/video/video_quality_test.cc')
| -rw-r--r-- | webrtc/video/video_quality_test.cc | 572 |
1 files changed, 390 insertions, 182 deletions
diff --git a/webrtc/video/video_quality_test.cc b/webrtc/video/video_quality_test.cc index 0f45fa6632..08ae0a9cee 100644 --- a/webrtc/video/video_quality_test.cc +++ b/webrtc/video/video_quality_test.cc @@ -12,16 +12,20 @@ #include <algorithm> #include <deque> #include <map> +#include <sstream> +#include <string> #include <vector> #include "testing/gtest/include/gtest/gtest.h" #include "webrtc/base/checks.h" +#include "webrtc/base/event.h" #include "webrtc/base/format_macros.h" #include "webrtc/base/scoped_ptr.h" #include "webrtc/call.h" #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h" -#include "webrtc/modules/rtp_rtcp/interface/rtp_header_parser.h" +#include "webrtc/modules/rtp_rtcp/include/rtp_header_parser.h" +#include "webrtc/modules/rtp_rtcp/source/rtp_utility.h" #include "webrtc/system_wrappers/include/cpu_info.h" #include "webrtc/test/layer_filtering_transport.h" #include "webrtc/test/run_loop.h" @@ -43,18 +47,22 @@ class VideoAnalyzer : public PacketReceiver, public EncodedFrameObserver, public EncodingTimeObserver { public: - VideoAnalyzer(Transport* transport, + VideoAnalyzer(test::LayerFilteringTransport* transport, const std::string& test_label, double avg_psnr_threshold, double avg_ssim_threshold, int duration_frames, - FILE* graph_data_output_file) + FILE* graph_data_output_file, + const std::string& graph_title, + uint32_t ssrc_to_analyze) : input_(nullptr), transport_(transport), receiver_(nullptr), send_stream_(nullptr), test_label_(test_label), graph_data_output_file_(graph_data_output_file), + graph_title_(graph_title), + ssrc_to_analyze_(ssrc_to_analyze), frames_to_process_(duration_frames), frames_recorded_(0), frames_processed_(0), @@ -63,8 +71,9 @@ class VideoAnalyzer : public PacketReceiver, rtp_timestamp_delta_(0), avg_psnr_threshold_(avg_psnr_threshold), avg_ssim_threshold_(avg_ssim_threshold), - comparison_available_event_(EventWrapper::Create()), - done_(EventWrapper::Create()) { + stats_polling_thread_(&PollStatsThread, this, "StatsPoller"), + comparison_available_event_(false, false), + done_(false, false) { // Create thread pool for CPU-expensive PSNR/SSIM calculations. // Try to use about as many threads as cores, but leave kMinCoresLeft alone, @@ -85,20 +94,16 @@ class VideoAnalyzer : public PacketReceiver, } for (uint32_t i = 0; i < num_cores; ++i) { - rtc::scoped_ptr<ThreadWrapper> thread = - ThreadWrapper::CreateThread(&FrameComparisonThread, this, "Analyzer"); - EXPECT_TRUE(thread->Start()); - comparison_thread_pool_.push_back(thread.release()); + rtc::PlatformThread* thread = + new rtc::PlatformThread(&FrameComparisonThread, this, "Analyzer"); + thread->Start(); + comparison_thread_pool_.push_back(thread); } - - stats_polling_thread_ = - ThreadWrapper::CreateThread(&PollStatsThread, this, "StatsPoller"); - EXPECT_TRUE(stats_polling_thread_->Start()); } ~VideoAnalyzer() { - for (ThreadWrapper* thread : comparison_thread_pool_) { - EXPECT_TRUE(thread->Stop()); + for (rtc::PlatformThread* thread : comparison_thread_pool_) { + thread->Stop(); delete thread; } } @@ -109,9 +114,9 @@ class VideoAnalyzer : public PacketReceiver, const uint8_t* packet, size_t length, const PacketTime& packet_time) override { - rtc::scoped_ptr<RtpHeaderParser> parser(RtpHeaderParser::Create()); + RtpUtility::RtpHeaderParser parser(packet, length); RTPHeader header; - parser->Parse(packet, length, &header); + parser.Parse(&header); { rtc::CritScope lock(&crit_); recv_times_[header.timestamp - rtp_timestamp_delta_] = @@ -145,10 +150,13 @@ class VideoAnalyzer : public PacketReceiver, bool SendRtp(const uint8_t* packet, size_t length, const PacketOptions& options) override { - rtc::scoped_ptr<RtpHeaderParser> parser(RtpHeaderParser::Create()); + RtpUtility::RtpHeaderParser parser(packet, length); RTPHeader header; - parser->Parse(packet, length, &header); + parser.Parse(&header); + int64_t current_time = + Clock::GetRealTimeClock()->CurrentNtpInMilliseconds(); + bool result = transport_->SendRtp(packet, length, options); { rtc::CritScope lock(&crit_); if (rtp_timestamp_delta_ == 0) { @@ -156,13 +164,14 @@ class VideoAnalyzer : public PacketReceiver, first_send_frame_.Reset(); } uint32_t timestamp = header.timestamp - rtp_timestamp_delta_; - send_times_[timestamp] = - Clock::GetRealTimeClock()->CurrentNtpInMilliseconds(); - encoded_frame_sizes_[timestamp] += - length - (header.headerLength + header.paddingLength); + send_times_[timestamp] = current_time; + if (!transport_->DiscardedLastPacket() && + header.ssrc == ssrc_to_analyze_) { + encoded_frame_sizes_[timestamp] += + length - (header.headerLength + header.paddingLength); + } } - - return transport_->SendRtp(packet, length, options); + return result; } bool SendRtcp(const uint8_t* packet, size_t length) override { @@ -192,6 +201,11 @@ class VideoAnalyzer : public PacketReceiver, VideoFrame reference_frame = frames_.front(); frames_.pop_front(); assert(!reference_frame.IsZeroSize()); + if (send_timestamp == reference_frame.timestamp() - 1) { + // TODO(ivica): Make this work for > 2 streams. + // Look at rtp_sender.c:RTPSender::BuildRTPHeader. + ++send_timestamp; + } EXPECT_EQ(reference_frame.timestamp(), send_timestamp); assert(reference_frame.timestamp() == send_timestamp); @@ -207,11 +221,11 @@ class VideoAnalyzer : public PacketReceiver, // at time-out check if frames_processed is going up. If so, give it more // time, otherwise fail. Hopefully this will reduce test flakiness. + stats_polling_thread_.Start(); + int last_frames_processed = -1; - EventTypeWrapper eventType; int iteration = 0; - while ((eventType = done_->Wait(VideoQualityTest::kDefaultTimeoutMs)) != - kEventSignaled) { + while (!done_.Wait(VideoQualityTest::kDefaultTimeoutMs)) { int frames_processed; { rtc::CritScope crit(&comparison_lock_); @@ -240,12 +254,12 @@ class VideoAnalyzer : public PacketReceiver, // Signal stats polling thread if that is still waiting and stop it now, // since it uses the send_stream_ reference that might be reclaimed after // returning from this method. - done_->Set(); - EXPECT_TRUE(stats_polling_thread_->Stop()); + done_.Set(); + stats_polling_thread_.Stop(); } VideoCaptureInput* input_; - Transport* const transport_; + test::LayerFilteringTransport* const transport_; PacketReceiver* receiver_; VideoSendStream* send_stream_; @@ -320,8 +334,13 @@ class VideoAnalyzer : public PacketReceiver, int64_t recv_time_ms = recv_times_[reference.timestamp()]; recv_times_.erase(reference.timestamp()); - size_t encoded_size = encoded_frame_sizes_[reference.timestamp()]; - encoded_frame_sizes_.erase(reference.timestamp()); + // TODO(ivica): Make this work for > 2 streams. + auto it = encoded_frame_sizes_.find(reference.timestamp()); + if (it == encoded_frame_sizes_.end()) + it = encoded_frame_sizes_.find(reference.timestamp() - 1); + size_t encoded_size = it == encoded_frame_sizes_.end() ? 0 : it->second; + if (it != encoded_frame_sizes_.end()) + encoded_frame_sizes_.erase(it); VideoFrame reference_copy; VideoFrame render_copy; @@ -332,7 +351,7 @@ class VideoAnalyzer : public PacketReceiver, comparisons_.push_back(FrameComparison(reference_copy, render_copy, dropped, send_time_ms, recv_time_ms, render_time_ms, encoded_size)); - comparison_available_event_->Set(); + comparison_available_event_.Set(); } static bool PollStatsThread(void* obj) { @@ -340,15 +359,11 @@ class VideoAnalyzer : public PacketReceiver, } bool PollStats() { - switch (done_->Wait(kSendStatsPollingIntervalMs)) { - case kEventSignaled: - case kEventError: - done_->Set(); // Make sure main thread is also signaled. - return false; - case kEventTimeout: - break; - default: - RTC_NOTREACHED(); + if (done_.Wait(kSendStatsPollingIntervalMs)) { + // Set event again to make sure main thread is also signaled, then we're + // done. + done_.Set(); + return false; } VideoSendStream::Stats stats = send_stream_->GetStats(); @@ -377,9 +392,9 @@ class VideoAnalyzer : public PacketReceiver, if (!PopComparison(&comparison)) { // Wait until new comparison task is available, or test is done. // If done, wake up remaining threads waiting. - comparison_available_event_->Wait(1000); + comparison_available_event_.Wait(1000); if (AllFramesRecorded()) { - comparison_available_event_->Set(); + comparison_available_event_.Set(); return false; } return true; // Try again. @@ -391,8 +406,8 @@ class VideoAnalyzer : public PacketReceiver, PrintResults(); if (graph_data_output_file_) PrintSamplesToFile(); - done_->Set(); - comparison_available_event_->Set(); + done_.Set(); + comparison_available_event_.Set(); return false; } @@ -509,7 +524,7 @@ class VideoAnalyzer : public PacketReceiver, return A.input_time_ms < B.input_time_ms; }); - fprintf(out, "%s\n", test_label_.c_str()); + fprintf(out, "%s\n", graph_title_.c_str()); fprintf(out, "%" PRIuS "\n", samples_.size()); fprintf(out, "dropped " @@ -547,6 +562,8 @@ class VideoAnalyzer : public PacketReceiver, const std::string test_label_; FILE* const graph_data_output_file_; + const std::string graph_title_; + const uint32_t ssrc_to_analyze_; std::vector<Sample> samples_ GUARDED_BY(comparison_lock_); std::map<int64_t, int> samples_encode_time_ms_ GUARDED_BY(comparison_lock_); test::Statistics sender_time_ GUARDED_BY(comparison_lock_); @@ -579,104 +596,260 @@ class VideoAnalyzer : public PacketReceiver, const double avg_ssim_threshold_; rtc::CriticalSection comparison_lock_; - std::vector<ThreadWrapper*> comparison_thread_pool_; - rtc::scoped_ptr<ThreadWrapper> stats_polling_thread_; - const rtc::scoped_ptr<EventWrapper> comparison_available_event_; + std::vector<rtc::PlatformThread*> comparison_thread_pool_; + rtc::PlatformThread stats_polling_thread_; + rtc::Event comparison_available_event_; std::deque<FrameComparison> comparisons_ GUARDED_BY(comparison_lock_); - const rtc::scoped_ptr<EventWrapper> done_; + rtc::Event done_; }; VideoQualityTest::VideoQualityTest() : clock_(Clock::GetRealTimeClock()) {} -void VideoQualityTest::ValidateParams(const Params& params) { - RTC_CHECK_GE(params.common.max_bitrate_bps, params.common.target_bitrate_bps); - RTC_CHECK_GE(params.common.target_bitrate_bps, params.common.min_bitrate_bps); - RTC_CHECK_LT(params.common.tl_discard_threshold, - params.common.num_temporal_layers); +void VideoQualityTest::TestBody() {} + +std::string VideoQualityTest::GenerateGraphTitle() const { + std::stringstream ss; + ss << params_.common.codec; + ss << " (" << params_.common.target_bitrate_bps / 1000 << "kbps"; + ss << ", " << params_.common.fps << " FPS"; + if (params_.screenshare.scroll_duration) + ss << ", " << params_.screenshare.scroll_duration << "s scroll"; + if (params_.ss.streams.size() > 1) + ss << ", Stream #" << params_.ss.selected_stream; + if (params_.ss.num_spatial_layers > 1) + ss << ", Layer #" << params_.ss.selected_sl; + ss << ")"; + return ss.str(); } -void VideoQualityTest::TestBody() {} +void VideoQualityTest::CheckParams() { + // Add a default stream in none specified. + if (params_.ss.streams.empty()) + params_.ss.streams.push_back(VideoQualityTest::DefaultVideoStream(params_)); + if (params_.ss.num_spatial_layers == 0) + params_.ss.num_spatial_layers = 1; + + if (params_.pipe.loss_percent != 0 || + params_.pipe.queue_length_packets != 0) { + // Since LayerFilteringTransport changes the sequence numbers, we can't + // use that feature with pack loss, since the NACK request would end up + // retransmitting the wrong packets. + RTC_CHECK(params_.ss.selected_sl == -1 || + params_.ss.selected_sl == params_.ss.num_spatial_layers - 1); + RTC_CHECK(params_.common.selected_tl == -1 || + params_.common.selected_tl == + params_.common.num_temporal_layers - 1); + } + + // TODO(ivica): Should max_bitrate_bps == -1 represent inf max bitrate, as it + // does in some parts of the code? + RTC_CHECK_GE(params_.common.max_bitrate_bps, + params_.common.target_bitrate_bps); + RTC_CHECK_GE(params_.common.target_bitrate_bps, + params_.common.min_bitrate_bps); + RTC_CHECK_LT(params_.common.selected_tl, params_.common.num_temporal_layers); + RTC_CHECK_LT(params_.ss.selected_stream, params_.ss.streams.size()); + for (const VideoStream& stream : params_.ss.streams) { + RTC_CHECK_GE(stream.min_bitrate_bps, 0); + RTC_CHECK_GE(stream.target_bitrate_bps, stream.min_bitrate_bps); + RTC_CHECK_GE(stream.max_bitrate_bps, stream.target_bitrate_bps); + RTC_CHECK_EQ(static_cast<int>(stream.temporal_layer_thresholds_bps.size()), + params_.common.num_temporal_layers - 1); + } + // TODO(ivica): Should we check if the sum of all streams/layers is equal to + // the total bitrate? We anyway have to update them in the case bitrate + // estimator changes the total bitrates. + RTC_CHECK_GE(params_.ss.num_spatial_layers, 1); + RTC_CHECK_LE(params_.ss.selected_sl, params_.ss.num_spatial_layers); + RTC_CHECK(params_.ss.spatial_layers.empty() || + params_.ss.spatial_layers.size() == + static_cast<size_t>(params_.ss.num_spatial_layers)); + if (params_.common.codec == "VP8") { + RTC_CHECK_EQ(params_.ss.num_spatial_layers, 1); + } else if (params_.common.codec == "VP9") { + RTC_CHECK_EQ(params_.ss.streams.size(), 1u); + } +} + +// Static. +std::vector<int> VideoQualityTest::ParseCSV(const std::string& str) { + // Parse comma separated nonnegative integers, where some elements may be + // empty. The empty values are replaced with -1. + // E.g. "10,-20,,30,40" --> {10, 20, -1, 30,40} + // E.g. ",,10,,20," --> {-1, -1, 10, -1, 20, -1} + std::vector<int> result; + if (str.empty()) + return result; + + const char* p = str.c_str(); + int value = -1; + int pos; + while (*p) { + if (*p == ',') { + result.push_back(value); + value = -1; + ++p; + continue; + } + RTC_CHECK_EQ(sscanf(p, "%d%n", &value, &pos), 1) + << "Unexpected non-number value."; + p += pos; + } + result.push_back(value); + return result; +} + +// Static. +VideoStream VideoQualityTest::DefaultVideoStream(const Params& params) { + VideoStream stream; + stream.width = params.common.width; + stream.height = params.common.height; + stream.max_framerate = params.common.fps; + stream.min_bitrate_bps = params.common.min_bitrate_bps; + stream.target_bitrate_bps = params.common.target_bitrate_bps; + stream.max_bitrate_bps = params.common.max_bitrate_bps; + stream.max_qp = 52; + if (params.common.num_temporal_layers == 2) + stream.temporal_layer_thresholds_bps.push_back(stream.target_bitrate_bps); + return stream; +} -void VideoQualityTest::SetupFullStack(const Params& params, - Transport* send_transport, - Transport* recv_transport) { - if (params.logs) +// Static. +void VideoQualityTest::FillScalabilitySettings( + Params* params, + const std::vector<std::string>& stream_descriptors, + size_t selected_stream, + int num_spatial_layers, + int selected_sl, + const std::vector<std::string>& sl_descriptors) { + // Read VideoStream and SpatialLayer elements from a list of comma separated + // lists. To use a default value for an element, use -1 or leave empty. + // Validity checks performed in CheckParams. + + RTC_CHECK(params->ss.streams.empty()); + for (auto descriptor : stream_descriptors) { + if (descriptor.empty()) + continue; + VideoStream stream = VideoQualityTest::DefaultVideoStream(*params); + std::vector<int> v = VideoQualityTest::ParseCSV(descriptor); + if (v[0] != -1) + stream.width = static_cast<size_t>(v[0]); + if (v[1] != -1) + stream.height = static_cast<size_t>(v[1]); + if (v[2] != -1) + stream.max_framerate = v[2]; + if (v[3] != -1) + stream.min_bitrate_bps = v[3]; + if (v[4] != -1) + stream.target_bitrate_bps = v[4]; + if (v[5] != -1) + stream.max_bitrate_bps = v[5]; + if (v.size() > 6 && v[6] != -1) + stream.max_qp = v[6]; + if (v.size() > 7) { + stream.temporal_layer_thresholds_bps.clear(); + stream.temporal_layer_thresholds_bps.insert( + stream.temporal_layer_thresholds_bps.end(), v.begin() + 7, v.end()); + } else { + // Automatic TL thresholds for more than two layers not supported. + RTC_CHECK_LE(params->common.num_temporal_layers, 2); + } + params->ss.streams.push_back(stream); + } + params->ss.selected_stream = selected_stream; + + params->ss.num_spatial_layers = num_spatial_layers ? num_spatial_layers : 1; + params->ss.selected_sl = selected_sl; + RTC_CHECK(params->ss.spatial_layers.empty()); + for (auto descriptor : sl_descriptors) { + if (descriptor.empty()) + continue; + std::vector<int> v = VideoQualityTest::ParseCSV(descriptor); + RTC_CHECK_GT(v[2], 0); + + SpatialLayer layer; + layer.scaling_factor_num = v[0] == -1 ? 1 : v[0]; + layer.scaling_factor_den = v[1] == -1 ? 1 : v[1]; + layer.target_bitrate_bps = v[2]; + params->ss.spatial_layers.push_back(layer); + } +} + +void VideoQualityTest::SetupCommon(Transport* send_transport, + Transport* recv_transport) { + if (params_.logs) trace_to_stderr_.reset(new test::TraceToStderr); - CreateSendConfig(1, send_transport); + size_t num_streams = params_.ss.streams.size(); + CreateSendConfig(num_streams, 0, send_transport); int payload_type; - if (params.common.codec == "VP8") { + if (params_.common.codec == "VP8") { encoder_.reset(VideoEncoder::Create(VideoEncoder::kVp8)); payload_type = kPayloadTypeVP8; - } else if (params.common.codec == "VP9") { + } else if (params_.common.codec == "VP9") { encoder_.reset(VideoEncoder::Create(VideoEncoder::kVp9)); payload_type = kPayloadTypeVP9; } else { RTC_NOTREACHED() << "Codec not supported!"; return; } - send_config_.encoder_settings.encoder = encoder_.get(); - send_config_.encoder_settings.payload_name = params.common.codec; - send_config_.encoder_settings.payload_type = payload_type; - - send_config_.rtp.nack.rtp_history_ms = kNackRtpHistoryMs; - send_config_.rtp.rtx.ssrcs.push_back(kSendRtxSsrcs[0]); - send_config_.rtp.rtx.payload_type = kSendRtxPayloadType; - - send_config_.rtp.extensions.clear(); - if (params.common.send_side_bwe) { - send_config_.rtp.extensions.push_back( + video_send_config_.encoder_settings.encoder = encoder_.get(); + video_send_config_.encoder_settings.payload_name = params_.common.codec; + video_send_config_.encoder_settings.payload_type = payload_type; + video_send_config_.rtp.nack.rtp_history_ms = kNackRtpHistoryMs; + video_send_config_.rtp.rtx.payload_type = kSendRtxPayloadType; + for (size_t i = 0; i < num_streams; ++i) + video_send_config_.rtp.rtx.ssrcs.push_back(kSendRtxSsrcs[i]); + + video_send_config_.rtp.extensions.clear(); + if (params_.common.send_side_bwe) { + video_send_config_.rtp.extensions.push_back( RtpExtension(RtpExtension::kTransportSequenceNumber, test::kTransportSequenceNumberExtensionId)); } else { - send_config_.rtp.extensions.push_back(RtpExtension( + video_send_config_.rtp.extensions.push_back(RtpExtension( RtpExtension::kAbsSendTime, test::kAbsSendTimeExtensionId)); } - // Automatically fill out streams[0] with params. - VideoStream* stream = &encoder_config_.streams[0]; - stream->width = params.common.width; - stream->height = params.common.height; - stream->min_bitrate_bps = params.common.min_bitrate_bps; - stream->target_bitrate_bps = params.common.target_bitrate_bps; - stream->max_bitrate_bps = params.common.max_bitrate_bps; - stream->max_framerate = static_cast<int>(params.common.fps); - - stream->temporal_layer_thresholds_bps.clear(); - if (params.common.num_temporal_layers > 1) { - stream->temporal_layer_thresholds_bps.push_back(stream->target_bitrate_bps); - } + video_encoder_config_.min_transmit_bitrate_bps = + params_.common.min_transmit_bps; + video_encoder_config_.streams = params_.ss.streams; + video_encoder_config_.spatial_layers = params_.ss.spatial_layers; CreateMatchingReceiveConfigs(recv_transport); - receive_configs_[0].rtp.nack.rtp_history_ms = kNackRtpHistoryMs; - receive_configs_[0].rtp.rtx[kSendRtxPayloadType].ssrc = kSendRtxSsrcs[0]; - receive_configs_[0].rtp.rtx[kSendRtxPayloadType].payload_type = - kSendRtxPayloadType; - - encoder_config_.min_transmit_bitrate_bps = params.common.min_transmit_bps; + for (size_t i = 0; i < num_streams; ++i) { + video_receive_configs_[i].rtp.nack.rtp_history_ms = kNackRtpHistoryMs; + video_receive_configs_[i].rtp.rtx[kSendRtxPayloadType].ssrc = + kSendRtxSsrcs[i]; + video_receive_configs_[i].rtp.rtx[kSendRtxPayloadType].payload_type = + kSendRtxPayloadType; + video_receive_configs_[i].rtp.transport_cc = params_.common.send_side_bwe; + } } -void VideoQualityTest::SetupScreenshare(const Params& params) { - RTC_CHECK(params.screenshare.enabled); +void VideoQualityTest::SetupScreenshare() { + RTC_CHECK(params_.screenshare.enabled); // Fill out codec settings. - encoder_config_.content_type = VideoEncoderConfig::ContentType::kScreen; - if (params.common.codec == "VP8") { + video_encoder_config_.content_type = VideoEncoderConfig::ContentType::kScreen; + if (params_.common.codec == "VP8") { codec_settings_.VP8 = VideoEncoder::GetDefaultVp8Settings(); codec_settings_.VP8.denoisingOn = false; codec_settings_.VP8.frameDroppingOn = false; codec_settings_.VP8.numberOfTemporalLayers = - static_cast<unsigned char>(params.common.num_temporal_layers); - encoder_config_.encoder_specific_settings = &codec_settings_.VP8; - } else if (params.common.codec == "VP9") { + static_cast<unsigned char>(params_.common.num_temporal_layers); + video_encoder_config_.encoder_specific_settings = &codec_settings_.VP8; + } else if (params_.common.codec == "VP9") { codec_settings_.VP9 = VideoEncoder::GetDefaultVp9Settings(); codec_settings_.VP9.denoisingOn = false; codec_settings_.VP9.frameDroppingOn = false; codec_settings_.VP9.numberOfTemporalLayers = - static_cast<unsigned char>(params.common.num_temporal_layers); - encoder_config_.encoder_specific_settings = &codec_settings_.VP9; + static_cast<unsigned char>(params_.common.num_temporal_layers); + video_encoder_config_.encoder_specific_settings = &codec_settings_.VP9; + codec_settings_.VP9.numberOfSpatialLayers = + static_cast<unsigned char>(params_.ss.num_spatial_layers); } // Setup frame generator. @@ -688,71 +861,67 @@ void VideoQualityTest::SetupScreenshare(const Params& params) { slides.push_back(test::ResourcePath("photo_1850_1110", "yuv")); slides.push_back(test::ResourcePath("difficult_photo_1850_1110", "yuv")); - if (params.screenshare.scroll_duration == 0) { + if (params_.screenshare.scroll_duration == 0) { // Cycle image every slide_change_interval seconds. frame_generator_.reset(test::FrameGenerator::CreateFromYuvFile( slides, kWidth, kHeight, - params.screenshare.slide_change_interval * params.common.fps)); + params_.screenshare.slide_change_interval * params_.common.fps)); } else { - RTC_CHECK_LE(params.common.width, kWidth); - RTC_CHECK_LE(params.common.height, kHeight); - RTC_CHECK_GT(params.screenshare.slide_change_interval, 0); - const int kPauseDurationMs = (params.screenshare.slide_change_interval - - params.screenshare.scroll_duration) * 1000; - RTC_CHECK_LE(params.screenshare.scroll_duration, - params.screenshare.slide_change_interval); - - if (params.screenshare.scroll_duration) { - frame_generator_.reset( - test::FrameGenerator::CreateScrollingInputFromYuvFiles( - clock_, slides, kWidth, kHeight, params.common.width, - params.common.height, params.screenshare.scroll_duration * 1000, - kPauseDurationMs)); - } else { - frame_generator_.reset(test::FrameGenerator::CreateFromYuvFile( - slides, kWidth, kHeight, - params.screenshare.slide_change_interval * params.common.fps)); - } + RTC_CHECK_LE(params_.common.width, kWidth); + RTC_CHECK_LE(params_.common.height, kHeight); + RTC_CHECK_GT(params_.screenshare.slide_change_interval, 0); + const int kPauseDurationMs = (params_.screenshare.slide_change_interval - + params_.screenshare.scroll_duration) * + 1000; + RTC_CHECK_LE(params_.screenshare.scroll_duration, + params_.screenshare.slide_change_interval); + + frame_generator_.reset( + test::FrameGenerator::CreateScrollingInputFromYuvFiles( + clock_, slides, kWidth, kHeight, params_.common.width, + params_.common.height, params_.screenshare.scroll_duration * 1000, + kPauseDurationMs)); } } -void VideoQualityTest::CreateCapturer(const Params& params, - VideoCaptureInput* input) { - if (params.screenshare.enabled) { - test::FrameGeneratorCapturer *frame_generator_capturer = +void VideoQualityTest::CreateCapturer(VideoCaptureInput* input) { + if (params_.screenshare.enabled) { + test::FrameGeneratorCapturer* frame_generator_capturer = new test::FrameGeneratorCapturer( - clock_, input, frame_generator_.release(), params.common.fps); + clock_, input, frame_generator_.release(), params_.common.fps); EXPECT_TRUE(frame_generator_capturer->Init()); capturer_.reset(frame_generator_capturer); } else { - if (params.video.clip_name.empty()) { - capturer_.reset(test::VideoCapturer::Create( - input, params.common.width, params.common.height, params.common.fps, - clock_)); + if (params_.video.clip_name.empty()) { + capturer_.reset(test::VideoCapturer::Create(input, params_.common.width, + params_.common.height, + params_.common.fps, clock_)); } else { capturer_.reset(test::FrameGeneratorCapturer::CreateFromYuvFile( - input, test::ResourcePath(params.video.clip_name, "yuv"), - params.common.width, params.common.height, params.common.fps, + input, test::ResourcePath(params_.video.clip_name, "yuv"), + params_.common.width, params_.common.height, params_.common.fps, clock_)); ASSERT_TRUE(capturer_.get() != nullptr) - << "Could not create capturer for " << params.video.clip_name + << "Could not create capturer for " << params_.video.clip_name << ".yuv. Is this resource file present?"; } } } void VideoQualityTest::RunWithAnalyzer(const Params& params) { + params_ = params; + // TODO(ivica): Merge with RunWithRenderer and use a flag / argument to // differentiate between the analyzer and the renderer case. - ValidateParams(params); + CheckParams(); FILE* graph_data_output_file = nullptr; - if (!params.analyzer.graph_data_output_filename.empty()) { + if (!params_.analyzer.graph_data_output_filename.empty()) { graph_data_output_file = - fopen(params.analyzer.graph_data_output_filename.c_str(), "w"); + fopen(params_.analyzer.graph_data_output_filename.c_str(), "w"); RTC_CHECK(graph_data_output_file != nullptr) - << "Can't open the file " - << params.analyzer.graph_data_output_filename << "!"; + << "Can't open the file " << params_.analyzer.graph_data_output_filename + << "!"; } Call::Config call_config; @@ -761,37 +930,64 @@ void VideoQualityTest::RunWithAnalyzer(const Params& params) { test::LayerFilteringTransport send_transport( params.pipe, sender_call_.get(), kPayloadTypeVP8, kPayloadTypeVP9, - static_cast<uint8_t>(params.common.tl_discard_threshold), 0); + params.common.selected_tl, params_.ss.selected_sl); test::DirectTransport recv_transport(params.pipe, receiver_call_.get()); + std::string graph_title = params_.analyzer.graph_title; + if (graph_title.empty()) + graph_title = VideoQualityTest::GenerateGraphTitle(); + + // In the case of different resolutions, the functions calculating PSNR and + // SSIM return -1.0, instead of a positive value as usual. VideoAnalyzer + // aborts if the average psnr/ssim are below the given threshold, which is + // 0.0 by default. Setting the thresholds to -1.1 prevents the unnecessary + // abort. + VideoStream& selected_stream = params_.ss.streams[params_.ss.selected_stream]; + int selected_sl = params_.ss.selected_sl != -1 + ? params_.ss.selected_sl + : params_.ss.num_spatial_layers - 1; + bool disable_quality_check = + selected_stream.width != params_.common.width || + selected_stream.height != params_.common.height || + (!params_.ss.spatial_layers.empty() && + params_.ss.spatial_layers[selected_sl].scaling_factor_num != + params_.ss.spatial_layers[selected_sl].scaling_factor_den); + if (disable_quality_check) { + fprintf(stderr, + "Warning: Calculating PSNR and SSIM for downsized resolution " + "not implemented yet! Skipping PSNR and SSIM calculations!"); + } + VideoAnalyzer analyzer( - &send_transport, params.analyzer.test_label, - params.analyzer.avg_psnr_threshold, params.analyzer.avg_ssim_threshold, - params.analyzer.test_durations_secs * params.common.fps, - graph_data_output_file); + &send_transport, params_.analyzer.test_label, + disable_quality_check ? -1.1 : params_.analyzer.avg_psnr_threshold, + disable_quality_check ? -1.1 : params_.analyzer.avg_ssim_threshold, + params_.analyzer.test_durations_secs * params_.common.fps, + graph_data_output_file, graph_title, + kVideoSendSsrcs[params_.ss.selected_stream]); analyzer.SetReceiver(receiver_call_->Receiver()); send_transport.SetReceiver(&analyzer); recv_transport.SetReceiver(sender_call_->Receiver()); - SetupFullStack(params, &analyzer, &recv_transport); - send_config_.encoding_time_observer = &analyzer; - receive_configs_[0].renderer = &analyzer; - for (auto& config : receive_configs_) + SetupCommon(&analyzer, &recv_transport); + video_send_config_.encoding_time_observer = &analyzer; + video_receive_configs_[params_.ss.selected_stream].renderer = &analyzer; + for (auto& config : video_receive_configs_) config.pre_decode_callback = &analyzer; - if (params.screenshare.enabled) - SetupScreenshare(params); + if (params_.screenshare.enabled) + SetupScreenshare(); - CreateStreams(); - analyzer.input_ = send_stream_->Input(); - analyzer.send_stream_ = send_stream_; + CreateVideoStreams(); + analyzer.input_ = video_send_stream_->Input(); + analyzer.send_stream_ = video_send_stream_; - CreateCapturer(params, &analyzer); + CreateCapturer(&analyzer); - send_stream_->Start(); - for (size_t i = 0; i < receive_streams_.size(); ++i) - receive_streams_[i]->Start(); + video_send_stream_->Start(); + for (VideoReceiveStream* receive_stream : video_receive_streams_) + receive_stream->Start(); capturer_->Start(); analyzer.Wait(); @@ -800,9 +996,9 @@ void VideoQualityTest::RunWithAnalyzer(const Params& params) { recv_transport.StopSending(); capturer_->Stop(); - for (size_t i = 0; i < receive_streams_.size(); ++i) - receive_streams_[i]->Stop(); - send_stream_->Stop(); + for (VideoReceiveStream* receive_stream : video_receive_streams_) + receive_stream->Stop(); + video_send_stream_->Stop(); DestroyStreams(); @@ -811,53 +1007,65 @@ void VideoQualityTest::RunWithAnalyzer(const Params& params) { } void VideoQualityTest::RunWithVideoRenderer(const Params& params) { - ValidateParams(params); + params_ = params; + CheckParams(); rtc::scoped_ptr<test::VideoRenderer> local_preview( - test::VideoRenderer::Create("Local Preview", params.common.width, - params.common.height)); + test::VideoRenderer::Create("Local Preview", params_.common.width, + params_.common.height)); + size_t stream_id = params_.ss.selected_stream; + std::string title = "Loopback Video"; + if (params_.ss.streams.size() > 1) { + std::ostringstream s; + s << stream_id; + title += " - Stream #" + s.str(); + } + rtc::scoped_ptr<test::VideoRenderer> loopback_video( - test::VideoRenderer::Create("Loopback Video", params.common.width, - params.common.height)); + test::VideoRenderer::Create(title.c_str(), + params_.ss.streams[stream_id].width, + params_.ss.streams[stream_id].height)); // TODO(ivica): Remove bitrate_config and use the default Call::Config(), to // match the full stack tests. Call::Config call_config; - call_config.bitrate_config = params.common.call_bitrate_config; + call_config.bitrate_config = params_.common.call_bitrate_config; rtc::scoped_ptr<Call> call(Call::Create(call_config)); test::LayerFilteringTransport transport( params.pipe, call.get(), kPayloadTypeVP8, kPayloadTypeVP9, - static_cast<uint8_t>(params.common.tl_discard_threshold), 0); + params.common.selected_tl, params_.ss.selected_sl); // TODO(ivica): Use two calls to be able to merge with RunWithAnalyzer or at // least share as much code as possible. That way this test would also match // the full stack tests better. transport.SetReceiver(call->Receiver()); - SetupFullStack(params, &transport, &transport); - send_config_.local_renderer = local_preview.get(); - receive_configs_[0].renderer = loopback_video.get(); + SetupCommon(&transport, &transport); + + video_send_config_.local_renderer = local_preview.get(); + video_receive_configs_[stream_id].renderer = loopback_video.get(); - if (params.screenshare.enabled) - SetupScreenshare(params); + if (params_.screenshare.enabled) + SetupScreenshare(); - send_stream_ = call->CreateVideoSendStream(send_config_, encoder_config_); + video_send_stream_ = + call->CreateVideoSendStream(video_send_config_, video_encoder_config_); VideoReceiveStream* receive_stream = - call->CreateVideoReceiveStream(receive_configs_[0]); - CreateCapturer(params, send_stream_->Input()); + call->CreateVideoReceiveStream(video_receive_configs_[stream_id]); + CreateCapturer(video_send_stream_->Input()); receive_stream->Start(); - send_stream_->Start(); + video_send_stream_->Start(); capturer_->Start(); test::PressEnterToContinue(); capturer_->Stop(); - send_stream_->Stop(); + video_send_stream_->Stop(); receive_stream->Stop(); call->DestroyVideoReceiveStream(receive_stream); - call->DestroyVideoSendStream(send_stream_); + call->DestroyVideoSendStream(video_send_stream_); transport.StopSending(); } |