diff options
Diffstat (limited to 'webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc')
| -rw-r--r-- | webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc | 968 |
1 files changed, 968 insertions, 0 deletions
diff --git a/webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc b/webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc new file mode 100644 index 0000000000..3d6aedb22a --- /dev/null +++ b/webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc @@ -0,0 +1,968 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include <math.h> + +#include "testing/gtest/include/gtest/gtest.h" + +#include "webrtc/modules/video_coding/codecs/interface/video_codec_interface.h" +#include "webrtc/modules/video_coding/codecs/test/packet_manipulator.h" +#include "webrtc/modules/video_coding/codecs/test/videoprocessor.h" +#include "webrtc/modules/video_coding/codecs/vp8/include/vp8.h" +#include "webrtc/modules/video_coding/codecs/vp9/include/vp9.h" +#include "webrtc/modules/video_coding/codecs/vp8/include/vp8_common_types.h" +#include "webrtc/modules/video_coding/main/interface/video_coding.h" +#include "webrtc/test/testsupport/fileutils.h" +#include "webrtc/test/testsupport/frame_reader.h" +#include "webrtc/test/testsupport/frame_writer.h" +#include "webrtc/test/testsupport/gtest_disable.h" +#include "webrtc/test/testsupport/metrics/video_metrics.h" +#include "webrtc/test/testsupport/packet_reader.h" +#include "webrtc/typedefs.h" + +namespace webrtc { + +// Maximum number of rate updates (i.e., calls to encoder to change bitrate +// and/or frame rate) for the current tests. +const int kMaxNumRateUpdates = 3; + +const int kPercTargetvsActualMismatch = 20; +const int kBaseKeyFrameInterval = 3000; + +// Codec and network settings. +struct CodecConfigPars { + VideoCodecType codec_type; + float packet_loss; + int num_temporal_layers; + int key_frame_interval; + bool error_concealment_on; + bool denoising_on; + bool frame_dropper_on; + bool spatial_resize_on; +}; + +// Quality metrics. +struct QualityMetrics { + double minimum_avg_psnr; + double minimum_min_psnr; + double minimum_avg_ssim; + double minimum_min_ssim; +}; + +// The sequence of bitrate and frame rate changes for the encoder, the frame +// number where the changes are made, and the total number of frames for the +// test. +struct RateProfile { + int target_bit_rate[kMaxNumRateUpdates]; + int input_frame_rate[kMaxNumRateUpdates]; + int frame_index_rate_update[kMaxNumRateUpdates + 1]; + int num_frames; +}; + +// Metrics for the rate control. The rate mismatch metrics are defined as +// percentages.|max_time_hit_target| is defined as number of frames, after a +// rate update is made to the encoder, for the encoder to reach within +// |kPercTargetvsActualMismatch| of new target rate. The metrics are defined for +// each rate update sequence. +struct RateControlMetrics { + int max_num_dropped_frames; + int max_key_frame_size_mismatch; + int max_delta_frame_size_mismatch; + int max_encoding_rate_mismatch; + int max_time_hit_target; + int num_spatial_resizes; + int num_key_frames; +}; + + +// Sequence used is foreman (CIF): may be better to use VGA for resize test. +const int kCIFWidth = 352; +const int kCIFHeight = 288; +const int kNbrFramesShort = 100; // Some tests are run for shorter sequence. +const int kNbrFramesLong = 299; + +// Parameters from VP8 wrapper, which control target size of key frames. +const float kInitialBufferSize = 0.5f; +const float kOptimalBufferSize = 0.6f; +const float kScaleKeyFrameSize = 0.5f; + +// Integration test for video processor. Encodes+decodes a clip and +// writes it to the output directory. After completion, quality metrics +// (PSNR and SSIM) and rate control metrics are computed to verify that the +// quality and encoder response is acceptable. The rate control tests allow us +// to verify the behavior for changing bitrate, changing frame rate, frame +// dropping/spatial resize, and temporal layers. The limits for the rate +// control metrics are set to be fairly conservative, so failure should only +// happen when some significant regression or breakdown occurs. +class VideoProcessorIntegrationTest: public testing::Test { + protected: + VideoEncoder* encoder_; + VideoDecoder* decoder_; + webrtc::test::FrameReader* frame_reader_; + webrtc::test::FrameWriter* frame_writer_; + webrtc::test::PacketReader packet_reader_; + webrtc::test::PacketManipulator* packet_manipulator_; + webrtc::test::Stats stats_; + webrtc::test::TestConfig config_; + VideoCodec codec_settings_; + webrtc::test::VideoProcessor* processor_; + + // Quantities defined/updated for every encoder rate update. + // Some quantities defined per temporal layer (at most 3 layers in this test). + int num_frames_per_update_[3]; + float sum_frame_size_mismatch_[3]; + float sum_encoded_frame_size_[3]; + float encoding_bitrate_[3]; + float per_frame_bandwidth_[3]; + float bit_rate_layer_[3]; + float frame_rate_layer_[3]; + int num_frames_total_; + float sum_encoded_frame_size_total_; + float encoding_bitrate_total_; + float perc_encoding_rate_mismatch_; + int num_frames_to_hit_target_; + bool encoding_rate_within_target_; + int bit_rate_; + int frame_rate_; + int layer_; + float target_size_key_frame_initial_; + float target_size_key_frame_; + float sum_key_frame_size_mismatch_; + int num_key_frames_; + float start_bitrate_; + + // Codec and network settings. + VideoCodecType codec_type_; + float packet_loss_; + int num_temporal_layers_; + int key_frame_interval_; + bool error_concealment_on_; + bool denoising_on_; + bool frame_dropper_on_; + bool spatial_resize_on_; + + + VideoProcessorIntegrationTest() {} + virtual ~VideoProcessorIntegrationTest() {} + + void SetUpCodecConfig() { + if (codec_type_ == kVideoCodecVP8) { + encoder_ = VP8Encoder::Create(); + decoder_ = VP8Decoder::Create(); + VideoCodingModule::Codec(kVideoCodecVP8, &codec_settings_); + } else if (codec_type_ == kVideoCodecVP9) { + encoder_ = VP9Encoder::Create(); + decoder_ = VP9Decoder::Create(); + VideoCodingModule::Codec(kVideoCodecVP9, &codec_settings_); + } + + // CIF is currently used for all tests below. + // Setup the TestConfig struct for processing of a clip in CIF resolution. + config_.input_filename = + webrtc::test::ResourcePath("foreman_cif", "yuv"); + + // Generate an output filename in a safe way. + config_.output_filename = webrtc::test::TempFilename( + webrtc::test::OutputPath(), "videoprocessor_integrationtest"); + config_.frame_length_in_bytes = CalcBufferSize(kI420, + kCIFWidth, kCIFHeight); + config_.verbose = false; + // Only allow encoder/decoder to use single core, for predictability. + config_.use_single_core = true; + // Key frame interval and packet loss are set for each test. + config_.keyframe_interval = key_frame_interval_; + config_.networking_config.packet_loss_probability = packet_loss_; + + // Configure codec settings. + config_.codec_settings = &codec_settings_; + config_.codec_settings->startBitrate = start_bitrate_; + config_.codec_settings->width = kCIFWidth; + config_.codec_settings->height = kCIFHeight; + + // These features may be set depending on the test. + switch (config_.codec_settings->codecType) { + case kVideoCodecVP8: + config_.codec_settings->codecSpecific.VP8.errorConcealmentOn = + error_concealment_on_; + config_.codec_settings->codecSpecific.VP8.denoisingOn = + denoising_on_; + config_.codec_settings->codecSpecific.VP8.numberOfTemporalLayers = + num_temporal_layers_; + config_.codec_settings->codecSpecific.VP8.frameDroppingOn = + frame_dropper_on_; + config_.codec_settings->codecSpecific.VP8.automaticResizeOn = + spatial_resize_on_; + config_.codec_settings->codecSpecific.VP8.keyFrameInterval = + kBaseKeyFrameInterval; + break; + case kVideoCodecVP9: + config_.codec_settings->codecSpecific.VP9.denoisingOn = + denoising_on_; + config_.codec_settings->codecSpecific.VP9.numberOfTemporalLayers = + num_temporal_layers_; + config_.codec_settings->codecSpecific.VP9.frameDroppingOn = + frame_dropper_on_; + config_.codec_settings->codecSpecific.VP9.automaticResizeOn = + spatial_resize_on_; + config_.codec_settings->codecSpecific.VP9.keyFrameInterval = + kBaseKeyFrameInterval; + break; + default: + assert(false); + break; + } + frame_reader_ = + new webrtc::test::FrameReaderImpl(config_.input_filename, + config_.frame_length_in_bytes); + frame_writer_ = + new webrtc::test::FrameWriterImpl(config_.output_filename, + config_.frame_length_in_bytes); + ASSERT_TRUE(frame_reader_->Init()); + ASSERT_TRUE(frame_writer_->Init()); + + packet_manipulator_ = new webrtc::test::PacketManipulatorImpl( + &packet_reader_, config_.networking_config, config_.verbose); + processor_ = new webrtc::test::VideoProcessorImpl(encoder_, decoder_, + frame_reader_, + frame_writer_, + packet_manipulator_, + config_, &stats_); + ASSERT_TRUE(processor_->Init()); + } + + // Reset quantities after each encoder update, update the target + // per-frame bandwidth. + void ResetRateControlMetrics(int num_frames) { + for (int i = 0; i < num_temporal_layers_; i++) { + num_frames_per_update_[i] = 0; + sum_frame_size_mismatch_[i] = 0.0f; + sum_encoded_frame_size_[i] = 0.0f; + encoding_bitrate_[i] = 0.0f; + // Update layer per-frame-bandwidth. + per_frame_bandwidth_[i] = static_cast<float>(bit_rate_layer_[i]) / + static_cast<float>(frame_rate_layer_[i]); + } + // Set maximum size of key frames, following setting in the VP8 wrapper. + float max_key_size = kScaleKeyFrameSize * kOptimalBufferSize * frame_rate_; + // We don't know exact target size of the key frames (except for first one), + // but the minimum in libvpx is ~|3 * per_frame_bandwidth| and maximum is + // set by |max_key_size_ * per_frame_bandwidth|. Take middle point/average + // as reference for mismatch. Note key frames always correspond to base + // layer frame in this test. + target_size_key_frame_ = 0.5 * (3 + max_key_size) * per_frame_bandwidth_[0]; + num_frames_total_ = 0; + sum_encoded_frame_size_total_ = 0.0f; + encoding_bitrate_total_ = 0.0f; + perc_encoding_rate_mismatch_ = 0.0f; + num_frames_to_hit_target_ = num_frames; + encoding_rate_within_target_ = false; + sum_key_frame_size_mismatch_ = 0.0; + num_key_frames_ = 0; + } + + // For every encoded frame, update the rate control metrics. + void UpdateRateControlMetrics(int frame_num, FrameType frame_type) { + float encoded_size_kbits = processor_->EncodedFrameSize() * 8.0f / 1000.0f; + // Update layer data. + // Update rate mismatch relative to per-frame bandwidth for delta frames. + if (frame_type == kVideoFrameDelta) { + // TODO(marpan): Should we count dropped (zero size) frames in mismatch? + sum_frame_size_mismatch_[layer_] += fabs(encoded_size_kbits - + per_frame_bandwidth_[layer_]) / + per_frame_bandwidth_[layer_]; + } else { + float target_size = (frame_num == 1) ? target_size_key_frame_initial_ : + target_size_key_frame_; + sum_key_frame_size_mismatch_ += fabs(encoded_size_kbits - target_size) / + target_size; + num_key_frames_ += 1; + } + sum_encoded_frame_size_[layer_] += encoded_size_kbits; + // Encoding bitrate per layer: from the start of the update/run to the + // current frame. + encoding_bitrate_[layer_] = sum_encoded_frame_size_[layer_] * + frame_rate_layer_[layer_] / + num_frames_per_update_[layer_]; + // Total encoding rate: from the start of the update/run to current frame. + sum_encoded_frame_size_total_ += encoded_size_kbits; + encoding_bitrate_total_ = sum_encoded_frame_size_total_ * frame_rate_ / + num_frames_total_; + perc_encoding_rate_mismatch_ = 100 * fabs(encoding_bitrate_total_ - + bit_rate_) / bit_rate_; + if (perc_encoding_rate_mismatch_ < kPercTargetvsActualMismatch && + !encoding_rate_within_target_) { + num_frames_to_hit_target_ = num_frames_total_; + encoding_rate_within_target_ = true; + } + } + + // Verify expected behavior of rate control and print out data. + void VerifyRateControl(int update_index, + int max_key_frame_size_mismatch, + int max_delta_frame_size_mismatch, + int max_encoding_rate_mismatch, + int max_time_hit_target, + int max_num_dropped_frames, + int num_spatial_resizes, + int num_key_frames) { + int num_dropped_frames = processor_->NumberDroppedFrames(); + int num_resize_actions = processor_->NumberSpatialResizes(); + printf("For update #: %d,\n " + " Target Bitrate: %d,\n" + " Encoding bitrate: %f,\n" + " Frame rate: %d \n", + update_index, bit_rate_, encoding_bitrate_total_, frame_rate_); + printf(" Number of frames to approach target rate = %d, \n" + " Number of dropped frames = %d, \n" + " Number of spatial resizes = %d, \n", + num_frames_to_hit_target_, num_dropped_frames, num_resize_actions); + EXPECT_LE(perc_encoding_rate_mismatch_, max_encoding_rate_mismatch); + if (num_key_frames_ > 0) { + int perc_key_frame_size_mismatch = 100 * sum_key_frame_size_mismatch_ / + num_key_frames_; + printf(" Number of Key frames: %d \n" + " Key frame rate mismatch: %d \n", + num_key_frames_, perc_key_frame_size_mismatch); + EXPECT_LE(perc_key_frame_size_mismatch, max_key_frame_size_mismatch); + } + printf("\n"); + printf("Rates statistics for Layer data \n"); + for (int i = 0; i < num_temporal_layers_ ; i++) { + printf("Layer #%d \n", i); + int perc_frame_size_mismatch = 100 * sum_frame_size_mismatch_[i] / + num_frames_per_update_[i]; + int perc_encoding_rate_mismatch = 100 * fabs(encoding_bitrate_[i] - + bit_rate_layer_[i]) / + bit_rate_layer_[i]; + printf(" Target Layer Bit rate: %f \n" + " Layer frame rate: %f, \n" + " Layer per frame bandwidth: %f, \n" + " Layer Encoding bit rate: %f, \n" + " Layer Percent frame size mismatch: %d, \n" + " Layer Percent encoding rate mismatch = %d, \n" + " Number of frame processed per layer = %d \n", + bit_rate_layer_[i], frame_rate_layer_[i], per_frame_bandwidth_[i], + encoding_bitrate_[i], perc_frame_size_mismatch, + perc_encoding_rate_mismatch, num_frames_per_update_[i]); + EXPECT_LE(perc_frame_size_mismatch, max_delta_frame_size_mismatch); + EXPECT_LE(perc_encoding_rate_mismatch, max_encoding_rate_mismatch); + } + printf("\n"); + EXPECT_LE(num_frames_to_hit_target_, max_time_hit_target); + EXPECT_LE(num_dropped_frames, max_num_dropped_frames); + EXPECT_EQ(num_resize_actions, num_spatial_resizes); + EXPECT_EQ(num_key_frames_, num_key_frames); + } + + // Layer index corresponding to frame number, for up to 3 layers. + void LayerIndexForFrame(int frame_number) { + if (num_temporal_layers_ == 1) { + layer_ = 0; + } else if (num_temporal_layers_ == 2) { + // layer 0: 0 2 4 ... + // layer 1: 1 3 + if (frame_number % 2 == 0) { + layer_ = 0; + } else { + layer_ = 1; + } + } else if (num_temporal_layers_ == 3) { + // layer 0: 0 4 8 ... + // layer 1: 2 6 + // layer 2: 1 3 5 7 + if (frame_number % 4 == 0) { + layer_ = 0; + } else if ((frame_number + 2) % 4 == 0) { + layer_ = 1; + } else if ((frame_number + 1) % 2 == 0) { + layer_ = 2; + } + } else { + assert(false); // Only up to 3 layers. + } + } + + // Set the bitrate and frame rate per layer, for up to 3 layers. + void SetLayerRates() { + assert(num_temporal_layers_<= 3); + for (int i = 0; i < num_temporal_layers_; i++) { + float bit_rate_ratio = + kVp8LayerRateAlloction[num_temporal_layers_ - 1][i]; + if (i > 0) { + float bit_rate_delta_ratio = kVp8LayerRateAlloction + [num_temporal_layers_ - 1][i] - + kVp8LayerRateAlloction[num_temporal_layers_ - 1][i - 1]; + bit_rate_layer_[i] = bit_rate_ * bit_rate_delta_ratio; + } else { + bit_rate_layer_[i] = bit_rate_ * bit_rate_ratio; + } + frame_rate_layer_[i] = frame_rate_ / static_cast<float>( + 1 << (num_temporal_layers_ - 1)); + } + if (num_temporal_layers_ == 3) { + frame_rate_layer_[2] = frame_rate_ / 2.0f; + } + } + + void TearDown() { + delete processor_; + delete packet_manipulator_; + delete frame_writer_; + delete frame_reader_; + delete decoder_; + delete encoder_; + } + + // Processes all frames in the clip and verifies the result. + void ProcessFramesAndVerify(QualityMetrics quality_metrics, + RateProfile rate_profile, + CodecConfigPars process, + RateControlMetrics* rc_metrics) { + // Codec/config settings. + codec_type_ = process.codec_type; + start_bitrate_ = rate_profile.target_bit_rate[0]; + packet_loss_ = process.packet_loss; + key_frame_interval_ = process.key_frame_interval; + num_temporal_layers_ = process.num_temporal_layers; + error_concealment_on_ = process.error_concealment_on; + denoising_on_ = process.denoising_on; + frame_dropper_on_ = process.frame_dropper_on; + spatial_resize_on_ = process.spatial_resize_on; + SetUpCodecConfig(); + // Update the layers and the codec with the initial rates. + bit_rate_ = rate_profile.target_bit_rate[0]; + frame_rate_ = rate_profile.input_frame_rate[0]; + SetLayerRates(); + // Set the initial target size for key frame. + target_size_key_frame_initial_ = 0.5 * kInitialBufferSize * + bit_rate_layer_[0]; + processor_->SetRates(bit_rate_, frame_rate_); + // Process each frame, up to |num_frames|. + int num_frames = rate_profile.num_frames; + int update_index = 0; + ResetRateControlMetrics( + rate_profile.frame_index_rate_update[update_index + 1]); + int frame_number = 0; + FrameType frame_type = kVideoFrameDelta; + while (processor_->ProcessFrame(frame_number) && + frame_number < num_frames) { + // Get the layer index for the frame |frame_number|. + LayerIndexForFrame(frame_number); + // Get the frame_type. + frame_type = processor_->EncodedFrameType(); + // Counter for whole sequence run. + ++frame_number; + // Counters for each rate update. + ++num_frames_per_update_[layer_]; + ++num_frames_total_; + UpdateRateControlMetrics(frame_number, frame_type); + // If we hit another/next update, verify stats for current state and + // update layers and codec with new rates. + if (frame_number == + rate_profile.frame_index_rate_update[update_index + 1]) { + VerifyRateControl( + update_index, + rc_metrics[update_index].max_key_frame_size_mismatch, + rc_metrics[update_index].max_delta_frame_size_mismatch, + rc_metrics[update_index].max_encoding_rate_mismatch, + rc_metrics[update_index].max_time_hit_target, + rc_metrics[update_index].max_num_dropped_frames, + rc_metrics[update_index].num_spatial_resizes, + rc_metrics[update_index].num_key_frames); + // Update layer rates and the codec with new rates. + ++update_index; + bit_rate_ = rate_profile.target_bit_rate[update_index]; + frame_rate_ = rate_profile.input_frame_rate[update_index]; + SetLayerRates(); + ResetRateControlMetrics(rate_profile. + frame_index_rate_update[update_index + 1]); + processor_->SetRates(bit_rate_, frame_rate_); + } + } + VerifyRateControl( + update_index, + rc_metrics[update_index].max_key_frame_size_mismatch, + rc_metrics[update_index].max_delta_frame_size_mismatch, + rc_metrics[update_index].max_encoding_rate_mismatch, + rc_metrics[update_index].max_time_hit_target, + rc_metrics[update_index].max_num_dropped_frames, + rc_metrics[update_index].num_spatial_resizes, + rc_metrics[update_index].num_key_frames); + EXPECT_EQ(num_frames, frame_number); + EXPECT_EQ(num_frames + 1, static_cast<int>(stats_.stats_.size())); + + // Release encoder and decoder to make sure they have finished processing: + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Release()); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, decoder_->Release()); + // Close the files before we start using them for SSIM/PSNR calculations. + frame_reader_->Close(); + frame_writer_->Close(); + + // TODO(marpan): should compute these quality metrics per SetRates update. + webrtc::test::QualityMetricsResult psnr_result, ssim_result; + EXPECT_EQ(0, webrtc::test::I420MetricsFromFiles( + config_.input_filename.c_str(), + config_.output_filename.c_str(), + config_.codec_settings->width, + config_.codec_settings->height, + &psnr_result, + &ssim_result)); + printf("PSNR avg: %f, min: %f SSIM avg: %f, min: %f\n", + psnr_result.average, psnr_result.min, + ssim_result.average, ssim_result.min); + stats_.PrintSummary(); + EXPECT_GT(psnr_result.average, quality_metrics.minimum_avg_psnr); + EXPECT_GT(psnr_result.min, quality_metrics.minimum_min_psnr); + EXPECT_GT(ssim_result.average, quality_metrics.minimum_avg_ssim); + EXPECT_GT(ssim_result.min, quality_metrics.minimum_min_ssim); + if (!remove(config_.output_filename.c_str())) { + fprintf(stderr, "Failed to remove temporary file!"); + } + } +}; + +void SetRateProfilePars(RateProfile* rate_profile, + int update_index, + int bit_rate, + int frame_rate, + int frame_index_rate_update) { + rate_profile->target_bit_rate[update_index] = bit_rate; + rate_profile->input_frame_rate[update_index] = frame_rate; + rate_profile->frame_index_rate_update[update_index] = frame_index_rate_update; +} + +void SetCodecParameters(CodecConfigPars* process_settings, + VideoCodecType codec_type, + float packet_loss, + int key_frame_interval, + int num_temporal_layers, + bool error_concealment_on, + bool denoising_on, + bool frame_dropper_on, + bool spatial_resize_on) { + process_settings->codec_type = codec_type; + process_settings->packet_loss = packet_loss; + process_settings->key_frame_interval = key_frame_interval; + process_settings->num_temporal_layers = num_temporal_layers, + process_settings->error_concealment_on = error_concealment_on; + process_settings->denoising_on = denoising_on; + process_settings->frame_dropper_on = frame_dropper_on; + process_settings->spatial_resize_on = spatial_resize_on; +} + +void SetQualityMetrics(QualityMetrics* quality_metrics, + double minimum_avg_psnr, + double minimum_min_psnr, + double minimum_avg_ssim, + double minimum_min_ssim) { + quality_metrics->minimum_avg_psnr = minimum_avg_psnr; + quality_metrics->minimum_min_psnr = minimum_min_psnr; + quality_metrics->minimum_avg_ssim = minimum_avg_ssim; + quality_metrics->minimum_min_ssim = minimum_min_ssim; +} + +void SetRateControlMetrics(RateControlMetrics* rc_metrics, + int update_index, + int max_num_dropped_frames, + int max_key_frame_size_mismatch, + int max_delta_frame_size_mismatch, + int max_encoding_rate_mismatch, + int max_time_hit_target, + int num_spatial_resizes, + int num_key_frames) { + rc_metrics[update_index].max_num_dropped_frames = max_num_dropped_frames; + rc_metrics[update_index].max_key_frame_size_mismatch = + max_key_frame_size_mismatch; + rc_metrics[update_index].max_delta_frame_size_mismatch = + max_delta_frame_size_mismatch; + rc_metrics[update_index].max_encoding_rate_mismatch = + max_encoding_rate_mismatch; + rc_metrics[update_index].max_time_hit_target = max_time_hit_target; + rc_metrics[update_index].num_spatial_resizes = num_spatial_resizes; + rc_metrics[update_index].num_key_frames = num_key_frames; +} + +// VP9: Run with no packet loss and fixed bitrate. Quality should be very high. +// One key frame (first frame only) in sequence. Setting |key_frame_interval| +// to -1 below means no periodic key frames in test. +TEST_F(VideoProcessorIntegrationTest, Process0PercentPacketLossVP9) { + // Bitrate and frame rate profile. + RateProfile rate_profile; + SetRateProfilePars(&rate_profile, 0, 500, 30, 0); + rate_profile.frame_index_rate_update[1] = kNbrFramesShort + 1; + rate_profile.num_frames = kNbrFramesShort; + // Codec/network settings. + CodecConfigPars process_settings; + SetCodecParameters(&process_settings, kVideoCodecVP9, 0.0f, -1, 1, false, + false, true, false); + // Metrics for expected quality. + QualityMetrics quality_metrics; + SetQualityMetrics(&quality_metrics, 37.0, 36.0, 0.93, 0.92); + // Metrics for rate control. + RateControlMetrics rc_metrics[1]; + SetRateControlMetrics(rc_metrics, 0, 0, 40, 20, 10, 20, 0, 1); + ProcessFramesAndVerify(quality_metrics, + rate_profile, + process_settings, + rc_metrics); +} + +// VP9: Run with 5% packet loss and fixed bitrate. Quality should be a bit +// lower. One key frame (first frame only) in sequence. +TEST_F(VideoProcessorIntegrationTest, Process5PercentPacketLossVP9) { + // Bitrate and frame rate profile. + RateProfile rate_profile; + SetRateProfilePars(&rate_profile, 0, 500, 30, 0); + rate_profile.frame_index_rate_update[1] = kNbrFramesShort + 1; + rate_profile.num_frames = kNbrFramesShort; + // Codec/network settings. + CodecConfigPars process_settings; + SetCodecParameters(&process_settings, kVideoCodecVP9, 0.05f, -1, 1, false, + false, true, false); + // Metrics for expected quality. + QualityMetrics quality_metrics; + SetQualityMetrics(&quality_metrics, 17.0, 14.0, 0.45, 0.36); + // Metrics for rate control. + RateControlMetrics rc_metrics[1]; + SetRateControlMetrics(rc_metrics, 0, 0, 40, 20, 10, 20, 0, 1); + ProcessFramesAndVerify(quality_metrics, + rate_profile, + process_settings, + rc_metrics); +} + + +// VP9: Run with no packet loss, with varying bitrate (3 rate updates): +// low to high to medium. Check that quality and encoder response to the new +// target rate/per-frame bandwidth (for each rate update) is within limits. +// One key frame (first frame only) in sequence. +TEST_F(VideoProcessorIntegrationTest, ProcessNoLossChangeBitRateVP9) { + // Bitrate and frame rate profile. + RateProfile rate_profile; + SetRateProfilePars(&rate_profile, 0, 200, 30, 0); + SetRateProfilePars(&rate_profile, 1, 700, 30, 100); + SetRateProfilePars(&rate_profile, 2, 500, 30, 200); + rate_profile.frame_index_rate_update[3] = kNbrFramesLong + 1; + rate_profile.num_frames = kNbrFramesLong; + // Codec/network settings. + CodecConfigPars process_settings; + SetCodecParameters(&process_settings, kVideoCodecVP9, 0.0f, -1, 1, false, + false, true, false); + // Metrics for expected quality. + QualityMetrics quality_metrics; + SetQualityMetrics(&quality_metrics, 35.9, 30.0, 0.90, 0.85); + // Metrics for rate control. + RateControlMetrics rc_metrics[3]; + SetRateControlMetrics(rc_metrics, 0, 0, 30, 20, 20, 30, 0, 1); + SetRateControlMetrics(rc_metrics, 1, 2, 0, 20, 20, 60, 0, 0); + SetRateControlMetrics(rc_metrics, 2, 0, 0, 25, 20, 40, 0, 0); + ProcessFramesAndVerify(quality_metrics, + rate_profile, + process_settings, + rc_metrics); +} + +// VP9: Run with no packet loss, with an update (decrease) in frame rate. +// Lower frame rate means higher per-frame-bandwidth, so easier to encode. +// At the low bitrate in this test, this means better rate control after the +// update(s) to lower frame rate. So expect less frame drops, and max values +// for the rate control metrics can be lower. One key frame (first frame only). +// Note: quality after update should be higher but we currently compute quality +// metrics averaged over whole sequence run. +TEST_F(VideoProcessorIntegrationTest, + ProcessNoLossChangeFrameRateFrameDropVP9) { + config_.networking_config.packet_loss_probability = 0; + // Bitrate and frame rate profile. + RateProfile rate_profile; + SetRateProfilePars(&rate_profile, 0, 100, 24, 0); + SetRateProfilePars(&rate_profile, 1, 100, 15, 100); + SetRateProfilePars(&rate_profile, 2, 100, 10, 200); + rate_profile.frame_index_rate_update[3] = kNbrFramesLong + 1; + rate_profile.num_frames = kNbrFramesLong; + // Codec/network settings. + CodecConfigPars process_settings; + SetCodecParameters(&process_settings, kVideoCodecVP9, 0.0f, -1, 1, false, + false, true, false); + // Metrics for expected quality. + QualityMetrics quality_metrics; + SetQualityMetrics(&quality_metrics, 31.5, 18.0, 0.80, 0.44); + // Metrics for rate control. + RateControlMetrics rc_metrics[3]; + SetRateControlMetrics(rc_metrics, 0, 35, 50, 70, 15, 45, 0, 1); + SetRateControlMetrics(rc_metrics, 1, 10, 0, 40, 10, 30, 0, 0); + SetRateControlMetrics(rc_metrics, 2, 5, 0, 30, 5, 20, 0, 0); + ProcessFramesAndVerify(quality_metrics, + rate_profile, + process_settings, + rc_metrics); +} + +// VP9: Run with no packet loss and denoiser on. One key frame (first frame). +TEST_F(VideoProcessorIntegrationTest, ProcessNoLossDenoiserOnVP9) { + // Bitrate and frame rate profile. + RateProfile rate_profile; + SetRateProfilePars(&rate_profile, 0, 500, 30, 0); + rate_profile.frame_index_rate_update[1] = kNbrFramesShort + 1; + rate_profile.num_frames = kNbrFramesShort; + // Codec/network settings. + CodecConfigPars process_settings; + SetCodecParameters(&process_settings, kVideoCodecVP9, 0.0f, -1, 1, false, + true, true, false); + // Metrics for expected quality. + QualityMetrics quality_metrics; + SetQualityMetrics(&quality_metrics, 36.8, 35.8, 0.92, 0.91); + // Metrics for rate control. + RateControlMetrics rc_metrics[1]; + SetRateControlMetrics(rc_metrics, 0, 0, 40, 20, 10, 20, 0, 1); + ProcessFramesAndVerify(quality_metrics, + rate_profile, + process_settings, + rc_metrics); +} + +// Run with no packet loss, at low bitrate. +// spatial_resize is on, and for this low bitrate expect two resizes during the +// sequence; first resize is 3/4, second is 1/2 (from original). +// Resize happens on delta frame. Expect only one key frame (first frame). +// Disable for msan, see +// https://code.google.com/p/webrtc/issues/detail?id=5110 for details. +#if !defined(MEMORY_SANITIZER) +TEST_F(VideoProcessorIntegrationTest, ProcessNoLossSpatialResizeFrameDropVP9) { + config_.networking_config.packet_loss_probability = 0; + // Bitrate and frame rate profile. + RateProfile rate_profile; + SetRateProfilePars(&rate_profile, 0, 50, 30, 0); + rate_profile.frame_index_rate_update[1] = kNbrFramesLong + 1; + rate_profile.num_frames = kNbrFramesLong; + // Codec/network settings. + CodecConfigPars process_settings; + SetCodecParameters(&process_settings, kVideoCodecVP9, 0.0f, -1, + 1, false, false, true, true); + // Metrics for expected quality. + QualityMetrics quality_metrics; + SetQualityMetrics(&quality_metrics, 25.0, 13.0, 0.70, 0.40); + // Metrics for rate control. + RateControlMetrics rc_metrics[1]; + SetRateControlMetrics(rc_metrics, 0, 180, 70, 130, 15, 80, 2, 1); + ProcessFramesAndVerify(quality_metrics, + rate_profile, + process_settings, + rc_metrics); +} +#endif + +// TODO(marpan): Add temporal layer test for VP9, once changes are in +// vp9 wrapper for this. + +// VP8: Run with no packet loss and fixed bitrate. Quality should be very high. +// One key frame (first frame only) in sequence. Setting |key_frame_interval| +// to -1 below means no periodic key frames in test. +TEST_F(VideoProcessorIntegrationTest, ProcessZeroPacketLoss) { + // Bitrate and frame rate profile. + RateProfile rate_profile; + SetRateProfilePars(&rate_profile, 0, 500, 30, 0); + rate_profile.frame_index_rate_update[1] = kNbrFramesShort + 1; + rate_profile.num_frames = kNbrFramesShort; + // Codec/network settings. + CodecConfigPars process_settings; + SetCodecParameters(&process_settings, kVideoCodecVP8, 0.0f, -1, 1, false, + true, true, false); + // Metrics for expected quality. + QualityMetrics quality_metrics; + SetQualityMetrics(&quality_metrics, 34.95, 33.0, 0.90, 0.89); + // Metrics for rate control. + RateControlMetrics rc_metrics[1]; + SetRateControlMetrics(rc_metrics, 0, 0, 40, 20, 10, 15, 0, 1); + ProcessFramesAndVerify(quality_metrics, + rate_profile, + process_settings, + rc_metrics); +} + +// VP8: Run with 5% packet loss and fixed bitrate. Quality should be a bit +// lower. One key frame (first frame only) in sequence. +TEST_F(VideoProcessorIntegrationTest, Process5PercentPacketLoss) { + // Bitrate and frame rate profile. + RateProfile rate_profile; + SetRateProfilePars(&rate_profile, 0, 500, 30, 0); + rate_profile.frame_index_rate_update[1] = kNbrFramesShort + 1; + rate_profile.num_frames = kNbrFramesShort; + // Codec/network settings. + CodecConfigPars process_settings; + SetCodecParameters(&process_settings, kVideoCodecVP8, 0.05f, -1, 1, false, + true, true, false); + // Metrics for expected quality. + QualityMetrics quality_metrics; + SetQualityMetrics(&quality_metrics, 20.0, 16.0, 0.60, 0.40); + // Metrics for rate control. + RateControlMetrics rc_metrics[1]; + SetRateControlMetrics(rc_metrics, 0, 0, 40, 20, 10, 15, 0, 1); + ProcessFramesAndVerify(quality_metrics, + rate_profile, + process_settings, + rc_metrics); +} + +// VP8: Run with 10% packet loss and fixed bitrate. Quality should be lower. +// One key frame (first frame only) in sequence. +TEST_F(VideoProcessorIntegrationTest, Process10PercentPacketLoss) { + // Bitrate and frame rate profile. + RateProfile rate_profile; + SetRateProfilePars(&rate_profile, 0, 500, 30, 0); + rate_profile.frame_index_rate_update[1] = kNbrFramesShort + 1; + rate_profile.num_frames = kNbrFramesShort; + // Codec/network settings. + CodecConfigPars process_settings; + SetCodecParameters(&process_settings, kVideoCodecVP8, 0.1f, -1, 1, false, + true, true, false); + // Metrics for expected quality. + QualityMetrics quality_metrics; + SetQualityMetrics(&quality_metrics, 19.0, 16.0, 0.50, 0.35); + // Metrics for rate control. + RateControlMetrics rc_metrics[1]; + SetRateControlMetrics(rc_metrics, 0, 0, 40, 20, 10, 15, 0, 1); + ProcessFramesAndVerify(quality_metrics, + rate_profile, + process_settings, + rc_metrics); +} + +// The tests below are currently disabled for Android. For ARM, the encoder +// uses |cpu_speed| = 12, as opposed to default |cpu_speed| <= 6 for x86, +// which leads to significantly different quality. The quality and rate control +// settings in the tests below are defined for encoder speed setting +// |cpu_speed| <= ~6. A number of settings would need to be significantly +// modified for the |cpu_speed| = 12 case. For now, keep the tests below +// disabled on Android. Some quality parameter in the above test has been +// adjusted to also pass for |cpu_speed| <= 12. + +// VP8: Run with no packet loss, with varying bitrate (3 rate updates): +// low to high to medium. Check that quality and encoder response to the new +// target rate/per-frame bandwidth (for each rate update) is within limits. +// One key frame (first frame only) in sequence. +TEST_F(VideoProcessorIntegrationTest, + DISABLED_ON_ANDROID(ProcessNoLossChangeBitRateVP8)) { + // Bitrate and frame rate profile. + RateProfile rate_profile; + SetRateProfilePars(&rate_profile, 0, 200, 30, 0); + SetRateProfilePars(&rate_profile, 1, 800, 30, 100); + SetRateProfilePars(&rate_profile, 2, 500, 30, 200); + rate_profile.frame_index_rate_update[3] = kNbrFramesLong + 1; + rate_profile.num_frames = kNbrFramesLong; + // Codec/network settings. + CodecConfigPars process_settings; + SetCodecParameters(&process_settings, kVideoCodecVP8, 0.0f, -1, 1, false, + true, true, false); + // Metrics for expected quality. + QualityMetrics quality_metrics; + SetQualityMetrics(&quality_metrics, 34.0, 32.0, 0.85, 0.80); + // Metrics for rate control. + RateControlMetrics rc_metrics[3]; + SetRateControlMetrics(rc_metrics, 0, 0, 45, 20, 10, 15, 0, 1); + SetRateControlMetrics(rc_metrics, 1, 0, 0, 25, 20, 10, 0, 0); + SetRateControlMetrics(rc_metrics, 2, 0, 0, 25, 15, 10, 0, 0); + ProcessFramesAndVerify(quality_metrics, + rate_profile, + process_settings, + rc_metrics); +} + +// VP8: Run with no packet loss, with an update (decrease) in frame rate. +// Lower frame rate means higher per-frame-bandwidth, so easier to encode. +// At the bitrate in this test, this means better rate control after the +// update(s) to lower frame rate. So expect less frame drops, and max values +// for the rate control metrics can be lower. One key frame (first frame only). +// Note: quality after update should be higher but we currently compute quality +// metrics averaged over whole sequence run. +TEST_F(VideoProcessorIntegrationTest, + DISABLED_ON_ANDROID(ProcessNoLossChangeFrameRateFrameDropVP8)) { + config_.networking_config.packet_loss_probability = 0; + // Bitrate and frame rate profile. + RateProfile rate_profile; + SetRateProfilePars(&rate_profile, 0, 80, 24, 0); + SetRateProfilePars(&rate_profile, 1, 80, 15, 100); + SetRateProfilePars(&rate_profile, 2, 80, 10, 200); + rate_profile.frame_index_rate_update[3] = kNbrFramesLong + 1; + rate_profile.num_frames = kNbrFramesLong; + // Codec/network settings. + CodecConfigPars process_settings; + SetCodecParameters(&process_settings, kVideoCodecVP8, 0.0f, -1, 1, false, + true, true, false); + // Metrics for expected quality. + QualityMetrics quality_metrics; + SetQualityMetrics(&quality_metrics, 31.0, 22.0, 0.80, 0.65); + // Metrics for rate control. + RateControlMetrics rc_metrics[3]; + SetRateControlMetrics(rc_metrics, 0, 40, 20, 75, 15, 60, 0, 1); + SetRateControlMetrics(rc_metrics, 1, 10, 0, 25, 10, 35, 0, 0); + SetRateControlMetrics(rc_metrics, 2, 0, 0, 20, 10, 15, 0, 0); + ProcessFramesAndVerify(quality_metrics, + rate_profile, + process_settings, + rc_metrics); +} + +// Run with no packet loss, at low bitrate. During this time we should've +// resized once. Expect 2 key frames generated (first and one for resize). +TEST_F(VideoProcessorIntegrationTest, + DISABLED_ON_ANDROID(ProcessNoLossSpatialResizeFrameDropVP8)) { + config_.networking_config.packet_loss_probability = 0; + // Bitrate and frame rate profile. + RateProfile rate_profile; + SetRateProfilePars(&rate_profile, 0, 50, 30, 0); + rate_profile.frame_index_rate_update[1] = kNbrFramesLong + 1; + rate_profile.num_frames = kNbrFramesLong; + // Codec/network settings. + CodecConfigPars process_settings; + SetCodecParameters(&process_settings, kVideoCodecVP8, 0.0f, -1, + 1, false, true, true, true); + // Metrics for expected quality. + QualityMetrics quality_metrics; + SetQualityMetrics(&quality_metrics, 25.0, 15.0, 0.70, 0.40); + // Metrics for rate control. + RateControlMetrics rc_metrics[1]; + SetRateControlMetrics(rc_metrics, 0, 160, 60, 120, 20, 70, 1, 2); + ProcessFramesAndVerify(quality_metrics, + rate_profile, + process_settings, + rc_metrics); +} + +// VP8: Run with no packet loss, with 3 temporal layers, with a rate update in +// the middle of the sequence. The max values for the frame size mismatch and +// encoding rate mismatch are applied to each layer. +// No dropped frames in this test, and internal spatial resizer is off. +// One key frame (first frame only) in sequence, so no spatial resizing. +TEST_F(VideoProcessorIntegrationTest, + DISABLED_ON_ANDROID(ProcessNoLossTemporalLayersVP8)) { + config_.networking_config.packet_loss_probability = 0; + // Bitrate and frame rate profile. + RateProfile rate_profile; + SetRateProfilePars(&rate_profile, 0, 200, 30, 0); + SetRateProfilePars(&rate_profile, 1, 400, 30, 150); + rate_profile.frame_index_rate_update[2] = kNbrFramesLong + 1; + rate_profile.num_frames = kNbrFramesLong; + // Codec/network settings. + CodecConfigPars process_settings; + SetCodecParameters(&process_settings, kVideoCodecVP8, 0.0f, -1, 3, false, + true, true, false); + // Metrics for expected quality. + QualityMetrics quality_metrics; + SetQualityMetrics(&quality_metrics, 32.5, 30.0, 0.85, 0.80); + // Metrics for rate control. + RateControlMetrics rc_metrics[2]; + SetRateControlMetrics(rc_metrics, 0, 0, 20, 30, 10, 10, 0, 1); + SetRateControlMetrics(rc_metrics, 1, 0, 0, 30, 15, 10, 0, 0); + ProcessFramesAndVerify(quality_metrics, + rate_profile, + process_settings, + rc_metrics); +} +} // namespace webrtc |