diff options
| author | Android Chromium Automerger <chromium-automerger@android> | 2014-11-04 15:21:41 +0000 |
|---|---|---|
| committer | Android Chromium Automerger <chromium-automerger@android> | 2014-11-04 15:21:41 +0000 |
| commit | 6651f18386d5bd9870d621ace2a11dec1cb68c80 (patch) | |
| tree | 940b095a69f4c43993cdf329d0aa6a8702b9ab3d | |
| parent | 2dc72e3be449734e64ff706df1765ac1bfe64b23 (diff) | |
| parent | a1fd19c12e4efdf4b8a5f92323070443d50dc34e (diff) | |
| download | webrtc-6651f18386d5bd9870d621ace2a11dec1cb68c80.tar.gz | |
Merge third_party/webrtc from https://chromium.googlesource.com/external/webrtc/trunk/webrtc.git at a1fd19c12e4efdf4b8a5f92323070443d50dc34e
This commit was generated by merge_from_chromium.py.
Change-Id: Ibd48eca2d93e6324a2e886e451f27307aab45e9b
184 files changed, 3682 insertions, 1951 deletions
@@ -120,6 +120,10 @@ config("common_config") { } } + if (cpu_arch == "arm64") { + defines += [ "WEBRTC_ARCH_ARM" ] + } + if (cpu_arch == "arm") { defines += [ "WEBRTC_ARCH_ARM" ] if (arm_version == 7) { diff --git a/build/common.gypi b/build/common.gypi index 18b33830..366e7e9f 100644 --- a/build/common.gypi +++ b/build/common.gypi @@ -42,6 +42,7 @@ 'modules_java_gyp_path%': '<(modules_java_gyp_path)', 'gen_core_neon_offsets_gyp%': '<(gen_core_neon_offsets_gyp)', 'webrtc_vp8_dir%': '<(webrtc_root)/modules/video_coding/codecs/vp8', + 'webrtc_vp9_dir%': '<(webrtc_root)/modules/video_coding/codecs/vp9', 'rbe_components_path%': '<(webrtc_root)/modules/remote_bitrate_estimator', 'include_opus%': 1, }, @@ -52,6 +53,7 @@ 'modules_java_gyp_path%': '<(modules_java_gyp_path)', 'gen_core_neon_offsets_gyp%': '<(gen_core_neon_offsets_gyp)', 'webrtc_vp8_dir%': '<(webrtc_vp8_dir)', + 'webrtc_vp9_dir%': '<(webrtc_vp9_dir)', 'include_opus%': '<(include_opus)', 'rtc_relative_path%': 1, 'rbe_components_path%': '<(rbe_components_path)', @@ -250,6 +252,11 @@ }], ], }], + ['target_arch=="arm64"', { + 'defines': [ + 'WEBRTC_ARCH_ARM', + ], + }], ['target_arch=="arm" or target_arch=="armv7"', { 'defines': [ 'WEBRTC_ARCH_ARM', @@ -267,7 +274,7 @@ }], ], }], - ['target_arch=="mipsel"', { + ['target_arch=="mipsel" and mips_arch_variant!="r6"', { 'defines': [ 'MIPS32_LE', ], diff --git a/build/isolate.gypi b/build/isolate.gypi index 2e86020e..86169fd0 100644 --- a/build/isolate.gypi +++ b/build/isolate.gypi @@ -62,18 +62,6 @@ # Files that are known to be involved in this step. '<(DEPTH)/tools/swarming_client/isolate.py', '<(DEPTH)/tools/swarming_client/run_isolated.py', - - # Disable file tracking by the build driver for now. This means the - # project must have the proper build-time dependency for their runtime - # dependency. This improves the runtime of the build driver since it - # doesn't have to stat() all these files. - # - # More importantly, it means that even if a isolate_dependency_tracked - # file is missing, for example if a file was deleted and the .isolate - # file was not updated, that won't break the build, especially in the - # case where foo_tests_run is not built! This should be reenabled once - # the switch-over to running tests on Swarm is completed. - #'<@(isolate_dependency_tracked)', ], 'outputs': [ '<(PRODUCT_DIR)/<(RULE_INPUT_ROOT).isolated', diff --git a/build/tsan_suppressions_webrtc.cc b/build/tsan_suppressions_webrtc.cc index 723e62a2..01658ed2 100644 --- a/build/tsan_suppressions_webrtc.cc +++ b/build/tsan_suppressions_webrtc.cc @@ -27,6 +27,7 @@ char kTSanDefaultSuppressions[] = "race:rtc::MessageQueue::Quit\n" "race:FileVideoCapturerTest::VideoCapturerListener::OnFrameCaptured\n" "race:vp8cx_remove_encoder_threads\n" +"race:third_party/libvpx/source/libvpx/vp9/common/vp9_scan.h\n" // Usage of trace callback and trace level is racy in libjingle_media_unittests. // https://code.google.com/p/webrtc/issues/detail?id=3372 diff --git a/common_audio/BUILD.gn b/common_audio/BUILD.gn index ad49d17f..ba1d1795 100644 --- a/common_audio/BUILD.gn +++ b/common_audio/BUILD.gn @@ -71,7 +71,9 @@ source_set("common_audio") { "signal_processing/splitting_filter.c", "signal_processing/sqrt_of_one_minus_x_squared.c", "signal_processing/vector_scaling_operations.c", + "vad/include/vad.h", "vad/include/webrtc_vad.h", + "vad/vad.cc", "vad/webrtc_vad.c", "vad/vad_core.c", "vad/vad_core.h", @@ -83,8 +85,8 @@ source_set("common_audio") { "vad/vad_sp.h", "wav_header.cc", "wav_header.h", - "wav_writer.cc", - "wav_writer.h", + "wav_file.cc", + "wav_file.h", "window_generator.cc", "window_generator.h", ] diff --git a/common_audio/audio_converter.cc b/common_audio/audio_converter.cc index 9e18033f..f085ff13 100644 --- a/common_audio/audio_converter.cc +++ b/common_audio/audio_converter.cc @@ -43,10 +43,13 @@ void UpmixFromMono(const float* src, } // namespace AudioConverter::AudioConverter(int src_channels, int src_frames, - int dst_channels, int dst_frames) { + int dst_channels, int dst_frames) + : src_channels_(src_channels), + src_frames_(src_frames), + dst_channels_(dst_channels), + dst_frames_(dst_frames) { CHECK(dst_channels == src_channels || dst_channels == 1 || src_channels == 1); - const int resample_channels = src_channels < dst_channels ? src_channels : - dst_channels; + const int resample_channels = std::min(src_channels, dst_channels); // Prepare buffers as needed for intermediate stages. if (dst_channels < src_channels) @@ -66,8 +69,11 @@ void AudioConverter::Convert(const float* const* src, int dst_channels, int dst_frames, float* const* dst) { - DCHECK(dst_channels == src_channels || dst_channels == 1 || - src_channels == 1); + DCHECK_EQ(src_channels_, src_channels); + DCHECK_EQ(src_frames_, src_frames); + DCHECK_EQ(dst_channels_, dst_channels); + DCHECK_EQ(dst_frames_, dst_frames);; + if (src_channels == dst_channels && src_frames == dst_frames) { // Shortcut copy. if (src != dst) { diff --git a/common_audio/audio_converter.h b/common_audio/audio_converter.h index df31755e..6365f587 100644 --- a/common_audio/audio_converter.h +++ b/common_audio/audio_converter.h @@ -40,6 +40,10 @@ class AudioConverter { float* const* dest); private: + const int src_channels_; + const int src_frames_; + const int dst_channels_; + const int dst_frames_; scoped_ptr<ChannelBuffer<float>> downmix_buffer_; ScopedVector<PushSincResampler> resamplers_; diff --git a/common_audio/audio_util.cc b/common_audio/audio_util.cc index f2936b07..2047295c 100644 --- a/common_audio/audio_util.cc +++ b/common_audio/audio_util.cc @@ -14,19 +14,29 @@ namespace webrtc { -void RoundToInt16(const float* src, size_t size, int16_t* dest) { +void FloatToS16(const float* src, size_t size, int16_t* dest) { for (size_t i = 0; i < size; ++i) - dest[i] = RoundToInt16(src[i]); + dest[i] = FloatToS16(src[i]); } -void ScaleAndRoundToInt16(const float* src, size_t size, int16_t* dest) { +void S16ToFloat(const int16_t* src, size_t size, float* dest) { for (size_t i = 0; i < size; ++i) - dest[i] = ScaleAndRoundToInt16(src[i]); + dest[i] = S16ToFloat(src[i]); } -void ScaleToFloat(const int16_t* src, size_t size, float* dest) { +void FloatS16ToS16(const float* src, size_t size, int16_t* dest) { for (size_t i = 0; i < size; ++i) - dest[i] = ScaleToFloat(src[i]); + dest[i] = FloatS16ToS16(src[i]); +} + +void FloatToFloatS16(const float* src, size_t size, float* dest) { + for (size_t i = 0; i < size; ++i) + dest[i] = FloatToFloatS16(src[i]); +} + +void FloatS16ToFloat(const float* src, size_t size, float* dest) { + for (size_t i = 0; i < size; ++i) + dest[i] = FloatS16ToFloat(src[i]); } } // namespace webrtc diff --git a/common_audio/audio_util_unittest.cc b/common_audio/audio_util_unittest.cc index bf9ad812..2cdf5381 100644 --- a/common_audio/audio_util_unittest.cc +++ b/common_audio/audio_util_unittest.cc @@ -26,35 +26,59 @@ void ExpectArraysEq(const float* ref, const float* test, int length) { } } -TEST(AudioUtilTest, RoundToInt16) { +TEST(AudioUtilTest, FloatToS16) { + const int kSize = 9; + const float kInput[kSize] = { + 0.f, 0.4f / 32767.f, 0.6f / 32767.f, -0.4f / 32768.f, -0.6f / 32768.f, + 1.f, -1.f, 1.1f, -1.1f}; + const int16_t kReference[kSize] = { + 0, 0, 1, 0, -1, 32767, -32768, 32767, -32768}; + int16_t output[kSize]; + FloatToS16(kInput, kSize, output); + ExpectArraysEq(kReference, output, kSize); +} + +TEST(AudioUtilTest, S16ToFloat) { + const int kSize = 7; + const int16_t kInput[kSize] = {0, 1, -1, 16384, -16384, 32767, -32768}; + const float kReference[kSize] = { + 0.f, 1.f / 32767.f, -1.f / 32768.f, 16384.f / 32767.f, -0.5f, 1.f, -1.f}; + float output[kSize]; + S16ToFloat(kInput, kSize, output); + ExpectArraysEq(kReference, output, kSize); +} + +TEST(AudioUtilTest, FloatS16ToS16) { const int kSize = 7; const float kInput[kSize] = { 0.f, 0.4f, 0.5f, -0.4f, -0.5f, 32768.f, -32769.f}; const int16_t kReference[kSize] = {0, 0, 1, 0, -1, 32767, -32768}; int16_t output[kSize]; - RoundToInt16(kInput, kSize, output); + FloatS16ToS16(kInput, kSize, output); ExpectArraysEq(kReference, output, kSize); } -TEST(AudioUtilTest, ScaleAndRoundToInt16) { +TEST(AudioUtilTest, FloatToFloatS16) { const int kSize = 9; const float kInput[kSize] = { 0.f, 0.4f / 32767.f, 0.6f / 32767.f, -0.4f / 32768.f, -0.6f / 32768.f, 1.f, -1.f, 1.1f, -1.1f}; - const int16_t kReference[kSize] = { - 0, 0, 1, 0, -1, 32767, -32768, 32767, -32768}; - int16_t output[kSize]; - ScaleAndRoundToInt16(kInput, kSize, output); + const float kReference[kSize] = { + 0.f, 0.4f, 0.6f, -0.4f, -0.6f, 32767.f, -32768.f, 36043.7f, -36044.8f}; + float output[kSize]; + FloatToFloatS16(kInput, kSize, output); ExpectArraysEq(kReference, output, kSize); } -TEST(AudioUtilTest, ScaleToFloat) { - const int kSize = 7; - const int16_t kInput[kSize] = {0, 1, -1, 16384, -16384, 32767, -32768}; +TEST(AudioUtilTest, FloatS16ToFloat) { + const int kSize = 9; + const float kInput[kSize] = { + 0.f, 0.4f, 0.6f, -0.4f, -0.6f, 32767.f, -32768.f, 36043.7f, -36044.8f}; const float kReference[kSize] = { - 0.f, 1.f / 32767.f, -1.f / 32768.f, 16384.f / 32767.f, -0.5f, 1.f, -1.f}; + 0.f, 0.4f / 32767.f, 0.6f / 32767.f, -0.4f / 32768.f, -0.6f / 32768.f, + 1.f, -1.f, 1.1f, -1.1f}; float output[kSize]; - ScaleToFloat(kInput, kSize, output); + FloatS16ToFloat(kInput, kSize, output); ExpectArraysEq(kReference, output, kSize); } diff --git a/common_audio/common_audio.gyp b/common_audio/common_audio.gyp index 6c1b7960..8f96674f 100644 --- a/common_audio/common_audio.gyp +++ b/common_audio/common_audio.gyp @@ -85,7 +85,9 @@ 'signal_processing/splitting_filter.c', 'signal_processing/sqrt_of_one_minus_x_squared.c', 'signal_processing/vector_scaling_operations.c', + 'vad/include/vad.h', 'vad/include/webrtc_vad.h', + 'vad/vad.cc', 'vad/webrtc_vad.c', 'vad/vad_core.c', 'vad/vad_core.h', @@ -97,8 +99,8 @@ 'vad/vad_sp.h', 'wav_header.cc', 'wav_header.h', - 'wav_writer.cc', - 'wav_writer.h', + 'wav_file.cc', + 'wav_file.h', 'window_generator.cc', 'window_generator.h', ], @@ -138,7 +140,7 @@ }], ], # conditions }], - ['target_arch=="mipsel"', { + ['target_arch=="mipsel" and mips_arch_variant!="r6"', { 'sources': [ 'signal_processing/include/spl_inl_mips.h', 'signal_processing/complex_bit_reverse_mips.c', @@ -243,7 +245,7 @@ 'vad/vad_unittest.cc', 'vad/vad_unittest.h', 'wav_header_unittest.cc', - 'wav_writer_unittest.cc', + 'wav_file_unittest.cc', 'window_generator_unittest.cc', ], 'conditions': [ @@ -283,7 +285,6 @@ ], 'includes': [ '../build/isolate.gypi', - 'common_audio_unittests.isolate', ], 'sources': [ 'common_audio_unittests.isolate', diff --git a/common_audio/common_audio_unittests.isolate b/common_audio/common_audio_unittests.isolate index cc5e6ab4..80eb0fc4 100644 --- a/common_audio/common_audio_unittests.isolate +++ b/common_audio/common_audio_unittests.isolate @@ -9,7 +9,7 @@ 'conditions': [ ['OS=="android"', { 'variables': { - 'isolate_dependency_untracked': [ + 'files': [ '<(DEPTH)/data/', '<(DEPTH)/resources/', ], @@ -21,13 +21,10 @@ '<(DEPTH)/testing/test_env.py', '<(PRODUCT_DIR)/common_audio_unittests<(EXECUTABLE_SUFFIX)', ], - 'isolate_dependency_tracked': [ + 'files': [ '<(DEPTH)/testing/test_env.py', '<(PRODUCT_DIR)/common_audio_unittests<(EXECUTABLE_SUFFIX)', ], - 'isolate_dependency_untracked': [ - '<(DEPTH)/tools/swarming_client/', - ], }, }], ], diff --git a/common_audio/include/audio_util.h b/common_audio/include/audio_util.h index 0ce034be..767b21c5 100644 --- a/common_audio/include/audio_util.h +++ b/common_audio/include/audio_util.h @@ -20,18 +20,11 @@ namespace webrtc { typedef std::numeric_limits<int16_t> limits_int16; -static inline int16_t RoundToInt16(float v) { - const float kMaxRound = limits_int16::max() - 0.5f; - const float kMinRound = limits_int16::min() + 0.5f; - if (v > 0) - return v >= kMaxRound ? limits_int16::max() : - static_cast<int16_t>(v + 0.5f); - return v <= kMinRound ? limits_int16::min() : - static_cast<int16_t>(v - 0.5f); -} - -// Scale (from [-1, 1]) and round to full-range int16 with clamping. -static inline int16_t ScaleAndRoundToInt16(float v) { +// The conversion functions use the following naming convention: +// S16: int16_t [-32768, 32767] +// Float: float [-1.0, 1.0] +// FloatS16: float [-32768.0, 32767.0] +static inline int16_t FloatToS16(float v) { if (v > 0) return v >= 1 ? limits_int16::max() : static_cast<int16_t>(v * limits_int16::max() + 0.5f); @@ -39,22 +32,37 @@ static inline int16_t ScaleAndRoundToInt16(float v) { static_cast<int16_t>(-v * limits_int16::min() - 0.5f); } -// Scale to float [-1, 1]. -static inline float ScaleToFloat(int16_t v) { - const float kMaxInt16Inverse = 1.f / limits_int16::max(); - const float kMinInt16Inverse = 1.f / limits_int16::min(); +static inline float S16ToFloat(int16_t v) { + static const float kMaxInt16Inverse = 1.f / limits_int16::max(); + static const float kMinInt16Inverse = 1.f / limits_int16::min(); return v * (v > 0 ? kMaxInt16Inverse : -kMinInt16Inverse); } -// Round |size| elements of |src| to int16 with clamping and write to |dest|. -void RoundToInt16(const float* src, size_t size, int16_t* dest); +static inline int16_t FloatS16ToS16(float v) { + static const float kMaxRound = limits_int16::max() - 0.5f; + static const float kMinRound = limits_int16::min() + 0.5f; + if (v > 0) + return v >= kMaxRound ? limits_int16::max() : + static_cast<int16_t>(v + 0.5f); + return v <= kMinRound ? limits_int16::min() : + static_cast<int16_t>(v - 0.5f); +} -// Scale (from [-1, 1]) and round |size| elements of |src| to full-range int16 -// with clamping and write to |dest|. -void ScaleAndRoundToInt16(const float* src, size_t size, int16_t* dest); +static inline float FloatToFloatS16(float v) { + return v * (v > 0 ? limits_int16::max() : -limits_int16::min()); +} + +static inline float FloatS16ToFloat(float v) { + static const float kMaxInt16Inverse = 1.f / limits_int16::max(); + static const float kMinInt16Inverse = 1.f / limits_int16::min(); + return v * (v > 0 ? kMaxInt16Inverse : -kMinInt16Inverse); +} -// Scale |size| elements of |src| to float [-1, 1] and write to |dest|. -void ScaleToFloat(const int16_t* src, size_t size, float* dest); +void FloatToS16(const float* src, size_t size, int16_t* dest); +void S16ToFloat(const int16_t* src, size_t size, float* dest); +void FloatS16ToS16(const float* src, size_t size, int16_t* dest); +void FloatToFloatS16(const float* src, size_t size, float* dest); +void FloatS16ToFloat(const float* src, size_t size, float* dest); // Deinterleave audio from |interleaved| to the channel buffers pointed to // by |deinterleaved|. There must be sufficient space allocated in the diff --git a/common_audio/resampler/push_sinc_resampler.cc b/common_audio/resampler/push_sinc_resampler.cc index 02755590..49e2e12e 100644 --- a/common_audio/resampler/push_sinc_resampler.cc +++ b/common_audio/resampler/push_sinc_resampler.cc @@ -40,7 +40,7 @@ int PushSincResampler::Resample(const int16_t* source, source_ptr_int_ = source; // Pass NULL as the float source to have Run() read from the int16 source. Resample(NULL, source_length, float_buffer_.get(), destination_frames_); - RoundToInt16(float_buffer_.get(), destination_frames_, destination); + FloatS16ToS16(float_buffer_.get(), destination_frames_, destination); source_ptr_int_ = NULL; return destination_frames_; } diff --git a/common_audio/resampler/push_sinc_resampler_unittest.cc b/common_audio/resampler/push_sinc_resampler_unittest.cc index 1ca4fdf9..90ac0cf0 100644 --- a/common_audio/resampler/push_sinc_resampler_unittest.cc +++ b/common_audio/resampler/push_sinc_resampler_unittest.cc @@ -160,16 +160,15 @@ void PushSincResamplerTest::ResampleTest(bool int_format) { resampler_source.Run(input_samples, source.get()); if (int_format) { for (int i = 0; i < kNumBlocks; ++i) { - ScaleAndRoundToInt16( - &source[i * input_block_size], input_block_size, source_int.get()); + FloatToS16(&source[i * input_block_size], input_block_size, + source_int.get()); EXPECT_EQ(output_block_size, resampler.Resample(source_int.get(), input_block_size, destination_int.get(), output_block_size)); - ScaleToFloat(destination_int.get(), - output_block_size, - &resampled_destination[i * output_block_size]); + S16ToFloat(destination_int.get(), output_block_size, + &resampled_destination[i * output_block_size]); } } else { for (int i = 0; i < kNumBlocks; ++i) { diff --git a/common_audio/signal_processing/complex_fft.c b/common_audio/signal_processing/complex_fft.c index c8230647..74b4258a 100644 --- a/common_audio/signal_processing/complex_fft.c +++ b/common_audio/signal_processing/complex_fft.c @@ -65,18 +65,16 @@ int WebRtcSpl_ComplexFFT(int16_t frfi[], int stages, int mode) { j = i + l; - tr32 = WEBRTC_SPL_RSHIFT_W32((WEBRTC_SPL_MUL_16_16(wr, frfi[2 * j]) - - WEBRTC_SPL_MUL_16_16(wi, frfi[2 * j + 1])), 15); + tr32 = (wr * frfi[2 * j] - wi * frfi[2 * j + 1]) >> 15; - ti32 = WEBRTC_SPL_RSHIFT_W32((WEBRTC_SPL_MUL_16_16(wr, frfi[2 * j + 1]) - + WEBRTC_SPL_MUL_16_16(wi, frfi[2 * j])), 15); + ti32 = (wr * frfi[2 * j + 1] + wi * frfi[2 * j]) >> 15; qr32 = (int32_t)frfi[2 * i]; qi32 = (int32_t)frfi[2 * i + 1]; - frfi[2 * j] = (int16_t)WEBRTC_SPL_RSHIFT_W32(qr32 - tr32, 1); - frfi[2 * j + 1] = (int16_t)WEBRTC_SPL_RSHIFT_W32(qi32 - ti32, 1); - frfi[2 * i] = (int16_t)WEBRTC_SPL_RSHIFT_W32(qr32 + tr32, 1); - frfi[2 * i + 1] = (int16_t)WEBRTC_SPL_RSHIFT_W32(qi32 + ti32, 1); + frfi[2 * j] = (int16_t)((qr32 - tr32) >> 1); + frfi[2 * j + 1] = (int16_t)((qi32 - ti32) >> 1); + frfi[2 * i] = (int16_t)((qr32 + tr32) >> 1); + frfi[2 * i + 1] = (int16_t)((qi32 + ti32) >> 1); } } @@ -135,20 +133,20 @@ int WebRtcSpl_ComplexFFT(int16_t frfi[], int stages, int mode) + WEBRTC_SPL_MUL_16_16(wi, frfi[2 * j]) + CFFTRND; #endif - tr32 = WEBRTC_SPL_RSHIFT_W32(tr32, 15 - CFFTSFT); - ti32 = WEBRTC_SPL_RSHIFT_W32(ti32, 15 - CFFTSFT); + tr32 >>= 15 - CFFTSFT; + ti32 >>= 15 - CFFTSFT; qr32 = ((int32_t)frfi[2 * i]) << CFFTSFT; qi32 = ((int32_t)frfi[2 * i + 1]) << CFFTSFT; - frfi[2 * j] = (int16_t)WEBRTC_SPL_RSHIFT_W32( - (qr32 - tr32 + CFFTRND2), 1 + CFFTSFT); - frfi[2 * j + 1] = (int16_t)WEBRTC_SPL_RSHIFT_W32( - (qi32 - ti32 + CFFTRND2), 1 + CFFTSFT); - frfi[2 * i] = (int16_t)WEBRTC_SPL_RSHIFT_W32( - (qr32 + tr32 + CFFTRND2), 1 + CFFTSFT); - frfi[2 * i + 1] = (int16_t)WEBRTC_SPL_RSHIFT_W32( - (qi32 + ti32 + CFFTRND2), 1 + CFFTSFT); + frfi[2 * j] = (int16_t)( + (qr32 - tr32 + CFFTRND2) >> (1 + CFFTSFT)); + frfi[2 * j + 1] = (int16_t)( + (qi32 - ti32 + CFFTRND2) >> (1 + CFFTSFT)); + frfi[2 * i] = (int16_t)( + (qr32 + tr32 + CFFTRND2) >> (1 + CFFTSFT)); + frfi[2 * i + 1] = (int16_t)( + (qi32 + ti32 + CFFTRND2) >> (1 + CFFTSFT)); } } @@ -219,19 +217,16 @@ int WebRtcSpl_ComplexIFFT(int16_t frfi[], int stages, int mode) { j = i + l; - tr32 = WEBRTC_SPL_RSHIFT_W32((WEBRTC_SPL_MUL_16_16_RSFT(wr, frfi[2 * j], 0) - - WEBRTC_SPL_MUL_16_16_RSFT(wi, frfi[2 * j + 1], 0)), 15); + tr32 = (wr * frfi[2 * j] - wi * frfi[2 * j + 1]) >> 15; - ti32 = WEBRTC_SPL_RSHIFT_W32( - (WEBRTC_SPL_MUL_16_16_RSFT(wr, frfi[2 * j + 1], 0) - + WEBRTC_SPL_MUL_16_16_RSFT(wi,frfi[2*j],0)), 15); + ti32 = (wr * frfi[2 * j + 1] + wi * frfi[2 * j]) >> 15; qr32 = (int32_t)frfi[2 * i]; qi32 = (int32_t)frfi[2 * i + 1]; - frfi[2 * j] = (int16_t)WEBRTC_SPL_RSHIFT_W32(qr32 - tr32, shift); - frfi[2 * j + 1] = (int16_t)WEBRTC_SPL_RSHIFT_W32(qi32 - ti32, shift); - frfi[2 * i] = (int16_t)WEBRTC_SPL_RSHIFT_W32(qr32 + tr32, shift); - frfi[2 * i + 1] = (int16_t)WEBRTC_SPL_RSHIFT_W32(qi32 + ti32, shift); + frfi[2 * j] = (int16_t)((qr32 - tr32) >> shift); + frfi[2 * j + 1] = (int16_t)((qi32 - ti32) >> shift); + frfi[2 * i] = (int16_t)((qr32 + tr32) >> shift); + frfi[2 * i + 1] = (int16_t)((qi32 + ti32) >> shift); } } } else @@ -281,20 +276,20 @@ int WebRtcSpl_ComplexIFFT(int16_t frfi[], int stages, int mode) ti32 = WEBRTC_SPL_MUL_16_16(wr, frfi[2 * j + 1]) + WEBRTC_SPL_MUL_16_16(wi, frfi[2 * j]) + CIFFTRND; #endif - tr32 = WEBRTC_SPL_RSHIFT_W32(tr32, 15 - CIFFTSFT); - ti32 = WEBRTC_SPL_RSHIFT_W32(ti32, 15 - CIFFTSFT); + tr32 >>= 15 - CIFFTSFT; + ti32 >>= 15 - CIFFTSFT; qr32 = ((int32_t)frfi[2 * i]) << CIFFTSFT; qi32 = ((int32_t)frfi[2 * i + 1]) << CIFFTSFT; - frfi[2 * j] = (int16_t)WEBRTC_SPL_RSHIFT_W32((qr32 - tr32+round2), - shift+CIFFTSFT); - frfi[2 * j + 1] = (int16_t)WEBRTC_SPL_RSHIFT_W32( - (qi32 - ti32 + round2), shift + CIFFTSFT); - frfi[2 * i] = (int16_t)WEBRTC_SPL_RSHIFT_W32((qr32 + tr32 + round2), - shift + CIFFTSFT); - frfi[2 * i + 1] = (int16_t)WEBRTC_SPL_RSHIFT_W32( - (qi32 + ti32 + round2), shift + CIFFTSFT); + frfi[2 * j] = (int16_t)( + (qr32 - tr32 + round2) >> (shift + CIFFTSFT)); + frfi[2 * j + 1] = (int16_t)( + (qi32 - ti32 + round2) >> (shift + CIFFTSFT)); + frfi[2 * i] = (int16_t)( + (qr32 + tr32 + round2) >> (shift + CIFFTSFT)); + frfi[2 * i + 1] = (int16_t)( + (qi32 + ti32 + round2) >> (shift + CIFFTSFT)); } } diff --git a/common_audio/signal_processing/division_operations.c b/common_audio/signal_processing/division_operations.c index e9554f44..6aeb0fb2 100644 --- a/common_audio/signal_processing/division_operations.c +++ b/common_audio/signal_processing/division_operations.c @@ -113,23 +113,20 @@ int32_t WebRtcSpl_DivW32HiLow(int32_t num, int16_t den_hi, int16_t den_low) tmpW32 = (int32_t)0x7fffffffL - tmpW32; // result in Q30 (tmpW32 = 2.0-(den*approx)) // Store tmpW32 in hi and low format - tmp_hi = (int16_t)WEBRTC_SPL_RSHIFT_W32(tmpW32, 16); - tmp_low = (int16_t)WEBRTC_SPL_RSHIFT_W32((tmpW32 - - WEBRTC_SPL_LSHIFT_W32((int32_t)tmp_hi, 16)), 1); + tmp_hi = (int16_t)(tmpW32 >> 16); + tmp_low = (int16_t)((tmpW32 - ((int32_t)tmp_hi << 16)) >> 1); // tmpW32 = 1/den in Q29 tmpW32 = ((WEBRTC_SPL_MUL_16_16(tmp_hi, approx) + (WEBRTC_SPL_MUL_16_16(tmp_low, approx) >> 15)) << 1); // 1/den in hi and low format - tmp_hi = (int16_t)WEBRTC_SPL_RSHIFT_W32(tmpW32, 16); - tmp_low = (int16_t)WEBRTC_SPL_RSHIFT_W32((tmpW32 - - WEBRTC_SPL_LSHIFT_W32((int32_t)tmp_hi, 16)), 1); + tmp_hi = (int16_t)(tmpW32 >> 16); + tmp_low = (int16_t)((tmpW32 - ((int32_t)tmp_hi << 16)) >> 1); // Store num in hi and low format - num_hi = (int16_t)WEBRTC_SPL_RSHIFT_W32(num, 16); - num_low = (int16_t)WEBRTC_SPL_RSHIFT_W32((num - - WEBRTC_SPL_LSHIFT_W32((int32_t)num_hi, 16)), 1); + num_hi = (int16_t)(num >> 16); + num_low = (int16_t)((num - ((int32_t)num_hi << 16)) >> 1); // num * (1/den) by 32 bit multiplication (result in Q28) diff --git a/common_audio/signal_processing/include/signal_processing_library.h b/common_audio/signal_processing/include/signal_processing_library.h index d13973ca..2bdfc23c 100644 --- a/common_audio/signal_processing/include/signal_processing_library.h +++ b/common_audio/signal_processing/include/signal_processing_library.h @@ -87,7 +87,6 @@ // Shifting with negative numbers not allowed // We cannot do casting here due to signed/unsigned problem -#define WEBRTC_SPL_RSHIFT_W32(x, c) ((x) >> (c)) #define WEBRTC_SPL_LSHIFT_W32(x, c) ((x) << (c)) #define WEBRTC_SPL_RSHIFT_U32(x, c) ((uint32_t)(x) >> (c)) diff --git a/common_audio/signal_processing/levinson_durbin.c b/common_audio/signal_processing/levinson_durbin.c index 5c5d2246..29f2398d 100644 --- a/common_audio/signal_processing/levinson_durbin.c +++ b/common_audio/signal_processing/levinson_durbin.c @@ -45,9 +45,8 @@ int16_t WebRtcSpl_LevinsonDurbin(int32_t *R, int16_t *A, int16_t *K, { temp1W32 = WEBRTC_SPL_LSHIFT_W32(R[i], norm); // Put R in hi and low format - R_hi[i] = (int16_t)WEBRTC_SPL_RSHIFT_W32(temp1W32, 16); - R_low[i] = (int16_t)WEBRTC_SPL_RSHIFT_W32((temp1W32 - - WEBRTC_SPL_LSHIFT_W32((int32_t)R_hi[i], 16)), 1); + R_hi[i] = (int16_t)(temp1W32 >> 16); + R_low[i] = (int16_t)((temp1W32 - ((int32_t)R_hi[i] << 16)) >> 1); } // K = A[1] = -R[1] / R[0] @@ -63,19 +62,17 @@ int16_t WebRtcSpl_LevinsonDurbin(int32_t *R, int16_t *A, int16_t *K, } // Put K in hi and low format - K_hi = (int16_t)WEBRTC_SPL_RSHIFT_W32(temp1W32, 16); - K_low = (int16_t)WEBRTC_SPL_RSHIFT_W32((temp1W32 - - WEBRTC_SPL_LSHIFT_W32((int32_t)K_hi, 16)), 1); + K_hi = (int16_t)(temp1W32 >> 16); + K_low = (int16_t)((temp1W32 - ((int32_t)K_hi << 16)) >> 1); // Store first reflection coefficient K[0] = K_hi; - temp1W32 = WEBRTC_SPL_RSHIFT_W32(temp1W32, 4); // A[1] in Q27 + temp1W32 >>= 4; // A[1] in Q27. // Put A[1] in hi and low format - A_hi[1] = (int16_t)WEBRTC_SPL_RSHIFT_W32(temp1W32, 16); - A_low[1] = (int16_t)WEBRTC_SPL_RSHIFT_W32((temp1W32 - - WEBRTC_SPL_LSHIFT_W32((int32_t)A_hi[1], 16)), 1); + A_hi[1] = (int16_t)(temp1W32 >> 16); + A_low[1] = (int16_t)((temp1W32 - ((int32_t)A_hi[1] << 16)) >> 1); // Alpha = R[0] * (1-K^2) @@ -86,9 +83,8 @@ int16_t WebRtcSpl_LevinsonDurbin(int32_t *R, int16_t *A, int16_t *K, temp1W32 = (int32_t)0x7fffffffL - temp1W32; // temp1W32 = (1 - K[0]*K[0]) in Q31 // Store temp1W32 = 1 - K[0]*K[0] on hi and low format - tmp_hi = (int16_t)WEBRTC_SPL_RSHIFT_W32(temp1W32, 16); - tmp_low = (int16_t)WEBRTC_SPL_RSHIFT_W32((temp1W32 - - WEBRTC_SPL_LSHIFT_W32((int32_t)tmp_hi, 16)), 1); + tmp_hi = (int16_t)(temp1W32 >> 16); + tmp_low = (int16_t)((temp1W32 - ((int32_t)tmp_hi << 16)) >> 1); // Calculate Alpha in Q31 temp1W32 = ((WEBRTC_SPL_MUL_16_16(R_hi[0], tmp_hi) @@ -99,9 +95,8 @@ int16_t WebRtcSpl_LevinsonDurbin(int32_t *R, int16_t *A, int16_t *K, Alpha_exp = WebRtcSpl_NormW32(temp1W32); temp1W32 = WEBRTC_SPL_LSHIFT_W32(temp1W32, Alpha_exp); - Alpha_hi = (int16_t)WEBRTC_SPL_RSHIFT_W32(temp1W32, 16); - Alpha_low = (int16_t)WEBRTC_SPL_RSHIFT_W32((temp1W32 - - WEBRTC_SPL_LSHIFT_W32((int32_t)Alpha_hi, 16)), 1); + Alpha_hi = (int16_t)(temp1W32 >> 16); + Alpha_low = (int16_t)((temp1W32 - ((int32_t)Alpha_hi << 16)) >> 1); // Perform the iterative calculations in the Levinson-Durbin algorithm @@ -155,9 +150,8 @@ int16_t WebRtcSpl_LevinsonDurbin(int32_t *R, int16_t *A, int16_t *K, } // Put K on hi and low format - K_hi = (int16_t)WEBRTC_SPL_RSHIFT_W32(temp3W32, 16); - K_low = (int16_t)WEBRTC_SPL_RSHIFT_W32((temp3W32 - - WEBRTC_SPL_LSHIFT_W32((int32_t)K_hi, 16)), 1); + K_hi = (int16_t)(temp3W32 >> 16); + K_low = (int16_t)((temp3W32 - ((int32_t)K_hi << 16)) >> 1); // Store Reflection coefficient in Q15 K[i - 1] = K_hi; @@ -188,18 +182,18 @@ int16_t WebRtcSpl_LevinsonDurbin(int32_t *R, int16_t *A, int16_t *K, + (WEBRTC_SPL_MUL_16_16(K_low, A_hi[i-j]) >> 15)) << 1); // Put Anew in hi and low format - A_upd_hi[j] = (int16_t)WEBRTC_SPL_RSHIFT_W32(temp1W32, 16); - A_upd_low[j] = (int16_t)WEBRTC_SPL_RSHIFT_W32((temp1W32 - - WEBRTC_SPL_LSHIFT_W32((int32_t)A_upd_hi[j], 16)), 1); + A_upd_hi[j] = (int16_t)(temp1W32 >> 16); + A_upd_low[j] = (int16_t)( + (temp1W32 - ((int32_t)A_upd_hi[j] << 16)) >> 1); } // temp3W32 = K in Q27 (Convert from Q31 to Q27) - temp3W32 = WEBRTC_SPL_RSHIFT_W32(temp3W32, 4); + temp3W32 >>= 4; // Store Anew in hi and low format - A_upd_hi[i] = (int16_t)WEBRTC_SPL_RSHIFT_W32(temp3W32, 16); - A_upd_low[i] = (int16_t)WEBRTC_SPL_RSHIFT_W32((temp3W32 - - WEBRTC_SPL_LSHIFT_W32((int32_t)A_upd_hi[i], 16)), 1); + A_upd_hi[i] = (int16_t)(temp3W32 >> 16); + A_upd_low[i] = (int16_t)( + (temp3W32 - ((int32_t)A_upd_hi[i] << 16)) >> 1); // Alpha = Alpha * (1-K^2) @@ -210,9 +204,8 @@ int16_t WebRtcSpl_LevinsonDurbin(int32_t *R, int16_t *A, int16_t *K, temp1W32 = (int32_t)0x7fffffffL - temp1W32; // 1 - K*K in Q31 // Convert 1- K^2 in hi and low format - tmp_hi = (int16_t)WEBRTC_SPL_RSHIFT_W32(temp1W32, 16); - tmp_low = (int16_t)WEBRTC_SPL_RSHIFT_W32((temp1W32 - - WEBRTC_SPL_LSHIFT_W32((int32_t)tmp_hi, 16)), 1); + tmp_hi = (int16_t)(temp1W32 >> 16); + tmp_low = (int16_t)((temp1W32 - ((int32_t)tmp_hi << 16)) >> 1); // Calculate Alpha = Alpha * (1-K^2) in Q31 temp1W32 = ((WEBRTC_SPL_MUL_16_16(Alpha_hi, tmp_hi) @@ -224,9 +217,8 @@ int16_t WebRtcSpl_LevinsonDurbin(int32_t *R, int16_t *A, int16_t *K, norm = WebRtcSpl_NormW32(temp1W32); temp1W32 = WEBRTC_SPL_LSHIFT_W32(temp1W32, norm); - Alpha_hi = (int16_t)WEBRTC_SPL_RSHIFT_W32(temp1W32, 16); - Alpha_low = (int16_t)WEBRTC_SPL_RSHIFT_W32((temp1W32 - - WEBRTC_SPL_LSHIFT_W32((int32_t)Alpha_hi, 16)), 1); + Alpha_hi = (int16_t)(temp1W32 >> 16); + Alpha_low = (int16_t)((temp1W32 - ((int32_t)Alpha_hi << 16)) >> 1); // Update the total normalization of Alpha Alpha_exp = Alpha_exp + norm; @@ -253,7 +245,7 @@ int16_t WebRtcSpl_LevinsonDurbin(int32_t *R, int16_t *A, int16_t *K, temp1W32 = WEBRTC_SPL_LSHIFT_W32((int32_t)A_hi[i], 16) + WEBRTC_SPL_LSHIFT_W32((int32_t)A_low[i], 1); // Round and store upper word - A[i] = (int16_t)WEBRTC_SPL_RSHIFT_W32((temp1W32<<1)+(int32_t)32768, 16); + A[i] = (int16_t)(((temp1W32 << 1) + 32768) >> 16); } return 1; // Stable filters } diff --git a/common_audio/signal_processing/lpc_to_refl_coef.c b/common_audio/signal_processing/lpc_to_refl_coef.c index b1a34d48..5fb4d859 100644 --- a/common_audio/signal_processing/lpc_to_refl_coef.c +++ b/common_audio/signal_processing/lpc_to_refl_coef.c @@ -32,7 +32,7 @@ void WebRtcSpl_LpcToReflCoef(int16_t* a16, int use_order, int16_t* k16) // (1 - k^2) in Q30 tmp_inv_denom32 = ((int32_t)1073741823) - WEBRTC_SPL_MUL_16_16(k16[m], k16[m]); // (1 - k^2) in Q15 - tmp_inv_denom16 = (int16_t)WEBRTC_SPL_RSHIFT_W32(tmp_inv_denom32, 15); + tmp_inv_denom16 = (int16_t)(tmp_inv_denom32 >> 15); for (k = 1; k <= m; k++) { @@ -47,7 +47,7 @@ void WebRtcSpl_LpcToReflCoef(int16_t* a16, int use_order, int16_t* k16) for (k = 1; k < m; k++) { - a16[k] = (int16_t)WEBRTC_SPL_RSHIFT_W32(tmp32[k], 1); //Q13>>1 => Q12 + a16[k] = (int16_t)(tmp32[k] >> 1); // Q13>>1 => Q12 } tmp32[m] = WEBRTC_SPL_SAT(8191, tmp32[m], -8191); diff --git a/common_audio/signal_processing/signal_processing_unittest.cc b/common_audio/signal_processing/signal_processing_unittest.cc index aa1f1787..611d2bfa 100644 --- a/common_audio/signal_processing/signal_processing_unittest.cc +++ b/common_audio/signal_processing/signal_processing_unittest.cc @@ -65,7 +65,6 @@ TEST_F(SplTest, MacroTest) { // Shifting with negative numbers not allowed // We cannot do casting here due to signed/unsigned problem - EXPECT_EQ(8191, WEBRTC_SPL_RSHIFT_W32(a, 1)); EXPECT_EQ(32766, WEBRTC_SPL_LSHIFT_W32(a, 1)); EXPECT_EQ(8191u, WEBRTC_SPL_RSHIFT_U32(a, 1)); diff --git a/common_audio/signal_processing/spl_sqrt.c b/common_audio/signal_processing/spl_sqrt.c index fff73c03..1de6ccd7 100644 --- a/common_audio/signal_processing/spl_sqrt.c +++ b/common_audio/signal_processing/spl_sqrt.c @@ -35,11 +35,10 @@ int32_t WebRtcSpl_SqrtLocal(int32_t in) + 0.875*((x_half)^5) */ - B = in; + B = in / 2; - B = WEBRTC_SPL_RSHIFT_W32(B, 1); // B = in/2 B = B - ((int32_t)0x40000000); // B = in/2 - 1/2 - x_half = (int16_t)WEBRTC_SPL_RSHIFT_W32(B, 16);// x_half = x/2 = (in-1)/2 + x_half = (int16_t)(B >> 16); // x_half = x/2 = (in-1)/2 B = B + ((int32_t)0x40000000); // B = 1 + x/2 B = B + ((int32_t)0x40000000); // Add 0.5 twice (since 1.0 does not exist in Q31) @@ -47,19 +46,18 @@ int32_t WebRtcSpl_SqrtLocal(int32_t in) A = -x2; // A = -(x/2)^2 B = B + (A >> 1); // B = 1 + x/2 - 0.5*(x/2)^2 - A = WEBRTC_SPL_RSHIFT_W32(A, 16); + A >>= 16; A = A * A * 2; // A = (x/2)^4 - t16 = (int16_t)WEBRTC_SPL_RSHIFT_W32(A, 16); + t16 = (int16_t)(A >> 16); B = B + WEBRTC_SPL_MUL_16_16(-20480, t16) * 2; // B = B - 0.625*A // After this, B = 1 + x/2 - 0.5*(x/2)^2 - 0.625*(x/2)^4 - t16 = (int16_t)WEBRTC_SPL_RSHIFT_W32(A, 16); A = WEBRTC_SPL_MUL_16_16(x_half, t16) * 2; // A = (x/2)^5 - t16 = (int16_t)WEBRTC_SPL_RSHIFT_W32(A, 16); + t16 = (int16_t)(A >> 16); B = B + WEBRTC_SPL_MUL_16_16(28672, t16) * 2; // B = B + 0.875*A // After this, B = 1 + x/2 - 0.5*(x/2)^2 - 0.625*(x/2)^4 + 0.875*(x/2)^5 - t16 = (int16_t)WEBRTC_SPL_RSHIFT_W32(x2, 16); + t16 = (int16_t)(x2 >> 16); A = WEBRTC_SPL_MUL_16_16(x_half, t16) * 2; // A = x/2^3 B = B + (A >> 1); // B = B + 0.5*A @@ -154,7 +152,7 @@ int32_t WebRtcSpl_Sqrt(int32_t value) A = WEBRTC_SPL_WORD32_MAX; } - x_norm = (int16_t)WEBRTC_SPL_RSHIFT_W32(A, 16); // x_norm = AH + x_norm = (int16_t)(A >> 16); // x_norm = AH nshift = (sh / 2); assert(nshift >= 0); @@ -166,17 +164,17 @@ int32_t WebRtcSpl_Sqrt(int32_t value) if (2 * nshift == sh) { // Even shift value case - t16 = (int16_t)WEBRTC_SPL_RSHIFT_W32(A, 16); // t16 = AH + t16 = (int16_t)(A >> 16); // t16 = AH A = WEBRTC_SPL_MUL_16_16(k_sqrt_2, t16) * 2; // A = 1/sqrt(2)*t16 A = A + ((int32_t)32768); // Round off A = A & ((int32_t)0x7fff0000); // Round off - A = WEBRTC_SPL_RSHIFT_W32(A, 15); // A = A>>16 + A >>= 15; // A = A>>16 } else { - A = WEBRTC_SPL_RSHIFT_W32(A, 16); // A = A>>16 + A >>= 16; // A = A>>16 } A = A & ((int32_t)0x0000ffff); diff --git a/common_audio/signal_processing/splitting_filter.c b/common_audio/signal_processing/splitting_filter.c index 4f6430c2..15c37240 100644 --- a/common_audio/signal_processing/splitting_filter.c +++ b/common_audio/signal_processing/splitting_filter.c @@ -156,12 +156,10 @@ void WebRtcSpl_AnalysisQMF(const int16_t* in_data, int in_data_length, // branches to get upper & lower band. for (i = 0; i < band_length; i++) { - tmp = filter1[i] + filter2[i] + 1024; - tmp = WEBRTC_SPL_RSHIFT_W32(tmp, 11); + tmp = (filter1[i] + filter2[i] + 1024) >> 11; low_band[i] = WebRtcSpl_SatW32ToW16(tmp); - tmp = filter1[i] - filter2[i] + 1024; - tmp = WEBRTC_SPL_RSHIFT_W32(tmp, 11); + tmp = (filter1[i] - filter2[i] + 1024) >> 11; high_band[i] = WebRtcSpl_SatW32ToW16(tmp); } } @@ -200,10 +198,10 @@ void WebRtcSpl_SynthesisQMF(const int16_t* low_band, const int16_t* high_band, // saturation. for (i = 0, k = 0; i < band_length; i++) { - tmp = WEBRTC_SPL_RSHIFT_W32(filter2[i] + 512, 10); + tmp = (filter2[i] + 512) >> 10; out_data[k++] = WebRtcSpl_SatW32ToW16(tmp); - tmp = WEBRTC_SPL_RSHIFT_W32(filter1[i] + 512, 10); + tmp = (filter1[i] + 512) >> 10; out_data[k++] = WebRtcSpl_SatW32ToW16(tmp); } diff --git a/common_audio/vad/include/vad.h b/common_audio/vad/include/vad.h new file mode 100644 index 00000000..f1d12123 --- /dev/null +++ b/common_audio/vad/include/vad.h @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef WEBRTC_COMMON_AUDIO_VAD_INCLUDE_VAD_H_ +#define WEBRTC_COMMON_AUDIO_VAD_INCLUDE_VAD_H_ + +#include "webrtc/base/checks.h" +#include "webrtc/common_audio/vad/include/webrtc_vad.h" +#include "webrtc/typedefs.h" + +namespace webrtc { + +// This is a C++ wrapper class for WebRtcVad. +class Vad { + public: + enum Aggressiveness { + kVadNormal = 0, + kVadLowBitrate = 1, + kVadAggressive = 2, + kVadVeryAggressive = 3 + }; + + enum Activity { kPassive = 0, kActive = 1, kError = -1 }; + + explicit Vad(enum Aggressiveness mode); + + virtual ~Vad(); + + enum Activity VoiceActivity(const int16_t* audio, + size_t num_samples, + int sample_rate_hz); + + private: + VadInst* handle_; +}; + +} // namespace webrtc +#endif // WEBRTC_COMMON_AUDIO_VAD_INCLUDE_VAD_H_ diff --git a/common_audio/vad/mock/mock_vad.h b/common_audio/vad/mock/mock_vad.h new file mode 100644 index 00000000..f1d8c226 --- /dev/null +++ b/common_audio/vad/mock/mock_vad.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef WEBRTC_COMMON_AUDIO_VAD_MOCK_MOCK_VAD_H_ +#define WEBRTC_COMMON_AUDIO_VAD_MOCK_MOCK_VAD_H_ + +#include "webrtc/common_audio/vad/include/vad.h" + +#include "testing/gmock/include/gmock/gmock.h" + +namespace webrtc { + +class MockVad : public Vad { + public: + explicit MockVad(enum Aggressiveness mode) {} + virtual ~MockVad() { Die(); } + MOCK_METHOD0(Die, void()); + + MOCK_METHOD3(VoiceActivity, + enum Activity(const int16_t* audio, + size_t num_samples, + int sample_rate_hz)); +}; + +} // namespace webrtc + +#endif // WEBRTC_COMMON_AUDIO_VAD_MOCK_MOCK_VAD_H_ diff --git a/common_audio/vad/vad.cc b/common_audio/vad/vad.cc new file mode 100644 index 00000000..9cc0c198 --- /dev/null +++ b/common_audio/vad/vad.cc @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "webrtc/common_audio/vad/include/vad.h" + +#include "webrtc/base/checks.h" + +namespace webrtc { + +Vad::Vad(enum Aggressiveness mode) { + CHECK_EQ(WebRtcVad_Create(&handle_), 0); + CHECK_EQ(WebRtcVad_Init(handle_), 0); + CHECK_EQ(WebRtcVad_set_mode(handle_, mode), 0); +} + +Vad::~Vad() { + WebRtcVad_Free(handle_); +} + +enum Vad::Activity Vad::VoiceActivity(const int16_t* audio, + size_t num_samples, + int sample_rate_hz) { + int ret = WebRtcVad_Process( + handle_, sample_rate_hz, audio, static_cast<int>(num_samples)); + switch (ret) { + case 0: + return kPassive; + case 1: + return kActive; + default: + DCHECK(false) << "WebRtcVad_Process returned an error."; + return kError; + } +} + +} // namespace webrtc diff --git a/common_audio/wav_file.cc b/common_audio/wav_file.cc new file mode 100644 index 00000000..880e1ec4 --- /dev/null +++ b/common_audio/wav_file.cc @@ -0,0 +1,166 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "webrtc/common_audio/wav_file.h" + +#include <algorithm> +#include <cstdio> +#include <limits> + +#include "webrtc/base/checks.h" +#include "webrtc/common_audio/include/audio_util.h" +#include "webrtc/common_audio/wav_header.h" + +namespace webrtc { + +// We write 16-bit PCM WAV files. +static const WavFormat kWavFormat = kWavFormatPcm; +static const int kBytesPerSample = 2; + +WavReader::WavReader(const std::string& filename) + : file_handle_(fopen(filename.c_str(), "rb")) { + CHECK(file_handle_); + uint8_t header[kWavHeaderSize]; + const size_t read = + fread(header, sizeof(*header), kWavHeaderSize, file_handle_); + CHECK_EQ(kWavHeaderSize, read); + + WavFormat format; + int bytes_per_sample; + CHECK(ReadWavHeader(header, &num_channels_, &sample_rate_, &format, + &bytes_per_sample, &num_samples_)); + CHECK_EQ(kWavFormat, format); + CHECK_EQ(kBytesPerSample, bytes_per_sample); +} + +WavReader::~WavReader() { + Close(); +} + +size_t WavReader::ReadSamples(size_t num_samples, int16_t* samples) { +#ifndef WEBRTC_ARCH_LITTLE_ENDIAN +#error "Need to convert samples to big-endian when reading from WAV file" +#endif + const size_t read = + fread(samples, sizeof(*samples), num_samples, file_handle_); + // If we didn't read what was requested, ensure we've reached the EOF. + CHECK(read == num_samples || feof(file_handle_)); + return read; +} + +size_t WavReader::ReadSamples(size_t num_samples, float* samples) { + static const size_t kChunksize = 4096 / sizeof(uint16_t); + size_t read = 0; + for (size_t i = 0; i < num_samples; i += kChunksize) { + int16_t isamples[kChunksize]; + size_t chunk = std::min(kChunksize, num_samples - i); + chunk = ReadSamples(chunk, isamples); + for (size_t j = 0; j < chunk; ++j) + samples[i + j] = isamples[j]; + read += chunk; + } + return read; +} + +void WavReader::Close() { + CHECK_EQ(0, fclose(file_handle_)); + file_handle_ = NULL; +} + +WavWriter::WavWriter(const std::string& filename, int sample_rate, + int num_channels) + : sample_rate_(sample_rate), + num_channels_(num_channels), + num_samples_(0), + file_handle_(fopen(filename.c_str(), "wb")) { + CHECK(file_handle_); + CHECK(CheckWavParameters(num_channels_, + sample_rate_, + kWavFormat, + kBytesPerSample, + num_samples_)); + + // Write a blank placeholder header, since we need to know the total number + // of samples before we can fill in the real data. + static const uint8_t blank_header[kWavHeaderSize] = {0}; + CHECK_EQ(1u, fwrite(blank_header, kWavHeaderSize, 1, file_handle_)); +} + +WavWriter::~WavWriter() { + Close(); +} + +void WavWriter::WriteSamples(const int16_t* samples, size_t num_samples) { +#ifndef WEBRTC_ARCH_LITTLE_ENDIAN +#error "Need to convert samples to little-endian when writing to WAV file" +#endif + const size_t written = + fwrite(samples, sizeof(*samples), num_samples, file_handle_); + CHECK_EQ(num_samples, written); + num_samples_ += static_cast<uint32_t>(written); + CHECK(written <= std::numeric_limits<uint32_t>::max() || + num_samples_ >= written); // detect uint32_t overflow + CHECK(CheckWavParameters(num_channels_, + sample_rate_, + kWavFormat, + kBytesPerSample, + num_samples_)); +} + +void WavWriter::WriteSamples(const float* samples, size_t num_samples) { + static const size_t kChunksize = 4096 / sizeof(uint16_t); + for (size_t i = 0; i < num_samples; i += kChunksize) { + int16_t isamples[kChunksize]; + const size_t chunk = std::min(kChunksize, num_samples - i); + FloatS16ToS16(samples + i, chunk, isamples); + WriteSamples(isamples, chunk); + } +} + +void WavWriter::Close() { + CHECK_EQ(0, fseek(file_handle_, 0, SEEK_SET)); + uint8_t header[kWavHeaderSize]; + WriteWavHeader(header, num_channels_, sample_rate_, kWavFormat, + kBytesPerSample, num_samples_); + CHECK_EQ(1u, fwrite(header, kWavHeaderSize, 1, file_handle_)); + CHECK_EQ(0, fclose(file_handle_)); + file_handle_ = NULL; +} + +} // namespace webrtc + +rtc_WavWriter* rtc_WavOpen(const char* filename, + int sample_rate, + int num_channels) { + return reinterpret_cast<rtc_WavWriter*>( + new webrtc::WavWriter(filename, sample_rate, num_channels)); +} + +void rtc_WavClose(rtc_WavWriter* wf) { + delete reinterpret_cast<webrtc::WavWriter*>(wf); +} + +void rtc_WavWriteSamples(rtc_WavWriter* wf, + const float* samples, + size_t num_samples) { + reinterpret_cast<webrtc::WavWriter*>(wf)->WriteSamples(samples, num_samples); +} + +int rtc_WavSampleRate(const rtc_WavWriter* wf) { + return reinterpret_cast<const webrtc::WavWriter*>(wf)->sample_rate(); +} + +int rtc_WavNumChannels(const rtc_WavWriter* wf) { + return reinterpret_cast<const webrtc::WavWriter*>(wf)->num_channels(); +} + +uint32_t rtc_WavNumSamples(const rtc_WavWriter* wf) { + return reinterpret_cast<const webrtc::WavWriter*>(wf)->num_samples(); +} diff --git a/common_audio/wav_file.h b/common_audio/wav_file.h new file mode 100644 index 00000000..c6c5d6b7 --- /dev/null +++ b/common_audio/wav_file.h @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef WEBRTC_COMMON_AUDIO_WAV_FILE_H_ +#define WEBRTC_COMMON_AUDIO_WAV_FILE_H_ + +#ifdef __cplusplus + +#include <stdint.h> +#include <cstddef> +#include <string> + +namespace webrtc { + +// Simple C++ class for writing 16-bit PCM WAV files. All error handling is +// by calls to CHECK(), making it unsuitable for anything but debug code. +class WavWriter { + public: + // Open a new WAV file for writing. + WavWriter(const std::string& filename, int sample_rate, int num_channels); + + // Close the WAV file, after writing its header. + ~WavWriter(); + + // Write additional samples to the file. Each sample is in the range + // [-32768,32767], and there must be the previously specified number of + // interleaved channels. + void WriteSamples(const float* samples, size_t num_samples); + void WriteSamples(const int16_t* samples, size_t num_samples); + + int sample_rate() const { return sample_rate_; } + int num_channels() const { return num_channels_; } + uint32_t num_samples() const { return num_samples_; } + + private: + void Close(); + const int sample_rate_; + const int num_channels_; + uint32_t num_samples_; // Total number of samples written to file. + FILE* file_handle_; // Output file, owned by this class +}; + +// Follows the conventions of WavWriter. +class WavReader { + public: + // Opens an existing WAV file for reading. + explicit WavReader(const std::string& filename); + + // Close the WAV file. + ~WavReader(); + + // Returns the number of samples read. If this is less than requested, + // verifies that the end of the file was reached. + size_t ReadSamples(size_t num_samples, float* samples); + size_t ReadSamples(size_t num_samples, int16_t* samples); + + int sample_rate() const { return sample_rate_; } + int num_channels() const { return num_channels_; } + uint32_t num_samples() const { return num_samples_; } + + private: + void Close(); + int sample_rate_; + int num_channels_; + uint32_t num_samples_; // Total number of samples in the file. + FILE* file_handle_; // Input file, owned by this class. +}; + +} // namespace webrtc + +extern "C" { +#endif // __cplusplus + +// C wrappers for the WavWriter class. +typedef struct rtc_WavWriter rtc_WavWriter; +rtc_WavWriter* rtc_WavOpen(const char* filename, + int sample_rate, + int num_channels); +void rtc_WavClose(rtc_WavWriter* wf); +void rtc_WavWriteSamples(rtc_WavWriter* wf, + const float* samples, + size_t num_samples); +int rtc_WavSampleRate(const rtc_WavWriter* wf); +int rtc_WavNumChannels(const rtc_WavWriter* wf); +uint32_t rtc_WavNumSamples(const rtc_WavWriter* wf); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // WEBRTC_COMMON_AUDIO_WAV_FILE_H_ diff --git a/common_audio/wav_writer_unittest.cc b/common_audio/wav_file_unittest.cc index 9c593be6..1bdb655d 100644 --- a/common_audio/wav_writer_unittest.cc +++ b/common_audio/wav_file_unittest.cc @@ -17,7 +17,7 @@ #include "testing/gtest/include/gtest/gtest.h" #include "webrtc/base/compile_assert.h" #include "webrtc/common_audio/wav_header.h" -#include "webrtc/common_audio/wav_writer.h" +#include "webrtc/common_audio/wav_file.h" #include "webrtc/test/testsupport/fileutils.h" static const float kSamples[] = {0.0, 10.0, 4e4, -1e9}; @@ -27,7 +27,7 @@ TEST(WavWriterTest, CPP) { const std::string outfile = webrtc::test::OutputPath() + "wavtest1.wav"; static const uint32_t kNumSamples = 3; { - webrtc::WavFile w(outfile, 14099, 1); + webrtc::WavWriter w(outfile, 14099, 1); EXPECT_EQ(14099, w.sample_rate()); EXPECT_EQ(1, w.num_channels()); EXPECT_EQ(0u, w.num_samples()); @@ -62,12 +62,24 @@ TEST(WavWriterTest, CPP) { ASSERT_EQ(1u, fread(contents, kContentSize, 1, f)); EXPECT_EQ(0, fclose(f)); EXPECT_EQ(0, memcmp(kExpectedContents, contents, kContentSize)); + + { + webrtc::WavReader r(outfile); + EXPECT_EQ(14099, r.sample_rate()); + EXPECT_EQ(1, r.num_channels()); + EXPECT_EQ(kNumSamples, r.num_samples()); + static const float kTruncatedSamples[] = {0.0, 10.0, 32767.0}; + float samples[kNumSamples]; + EXPECT_EQ(kNumSamples, r.ReadSamples(kNumSamples, samples)); + EXPECT_EQ(0, memcmp(kTruncatedSamples, samples, sizeof(samples))); + EXPECT_EQ(0u, r.ReadSamples(kNumSamples, samples)); + } } // Write a tiny WAV file with the C interface and verify the result. TEST(WavWriterTest, C) { const std::string outfile = webrtc::test::OutputPath() + "wavtest2.wav"; - rtc_WavFile *w = rtc_WavOpen(outfile.c_str(), 11904, 2); + rtc_WavWriter *w = rtc_WavOpen(outfile.c_str(), 11904, 2); EXPECT_EQ(11904, rtc_WavSampleRate(w)); EXPECT_EQ(2, rtc_WavNumChannels(w)); EXPECT_EQ(0u, rtc_WavNumSamples(w)); @@ -125,7 +137,7 @@ TEST(WavWriterTest, LargeFile) { samples[i + 1] = std::pow(std::cos(t * 2 * 2 * M_PI), 10) * x; } { - webrtc::WavFile w(outfile, kSampleRate, kNumChannels); + webrtc::WavWriter w(outfile, kSampleRate, kNumChannels); EXPECT_EQ(kSampleRate, w.sample_rate()); EXPECT_EQ(kNumChannels, w.num_channels()); EXPECT_EQ(0u, w.num_samples()); @@ -134,4 +146,18 @@ TEST(WavWriterTest, LargeFile) { } EXPECT_EQ(sizeof(int16_t) * kNumSamples + webrtc::kWavHeaderSize, webrtc::test::GetFileSize(outfile)); + + { + webrtc::WavReader r(outfile); + EXPECT_EQ(kSampleRate, r.sample_rate()); + EXPECT_EQ(kNumChannels, r.num_channels()); + EXPECT_EQ(kNumSamples, r.num_samples()); + + float read_samples[kNumSamples]; + EXPECT_EQ(kNumSamples, r.ReadSamples(kNumSamples, read_samples)); + for (size_t i = 0; i < kNumSamples; ++i) + EXPECT_NEAR(samples[i], read_samples[i], 1); + + EXPECT_EQ(0u, r.ReadSamples(kNumSamples, read_samples)); + } } diff --git a/common_audio/wav_header.cc b/common_audio/wav_header.cc index ce43896f..8c781fb4 100644 --- a/common_audio/wav_header.cc +++ b/common_audio/wav_header.cc @@ -18,9 +18,11 @@ #include <cstring> #include <limits> +#include "webrtc/base/checks.h" #include "webrtc/common_audio/include/audio_util.h" namespace webrtc { +namespace { struct ChunkHeader { uint32_t ID; @@ -28,6 +30,34 @@ struct ChunkHeader { }; COMPILE_ASSERT(sizeof(ChunkHeader) == 8, chunk_header_size); +// We can't nest this definition in WavHeader, because VS2013 gives an error +// on sizeof(WavHeader::fmt): "error C2070: 'unknown': illegal sizeof operand". +struct FmtSubchunk { + ChunkHeader header; + uint16_t AudioFormat; + uint16_t NumChannels; + uint32_t SampleRate; + uint32_t ByteRate; + uint16_t BlockAlign; + uint16_t BitsPerSample; +}; +COMPILE_ASSERT(sizeof(FmtSubchunk) == 24, fmt_subchunk_size); +const uint32_t kFmtSubchunkSize = sizeof(FmtSubchunk) - sizeof(ChunkHeader); + +struct WavHeader { + struct { + ChunkHeader header; + uint32_t Format; + } riff; + FmtSubchunk fmt; + struct { + ChunkHeader header; + } data; +}; +COMPILE_ASSERT(sizeof(WavHeader) == kWavHeaderSize, no_padding_in_header); + +} // namespace + bool CheckWavParameters(int num_channels, int sample_rate, WavFormat format, @@ -91,54 +121,53 @@ static inline void WriteFourCC(uint32_t* f, char a, char b, char c, char d) { | static_cast<uint32_t>(c) << 16 | static_cast<uint32_t>(d) << 24; } + +static inline uint16_t ReadLE16(uint16_t x) { return x; } +static inline uint32_t ReadLE32(uint32_t x) { return x; } +static inline std::string ReadFourCC(uint32_t x) { + return std::string(reinterpret_cast<char*>(&x), 4); +} #else #error "Write be-to-le conversion functions" #endif +static inline uint32_t RiffChunkSize(uint32_t bytes_in_payload) { + return bytes_in_payload + kWavHeaderSize - sizeof(ChunkHeader); +} + +static inline uint32_t ByteRate(int num_channels, int sample_rate, + int bytes_per_sample) { + return static_cast<uint32_t>(num_channels) * sample_rate * bytes_per_sample; +} + +static inline uint16_t BlockAlign(int num_channels, int bytes_per_sample) { + return num_channels * bytes_per_sample; +} + void WriteWavHeader(uint8_t* buf, int num_channels, int sample_rate, WavFormat format, int bytes_per_sample, uint32_t num_samples) { - assert(CheckWavParameters(num_channels, sample_rate, format, - bytes_per_sample, num_samples)); - - struct { - struct { - ChunkHeader header; - uint32_t Format; - } riff; - struct { - ChunkHeader header; - uint16_t AudioFormat; - uint16_t NumChannels; - uint32_t SampleRate; - uint32_t ByteRate; - uint16_t BlockAlign; - uint16_t BitsPerSample; - } fmt; - struct { - ChunkHeader header; - } data; - } header; - COMPILE_ASSERT(sizeof(header) == kWavHeaderSize, no_padding_in_header); + CHECK(CheckWavParameters(num_channels, sample_rate, format, + bytes_per_sample, num_samples)); + WavHeader header; const uint32_t bytes_in_payload = bytes_per_sample * num_samples; WriteFourCC(&header.riff.header.ID, 'R', 'I', 'F', 'F'); - WriteLE32(&header.riff.header.Size, - bytes_in_payload + kWavHeaderSize - sizeof(ChunkHeader)); + WriteLE32(&header.riff.header.Size, RiffChunkSize(bytes_in_payload)); WriteFourCC(&header.riff.Format, 'W', 'A', 'V', 'E'); WriteFourCC(&header.fmt.header.ID, 'f', 'm', 't', ' '); - WriteLE32(&header.fmt.header.Size, sizeof(header.fmt) - sizeof(ChunkHeader)); + WriteLE32(&header.fmt.header.Size, kFmtSubchunkSize); WriteLE16(&header.fmt.AudioFormat, format); WriteLE16(&header.fmt.NumChannels, num_channels); WriteLE32(&header.fmt.SampleRate, sample_rate); - WriteLE32(&header.fmt.ByteRate, (static_cast<uint32_t>(num_channels) - * sample_rate * bytes_per_sample)); - WriteLE16(&header.fmt.BlockAlign, num_channels * bytes_per_sample); + WriteLE32(&header.fmt.ByteRate, ByteRate(num_channels, sample_rate, + bytes_per_sample)); + WriteLE16(&header.fmt.BlockAlign, BlockAlign(num_channels, bytes_per_sample)); WriteLE16(&header.fmt.BitsPerSample, 8 * bytes_per_sample); WriteFourCC(&header.data.header.ID, 'd', 'a', 't', 'a'); @@ -149,4 +178,49 @@ void WriteWavHeader(uint8_t* buf, memcpy(buf, &header, kWavHeaderSize); } +bool ReadWavHeader(const uint8_t* buf, + int* num_channels, + int* sample_rate, + WavFormat* format, + int* bytes_per_sample, + uint32_t* num_samples) { + WavHeader header; + memcpy(&header, buf, kWavHeaderSize); + + // Parse needed fields. + *format = static_cast<WavFormat>(ReadLE16(header.fmt.AudioFormat)); + *num_channels = ReadLE16(header.fmt.NumChannels); + *sample_rate = ReadLE32(header.fmt.SampleRate); + *bytes_per_sample = ReadLE16(header.fmt.BitsPerSample) / 8; + const uint32_t bytes_in_payload = ReadLE32(header.data.header.Size); + if (*bytes_per_sample <= 0) + return false; + *num_samples = bytes_in_payload / *bytes_per_sample; + + // Sanity check remaining fields. + if (ReadFourCC(header.riff.header.ID) != "RIFF") + return false; + if (ReadFourCC(header.riff.Format) != "WAVE") + return false; + if (ReadFourCC(header.fmt.header.ID) != "fmt ") + return false; + if (ReadFourCC(header.data.header.ID) != "data") + return false; + + if (ReadLE32(header.riff.header.Size) != RiffChunkSize(bytes_in_payload)) + return false; + if (ReadLE32(header.fmt.header.Size) != kFmtSubchunkSize) + return false; + if (ReadLE32(header.fmt.ByteRate) != + ByteRate(*num_channels, *sample_rate, *bytes_per_sample)) + return false; + if (ReadLE16(header.fmt.BlockAlign) != + BlockAlign(*num_channels, *bytes_per_sample)) + return false; + + return CheckWavParameters(*num_channels, *sample_rate, *format, + *bytes_per_sample, *num_samples); +} + + } // namespace webrtc diff --git a/common_audio/wav_header.h b/common_audio/wav_header.h index f9ed8a57..37f78a6f 100644 --- a/common_audio/wav_header.h +++ b/common_audio/wav_header.h @@ -11,11 +11,12 @@ #ifndef WEBRTC_COMMON_AUDIO_WAV_HEADER_H_ #define WEBRTC_COMMON_AUDIO_WAV_HEADER_H_ +#include <stddef.h> #include <stdint.h> namespace webrtc { -static const int kWavHeaderSize = 44; +static const size_t kWavHeaderSize = 44; enum WavFormat { kWavFormatPcm = 1, // PCM, each sample of size bytes_per_sample @@ -33,7 +34,7 @@ bool CheckWavParameters(int num_channels, // Write a kWavHeaderSize bytes long WAV header to buf. The payload that // follows the header is supposed to have the specified number of interleaved // channels and contain the specified total number of samples of the specified -// type. +// type. CHECKs the input parameters for validity. void WriteWavHeader(uint8_t* buf, int num_channels, int sample_rate, @@ -41,6 +42,15 @@ void WriteWavHeader(uint8_t* buf, int bytes_per_sample, uint32_t num_samples); +// Read a kWavHeaderSize bytes long WAV header from buf and parse the values +// into the provided output parameters. Returns false if the header is invalid. +bool ReadWavHeader(const uint8_t* buf, + int* num_channels, + int* sample_rate, + WavFormat* format, + int* bytes_per_sample, + uint32_t* num_samples); + } // namespace webrtc #endif // WEBRTC_COMMON_AUDIO_WAV_HEADER_H_ diff --git a/common_audio/wav_header_unittest.cc b/common_audio/wav_header_unittest.cc index f05160ea..677affa5 100644 --- a/common_audio/wav_header_unittest.cc +++ b/common_audio/wav_header_unittest.cc @@ -48,8 +48,80 @@ TEST(WavHeaderTest, CheckWavParameters) { webrtc::CheckWavParameters(3, 8000, webrtc::kWavFormatPcm, 1, 5)); } -// Try writing a WAV header and make sure it looks OK. -TEST(WavHeaderTest, WriteWavHeader) { +TEST(WavHeaderTest, ReadWavHeaderWithErrors) { + int num_channels = 0; + int sample_rate = 0; + webrtc::WavFormat format = webrtc::kWavFormatPcm; + int bytes_per_sample = 0; + uint32_t num_samples = 0; + + // Test a few ways the header can be invalid. We start with the valid header + // used in WriteAndReadWavHeader, and invalidate one field per test. The + // invalid field is indicated in the array name, and in the comments with + // *BAD*. + static const uint8_t kBadRiffID[] = { + 'R', 'i', 'f', 'f', // *BAD* + 0xbd, 0xd0, 0x5b, 0x07, // size of whole file - 8: 123457689 + 44 - 8 + 'W', 'A', 'V', 'E', + 'f', 'm', 't', ' ', + 16, 0, 0, 0, // size of fmt block - 8: 24 - 8 + 6, 0, // format: A-law (6) + 17, 0, // channels: 17 + 0x39, 0x30, 0, 0, // sample rate: 12345 + 0xc9, 0x33, 0x03, 0, // byte rate: 1 * 17 * 12345 + 17, 0, // block align: NumChannels * BytesPerSample + 8, 0, // bits per sample: 1 * 8 + 'd', 'a', 't', 'a', + 0x99, 0xd0, 0x5b, 0x07, // size of payload: 123457689 + 0xa4, 0xa4, 0xa4, 0xa4, // untouched bytes after header + }; + EXPECT_FALSE( + webrtc::ReadWavHeader(kBadRiffID, &num_channels, &sample_rate, + &format, &bytes_per_sample, &num_samples)); + + static const uint8_t kBadBitsPerSample[] = { + 'R', 'I', 'F', 'F', + 0xbd, 0xd0, 0x5b, 0x07, // size of whole file - 8: 123457689 + 44 - 8 + 'W', 'A', 'V', 'E', + 'f', 'm', 't', ' ', + 16, 0, 0, 0, // size of fmt block - 8: 24 - 8 + 6, 0, // format: A-law (6) + 17, 0, // channels: 17 + 0x39, 0x30, 0, 0, // sample rate: 12345 + 0xc9, 0x33, 0x03, 0, // byte rate: 1 * 17 * 12345 + 17, 0, // block align: NumChannels * BytesPerSample + 1, 0, // bits per sample: *BAD* + 'd', 'a', 't', 'a', + 0x99, 0xd0, 0x5b, 0x07, // size of payload: 123457689 + 0xa4, 0xa4, 0xa4, 0xa4, // untouched bytes after header + }; + EXPECT_FALSE( + webrtc::ReadWavHeader(kBadBitsPerSample, &num_channels, &sample_rate, + &format, &bytes_per_sample, &num_samples)); + + static const uint8_t kBadByteRate[] = { + 'R', 'I', 'F', 'F', + 0xbd, 0xd0, 0x5b, 0x07, // size of whole file - 8: 123457689 + 44 - 8 + 'W', 'A', 'V', 'E', + 'f', 'm', 't', ' ', + 16, 0, 0, 0, // size of fmt block - 8: 24 - 8 + 6, 0, // format: A-law (6) + 17, 0, // channels: 17 + 0x39, 0x30, 0, 0, // sample rate: 12345 + 0x00, 0x33, 0x03, 0, // byte rate: *BAD* + 17, 0, // block align: NumChannels * BytesPerSample + 8, 0, // bits per sample: 1 * 8 + 'd', 'a', 't', 'a', + 0x99, 0xd0, 0x5b, 0x07, // size of payload: 123457689 + 0xa4, 0xa4, 0xa4, 0xa4, // untouched bytes after header + }; + EXPECT_FALSE( + webrtc::ReadWavHeader(kBadByteRate, &num_channels, &sample_rate, + &format, &bytes_per_sample, &num_samples)); +} + +// Try writing and reading a valid WAV header and make sure it looks OK. +TEST(WavHeaderTest, WriteAndReadWavHeader) { static const int kSize = 4 + webrtc::kWavHeaderSize + 4; uint8_t buf[kSize]; memset(buf, 0xa4, sizeof(buf)); @@ -74,4 +146,18 @@ TEST(WavHeaderTest, WriteWavHeader) { }; COMPILE_ASSERT(sizeof(kExpectedBuf) == kSize, buf_size); EXPECT_EQ(0, memcmp(kExpectedBuf, buf, kSize)); + + int num_channels = 0; + int sample_rate = 0; + webrtc::WavFormat format = webrtc::kWavFormatPcm; + int bytes_per_sample = 0; + uint32_t num_samples = 0; + EXPECT_TRUE( + webrtc::ReadWavHeader(buf + 4, &num_channels, &sample_rate, &format, + &bytes_per_sample, &num_samples)); + EXPECT_EQ(17, num_channels); + EXPECT_EQ(12345, sample_rate); + EXPECT_EQ(webrtc::kWavFormatALaw, format); + EXPECT_EQ(1, bytes_per_sample); + EXPECT_EQ(123457689u, num_samples); } diff --git a/common_audio/wav_writer.cc b/common_audio/wav_writer.cc deleted file mode 100644 index 30a220c2..00000000 --- a/common_audio/wav_writer.cc +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "webrtc/common_audio/wav_writer.h" - -#include <algorithm> -#include <cstdio> -#include <limits> - -#include "webrtc/base/checks.h" -#include "webrtc/common_audio/include/audio_util.h" -#include "webrtc/common_audio/wav_header.h" - -namespace webrtc { - -// We write 16-bit PCM WAV files. -static const WavFormat kWavFormat = kWavFormatPcm; -static const int kBytesPerSample = 2; - -WavFile::WavFile(const std::string& filename, int sample_rate, int num_channels) - : sample_rate_(sample_rate), - num_channels_(num_channels), - num_samples_(0), - file_handle_(fopen(filename.c_str(), "wb")) { - CHECK(file_handle_); - CHECK(CheckWavParameters(num_channels_, - sample_rate_, - kWavFormat, - kBytesPerSample, - num_samples_)); - - // Write a blank placeholder header, since we need to know the total number - // of samples before we can fill in the real data. - static const uint8_t blank_header[kWavHeaderSize] = {0}; - CHECK_EQ(1u, fwrite(blank_header, kWavHeaderSize, 1, file_handle_)); -} - -WavFile::~WavFile() { - Close(); -} - -void WavFile::WriteSamples(const int16_t* samples, size_t num_samples) { -#ifndef WEBRTC_ARCH_LITTLE_ENDIAN -#error "Need to convert samples to little-endian when writing to WAV file" -#endif - const size_t written = - fwrite(samples, sizeof(*samples), num_samples, file_handle_); - CHECK_EQ(num_samples, written); - num_samples_ += static_cast<uint32_t>(written); - CHECK(written <= std::numeric_limits<uint32_t>::max() || - num_samples_ >= written); // detect uint32_t overflow - CHECK(CheckWavParameters(num_channels_, - sample_rate_, - kWavFormat, - kBytesPerSample, - num_samples_)); -} - -void WavFile::WriteSamples(const float* samples, size_t num_samples) { - static const size_t kChunksize = 4096 / sizeof(uint16_t); - for (size_t i = 0; i < num_samples; i += kChunksize) { - int16_t isamples[kChunksize]; - const size_t chunk = std::min(kChunksize, num_samples - i); - RoundToInt16(samples + i, chunk, isamples); - WriteSamples(isamples, chunk); - } -} - -void WavFile::Close() { - CHECK_EQ(0, fseek(file_handle_, 0, SEEK_SET)); - uint8_t header[kWavHeaderSize]; - WriteWavHeader(header, num_channels_, sample_rate_, kWavFormat, - kBytesPerSample, num_samples_); - CHECK_EQ(1u, fwrite(header, kWavHeaderSize, 1, file_handle_)); - CHECK_EQ(0, fclose(file_handle_)); - file_handle_ = NULL; -} - -} // namespace webrtc - -rtc_WavFile* rtc_WavOpen(const char* filename, - int sample_rate, - int num_channels) { - return reinterpret_cast<rtc_WavFile*>( - new webrtc::WavFile(filename, sample_rate, num_channels)); -} - -void rtc_WavClose(rtc_WavFile* wf) { - delete reinterpret_cast<webrtc::WavFile*>(wf); -} - -void rtc_WavWriteSamples(rtc_WavFile* wf, - const float* samples, - size_t num_samples) { - reinterpret_cast<webrtc::WavFile*>(wf)->WriteSamples(samples, num_samples); -} - -int rtc_WavSampleRate(const rtc_WavFile* wf) { - return reinterpret_cast<const webrtc::WavFile*>(wf)->sample_rate(); -} - -int rtc_WavNumChannels(const rtc_WavFile* wf) { - return reinterpret_cast<const webrtc::WavFile*>(wf)->num_channels(); -} - -uint32_t rtc_WavNumSamples(const rtc_WavFile* wf) { - return reinterpret_cast<const webrtc::WavFile*>(wf)->num_samples(); -} diff --git a/common_audio/wav_writer.h b/common_audio/wav_writer.h deleted file mode 100644 index 09667279..00000000 --- a/common_audio/wav_writer.h +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#ifndef WEBRTC_COMMON_AUDIO_WAV_WRITER_H_ -#define WEBRTC_COMMON_AUDIO_WAV_WRITER_H_ - -#ifdef __cplusplus - -#include <stdint.h> -#include <cstddef> -#include <string> - -namespace webrtc { - -// Simple C++ class for writing 16-bit PCM WAV files. All error handling is -// by calls to CHECK(), making it unsuitable for anything but debug code. -class WavFile { - public: - // Open a new WAV file for writing. - WavFile(const std::string& filename, int sample_rate, int num_channels); - - // Close the WAV file, after writing its header. - ~WavFile(); - - // Write additional samples to the file. Each sample is in the range - // [-32768,32767], and there must be the previously specified number of - // interleaved channels. - void WriteSamples(const float* samples, size_t num_samples); - void WriteSamples(const int16_t* samples, size_t num_samples); - - int sample_rate() const { return sample_rate_; } - int num_channels() const { return num_channels_; } - uint32_t num_samples() const { return num_samples_; } - - private: - void Close(); - const int sample_rate_; - const int num_channels_; - uint32_t num_samples_; // total number of samples written to file - FILE* file_handle_; // output file, owned by this class -}; - -} // namespace webrtc - -extern "C" { -#endif // __cplusplus - -// C wrappers for the WavFile class. -typedef struct rtc_WavFile rtc_WavFile; -rtc_WavFile* rtc_WavOpen(const char* filename, - int sample_rate, - int num_channels); -void rtc_WavClose(rtc_WavFile* wf); -void rtc_WavWriteSamples(rtc_WavFile* wf, - const float* samples, - size_t num_samples); -int rtc_WavSampleRate(const rtc_WavFile* wf); -int rtc_WavNumChannels(const rtc_WavFile* wf); -uint32_t rtc_WavNumSamples(const rtc_WavFile* wf); - -#ifdef __cplusplus -} // extern "C" -#endif - -#endif // WEBRTC_COMMON_AUDIO_WAV_WRITER_H_ diff --git a/common_types.h b/common_types.h index e607ddc7..7bcfd6d4 100644 --- a/common_types.h +++ b/common_types.h @@ -202,17 +202,31 @@ struct RtcpPacketTypeCounter { RtcpPacketTypeCounter() : nack_packets(0), fir_packets(0), - pli_packets(0) {} + pli_packets(0), + nack_requests(0), + unique_nack_requests(0) {} void Add(const RtcpPacketTypeCounter& other) { nack_packets += other.nack_packets; fir_packets += other.fir_packets; pli_packets += other.pli_packets; + nack_requests += other.nack_requests; + unique_nack_requests += other.unique_nack_requests; } - uint32_t nack_packets; - uint32_t fir_packets; - uint32_t pli_packets; + int UniqueNackRequestsInPercent() const { + if (nack_requests == 0) { + return 0; + } + return static_cast<int>( + (unique_nack_requests * 100.0f / nack_requests) + 0.5f); + } + + uint32_t nack_packets; // Number of RTCP NACK packets. + uint32_t fir_packets; // Number of RTCP FIR packets. + uint32_t pli_packets; // Number of RTCP PLI packets. + uint32_t nack_requests; // Number of NACKed RTP packets. + uint32_t unique_nack_requests; // Number of unique NACKed RTP packets. }; // Data usage statistics for a (rtp) stream @@ -597,35 +611,45 @@ struct VideoCodecVP8 { } }; -// H264 specific. -struct VideoCodecH264 -{ - VideoCodecProfile profile; - bool frameDroppingOn; - int keyFrameInterval; - // These are NULL/0 if not externally negotiated. - const uint8_t* spsData; - size_t spsLen; - const uint8_t* ppsData; - size_t ppsLen; +// VP9 specific +struct VideoCodecVP9 { + VideoCodecComplexity complexity; + int resilience; + unsigned char numberOfTemporalLayers; + bool denoisingOn; + bool frameDroppingOn; + int keyFrameInterval; + bool adaptiveQpMode; }; -// Video codec types -enum VideoCodecType -{ - kVideoCodecVP8, - kVideoCodecH264, - kVideoCodecI420, - kVideoCodecRED, - kVideoCodecULPFEC, - kVideoCodecGeneric, - kVideoCodecUnknown +// H264 specific. +struct VideoCodecH264 { + VideoCodecProfile profile; + bool frameDroppingOn; + int keyFrameInterval; + // These are NULL/0 if not externally negotiated. + const uint8_t* spsData; + size_t spsLen; + const uint8_t* ppsData; + size_t ppsLen; }; -union VideoCodecUnion -{ - VideoCodecVP8 VP8; - VideoCodecH264 H264; +// Video codec types +enum VideoCodecType { + kVideoCodecVP8, + kVideoCodecVP9, + kVideoCodecH264, + kVideoCodecI420, + kVideoCodecRED, + kVideoCodecULPFEC, + kVideoCodecGeneric, + kVideoCodecUnknown +}; + +union VideoCodecUnion { + VideoCodecVP8 VP8; + VideoCodecVP9 VP9; + VideoCodecH264 H264; }; diff --git a/common_video/common_video_unittests.gyp b/common_video/common_video_unittests.gyp index 0405ba0e..9189991e 100644 --- a/common_video/common_video_unittests.gyp +++ b/common_video/common_video_unittests.gyp @@ -60,7 +60,6 @@ ], 'includes': [ '../build/isolate.gypi', - 'common_video_unittests.isolate', ], 'sources': [ 'common_video_unittests.isolate', diff --git a/common_video/common_video_unittests.isolate b/common_video/common_video_unittests.isolate index d33366c2..70823654 100644 --- a/common_video/common_video_unittests.isolate +++ b/common_video/common_video_unittests.isolate @@ -9,7 +9,7 @@ 'conditions': [ ['OS=="android"', { 'variables': { - 'isolate_dependency_untracked': [ + 'files': [ '<(DEPTH)/data/', '<(DEPTH)/resources/', ], @@ -21,15 +21,12 @@ '<(DEPTH)/testing/test_env.py', '<(PRODUCT_DIR)/common_video_unittests<(EXECUTABLE_SUFFIX)', ], - 'isolate_dependency_tracked': [ + 'files': [ '<(DEPTH)/DEPS', '<(DEPTH)/resources/foreman_cif.yuv', '<(DEPTH)/testing/test_env.py', '<(PRODUCT_DIR)/common_video_unittests<(EXECUTABLE_SUFFIX)', ], - 'isolate_dependency_untracked': [ - '<(DEPTH)/tools/swarming_client/', - ], }, }], ], @@ -39,10 +39,10 @@ std::string VideoStream::ToString() const { ss << ", max_bitrate_bps:" << max_bitrate_bps; ss << ", max_qp: " << max_qp; - ss << ", temporal_layers: {"; - for (size_t i = 0; i < temporal_layers.size(); ++i) { - ss << temporal_layers[i]; - if (i != temporal_layers.size() - 1) + ss << ", temporal_layer_thresholds_bps: {"; + for (size_t i = 0; i < temporal_layer_thresholds_bps.size(); ++i) { + ss << temporal_layer_thresholds_bps[i]; + if (i != temporal_layer_thresholds_bps.size() - 1) ss << "}, {"; } ss << '}'; @@ -104,8 +104,17 @@ struct VideoStream { int max_qp; - // Bitrate thresholds for enabling additional temporal layers. - std::vector<int> temporal_layers; + // Bitrate thresholds for enabling additional temporal layers. Since these are + // thresholds in between layers, we have one additional layer. One threshold + // gives two temporal layers, one below the threshold and one above, two give + // three, and so on. + // The VideoEncoder may redistribute bitrates over the temporal layers so a + // bitrate threshold of 100k and an estimate of 105k does not imply that we + // get 100k in one temporal layer and 5k in the other, just that the bitrate + // in the first temporal layer should not exceed 100k. + // TODO(pbos): Apart from a special case for two-layer screencast these + // thresholds are not propagated to the VideoEncoder. To be implemented. + std::vector<int> temporal_layer_thresholds_bps; }; struct VideoEncoderConfig { diff --git a/engine_configurations.h b/engine_configurations.h index e9f23097..5b093e5c 100644 --- a/engine_configurations.h +++ b/engine_configurations.h @@ -21,13 +21,14 @@ // [Voice] Codec settings // ---------------------------------------------------------------------------- -// iSAC is not included in the Mozilla build, but in all other builds. +// iSAC and G722 are not included in the Mozilla build, but in all other builds. #ifndef WEBRTC_MOZILLA_BUILD #ifdef WEBRTC_ARCH_ARM #define WEBRTC_CODEC_ISACFX // Fix-point iSAC implementation. #else #define WEBRTC_CODEC_ISAC // Floating-point iSAC implementation (default). #endif // WEBRTC_ARCH_ARM +#define WEBRTC_CODEC_G722 #endif // !WEBRTC_MOZILLA_BUILD // AVT is included in all builds, along with G.711, NetEQ and CNG @@ -37,11 +38,10 @@ // PCM16 is useful for testing and incurs only a small binary size cost. #define WEBRTC_CODEC_PCM16 -// iLBC, G.722, and Redundancy coding are excluded from Chromium and Mozilla +// iLBC and Redundancy coding are excluded from Chromium and Mozilla // builds to reduce binary size. #if !defined(WEBRTC_CHROMIUM_BUILD) && !defined(WEBRTC_MOZILLA_BUILD) #define WEBRTC_CODEC_ILBC -#define WEBRTC_CODEC_G722 #define WEBRTC_CODEC_RED #endif // !WEBRTC_CHROMIUM_BUILD && !WEBRTC_MOZILLA_BUILD @@ -51,6 +51,7 @@ #define VIDEOCODEC_I420 #define VIDEOCODEC_VP8 +#define VIDEOCODEC_VP9 #define VIDEOCODEC_H264 // ============================================================================ diff --git a/examples/android/media_demo/build.xml b/examples/android/media_demo/build.xml index c8a51dd5..17734886 100644 --- a/examples/android/media_demo/build.xml +++ b/examples/android/media_demo/build.xml @@ -1,15 +1,92 @@ <?xml version="1.0" encoding="UTF-8"?> <project name="WebRTCDemo" default="help"> + + <!-- The local.properties file is created and updated by the 'android' tool. + It contains the path to the SDK. It should *NOT* be checked into + Version Control Systems. --> <property file="local.properties" /> + + <!-- The ant.properties file can be created by you. It is only edited by the + 'android' tool to add properties to it. + This is the place to change some Ant specific build properties. + Here are some properties you may want to change/update: + + source.dir + The name of the source directory. Default is 'src'. + out.dir + The name of the output directory. Default is 'bin'. + + For other overridable properties, look at the beginning of the rules + files in the SDK, at tools/ant/build.xml + + Properties related to the SDK location or the project target should + be updated using the 'android' tool with the 'update' action. + + This file is an integral part of the build system for your + application and should be checked into Version Control Systems. + + --> + <property file="ant.properties" /> + + <!-- if sdk.dir was not set from one of the property file, then + get it from the ANDROID_HOME env var. + This must be done before we load project.properties since + the proguard config can use sdk.dir --> <property environment="env" /> <condition property="sdk.dir" value="${env.ANDROID_SDK_ROOT}"> <isset property="env.ANDROID_SDK_ROOT" /> </condition> + + <!-- The project.properties file is created and updated by the 'android' + tool, as well as ADT. + + This contains project specific properties such as project target, and library + dependencies. Lower level build properties are stored in ant.properties + (or in .classpath for Eclipse projects). + + This file is an integral part of the build system for your + application and should be checked into Version Control Systems. --> <loadproperties srcFile="project.properties" /> + + <!-- quick check on sdk.dir --> <fail message="sdk.dir is missing. Make sure to generate local.properties using 'android update project' or to inject it through the ANDROID_SDK_ROOT environment variable." unless="sdk.dir" /> + + <!-- + Import per project custom build rules if present at the root of the project. + This is the place to put custom intermediary targets such as: + -pre-build + -pre-compile + -post-compile (This is typically used for code obfuscation. + Compiled code location: ${out.classes.absolute.dir} + If this is not done in place, override ${out.dex.input.absolute.dir}) + -post-package + -post-build + -pre-clean + --> <import file="custom_rules.xml" optional="true" /> + + <!-- Import the actual build file. + + To customize existing targets, there are two options: + - Customize only one target: + - copy/paste the target into this file, *before* the + <import> task. + - customize it to your needs. + - Customize the whole content of build.xml + - copy/paste the content of the rules files (minus the top node) + into this file, replacing the <import> task. + - customize to your needs. + + *********************** + ****** IMPORTANT ****** + *********************** + In all cases you must update the value of version-tag below to read 'custom' instead of an integer, + in order to avoid having your file be overridden by tools such as "android update project" + --> + <!-- version-tag: 1 --> <import file="${sdk.dir}/tools/ant/build.xml" /> + </project> diff --git a/examples/android/media_demo/project.properties b/examples/android/media_demo/project.properties index 162fe605..8ee39b95 100644 --- a/examples/android/media_demo/project.properties +++ b/examples/android/media_demo/project.properties @@ -11,4 +11,4 @@ #proguard.config=${sdk.dir}/tools/proguard/proguard-android.txt:proguard-project.txt # Project target. -target=android-19 +target=android-21 diff --git a/examples/android/opensl_loopback/project.properties b/examples/android/opensl_loopback/project.properties index 8459f9b8..47b70783 100644 --- a/examples/android/opensl_loopback/project.properties +++ b/examples/android/opensl_loopback/project.properties @@ -11,6 +11,6 @@ #proguard.config=${sdk.dir}/tools/proguard/proguard-android.txt:proguard-project.txt # Project target. -target=android-19 +target=android-21 java.compilerargs=-Xlint:all -Werror diff --git a/modules/audio_coding/BUILD.gn b/modules/audio_coding/BUILD.gn index d1f70fa1..547f15f0 100644 --- a/modules/audio_coding/BUILD.gn +++ b/modules/audio_coding/BUILD.gn @@ -608,6 +608,8 @@ config("opus_config") { source_set("webrtc_opus") { sources = [ + "codecs/opus/audio_encoder_opus.cc", + "codecs/opus/interface/audio_encoder_opus.h", "codecs/opus/interface/opus_interface.h", "codecs/opus/opus_inst.h", "codecs/opus/opus_interface.c", diff --git a/modules/audio_coding/codecs/audio_encoder.h b/modules/audio_coding/codecs/audio_encoder.h index f8142e2b..45c0a855 100644 --- a/modules/audio_coding/codecs/audio_encoder.h +++ b/modules/audio_coding/codecs/audio_encoder.h @@ -33,13 +33,13 @@ class AudioEncoder { // output. bool Encode(uint32_t timestamp, const int16_t* audio, - size_t num_samples, + size_t num_samples_per_channel, size_t max_encoded_bytes, uint8_t* encoded, size_t* encoded_bytes, uint32_t* encoded_timestamp) { - CHECK_EQ(num_samples, - static_cast<size_t>(sample_rate_hz() / 100 * num_channels())); + CHECK_EQ(num_samples_per_channel, + static_cast<size_t>(sample_rate_hz() / 100)); bool ret = Encode(timestamp, audio, max_encoded_bytes, @@ -50,12 +50,17 @@ class AudioEncoder { return ret; } - // Returns the input sample rate in Hz, the number of input channels, and the - // number of 10 ms frames the encoder puts in one output packet. These are - // constants set at instantiation time. + // Return the input sample rate in Hz and the number of input channels. + // These are constants set at instantiation time. virtual int sample_rate_hz() const = 0; virtual int num_channels() const = 0; - virtual int num_10ms_frames_per_packet() const = 0; + + // Returns the number of 10 ms frames the encoder will put in the next + // packet. This value may only change when Encode() outputs a packet; i.e., + // the encoder may vary the number of 10 ms frames from packet to packet, but + // it must decide the length of the next packet no later than when outputting + // the preceding packet. + virtual int Num10MsFramesInNextPacket() const = 0; protected: virtual bool Encode(uint32_t timestamp, diff --git a/modules/audio_coding/codecs/g711/audio_encoder_pcm.cc b/modules/audio_coding/codecs/g711/audio_encoder_pcm.cc index ef22a277..097e11f1 100644 --- a/modules/audio_coding/codecs/g711/audio_encoder_pcm.cc +++ b/modules/audio_coding/codecs/g711/audio_encoder_pcm.cc @@ -48,7 +48,7 @@ int AudioEncoderPcm::sample_rate_hz() const { int AudioEncoderPcm::num_channels() const { return num_channels_; } -int AudioEncoderPcm::num_10ms_frames_per_packet() const { +int AudioEncoderPcm::Num10MsFramesInNextPacket() const { return num_10ms_frames_per_packet_; } diff --git a/modules/audio_coding/codecs/g711/include/audio_encoder_pcm.h b/modules/audio_coding/codecs/g711/include/audio_encoder_pcm.h index 8133987a..f6682969 100644 --- a/modules/audio_coding/codecs/g711/include/audio_encoder_pcm.h +++ b/modules/audio_coding/codecs/g711/include/audio_encoder_pcm.h @@ -32,7 +32,7 @@ class AudioEncoderPcm : public AudioEncoder { virtual int sample_rate_hz() const OVERRIDE; virtual int num_channels() const OVERRIDE; - virtual int num_10ms_frames_per_packet() const OVERRIDE; + virtual int Num10MsFramesInNextPacket() const OVERRIDE; protected: virtual bool Encode(uint32_t timestamp, diff --git a/modules/audio_coding/codecs/isac/fix/source/isacfix.gypi b/modules/audio_coding/codecs/isac/fix/source/isacfix.gypi index 7bef170d..2a36309f 100644 --- a/modules/audio_coding/codecs/isac/fix/source/isacfix.gypi +++ b/modules/audio_coding/codecs/isac/fix/source/isacfix.gypi @@ -87,7 +87,7 @@ 'pitch_filter_c.c', ], }], - ['target_arch=="mipsel"', { + ['target_arch=="mipsel" and mips_arch_variant!="r6"', { 'sources': [ 'entropy_coding_mips.c', 'filters_mips.c', diff --git a/modules/audio_coding/codecs/isac/main/test/ReleaseTest-API/ReleaseTest-API.cc b/modules/audio_coding/codecs/isac/main/test/ReleaseTest-API/ReleaseTest-API.cc index 567ec85b..ffdcc0c1 100644 --- a/modules/audio_coding/codecs/isac/main/test/ReleaseTest-API/ReleaseTest-API.cc +++ b/modules/audio_coding/codecs/isac/main/test/ReleaseTest-API/ReleaseTest-API.cc @@ -448,7 +448,7 @@ int main(int argc, char* argv[]) { printf(" Error iSAC Cannot write file %s.\n", outname); cout << flush; - getchar(); + getc(stdin); exit(1); } if(VADusage) diff --git a/modules/audio_coding/codecs/isac/main/test/simpleKenny.c b/modules/audio_coding/codecs/isac/main/test/simpleKenny.c index 1e752a18..d10b4add 100644 --- a/modules/audio_coding/codecs/isac/main/test/simpleKenny.c +++ b/modules/audio_coding/codecs/isac/main/test/simpleKenny.c @@ -383,7 +383,7 @@ valid values are 8 and 16.\n", sampFreqKHz); // exit if returned with error //errType=WebRtcIsac_GetErrorCode(ISAC_main_inst); fprintf(stderr,"\nError in encoder\n"); - getchar(); + getc(stdin); exit(EXIT_FAILURE); } @@ -479,7 +479,7 @@ valid values are 8 and 16.\n", sampFreqKHz); { //errType=WebRtcIsac_GetErrorCode(ISAC_main_inst); fprintf(stderr,"\nError in decoder.\n"); - getchar(); + getc(stdin); exit(1); } diff --git a/modules/audio_coding/codecs/opus/audio_encoder_opus.cc b/modules/audio_coding/codecs/opus/audio_encoder_opus.cc new file mode 100644 index 00000000..6349b5c2 --- /dev/null +++ b/modules/audio_coding/codecs/opus/audio_encoder_opus.cc @@ -0,0 +1,104 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "webrtc/modules/audio_coding/codecs/opus/interface/audio_encoder_opus.h" + +#include "webrtc/modules/audio_coding/codecs/opus/interface/opus_interface.h" + +namespace webrtc { + +namespace { + +// We always encode at 48 kHz. +const int kSampleRateHz = 48000; + +int DivExact(int a, int b) { + CHECK_EQ(a % b, 0); + return a / b; +} + +int16_t ClampInt16(size_t x) { + return static_cast<int16_t>( + std::min(x, static_cast<size_t>(std::numeric_limits<int16_t>::max()))); +} + +int16_t CastInt16(size_t x) { + DCHECK_LE(x, static_cast<size_t>(std::numeric_limits<int16_t>::max())); + return static_cast<int16_t>(x); +} + +} // namespace + +AudioEncoderOpus::Config::Config() : frame_size_ms(20), num_channels(1) {} + +bool AudioEncoderOpus::Config::IsOk() const { + if (frame_size_ms <= 0 || frame_size_ms % 10 != 0) + return false; + if (num_channels <= 0) + return false; + return true; +} + +AudioEncoderOpus::AudioEncoderOpus(const Config& config) + : num_10ms_frames_per_packet_(DivExact(config.frame_size_ms, 10)), + num_channels_(config.num_channels), + samples_per_10ms_frame_(DivExact(kSampleRateHz, 100) * num_channels_) { + CHECK(config.IsOk()); + input_buffer_.reserve(num_10ms_frames_per_packet_ * samples_per_10ms_frame_); + CHECK_EQ(0, WebRtcOpus_EncoderCreate(&inst_, num_channels_)); +} + +AudioEncoderOpus::~AudioEncoderOpus() { + CHECK_EQ(0, WebRtcOpus_EncoderFree(inst_)); +} + +int AudioEncoderOpus::sample_rate_hz() const { + return kSampleRateHz; +} + +int AudioEncoderOpus::num_channels() const { + return num_channels_; +} + +int AudioEncoderOpus::Num10MsFramesInNextPacket() const { + return num_10ms_frames_per_packet_; +} + +bool AudioEncoderOpus::Encode(uint32_t timestamp, + const int16_t* audio, + size_t max_encoded_bytes, + uint8_t* encoded, + size_t* encoded_bytes, + uint32_t* encoded_timestamp) { + if (input_buffer_.empty()) + first_timestamp_in_buffer_ = timestamp; + input_buffer_.insert(input_buffer_.end(), audio, + audio + samples_per_10ms_frame_); + if (input_buffer_.size() < (static_cast<size_t>(num_10ms_frames_per_packet_) * + samples_per_10ms_frame_)) { + *encoded_bytes = 0; + return true; + } + CHECK_EQ(input_buffer_.size(), + static_cast<size_t>(num_10ms_frames_per_packet_) * + samples_per_10ms_frame_); + int16_t r = WebRtcOpus_Encode( + inst_, &input_buffer_[0], + DivExact(CastInt16(input_buffer_.size()), num_channels_), + ClampInt16(max_encoded_bytes), encoded); + input_buffer_.clear(); + if (r < 0) + return false; + *encoded_bytes = r; + *encoded_timestamp = first_timestamp_in_buffer_; + return true; +} + +} // namespace webrtc diff --git a/modules/audio_coding/codecs/opus/interface/audio_encoder_opus.h b/modules/audio_coding/codecs/opus/interface/audio_encoder_opus.h new file mode 100644 index 00000000..e2e5c73f --- /dev/null +++ b/modules/audio_coding/codecs/opus/interface/audio_encoder_opus.h @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef WEBRTC_MODULES_AUDIO_CODING_CODECS_OPUS_INTERFACE_AUDIO_ENCODER_OPUS_H_ +#define WEBRTC_MODULES_AUDIO_CODING_CODECS_OPUS_INTERFACE_AUDIO_ENCODER_OPUS_H_ + +#include <vector> + +#include "webrtc/modules/audio_coding/codecs/opus/interface/opus_interface.h" +#include "webrtc/modules/audio_coding/codecs/audio_encoder.h" + +namespace webrtc { + +class AudioEncoderOpus : public AudioEncoder { + public: + struct Config { + Config(); + bool IsOk() const; + int frame_size_ms; + int num_channels; + }; + + explicit AudioEncoderOpus(const Config& config); + virtual ~AudioEncoderOpus() OVERRIDE; + + virtual int sample_rate_hz() const OVERRIDE; + virtual int num_channels() const OVERRIDE; + virtual int Num10MsFramesInNextPacket() const OVERRIDE; + + protected: + virtual bool Encode(uint32_t timestamp, + const int16_t* audio, + size_t max_encoded_bytes, + uint8_t* encoded, + size_t* encoded_bytes, + uint32_t* encoded_timestamp) OVERRIDE; + + private: + const int num_10ms_frames_per_packet_; + const int num_channels_; + const int samples_per_10ms_frame_; + std::vector<int16_t> input_buffer_; + OpusEncInst* inst_; + uint32_t first_timestamp_in_buffer_; +}; + +} // namespace webrtc +#endif // WEBRTC_MODULES_AUDIO_CODING_CODECS_OPUS_INTERFACE_AUDIO_ENCODER_OPUS_H_ diff --git a/modules/audio_coding/codecs/opus/opus.gypi b/modules/audio_coding/codecs/opus/opus.gypi index 89f0a54a..b537285a 100644 --- a/modules/audio_coding/codecs/opus/opus.gypi +++ b/modules/audio_coding/codecs/opus/opus.gypi @@ -27,6 +27,8 @@ '<(webrtc_root)', ], 'sources': [ + 'audio_encoder_opus.cc', + 'interface/audio_encoder_opus.h', 'interface/opus_interface.h', 'opus_inst.h', 'opus_interface.c', diff --git a/modules/audio_coding/codecs/tools/audio_codec_speed_tests.gypi b/modules/audio_coding/codecs/tools/audio_codec_speed_tests.gypi index 6503a51e..a9a5bb94 100644 --- a/modules/audio_coding/codecs/tools/audio_codec_speed_tests.gypi +++ b/modules/audio_coding/codecs/tools/audio_codec_speed_tests.gypi @@ -55,7 +55,6 @@ ], 'includes': [ '../../../../build/isolate.gypi', - 'audio_codec_speed_tests.isolate', ], 'sources': [ 'audio_codec_speed_tests.isolate', diff --git a/modules/audio_coding/codecs/tools/audio_codec_speed_tests.isolate b/modules/audio_coding/codecs/tools/audio_codec_speed_tests.isolate index 8c5a2bd0..ed7599d7 100644 --- a/modules/audio_coding/codecs/tools/audio_codec_speed_tests.isolate +++ b/modules/audio_coding/codecs/tools/audio_codec_speed_tests.isolate @@ -9,7 +9,7 @@ 'conditions': [ ['OS=="android"', { 'variables': { - 'isolate_dependency_untracked': [ + 'files': [ '<(DEPTH)/resources/', '<(DEPTH)/data/', ], @@ -21,19 +21,14 @@ '<(DEPTH)/testing/test_env.py', '<(PRODUCT_DIR)/audio_codec_speed_tests<(EXECUTABLE_SUFFIX)', ], - 'isolate_dependency_touched': [ + 'files': [ '<(DEPTH)/DEPS', - ], - 'isolate_dependency_tracked': [ '<(DEPTH)/resources/audio_coding/music_stereo_48kHz.pcm', '<(DEPTH)/resources/audio_coding/speech_mono_16kHz.pcm', '<(DEPTH)/resources/audio_coding/speech_mono_32_48kHz.pcm', '<(DEPTH)/testing/test_env.py', '<(PRODUCT_DIR)/audio_codec_speed_tests<(EXECUTABLE_SUFFIX)', ], - 'isolate_dependency_untracked': [ - '<(DEPTH)/tools/swarming_client/', - ], }, }], ], diff --git a/modules/audio_coding/main/acm2/acm_isac.cc b/modules/audio_coding/main/acm2/acm_isac.cc index 6ee26825..bc20c961 100644 --- a/modules/audio_coding/main/acm2/acm_isac.cc +++ b/modules/audio_coding/main/acm2/acm_isac.cc @@ -262,8 +262,7 @@ static uint16_t ACMISACFixGetDecSampRate(ACM_ISAC_STRUCT* /* inst */) { #endif ACMISAC::ACMISAC(int16_t codec_id) - : AudioDecoder(ACMCodecDB::neteq_decoders_[codec_id]), - codec_inst_crit_sect_(CriticalSectionWrapper::CreateCriticalSection()), + : codec_inst_crit_sect_(CriticalSectionWrapper::CreateCriticalSection()), is_enc_initialized_(false), isac_coding_mode_(CHANNEL_INDEPENDENT), enforce_frame_size_(false), diff --git a/modules/audio_coding/main/test/APITest.cc b/modules/audio_coding/main/test/APITest.cc index 82940fa3..b6e47145 100644 --- a/modules/audio_coding/main/test/APITest.cc +++ b/modules/audio_coding/main/test/APITest.cc @@ -503,7 +503,7 @@ void APITest::RunTest(char thread) { break; default: fprintf(stderr, "Wrong Test Number\n"); - getchar(); + getc(stdin); exit(1); } } diff --git a/modules/audio_coding/neteq/audio_decoder.cc b/modules/audio_coding/neteq/audio_decoder.cc index 0fdaa44b..04a74eef 100644 --- a/modules/audio_coding/neteq/audio_decoder.cc +++ b/modules/audio_coding/neteq/audio_decoder.cc @@ -51,8 +51,6 @@ bool AudioDecoder::PacketHasFec(const uint8_t* encoded, return false; } -NetEqDecoder AudioDecoder::codec_type() const { return codec_type_; } - bool AudioDecoder::CodecSupported(NetEqDecoder codec_type) { switch (codec_type) { case kDecoderPCMu: @@ -197,26 +195,24 @@ AudioDecoder* AudioDecoder::CreateAudioDecoder(NetEqDecoder codec_type) { return new AudioDecoderIsacFix; #elif defined(WEBRTC_CODEC_ISAC) case kDecoderISAC: - return new AudioDecoderIsac; -#endif -#ifdef WEBRTC_CODEC_ISAC + return new AudioDecoderIsac(16000); case kDecoderISACswb: - return new AudioDecoderIsacSwb; case kDecoderISACfb: - return new AudioDecoderIsacFb; + return new AudioDecoderIsac(32000); #endif #ifdef WEBRTC_CODEC_PCM16 case kDecoderPCM16B: case kDecoderPCM16Bwb: case kDecoderPCM16Bswb32kHz: case kDecoderPCM16Bswb48kHz: - return new AudioDecoderPcm16B(codec_type); + return new AudioDecoderPcm16B; case kDecoderPCM16B_2ch: case kDecoderPCM16Bwb_2ch: case kDecoderPCM16Bswb32kHz_2ch: case kDecoderPCM16Bswb48kHz_2ch: + return new AudioDecoderPcm16BMultiCh(2); case kDecoderPCM16B_5ch: - return new AudioDecoderPcm16BMultiCh(codec_type); + return new AudioDecoderPcm16BMultiCh(5); #endif #ifdef WEBRTC_CODEC_G722 case kDecoderG722: @@ -226,19 +222,21 @@ AudioDecoder* AudioDecoder::CreateAudioDecoder(NetEqDecoder codec_type) { #endif #ifdef WEBRTC_CODEC_CELT case kDecoderCELT_32: + return new AudioDecoderCelt(1); case kDecoderCELT_32_2ch: - return new AudioDecoderCelt(codec_type); + return new AudioDecoderCelt(2); #endif #ifdef WEBRTC_CODEC_OPUS case kDecoderOpus: + return new AudioDecoderOpus(1); case kDecoderOpus_2ch: - return new AudioDecoderOpus(codec_type); + return new AudioDecoderOpus(2); #endif case kDecoderCNGnb: case kDecoderCNGwb: case kDecoderCNGswb32kHz: case kDecoderCNGswb48kHz: - return new AudioDecoderCng(codec_type); + return new AudioDecoderCng; case kDecoderRED: case kDecoderAVT: case kDecoderArbitrary: diff --git a/modules/audio_coding/neteq/audio_decoder_impl.cc b/modules/audio_coding/neteq/audio_decoder_impl.cc index 661f2b11..0215f36d 100644 --- a/modules/audio_coding/neteq/audio_decoder_impl.cc +++ b/modules/audio_coding/neteq/audio_decoder_impl.cc @@ -13,6 +13,7 @@ #include <assert.h> #include <string.h> // memmove +#include "webrtc/base/checks.h" #ifdef WEBRTC_CODEC_CELT #include "webrtc/modules/audio_coding/codecs/celt/include/celt_interface.h" #endif @@ -75,13 +76,7 @@ int AudioDecoderPcmA::PacketDuration(const uint8_t* encoded, // PCM16B #ifdef WEBRTC_CODEC_PCM16 -AudioDecoderPcm16B::AudioDecoderPcm16B(enum NetEqDecoder type) - : AudioDecoder(type) { - assert(type == kDecoderPCM16B || - type == kDecoderPCM16Bwb || - type == kDecoderPCM16Bswb32kHz || - type == kDecoderPCM16Bswb48kHz); -} +AudioDecoderPcm16B::AudioDecoderPcm16B() {} int AudioDecoderPcm16B::Decode(const uint8_t* encoded, size_t encoded_len, int16_t* decoded, SpeechType* speech_type) { @@ -99,29 +94,15 @@ int AudioDecoderPcm16B::PacketDuration(const uint8_t* encoded, return static_cast<int>(encoded_len / (2 * channels_)); } -AudioDecoderPcm16BMultiCh::AudioDecoderPcm16BMultiCh( - enum NetEqDecoder type) - : AudioDecoderPcm16B(kDecoderPCM16B) { // This will be changed below. - codec_type_ = type; // Changing to actual type here. - switch (codec_type_) { - case kDecoderPCM16B_2ch: - case kDecoderPCM16Bwb_2ch: - case kDecoderPCM16Bswb32kHz_2ch: - case kDecoderPCM16Bswb48kHz_2ch: - channels_ = 2; - break; - case kDecoderPCM16B_5ch: - channels_ = 5; - break; - default: - assert(false); - } +AudioDecoderPcm16BMultiCh::AudioDecoderPcm16BMultiCh(int num_channels) { + DCHECK(num_channels > 0); + channels_ = num_channels; } #endif // iLBC #ifdef WEBRTC_CODEC_ILBC -AudioDecoderIlbc::AudioDecoderIlbc() : AudioDecoder(kDecoderILBC) { +AudioDecoderIlbc::AudioDecoderIlbc() { WebRtcIlbcfix_DecoderCreate(reinterpret_cast<iLBC_decinst_t**>(&state_)); } @@ -152,9 +133,11 @@ int AudioDecoderIlbc::Init() { // iSAC float #ifdef WEBRTC_CODEC_ISAC -AudioDecoderIsac::AudioDecoderIsac() : AudioDecoder(kDecoderISAC) { +AudioDecoderIsac::AudioDecoderIsac(int decode_sample_rate_hz) { + DCHECK(decode_sample_rate_hz == 16000 || decode_sample_rate_hz == 32000); WebRtcIsac_Create(reinterpret_cast<ISACStruct**>(&state_)); - WebRtcIsac_SetDecSampRate(static_cast<ISACStruct*>(state_), 16000); + WebRtcIsac_SetDecSampRate(static_cast<ISACStruct*>(state_), + decode_sample_rate_hz); } AudioDecoderIsac::~AudioDecoderIsac() { @@ -209,22 +192,11 @@ int AudioDecoderIsac::IncomingPacket(const uint8_t* payload, int AudioDecoderIsac::ErrorCode() { return WebRtcIsac_GetErrorCode(static_cast<ISACStruct*>(state_)); } - -// iSAC SWB -AudioDecoderIsacSwb::AudioDecoderIsacSwb() : AudioDecoderIsac() { - codec_type_ = kDecoderISACswb; - WebRtcIsac_SetDecSampRate(static_cast<ISACStruct*>(state_), 32000); -} - -// iSAC FB -AudioDecoderIsacFb::AudioDecoderIsacFb() : AudioDecoderIsacSwb() { - codec_type_ = kDecoderISACfb; -} #endif // iSAC fix #ifdef WEBRTC_CODEC_ISACFX -AudioDecoderIsacFix::AudioDecoderIsacFix() : AudioDecoder(kDecoderISAC) { +AudioDecoderIsacFix::AudioDecoderIsacFix() { WebRtcIsacfix_Create(reinterpret_cast<ISACFIX_MainStruct**>(&state_)); } @@ -266,7 +238,7 @@ int AudioDecoderIsacFix::ErrorCode() { // G.722 #ifdef WEBRTC_CODEC_G722 -AudioDecoderG722::AudioDecoderG722() : AudioDecoder(kDecoderG722) { +AudioDecoderG722::AudioDecoderG722() { WebRtcG722_CreateDecoder(reinterpret_cast<G722DecInst**>(&state_)); } @@ -382,14 +354,9 @@ void AudioDecoderG722Stereo::SplitStereoPacket(const uint8_t* encoded, // CELT #ifdef WEBRTC_CODEC_CELT -AudioDecoderCelt::AudioDecoderCelt(enum NetEqDecoder type) - : AudioDecoder(type) { - assert(type == kDecoderCELT_32 || type == kDecoderCELT_32_2ch); - if (type == kDecoderCELT_32) { - channels_ = 1; - } else { - channels_ = 2; - } +AudioDecoderCelt::AudioDecoderCelt(int num_channels) { + DCHECK(num_channels == 1 || num_channels == 2); + channels_ = num_channels; WebRtcCelt_CreateDec(reinterpret_cast<CELT_decinst_t**>(&state_), static_cast<int>(channels_)); } @@ -431,13 +398,9 @@ int AudioDecoderCelt::DecodePlc(int num_frames, int16_t* decoded) { // Opus #ifdef WEBRTC_CODEC_OPUS -AudioDecoderOpus::AudioDecoderOpus(enum NetEqDecoder type) - : AudioDecoder(type) { - if (type == kDecoderOpus_2ch) { - channels_ = 2; - } else { - channels_ = 1; - } +AudioDecoderOpus::AudioDecoderOpus(int num_channels) { + DCHECK(num_channels == 1 || num_channels == 2); + channels_ = num_channels; WebRtcOpus_DecoderCreate(reinterpret_cast<OpusDecInst**>(&state_), static_cast<int>(channels_)); } @@ -494,10 +457,7 @@ bool AudioDecoderOpus::PacketHasFec(const uint8_t* encoded, } #endif -AudioDecoderCng::AudioDecoderCng(enum NetEqDecoder type) - : AudioDecoder(type) { - assert(type == kDecoderCNGnb || type == kDecoderCNGwb || - kDecoderCNGswb32kHz || type == kDecoderCNGswb48kHz); +AudioDecoderCng::AudioDecoderCng() { WebRtcCng_CreateDec(reinterpret_cast<CNG_dec_inst**>(&state_)); assert(state_); } diff --git a/modules/audio_coding/neteq/audio_decoder_impl.h b/modules/audio_coding/neteq/audio_decoder_impl.h index 265d660b..214392e7 100644 --- a/modules/audio_coding/neteq/audio_decoder_impl.h +++ b/modules/audio_coding/neteq/audio_decoder_impl.h @@ -26,7 +26,7 @@ namespace webrtc { class AudioDecoderPcmU : public AudioDecoder { public: - AudioDecoderPcmU() : AudioDecoder(kDecoderPCMu) {} + AudioDecoderPcmU() {} virtual int Decode(const uint8_t* encoded, size_t encoded_len, int16_t* decoded, SpeechType* speech_type); virtual int Init() { return 0; } @@ -38,7 +38,7 @@ class AudioDecoderPcmU : public AudioDecoder { class AudioDecoderPcmA : public AudioDecoder { public: - AudioDecoderPcmA() : AudioDecoder(kDecoderPCMa) {} + AudioDecoderPcmA() {} virtual int Decode(const uint8_t* encoded, size_t encoded_len, int16_t* decoded, SpeechType* speech_type); virtual int Init() { return 0; } @@ -75,7 +75,7 @@ class AudioDecoderPcmAMultiCh : public AudioDecoderPcmA { // The type is specified in the constructor parameter |type|. class AudioDecoderPcm16B : public AudioDecoder { public: - explicit AudioDecoderPcm16B(enum NetEqDecoder type); + AudioDecoderPcm16B(); virtual int Decode(const uint8_t* encoded, size_t encoded_len, int16_t* decoded, SpeechType* speech_type); virtual int Init() { return 0; } @@ -90,7 +90,7 @@ class AudioDecoderPcm16B : public AudioDecoder { // of channels is derived from the type. class AudioDecoderPcm16BMultiCh : public AudioDecoderPcm16B { public: - explicit AudioDecoderPcm16BMultiCh(enum NetEqDecoder type); + explicit AudioDecoderPcm16BMultiCh(int num_channels); private: DISALLOW_COPY_AND_ASSIGN(AudioDecoderPcm16BMultiCh); @@ -116,7 +116,7 @@ class AudioDecoderIlbc : public AudioDecoder { #ifdef WEBRTC_CODEC_ISAC class AudioDecoderIsac : public AudioDecoder { public: - AudioDecoderIsac(); + explicit AudioDecoderIsac(int decode_sample_rate_hz); virtual ~AudioDecoderIsac(); virtual int Decode(const uint8_t* encoded, size_t encoded_len, int16_t* decoded, SpeechType* speech_type); @@ -135,22 +135,6 @@ class AudioDecoderIsac : public AudioDecoder { private: DISALLOW_COPY_AND_ASSIGN(AudioDecoderIsac); }; - -class AudioDecoderIsacSwb : public AudioDecoderIsac { - public: - AudioDecoderIsacSwb(); - - private: - DISALLOW_COPY_AND_ASSIGN(AudioDecoderIsacSwb); -}; - -class AudioDecoderIsacFb : public AudioDecoderIsacSwb { - public: - AudioDecoderIsacFb(); - - private: - DISALLOW_COPY_AND_ASSIGN(AudioDecoderIsacFb); -}; #endif #ifdef WEBRTC_CODEC_ISACFX @@ -215,7 +199,7 @@ class AudioDecoderG722Stereo : public AudioDecoderG722 { #ifdef WEBRTC_CODEC_CELT class AudioDecoderCelt : public AudioDecoder { public: - explicit AudioDecoderCelt(enum NetEqDecoder type); + explicit AudioDecoderCelt(int num_channels); virtual ~AudioDecoderCelt(); virtual int Decode(const uint8_t* encoded, size_t encoded_len, @@ -232,7 +216,7 @@ class AudioDecoderCelt : public AudioDecoder { #ifdef WEBRTC_CODEC_OPUS class AudioDecoderOpus : public AudioDecoder { public: - explicit AudioDecoderOpus(enum NetEqDecoder type); + explicit AudioDecoderOpus(int num_channels); virtual ~AudioDecoderOpus(); virtual int Decode(const uint8_t* encoded, size_t encoded_len, int16_t* decoded, SpeechType* speech_type); @@ -257,7 +241,7 @@ class AudioDecoderOpus : public AudioDecoder { // specific CngDecoder class could both inherit from that class. class AudioDecoderCng : public AudioDecoder { public: - explicit AudioDecoderCng(enum NetEqDecoder type); + explicit AudioDecoderCng(); virtual ~AudioDecoderCng(); virtual int Decode(const uint8_t* encoded, size_t encoded_len, int16_t* decoded, SpeechType* speech_type) { return -1; } diff --git a/modules/audio_coding/neteq/audio_decoder_unittest.cc b/modules/audio_coding/neteq/audio_decoder_unittest.cc index 3a5a13ff..c95214b2 100644 --- a/modules/audio_coding/neteq/audio_decoder_unittest.cc +++ b/modules/audio_coding/neteq/audio_decoder_unittest.cc @@ -26,7 +26,7 @@ #include "webrtc/modules/audio_coding/codecs/ilbc/interface/ilbc.h" #include "webrtc/modules/audio_coding/codecs/isac/fix/interface/isacfix.h" #include "webrtc/modules/audio_coding/codecs/isac/main/interface/isac.h" -#include "webrtc/modules/audio_coding/codecs/opus/interface/opus_interface.h" +#include "webrtc/modules/audio_coding/codecs/opus/interface/audio_encoder_opus.h" #include "webrtc/modules/audio_coding/codecs/pcm16b/include/pcm16b.h" #include "webrtc/modules/audio_coding/neteq/tools/resample_input_audio_file.h" #include "webrtc/system_wrappers/interface/data_log.h" @@ -140,17 +140,20 @@ class AudioDecoderTest : public ::testing::Test { size_t input_len_samples, uint8_t* output) { size_t enc_len_bytes = 0; - for (int i = 0; i < audio_encoder_->num_10ms_frames_per_packet(); ++i) { + scoped_ptr<int16_t[]> interleaved_input( + new int16_t[channels_ * input_len_samples]); + for (int i = 0; i < audio_encoder_->Num10MsFramesInNextPacket(); ++i) { EXPECT_EQ(0u, enc_len_bytes); - EXPECT_TRUE(audio_encoder_->Encode(0, - input, - audio_encoder_->sample_rate_hz() / 100, - data_length_ * 2, - output, - &enc_len_bytes, - &output_timestamp_)); + + // Duplicate the mono input signal to however many channels the test + // wants. + test::InputAudioFile::DuplicateInterleaved( + input, input_len_samples, channels_, interleaved_input.get()); + + EXPECT_TRUE(audio_encoder_->Encode( + 0, interleaved_input.get(), audio_encoder_->sample_rate_hz() / 100, + data_length_ * 2, output, &enc_len_bytes, &output_timestamp_)); } - EXPECT_EQ(input_len_samples, enc_len_bytes); return static_cast<int>(enc_len_bytes); } @@ -295,7 +298,7 @@ class AudioDecoderPcm16BTest : public AudioDecoderTest { codec_input_rate_hz_ = 8000; frame_size_ = 160; data_length_ = 10 * frame_size_; - decoder_ = new AudioDecoderPcm16B(kDecoderPCM16B); + decoder_ = new AudioDecoderPcm16B; assert(decoder_); } @@ -366,7 +369,7 @@ class AudioDecoderIsacFloatTest : public AudioDecoderTest { input_size_ = 160; frame_size_ = 480; data_length_ = 10 * frame_size_; - decoder_ = new AudioDecoderIsac; + decoder_ = new AudioDecoderIsac(16000); assert(decoder_); WebRtcIsac_Create(&encoder_); WebRtcIsac_SetEncSampRate(encoder_, 16000); @@ -404,7 +407,7 @@ class AudioDecoderIsacSwbTest : public AudioDecoderTest { input_size_ = 320; frame_size_ = 960; data_length_ = 10 * frame_size_; - decoder_ = new AudioDecoderIsacSwb; + decoder_ = new AudioDecoderIsac(32000); assert(decoder_); WebRtcIsac_Create(&encoder_); WebRtcIsac_SetEncSampRate(encoder_, 32000); @@ -435,19 +438,6 @@ class AudioDecoderIsacSwbTest : public AudioDecoderTest { int input_size_; }; -// This test is identical to AudioDecoderIsacSwbTest, except that it creates -// an AudioDecoderIsacFb decoder object. -class AudioDecoderIsacFbTest : public AudioDecoderIsacSwbTest { - protected: - AudioDecoderIsacFbTest() : AudioDecoderIsacSwbTest() { - // Delete the |decoder_| that was created by AudioDecoderIsacSwbTest and - // create an AudioDecoderIsacFb object instead. - delete decoder_; - decoder_ = new AudioDecoderIsacFb; - assert(decoder_); - } -}; - class AudioDecoderIsacFixTest : public AudioDecoderTest { protected: AudioDecoderIsacFixTest() : AudioDecoderTest() { @@ -635,57 +625,23 @@ class AudioDecoderOpusTest : public AudioDecoderTest { codec_input_rate_hz_ = 48000; frame_size_ = 480; data_length_ = 10 * frame_size_; - decoder_ = new AudioDecoderOpus(kDecoderOpus); - assert(decoder_); - WebRtcOpus_EncoderCreate(&encoder_, 1); + decoder_ = new AudioDecoderOpus(1); + AudioEncoderOpus::Config config; + config.frame_size_ms = static_cast<int>(frame_size_) / 48; + audio_encoder_.reset(new AudioEncoderOpus(config)); } - - ~AudioDecoderOpusTest() { - WebRtcOpus_EncoderFree(encoder_); - } - - virtual void InitEncoder() {} - - virtual int EncodeFrame(const int16_t* input, size_t input_len_samples, - uint8_t* output) OVERRIDE { - int enc_len_bytes = WebRtcOpus_Encode(encoder_, const_cast<int16_t*>(input), - static_cast<int16_t>(input_len_samples), - static_cast<int16_t>(data_length_), output); - EXPECT_GT(enc_len_bytes, 0); - return enc_len_bytes; - } - - OpusEncInst* encoder_; }; class AudioDecoderOpusStereoTest : public AudioDecoderOpusTest { protected: AudioDecoderOpusStereoTest() : AudioDecoderOpusTest() { channels_ = 2; - WebRtcOpus_EncoderFree(encoder_); delete decoder_; - decoder_ = new AudioDecoderOpus(kDecoderOpus_2ch); - assert(decoder_); - WebRtcOpus_EncoderCreate(&encoder_, 2); - } - - virtual int EncodeFrame(const int16_t* input, size_t input_len_samples, - uint8_t* output) OVERRIDE { - // Create stereo by duplicating each sample in |input|. - const int input_stereo_samples = static_cast<int>(input_len_samples) * 2; - scoped_ptr<int16_t[]> input_stereo(new int16_t[input_stereo_samples]); - test::InputAudioFile::DuplicateInterleaved( - input, input_len_samples, 2, input_stereo.get()); - - // Note that the input length is given as samples per channel. - int enc_len_bytes = - WebRtcOpus_Encode(encoder_, - input_stereo.get(), - static_cast<int16_t>(input_len_samples), - static_cast<int16_t>(data_length_), - output); - EXPECT_GT(enc_len_bytes, 0); - return enc_len_bytes; + decoder_ = new AudioDecoderOpus(2); + AudioEncoderOpus::Config config; + config.frame_size_ms = static_cast<int>(frame_size_) / 48; + config.num_channels = 2; + audio_encoder_.reset(new AudioEncoderOpus(config)); } }; @@ -752,17 +708,6 @@ TEST_F(AudioDecoderIsacSwbTest, EncodeDecode) { DecodePlcTest(); } -TEST_F(AudioDecoderIsacFbTest, EncodeDecode) { - int tolerance = 19757; - double mse = 8.18e6; - int delay = 160; // Delay from input to output. - EXPECT_TRUE(AudioDecoder::CodecSupported(kDecoderISACswb)); - EncodeDecodeTest(0, tolerance, mse, delay); - ReInitTest(); - EXPECT_TRUE(decoder_->HasDecodePlc()); - DecodePlcTest(); -} - TEST_F(AudioDecoderIsacFixTest, DISABLED_EncodeDecode) { int tolerance = 11034; double mse = 3.46e6; diff --git a/modules/audio_coding/neteq/audio_decoder_unittests.isolate b/modules/audio_coding/neteq/audio_decoder_unittests.isolate index e4a18ca8..f10b327b 100644 --- a/modules/audio_coding/neteq/audio_decoder_unittests.isolate +++ b/modules/audio_coding/neteq/audio_decoder_unittests.isolate @@ -9,7 +9,7 @@ 'conditions': [ ['OS=="android"', { 'variables': { - 'isolate_dependency_untracked': [ + 'files': [ '<(DEPTH)/resources/', '<(DEPTH)/data/', ], @@ -21,17 +21,12 @@ '<(DEPTH)/testing/test_env.py', '<(PRODUCT_DIR)/audio_decoder_unittests<(EXECUTABLE_SUFFIX)', ], - 'isolate_dependency_touched': [ + 'files': [ '<(DEPTH)/DEPS', - ], - 'isolate_dependency_tracked': [ '<(DEPTH)/resources/audio_coding/testfile32kHz.pcm', '<(DEPTH)/testing/test_env.py', '<(PRODUCT_DIR)/audio_decoder_unittests<(EXECUTABLE_SUFFIX)', ], - 'isolate_dependency_untracked': [ - '<(DEPTH)/tools/swarming_client/', - ], }, }], ], diff --git a/modules/audio_coding/neteq/decoder_database_unittest.cc b/modules/audio_coding/neteq/decoder_database_unittest.cc index 6f497199..1e4e58af 100644 --- a/modules/audio_coding/neteq/decoder_database_unittest.cc +++ b/modules/audio_coding/neteq/decoder_database_unittest.cc @@ -189,21 +189,18 @@ TEST(DecoderDatabase, ActiveDecoders) { EXPECT_TRUE(changed); AudioDecoder* decoder = db.GetActiveDecoder(); ASSERT_FALSE(decoder == NULL); // Should get a decoder here. - EXPECT_EQ(kDecoderPCMu, decoder->codec_type()); // Set the same again. Expect no change. EXPECT_EQ(DecoderDatabase::kOK, db.SetActiveDecoder(0, &changed)); EXPECT_FALSE(changed); decoder = db.GetActiveDecoder(); ASSERT_FALSE(decoder == NULL); // Should get a decoder here. - EXPECT_EQ(kDecoderPCMu, decoder->codec_type()); // Change active decoder. EXPECT_EQ(DecoderDatabase::kOK, db.SetActiveDecoder(103, &changed)); EXPECT_TRUE(changed); decoder = db.GetActiveDecoder(); ASSERT_FALSE(decoder == NULL); // Should get a decoder here. - EXPECT_EQ(kDecoderISAC, decoder->codec_type()); // Remove the active decoder, and verify that the active becomes NULL. EXPECT_EQ(DecoderDatabase::kOK, db.Remove(103)); @@ -213,7 +210,6 @@ TEST(DecoderDatabase, ActiveDecoders) { EXPECT_EQ(DecoderDatabase::kOK, db.SetActiveCngDecoder(13)); decoder = db.GetActiveCngDecoder(); ASSERT_FALSE(decoder == NULL); // Should get a decoder here. - EXPECT_EQ(kDecoderCNGnb, decoder->codec_type()); // Remove the active CNG decoder, and verify that the active becomes NULL. EXPECT_EQ(DecoderDatabase::kOK, db.Remove(13)); diff --git a/modules/audio_coding/neteq/interface/audio_decoder.h b/modules/audio_coding/neteq/interface/audio_decoder.h index 9a2fb8b4..16d78c9e 100644 --- a/modules/audio_coding/neteq/interface/audio_decoder.h +++ b/modules/audio_coding/neteq/interface/audio_decoder.h @@ -63,12 +63,7 @@ class AudioDecoder { // Used by PacketDuration below. Save the value -1 for errors. enum { kNotImplemented = -2 }; - explicit AudioDecoder(enum NetEqDecoder type) - : codec_type_(type), - channels_(1), - state_(NULL) { - } - + AudioDecoder() : channels_(1), state_(NULL) {} virtual ~AudioDecoder() {} // Decodes |encode_len| bytes from |encoded| and writes the result in @@ -119,8 +114,6 @@ class AudioDecoder { // Returns true if the packet has FEC and false otherwise. virtual bool PacketHasFec(const uint8_t* encoded, size_t encoded_len) const; - virtual NetEqDecoder codec_type() const; - // Returns the underlying decoder state. void* state() { return state_; } @@ -140,7 +133,6 @@ class AudioDecoder { protected: static SpeechType ConvertSpeechType(int16_t type); - enum NetEqDecoder codec_type_; size_t channels_; void* state_; diff --git a/modules/audio_coding/neteq/interface/neteq.h b/modules/audio_coding/neteq/interface/neteq.h index 925cb231..560e77ba 100644 --- a/modules/audio_coding/neteq/interface/neteq.h +++ b/modules/audio_coding/neteq/interface/neteq.h @@ -248,7 +248,7 @@ class NetEq { // Returns the error code for the last occurred error. If no error has // occurred, 0 is returned. - virtual int LastError() = 0; + virtual int LastError() const = 0; // Returns the error code last returned by a decoder (audio or comfort noise). // When LastError() returns kDecoderErrorCode or kComfortNoiseErrorCode, check diff --git a/modules/audio_coding/neteq/mock/mock_audio_decoder.h b/modules/audio_coding/neteq/mock/mock_audio_decoder.h index f3cecc24..95b564d1 100644 --- a/modules/audio_coding/neteq/mock/mock_audio_decoder.h +++ b/modules/audio_coding/neteq/mock/mock_audio_decoder.h @@ -19,7 +19,7 @@ namespace webrtc { class MockAudioDecoder : public AudioDecoder { public: - MockAudioDecoder() : AudioDecoder(kDecoderArbitrary) {} + MockAudioDecoder() {} virtual ~MockAudioDecoder() { Die(); } MOCK_METHOD0(Die, void()); MOCK_METHOD4(Decode, int(const uint8_t*, size_t, int16_t*, diff --git a/modules/audio_coding/neteq/mock/mock_external_decoder_pcm16b.h b/modules/audio_coding/neteq/mock/mock_external_decoder_pcm16b.h index ef5e03e5..c15fa1ac 100644 --- a/modules/audio_coding/neteq/mock/mock_external_decoder_pcm16b.h +++ b/modules/audio_coding/neteq/mock/mock_external_decoder_pcm16b.h @@ -27,9 +27,7 @@ using ::testing::Invoke; // audio_decoder_impl.{cc, h}. class ExternalPcm16B : public AudioDecoder { public: - explicit ExternalPcm16B(enum NetEqDecoder type) - : AudioDecoder(type) { - } + ExternalPcm16B() {} virtual int Decode(const uint8_t* encoded, size_t encoded_len, int16_t* decoded, SpeechType* speech_type) { @@ -51,9 +49,7 @@ class ExternalPcm16B : public AudioDecoder { // The reason is that we can then track that the correct calls are being made. class MockExternalPcm16B : public ExternalPcm16B { public: - explicit MockExternalPcm16B(enum NetEqDecoder type) - : ExternalPcm16B(type), - real_(type) { + MockExternalPcm16B() { // By default, all calls are delegated to the real object. ON_CALL(*this, Decode(_, _, _, _)) .WillByDefault(Invoke(&real_, &ExternalPcm16B::Decode)); @@ -67,8 +63,6 @@ class MockExternalPcm16B : public ExternalPcm16B { .WillByDefault(Invoke(&real_, &ExternalPcm16B::IncomingPacket)); ON_CALL(*this, ErrorCode()) .WillByDefault(Invoke(&real_, &ExternalPcm16B::ErrorCode)); - ON_CALL(*this, codec_type()) - .WillByDefault(Invoke(&real_, &ExternalPcm16B::codec_type)); } virtual ~MockExternalPcm16B() { Die(); } diff --git a/modules/audio_coding/neteq/neteq.gypi b/modules/audio_coding/neteq/neteq.gypi index 73cb0eb3..c68651d6 100644 --- a/modules/audio_coding/neteq/neteq.gypi +++ b/modules/audio_coding/neteq/neteq.gypi @@ -225,7 +225,6 @@ ], 'includes': [ '../../../build/isolate.gypi', - 'audio_decoder_unittests.isolate', ], 'sources': [ 'audio_decoder_unittests.isolate', diff --git a/modules/audio_coding/neteq/neteq_external_decoder_unittest.cc b/modules/audio_coding/neteq/neteq_external_decoder_unittest.cc index 6a8eafa1..2e07b490 100644 --- a/modules/audio_coding/neteq/neteq_external_decoder_unittest.cc +++ b/modules/audio_coding/neteq/neteq_external_decoder_unittest.cc @@ -47,7 +47,7 @@ class NetEqExternalDecoderTest : public ::testing::Test { frame_size_ms_(10), frame_size_samples_(frame_size_ms_ * samples_per_ms_), output_size_samples_(frame_size_ms_ * samples_per_ms_), - external_decoder_(new MockExternalPcm16B(kDecoderPCM16Bswb32kHz)), + external_decoder_(new MockExternalPcm16B), rtp_generator_(new test::RtpGenerator(samples_per_ms_)), payload_size_bytes_(0), last_send_time_(0), @@ -241,7 +241,7 @@ class LargeTimestampJumpTest : public NetEqExternalDecoderTest { frame_size_samples_ = frame_size_ms_ * samples_per_ms_; output_size_samples_ = frame_size_ms_ * samples_per_ms_; EXPECT_CALL(*external_decoder_, Die()).Times(1); - external_decoder_.reset(new MockExternalPcm16B(kDecoderPCM16B)); + external_decoder_.reset(new MockExternalPcm16B); } void SetUp() OVERRIDE { diff --git a/modules/audio_coding/neteq/neteq_impl.cc b/modules/audio_coding/neteq/neteq_impl.cc index edf618ef..44faa22a 100644 --- a/modules/audio_coding/neteq/neteq_impl.cc +++ b/modules/audio_coding/neteq/neteq_impl.cc @@ -352,7 +352,7 @@ bool NetEqImpl::GetPlayoutTimestamp(uint32_t* timestamp) { return true; } -int NetEqImpl::LastError() { +int NetEqImpl::LastError() const { CriticalSectionScoped lock(crit_sect_.get()); return error_code_; } diff --git a/modules/audio_coding/neteq/neteq_impl.h b/modules/audio_coding/neteq/neteq_impl.h index fc2284d9..348f483c 100644 --- a/modules/audio_coding/neteq/neteq_impl.h +++ b/modules/audio_coding/neteq/neteq_impl.h @@ -178,7 +178,7 @@ class NetEqImpl : public webrtc::NetEq { // Returns the error code for the last occurred error. If no error has // occurred, 0 is returned. - virtual int LastError() OVERRIDE; + virtual int LastError() const OVERRIDE; // Returns the error code last returned by a decoder (audio or comfort noise). // When LastError() returns kDecoderErrorCode or kComfortNoiseErrorCode, check diff --git a/modules/audio_coding/neteq/neteq_impl_unittest.cc b/modules/audio_coding/neteq/neteq_impl_unittest.cc index d5676d7b..b3bd69ba 100644 --- a/modules/audio_coding/neteq/neteq_impl_unittest.cc +++ b/modules/audio_coding/neteq/neteq_impl_unittest.cc @@ -422,8 +422,7 @@ TEST_F(NetEqImplTest, VerifyTimestampPropagation) { // sample, and then increasing by 1 for each sample. class CountingSamplesDecoder : public AudioDecoder { public: - explicit CountingSamplesDecoder(enum NetEqDecoder type) - : AudioDecoder(type), next_value_(1) {} + CountingSamplesDecoder() : next_value_(1) {} // Produce as many samples as input bytes (|encoded_len|). virtual int Decode(const uint8_t* encoded, @@ -446,7 +445,7 @@ TEST_F(NetEqImplTest, VerifyTimestampPropagation) { private: int16_t next_value_; - } decoder_(kDecoderPCM16B); + } decoder_; EXPECT_EQ(NetEq::kOK, neteq_->RegisterExternalDecoder( diff --git a/modules/audio_coding/neteq/tools/neteq_rtpplay.cc b/modules/audio_coding/neteq/tools/neteq_rtpplay.cc index 46ef3d08..ef2c0b6b 100644 --- a/modules/audio_coding/neteq/tools/neteq_rtpplay.cc +++ b/modules/audio_coding/neteq/tools/neteq_rtpplay.cc @@ -92,6 +92,9 @@ static const bool isac_dummy = DEFINE_int32(isac_swb, 104, "RTP payload type for iSAC-swb (32 kHz)"); static const bool isac_swb_dummy = google::RegisterFlagValidator(&FLAGS_isac_swb, &ValidatePayloadType); +DEFINE_int32(opus, 111, "RTP payload type for Opus"); +static const bool opus_dummy = + google::RegisterFlagValidator(&FLAGS_opus, &ValidatePayloadType); DEFINE_int32(pcm16b, 93, "RTP payload type for PCM16b-nb (8 kHz)"); static const bool pcm16b_dummy = google::RegisterFlagValidator(&FLAGS_pcm16b, &ValidatePayloadType); @@ -286,8 +289,24 @@ int main(int argc, char* argv[]) { static_cast<int>(payload_len), packet->time_ms() * sample_rate_hz / 1000); if (error != NetEq::kOK) { - std::cerr << "InsertPacket returned error code " << neteq->LastError() - << std::endl; + if (neteq->LastError() == NetEq::kUnknownRtpPayloadType) { + std::cerr << "RTP Payload type " + << static_cast<int>(rtp_header.header.payloadType) + << " is unknown." << std::endl; + std::cerr << "Use --codec_map to view default mapping." << std::endl; + std::cerr << "Use --helpshort for information on how to make custom " + "mappings." << std::endl; + } else { + std::cerr << "InsertPacket returned error code " << neteq->LastError() + << std::endl; + std::cerr << "Header data:" << std::endl; + std::cerr << " PT = " + << static_cast<int>(rtp_header.header.payloadType) + << std::endl; + std::cerr << " SN = " << rtp_header.header.sequenceNumber + << std::endl; + std::cerr << " TS = " << rtp_header.header.timestamp << std::endl; + } } // Get next packet from file. @@ -366,6 +385,8 @@ std::string CodecName(webrtc::NetEqDecoder codec) { return "iSAC"; case webrtc::kDecoderISACswb: return "iSAC-swb (32 kHz)"; + case webrtc::kDecoderOpus: + return "Opus"; case webrtc::kDecoderPCM16B: return "PCM16b-nb (8 kHz)"; case webrtc::kDecoderPCM16Bwb: @@ -428,6 +449,12 @@ void RegisterPayloadTypes(NetEq* neteq) { " as " << CodecName(webrtc::kDecoderISACswb).c_str() << std::endl; exit(1); } + error = neteq->RegisterPayloadType(webrtc::kDecoderOpus, FLAGS_opus); + if (error) { + std::cerr << "Cannot register payload type " << FLAGS_opus << " as " + << CodecName(webrtc::kDecoderOpus).c_str() << std::endl; + exit(1); + } error = neteq->RegisterPayloadType(webrtc::kDecoderPCM16B, FLAGS_pcm16b); if (error) { std::cerr << "Cannot register payload type " << FLAGS_pcm16b << @@ -514,6 +541,8 @@ void PrintCodecMapping() { std::endl; std::cout << CodecName(webrtc::kDecoderISACswb).c_str() << ": " << FLAGS_isac_swb << std::endl; + std::cout << CodecName(webrtc::kDecoderOpus).c_str() << ": " << FLAGS_opus + << std::endl; std::cout << CodecName(webrtc::kDecoderPCM16B).c_str() << ": " << FLAGS_pcm16b << std::endl; std::cout << CodecName(webrtc::kDecoderPCM16Bwb).c_str() << ": " << @@ -637,8 +666,8 @@ int CodecSampleRate(uint8_t payload_type) { payload_type == FLAGS_pcm16b_swb32 || payload_type == FLAGS_cn_swb32) { return 32000; - } else if (payload_type == FLAGS_pcm16b_swb48 || - payload_type == FLAGS_cn_swb48) { + } else if (payload_type == FLAGS_opus || payload_type == FLAGS_pcm16b_swb48 || + payload_type == FLAGS_cn_swb48) { return 48000; } else if (payload_type == FLAGS_avt || payload_type == FLAGS_red) { diff --git a/modules/audio_device/audio_device.gypi b/modules/audio_device/audio_device.gypi index 23f417f9..add3be2a 100644 --- a/modules/audio_device/audio_device.gypi +++ b/modules/audio_device/audio_device.gypi @@ -260,7 +260,6 @@ ], 'includes': [ '../../build/isolate.gypi', - 'audio_device_tests.isolate', ], 'sources': [ 'audio_device_tests.isolate', diff --git a/modules/audio_device/audio_device_tests.isolate b/modules/audio_device/audio_device_tests.isolate index ebe8bfb4..a3550b79 100644 --- a/modules/audio_device/audio_device_tests.isolate +++ b/modules/audio_device/audio_device_tests.isolate @@ -9,7 +9,7 @@ 'conditions': [ ['OS=="android"', { 'variables': { - 'isolate_dependency_untracked': [ + 'files': [ '<(DEPTH)/data/', '<(DEPTH)/resources/', ], @@ -21,13 +21,10 @@ '<(DEPTH)/testing/test_env.py', '<(PRODUCT_DIR)/audio_device_tests<(EXECUTABLE_SUFFIX)', ], - 'isolate_dependency_tracked': [ + 'files': [ '<(DEPTH)/testing/test_env.py', '<(PRODUCT_DIR)/audio_device_tests<(EXECUTABLE_SUFFIX)', ], - 'isolate_dependency_untracked': [ - '<(DEPTH)/tools/swarming_client/', - ], }, }], ], diff --git a/modules/audio_device/audio_device_utility.cc b/modules/audio_device/audio_device_utility.cc index b6c5c482..182329c4 100644 --- a/modules/audio_device/audio_device_utility.cc +++ b/modules/audio_device/audio_device_utility.cc @@ -82,9 +82,9 @@ void AudioDeviceUtility::WaitForKey() // choose enter out of all available keys - if (getchar() == '\n') + if (getc(stdin) == '\n') { - getchar(); + getc(stdin); } tcsetattr( STDIN_FILENO, TCSANOW, &oldt ); diff --git a/modules/audio_device/win/audio_device_core_win.cc b/modules/audio_device/win/audio_device_core_win.cc index 3708c540..bcf1c1bb 100644 --- a/modules/audio_device/win/audio_device_core_win.cc +++ b/modules/audio_device/win/audio_device_core_win.cc @@ -3893,6 +3893,12 @@ DWORD AudioDeviceWindowsCore::DoCaptureThread() // This value is fixed during the capturing session. // UINT32 bufferLength = 0; + if (_ptrClientIn == NULL) + { + WEBRTC_TRACE(kTraceError, kTraceAudioDevice, _id, + "input state has been modified before capture loop starts."); + return 1; + } hr = _ptrClientIn->GetBufferSize(&bufferLength); EXIT_ON_ERROR(hr); WEBRTC_TRACE(kTraceInfo, kTraceAudioDevice, _id, "[CAPT] size of buffer : %u", bufferLength); @@ -4113,7 +4119,10 @@ DWORD AudioDeviceWindowsCore::DoCaptureThread() // ---------------------------- THREAD LOOP ---------------------------- << - hr = _ptrClientIn->Stop(); + if (_ptrClientIn) + { + hr = _ptrClientIn->Stop(); + } Exit: if (FAILED(hr)) diff --git a/modules/audio_processing/aec/aec_core.c b/modules/audio_processing/aec/aec_core.c index 1e217eb5..50457d9b 100644 --- a/modules/audio_processing/aec/aec_core.c +++ b/modules/audio_processing/aec/aec_core.c @@ -1351,7 +1351,7 @@ int WebRtcAec_FreeAec(AecCore* aec) { #ifdef WEBRTC_AEC_DEBUG_DUMP // Open a new Wav file for writing. If it was already open with a different // sample frequency, close it first. -static void ReopenWav(rtc_WavFile** wav_file, +static void ReopenWav(rtc_WavWriter** wav_file, const char* name, int seq1, int seq2, diff --git a/modules/audio_processing/aec/aec_core_internal.h b/modules/audio_processing/aec/aec_core_internal.h index 6adc4d68..5e30366d 100644 --- a/modules/audio_processing/aec/aec_core_internal.h +++ b/modules/audio_processing/aec/aec_core_internal.h @@ -11,7 +11,7 @@ #ifndef WEBRTC_MODULES_AUDIO_PROCESSING_AEC_AEC_CORE_INTERNAL_H_ #define WEBRTC_MODULES_AUDIO_PROCESSING_AEC_AEC_CORE_INTERNAL_H_ -#include "webrtc/common_audio/wav_writer.h" +#include "webrtc/common_audio/wav_file.h" #include "webrtc/modules/audio_processing/aec/aec_common.h" #include "webrtc/modules/audio_processing/aec/aec_core.h" #include "webrtc/modules/audio_processing/utility/ring_buffer.h" @@ -147,10 +147,10 @@ struct AecCore { int debug_dump_count; RingBuffer* far_time_buf; - rtc_WavFile* farFile; - rtc_WavFile* nearFile; - rtc_WavFile* outFile; - rtc_WavFile* outLinearFile; + rtc_WavWriter* farFile; + rtc_WavWriter* nearFile; + rtc_WavWriter* outFile; + rtc_WavWriter* outLinearFile; #endif }; diff --git a/modules/audio_processing/audio_buffer.cc b/modules/audio_processing/audio_buffer.cc index 8aff61cc..63d69cfb 100644 --- a/modules/audio_processing/audio_buffer.cc +++ b/modules/audio_processing/audio_buffer.cc @@ -51,18 +51,11 @@ int KeyboardChannelIndex(AudioProcessing::ChannelLayout layout) { return -1; } -void StereoToMono(const float* left, const float* right, float* out, +template <typename T> +void StereoToMono(const T* left, const T* right, T* out, int samples_per_channel) { - for (int i = 0; i < samples_per_channel; ++i) { + for (int i = 0; i < samples_per_channel; ++i) out[i] = (left[i] + right[i]) / 2; - } -} - -void StereoToMono(const int16_t* left, const int16_t* right, int16_t* out, - int samples_per_channel) { - for (int i = 0; i < samples_per_channel; ++i) { - out[i] = (left[i] + right[i]) >> 1; - } } } // namespace @@ -114,13 +107,7 @@ class IFChannelBuffer { void RefreshI() { if (!ivalid_) { assert(fvalid_); - const float* const float_data = fbuf_.data(); - int16_t* const int_data = ibuf_.data(); - const int length = ibuf_.length(); - for (int i = 0; i < length; ++i) - int_data[i] = WEBRTC_SPL_SAT(std::numeric_limits<int16_t>::max(), - float_data[i], - std::numeric_limits<int16_t>::min()); + FloatS16ToS16(fbuf_.data(), ibuf_.length(), ibuf_.data()); ivalid_ = true; } } @@ -228,10 +215,10 @@ void AudioBuffer::CopyFrom(const float* const* data, data_ptr = process_buffer_->channels(); } - // Convert to int16. + // Convert to the S16 range. for (int i = 0; i < num_proc_channels_; ++i) { - ScaleAndRoundToInt16(data_ptr[i], proc_samples_per_channel_, - channels_->ibuf()->channel(i)); + FloatToFloatS16(data_ptr[i], proc_samples_per_channel_, + channels_->fbuf()->channel(i)); } } @@ -241,16 +228,15 @@ void AudioBuffer::CopyTo(int samples_per_channel, assert(samples_per_channel == output_samples_per_channel_); assert(ChannelsFromLayout(layout) == num_proc_channels_); - // Convert to float. + // Convert to the float range. float* const* data_ptr = data; if (output_samples_per_channel_ != proc_samples_per_channel_) { // Convert to an intermediate buffer for subsequent resampling. data_ptr = process_buffer_->channels(); } for (int i = 0; i < num_proc_channels_; ++i) { - ScaleToFloat(channels_->ibuf()->channel(i), - proc_samples_per_channel_, - data_ptr[i]); + FloatS16ToFloat(channels_->fbuf()->channel(i), proc_samples_per_channel_, + data_ptr[i]); } // Resample. @@ -449,12 +435,7 @@ void AudioBuffer::DeinterleaveFrom(AudioFrame* frame) { // Downmix directly; no explicit deinterleaving needed. int16_t* downmixed = channels_->ibuf()->channel(0); for (int i = 0; i < input_samples_per_channel_; ++i) { - // HACK(ajm): The downmixing in the int16_t path is in practice never - // called from production code. We do this weird scaling to and from float - // to satisfy tests checking for bit-exactness with the float path. - float downmix_float = (ScaleToFloat(frame->data_[i * 2]) + - ScaleToFloat(frame->data_[i * 2 + 1])) / 2; - downmixed[i] = ScaleAndRoundToInt16(downmix_float); + downmixed[i] = (frame->data_[i * 2] + frame->data_[i * 2 + 1]) / 2; } } else { assert(num_proc_channels_ == num_input_channels_); diff --git a/modules/audio_processing/audio_processing.gypi b/modules/audio_processing/audio_processing.gypi index ce65f643..2ddcffc9 100644 --- a/modules/audio_processing/audio_processing.gypi +++ b/modules/audio_processing/audio_processing.gypi @@ -112,7 +112,7 @@ 'ns/nsx_defines.h', ], 'conditions': [ - ['target_arch=="mipsel"', { + ['target_arch=="mipsel" and mips_arch_variant!="r6"', { 'sources': [ 'ns/nsx_core_mips.c', ], @@ -139,7 +139,7 @@ ['(target_arch=="arm" and arm_version==7) or target_arch=="armv7"', { 'dependencies': ['audio_processing_neon',], }], - ['target_arch=="mipsel"', { + ['target_arch=="mipsel" and mips_arch_variant!="r6"', { 'sources': [ 'aecm/aecm_core_mips.c', ], diff --git a/modules/audio_processing/ns/ns_core.c b/modules/audio_processing/ns/ns_core.c index 1195a8cb..e026c29e 100644 --- a/modules/audio_processing/ns/ns_core.c +++ b/modules/audio_processing/ns/ns_core.c @@ -20,279 +20,279 @@ #include "webrtc/modules/audio_processing/utility/fft4g.h" // Set Feature Extraction Parameters. -static void set_feature_extraction_parameters(NSinst_t* inst) { +static void set_feature_extraction_parameters(NSinst_t* self) { // Bin size of histogram. - inst->featureExtractionParams.binSizeLrt = 0.1f; - inst->featureExtractionParams.binSizeSpecFlat = 0.05f; - inst->featureExtractionParams.binSizeSpecDiff = 0.1f; + self->featureExtractionParams.binSizeLrt = 0.1f; + self->featureExtractionParams.binSizeSpecFlat = 0.05f; + self->featureExtractionParams.binSizeSpecDiff = 0.1f; // Range of histogram over which LRT threshold is computed. - inst->featureExtractionParams.rangeAvgHistLrt = 1.f; + self->featureExtractionParams.rangeAvgHistLrt = 1.f; // Scale parameters: multiply dominant peaks of the histograms by scale factor // to obtain thresholds for prior model. // For LRT and spectral difference. - inst->featureExtractionParams.factor1ModelPars = 1.2f; + self->featureExtractionParams.factor1ModelPars = 1.2f; // For spectral_flatness: used when noise is flatter than speech. - inst->featureExtractionParams.factor2ModelPars = 0.9f; + self->featureExtractionParams.factor2ModelPars = 0.9f; // Peak limit for spectral flatness (varies between 0 and 1). - inst->featureExtractionParams.thresPosSpecFlat = 0.6f; + self->featureExtractionParams.thresPosSpecFlat = 0.6f; // Limit on spacing of two highest peaks in histogram: spacing determined by // bin size. - inst->featureExtractionParams.limitPeakSpacingSpecFlat = - 2 * inst->featureExtractionParams.binSizeSpecFlat; - inst->featureExtractionParams.limitPeakSpacingSpecDiff = - 2 * inst->featureExtractionParams.binSizeSpecDiff; + self->featureExtractionParams.limitPeakSpacingSpecFlat = + 2 * self->featureExtractionParams.binSizeSpecFlat; + self->featureExtractionParams.limitPeakSpacingSpecDiff = + 2 * self->featureExtractionParams.binSizeSpecDiff; // Limit on relevance of second peak. - inst->featureExtractionParams.limitPeakWeightsSpecFlat = 0.5f; - inst->featureExtractionParams.limitPeakWeightsSpecDiff = 0.5f; + self->featureExtractionParams.limitPeakWeightsSpecFlat = 0.5f; + self->featureExtractionParams.limitPeakWeightsSpecDiff = 0.5f; // Fluctuation limit of LRT feature. - inst->featureExtractionParams.thresFluctLrt = 0.05f; + self->featureExtractionParams.thresFluctLrt = 0.05f; // Limit on the max and min values for the feature thresholds. - inst->featureExtractionParams.maxLrt = 1.f; - inst->featureExtractionParams.minLrt = 0.2f; + self->featureExtractionParams.maxLrt = 1.f; + self->featureExtractionParams.minLrt = 0.2f; - inst->featureExtractionParams.maxSpecFlat = 0.95f; - inst->featureExtractionParams.minSpecFlat = 0.1f; + self->featureExtractionParams.maxSpecFlat = 0.95f; + self->featureExtractionParams.minSpecFlat = 0.1f; - inst->featureExtractionParams.maxSpecDiff = 1.f; - inst->featureExtractionParams.minSpecDiff = 0.16f; + self->featureExtractionParams.maxSpecDiff = 1.f; + self->featureExtractionParams.minSpecDiff = 0.16f; // Criteria of weight of histogram peak to accept/reject feature. - inst->featureExtractionParams.thresWeightSpecFlat = - (int)(0.3 * (inst->modelUpdatePars[1])); // For spectral flatness. - inst->featureExtractionParams.thresWeightSpecDiff = - (int)(0.3 * (inst->modelUpdatePars[1])); // For spectral difference. + self->featureExtractionParams.thresWeightSpecFlat = + (int)(0.3 * (self->modelUpdatePars[1])); // For spectral flatness. + self->featureExtractionParams.thresWeightSpecDiff = + (int)(0.3 * (self->modelUpdatePars[1])); // For spectral difference. } // Initialize state. -int WebRtcNs_InitCore(NSinst_t* inst, uint32_t fs) { +int WebRtcNs_InitCore(NSinst_t* self, uint32_t fs) { int i; // Check for valid pointer. - if (inst == NULL) { + if (self == NULL) { return -1; } // Initialization of struct. if (fs == 8000 || fs == 16000 || fs == 32000) { - inst->fs = fs; + self->fs = fs; } else { return -1; } - inst->windShift = 0; + self->windShift = 0; if (fs == 8000) { // We only support 10ms frames. - inst->blockLen = 80; - inst->anaLen = 128; - inst->window = kBlocks80w128; + self->blockLen = 80; + self->anaLen = 128; + self->window = kBlocks80w128; } else if (fs == 16000) { // We only support 10ms frames. - inst->blockLen = 160; - inst->anaLen = 256; - inst->window = kBlocks160w256; + self->blockLen = 160; + self->anaLen = 256; + self->window = kBlocks160w256; } else if (fs == 32000) { // We only support 10ms frames. - inst->blockLen = 160; - inst->anaLen = 256; - inst->window = kBlocks160w256; + self->blockLen = 160; + self->anaLen = 256; + self->window = kBlocks160w256; } - inst->magnLen = inst->anaLen / 2 + 1; // Number of frequency bins. + self->magnLen = self->anaLen / 2 + 1; // Number of frequency bins. // Initialize FFT work arrays. - inst->ip[0] = 0; // Setting this triggers initialization. - memset(inst->dataBuf, 0, sizeof(float) * ANAL_BLOCKL_MAX); - WebRtc_rdft(inst->anaLen, 1, inst->dataBuf, inst->ip, inst->wfft); + self->ip[0] = 0; // Setting this triggers initialization. + memset(self->dataBuf, 0, sizeof(float) * ANAL_BLOCKL_MAX); + WebRtc_rdft(self->anaLen, 1, self->dataBuf, self->ip, self->wfft); - memset(inst->analyzeBuf, 0, sizeof(float) * ANAL_BLOCKL_MAX); - memset(inst->dataBuf, 0, sizeof(float) * ANAL_BLOCKL_MAX); - memset(inst->syntBuf, 0, sizeof(float) * ANAL_BLOCKL_MAX); + memset(self->analyzeBuf, 0, sizeof(float) * ANAL_BLOCKL_MAX); + memset(self->dataBuf, 0, sizeof(float) * ANAL_BLOCKL_MAX); + memset(self->syntBuf, 0, sizeof(float) * ANAL_BLOCKL_MAX); // For HB processing. - memset(inst->dataBufHB, 0, sizeof(float) * ANAL_BLOCKL_MAX); + memset(self->dataBufHB, 0, sizeof(float) * ANAL_BLOCKL_MAX); // For quantile noise estimation. - memset(inst->quantile, 0, sizeof(float) * HALF_ANAL_BLOCKL); + memset(self->quantile, 0, sizeof(float) * HALF_ANAL_BLOCKL); for (i = 0; i < SIMULT * HALF_ANAL_BLOCKL; i++) { - inst->lquantile[i] = 8.f; - inst->density[i] = 0.3f; + self->lquantile[i] = 8.f; + self->density[i] = 0.3f; } for (i = 0; i < SIMULT; i++) { - inst->counter[i] = + self->counter[i] = (int)floor((float)(END_STARTUP_LONG * (i + 1)) / (float)SIMULT); } - inst->updates = 0; + self->updates = 0; // Wiener filter initialization. for (i = 0; i < HALF_ANAL_BLOCKL; i++) { - inst->smooth[i] = 1.f; + self->smooth[i] = 1.f; } // Set the aggressiveness: default. - inst->aggrMode = 0; + self->aggrMode = 0; // Initialize variables for new method. - inst->priorSpeechProb = 0.5f; // Prior prob for speech/noise. + self->priorSpeechProb = 0.5f; // Prior prob for speech/noise. // Previous analyze mag spectrum. - memset(inst->magnPrevAnalyze, 0, sizeof(float) * HALF_ANAL_BLOCKL); + memset(self->magnPrevAnalyze, 0, sizeof(float) * HALF_ANAL_BLOCKL); // Previous process mag spectrum. - memset(inst->magnPrevProcess, 0, sizeof(float) * HALF_ANAL_BLOCKL); + memset(self->magnPrevProcess, 0, sizeof(float) * HALF_ANAL_BLOCKL); // Current noise-spectrum. - memset(inst->noise, 0, sizeof(float) * HALF_ANAL_BLOCKL); + memset(self->noise, 0, sizeof(float) * HALF_ANAL_BLOCKL); // Previous noise-spectrum. - memset(inst->noisePrev, 0, sizeof(float) * HALF_ANAL_BLOCKL); + memset(self->noisePrev, 0, sizeof(float) * HALF_ANAL_BLOCKL); // Conservative noise spectrum estimate. - memset(inst->magnAvgPause, 0, sizeof(float) * HALF_ANAL_BLOCKL); + memset(self->magnAvgPause, 0, sizeof(float) * HALF_ANAL_BLOCKL); // For estimation of HB in second pass. - memset(inst->speechProb, 0, sizeof(float) * HALF_ANAL_BLOCKL); + memset(self->speechProb, 0, sizeof(float) * HALF_ANAL_BLOCKL); // Initial average magnitude spectrum. - memset(inst->initMagnEst, 0, sizeof(float) * HALF_ANAL_BLOCKL); + memset(self->initMagnEst, 0, sizeof(float) * HALF_ANAL_BLOCKL); for (i = 0; i < HALF_ANAL_BLOCKL; i++) { // Smooth LR (same as threshold). - inst->logLrtTimeAvg[i] = LRT_FEATURE_THR; + self->logLrtTimeAvg[i] = LRT_FEATURE_THR; } // Feature quantities. // Spectral flatness (start on threshold). - inst->featureData[0] = SF_FEATURE_THR; - inst->featureData[1] = 0.f; // Spectral entropy: not used in this version. - inst->featureData[2] = 0.f; // Spectral variance: not used in this version. + self->featureData[0] = SF_FEATURE_THR; + self->featureData[1] = 0.f; // Spectral entropy: not used in this version. + self->featureData[2] = 0.f; // Spectral variance: not used in this version. // Average LRT factor (start on threshold). - inst->featureData[3] = LRT_FEATURE_THR; + self->featureData[3] = LRT_FEATURE_THR; // Spectral template diff (start on threshold). - inst->featureData[4] = SF_FEATURE_THR; - inst->featureData[5] = 0.f; // Normalization for spectral difference. + self->featureData[4] = SF_FEATURE_THR; + self->featureData[5] = 0.f; // Normalization for spectral difference. // Window time-average of input magnitude spectrum. - inst->featureData[6] = 0.f; + self->featureData[6] = 0.f; // Histogram quantities: used to estimate/update thresholds for features. - memset(inst->histLrt, 0, sizeof(int) * HIST_PAR_EST); - memset(inst->histSpecFlat, 0, sizeof(int) * HIST_PAR_EST); - memset(inst->histSpecDiff, 0, sizeof(int) * HIST_PAR_EST); + memset(self->histLrt, 0, sizeof(int) * HIST_PAR_EST); + memset(self->histSpecFlat, 0, sizeof(int) * HIST_PAR_EST); + memset(self->histSpecDiff, 0, sizeof(int) * HIST_PAR_EST); - inst->blockInd = -1; // Frame counter. + self->blockInd = -1; // Frame counter. // Default threshold for LRT feature. - inst->priorModelPars[0] = LRT_FEATURE_THR; + self->priorModelPars[0] = LRT_FEATURE_THR; // Threshold for spectral flatness: determined on-line. - inst->priorModelPars[1] = 0.5f; + self->priorModelPars[1] = 0.5f; // sgn_map par for spectral measure: 1 for flatness measure. - inst->priorModelPars[2] = 1.f; + self->priorModelPars[2] = 1.f; // Threshold for template-difference feature: determined on-line. - inst->priorModelPars[3] = 0.5f; + self->priorModelPars[3] = 0.5f; // Default weighting parameter for LRT feature. - inst->priorModelPars[4] = 1.f; + self->priorModelPars[4] = 1.f; // Default weighting parameter for spectral flatness feature. - inst->priorModelPars[5] = 0.f; + self->priorModelPars[5] = 0.f; // Default weighting parameter for spectral difference feature. - inst->priorModelPars[6] = 0.f; + self->priorModelPars[6] = 0.f; // Update flag for parameters: // 0 no update, 1 = update once, 2 = update every window. - inst->modelUpdatePars[0] = 2; - inst->modelUpdatePars[1] = 500; // Window for update. + self->modelUpdatePars[0] = 2; + self->modelUpdatePars[1] = 500; // Window for update. // Counter for update of conservative noise spectrum. - inst->modelUpdatePars[2] = 0; + self->modelUpdatePars[2] = 0; // Counter if the feature thresholds are updated during the sequence. - inst->modelUpdatePars[3] = inst->modelUpdatePars[1]; + self->modelUpdatePars[3] = self->modelUpdatePars[1]; - inst->signalEnergy = 0.0; - inst->sumMagn = 0.0; - inst->whiteNoiseLevel = 0.0; - inst->pinkNoiseNumerator = 0.0; - inst->pinkNoiseExp = 0.0; + self->signalEnergy = 0.0; + self->sumMagn = 0.0; + self->whiteNoiseLevel = 0.0; + self->pinkNoiseNumerator = 0.0; + self->pinkNoiseExp = 0.0; - set_feature_extraction_parameters(inst); + set_feature_extraction_parameters(self); // Default mode. - WebRtcNs_set_policy_core(inst, 0); + WebRtcNs_set_policy_core(self, 0); - inst->initFlag = 1; + self->initFlag = 1; return 0; } // Estimate noise. -static void NoiseEstimation(NSinst_t* inst, float* magn, float* noise) { +static void NoiseEstimation(NSinst_t* self, float* magn, float* noise) { int i, s, offset; float lmagn[HALF_ANAL_BLOCKL], delta; - if (inst->updates < END_STARTUP_LONG) { - inst->updates++; + if (self->updates < END_STARTUP_LONG) { + self->updates++; } - for (i = 0; i < inst->magnLen; i++) { + for (i = 0; i < self->magnLen; i++) { lmagn[i] = (float)log(magn[i]); } // Loop over simultaneous estimates. for (s = 0; s < SIMULT; s++) { - offset = s * inst->magnLen; + offset = s * self->magnLen; // newquantest(...) - for (i = 0; i < inst->magnLen; i++) { + for (i = 0; i < self->magnLen; i++) { // Compute delta. - if (inst->density[offset + i] > 1.0) { - delta = FACTOR * 1.f / inst->density[offset + i]; + if (self->density[offset + i] > 1.0) { + delta = FACTOR * 1.f / self->density[offset + i]; } else { delta = FACTOR; } // Update log quantile estimate. - if (lmagn[i] > inst->lquantile[offset + i]) { - inst->lquantile[offset + i] += - QUANTILE * delta / (float)(inst->counter[s] + 1); + if (lmagn[i] > self->lquantile[offset + i]) { + self->lquantile[offset + i] += + QUANTILE * delta / (float)(self->counter[s] + 1); } else { - inst->lquantile[offset + i] -= - (1.f - QUANTILE) * delta / (float)(inst->counter[s] + 1); + self->lquantile[offset + i] -= + (1.f - QUANTILE) * delta / (float)(self->counter[s] + 1); } // Update density estimate. - if (fabs(lmagn[i] - inst->lquantile[offset + i]) < WIDTH) { - inst->density[offset + i] = - ((float)inst->counter[s] * inst->density[offset + i] + + if (fabs(lmagn[i] - self->lquantile[offset + i]) < WIDTH) { + self->density[offset + i] = + ((float)self->counter[s] * self->density[offset + i] + 1.f / (2.f * WIDTH)) / - (float)(inst->counter[s] + 1); + (float)(self->counter[s] + 1); } } // End loop over magnitude spectrum. - if (inst->counter[s] >= END_STARTUP_LONG) { - inst->counter[s] = 0; - if (inst->updates >= END_STARTUP_LONG) { - for (i = 0; i < inst->magnLen; i++) { - inst->quantile[i] = (float)exp(inst->lquantile[offset + i]); + if (self->counter[s] >= END_STARTUP_LONG) { + self->counter[s] = 0; + if (self->updates >= END_STARTUP_LONG) { + for (i = 0; i < self->magnLen; i++) { + self->quantile[i] = (float)exp(self->lquantile[offset + i]); } } } - inst->counter[s]++; + self->counter[s]++; } // End loop over simultaneous estimates. // Sequentially update the noise during startup. - if (inst->updates < END_STARTUP_LONG) { + if (self->updates < END_STARTUP_LONG) { // Use the last "s" to get noise during startup that differ from zero. - for (i = 0; i < inst->magnLen; i++) { - inst->quantile[i] = (float)exp(inst->lquantile[offset + i]); + for (i = 0; i < self->magnLen; i++) { + self->quantile[i] = (float)exp(self->lquantile[offset + i]); } } - for (i = 0; i < inst->magnLen; i++) { - noise[i] = inst->quantile[i]; + for (i = 0; i < self->magnLen; i++) { + noise[i] = self->quantile[i]; } } // Extract thresholds for feature parameters. // Histograms are computed over some window size (given by -// inst->modelUpdatePars[1]). +// self->modelUpdatePars[1]). // Thresholds and weights are extracted every window. // |flag| = 0 updates histogram only, |flag| = 1 computes the threshold/weights. -// Threshold and weights are returned in: inst->priorModelPars. -static void FeatureParameterExtraction(NSinst_t* const self, int flag) { +// Threshold and weights are returned in: self->priorModelPars. +static void FeatureParameterExtraction(NSinst_t* self, int flag) { int i, useFeatureSpecFlat, useFeatureSpecDiff, numHistLrt; int maxPeak1, maxPeak2; int weightPeak1SpecFlat, weightPeak2SpecFlat, weightPeak1SpecDiff, @@ -521,8 +521,8 @@ static void FeatureParameterExtraction(NSinst_t* const self, int flag) { // Compute spectral flatness on input spectrum. // |magnIn| is the magnitude spectrum. -// Spectral flatness is returned in inst->featureData[0]. -static void ComputeSpectralFlatness(NSinst_t* const self, const float* magnIn) { +// Spectral flatness is returned in self->featureData[0]. +static void ComputeSpectralFlatness(NSinst_t* self, const float* magnIn) { int i; int shiftLP = 1; // Option to remove first bin(s) from spectral measures. float avgSpectralFlatnessNum, avgSpectralFlatnessDen, spectralTmp; @@ -564,7 +564,7 @@ static void ComputeSpectralFlatness(NSinst_t* const self, const float* magnIn) { // Outputs: // * |snrLocPrior| is the computed prior SNR. // * |snrLocPost| is the computed post SNR. -static void ComputeSnr(const NSinst_t* const self, +static void ComputeSnr(const NSinst_t* self, const float* magn, const float* noise, float* snrLocPrior, @@ -591,9 +591,9 @@ static void ComputeSnr(const NSinst_t* const self, // Compute the difference measure between input spectrum and a template/learned // noise spectrum. // |magnIn| is the input spectrum. -// The reference/template spectrum is inst->magnAvgPause[i]. -// Returns (normalized) spectral difference in inst->featureData[4]. -static void ComputeSpectralDifference(NSinst_t* const self, +// The reference/template spectrum is self->magnAvgPause[i]. +// Returns (normalized) spectral difference in self->featureData[4]. +static void ComputeSpectralDifference(NSinst_t* self, const float* magnIn) { // avgDiffNormMagn = var(magnIn) - cov(magnIn, magnAvgPause)^2 / // var(magnAvgPause) @@ -640,7 +640,7 @@ static void ComputeSpectralDifference(NSinst_t* const self, // |noise| is the noise spectrum. // |snrLocPrior| is the prior SNR for each frequency. // |snrLocPost| is the post SNR for each frequency. -static void SpeechNoiseProb(NSinst_t* const self, +static void SpeechNoiseProb(NSinst_t* self, float* probSpeechFinal, const float* snrLocPrior, const float* snrLocPost) { @@ -753,7 +753,7 @@ static void SpeechNoiseProb(NSinst_t* const self, // Inputs: // * |magn| is the signal magnitude spectrum estimate. // * |updateParsFlag| is an update flag for parameters. -static void FeatureUpdate(NSinst_t* const self, +static void FeatureUpdate(NSinst_t* self, const float* magn, int updateParsFlag) { // Compute spectral flatness on input spectrum. @@ -798,7 +798,7 @@ static void FeatureUpdate(NSinst_t* const self, // * |snrLocPost| is the post SNR. // Output: // * |noise| is the updated noise magnitude spectrum estimate. -static void UpdateNoiseEstimate(NSinst_t* const self, +static void UpdateNoiseEstimate(NSinst_t* self, const float* magn, const float* snrLocPrior, const float* snrLocPost, @@ -884,7 +884,7 @@ static void UpdateBuffer(const float* frame, // * |real| is the real part of the frequency domain. // * |imag| is the imaginary part of the frequency domain. // * |magn| is the calculated signal magnitude in the frequency domain. -static void FFT(NSinst_t* const self, +static void FFT(NSinst_t* self, float* time_data, int time_data_length, int magnitude_length, @@ -921,7 +921,7 @@ static void FFT(NSinst_t* const self, // (2 * (magnitude_length - 1)). // Output: // * |time_data| is the signal in the time domain. -static void IFFT(NSinst_t* const self, +static void IFFT(NSinst_t* self, const float* real, const float* imag, int magnitude_length, @@ -983,7 +983,7 @@ static void Windowing(const float* window, // * |magn| is the signal magnitude spectrum estimate. // Output: // * |theFilter| is the frequency response of the computed Wiener filter. -static void ComputeDdBasedWienerFilter(const NSinst_t* const self, +static void ComputeDdBasedWienerFilter(const NSinst_t* self, const float* magn, float* theFilter) { int i; @@ -1011,37 +1011,37 @@ static void ComputeDdBasedWienerFilter(const NSinst_t* const self, // |mode| = 0 is mild (6dB), |mode| = 1 is medium (10dB) and |mode| = 2 is // aggressive (15dB). // Returns 0 on success and -1 otherwise. -int WebRtcNs_set_policy_core(NSinst_t* inst, int mode) { +int WebRtcNs_set_policy_core(NSinst_t* self, int mode) { // Allow for modes: 0, 1, 2, 3. if (mode < 0 || mode > 3) { return (-1); } - inst->aggrMode = mode; + self->aggrMode = mode; if (mode == 0) { - inst->overdrive = 1.f; - inst->denoiseBound = 0.5f; - inst->gainmap = 0; + self->overdrive = 1.f; + self->denoiseBound = 0.5f; + self->gainmap = 0; } else if (mode == 1) { - // inst->overdrive = 1.25f; - inst->overdrive = 1.f; - inst->denoiseBound = 0.25f; - inst->gainmap = 1; + // self->overdrive = 1.25f; + self->overdrive = 1.f; + self->denoiseBound = 0.25f; + self->gainmap = 1; } else if (mode == 2) { - // inst->overdrive = 1.25f; - inst->overdrive = 1.1f; - inst->denoiseBound = 0.125f; - inst->gainmap = 1; + // self->overdrive = 1.25f; + self->overdrive = 1.1f; + self->denoiseBound = 0.125f; + self->gainmap = 1; } else if (mode == 3) { - // inst->overdrive = 1.3f; - inst->overdrive = 1.25f; - inst->denoiseBound = 0.09f; - inst->gainmap = 1; + // self->overdrive = 1.3f; + self->overdrive = 1.25f; + self->denoiseBound = 0.09f; + self->gainmap = 1; } return 0; } -int WebRtcNs_AnalyzeCore(NSinst_t* inst, float* speechFrame) { +int WebRtcNs_AnalyzeCore(NSinst_t* self, float* speechFrame) { int i; const int kStartBand = 5; // Skip first frequency bins during estimation. int updateParsFlag; @@ -1062,16 +1062,16 @@ int WebRtcNs_AnalyzeCore(NSinst_t* inst, float* speechFrame) { float parametric_num = 0.0; // Check that initiation has been done. - if (inst->initFlag != 1) { + if (self->initFlag != 1) { return (-1); } - updateParsFlag = inst->modelUpdatePars[0]; + updateParsFlag = self->modelUpdatePars[0]; // Update analysis buffer for L band. - UpdateBuffer(speechFrame, inst->blockLen, inst->anaLen, inst->analyzeBuf); + UpdateBuffer(speechFrame, self->blockLen, self->anaLen, self->analyzeBuf); - Windowing(inst->window, inst->analyzeBuf, inst->anaLen, winData); - energy = Energy(winData, inst->anaLen); + Windowing(self->window, self->analyzeBuf, self->anaLen, winData); + energy = Energy(winData, self->anaLen); if (energy == 0.0) { // We want to avoid updating statistics in this case: // Updating feature statistics when we have zeros only will cause @@ -1084,14 +1084,14 @@ int WebRtcNs_AnalyzeCore(NSinst_t* inst, float* speechFrame) { return 0; } - inst->blockInd++; // Update the block index only when we process a block. + self->blockInd++; // Update the block index only when we process a block. - FFT(inst, winData, inst->anaLen, inst->magnLen, real, imag, magn); + FFT(self, winData, self->anaLen, self->magnLen, real, imag, magn); - for (i = 0; i < inst->magnLen; i++) { + for (i = 0; i < self->magnLen; i++) { signalEnergy += real[i] * real[i] + imag[i] * imag[i]; sumMagn += magn[i]; - if (inst->blockInd < END_STARTUP_SHORT) { + if (self->blockInd < END_STARTUP_SHORT) { if (i >= kStartBand) { tmpFloat2 = log((float)i); sum_log_i += tmpFloat2; @@ -1102,18 +1102,18 @@ int WebRtcNs_AnalyzeCore(NSinst_t* inst, float* speechFrame) { } } } - signalEnergy = signalEnergy / ((float)inst->magnLen); - inst->signalEnergy = signalEnergy; - inst->sumMagn = sumMagn; + signalEnergy = signalEnergy / ((float)self->magnLen); + self->signalEnergy = signalEnergy; + self->sumMagn = sumMagn; // Quantile noise estimate. - NoiseEstimation(inst, magn, noise); + NoiseEstimation(self, magn, noise); // Compute simplified noise model during startup. - if (inst->blockInd < END_STARTUP_SHORT) { + if (self->blockInd < END_STARTUP_SHORT) { // Estimate White noise. - inst->whiteNoiseLevel += sumMagn / ((float)inst->magnLen) * inst->overdrive; + self->whiteNoiseLevel += sumMagn / ((float)self->magnLen) * self->overdrive; // Estimate Pink noise parameters. - tmpFloat1 = sum_log_i_square * ((float)(inst->magnLen - kStartBand)); + tmpFloat1 = sum_log_i_square * ((float)(self->magnLen - kStartBand)); tmpFloat1 -= (sum_log_i * sum_log_i); tmpFloat2 = (sum_log_i_square * sum_log_magn - sum_log_i * sum_log_i_log_magn); @@ -1122,9 +1122,9 @@ int WebRtcNs_AnalyzeCore(NSinst_t* inst, float* speechFrame) { if (tmpFloat3 < 0.f) { tmpFloat3 = 0.f; } - inst->pinkNoiseNumerator += tmpFloat3; + self->pinkNoiseNumerator += tmpFloat3; tmpFloat2 = (sum_log_i * sum_log_magn); - tmpFloat2 -= ((float)(inst->magnLen - kStartBand)) * sum_log_i_log_magn; + tmpFloat2 -= ((float)(self->magnLen - kStartBand)) * sum_log_i_log_magn; tmpFloat3 = tmpFloat2 / tmpFloat1; // Constrain the pink noise power to be in the interval [0, 1]. if (tmpFloat3 < 0.f) { @@ -1133,59 +1133,59 @@ int WebRtcNs_AnalyzeCore(NSinst_t* inst, float* speechFrame) { if (tmpFloat3 > 1.f) { tmpFloat3 = 1.f; } - inst->pinkNoiseExp += tmpFloat3; + self->pinkNoiseExp += tmpFloat3; // Calculate frequency independent parts of parametric noise estimate. - if (inst->pinkNoiseExp > 0.f) { + if (self->pinkNoiseExp > 0.f) { // Use pink noise estimate. parametric_num = - exp(inst->pinkNoiseNumerator / (float)(inst->blockInd + 1)); - parametric_num *= (float)(inst->blockInd + 1); - parametric_exp = inst->pinkNoiseExp / (float)(inst->blockInd + 1); + exp(self->pinkNoiseNumerator / (float)(self->blockInd + 1)); + parametric_num *= (float)(self->blockInd + 1); + parametric_exp = self->pinkNoiseExp / (float)(self->blockInd + 1); } - for (i = 0; i < inst->magnLen; i++) { + for (i = 0; i < self->magnLen; i++) { // Estimate the background noise using the white and pink noise // parameters. - if (inst->pinkNoiseExp == 0.f) { + if (self->pinkNoiseExp == 0.f) { // Use white noise estimate. - inst->parametricNoise[i] = inst->whiteNoiseLevel; + self->parametricNoise[i] = self->whiteNoiseLevel; } else { // Use pink noise estimate. float use_band = (float)(i < kStartBand ? kStartBand : i); - inst->parametricNoise[i] = + self->parametricNoise[i] = parametric_num / pow(use_band, parametric_exp); } // Weight quantile noise with modeled noise. - noise[i] *= (inst->blockInd); + noise[i] *= (self->blockInd); tmpFloat2 = - inst->parametricNoise[i] * (END_STARTUP_SHORT - inst->blockInd); - noise[i] += (tmpFloat2 / (float)(inst->blockInd + 1)); + self->parametricNoise[i] * (END_STARTUP_SHORT - self->blockInd); + noise[i] += (tmpFloat2 / (float)(self->blockInd + 1)); noise[i] /= END_STARTUP_SHORT; } } // Compute average signal during END_STARTUP_LONG time: // used to normalize spectral difference measure. - if (inst->blockInd < END_STARTUP_LONG) { - inst->featureData[5] *= inst->blockInd; - inst->featureData[5] += signalEnergy; - inst->featureData[5] /= (inst->blockInd + 1); + if (self->blockInd < END_STARTUP_LONG) { + self->featureData[5] *= self->blockInd; + self->featureData[5] += signalEnergy; + self->featureData[5] /= (self->blockInd + 1); } // Post and prior SNR needed for SpeechNoiseProb. - ComputeSnr(inst, magn, noise, snrLocPrior, snrLocPost); + ComputeSnr(self, magn, noise, snrLocPrior, snrLocPost); - FeatureUpdate(inst, magn, updateParsFlag); - SpeechNoiseProb(inst, inst->speechProb, snrLocPrior, snrLocPost); - UpdateNoiseEstimate(inst, magn, snrLocPrior, snrLocPost, noise); + FeatureUpdate(self, magn, updateParsFlag); + SpeechNoiseProb(self, self->speechProb, snrLocPrior, snrLocPost); + UpdateNoiseEstimate(self, magn, snrLocPrior, snrLocPost, noise); // Keep track of noise spectrum for next frame. - memcpy(inst->noise, noise, sizeof(*noise) * inst->magnLen); - memcpy(inst->magnPrevAnalyze, magn, sizeof(*magn) * inst->magnLen); + memcpy(self->noise, noise, sizeof(*noise) * self->magnLen); + memcpy(self->magnPrevAnalyze, magn, sizeof(*magn) * self->magnLen); return 0; } -int WebRtcNs_ProcessCore(NSinst_t* inst, +int WebRtcNs_ProcessCore(NSinst_t* self, float* speechFrame, float* speechFrameHB, float* outFrame, @@ -1211,107 +1211,107 @@ int WebRtcNs_ProcessCore(NSinst_t* inst, float sumMagnAnalyze, sumMagnProcess; // Check that initiation has been done. - if (inst->initFlag != 1) { + if (self->initFlag != 1) { return (-1); } // Check for valid pointers based on sampling rate. - if (inst->fs == 32000) { + if (self->fs == 32000) { if (speechFrameHB == NULL) { return -1; } flagHB = 1; // Range for averaging low band quantities for H band gain. - deltaBweHB = (int)inst->magnLen / 4; + deltaBweHB = (int)self->magnLen / 4; deltaGainHB = deltaBweHB; } // Update analysis buffer for L band. - UpdateBuffer(speechFrame, inst->blockLen, inst->anaLen, inst->dataBuf); + UpdateBuffer(speechFrame, self->blockLen, self->anaLen, self->dataBuf); if (flagHB == 1) { // Update analysis buffer for H band. - UpdateBuffer(speechFrameHB, inst->blockLen, inst->anaLen, inst->dataBufHB); + UpdateBuffer(speechFrameHB, self->blockLen, self->anaLen, self->dataBufHB); } - Windowing(inst->window, inst->dataBuf, inst->anaLen, winData); - energy1 = Energy(winData, inst->anaLen); + Windowing(self->window, self->dataBuf, self->anaLen, winData); + energy1 = Energy(winData, self->anaLen); if (energy1 == 0.0) { // Synthesize the special case of zero input. // Read out fully processed segment. - for (i = inst->windShift; i < inst->blockLen + inst->windShift; i++) { - fout[i - inst->windShift] = inst->syntBuf[i]; + for (i = self->windShift; i < self->blockLen + self->windShift; i++) { + fout[i - self->windShift] = self->syntBuf[i]; } // Update synthesis buffer. - UpdateBuffer(NULL, inst->blockLen, inst->anaLen, inst->syntBuf); + UpdateBuffer(NULL, self->blockLen, self->anaLen, self->syntBuf); - for (i = 0; i < inst->blockLen; ++i) + for (i = 0; i < self->blockLen; ++i) outFrame[i] = WEBRTC_SPL_SAT(WEBRTC_SPL_WORD16_MAX, fout[i], WEBRTC_SPL_WORD16_MIN); // For time-domain gain of HB. if (flagHB == 1) - for (i = 0; i < inst->blockLen; ++i) + for (i = 0; i < self->blockLen; ++i) outFrameHB[i] = WEBRTC_SPL_SAT( - WEBRTC_SPL_WORD16_MAX, inst->dataBufHB[i], WEBRTC_SPL_WORD16_MIN); + WEBRTC_SPL_WORD16_MAX, self->dataBufHB[i], WEBRTC_SPL_WORD16_MIN); return 0; } - FFT(inst, winData, inst->anaLen, inst->magnLen, real, imag, magn); + FFT(self, winData, self->anaLen, self->magnLen, real, imag, magn); - if (inst->blockInd < END_STARTUP_SHORT) { - for (i = 0; i < inst->magnLen; i++) { - inst->initMagnEst[i] += magn[i]; + if (self->blockInd < END_STARTUP_SHORT) { + for (i = 0; i < self->magnLen; i++) { + self->initMagnEst[i] += magn[i]; } } - ComputeDdBasedWienerFilter(inst, magn, theFilter); + ComputeDdBasedWienerFilter(self, magn, theFilter); - for (i = 0; i < inst->magnLen; i++) { + for (i = 0; i < self->magnLen; i++) { // Flooring bottom. - if (theFilter[i] < inst->denoiseBound) { - theFilter[i] = inst->denoiseBound; + if (theFilter[i] < self->denoiseBound) { + theFilter[i] = self->denoiseBound; } // Flooring top. if (theFilter[i] > 1.f) { theFilter[i] = 1.f; } - if (inst->blockInd < END_STARTUP_SHORT) { + if (self->blockInd < END_STARTUP_SHORT) { theFilterTmp[i] = - (inst->initMagnEst[i] - inst->overdrive * inst->parametricNoise[i]); - theFilterTmp[i] /= (inst->initMagnEst[i] + 0.0001f); + (self->initMagnEst[i] - self->overdrive * self->parametricNoise[i]); + theFilterTmp[i] /= (self->initMagnEst[i] + 0.0001f); // Flooring bottom. - if (theFilterTmp[i] < inst->denoiseBound) { - theFilterTmp[i] = inst->denoiseBound; + if (theFilterTmp[i] < self->denoiseBound) { + theFilterTmp[i] = self->denoiseBound; } // Flooring top. if (theFilterTmp[i] > 1.f) { theFilterTmp[i] = 1.f; } // Weight the two suppression filters. - theFilter[i] *= (inst->blockInd); - theFilterTmp[i] *= (END_STARTUP_SHORT - inst->blockInd); + theFilter[i] *= (self->blockInd); + theFilterTmp[i] *= (END_STARTUP_SHORT - self->blockInd); theFilter[i] += theFilterTmp[i]; theFilter[i] /= (END_STARTUP_SHORT); } - inst->smooth[i] = theFilter[i]; - real[i] *= inst->smooth[i]; - imag[i] *= inst->smooth[i]; + self->smooth[i] = theFilter[i]; + real[i] *= self->smooth[i]; + imag[i] *= self->smooth[i]; } // Keep track of |magn| spectrum for next frame. - memcpy(inst->magnPrevProcess, magn, sizeof(*magn) * inst->magnLen); - memcpy(inst->noisePrev, inst->noise, sizeof(inst->noise[0]) * inst->magnLen); + memcpy(self->magnPrevProcess, magn, sizeof(*magn) * self->magnLen); + memcpy(self->noisePrev, self->noise, sizeof(self->noise[0]) * self->magnLen); // Back to time domain. - IFFT(inst, real, imag, inst->magnLen, inst->anaLen, winData); + IFFT(self, real, imag, self->magnLen, self->anaLen, winData); // Scale factor: only do it after END_STARTUP_LONG time. factor = 1.f; - if (inst->gainmap == 1 && inst->blockInd > END_STARTUP_LONG) { + if (self->gainmap == 1 && self->blockInd > END_STARTUP_LONG) { factor1 = 1.f; factor2 = 1.f; - energy2 = Energy(winData, inst->anaLen); + energy2 = Energy(winData, self->anaLen); gain = (float)sqrt(energy2 / (energy1 + 1.f)); // Scaling for new version. @@ -1324,31 +1324,31 @@ int WebRtcNs_ProcessCore(NSinst_t* inst, if (gain < B_LIM) { // Don't reduce scale too much for pause regions: // attenuation here should be controlled by flooring. - if (gain <= inst->denoiseBound) { - gain = inst->denoiseBound; + if (gain <= self->denoiseBound) { + gain = self->denoiseBound; } factor2 = 1.f - 0.3f * (B_LIM - gain); } // Combine both scales with speech/noise prob: // note prior (priorSpeechProb) is not frequency dependent. - factor = inst->priorSpeechProb * factor1 + - (1.f - inst->priorSpeechProb) * factor2; - } // Out of inst->gainmap == 1. + factor = self->priorSpeechProb * factor1 + + (1.f - self->priorSpeechProb) * factor2; + } // Out of self->gainmap == 1. - Windowing(inst->window, winData, inst->anaLen, winData); + Windowing(self->window, winData, self->anaLen, winData); // Synthesis. - for (i = 0; i < inst->anaLen; i++) { - inst->syntBuf[i] += factor * winData[i]; + for (i = 0; i < self->anaLen; i++) { + self->syntBuf[i] += factor * winData[i]; } // Read out fully processed segment. - for (i = inst->windShift; i < inst->blockLen + inst->windShift; i++) { - fout[i - inst->windShift] = inst->syntBuf[i]; + for (i = self->windShift; i < self->blockLen + self->windShift; i++) { + fout[i - self->windShift] = self->syntBuf[i]; } // Update synthesis buffer. - UpdateBuffer(NULL, inst->blockLen, inst->anaLen, inst->syntBuf); + UpdateBuffer(NULL, self->blockLen, self->anaLen, self->syntBuf); - for (i = 0; i < inst->blockLen; ++i) + for (i = 0; i < self->blockLen; ++i) outFrame[i] = WEBRTC_SPL_SAT(WEBRTC_SPL_WORD16_MAX, fout[i], WEBRTC_SPL_WORD16_MIN); @@ -1357,8 +1357,8 @@ int WebRtcNs_ProcessCore(NSinst_t* inst, // Average speech prob from low band. // Average over second half (i.e., 4->8kHz) of frequencies spectrum. avgProbSpeechHB = 0.0; - for (i = inst->magnLen - deltaBweHB - 1; i < inst->magnLen - 1; i++) { - avgProbSpeechHB += inst->speechProb[i]; + for (i = self->magnLen - deltaBweHB - 1; i < self->magnLen - 1; i++) { + avgProbSpeechHB += self->speechProb[i]; } avgProbSpeechHB = avgProbSpeechHB / ((float)deltaBweHB); // If the speech was suppressed by a component between Analyze and @@ -1366,16 +1366,16 @@ int WebRtcNs_ProcessCore(NSinst_t* inst, // for high band suppression purposes. sumMagnAnalyze = 0; sumMagnProcess = 0; - for (i = 0; i < inst->magnLen; ++i) { - sumMagnAnalyze += inst->magnPrevAnalyze[i]; - sumMagnProcess += inst->magnPrevProcess[i]; + for (i = 0; i < self->magnLen; ++i) { + sumMagnAnalyze += self->magnPrevAnalyze[i]; + sumMagnProcess += self->magnPrevProcess[i]; } avgProbSpeechHB *= sumMagnProcess / sumMagnAnalyze; // Average filter gain from low band. // Average over second half (i.e., 4->8kHz) of frequencies spectrum. avgFilterGainHB = 0.0; - for (i = inst->magnLen - deltaGainHB - 1; i < inst->magnLen - 1; i++) { - avgFilterGainHB += inst->smooth[i]; + for (i = self->magnLen - deltaGainHB - 1; i < self->magnLen - 1; i++) { + avgFilterGainHB += self->smooth[i]; } avgFilterGainHB = avgFilterGainHB / ((float)(deltaGainHB)); avgProbSpeechHBTmp = 2.f * avgProbSpeechHB - 1.f; @@ -1389,16 +1389,16 @@ int WebRtcNs_ProcessCore(NSinst_t* inst, gainTimeDomainHB = gainTimeDomainHB * decayBweHB; // Make sure gain is within flooring range. // Flooring bottom. - if (gainTimeDomainHB < inst->denoiseBound) { - gainTimeDomainHB = inst->denoiseBound; + if (gainTimeDomainHB < self->denoiseBound) { + gainTimeDomainHB = self->denoiseBound; } // Flooring top. if (gainTimeDomainHB > 1.f) { gainTimeDomainHB = 1.f; } // Apply gain. - for (i = 0; i < inst->blockLen; i++) { - float o = gainTimeDomainHB * inst->dataBufHB[i]; + for (i = 0; i < self->blockLen; i++) { + float o = gainTimeDomainHB * self->dataBufHB[i]; outFrameHB[i] = WEBRTC_SPL_SAT(WEBRTC_SPL_WORD16_MAX, o, WEBRTC_SPL_WORD16_MIN); } diff --git a/modules/audio_processing/ns/ns_core.h b/modules/audio_processing/ns/ns_core.h index 601f40ba..d20c60bf 100644 --- a/modules/audio_processing/ns/ns_core.h +++ b/modules/audio_processing/ns/ns_core.h @@ -122,16 +122,16 @@ extern "C" { * This function initializes a noise suppression instance * * Input: - * - inst : Instance that should be initialized + * - self : Instance that should be initialized * - fs : Sampling frequency * * Output: - * - inst : Initialized instance + * - self : Initialized instance * * Return value : 0 - Ok * -1 - Error */ -int WebRtcNs_InitCore(NSinst_t* inst, uint32_t fs); +int WebRtcNs_InitCore(NSinst_t* self, uint32_t fs); /**************************************************************************** * WebRtcNs_set_policy_core(...) @@ -139,16 +139,16 @@ int WebRtcNs_InitCore(NSinst_t* inst, uint32_t fs); * This changes the aggressiveness of the noise suppression method. * * Input: - * - inst : Instance that should be initialized + * - self : Instance that should be initialized * - mode : 0: Mild (6dB), 1: Medium (10dB), 2: Aggressive (15dB) * * Output: - * - NS_inst : Initialized instance + * - self : Initialized instance * * Return value : 0 - Ok * -1 - Error */ -int WebRtcNs_set_policy_core(NSinst_t* inst, int mode); +int WebRtcNs_set_policy_core(NSinst_t* self, int mode); /**************************************************************************** * WebRtcNs_AnalyzeCore @@ -156,16 +156,16 @@ int WebRtcNs_set_policy_core(NSinst_t* inst, int mode); * Estimate the background noise. * * Input: - * - inst : Instance that should be initialized + * - self : Instance that should be initialized * - speechFrame : Input speech frame for lower band * * Output: - * - inst : Updated instance + * - self : Updated instance * * Return value : 0 - OK * -1 - Error */ -int WebRtcNs_AnalyzeCore(NSinst_t* inst, float* speechFrame); +int WebRtcNs_AnalyzeCore(NSinst_t* self, float* speechFrame); /**************************************************************************** * WebRtcNs_ProcessCore @@ -173,19 +173,19 @@ int WebRtcNs_AnalyzeCore(NSinst_t* inst, float* speechFrame); * Do noise suppression. * * Input: - * - inst : Instance that should be initialized + * - self : Instance that should be initialized * - inFrameLow : Input speech frame for lower band * - inFrameHigh : Input speech frame for higher band * * Output: - * - inst : Updated instance + * - self : Updated instance * - outFrameLow : Output speech frame for lower band * - outFrameHigh : Output speech frame for higher band * * Return value : 0 - OK * -1 - Error */ -int WebRtcNs_ProcessCore(NSinst_t* inst, +int WebRtcNs_ProcessCore(NSinst_t* self, float* inFrameLow, float* inFrameHigh, float* outFrameLow, diff --git a/modules/audio_processing/ns/nsx_core_neon.c b/modules/audio_processing/ns/nsx_core_neon.c index f2491417..93099dbf 100644 --- a/modules/audio_processing/ns/nsx_core_neon.c +++ b/modules/audio_processing/ns/nsx_core_neon.c @@ -490,110 +490,91 @@ void WebRtcNsx_DenormalizeNeon(NsxInst_t* inst, int16_t* in, int factor) { void WebRtcNsx_SynthesisUpdateNeon(NsxInst_t* inst, int16_t* out_frame, int16_t gain_factor) { - int16_t* ptr_real = &inst->real[0]; - int16_t* ptr_syn = &inst->synthesisBuffer[0]; - const int16_t* ptr_window = &inst->window[0]; - - // synthesis - __asm__ __volatile__("vdup.16 d24, %0" : : "r"(gain_factor) : "d24"); - // Loop unrolled once. All pointers are incremented in the assembly code. - for (; ptr_syn < &inst->synthesisBuffer[inst->anaLen];) { - __asm__ __volatile__( - // Load variables. - "vld1.16 d22, [%[ptr_real]]!\n\t" - "vld1.16 d23, [%[ptr_window]]!\n\t" - "vld1.16 d25, [%[ptr_syn]]\n\t" - // tmp16a = (int16_t)WEBRTC_SPL_MUL_16_16_RSFT_WITH_ROUND( - // inst->window[i], inst->real[i], 14); // Q0, window in Q14 - "vmull.s16 q11, d22, d23\n\t" - "vrshrn.i32 d22, q11, #14\n\t" - // tmp32 = WEBRTC_SPL_MUL_16_16_RSFT_WITH_ROUND(tmp16a, gain_factor, 13); - "vmull.s16 q11, d24, d22\n\t" - // tmp16b = WebRtcSpl_SatW32ToW16(tmp32); // Q0 - "vqrshrn.s32 d22, q11, #13\n\t" - // inst->synthesisBuffer[i] = WebRtcSpl_AddSatW16( - // inst->synthesisBuffer[i], tmp16b); // Q0 - "vqadd.s16 d25, d22\n\t" - "vst1.16 d25, [%[ptr_syn]]!\n\t" - - // Load variables. - "vld1.16 d26, [%[ptr_real]]!\n\t" - "vld1.16 d27, [%[ptr_window]]!\n\t" - "vld1.16 d28, [%[ptr_syn]]\n\t" - // tmp16a = (int16_t)WEBRTC_SPL_MUL_16_16_RSFT_WITH_ROUND( - // inst->window[i], inst->real[i], 14); // Q0, window in Q14 - "vmull.s16 q13, d26, d27\n\t" - "vrshrn.i32 d26, q13, #14\n\t" - // tmp32 = WEBRTC_SPL_MUL_16_16_RSFT_WITH_ROUND(tmp16a, gain_factor, 13); - "vmull.s16 q13, d24, d26\n\t" - // tmp16b = WebRtcSpl_SatW32ToW16(tmp32); // Q0 - "vqrshrn.s32 d26, q13, #13\n\t" - // inst->synthesisBuffer[i] = WebRtcSpl_AddSatW16( - // inst->synthesisBuffer[i], tmp16b); // Q0 - "vqadd.s16 d28, d26\n\t" - "vst1.16 d28, [%[ptr_syn]]!\n\t" - - // Specify constraints. - :[ptr_real]"+r"(ptr_real), - [ptr_window]"+r"(ptr_window), - [ptr_syn]"+r"(ptr_syn) - : - :"d22", "d23", "d24", "d25", "d26", "d27", "d28", "q11", "q12", "q13" - ); + assert(inst->anaLen % 16 == 0); + assert(inst->blockLen10ms % 16 == 0); + + int16_t* preal_start = inst->real; + const int16_t* pwindow = inst->window; + int16_t* preal_end = preal_start + inst->anaLen; + int16_t* psynthesis_buffer = inst->synthesisBuffer; + + while (preal_start < preal_end) { + // Loop unroll. + int16x8_t window_0 = vld1q_s16(pwindow); + int16x8_t real_0 = vld1q_s16(preal_start); + int16x8_t synthesis_buffer_0 = vld1q_s16(psynthesis_buffer); + + int16x8_t window_1 = vld1q_s16(pwindow + 8); + int16x8_t real_1 = vld1q_s16(preal_start + 8); + int16x8_t synthesis_buffer_1 = vld1q_s16(psynthesis_buffer + 8); + + int32x4_t tmp32a_0_low = vmull_s16(vget_low_s16(real_0), + vget_low_s16(window_0)); + int32x4_t tmp32a_0_high = vmull_s16(vget_high_s16(real_0), + vget_high_s16(window_0)); + + int32x4_t tmp32a_1_low = vmull_s16(vget_low_s16(real_1), + vget_low_s16(window_1)); + int32x4_t tmp32a_1_high = vmull_s16(vget_high_s16(real_1), + vget_high_s16(window_1)); + + int16x4_t tmp16a_0_low = vqrshrn_n_s32(tmp32a_0_low, 14); + int16x4_t tmp16a_0_high = vqrshrn_n_s32(tmp32a_0_high, 14); + + int16x4_t tmp16a_1_low = vqrshrn_n_s32(tmp32a_1_low, 14); + int16x4_t tmp16a_1_high = vqrshrn_n_s32(tmp32a_1_high, 14); + + int32x4_t tmp32b_0_low = vmull_n_s16(tmp16a_0_low, gain_factor); + int32x4_t tmp32b_0_high = vmull_n_s16(tmp16a_0_high, gain_factor); + + int32x4_t tmp32b_1_low = vmull_n_s16(tmp16a_1_low, gain_factor); + int32x4_t tmp32b_1_high = vmull_n_s16(tmp16a_1_high, gain_factor); + + int16x4_t tmp16b_0_low = vqrshrn_n_s32(tmp32b_0_low, 13); + int16x4_t tmp16b_0_high = vqrshrn_n_s32(tmp32b_0_high, 13); + + int16x4_t tmp16b_1_low = vqrshrn_n_s32(tmp32b_1_low, 13); + int16x4_t tmp16b_1_high = vqrshrn_n_s32(tmp32b_1_high, 13); + + synthesis_buffer_0 = vqaddq_s16(vcombine_s16(tmp16b_0_low, tmp16b_0_high), + synthesis_buffer_0); + synthesis_buffer_1 = vqaddq_s16(vcombine_s16(tmp16b_1_low, tmp16b_1_high), + synthesis_buffer_1); + vst1q_s16(psynthesis_buffer, synthesis_buffer_0); + vst1q_s16(psynthesis_buffer + 8, synthesis_buffer_1); + + pwindow += 16; + preal_start += 16; + psynthesis_buffer += 16; } - int16_t* ptr_out = &out_frame[0]; - ptr_syn = &inst->synthesisBuffer[0]; - // read out fully processed segment - for (; ptr_syn < &inst->synthesisBuffer[inst->blockLen10ms];) { - // Loop unrolled once. Both pointers are incremented in the assembly code. - __asm__ __volatile__( - // out_frame[i] = inst->synthesisBuffer[i]; // Q0 - "vld1.16 {d22, d23}, [%[ptr_syn]]!\n\t" - "vld1.16 {d24, d25}, [%[ptr_syn]]!\n\t" - "vst1.16 {d22, d23}, [%[ptr_out]]!\n\t" - "vst1.16 {d24, d25}, [%[ptr_out]]!\n\t" - :[ptr_syn]"+r"(ptr_syn), - [ptr_out]"+r"(ptr_out) - : - :"d22", "d23", "d24", "d25" - ); + // Read out fully processed segment. + int16_t * p_start = inst->synthesisBuffer; + int16_t * p_end = inst->synthesisBuffer + inst->blockLen10ms; + int16_t * p_frame = out_frame; + while (p_start < p_end) { + int16x8_t frame_0 = vld1q_s16(p_start); + vst1q_s16(p_frame, frame_0); + p_start += 8; + p_frame += 8; } // Update synthesis buffer. - // C code: - // WEBRTC_SPL_MEMCPY_W16(inst->synthesisBuffer, - // inst->synthesisBuffer + inst->blockLen10ms, - // inst->anaLen - inst->blockLen10ms); - ptr_out = &inst->synthesisBuffer[0], - ptr_syn = &inst->synthesisBuffer[inst->blockLen10ms]; - for (; ptr_syn < &inst->synthesisBuffer[inst->anaLen];) { - // Loop unrolled once. Both pointers are incremented in the assembly code. - __asm__ __volatile__( - "vld1.16 {d22, d23}, [%[ptr_syn]]!\n\t" - "vld1.16 {d24, d25}, [%[ptr_syn]]!\n\t" - "vst1.16 {d22, d23}, [%[ptr_out]]!\n\t" - "vst1.16 {d24, d25}, [%[ptr_out]]!\n\t" - :[ptr_syn]"+r"(ptr_syn), - [ptr_out]"+r"(ptr_out) - : - :"d22", "d23", "d24", "d25" - ); + int16_t* p_start_src = inst->synthesisBuffer + inst->blockLen10ms; + int16_t* p_end_src = inst->synthesisBuffer + inst->anaLen; + int16_t* p_start_dst = inst->synthesisBuffer; + while (p_start_src < p_end_src) { + int16x8_t frame = vld1q_s16(p_start_src); + vst1q_s16(p_start_dst, frame); + p_start_src += 8; + p_start_dst += 8; } - // C code: - // WebRtcSpl_ZerosArrayW16(inst->synthesisBuffer - // + inst->anaLen - inst->blockLen10ms, inst->blockLen10ms); - __asm__ __volatile__("vdup.16 q10, %0" : : "r"(0) : "q10"); - for (; ptr_out < &inst->synthesisBuffer[inst->anaLen];) { - // Loop unrolled once. Pointer is incremented in the assembly code. - __asm__ __volatile__( - "vst1.16 {d20, d21}, [%[ptr_out]]!\n\t" - "vst1.16 {d20, d21}, [%[ptr_out]]!\n\t" - :[ptr_out]"+r"(ptr_out) - : - :"d20", "d21" - ); + p_start = inst->synthesisBuffer + inst->anaLen - inst->blockLen10ms; + p_end = p_start + inst->blockLen10ms; + int16x8_t zero = vdupq_n_s16(0); + for (;p_start < p_end; p_start += 8) { + vst1q_s16(p_start, zero); } } @@ -601,75 +582,64 @@ void WebRtcNsx_SynthesisUpdateNeon(NsxInst_t* inst, void WebRtcNsx_AnalysisUpdateNeon(NsxInst_t* inst, int16_t* out, int16_t* new_speech) { - - int16_t* ptr_ana = &inst->analysisBuffer[inst->blockLen10ms]; - int16_t* ptr_out = &inst->analysisBuffer[0]; + assert(inst->blockLen10ms % 16 == 0); + assert(inst->anaLen % 16 == 0); // For lower band update analysis buffer. // WEBRTC_SPL_MEMCPY_W16(inst->analysisBuffer, // inst->analysisBuffer + inst->blockLen10ms, // inst->anaLen - inst->blockLen10ms); - for (; ptr_out < &inst->analysisBuffer[inst->anaLen - inst->blockLen10ms];) { - // Loop unrolled once, so both pointers are incremented by 8 twice. - __asm__ __volatile__( - "vld1.16 {d20, d21}, [%[ptr_ana]]!\n\t" - "vst1.16 {d20, d21}, [%[ptr_out]]!\n\t" - "vld1.16 {d22, d23}, [%[ptr_ana]]!\n\t" - "vst1.16 {d22, d23}, [%[ptr_out]]!\n\t" - :[ptr_ana]"+r"(ptr_ana), - [ptr_out]"+r"(ptr_out) - : - :"d20", "d21", "d22", "d23" - ); + int16_t* p_start_src = inst->analysisBuffer + inst->blockLen10ms; + int16_t* p_end_src = inst->analysisBuffer + inst->anaLen; + int16_t* p_start_dst = inst->analysisBuffer; + while (p_start_src < p_end_src) { + int16x8_t frame = vld1q_s16(p_start_src); + vst1q_s16(p_start_dst, frame); + + p_start_src += 8; + p_start_dst += 8; } // WEBRTC_SPL_MEMCPY_W16(inst->analysisBuffer // + inst->anaLen - inst->blockLen10ms, new_speech, inst->blockLen10ms); - for (ptr_ana = new_speech; ptr_out < &inst->analysisBuffer[inst->anaLen];) { - // Loop unrolled once, so both pointers are incremented by 8 twice. - __asm__ __volatile__( - "vld1.16 {d20, d21}, [%[ptr_ana]]!\n\t" - "vst1.16 {d20, d21}, [%[ptr_out]]!\n\t" - "vld1.16 {d22, d23}, [%[ptr_ana]]!\n\t" - "vst1.16 {d22, d23}, [%[ptr_out]]!\n\t" - :[ptr_ana]"+r"(ptr_ana), - [ptr_out]"+r"(ptr_out) - : - :"d20", "d21", "d22", "d23" - ); + p_start_src = new_speech; + p_end_src = new_speech + inst->blockLen10ms; + p_start_dst = inst->analysisBuffer + inst->anaLen - inst->blockLen10ms; + while (p_start_src < p_end_src) { + int16x8_t frame = vld1q_s16(p_start_src); + vst1q_s16(p_start_dst, frame); + + p_start_src += 8; + p_start_dst += 8; } - // Window data before FFT - const int16_t* ptr_window = &inst->window[0]; - ptr_out = &out[0]; - ptr_ana = &inst->analysisBuffer[0]; - for (; ptr_out < &out[inst->anaLen];) { - - // Loop unrolled once, so all pointers are incremented by 4 twice. - __asm__ __volatile__( - "vld1.16 d20, [%[ptr_ana]]!\n\t" - "vld1.16 d21, [%[ptr_window]]!\n\t" - // out[i] = (int16_t)WEBRTC_SPL_MUL_16_16_RSFT_WITH_ROUND( - // inst->window[i], inst->analysisBuffer[i], 14); // Q0 - "vmull.s16 q10, d20, d21\n\t" - "vrshrn.i32 d20, q10, #14\n\t" - "vst1.16 d20, [%[ptr_out]]!\n\t" - - "vld1.16 d22, [%[ptr_ana]]!\n\t" - "vld1.16 d23, [%[ptr_window]]!\n\t" - // out[i] = (int16_t)WEBRTC_SPL_MUL_16_16_RSFT_WITH_ROUND( - // inst->window[i], inst->analysisBuffer[i], 14); // Q0 - "vmull.s16 q11, d22, d23\n\t" - "vrshrn.i32 d22, q11, #14\n\t" - "vst1.16 d22, [%[ptr_out]]!\n\t" - - // Specify constraints. - :[ptr_ana]"+r"(ptr_ana), - [ptr_window]"+r"(ptr_window), - [ptr_out]"+r"(ptr_out) - : - :"d20", "d21", "d22", "d23", "q10", "q11" - ); + // Window data before FFT. + int16_t* p_start_window = (int16_t*) inst->window; + int16_t* p_start_buffer = inst->analysisBuffer; + int16_t* p_start_out = out; + const int16_t* p_end_out = out + inst->anaLen; + + // Load the first element to reduce pipeline bubble. + int16x8_t window = vld1q_s16(p_start_window); + int16x8_t buffer = vld1q_s16(p_start_buffer); + p_start_window += 8; + p_start_buffer += 8; + + while (p_start_out < p_end_out) { + // Unroll loop. + int32x4_t tmp32_low = vmull_s16(vget_low_s16(window), vget_low_s16(buffer)); + int32x4_t tmp32_high = vmull_s16(vget_high_s16(window), + vget_high_s16(buffer)); + window = vld1q_s16(p_start_window); + buffer = vld1q_s16(p_start_buffer); + + int16x4_t result_low = vrshrn_n_s32(tmp32_low, 14); + int16x4_t result_high = vrshrn_n_s32(tmp32_high, 14); + vst1q_s16(p_start_out, vcombine_s16(result_low, result_high)); + + p_start_buffer += 8; + p_start_window += 8; + p_start_out += 8; } } diff --git a/modules/audio_processing/test/audio_processing_unittest.cc b/modules/audio_processing/test/audio_processing_unittest.cc index a0fb303b..401391aa 100644 --- a/modules/audio_processing/test/audio_processing_unittest.cc +++ b/modules/audio_processing/test/audio_processing_unittest.cc @@ -66,9 +66,9 @@ void ConvertToFloat(const int16_t* int_data, ChannelBuffer<float>* cb) { cb->samples_per_channel(), cb->num_channels(), cb_int.channels()); - ScaleToFloat(cb_int.data(), - cb->samples_per_channel() * cb->num_channels(), - cb->data()); + S16ToFloat(cb_int.data(), + cb->samples_per_channel() * cb->num_channels(), + cb->data()); } void ConvertToFloat(const AudioFrame& frame, ChannelBuffer<float>* cb) { @@ -96,14 +96,13 @@ int TruncateToMultipleOf10(int value) { void MixStereoToMono(const float* stereo, float* mono, int samples_per_channel) { - for (int i = 0; i < samples_per_channel; ++i) { + for (int i = 0; i < samples_per_channel; ++i) mono[i] = (stereo[i * 2] + stereo[i * 2 + 1]) / 2; - } } void MixStereoToMono(const int16_t* stereo, int16_t* mono, int samples_per_channel) { - for (int i = 0; i < samples_per_channel; i++) + for (int i = 0; i < samples_per_channel; ++i) mono[i] = (stereo[i * 2] + stereo[i * 2 + 1]) >> 1; } @@ -135,7 +134,7 @@ void SetFrameTo(AudioFrame* frame, int16_t left, int16_t right) { void ScaleFrame(AudioFrame* frame, float scale) { for (int i = 0; i < frame->samples_per_channel_ * frame->num_channels_; ++i) { - frame->data_[i] = RoundToInt16(frame->data_[i] * scale); + frame->data_[i] = FloatS16ToS16(frame->data_[i] * scale); } } @@ -1650,7 +1649,7 @@ TEST_F(ApmTest, DebugDumpFromFileHandle) { #endif // WEBRTC_AUDIOPROC_DEBUG_DUMP } -TEST_F(ApmTest, FloatAndIntInterfacesGiveIdenticalResults) { +TEST_F(ApmTest, FloatAndIntInterfacesGiveSimilarResults) { audioproc::OutputData ref_data; OpenFileAndReadMessage(ref_filename_, &ref_data); @@ -1679,7 +1678,8 @@ TEST_F(ApmTest, FloatAndIntInterfacesGiveIdenticalResults) { Init(fapm.get()); ChannelBuffer<int16_t> output_cb(samples_per_channel, num_input_channels); - scoped_ptr<int16_t[]> output_int16(new int16_t[output_length]); + ChannelBuffer<int16_t> output_int16(samples_per_channel, + num_input_channels); int analog_level = 127; while (ReadFrame(far_file_, revframe_, revfloat_cb_.get()) && @@ -1701,7 +1701,9 @@ TEST_F(ApmTest, FloatAndIntInterfacesGiveIdenticalResults) { EXPECT_NOERR(fapm->gain_control()->set_stream_analog_level(analog_level)); EXPECT_NOERR(apm_->ProcessStream(frame_)); - // TODO(ajm): Update to support different output rates. + Deinterleave(frame_->data_, samples_per_channel, num_output_channels, + output_int16.channels()); + EXPECT_NOERR(fapm->ProcessStream( float_cb_->channels(), samples_per_channel, @@ -1711,24 +1713,34 @@ TEST_F(ApmTest, FloatAndIntInterfacesGiveIdenticalResults) { LayoutFromChannels(num_output_channels), float_cb_->channels())); - // Convert to interleaved int16. - ScaleAndRoundToInt16(float_cb_->data(), output_length, output_cb.data()); - Interleave(output_cb.channels(), - samples_per_channel, - num_output_channels, - output_int16.get()); - // Verify float and int16 paths produce identical output. - EXPECT_EQ(0, memcmp(frame_->data_, output_int16.get(), output_length)); + FloatToS16(float_cb_->data(), output_length, output_cb.data()); + for (int j = 0; j < num_output_channels; ++j) { + float variance = 0; + float snr = ComputeSNR(output_int16.channel(j), output_cb.channel(j), + samples_per_channel, &variance); + #if defined(WEBRTC_AUDIOPROC_FIXED_PROFILE) + // There are a few chunks in the fixed-point profile that give low SNR. + // Listening confirmed the difference is acceptable. + const float kVarianceThreshold = 150; + const float kSNRThreshold = 10; + #else + const float kVarianceThreshold = 20; + const float kSNRThreshold = 20; + #endif + // Skip frames with low energy. + if (sqrt(variance) > kVarianceThreshold) { + EXPECT_LT(kSNRThreshold, snr); + } + } analog_level = fapm->gain_control()->stream_analog_level(); EXPECT_EQ(apm_->gain_control()->stream_analog_level(), fapm->gain_control()->stream_analog_level()); EXPECT_EQ(apm_->echo_cancellation()->stream_has_echo(), fapm->echo_cancellation()->stream_has_echo()); - EXPECT_EQ(apm_->voice_detection()->stream_has_voice(), - fapm->voice_detection()->stream_has_voice()); - EXPECT_EQ(apm_->noise_suppression()->speech_probability(), - fapm->noise_suppression()->speech_probability()); + EXPECT_NEAR(apm_->noise_suppression()->speech_probability(), + fapm->noise_suppression()->speech_probability(), + 0.0005); // Reset in case of downmixing. frame_->num_channels_ = test->num_input_channels(); @@ -2002,7 +2014,7 @@ bool ReadChunk(FILE* file, int16_t* int_data, float* float_data, return false; // This is expected. } - ScaleToFloat(int_data, frame_size, float_data); + S16ToFloat(int_data, frame_size, float_data); if (cb->num_channels() == 1) { MixStereoToMono(float_data, cb->data(), cb->samples_per_channel()); } else { @@ -2109,7 +2121,9 @@ class AudioProcessingTest int num_output_channels, int num_reverse_channels, std::string output_file_prefix) { - scoped_ptr<AudioProcessing> ap(AudioProcessing::Create()); + Config config; + config.Set<ExperimentalAgc>(new ExperimentalAgc(false)); + scoped_ptr<AudioProcessing> ap(AudioProcessing::Create(config)); EnableAllAPComponents(ap.get()); ap->Initialize(input_rate, output_rate, diff --git a/modules/audio_processing/test/process_test.cc b/modules/audio_processing/test/process_test.cc index b6d51e47..76a916da 100644 --- a/modules/audio_processing/test/process_test.cc +++ b/modules/audio_processing/test/process_test.cc @@ -489,7 +489,7 @@ void void_main(int argc, char* argv[]) { FILE* aecm_echo_path_in_file = NULL; FILE* aecm_echo_path_out_file = NULL; - scoped_ptr<WavFile> output_wav_file; + scoped_ptr<WavWriter> output_wav_file; scoped_ptr<RawFile> output_raw_file; if (pb_filename) { @@ -637,9 +637,9 @@ void void_main(int argc, char* argv[]) { if (!raw_output) { // The WAV file needs to be reset every time, because it cant change // it's sample rate or number of channels. - output_wav_file.reset(new WavFile(out_filename + ".wav", - output_sample_rate, - msg.num_output_channels())); + output_wav_file.reset(new WavWriter(out_filename + ".wav", + output_sample_rate, + msg.num_output_channels())); } } else if (event_msg.type() == Event::REVERSE_STREAM) { @@ -876,9 +876,9 @@ void void_main(int argc, char* argv[]) { if (!raw_output) { // The WAV file needs to be reset every time, because it can't change // it's sample rate or number of channels. - output_wav_file.reset(new WavFile(out_filename + ".wav", - sample_rate_hz, - num_capture_output_channels)); + output_wav_file.reset(new WavWriter(out_filename + ".wav", + sample_rate_hz, + num_capture_output_channels)); } if (verbose) { @@ -1029,9 +1029,9 @@ void void_main(int argc, char* argv[]) { output_raw_file.reset(new RawFile(out_filename + ".pcm")); } if (!raw_output && !output_wav_file) { - output_wav_file.reset(new WavFile(out_filename + ".wav", - sample_rate_hz, - num_capture_output_channels)); + output_wav_file.reset(new WavWriter(out_filename + ".wav", + sample_rate_hz, + num_capture_output_channels)); } WriteIntData(near_frame.data_, size, diff --git a/modules/audio_processing/test/test_utils.h b/modules/audio_processing/test/test_utils.h index 61edd8f3..d0d08cb8 100644 --- a/modules/audio_processing/test/test_utils.h +++ b/modules/audio_processing/test/test_utils.h @@ -8,11 +8,12 @@ * be found in the AUTHORS file in the root of the source tree. */ +#include <math.h> #include <limits> #include "webrtc/audio_processing/debug.pb.h" #include "webrtc/common_audio/include/audio_util.h" -#include "webrtc/common_audio/wav_writer.h" +#include "webrtc/common_audio/wav_file.h" #include "webrtc/modules/audio_processing/common.h" #include "webrtc/modules/audio_processing/include/audio_processing.h" #include "webrtc/modules/interface/module_common_types.h" @@ -49,7 +50,7 @@ class RawFile { static inline void WriteIntData(const int16_t* data, size_t length, - WavFile* wav_file, + WavWriter* wav_file, RawFile* raw_file) { if (wav_file) { wav_file->WriteSamples(data, length); @@ -62,7 +63,7 @@ static inline void WriteIntData(const int16_t* data, static inline void WriteFloatData(const float* const* data, size_t samples_per_channel, int num_channels, - WavFile* wav_file, + WavWriter* wav_file, RawFile* raw_file) { size_t length = num_channels * samples_per_channel; scoped_ptr<float[]> buffer(new float[length]); @@ -153,4 +154,26 @@ static inline bool ReadMessageFromFile(FILE* file, return msg->ParseFromArray(bytes.get(), size); } +template <typename T> +float ComputeSNR(const T* ref, const T* test, int length, float* variance) { + float mse = 0; + float mean = 0; + *variance = 0; + for (int i = 0; i < length; ++i) { + T error = ref[i] - test[i]; + mse += error * error; + *variance += ref[i] * ref[i]; + mean += ref[i]; + } + mse /= length; + *variance /= length; + mean /= length; + *variance -= mean * mean; + + float snr = 100; // We assign 100 dB to the zero-error case. + if (mse > 0) + snr = 10 * log10(*variance / mse); + return snr; +} + } // namespace webrtc diff --git a/modules/audio_processing/test/unpack.cc b/modules/audio_processing/test/unpack.cc index 249b6682..ee4e2632 100644 --- a/modules/audio_processing/test/unpack.cc +++ b/modules/audio_processing/test/unpack.cc @@ -73,9 +73,9 @@ int do_main(int argc, char* argv[]) { int num_reverse_channels = 0; int num_input_channels = 0; int num_output_channels = 0; - scoped_ptr<WavFile> reverse_wav_file; - scoped_ptr<WavFile> input_wav_file; - scoped_ptr<WavFile> output_wav_file; + scoped_ptr<WavWriter> reverse_wav_file; + scoped_ptr<WavWriter> input_wav_file; + scoped_ptr<WavWriter> output_wav_file; scoped_ptr<RawFile> reverse_raw_file; scoped_ptr<RawFile> input_raw_file; scoped_ptr<RawFile> output_raw_file; @@ -235,15 +235,15 @@ int do_main(int argc, char* argv[]) { if (!FLAGS_raw) { // The WAV files need to be reset every time, because they cant change // their sample rate or number of channels. - reverse_wav_file.reset(new WavFile(FLAGS_reverse_file + ".wav", - reverse_sample_rate, - num_reverse_channels)); - input_wav_file.reset(new WavFile(FLAGS_input_file + ".wav", - input_sample_rate, - num_input_channels)); - output_wav_file.reset(new WavFile(FLAGS_output_file + ".wav", - output_sample_rate, - num_output_channels)); + reverse_wav_file.reset(new WavWriter(FLAGS_reverse_file + ".wav", + reverse_sample_rate, + num_reverse_channels)); + input_wav_file.reset(new WavWriter(FLAGS_input_file + ".wav", + input_sample_rate, + num_input_channels)); + output_wav_file.reset(new WavWriter(FLAGS_output_file + ".wav", + output_sample_rate, + num_output_channels)); } } } diff --git a/modules/bitrate_controller/send_side_bandwidth_estimation.cc b/modules/bitrate_controller/send_side_bandwidth_estimation.cc index 406322a3..47a79ad2 100644 --- a/modules/bitrate_controller/send_side_bandwidth_estimation.cc +++ b/modules/bitrate_controller/send_side_bandwidth_estimation.cc @@ -13,6 +13,7 @@ #include <cmath> #include "webrtc/system_wrappers/interface/logging.h" +#include "webrtc/system_wrappers/interface/metrics.h" namespace webrtc { namespace { @@ -20,6 +21,7 @@ enum { kBweIncreaseIntervalMs = 1000 }; enum { kBweDecreaseIntervalMs = 300 }; enum { kLimitNumPackets = 20 }; enum { kAvgPacketSizeBytes = 1000 }; +enum { kStartPhaseMs = 2000 }; // Calculate the rate that TCP-Friendly Rate Control (TFRC) would apply. // The formula in RFC 3448, Section 3.1, is used. @@ -57,7 +59,9 @@ SendSideBandwidthEstimation::SendSideBandwidthEstimation() last_round_trip_time_ms_(0), bwe_incoming_(0), time_last_decrease_ms_(0), - first_report_time_ms_(-1) { + first_report_time_ms_(-1), + initially_lost_packets_(0), + uma_updated_(false) { } SendSideBandwidthEstimation::~SendSideBandwidthEstimation() {} @@ -97,8 +101,6 @@ void SendSideBandwidthEstimation::UpdateReceiverBlock(uint8_t fraction_loss, uint32_t rtt, int number_of_packets, uint32_t now_ms) { - if (first_report_time_ms_ == -1) - first_report_time_ms_ = now_ms; // Update RTT. last_round_trip_time_ms_ = rtt; @@ -125,12 +127,28 @@ void SendSideBandwidthEstimation::UpdateReceiverBlock(uint8_t fraction_loss, } time_last_receiver_block_ms_ = now_ms; UpdateEstimate(now_ms); + + if (first_report_time_ms_ == -1) { + first_report_time_ms_ = now_ms; + } else if (IsInStartPhase(now_ms)) { + initially_lost_packets_ += (fraction_loss * number_of_packets) >> 8; + } else if (!uma_updated_) { + uma_updated_ = true; + RTC_HISTOGRAM_COUNTS( + "WebRTC.BWE.InitiallyLostPackets", initially_lost_packets_, 0, 100, 50); + RTC_HISTOGRAM_COUNTS("WebRTC.BWE.InitialRtt", rtt, 0, 2000, 50); + RTC_HISTOGRAM_COUNTS("WebRTC.BWE.InitialBandwidthEstimate", + (bitrate_ + 500) / 1000, + 0, + 2000, + 50); + } } void SendSideBandwidthEstimation::UpdateEstimate(uint32_t now_ms) { // We trust the REMB during the first 2 seconds if we haven't had any // packet loss reported, to allow startup bitrate probing. - if (last_fraction_loss_ == 0 && now_ms - first_report_time_ms_ < 2000 && + if (last_fraction_loss_ == 0 && IsInStartPhase(now_ms) && bwe_incoming_ > bitrate_) { bitrate_ = CapBitrateToThresholds(bwe_incoming_); min_bitrate_history_.clear(); @@ -187,6 +205,11 @@ void SendSideBandwidthEstimation::UpdateEstimate(uint32_t now_ms) { bitrate_ = CapBitrateToThresholds(bitrate_); } +bool SendSideBandwidthEstimation::IsInStartPhase(int64_t now_ms) const { + return first_report_time_ms_ == -1 || + now_ms - first_report_time_ms_ < kStartPhaseMs; +} + void SendSideBandwidthEstimation::UpdateMinHistory(uint32_t now_ms) { // Remove old data points from history. // Since history precision is in ms, add one so it is able to increase diff --git a/modules/bitrate_controller/send_side_bandwidth_estimation.h b/modules/bitrate_controller/send_side_bandwidth_estimation.h index 6b323852..0fe3ae67 100644 --- a/modules/bitrate_controller/send_side_bandwidth_estimation.h +++ b/modules/bitrate_controller/send_side_bandwidth_estimation.h @@ -43,6 +43,8 @@ class SendSideBandwidthEstimation { void SetMinBitrate(uint32_t min_bitrate); private: + bool IsInStartPhase(int64_t now_ms) const; + // Returns the input bitrate capped to the thresholds defined by the max, // min and incoming bandwidth. uint32_t CapBitrateToThresholds(uint32_t bitrate); @@ -69,6 +71,8 @@ class SendSideBandwidthEstimation { uint32_t bwe_incoming_; uint32_t time_last_decrease_ms_; int64_t first_report_time_ms_; + int initially_lost_packets_; + bool uma_updated_; }; } // namespace webrtc #endif // WEBRTC_MODULES_BITRATE_CONTROLLER_SEND_SIDE_BANDWIDTH_ESTIMATION_H_ diff --git a/modules/modules.gyp b/modules/modules.gyp index 12200fe8..3caf41f9 100644 --- a/modules/modules.gyp +++ b/modules/modules.gyp @@ -97,6 +97,7 @@ '<(webrtc_root)/common_audio/common_audio.gyp:common_audio', '<(webrtc_root)/modules/modules.gyp:video_capture_module_impl', '<(webrtc_root)/modules/video_coding/codecs/vp8/vp8.gyp:webrtc_vp8', + '<(webrtc_root)/modules/video_coding/codecs/vp9/vp9.gyp:webrtc_vp9', '<(webrtc_root)/system_wrappers/source/system_wrappers.gyp:system_wrappers', '<(webrtc_root)/test/test.gyp:frame_generator', '<(webrtc_root)/test/test.gyp:rtp_test_utils', @@ -213,6 +214,7 @@ 'rtp_rtcp/source/rtcp_packet_unittest.cc', 'rtp_rtcp/source/rtcp_receiver_unittest.cc', 'rtp_rtcp/source/rtcp_sender_unittest.cc', + 'rtp_rtcp/source/rtcp_utility_unittest.cc', 'rtp_rtcp/source/rtp_fec_unittest.cc', 'rtp_rtcp/source/rtp_format_h264_unittest.cc', 'rtp_rtcp/source/rtp_format_vp8_unittest.cc', @@ -325,6 +327,7 @@ '<(DEPTH)/testing/gtest.gyp:gtest', '<(webrtc_root)/common_video/common_video.gyp:common_video', '<(webrtc_root)/modules/video_coding/codecs/vp8/vp8.gyp:webrtc_vp8', + '<(webrtc_root)/modules/video_coding/codecs/vp9/vp9.gyp:webrtc_vp9', '<(webrtc_root)/system_wrappers/source/system_wrappers.gyp:system_wrappers', '<(webrtc_root)/test/metrics.gyp:metrics', '<(webrtc_root)/test/test.gyp:test_support', @@ -396,7 +399,6 @@ ], 'includes': [ '../build/isolate.gypi', - 'modules_tests.isolate', ], 'sources': [ 'modules_tests.isolate', @@ -410,7 +412,6 @@ ], 'includes': [ '../build/isolate.gypi', - 'modules_unittests.isolate', ], 'sources': [ 'modules_unittests.isolate', diff --git a/modules/modules_tests.isolate b/modules/modules_tests.isolate index e5055f0d..c29c65b7 100644 --- a/modules/modules_tests.isolate +++ b/modules/modules_tests.isolate @@ -9,7 +9,7 @@ 'conditions': [ ['OS=="android"', { 'variables': { - 'isolate_dependency_untracked': [ + 'files': [ '<(DEPTH)/data/', '<(DEPTH)/resources/', ], @@ -21,7 +21,7 @@ '<(DEPTH)/testing/test_env.py', '<(PRODUCT_DIR)/modules_tests<(EXECUTABLE_SUFFIX)', ], - 'isolate_dependency_tracked': [ + 'files': [ '<(DEPTH)/DEPS', '<(DEPTH)/resources/audio_coding/testfile32kHz.pcm', '<(DEPTH)/resources/audio_coding/teststereo32kHz.pcm', @@ -30,9 +30,6 @@ '<(DEPTH)/testing/test_env.py', '<(PRODUCT_DIR)/modules_tests<(EXECUTABLE_SUFFIX)', ], - 'isolate_dependency_untracked': [ - '<(DEPTH)/tools/swarming_client/', - ], }, }], ], diff --git a/modules/modules_unittests.isolate b/modules/modules_unittests.isolate index 09ace1c1..967ff32f 100644 --- a/modules/modules_unittests.isolate +++ b/modules/modules_unittests.isolate @@ -9,16 +9,10 @@ 'conditions': [ ['OS=="android"', { 'variables': { - 'isolate_dependency_untracked': [ + 'files': [ '<(DEPTH)/data/', '<(DEPTH)/resources/', ], - 'isolate_dependency_tracked': [ - '<(DEPTH)/resources/short_mixed_mono_48.dat', - '<(DEPTH)/resources/short_mixed_mono_48.pcm', - '<(DEPTH)/resources/short_mixed_stereo_48.dat', - '<(DEPTH)/resources/short_mixed_stereo_48.pcm', - ], }, }], ['OS=="linux" or OS=="mac" or OS=="win"', { @@ -27,7 +21,7 @@ '<(DEPTH)/testing/test_env.py', '<(PRODUCT_DIR)/modules_unittests<(EXECUTABLE_SUFFIX)', ], - 'isolate_dependency_tracked': [ + 'files': [ '<(DEPTH)/DEPS', '<(DEPTH)/data/audio_processing/output_data_float.pb', '<(DEPTH)/data/voice_engine/audio_tiny48.wav', @@ -105,9 +99,6 @@ '<(DEPTH)/testing/test_env.py', '<(PRODUCT_DIR)/modules_unittests<(EXECUTABLE_SUFFIX)', ], - 'isolate_dependency_untracked': [ - '<(DEPTH)/tools/swarming_client/', - ], }, }], ], diff --git a/modules/remote_bitrate_estimator/remote_bitrate_estimator_single_stream_unittest.cc b/modules/remote_bitrate_estimator/remote_bitrate_estimator_single_stream_unittest.cc index f67c7f34..08ba49d4 100644 --- a/modules/remote_bitrate_estimator/remote_bitrate_estimator_single_stream_unittest.cc +++ b/modules/remote_bitrate_estimator/remote_bitrate_estimator_single_stream_unittest.cc @@ -40,45 +40,45 @@ TEST_F(RemoteBitrateEstimatorSingleTest, RateIncreaseReordering) { } TEST_F(RemoteBitrateEstimatorSingleTest, RateIncreaseRtpTimestamps) { - RateIncreaseRtpTimestampsTestHelper(); + RateIncreaseRtpTimestampsTestHelper(1621); } // Verify that the time it takes for the estimator to reduce the bitrate when // the capacity is tightened stays the same. TEST_F(RemoteBitrateEstimatorSingleTest, CapacityDropOneStream) { - CapacityDropTestHelper(1, false, 956214, 367); + CapacityDropTestHelper(1, false, 367); } // Verify that the time it takes for the estimator to reduce the bitrate when // the capacity is tightened stays the same. This test also verifies that we // handle wrap-arounds in this scenario. TEST_F(RemoteBitrateEstimatorSingleTest, CapacityDropOneStreamWrap) { - CapacityDropTestHelper(1, true, 956214, 367); + CapacityDropTestHelper(1, true, 367); } // Verify that the time it takes for the estimator to reduce the bitrate when // the capacity is tightened stays the same. This test also verifies that we // handle wrap-arounds in this scenario. This is a multi-stream test. TEST_F(RemoteBitrateEstimatorSingleTest, CapacityDropTwoStreamsWrap) { - CapacityDropTestHelper(2, true, 927088, 267); + CapacityDropTestHelper(2, true, 267); } // Verify that the time it takes for the estimator to reduce the bitrate when // the capacity is tightened stays the same. This test also verifies that we // handle wrap-arounds in this scenario. This is a multi-stream test. TEST_F(RemoteBitrateEstimatorSingleTest, CapacityDropThreeStreamsWrap) { - CapacityDropTestHelper(3, true, 920944, 333); + CapacityDropTestHelper(3, true, 333); } TEST_F(RemoteBitrateEstimatorSingleTest, CapacityDropThirteenStreamsWrap) { - CapacityDropTestHelper(13, true, 938944, 300); + CapacityDropTestHelper(13, true, 300); } TEST_F(RemoteBitrateEstimatorSingleTest, CapacityDropNineteenStreamsWrap) { - CapacityDropTestHelper(19, true, 926718, 300); + CapacityDropTestHelper(19, true, 300); } TEST_F(RemoteBitrateEstimatorSingleTest, CapacityDropThirtyStreamsWrap) { - CapacityDropTestHelper(30, true, 927016, 300); + CapacityDropTestHelper(30, true, 300); } } // namespace webrtc diff --git a/modules/remote_bitrate_estimator/remote_bitrate_estimator_unittest_helper.cc b/modules/remote_bitrate_estimator/remote_bitrate_estimator_unittest_helper.cc index 1b38a1ea..4f9e16c8 100644 --- a/modules/remote_bitrate_estimator/remote_bitrate_estimator_unittest_helper.cc +++ b/modules/remote_bitrate_estimator/remote_bitrate_estimator_unittest_helper.cc @@ -14,7 +14,7 @@ namespace webrtc { -enum { kMtu = 1200 }; +enum { kMtu = 1200, kAcceptedBitrateErrorBps = 50000u }; namespace testing { @@ -225,6 +225,7 @@ void RemoteBitrateEstimatorTest::IncomingPacket(uint32_t ssrc, memset(&header, 0, sizeof(header)); header.ssrc = ssrc; header.timestamp = rtp_timestamp; + header.extension.hasAbsoluteSendTime = true; header.extension.absoluteSendTime = absolute_send_time; bitrate_estimator_->IncomingPacket(arrival_time + kArrivalTimeClockOffsetMs, payload_size, header); @@ -340,7 +341,7 @@ void RemoteBitrateEstimatorTest::InitialBehaviorTestHelper( EXPECT_TRUE(bitrate_estimator_->LatestEstimate(&ssrcs, &bitrate_bps)); ASSERT_EQ(1u, ssrcs.size()); EXPECT_EQ(kDefaultSsrc, ssrcs.front()); - EXPECT_EQ(expected_converge_bitrate, bitrate_bps); + EXPECT_NEAR(expected_converge_bitrate, bitrate_bps, kAcceptedBitrateErrorBps); EXPECT_TRUE(bitrate_observer_->updated()); bitrate_observer_->Reset(); EXPECT_EQ(bitrate_observer_->latest_bitrate(), bitrate_bps); @@ -372,7 +373,9 @@ void RemoteBitrateEstimatorTest::RateIncreaseReorderingTestHelper( } bitrate_estimator_->Process(); EXPECT_TRUE(bitrate_observer_->updated()); - EXPECT_EQ(expected_bitrate_bps, bitrate_observer_->latest_bitrate()); + EXPECT_NEAR(expected_bitrate_bps, + bitrate_observer_->latest_bitrate(), + kAcceptedBitrateErrorBps); for (int i = 0; i < 10; ++i) { clock_.AdvanceTimeMilliseconds(2 * kFrameIntervalMs); timestamp += 2 * 90 * kFrameIntervalMs; @@ -387,15 +390,17 @@ void RemoteBitrateEstimatorTest::RateIncreaseReorderingTestHelper( } bitrate_estimator_->Process(); EXPECT_TRUE(bitrate_observer_->updated()); - EXPECT_EQ(expected_bitrate_bps, bitrate_observer_->latest_bitrate()); + EXPECT_NEAR(expected_bitrate_bps, + bitrate_observer_->latest_bitrate(), + kAcceptedBitrateErrorBps); } // Make sure we initially increase the bitrate as expected. -void RemoteBitrateEstimatorTest::RateIncreaseRtpTimestampsTestHelper() { +void RemoteBitrateEstimatorTest::RateIncreaseRtpTimestampsTestHelper( + int expected_iterations) { // This threshold corresponds approximately to increasing linearly with // bitrate(i) = 1.04 * bitrate(i-1) + 1000 // until bitrate(i) > 500000, with bitrate(1) ~= 30000. - const int kExpectedIterations = 1621; unsigned int bitrate_bps = 30000; int iterations = 0; AddDefaultStream(); @@ -412,15 +417,14 @@ void RemoteBitrateEstimatorTest::RateIncreaseRtpTimestampsTestHelper() { bitrate_observer_->Reset(); } ++iterations; - ASSERT_LE(iterations, kExpectedIterations); + ASSERT_LE(iterations, expected_iterations); } - ASSERT_EQ(kExpectedIterations, iterations); + ASSERT_EQ(expected_iterations, iterations); } void RemoteBitrateEstimatorTest::CapacityDropTestHelper( int number_of_streams, bool wrap_time_stamp, - unsigned int expected_converge_bitrate, unsigned int expected_bitrate_drop_delta) { const int kFramerate = 30; const int kStartBitrate = 900e3; @@ -430,14 +434,11 @@ void RemoteBitrateEstimatorTest::CapacityDropTestHelper( const unsigned int kReducedCapacityBps = 500e3; int steady_state_time = 0; - int expected_overuse_start_time = 0; if (number_of_streams <= 1) { steady_state_time = 10; - expected_overuse_start_time = 10000; AddDefaultStream(); } else { steady_state_time = 8 * number_of_streams; - expected_overuse_start_time = 8000; int bitrate_sum = 0; int kBitrateDenom = number_of_streams * (number_of_streams - 1); for (int i = 0; i < number_of_streams; i++) { @@ -471,13 +472,12 @@ void RemoteBitrateEstimatorTest::CapacityDropTestHelper( kMinExpectedBitrate, kMaxExpectedBitrate, kInitialCapacityBps); - EXPECT_EQ(expected_converge_bitrate, bitrate_bps); + EXPECT_NEAR(kInitialCapacityBps, bitrate_bps, 100000u); bitrate_observer_->Reset(); // Reduce the capacity and verify the decrease time. stream_generator_->set_capacity_bps(kReducedCapacityBps); int64_t overuse_start_time = clock_.TimeInMilliseconds(); - EXPECT_EQ(expected_overuse_start_time, overuse_start_time); int64_t bitrate_drop_time = -1; for (int i = 0; i < 100 * number_of_streams; ++i) { GenerateAndProcessFrame(kDefaultSsrc, bitrate_bps); diff --git a/modules/remote_bitrate_estimator/remote_bitrate_estimator_unittest_helper.h b/modules/remote_bitrate_estimator/remote_bitrate_estimator_unittest_helper.h index 1d748c57..2ef2f45c 100644 --- a/modules/remote_bitrate_estimator/remote_bitrate_estimator_unittest_helper.h +++ b/modules/remote_bitrate_estimator/remote_bitrate_estimator_unittest_helper.h @@ -191,10 +191,9 @@ class RemoteBitrateEstimatorTest : public ::testing::Test { void InitialBehaviorTestHelper(unsigned int expected_converge_bitrate); void RateIncreaseReorderingTestHelper(unsigned int expected_bitrate); - void RateIncreaseRtpTimestampsTestHelper(); + void RateIncreaseRtpTimestampsTestHelper(int expected_iterations); void CapacityDropTestHelper(int number_of_streams, bool wrap_time_stamp, - unsigned int expected_converge_bitrate, unsigned int expected_bitrate_drop_delta); static const unsigned int kDefaultSsrc; diff --git a/modules/rtp_rtcp/source/rtcp_receiver.cc b/modules/rtp_rtcp/source/rtcp_receiver.cc index a9343145..1d1e2916 100644 --- a/modules/rtp_rtcp/source/rtcp_receiver.cc +++ b/modules/rtp_rtcp/source/rtcp_receiver.cc @@ -833,6 +833,8 @@ RTCPReceiver::HandleNACK(RTCPUtility::RTCPParserV2& rtcpParser, if (rtcpPacketInformation.rtcpPacketTypeFlags & kRtcpNack) { ++packet_type_counter_.nack_packets; + packet_type_counter_.nack_requests = nack_stats_.requests(); + packet_type_counter_.unique_nack_requests = nack_stats_.unique_requests(); } } @@ -842,6 +844,7 @@ RTCPReceiver::HandleNACKItem(const RTCPUtility::RTCPPacket& rtcpPacket, RTCPPacketInformation& rtcpPacketInformation) { rtcpPacketInformation.AddNACKPacket(rtcpPacket.NACKItem.PacketID); + nack_stats_.ReportRequest(rtcpPacket.NACKItem.PacketID); uint16_t bitMask = rtcpPacket.NACKItem.BitMask; if(bitMask) @@ -851,6 +854,7 @@ RTCPReceiver::HandleNACKItem(const RTCPUtility::RTCPPacket& rtcpPacket, if(bitMask & 0x01) { rtcpPacketInformation.AddNACKPacket(rtcpPacket.NACKItem.PacketID + i); + nack_stats_.ReportRequest(rtcpPacket.NACKItem.PacketID + i); } bitMask = bitMask >>1; } diff --git a/modules/rtp_rtcp/source/rtcp_receiver.h b/modules/rtp_rtcp/source/rtcp_receiver.h index 84eb24c7..087722c4 100644 --- a/modules/rtp_rtcp/source/rtcp_receiver.h +++ b/modules/rtp_rtcp/source/rtcp_receiver.h @@ -270,6 +270,8 @@ protected: RtcpStatisticsCallback* stats_callback_; RtcpPacketTypeCounter packet_type_counter_; + + RTCPUtility::NackStats nack_stats_; }; } // namespace webrtc #endif // WEBRTC_MODULES_RTP_RTCP_SOURCE_RTCP_RECEIVER_H_ diff --git a/modules/rtp_rtcp/source/rtcp_receiver_unittest.cc b/modules/rtp_rtcp/source/rtcp_receiver_unittest.cc index b3a9d093..2a615734 100644 --- a/modules/rtp_rtcp/source/rtcp_receiver_unittest.cc +++ b/modules/rtp_rtcp/source/rtcp_receiver_unittest.cc @@ -592,13 +592,6 @@ TEST_F(RtcpReceiverTest, InjectXrPacketWithUnknownReportBlock) { EXPECT_FALSE(rtcp_packet_info_.xr_dlrr_item); } -TEST(RtcpUtilityTest, MidNtp) { - const uint32_t kNtpSec = 0x12345678; - const uint32_t kNtpFrac = 0x23456789; - const uint32_t kNtpMid = 0x56782345; - EXPECT_EQ(kNtpMid, RTCPUtility::MidNtp(kNtpSec, kNtpFrac)); -} - TEST_F(RtcpReceiverTest, TestXrRrRttInitiallyFalse) { uint16_t rtt_ms; EXPECT_FALSE(rtcp_receiver_->GetAndResetXrRrRtt(&rtt_ms)); diff --git a/modules/rtp_rtcp/source/rtcp_sender.cc b/modules/rtp_rtcp/source/rtcp_sender.cc index 1edbee43..afe80195 100644 --- a/modules/rtp_rtcp/source/rtcp_sender.cc +++ b/modules/rtp_rtcp/source/rtcp_sender.cc @@ -1354,7 +1354,6 @@ RTCPSender::BuildNACK(uint8_t* rtcpbuffer, RtpUtility::AssignUWord32ToBuffer(rtcpbuffer + pos, _remoteSSRC); pos += 4; - NACKStringBuilder stringBuilder; // Build NACK bitmasks and write them to the RTCP message. // The nack list should be sorted and not contain duplicates if one // wants to build the smallest rtcp nack packet. @@ -1363,13 +1362,11 @@ RTCPSender::BuildNACK(uint8_t* rtcpbuffer, (IP_PACKET_SIZE - pos) / 4); int i = 0; while (i < nackSize && numOfNackFields < maxNackFields) { - stringBuilder.PushNACK(nackList[i]); uint16_t nack = nackList[i++]; uint16_t bitmask = 0; while (i < nackSize) { int shift = static_cast<uint16_t>(nackList[i] - nack) - 1; if (shift >= 0 && shift <= 15) { - stringBuilder.PushNACK(nackList[i]); bitmask |= (1 << shift); ++i; } else { @@ -1384,11 +1381,21 @@ RTCPSender::BuildNACK(uint8_t* rtcpbuffer, pos += 2; numOfNackFields++; } + rtcpbuffer[nackSizePos] = static_cast<uint8_t>(2 + numOfNackFields); + if (i != nackSize) { - LOG(LS_WARNING) << "Nack list to large for one packet."; + LOG(LS_WARNING) << "Nack list too large for one packet."; + } + + // Report stats. + NACKStringBuilder stringBuilder; + for (int idx = 0; idx < i; ++idx) { + stringBuilder.PushNACK(nackList[idx]); + nack_stats_.ReportRequest(nackList[idx]); } - rtcpbuffer[nackSizePos] = static_cast<uint8_t>(2 + numOfNackFields); *nackString = stringBuilder.GetResult(); + packet_type_counter_.nack_requests = nack_stats_.requests(); + packet_type_counter_.unique_nack_requests = nack_stats_.unique_requests(); return 0; } diff --git a/modules/rtp_rtcp/source/rtcp_sender.h b/modules/rtp_rtcp/source/rtcp_sender.h index c0b8ebdb..06c373b6 100644 --- a/modules/rtp_rtcp/source/rtcp_sender.h +++ b/modules/rtp_rtcp/source/rtcp_sender.h @@ -360,6 +360,8 @@ private: RtcpPacketTypeCounter packet_type_counter_ GUARDED_BY(_criticalSectionRTCPSender); + + RTCPUtility::NackStats nack_stats_ GUARDED_BY(_criticalSectionRTCPSender); }; } // namespace webrtc diff --git a/modules/rtp_rtcp/source/rtcp_utility.cc b/modules/rtp_rtcp/source/rtcp_utility.cc index 9acab735..04233896 100644 --- a/modules/rtp_rtcp/source/rtcp_utility.cc +++ b/modules/rtp_rtcp/source/rtcp_utility.cc @@ -17,6 +17,23 @@ namespace webrtc { namespace RTCPUtility { + +NackStats::NackStats() + : max_sequence_number_(0), + requests_(0), + unique_requests_(0) {} + +NackStats::~NackStats() {} + +void NackStats::ReportRequest(uint16_t sequence_number) { + if (requests_ == 0 || + webrtc::IsNewerSequenceNumber(sequence_number, max_sequence_number_)) { + max_sequence_number_ = sequence_number; + ++unique_requests_; + } + ++requests_; +} + uint32_t MidNtp(uint32_t ntp_sec, uint32_t ntp_frac) { return (ntp_sec << 16) + (ntp_frac >> 16); } // end RTCPUtility diff --git a/modules/rtp_rtcp/source/rtcp_utility.h b/modules/rtp_rtcp/source/rtcp_utility.h index f0867a75..804e8b94 100644 --- a/modules/rtp_rtcp/source/rtcp_utility.h +++ b/modules/rtp_rtcp/source/rtcp_utility.h @@ -19,6 +19,29 @@ namespace webrtc { namespace RTCPUtility { + +class NackStats { + public: + NackStats(); + ~NackStats(); + + // Updates stats with requested sequence number. + // This function should be called for each NACK request to calculate the + // number of unique NACKed RTP packets. + void ReportRequest(uint16_t sequence_number); + + // Gets the number of NACKed RTP packets. + uint32_t requests() const { return requests_; } + + // Gets the number of unique NACKed RTP packets. + uint32_t unique_requests() const { return unique_requests_; } + + private: + uint16_t max_sequence_number_; + uint32_t requests_; + uint32_t unique_requests_; +}; + uint32_t MidNtp(uint32_t ntp_sec, uint32_t ntp_frac); // CNAME diff --git a/modules/rtp_rtcp/source/rtcp_utility_unittest.cc b/modules/rtp_rtcp/source/rtcp_utility_unittest.cc new file mode 100644 index 00000000..275b007b --- /dev/null +++ b/modules/rtp_rtcp/source/rtcp_utility_unittest.cc @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "testing/gtest/include/gtest/gtest.h" + +#include "webrtc/modules/rtp_rtcp/source/rtcp_utility.h" + +namespace webrtc { + +TEST(RtcpUtilityTest, MidNtp) { + const uint32_t kNtpSec = 0x12345678; + const uint32_t kNtpFrac = 0x23456789; + const uint32_t kNtpMid = 0x56782345; + EXPECT_EQ(kNtpMid, RTCPUtility::MidNtp(kNtpSec, kNtpFrac)); +} + +TEST(RtcpUtilityTest, NackRequests) { + RTCPUtility::NackStats stats; + EXPECT_EQ(0U, stats.unique_requests()); + EXPECT_EQ(0U, stats.requests()); + stats.ReportRequest(10); + EXPECT_EQ(1U, stats.unique_requests()); + EXPECT_EQ(1U, stats.requests()); + + stats.ReportRequest(10); + EXPECT_EQ(1U, stats.unique_requests()); + stats.ReportRequest(11); + EXPECT_EQ(2U, stats.unique_requests()); + + stats.ReportRequest(11); + EXPECT_EQ(2U, stats.unique_requests()); + stats.ReportRequest(13); + EXPECT_EQ(3U, stats.unique_requests()); + + stats.ReportRequest(11); + EXPECT_EQ(3U, stats.unique_requests()); + EXPECT_EQ(6U, stats.requests()); +} + +TEST(RtcpUtilityTest, NackRequestsWithWrap) { + RTCPUtility::NackStats stats; + stats.ReportRequest(65534); + EXPECT_EQ(1U, stats.unique_requests()); + + stats.ReportRequest(65534); + EXPECT_EQ(1U, stats.unique_requests()); + stats.ReportRequest(65535); + EXPECT_EQ(2U, stats.unique_requests()); + + stats.ReportRequest(65535); + EXPECT_EQ(2U, stats.unique_requests()); + stats.ReportRequest(0); + EXPECT_EQ(3U, stats.unique_requests()); + + stats.ReportRequest(65535); + EXPECT_EQ(3U, stats.unique_requests()); + stats.ReportRequest(0); + EXPECT_EQ(3U, stats.unique_requests()); + stats.ReportRequest(1); + EXPECT_EQ(4U, stats.unique_requests()); + EXPECT_EQ(8U, stats.requests()); +} + +} // namespace webrtc + diff --git a/modules/rtp_rtcp/source/rtp_rtcp_impl_unittest.cc b/modules/rtp_rtcp/source/rtp_rtcp_impl_unittest.cc index 4868b71f..24ff227a 100644 --- a/modules/rtp_rtcp/source/rtp_rtcp_impl_unittest.cc +++ b/modules/rtp_rtcp/source/rtp_rtcp_impl_unittest.cc @@ -18,8 +18,10 @@ #include "webrtc/modules/rtp_rtcp/source/rtcp_packet.h" #include "webrtc/modules/rtp_rtcp/source/rtp_rtcp_impl.h" #include "webrtc/system_wrappers/interface/scoped_vector.h" +#include "webrtc/test/rtcp_packet_parser.h" using ::testing::_; +using ::testing::ElementsAre; using ::testing::NiceMock; using ::testing::Return; using ::testing::SaveArg; @@ -76,6 +78,10 @@ class SendTransport : public Transport, return len; } virtual int SendRTCPPacket(int /*ch*/, const void *data, int len) OVERRIDE { + test::RtcpPacketParser parser; + parser.Parse(static_cast<const uint8_t*>(data), len); + last_nack_list_ = parser.nack_item()->last_nack_list(); + if (clock_) { clock_->AdvanceTimeMilliseconds(delay_ms_); } @@ -89,6 +95,7 @@ class SendTransport : public Transport, uint32_t delay_ms_; int rtp_packets_sent_; RTPHeader last_rtp_header_; + std::vector<uint16_t> last_nack_list_; }; class RtpRtcpModule { @@ -129,6 +136,9 @@ class RtpRtcpModule { uint16_t LastRtpSequenceNumber() { return transport_.last_rtp_header_.sequenceNumber; } + std::vector<uint16_t> LastNackListSent() { + return transport_.last_nack_list_; + } }; } // namespace @@ -344,6 +354,46 @@ TEST_F(RtpRtcpImplTest, RtcpPacketTypeCounter_FirAndPli) { EXPECT_EQ(1U, sender_.RtcpReceived().pli_packets); } +TEST_F(RtpRtcpImplTest, UniqueNackRequests) { + receiver_.transport_.SimulateNetworkDelay(0, &clock_); + EXPECT_EQ(0U, receiver_.RtcpSent().nack_packets); + EXPECT_EQ(0U, receiver_.RtcpSent().nack_requests); + EXPECT_EQ(0U, receiver_.RtcpSent().unique_nack_requests); + EXPECT_EQ(0, receiver_.RtcpSent().UniqueNackRequestsInPercent()); + + // Receive module sends NACK request. + const uint16_t kNackLength = 4; + uint16_t nack_list[kNackLength] = {10, 11, 13, 18}; + EXPECT_EQ(0, receiver_.impl_->SendNACK(nack_list, kNackLength)); + EXPECT_EQ(1U, receiver_.RtcpSent().nack_packets); + EXPECT_EQ(4U, receiver_.RtcpSent().nack_requests); + EXPECT_EQ(4U, receiver_.RtcpSent().unique_nack_requests); + EXPECT_THAT(receiver_.LastNackListSent(), ElementsAre(10, 11, 13, 18)); + + // Send module receives the request. + EXPECT_EQ(1U, sender_.RtcpReceived().nack_packets); + EXPECT_EQ(4U, sender_.RtcpReceived().nack_requests); + EXPECT_EQ(4U, sender_.RtcpReceived().unique_nack_requests); + EXPECT_EQ(100, sender_.RtcpReceived().UniqueNackRequestsInPercent()); + + // Receive module sends new request with duplicated packets. + const int kStartupRttMs = 100; + clock_.AdvanceTimeMilliseconds(kStartupRttMs + 1); + const uint16_t kNackLength2 = 4; + uint16_t nack_list2[kNackLength2] = {11, 18, 20, 21}; + EXPECT_EQ(0, receiver_.impl_->SendNACK(nack_list2, kNackLength2)); + EXPECT_EQ(2U, receiver_.RtcpSent().nack_packets); + EXPECT_EQ(8U, receiver_.RtcpSent().nack_requests); + EXPECT_EQ(6U, receiver_.RtcpSent().unique_nack_requests); + EXPECT_THAT(receiver_.LastNackListSent(), ElementsAre(11, 18, 20, 21)); + + // Send module receives the request. + EXPECT_EQ(2U, sender_.RtcpReceived().nack_packets); + EXPECT_EQ(8U, sender_.RtcpReceived().nack_requests); + EXPECT_EQ(6U, sender_.RtcpReceived().unique_nack_requests); + EXPECT_EQ(75, sender_.RtcpReceived().UniqueNackRequestsInPercent()); +} + class RtpSendingTestTransport : public Transport { public: void ResetCounters() { bytes_received_.clear(); } diff --git a/modules/video_capture/android/java/src/org/webrtc/videoengine/VideoCaptureAndroid.java b/modules/video_capture/android/java/src/org/webrtc/videoengine/VideoCaptureAndroid.java index 926b350e..2ebf911f 100644 --- a/modules/video_capture/android/java/src/org/webrtc/videoengine/VideoCaptureAndroid.java +++ b/modules/video_capture/android/java/src/org/webrtc/videoengine/VideoCaptureAndroid.java @@ -11,11 +11,13 @@ package org.webrtc.videoengine; import java.io.IOException; +import java.util.List; import java.util.concurrent.Exchanger; import android.content.Context; import android.graphics.ImageFormat; import android.graphics.SurfaceTexture; +import android.hardware.Camera.Parameters; import android.hardware.Camera.PreviewCallback; import android.hardware.Camera; import android.opengl.GLES11Ext; @@ -58,6 +60,7 @@ public class VideoCaptureAndroid implements PreviewCallback, Callback { private double averageDurationMs; private long lastCaptureTimeMs; private int frameCount; + private int frameDropRatio; // Requests future capturers to send their frames to |localPreview| directly. public static void setLocalPreview(SurfaceHolder localPreview) { @@ -169,8 +172,40 @@ public class VideoCaptureAndroid implements PreviewCallback, Callback { if (parameters.isVideoStabilizationSupported()) { parameters.setVideoStabilization(true); } + parameters.setPictureSize(width, height); parameters.setPreviewSize(width, height); + + // Check if requested fps range is supported by camera, + // otherwise calculate frame drop ratio. + List<int[]> supportedFpsRanges = parameters.getSupportedPreviewFpsRange(); + frameDropRatio = Integer.MAX_VALUE; + for (int i = 0; i < supportedFpsRanges.size(); i++) { + int[] range = supportedFpsRanges.get(i); + if (range[Parameters.PREVIEW_FPS_MIN_INDEX] == min_mfps && + range[Parameters.PREVIEW_FPS_MAX_INDEX] == max_mfps) { + frameDropRatio = 1; + break; + } + if (range[Parameters.PREVIEW_FPS_MIN_INDEX] % min_mfps == 0 && + range[Parameters.PREVIEW_FPS_MAX_INDEX] % max_mfps == 0) { + int dropRatio = range[Parameters.PREVIEW_FPS_MAX_INDEX] / max_mfps; + frameDropRatio = Math.min(dropRatio, frameDropRatio); + } + } + if (frameDropRatio == Integer.MAX_VALUE) { + Log.e(TAG, "Can not find camera fps range"); + error = new RuntimeException("Can not find camera fps range"); + exchange(result, false); + return; + } + if (frameDropRatio > 1) { + Log.d(TAG, "Frame dropper is enabled. Ratio: " + frameDropRatio); + } + min_mfps *= frameDropRatio; + max_mfps *= frameDropRatio; + Log.d(TAG, "Camera preview mfps range: " + min_mfps + " - " + max_mfps); parameters.setPreviewFpsRange(min_mfps, max_mfps); + int format = ImageFormat.NV21; parameters.setPreviewFormat(format); camera.setParameters(parameters); @@ -180,7 +215,7 @@ public class VideoCaptureAndroid implements PreviewCallback, Callback { } camera.setPreviewCallbackWithBuffer(this); frameCount = 0; - averageDurationMs = 1000 / max_mfps; + averageDurationMs = 1000000.0f / (max_mfps / frameDropRatio); camera.startPreview(); exchange(result, true); return; @@ -296,8 +331,13 @@ public class VideoCaptureAndroid implements PreviewCallback, Callback { throw new RuntimeException("Unexpected camera in callback!"); } frameCount++; + // Check if frame needs to be dropped. + if ((frameDropRatio > 1) && (frameCount % frameDropRatio) > 0) { + camera.addCallbackBuffer(data); + return; + } long captureTimeMs = SystemClock.elapsedRealtime(); - if (frameCount > 1) { + if (frameCount > frameDropRatio) { double durationMs = captureTimeMs - lastCaptureTimeMs; averageDurationMs = 0.9 * averageDurationMs + 0.1 * durationMs; if ((frameCount % 30) == 0) { diff --git a/modules/video_capture/android/java/src/org/webrtc/videoengine/VideoCaptureDeviceInfoAndroid.java b/modules/video_capture/android/java/src/org/webrtc/videoengine/VideoCaptureDeviceInfoAndroid.java index fe207ca3..4b0089b1 100644 --- a/modules/video_capture/android/java/src/org/webrtc/videoengine/VideoCaptureDeviceInfoAndroid.java +++ b/modules/video_capture/android/java/src/org/webrtc/videoengine/VideoCaptureDeviceInfoAndroid.java @@ -75,6 +75,36 @@ public class VideoCaptureDeviceInfoAndroid { sizes.put(size); } + boolean is30fpsRange = false; + boolean is15fpsRange = false; + // If there is constant 30 fps mode, but no 15 fps - add 15 fps + // mode to the list of supported ranges. Frame drop will be done + // in software. + for (int[] range : supportedFpsRanges) { + if (range[Parameters.PREVIEW_FPS_MIN_INDEX] == 30000 && + range[Parameters.PREVIEW_FPS_MAX_INDEX] == 30000) { + is30fpsRange = true; + } + if (range[Parameters.PREVIEW_FPS_MIN_INDEX] == 15000 && + range[Parameters.PREVIEW_FPS_MAX_INDEX] == 15000) { + is15fpsRange = true; + } + } + if (is30fpsRange && !is15fpsRange) { + Log.d(TAG, "Adding 15 fps support"); + int[] newRange = new int [Parameters.PREVIEW_FPS_MAX_INDEX + 1]; + newRange[Parameters.PREVIEW_FPS_MIN_INDEX] = 15000; + newRange[Parameters.PREVIEW_FPS_MAX_INDEX] = 15000; + for (int j = 0; j < supportedFpsRanges.size(); j++ ) { + int[] range = supportedFpsRanges.get(j); + if (range[Parameters.PREVIEW_FPS_MAX_INDEX] > + newRange[Parameters.PREVIEW_FPS_MAX_INDEX]) { + supportedFpsRanges.add(j, newRange); + break; + } + } + } + JSONArray mfpsRanges = new JSONArray(); for (int[] range : supportedFpsRanges) { JSONObject mfpsRange = new JSONObject(); diff --git a/modules/video_capture/video_capture.gypi b/modules/video_capture/video_capture.gypi index 888d7727..22b8e02c 100644 --- a/modules/video_capture/video_capture.gypi +++ b/modules/video_capture/video_capture.gypi @@ -161,7 +161,7 @@ 'type': '<(gtest_target_type)', 'dependencies': [ 'video_capture_module', - 'video_capture_module_internal_impl', + 'video_capture_module_internal_impl', 'webrtc_utility', '<(webrtc_root)/system_wrappers/source/system_wrappers.gyp:system_wrappers', '<(DEPTH)/testing/gtest.gyp:gtest', @@ -192,13 +192,13 @@ 'dependencies': [ '<(DEPTH)/testing/android/native_test.gyp:native_test_native_code', ], - # Need to disable error due to the line in - # base/android/jni_android.h triggering it: - # const BASE_EXPORT jobject GetApplicationContext() - # error: type qualifiers ignored on function return type - 'cflags': [ - '-Wno-ignored-qualifiers', - ], + # Need to disable error due to the line in + # base/android/jni_android.h triggering it: + # const BASE_EXPORT jobject GetApplicationContext() + # error: type qualifiers ignored on function return type + 'cflags': [ + '-Wno-ignored-qualifiers', + ], }], ['OS=="mac"', { 'dependencies': [ @@ -243,7 +243,6 @@ ], 'includes': [ '../../build/isolate.gypi', - 'video_capture_tests.isolate', ], 'sources': [ 'video_capture_tests.isolate', diff --git a/modules/video_capture/video_capture_tests.isolate b/modules/video_capture/video_capture_tests.isolate index 57dd6673..be104376 100644 --- a/modules/video_capture/video_capture_tests.isolate +++ b/modules/video_capture/video_capture_tests.isolate @@ -9,7 +9,7 @@ 'conditions': [ ['OS=="android"', { 'variables': { - 'isolate_dependency_untracked': [ + 'files': [ '<(DEPTH)/data/', '<(DEPTH)/resources/', ], @@ -21,13 +21,10 @@ '<(DEPTH)/testing/test_env.py', '<(PRODUCT_DIR)/video_capture_tests<(EXECUTABLE_SUFFIX)', ], - 'isolate_dependency_tracked': [ + 'files': [ '<(DEPTH)/testing/test_env.py', '<(PRODUCT_DIR)/video_capture_tests<(EXECUTABLE_SUFFIX)', ], - 'isolate_dependency_untracked': [ - '<(DEPTH)/tools/swarming_client/', - ], }, }], ], diff --git a/modules/video_coding/BUILD.gn b/modules/video_coding/BUILD.gn index 07e7afc0..706e9d99 100644 --- a/modules/video_coding/BUILD.gn +++ b/modules/video_coding/BUILD.gn @@ -77,6 +77,7 @@ source_set("video_coding") { ":video_coding_utility", ":webrtc_i420", ":webrtc_vp8", + ":webrtc_vp9", "../../common_video", "../../system_wrappers", ] @@ -159,3 +160,35 @@ source_set("webrtc_vp8") { ] } } + +source_set("webrtc_vp9") { + sources = [ + "codecs/vp9/include/vp9.h", + "codecs/vp9/vp9_impl.cc", + "codecs/vp9/vp9_impl.h", + ] + + configs += [ "../..:common_config" ] + public_configs = [ "../..:common_inherited_config" ] + + if (is_clang) { + # Suppress warnings from Chrome's Clang plugins. + # See http://code.google.com/p/webrtc/issues/detail?id=163 for details. + configs -= [ "//build/config/clang:find_bad_constructs" ] + } + + # TODO(kjellander): Remove once libvpx has changed it's libvpx_config to be + # in direct_dependent_configs. + configs += [ "//third_party/libvpx:libvpx_config" ] + + deps = [ + ":video_coding_utility", + "../../common_video", + "../../system_wrappers", + ] + if (rtc_build_libvpx) { + deps += [ + "//third_party/libvpx", + ] + } +} diff --git a/modules/video_coding/codecs/interface/video_codec_interface.h b/modules/video_coding/codecs/interface/video_codec_interface.h index 82bcd26d..da72febb 100644 --- a/modules/video_coding/codecs/interface/video_codec_interface.h +++ b/modules/video_coding/codecs/interface/video_codec_interface.h @@ -18,6 +18,7 @@ #include "webrtc/modules/interface/module_common_types.h" #include "webrtc/modules/video_coding/codecs/interface/video_error_codes.h" #include "webrtc/typedefs.h" +#include "webrtc/video_decoder.h" #include "webrtc/video_encoder.h" namespace webrtc @@ -41,6 +42,19 @@ struct CodecSpecificInfoVP8 { int8_t keyIdx; // Negative value to skip keyIdx. }; +struct CodecSpecificInfoVP9 { + bool hasReceivedSLI; + uint8_t pictureIdSLI; + bool hasReceivedRPSI; + uint64_t pictureIdRPSI; + int16_t pictureId; // Negative value to skip pictureId. + bool nonReference; + uint8_t temporalIdx; + bool layerSync; + int tl0PicIdx; // Negative value to skip tl0PicIdx. + int8_t keyIdx; // Negative value to skip keyIdx. +}; + struct CodecSpecificInfoGeneric { uint8_t simulcast_idx; }; @@ -50,6 +64,7 @@ struct CodecSpecificInfoH264 {}; union CodecSpecificInfoUnion { CodecSpecificInfoGeneric generic; CodecSpecificInfoVP8 VP8; + CodecSpecificInfoVP9 VP9; CodecSpecificInfoH264 H264; }; @@ -62,96 +77,6 @@ struct CodecSpecificInfo CodecSpecificInfoUnion codecSpecific; }; -class DecodedImageCallback -{ -public: - virtual ~DecodedImageCallback() {}; - - // Callback function which is called when an image has been decoded. - // - // Input: - // - decodedImage : The decoded image. - // - // Return value : 0 if OK, < 0 otherwise. - virtual int32_t Decoded(I420VideoFrame& decodedImage) = 0; - - virtual int32_t ReceivedDecodedReferenceFrame(const uint64_t pictureId) {return -1;} - - virtual int32_t ReceivedDecodedFrame(const uint64_t pictureId) {return -1;} -}; - -class VideoDecoder -{ -public: - virtual ~VideoDecoder() {}; - - // Initialize the decoder with the information from the VideoCodec. - // - // Input: - // - inst : Codec settings - // - numberOfCores : Number of cores available for the decoder - // - // Return value : WEBRTC_VIDEO_CODEC_OK if OK, < 0 otherwise. - virtual int32_t InitDecode(const VideoCodec* codecSettings, int32_t numberOfCores) = 0; - - // Decode encoded image (as a part of a video stream). The decoded image - // will be returned to the user through the decode complete callback. - // - // Input: - // - inputImage : Encoded image to be decoded - // - missingFrames : True if one or more frames have been lost - // since the previous decode call. - // - fragmentation : Specifies where the encoded frame can be - // split into separate fragments. The meaning - // of fragment is codec specific, but often - // means that each fragment is decodable by - // itself. - // - codecSpecificInfo : Pointer to codec specific data - // - renderTimeMs : System time to render in milliseconds. Only - // used by decoders with internal rendering. - // - // Return value : WEBRTC_VIDEO_CODEC_OK if OK, < 0 otherwise. - virtual int32_t - Decode(const EncodedImage& inputImage, - bool missingFrames, - const RTPFragmentationHeader* fragmentation, - const CodecSpecificInfo* codecSpecificInfo = NULL, - int64_t renderTimeMs = -1) = 0; - - // Register an decode complete callback object. - // - // Input: - // - callback : Callback object which handles decoded images. - // - // Return value : WEBRTC_VIDEO_CODEC_OK if OK, < 0 otherwise. - virtual int32_t RegisterDecodeCompleteCallback(DecodedImageCallback* callback) = 0; - - // Free decoder memory. - // - // Return value : WEBRTC_VIDEO_CODEC_OK if OK, < 0 otherwise. - virtual int32_t Release() = 0; - - // Reset decoder state and prepare for a new call. - // - // Return value : WEBRTC_VIDEO_CODEC_OK if OK, < 0 otherwise. - virtual int32_t Reset() = 0; - - // Codec configuration data sent out-of-band, i.e. in SIP call setup - // - // Input/Output: - // - buffer : Buffer pointer to the configuration data - // - size : The size of the configuration data in - // bytes - // - // Return value : WEBRTC_VIDEO_CODEC_OK if OK, < 0 otherwise. - virtual int32_t SetCodecConfigParameters(const uint8_t* /*buffer*/, int32_t /*size*/) { return WEBRTC_VIDEO_CODEC_ERROR; } - - // Create a copy of the codec and its internal state. - // - // Return value : A copy of the instance if OK, NULL otherwise. - virtual VideoDecoder* Copy() { return NULL; } -}; - } // namespace webrtc #endif // WEBRTC_MODULES_VIDEO_CODING_CODECS_INTERFACE_VIDEO_CODEC_INTERFACE_H diff --git a/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc b/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc index bd4a563f..73f774d6 100644 --- a/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc +++ b/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc @@ -16,6 +16,7 @@ #include "webrtc/modules/video_coding/codecs/test/packet_manipulator.h" #include "webrtc/modules/video_coding/codecs/test/videoprocessor.h" #include "webrtc/modules/video_coding/codecs/vp8/include/vp8.h" +#include "webrtc/modules/video_coding/codecs/vp9/include/vp9.h" #include "webrtc/modules/video_coding/codecs/vp8/include/vp8_common_types.h" #include "webrtc/modules/video_coding/main/interface/video_coding.h" #include "webrtc/test/testsupport/fileutils.h" @@ -37,6 +38,7 @@ const int kBaseKeyFrameInterval = 3000; // Codec and network settings. struct CodecConfigPars { + VideoCodecType codec_type; float packet_loss; int num_temporal_layers; int key_frame_interval; @@ -136,6 +138,7 @@ class VideoProcessorIntegrationTest: public testing::Test { float start_bitrate_; // Codec and network settings. + VideoCodecType codec_type_; float packet_loss_; int num_temporal_layers_; int key_frame_interval_; @@ -149,8 +152,15 @@ class VideoProcessorIntegrationTest: public testing::Test { virtual ~VideoProcessorIntegrationTest() {} void SetUpCodecConfig() { - encoder_ = VP8Encoder::Create(); - decoder_ = VP8Decoder::Create(); + if (codec_type_ == kVideoCodecVP8) { + encoder_ = VP8Encoder::Create(); + decoder_ = VP8Decoder::Create(); + VideoCodingModule::Codec(kVideoCodecVP8, &codec_settings_); + } else if (codec_type_ == kVideoCodecVP9) { + encoder_ = VP9Encoder::Create(); + decoder_ = VP9Decoder::Create(); + VideoCodingModule::Codec(kVideoCodecVP9, &codec_settings_); + } // CIF is currently used for all tests below. // Setup the TestConfig struct for processing of a clip in CIF resolution. @@ -169,26 +179,42 @@ class VideoProcessorIntegrationTest: public testing::Test { config_.keyframe_interval = key_frame_interval_; config_.networking_config.packet_loss_probability = packet_loss_; - // Get a codec configuration struct and configure it. - VideoCodingModule::Codec(kVideoCodecVP8, &codec_settings_); + // Configure codec settings. config_.codec_settings = &codec_settings_; config_.codec_settings->startBitrate = start_bitrate_; config_.codec_settings->width = kCIFWidth; config_.codec_settings->height = kCIFHeight; - // These features may be set depending on the test. - config_.codec_settings->codecSpecific.VP8.errorConcealmentOn = - error_concealment_on_; - config_.codec_settings->codecSpecific.VP8.denoisingOn = - denoising_on_; - config_.codec_settings->codecSpecific.VP8.numberOfTemporalLayers = - num_temporal_layers_; - config_.codec_settings->codecSpecific.VP8.frameDroppingOn = - frame_dropper_on_; - config_.codec_settings->codecSpecific.VP8.automaticResizeOn = - spatial_resize_on_; - config_.codec_settings->codecSpecific.VP8.keyFrameInterval = - kBaseKeyFrameInterval; + // These features may be set depending on the test. + switch (config_.codec_settings->codecType) { + case kVideoCodecVP8: + config_.codec_settings->codecSpecific.VP8.errorConcealmentOn = + error_concealment_on_; + config_.codec_settings->codecSpecific.VP8.denoisingOn = + denoising_on_; + config_.codec_settings->codecSpecific.VP8.numberOfTemporalLayers = + num_temporal_layers_; + config_.codec_settings->codecSpecific.VP8.frameDroppingOn = + frame_dropper_on_; + config_.codec_settings->codecSpecific.VP8.automaticResizeOn = + spatial_resize_on_; + config_.codec_settings->codecSpecific.VP8.keyFrameInterval = + kBaseKeyFrameInterval; + break; + case kVideoCodecVP9: + config_.codec_settings->codecSpecific.VP9.denoisingOn = + denoising_on_; + config_.codec_settings->codecSpecific.VP9.numberOfTemporalLayers = + num_temporal_layers_; + config_.codec_settings->codecSpecific.VP9.frameDroppingOn = + frame_dropper_on_; + config_.codec_settings->codecSpecific.VP9.keyFrameInterval = + kBaseKeyFrameInterval; + break; + default: + assert(false); + break; + } frame_reader_ = new webrtc::test::FrameReaderImpl(config_.input_filename, config_.frame_length_in_bytes); @@ -405,6 +431,7 @@ class VideoProcessorIntegrationTest: public testing::Test { CodecConfigPars process, RateControlMetrics* rc_metrics) { // Codec/config settings. + codec_type_ = process.codec_type; start_bitrate_ = rate_profile.target_bit_rate[0]; packet_loss_ = process.packet_loss; key_frame_interval_ = process.key_frame_interval; @@ -514,6 +541,7 @@ void SetRateProfilePars(RateProfile* rate_profile, } void SetCodecParameters(CodecConfigPars* process_settings, + VideoCodecType codec_type, float packet_loss, int key_frame_interval, int num_temporal_layers, @@ -521,6 +549,7 @@ void SetCodecParameters(CodecConfigPars* process_settings, bool denoising_on, bool frame_dropper_on, bool spatial_resize_on) { + process_settings->codec_type = codec_type; process_settings->packet_loss = packet_loss; process_settings->key_frame_interval = key_frame_interval; process_settings->num_temporal_layers = num_temporal_layers, @@ -560,7 +589,126 @@ void SetRateControlMetrics(RateControlMetrics* rc_metrics, rc_metrics[update_index].num_spatial_resizes = num_spatial_resizes; } -// Run with no packet loss and fixed bitrate. Quality should be very high. +// VP9: Run with no packet loss and fixed bitrate. Quality should be very high. +// One key frame (first frame only) in sequence. Setting |key_frame_interval| +// to -1 below means no periodic key frames in test. +TEST_F(VideoProcessorIntegrationTest, Process0PercentPacketLossVP9) { + // Bitrate and frame rate profile. + RateProfile rate_profile; + SetRateProfilePars(&rate_profile, 0, 500, 30, 0); + rate_profile.frame_index_rate_update[1] = kNbrFramesShort + 1; + rate_profile.num_frames = kNbrFramesShort; + // Codec/network settings. + CodecConfigPars process_settings; + SetCodecParameters(&process_settings, kVideoCodecVP9, 0.0f, -1, 1, false, + false, true, false); + // Metrics for expected quality. + QualityMetrics quality_metrics; + SetQualityMetrics(&quality_metrics, 37.5, 36.0, 0.94, 0.93); + // Metrics for rate control. + RateControlMetrics rc_metrics[1]; + SetRateControlMetrics(rc_metrics, 0, 0, 40, 20, 10, 15, 0); + ProcessFramesAndVerify(quality_metrics, + rate_profile, + process_settings, + rc_metrics); +} + +// VP9: Run with 5% packet loss and fixed bitrate. Quality should be a bit +// lower. One key frame (first frame only) in sequence. +TEST_F(VideoProcessorIntegrationTest, Process5PercentPacketLossVP9) { + // Bitrate and frame rate profile. + RateProfile rate_profile; + SetRateProfilePars(&rate_profile, 0, 500, 30, 0); + rate_profile.frame_index_rate_update[1] = kNbrFramesShort + 1; + rate_profile.num_frames = kNbrFramesShort; + // Codec/network settings. + CodecConfigPars process_settings; + SetCodecParameters(&process_settings, kVideoCodecVP9, 0.05f, -1, 1, false, + false, true, false); + // Metrics for expected quality. + QualityMetrics quality_metrics; + SetQualityMetrics(&quality_metrics, 17.0, 15.0, 0.45, 0.38); + // Metrics for rate control. + RateControlMetrics rc_metrics[1]; + SetRateControlMetrics(rc_metrics, 0, 0, 40, 20, 10, 15, 0); + ProcessFramesAndVerify(quality_metrics, + rate_profile, + process_settings, + rc_metrics); +} + + +// VP9: Run with no packet loss, with varying bitrate (3 rate updates): +// low to high to medium. Check that quality and encoder response to the new +// target rate/per-frame bandwidth (for each rate update) is within limits. +// One key frame (first frame only) in sequence. +TEST_F(VideoProcessorIntegrationTest, ProcessNoLossChangeBitRateVP9) { + // Bitrate and frame rate profile. + RateProfile rate_profile; + SetRateProfilePars(&rate_profile, 0, 200, 30, 0); + SetRateProfilePars(&rate_profile, 1, 800, 30, 100); + SetRateProfilePars(&rate_profile, 2, 500, 30, 200); + rate_profile.frame_index_rate_update[3] = kNbrFramesLong + 1; + rate_profile.num_frames = kNbrFramesLong; + // Codec/network settings. + CodecConfigPars process_settings; + SetCodecParameters(&process_settings, kVideoCodecVP9, 0.0f, -1, 1, false, + false, true, false); + // Metrics for expected quality. + QualityMetrics quality_metrics; + SetQualityMetrics(&quality_metrics, 36.0, 32.0, 0.90, 0.85); + // Metrics for rate control. + RateControlMetrics rc_metrics[3]; + SetRateControlMetrics(rc_metrics, 0, 0, 30, 20, 20, 20, 0); + SetRateControlMetrics(rc_metrics, 1, 2, 0, 20, 20, 60, 0); + SetRateControlMetrics(rc_metrics, 2, 0, 0, 20, 20, 30, 0); + ProcessFramesAndVerify(quality_metrics, + rate_profile, + process_settings, + rc_metrics); +} + +// VP9: Run with no packet loss, with an update (decrease) in frame rate. +// Lower frame rate means higher per-frame-bandwidth, so easier to encode. +// At the low bitrate in this test, this means better rate control after the +// update(s) to lower frame rate. So expect less frame drops, and max values +// for the rate control metrics can be lower. One key frame (first frame only). +// Note: quality after update should be higher but we currently compute quality +// metrics averaged over whole sequence run. +TEST_F(VideoProcessorIntegrationTest, + ProcessNoLossChangeFrameRateFrameDropVP9) { + config_.networking_config.packet_loss_probability = 0; + // Bitrate and frame rate profile. + RateProfile rate_profile; + SetRateProfilePars(&rate_profile, 0, 50, 24, 0); + SetRateProfilePars(&rate_profile, 1, 50, 15, 100); + SetRateProfilePars(&rate_profile, 2, 50, 10, 200); + rate_profile.frame_index_rate_update[3] = kNbrFramesLong + 1; + rate_profile.num_frames = kNbrFramesLong; + // Codec/network settings. + CodecConfigPars process_settings; + SetCodecParameters(&process_settings, kVideoCodecVP9, 0.0f, -1, 1, false, + false, true, false); + // Metrics for expected quality. + QualityMetrics quality_metrics; + SetQualityMetrics(&quality_metrics, 30.0, 18.0, 0.80, 0.40); + // Metrics for rate control. + RateControlMetrics rc_metrics[3]; + SetRateControlMetrics(rc_metrics, 0, 35, 55, 70, 15, 40, 0); + SetRateControlMetrics(rc_metrics, 1, 15, 0, 50, 10, 30, 0); + SetRateControlMetrics(rc_metrics, 2, 5, 0, 38, 10, 30, 0); + ProcessFramesAndVerify(quality_metrics, + rate_profile, + process_settings, + rc_metrics); +} + + +// TODO(marpan): Add temporal layer test for VP9, once changes are in +// vp9 wrapper for this. + +// VP8: Run with no packet loss and fixed bitrate. Quality should be very high. // One key frame (first frame only) in sequence. Setting |key_frame_interval| // to -1 below means no periodic key frames in test. TEST_F(VideoProcessorIntegrationTest, ProcessZeroPacketLoss) { @@ -571,7 +719,8 @@ TEST_F(VideoProcessorIntegrationTest, ProcessZeroPacketLoss) { rate_profile.num_frames = kNbrFramesShort; // Codec/network settings. CodecConfigPars process_settings; - SetCodecParameters(&process_settings, 0.0f, -1, 1, false, true, true, false); + SetCodecParameters(&process_settings, kVideoCodecVP8, 0.0f, -1, 1, false, + true, true, false); // Metrics for expected quality. QualityMetrics quality_metrics; SetQualityMetrics(&quality_metrics, 34.95, 33.0, 0.90, 0.89); @@ -584,8 +733,8 @@ TEST_F(VideoProcessorIntegrationTest, ProcessZeroPacketLoss) { rc_metrics); } -// Run with 5% packet loss and fixed bitrate. Quality should be a bit lower. -// One key frame (first frame only) in sequence. +// VP8: Run with 5% packet loss and fixed bitrate. Quality should be a bit +// lower. One key frame (first frame only) in sequence. TEST_F(VideoProcessorIntegrationTest, Process5PercentPacketLoss) { // Bitrate and frame rate profile. RateProfile rate_profile; @@ -594,7 +743,8 @@ TEST_F(VideoProcessorIntegrationTest, Process5PercentPacketLoss) { rate_profile.num_frames = kNbrFramesShort; // Codec/network settings. CodecConfigPars process_settings; - SetCodecParameters(&process_settings, 0.05f, -1, 1, false, true, true, false); + SetCodecParameters(&process_settings, kVideoCodecVP8, 0.05f, -1, 1, false, + true, true, false); // Metrics for expected quality. QualityMetrics quality_metrics; SetQualityMetrics(&quality_metrics, 20.0, 16.0, 0.60, 0.40); @@ -607,7 +757,7 @@ TEST_F(VideoProcessorIntegrationTest, Process5PercentPacketLoss) { rc_metrics); } -// Run with 10% packet loss and fixed bitrate. Quality should be even lower. +// VP8: Run with 10% packet loss and fixed bitrate. Quality should be lower. // One key frame (first frame only) in sequence. TEST_F(VideoProcessorIntegrationTest, Process10PercentPacketLoss) { // Bitrate and frame rate profile. @@ -617,7 +767,8 @@ TEST_F(VideoProcessorIntegrationTest, Process10PercentPacketLoss) { rate_profile.num_frames = kNbrFramesShort; // Codec/network settings. CodecConfigPars process_settings; - SetCodecParameters(&process_settings, 0.1f, -1, 1, false, true, true, false); + SetCodecParameters(&process_settings, kVideoCodecVP8, 0.1f, -1, 1, false, + true, true, false); // Metrics for expected quality. QualityMetrics quality_metrics; SetQualityMetrics(&quality_metrics, 19.0, 16.0, 0.50, 0.35); @@ -639,12 +790,12 @@ TEST_F(VideoProcessorIntegrationTest, Process10PercentPacketLoss) { // disabled on Android. Some quality parameter in the above test has been // adjusted to also pass for |cpu_speed| <= 12. -// Run with no packet loss, with varying bitrate (3 rate updates): +// VP8: Run with no packet loss, with varying bitrate (3 rate updates): // low to high to medium. Check that quality and encoder response to the new // target rate/per-frame bandwidth (for each rate update) is within limits. // One key frame (first frame only) in sequence. TEST_F(VideoProcessorIntegrationTest, - DISABLED_ON_ANDROID(ProcessNoLossChangeBitRate)) { + DISABLED_ON_ANDROID(ProcessNoLossChangeBitRateVP8)) { // Bitrate and frame rate profile. RateProfile rate_profile; SetRateProfilePars(&rate_profile, 0, 200, 30, 0); @@ -654,7 +805,8 @@ TEST_F(VideoProcessorIntegrationTest, rate_profile.num_frames = kNbrFramesLong; // Codec/network settings. CodecConfigPars process_settings; - SetCodecParameters(&process_settings, 0.0f, -1, 1, false, true, true, false); + SetCodecParameters(&process_settings, kVideoCodecVP8, 0.0f, -1, 1, false, + true, true, false); // Metrics for expected quality. QualityMetrics quality_metrics; SetQualityMetrics(&quality_metrics, 34.0, 32.0, 0.85, 0.80); @@ -669,15 +821,15 @@ TEST_F(VideoProcessorIntegrationTest, rc_metrics); } -// Run with no packet loss, with an update (decrease) in frame rate. +// VP8: Run with no packet loss, with an update (decrease) in frame rate. // Lower frame rate means higher per-frame-bandwidth, so easier to encode. // At the bitrate in this test, this means better rate control after the // update(s) to lower frame rate. So expect less frame drops, and max values // for the rate control metrics can be lower. One key frame (first frame only). // Note: quality after update should be higher but we currently compute quality -// metrics avergaed over whole sequence run. +// metrics averaged over whole sequence run. TEST_F(VideoProcessorIntegrationTest, - DISABLED_ON_ANDROID(ProcessNoLossChangeFrameRateFrameDrop)) { + DISABLED_ON_ANDROID(ProcessNoLossChangeFrameRateFrameDropVP8)) { config_.networking_config.packet_loss_probability = 0; // Bitrate and frame rate profile. RateProfile rate_profile; @@ -688,7 +840,8 @@ TEST_F(VideoProcessorIntegrationTest, rate_profile.num_frames = kNbrFramesLong; // Codec/network settings. CodecConfigPars process_settings; - SetCodecParameters(&process_settings, 0.0f, -1, 1, false, true, true, false); + SetCodecParameters(&process_settings, kVideoCodecVP8, 0.0f, -1, 1, false, + true, true, false); // Metrics for expected quality. QualityMetrics quality_metrics; SetQualityMetrics(&quality_metrics, 31.0, 22.0, 0.80, 0.65); @@ -706,7 +859,7 @@ TEST_F(VideoProcessorIntegrationTest, // Run with no packet loss, at low bitrate. During this time we should've // resized once. TEST_F(VideoProcessorIntegrationTest, - DISABLED_ON_ANDROID(ProcessNoLossSpatialResizeFrameDrop)) { + DISABLED_ON_ANDROID(ProcessNoLossSpatialResizeFrameDropVP8)) { config_.networking_config.packet_loss_probability = 0; // Bitrate and frame rate profile. RateProfile rate_profile; @@ -715,8 +868,8 @@ TEST_F(VideoProcessorIntegrationTest, rate_profile.num_frames = kNbrFramesLong; // Codec/network settings. CodecConfigPars process_settings; - SetCodecParameters( - &process_settings, 0.0f, kNbrFramesLong, 1, false, true, true, true); + SetCodecParameters(&process_settings, kVideoCodecVP8, 0.0f, kNbrFramesLong, + 1, false, true, true, true); // Metrics for expected quality. QualityMetrics quality_metrics; SetQualityMetrics(&quality_metrics, 25.0, 15.0, 0.70, 0.40); @@ -729,13 +882,13 @@ TEST_F(VideoProcessorIntegrationTest, rc_metrics); } -// Run with no packet loss, with 3 temporal layers, with a rate update in the -// middle of the sequence. The max values for the frame size mismatch and +// VP8: Run with no packet loss, with 3 temporal layers, with a rate update in +// the middle of the sequence. The max values for the frame size mismatch and // encoding rate mismatch are applied to each layer. // No dropped frames in this test, and internal spatial resizer is off. // One key frame (first frame only) in sequence, so no spatial resizing. TEST_F(VideoProcessorIntegrationTest, - DISABLED_ON_ANDROID(ProcessNoLossTemporalLayers)) { + DISABLED_ON_ANDROID(ProcessNoLossTemporalLayersVP8)) { config_.networking_config.packet_loss_probability = 0; // Bitrate and frame rate profile. RateProfile rate_profile; @@ -745,7 +898,8 @@ TEST_F(VideoProcessorIntegrationTest, rate_profile.num_frames = kNbrFramesLong; // Codec/network settings. CodecConfigPars process_settings; - SetCodecParameters(&process_settings, 0.0f, -1, 3, false, true, true, false); + SetCodecParameters(&process_settings, kVideoCodecVP8, 0.0f, -1, 3, false, + true, true, false); // Metrics for expected quality. QualityMetrics quality_metrics; SetQualityMetrics(&quality_metrics, 32.5, 30.0, 0.85, 0.80); diff --git a/modules/video_coding/codecs/vp8/vp8_impl.h b/modules/video_coding/codecs/vp8/vp8_impl.h index 08ce3c91..fec53d53 100644 --- a/modules/video_coding/codecs/vp8/vp8_impl.h +++ b/modules/video_coding/codecs/vp8/vp8_impl.h @@ -35,73 +35,20 @@ class VP8EncoderImpl : public VP8Encoder { virtual ~VP8EncoderImpl(); - // Free encoder memory. - // - // Return value : WEBRTC_VIDEO_CODEC_OK if OK, < 0 otherwise. virtual int Release(); - // Initialize the encoder with the information from the codecSettings - // - // Input: - // - codec_settings : Codec settings - // - number_of_cores : Number of cores available for the encoder - // - max_payload_size : The maximum size each payload is allowed - // to have. Usually MTU - overhead. - // - // Return value : Set bit rate if OK - // <0 - Errors: - // WEBRTC_VIDEO_CODEC_ERR_PARAMETER - // WEBRTC_VIDEO_CODEC_ERR_SIZE - // WEBRTC_VIDEO_CODEC_LEVEL_EXCEEDED - // WEBRTC_VIDEO_CODEC_MEMORY - // WEBRTC_VIDEO_CODEC_ERROR virtual int InitEncode(const VideoCodec* codec_settings, int number_of_cores, uint32_t max_payload_size); - // Encode an I420 image (as a part of a video stream). The encoded image - // will be returned to the user through the encode complete callback. - // - // Input: - // - input_image : Image to be encoded - // - frame_types : Frame type to be generated by the encoder. - // - // Return value : WEBRTC_VIDEO_CODEC_OK if OK - // <0 - Errors: - // WEBRTC_VIDEO_CODEC_ERR_PARAMETER - // WEBRTC_VIDEO_CODEC_MEMORY - // WEBRTC_VIDEO_CODEC_ERROR - // WEBRTC_VIDEO_CODEC_TIMEOUT - virtual int Encode(const I420VideoFrame& input_image, const CodecSpecificInfo* codec_specific_info, const std::vector<VideoFrameType>* frame_types); - // Register an encode complete callback object. - // - // Input: - // - callback : Callback object which handles encoded images. - // - // Return value : WEBRTC_VIDEO_CODEC_OK if OK, < 0 otherwise. virtual int RegisterEncodeCompleteCallback(EncodedImageCallback* callback); - // Inform the encoder of the new packet loss rate and the round-trip time of - // the network. - // - // - packet_loss : Fraction lost - // (loss rate in percent = 100 * packetLoss / 255) - // - rtt : Round-trip time in milliseconds - // Return value : WEBRTC_VIDEO_CODEC_OK if OK - // <0 - Errors: WEBRTC_VIDEO_CODEC_ERROR - // virtual int SetChannelParameters(uint32_t packet_loss, int rtt); - // Inform the encoder about the new target bit rate. - // - // - new_bitrate_kbit : New target bit rate - // - frame_rate : The target frame rate - // - // Return value : WEBRTC_VIDEO_CODEC_OK if OK, < 0 otherwise. virtual int SetRates(uint32_t new_bitrate_kbit, uint32_t frame_rate); private: @@ -150,61 +97,20 @@ class VP8DecoderImpl : public VP8Decoder { virtual ~VP8DecoderImpl(); - // Initialize the decoder. - // - // Return value : WEBRTC_VIDEO_CODEC_OK. - // <0 - Errors: - // WEBRTC_VIDEO_CODEC_ERROR virtual int InitDecode(const VideoCodec* inst, int number_of_cores); - // Decode encoded image (as a part of a video stream). The decoded image - // will be returned to the user through the decode complete callback. - // - // Input: - // - input_image : Encoded image to be decoded - // - missing_frames : True if one or more frames have been lost - // since the previous decode call. - // - fragmentation : Specifies the start and length of each VP8 - // partition. - // - codec_specific_info : pointer to specific codec data - // - render_time_ms : Render time in Ms - // - // Return value : WEBRTC_VIDEO_CODEC_OK if OK - // <0 - Errors: - // WEBRTC_VIDEO_CODEC_ERROR - // WEBRTC_VIDEO_CODEC_ERR_PARAMETER virtual int Decode(const EncodedImage& input_image, bool missing_frames, const RTPFragmentationHeader* fragmentation, const CodecSpecificInfo* codec_specific_info, int64_t /*render_time_ms*/); - // Register a decode complete callback object. - // - // Input: - // - callback : Callback object which handles decoded images. - // - // Return value : WEBRTC_VIDEO_CODEC_OK if OK, < 0 otherwise. virtual int RegisterDecodeCompleteCallback(DecodedImageCallback* callback); - // Free decoder memory. - // - // Return value : WEBRTC_VIDEO_CODEC_OK if OK - // <0 - Errors: - // WEBRTC_VIDEO_CODEC_ERROR virtual int Release(); - // Reset decoder state and prepare for a new call. - // - // Return value : WEBRTC_VIDEO_CODEC_OK. - // <0 - Errors: - // WEBRTC_VIDEO_CODEC_UNINITIALIZED - // WEBRTC_VIDEO_CODEC_ERROR virtual int Reset(); - // Create a copy of the codec and its internal state. - // - // Return value : A copy of the instance if OK, NULL otherwise. virtual VideoDecoder* Copy(); private: diff --git a/modules/video_coding/codecs/vp9/include/vp9.h b/modules/video_coding/codecs/vp9/include/vp9.h new file mode 100644 index 00000000..cd77f72d --- /dev/null +++ b/modules/video_coding/codecs/vp9/include/vp9.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + * + */ + +#ifndef WEBRTC_MODULES_VIDEO_CODING_CODECS_VP9_INCLUDE_VP9_H_ +#define WEBRTC_MODULES_VIDEO_CODING_CODECS_VP9_INCLUDE_VP9_H_ + +#include "webrtc/modules/video_coding/codecs/interface/video_codec_interface.h" + +namespace webrtc { + +class VP9Encoder : public VideoEncoder { + public: + static VP9Encoder* Create(); + + virtual ~VP9Encoder() {} +}; + + +class VP9Decoder : public VideoDecoder { + public: + static VP9Decoder* Create(); + + virtual ~VP9Decoder() {} +}; +} // namespace webrtc + +#endif // WEBRTC_MODULES_VIDEO_CODING_CODECS_VP9_INCLUDE_VP9_H_ diff --git a/modules/video_coding/codecs/vp9/vp9.gyp b/modules/video_coding/codecs/vp9/vp9.gyp new file mode 100644 index 00000000..2bd46feb --- /dev/null +++ b/modules/video_coding/codecs/vp9/vp9.gyp @@ -0,0 +1,36 @@ +# Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +{ + 'includes': [ + '../../../../build/common.gypi', + ], + 'targets': [ + { + 'target_name': 'webrtc_vp9', + 'type': 'static_library', + 'dependencies': [ + '<(webrtc_root)/common_video/common_video.gyp:common_video', + '<(webrtc_root)/modules/video_coding/utility/video_coding_utility.gyp:video_coding_utility', + '<(webrtc_root)/system_wrappers/source/system_wrappers.gyp:system_wrappers', + ], + 'conditions': [ + ['build_libvpx==1', { + 'dependencies': [ + '<(DEPTH)/third_party/libvpx/libvpx.gyp:libvpx', + ], + }], + ], + 'sources': [ + 'include/vp9.h', + 'vp9_impl.cc', + 'vp9_impl.h', + ], + }, + ], +} diff --git a/modules/video_coding/codecs/vp9/vp9_impl.cc b/modules/video_coding/codecs/vp9/vp9_impl.cc new file mode 100644 index 00000000..33f11a37 --- /dev/null +++ b/modules/video_coding/codecs/vp9/vp9_impl.cc @@ -0,0 +1,487 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + * + */ + +#include "webrtc/modules/video_coding/codecs/vp9/vp9_impl.h" + +#include <stdlib.h> +#include <string.h> +#include <time.h> +#include <vector> + +#include "vpx/vpx_encoder.h" +#include "vpx/vpx_decoder.h" +#include "vpx/vp8cx.h" +#include "vpx/vp8dx.h" + +#include "webrtc/common.h" +#include "webrtc/common_video/libyuv/include/webrtc_libyuv.h" +#include "webrtc/modules/interface/module_common_types.h" +#include "webrtc/system_wrappers/interface/tick_util.h" +#include "webrtc/system_wrappers/interface/trace_event.h" + +namespace webrtc { + +VP9Encoder* VP9Encoder::Create() { + return new VP9EncoderImpl(); +} + +VP9EncoderImpl::VP9EncoderImpl() + : encoded_image_(), + encoded_complete_callback_(NULL), + inited_(false), + timestamp_(0), + picture_id_(0), + cpu_speed_(3), + rc_max_intra_target_(0), + encoder_(NULL), + config_(NULL), + raw_(NULL) { + memset(&codec_, 0, sizeof(codec_)); + uint32_t seed = static_cast<uint32_t>(TickTime::MillisecondTimestamp()); + srand(seed); +} + +VP9EncoderImpl::~VP9EncoderImpl() { + Release(); +} + +int VP9EncoderImpl::Release() { + if (encoded_image_._buffer != NULL) { + delete [] encoded_image_._buffer; + encoded_image_._buffer = NULL; + } + if (encoder_ != NULL) { + if (vpx_codec_destroy(encoder_)) { + return WEBRTC_VIDEO_CODEC_MEMORY; + } + delete encoder_; + encoder_ = NULL; + } + if (config_ != NULL) { + delete config_; + config_ = NULL; + } + if (raw_ != NULL) { + vpx_img_free(raw_); + raw_ = NULL; + } + inited_ = false; + return WEBRTC_VIDEO_CODEC_OK; +} + +int VP9EncoderImpl::SetRates(uint32_t new_bitrate_kbit, + uint32_t new_framerate) { + if (!inited_) { + return WEBRTC_VIDEO_CODEC_UNINITIALIZED; + } + if (encoder_->err) { + return WEBRTC_VIDEO_CODEC_ERROR; + } + if (new_framerate < 1) { + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + // Update bit rate + if (codec_.maxBitrate > 0 && new_bitrate_kbit > codec_.maxBitrate) { + new_bitrate_kbit = codec_.maxBitrate; + } + config_->rc_target_bitrate = new_bitrate_kbit; + codec_.maxFramerate = new_framerate; + // Update encoder context + if (vpx_codec_enc_config_set(encoder_, config_)) { + return WEBRTC_VIDEO_CODEC_ERROR; + } + return WEBRTC_VIDEO_CODEC_OK; +} + +int VP9EncoderImpl::InitEncode(const VideoCodec* inst, + int number_of_cores, + uint32_t /*max_payload_size*/) { + if (inst == NULL) { + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + if (inst->maxFramerate < 1) { + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + // Allow zero to represent an unspecified maxBitRate + if (inst->maxBitrate > 0 && inst->startBitrate > inst->maxBitrate) { + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + if (inst->width < 1 || inst->height < 1) { + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + if (number_of_cores < 1) { + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + int retVal = Release(); + if (retVal < 0) { + return retVal; + } + if (encoder_ == NULL) { + encoder_ = new vpx_codec_ctx_t; + } + if (config_ == NULL) { + config_ = new vpx_codec_enc_cfg_t; + } + timestamp_ = 0; + if (&codec_ != inst) { + codec_ = *inst; + } + // Random start 16 bits is enough. + picture_id_ = static_cast<uint16_t>(rand()) & 0x7FFF; + // Allocate memory for encoded image + if (encoded_image_._buffer != NULL) { + delete [] encoded_image_._buffer; + } + encoded_image_._size = CalcBufferSize(kI420, codec_.width, codec_.height); + encoded_image_._buffer = new uint8_t[encoded_image_._size]; + encoded_image_._completeFrame = true; + // Creating a wrapper to the image - setting image data to NULL. Actual + // pointer will be set in encode. Setting align to 1, as it is meaningless + // (actual memory is not allocated). + raw_ = vpx_img_wrap(NULL, VPX_IMG_FMT_I420, codec_.width, codec_.height, + 1, NULL); + // Populate encoder configuration with default values. + if (vpx_codec_enc_config_default(vpx_codec_vp9_cx(), config_, 0)) { + return WEBRTC_VIDEO_CODEC_ERROR; + } + config_->g_w = codec_.width; + config_->g_h = codec_.height; + config_->rc_target_bitrate = inst->startBitrate; // in kbit/s + config_->g_error_resilient = 1; + // Setting the time base of the codec. + config_->g_timebase.num = 1; + config_->g_timebase.den = 90000; + config_->g_lag_in_frames = 0; // 0- no frame lagging + config_->g_threads = 1; + // Rate control settings. + config_->rc_dropframe_thresh = inst->codecSpecific.VP9.frameDroppingOn ? + 30 : 0; + config_->rc_end_usage = VPX_CBR; + config_->g_pass = VPX_RC_ONE_PASS; + config_->rc_min_quantizer = 2; + config_->rc_max_quantizer = 56; + config_->rc_undershoot_pct = 50; + config_->rc_overshoot_pct = 50; + config_->rc_buf_initial_sz = 500; + config_->rc_buf_optimal_sz = 600; + config_->rc_buf_sz = 1000; + // Set the maximum target size of any key-frame. + rc_max_intra_target_ = MaxIntraTarget(config_->rc_buf_optimal_sz); + if (inst->codecSpecific.VP9.keyFrameInterval > 0) { + config_->kf_mode = VPX_KF_AUTO; + config_->kf_max_dist = inst->codecSpecific.VP9.keyFrameInterval; + } else { + config_->kf_mode = VPX_KF_DISABLED; + } + return InitAndSetControlSettings(inst); +} + +int VP9EncoderImpl::InitAndSetControlSettings(const VideoCodec* inst) { + if (vpx_codec_enc_init(encoder_, vpx_codec_vp9_cx(), config_, 0)) { + return WEBRTC_VIDEO_CODEC_UNINITIALIZED; + } + // Only positive speeds, currently: 0 - 7. + // O means slowest/best quality, 7 means fastest/lowest quality. + // TODO(marpan): Speeds 5-7 are speed settings for real-time mode, on desktop. + // Currently set to 5, update to 6 (for faster encoding) after some subjective + // quality tests. + cpu_speed_ = 5; + // Note: some of these codec controls still use "VP8" in the control name. + // TODO(marpan): Update this in the next/future libvpx version. + vpx_codec_control(encoder_, VP8E_SET_CPUUSED, cpu_speed_); + vpx_codec_control(encoder_, VP8E_SET_MAX_INTRA_BITRATE_PCT, + rc_max_intra_target_); + vpx_codec_control(encoder_, VP9E_SET_AQ_MODE, + inst->codecSpecific.VP9.adaptiveQpMode ? 3 : 0); + // TODO(marpan): Enable in future libvpx roll: waiting for SSE2 optimization. +// #if !defined(WEBRTC_ARCH_ARM) + // vpx_codec_control(encoder_, VP9E_SET_NOISE_SENSITIVITY, + // inst->codecSpecific.VP9.denoisingOn ? 1 : 0); +// #endif + inited_ = true; + return WEBRTC_VIDEO_CODEC_OK; +} + +uint32_t VP9EncoderImpl::MaxIntraTarget(uint32_t optimal_buffer_size) { + // Set max to the optimal buffer level (normalized by target BR), + // and scaled by a scale_par. + // Max target size = scale_par * optimal_buffer_size * targetBR[Kbps]. + // This value is presented in percentage of perFrameBw: + // perFrameBw = targetBR[Kbps] * 1000 / framerate. + // The target in % is as follows: + float scale_par = 0.5; + uint32_t target_pct = + optimal_buffer_size * scale_par * codec_.maxFramerate / 10; + // Don't go below 3 times the per frame bandwidth. + const uint32_t min_intra_size = 300; + return (target_pct < min_intra_size) ? min_intra_size: target_pct; +} + +int VP9EncoderImpl::Encode(const I420VideoFrame& input_image, + const CodecSpecificInfo* codec_specific_info, + const std::vector<VideoFrameType>* frame_types) { + if (!inited_) { + return WEBRTC_VIDEO_CODEC_UNINITIALIZED; + } + if (input_image.IsZeroSize()) { + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + if (encoded_complete_callback_ == NULL) { + return WEBRTC_VIDEO_CODEC_UNINITIALIZED; + } + VideoFrameType frame_type = kDeltaFrame; + // We only support one stream at the moment. + if (frame_types && frame_types->size() > 0) { + frame_type = (*frame_types)[0]; + } + // Image in vpx_image_t format. + // Input image is const. VPX's raw image is not defined as const. + raw_->planes[VPX_PLANE_Y] = const_cast<uint8_t*>(input_image.buffer(kYPlane)); + raw_->planes[VPX_PLANE_U] = const_cast<uint8_t*>(input_image.buffer(kUPlane)); + raw_->planes[VPX_PLANE_V] = const_cast<uint8_t*>(input_image.buffer(kVPlane)); + raw_->stride[VPX_PLANE_Y] = input_image.stride(kYPlane); + raw_->stride[VPX_PLANE_U] = input_image.stride(kUPlane); + raw_->stride[VPX_PLANE_V] = input_image.stride(kVPlane); + + int flags = 0; + bool send_keyframe = (frame_type == kKeyFrame); + if (send_keyframe) { + // Key frame request from caller. + flags = VPX_EFLAG_FORCE_KF; + } + assert(codec_.maxFramerate > 0); + uint32_t duration = 90000 / codec_.maxFramerate; + if (vpx_codec_encode(encoder_, raw_, timestamp_, duration, flags, + VPX_DL_REALTIME)) { + return WEBRTC_VIDEO_CODEC_ERROR; + } + timestamp_ += duration; + return GetEncodedPartitions(input_image); +} + +void VP9EncoderImpl::PopulateCodecSpecific(CodecSpecificInfo* codec_specific, + const vpx_codec_cx_pkt& pkt, + uint32_t timestamp) { + assert(codec_specific != NULL); + codec_specific->codecType = kVideoCodecVP9; + CodecSpecificInfoVP9 *vp9_info = &(codec_specific->codecSpecific.VP9); + vp9_info->pictureId = picture_id_; + vp9_info->keyIdx = kNoKeyIdx; + vp9_info->nonReference = (pkt.data.frame.flags & VPX_FRAME_IS_DROPPABLE) != 0; + // TODO(marpan): Temporal layers are supported in the current VP9 version, + // but for now use 1 temporal layer encoding. Will update this when temporal + // layer support for VP9 is added in webrtc. + vp9_info->temporalIdx = kNoTemporalIdx; + vp9_info->layerSync = false; + vp9_info->tl0PicIdx = kNoTl0PicIdx; + picture_id_ = (picture_id_ + 1) & 0x7FFF; +} + +int VP9EncoderImpl::GetEncodedPartitions(const I420VideoFrame& input_image) { + vpx_codec_iter_t iter = NULL; + encoded_image_._length = 0; + encoded_image_._frameType = kDeltaFrame; + RTPFragmentationHeader frag_info; + // Note: no data partitioning in VP9, so 1 partition only. We keep this + // fragmentation data for now, until VP9 packetizer is implemented. + frag_info.VerifyAndAllocateFragmentationHeader(1); + int part_idx = 0; + CodecSpecificInfo codec_specific; + const vpx_codec_cx_pkt_t *pkt = NULL; + while ((pkt = vpx_codec_get_cx_data(encoder_, &iter)) != NULL) { + switch (pkt->kind) { + case VPX_CODEC_CX_FRAME_PKT: { + memcpy(&encoded_image_._buffer[encoded_image_._length], + pkt->data.frame.buf, + pkt->data.frame.sz); + frag_info.fragmentationOffset[part_idx] = encoded_image_._length; + frag_info.fragmentationLength[part_idx] = + static_cast<uint32_t>(pkt->data.frame.sz); + frag_info.fragmentationPlType[part_idx] = 0; + frag_info.fragmentationTimeDiff[part_idx] = 0; + encoded_image_._length += static_cast<uint32_t>(pkt->data.frame.sz); + assert(encoded_image_._length <= encoded_image_._size); + break; + } + default: { + break; + } + } + // End of frame. + if ((pkt->data.frame.flags & VPX_FRAME_IS_FRAGMENT) == 0) { + // Check if encoded frame is a key frame. + if (pkt->data.frame.flags & VPX_FRAME_IS_KEY) { + encoded_image_._frameType = kKeyFrame; + } + PopulateCodecSpecific(&codec_specific, *pkt, input_image.timestamp()); + break; + } + } + if (encoded_image_._length > 0) { + TRACE_COUNTER1("webrtc", "EncodedFrameSize", encoded_image_._length); + encoded_image_._timeStamp = input_image.timestamp(); + encoded_image_.capture_time_ms_ = input_image.render_time_ms(); + encoded_image_._encodedHeight = raw_->d_h; + encoded_image_._encodedWidth = raw_->d_w; + encoded_complete_callback_->Encoded(encoded_image_, &codec_specific, + &frag_info); + } + return WEBRTC_VIDEO_CODEC_OK; +} + +int VP9EncoderImpl::SetChannelParameters(uint32_t packet_loss, int rtt) { + return WEBRTC_VIDEO_CODEC_OK; +} + +int VP9EncoderImpl::RegisterEncodeCompleteCallback( + EncodedImageCallback* callback) { + encoded_complete_callback_ = callback; + return WEBRTC_VIDEO_CODEC_OK; +} + +VP9Decoder* VP9Decoder::Create() { + return new VP9DecoderImpl(); +} + +VP9DecoderImpl::VP9DecoderImpl() + : decode_complete_callback_(NULL), + inited_(false), + decoder_(NULL), + key_frame_required_(true) { + memset(&codec_, 0, sizeof(codec_)); +} + +VP9DecoderImpl::~VP9DecoderImpl() { + inited_ = true; // in order to do the actual release + Release(); +} + +int VP9DecoderImpl::Reset() { + if (!inited_) { + return WEBRTC_VIDEO_CODEC_UNINITIALIZED; + } + InitDecode(&codec_, 1); + return WEBRTC_VIDEO_CODEC_OK; +} + +int VP9DecoderImpl::InitDecode(const VideoCodec* inst, int number_of_cores) { + if (inst == NULL) { + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + int ret_val = Release(); + if (ret_val < 0) { + return ret_val; + } + if (decoder_ == NULL) { + decoder_ = new vpx_dec_ctx_t; + } + vpx_codec_dec_cfg_t cfg; + // Setting number of threads to a constant value (1) + cfg.threads = 1; + cfg.h = cfg.w = 0; // set after decode + vpx_codec_flags_t flags = 0; + if (vpx_codec_dec_init(decoder_, vpx_codec_vp9_dx(), &cfg, flags)) { + return WEBRTC_VIDEO_CODEC_MEMORY; + } + if (&codec_ != inst) { + // Save VideoCodec instance for later; mainly for duplicating the decoder. + codec_ = *inst; + } + inited_ = true; + // Always start with a complete key frame. + key_frame_required_ = true; + return WEBRTC_VIDEO_CODEC_OK; +} + +int VP9DecoderImpl::Decode(const EncodedImage& input_image, + bool missing_frames, + const RTPFragmentationHeader* fragmentation, + const CodecSpecificInfo* codec_specific_info, + int64_t /*render_time_ms*/) { + if (!inited_) { + return WEBRTC_VIDEO_CODEC_UNINITIALIZED; + } + if (decode_complete_callback_ == NULL) { + return WEBRTC_VIDEO_CODEC_UNINITIALIZED; + } + // Always start with a complete key frame. + if (key_frame_required_) { + if (input_image._frameType != kKeyFrame) + return WEBRTC_VIDEO_CODEC_ERROR; + // We have a key frame - is it complete? + if (input_image._completeFrame) { + key_frame_required_ = false; + } else { + return WEBRTC_VIDEO_CODEC_ERROR; + } + } + vpx_codec_iter_t iter = NULL; + vpx_image_t* img; + uint8_t* buffer = input_image._buffer; + if (input_image._length == 0) { + buffer = NULL; // Triggers full frame concealment. + } + if (vpx_codec_decode(decoder_, + buffer, + input_image._length, + 0, + VPX_DL_REALTIME)) { + return WEBRTC_VIDEO_CODEC_ERROR; + } + img = vpx_codec_get_frame(decoder_, &iter); + int ret = ReturnFrame(img, input_image._timeStamp); + if (ret != 0) { + return ret; + } + return WEBRTC_VIDEO_CODEC_OK; +} + +int VP9DecoderImpl::ReturnFrame(const vpx_image_t* img, uint32_t timestamp) { + if (img == NULL) { + // Decoder OK and NULL image => No show frame. + return WEBRTC_VIDEO_CODEC_NO_OUTPUT; + } + int half_height = (img->d_h + 1) / 2; + int size_y = img->stride[VPX_PLANE_Y] * img->d_h; + int size_u = img->stride[VPX_PLANE_U] * half_height; + int size_v = img->stride[VPX_PLANE_V] * half_height; + decoded_image_.CreateFrame(size_y, img->planes[VPX_PLANE_Y], + size_u, img->planes[VPX_PLANE_U], + size_v, img->planes[VPX_PLANE_V], + img->d_w, img->d_h, + img->stride[VPX_PLANE_Y], + img->stride[VPX_PLANE_U], + img->stride[VPX_PLANE_V]); + decoded_image_.set_timestamp(timestamp); + int ret = decode_complete_callback_->Decoded(decoded_image_); + if (ret != 0) + return ret; + return WEBRTC_VIDEO_CODEC_OK; +} + +int VP9DecoderImpl::RegisterDecodeCompleteCallback( + DecodedImageCallback* callback) { + decode_complete_callback_ = callback; + return WEBRTC_VIDEO_CODEC_OK; +} + +int VP9DecoderImpl::Release() { + if (decoder_ != NULL) { + if (vpx_codec_destroy(decoder_)) { + return WEBRTC_VIDEO_CODEC_MEMORY; + } + delete decoder_; + decoder_ = NULL; + } + inited_ = false; + return WEBRTC_VIDEO_CODEC_OK; +} +} // namespace webrtc diff --git a/modules/video_coding/codecs/vp9/vp9_impl.h b/modules/video_coding/codecs/vp9/vp9_impl.h new file mode 100644 index 00000000..94788db5 --- /dev/null +++ b/modules/video_coding/codecs/vp9/vp9_impl.h @@ -0,0 +1,115 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + * + */ + +#ifndef WEBRTC_MODULES_VIDEO_CODING_CODECS_VP9_IMPL_H_ +#define WEBRTC_MODULES_VIDEO_CODING_CODECS_VP9_IMPL_H_ + +#include "webrtc/modules/video_coding/codecs/vp9/include/vp9.h" + +// VPX forward declaration +typedef struct vpx_codec_ctx vpx_codec_ctx_t; +typedef struct vpx_codec_ctx vpx_dec_ctx_t; +typedef struct vpx_codec_enc_cfg vpx_codec_enc_cfg_t; +typedef struct vpx_image vpx_image_t; +typedef struct vpx_ref_frame vpx_ref_frame_t; +struct vpx_codec_cx_pkt; + +namespace webrtc { + +class VP9EncoderImpl : public VP9Encoder { + public: + VP9EncoderImpl(); + + virtual ~VP9EncoderImpl(); + + virtual int Release() OVERRIDE; + + virtual int InitEncode(const VideoCodec* codec_settings, + int number_of_cores, + uint32_t max_payload_size) OVERRIDE; + + virtual int Encode(const I420VideoFrame& input_image, + const CodecSpecificInfo* codec_specific_info, + const std::vector<VideoFrameType>* frame_types) OVERRIDE; + + virtual int RegisterEncodeCompleteCallback(EncodedImageCallback* callback) + OVERRIDE; + + virtual int SetChannelParameters(uint32_t packet_loss, int rtt) OVERRIDE; + + virtual int SetRates(uint32_t new_bitrate_kbit, uint32_t frame_rate) OVERRIDE; + + private: + // Call encoder initialize function and set control settings. + int InitAndSetControlSettings(const VideoCodec* inst); + + void PopulateCodecSpecific(CodecSpecificInfo* codec_specific, + const vpx_codec_cx_pkt& pkt, + uint32_t timestamp); + + int GetEncodedPartitions(const I420VideoFrame& input_image); + + // Determine maximum target for Intra frames + // + // Input: + // - optimal_buffer_size : Optimal buffer size + // Return Value : Max target size for Intra frames represented as + // percentage of the per frame bandwidth + uint32_t MaxIntraTarget(uint32_t optimal_buffer_size); + + EncodedImage encoded_image_; + EncodedImageCallback* encoded_complete_callback_; + VideoCodec codec_; + bool inited_; + int64_t timestamp_; + uint16_t picture_id_; + int cpu_speed_; + uint32_t rc_max_intra_target_; + vpx_codec_ctx_t* encoder_; + vpx_codec_enc_cfg_t* config_; + vpx_image_t* raw_; +}; + + +class VP9DecoderImpl : public VP9Decoder { + public: + VP9DecoderImpl(); + + virtual ~VP9DecoderImpl(); + + virtual int InitDecode(const VideoCodec* inst, int number_of_cores) OVERRIDE; + + virtual int Decode(const EncodedImage& input_image, + bool missing_frames, + const RTPFragmentationHeader* fragmentation, + const CodecSpecificInfo* codec_specific_info, + int64_t /*render_time_ms*/) OVERRIDE; + + virtual int RegisterDecodeCompleteCallback(DecodedImageCallback* callback) + OVERRIDE; + + virtual int Release() OVERRIDE; + + virtual int Reset() OVERRIDE; + + private: + int ReturnFrame(const vpx_image_t* img, uint32_t timeStamp); + + I420VideoFrame decoded_image_; + DecodedImageCallback* decode_complete_callback_; + bool inited_; + vpx_dec_ctx_t* decoder_; + VideoCodec codec_; + bool key_frame_required_; +}; +} // namespace webrtc + +#endif // WEBRTC_MODULES_VIDEO_CODING_CODECS_VP9_IMPL_H_ diff --git a/modules/video_coding/main/interface/video_coding_defines.h b/modules/video_coding/main/interface/video_coding_defines.h index 0919c892..efdc41b4 100644 --- a/modules/video_coding/main/interface/video_coding_defines.h +++ b/modules/video_coding/main/interface/video_coding_defines.h @@ -39,6 +39,7 @@ namespace webrtc { #define VCM_RED_PAYLOAD_TYPE 96 #define VCM_ULPFEC_PAYLOAD_TYPE 97 #define VCM_VP8_PAYLOAD_TYPE 100 +#define VCM_VP9_PAYLOAD_TYPE 101 #define VCM_I420_PAYLOAD_TYPE 124 #define VCM_H264_PAYLOAD_TYPE 127 diff --git a/modules/video_coding/main/source/codec_database.cc b/modules/video_coding/main/source/codec_database.cc index e99cc528..2fc92461 100644 --- a/modules/video_coding/main/source/codec_database.cc +++ b/modules/video_coding/main/source/codec_database.cc @@ -19,6 +19,9 @@ #ifdef VIDEOCODEC_VP8 #include "webrtc/modules/video_coding/codecs/vp8/include/vp8.h" #endif +#ifdef VIDEOCODEC_VP9 +#include "webrtc/modules/video_coding/codecs/vp9/include/vp9.h" +#endif #include "webrtc/modules/video_coding/main/source/internal_defines.h" #include "webrtc/system_wrappers/interface/logging.h" @@ -39,6 +42,20 @@ VideoCodecVP8 VideoEncoder::GetDefaultVp8Settings() { return vp8_settings; } +VideoCodecVP9 VideoEncoder::GetDefaultVp9Settings() { + VideoCodecVP9 vp9_settings; + memset(&vp9_settings, 0, sizeof(vp9_settings)); + + vp9_settings.resilience = 1; + vp9_settings.numberOfTemporalLayers = 1; + vp9_settings.denoisingOn = false; + vp9_settings.frameDroppingOn = true; + vp9_settings.keyFrameInterval = 3000; + vp9_settings.adaptiveQpMode = true; + + return vp9_settings; +} + VideoCodecH264 VideoEncoder::GetDefaultH264Settings() { VideoCodecH264 h264_settings; memset(&h264_settings, 0, sizeof(h264_settings)); @@ -126,6 +143,24 @@ bool VCMCodecDataBase::Codec(int list_id, return true; } #endif +#ifdef VIDEOCODEC_VP9 + case VCM_VP9_IDX: { + strncpy(settings->plName, "VP9", 4); + settings->codecType = kVideoCodecVP9; + // 96 to 127 dynamic payload types for video codecs. + settings->plType = VCM_VP9_PAYLOAD_TYPE; + settings->startBitrate = 100; + settings->minBitrate = VCM_MIN_BITRATE; + settings->maxBitrate = 0; + settings->maxFramerate = VCM_DEFAULT_FRAME_RATE; + settings->width = VCM_DEFAULT_CODEC_WIDTH; + settings->height = VCM_DEFAULT_CODEC_HEIGHT; + settings->numberOfSimulcastStreams = 0; + settings->qpMax = 56; + settings->codecSpecific.VP9 = VideoEncoder::GetDefaultVp9Settings(); + return true; + } +#endif #ifdef VIDEOCODEC_H264 case VCM_H264_IDX: { strncpy(settings->plName, "H264", 5); @@ -362,6 +397,13 @@ bool VCMCodecDataBase::RequiresEncoderReset(const VideoCodec& new_send_codec) { return true; } break; + case kVideoCodecVP9: + if (memcmp(&new_send_codec.codecSpecific.VP9, + &send_codec_.codecSpecific.VP9, + sizeof(new_send_codec.codecSpecific.VP9)) != 0) { + return true; + } + break; case kVideoCodecH264: if (memcmp(&new_send_codec.codecSpecific.H264, &send_codec_.codecSpecific.H264, @@ -635,6 +677,10 @@ VCMGenericEncoder* VCMCodecDataBase::CreateEncoder( case kVideoCodecVP8: return new VCMGenericEncoder(*(VP8Encoder::Create())); #endif +#ifdef VIDEOCODEC_VP9 + case kVideoCodecVP9: + return new VCMGenericEncoder(*(VP9Encoder::Create())); +#endif #ifdef VIDEOCODEC_I420 case kVideoCodecI420: return new VCMGenericEncoder(*(new I420Encoder)); @@ -662,6 +708,10 @@ VCMGenericDecoder* VCMCodecDataBase::CreateDecoder(VideoCodecType type) const { case kVideoCodecVP8: return new VCMGenericDecoder(*(VP8Decoder::Create())); #endif +#ifdef VIDEOCODEC_VP9 + case kVideoCodecVP9: + return new VCMGenericDecoder(*(VP9Decoder::Create())); +#endif #ifdef VIDEOCODEC_I420 case kVideoCodecI420: return new VCMGenericDecoder(*(new I420Decoder)); diff --git a/modules/video_coding/main/source/internal_defines.h b/modules/video_coding/main/source/internal_defines.h index ef42c628..adc940f2 100644 --- a/modules/video_coding/main/source/internal_defines.h +++ b/modules/video_coding/main/source/internal_defines.h @@ -39,10 +39,15 @@ inline uint32_t MaskWord64ToUWord32(int64_t w64) #else #define VCM_VP8_IDX VCM_NO_CODEC_IDX #endif +#ifdef VIDEOCODEC_VP9 + #define VCM_VP9_IDX (VCM_VP8_IDX + 1) +#else + #define VCM_VP9_IDX VCM_VP8_IDX +#endif #ifdef VIDEOCODEC_H264 - #define VCM_H264_IDX (VCM_VP8_IDX + 1) + #define VCM_H264_IDX (VCM_VP9_IDX + 1) #else - #define VCM_H264_IDX VCM_VP8_IDX + #define VCM_H264_IDX VCM_VP9_IDX #endif #ifdef VIDEOCODEC_I420 #define VCM_I420_IDX (VCM_H264_IDX + 1) diff --git a/modules/video_coding/main/source/jitter_buffer.cc b/modules/video_coding/main/source/jitter_buffer.cc index d09fccd9..e3a4a8a7 100644 --- a/modules/video_coding/main/source/jitter_buffer.cc +++ b/modules/video_coding/main/source/jitter_buffer.cc @@ -25,6 +25,7 @@ #include "webrtc/system_wrappers/interface/critical_section_wrapper.h" #include "webrtc/system_wrappers/interface/event_wrapper.h" #include "webrtc/system_wrappers/interface/logging.h" +#include "webrtc/system_wrappers/interface/metrics.h" #include "webrtc/system_wrappers/interface/trace_event.h" namespace webrtc { @@ -143,6 +144,8 @@ VCMJitterBuffer::VCMJitterBuffer(Clock* clock, EventFactory* event_factory) drop_count_(0), num_consecutive_old_frames_(0), num_consecutive_old_packets_(0), + num_packets_(0), + num_duplicated_packets_(0), num_discarded_packets_(0), jitter_estimate_(clock), inter_frame_delay_(clock_->TimeInMilliseconds()), @@ -190,6 +193,8 @@ void VCMJitterBuffer::CopyFrom(const VCMJitterBuffer& rhs) { drop_count_ = rhs.drop_count_; num_consecutive_old_frames_ = rhs.num_consecutive_old_frames_; num_consecutive_old_packets_ = rhs.num_consecutive_old_packets_; + num_packets_ = rhs.num_packets_; + num_duplicated_packets_ = rhs.num_duplicated_packets_; num_discarded_packets_ = rhs.num_discarded_packets_; jitter_estimate_ = rhs.jitter_estimate_; inter_frame_delay_ = rhs.inter_frame_delay_; @@ -238,6 +243,15 @@ void VCMJitterBuffer::CopyFrames(FrameList* to_list, } } +void VCMJitterBuffer::UpdateHistograms() { + if (num_packets_ > 0) { + RTC_HISTOGRAM_PERCENTAGE("WebRTC.Video.DiscardedPacketsInPercent", + num_discarded_packets_ * 100 / num_packets_); + RTC_HISTOGRAM_PERCENTAGE("WebRTC.Video.DuplicatedPacketsInPercent", + num_duplicated_packets_ * 100 / num_packets_); + } +} + void VCMJitterBuffer::Start() { CriticalSectionScoped cs(crit_sect_); running_ = true; @@ -250,6 +264,8 @@ void VCMJitterBuffer::Start() { num_consecutive_old_frames_ = 0; num_consecutive_old_packets_ = 0; + num_packets_ = 0; + num_duplicated_packets_ = 0; num_discarded_packets_ = 0; // Start in a non-signaled state. @@ -265,6 +281,7 @@ void VCMJitterBuffer::Start() { void VCMJitterBuffer::Stop() { crit_sect_->Enter(); + UpdateHistograms(); running_ = false; last_decoded_state_.Reset(); free_frames_.clear(); @@ -313,6 +330,16 @@ std::map<FrameType, uint32_t> VCMJitterBuffer::FrameStatistics() const { return receive_statistics_; } +int VCMJitterBuffer::num_packets() const { + CriticalSectionScoped cs(crit_sect_); + return num_packets_; +} + +int VCMJitterBuffer::num_duplicated_packets() const { + CriticalSectionScoped cs(crit_sect_); + return num_duplicated_packets_; +} + int VCMJitterBuffer::num_discarded_packets() const { CriticalSectionScoped cs(crit_sect_); return num_discarded_packets_; @@ -543,6 +570,7 @@ void VCMJitterBuffer::ReleaseFrame(VCMEncodedFrame* frame) { // Gets frame to use for this timestamp. If no match, get empty frame. VCMFrameBufferEnum VCMJitterBuffer::GetFrame(const VCMPacket& packet, VCMFrameBuffer** frame) { + ++num_packets_; // Does this packet belong to an old frame? if (last_decoded_state_.IsOldPacket(&packet)) { // Account only for media packets. @@ -747,6 +775,7 @@ VCMFrameBufferEnum VCMJitterBuffer::InsertPacket(const VCMPacket& packet, case kNoError: case kOutOfBoundsPacket: case kDuplicatePacket: { + ++num_duplicated_packets_; break; } case kFlushIndicator: diff --git a/modules/video_coding/main/source/jitter_buffer.h b/modules/video_coding/main/source/jitter_buffer.h index 6ed9cfb8..182b80b4 100644 --- a/modules/video_coding/main/source/jitter_buffer.h +++ b/modules/video_coding/main/source/jitter_buffer.h @@ -103,6 +103,12 @@ class VCMJitterBuffer { // won't be able to decode them. int num_not_decodable_packets() const; + // Gets number of packets received. + int num_packets() const; + + // Gets number of duplicated packets received. + int num_duplicated_packets() const; + // Gets number of packets discarded by the jitter buffer. int num_discarded_packets() const; @@ -271,6 +277,8 @@ class VCMJitterBuffer { uint16_t EstimatedLowSequenceNumber(const VCMFrameBuffer& frame) const; + void UpdateHistograms(); + Clock* clock_; // If we are running (have started) or not. bool running_; @@ -303,6 +311,10 @@ class VCMJitterBuffer { int num_consecutive_old_frames_; // Number of packets in a row that have been too old. int num_consecutive_old_packets_; + // Number of packets received. + int num_packets_; + // Number of duplicated packets received. + int num_duplicated_packets_; // Number of packets discarded by the jitter buffer. int num_discarded_packets_; diff --git a/modules/video_coding/main/source/jitter_buffer_unittest.cc b/modules/video_coding/main/source/jitter_buffer_unittest.cc index 0490658b..899a3eec 100644 --- a/modules/video_coding/main/source/jitter_buffer_unittest.cc +++ b/modules/video_coding/main/source/jitter_buffer_unittest.cc @@ -512,6 +512,8 @@ TEST_F(TestBasicJitterBuffer, DuplicatePackets) { packet_->markerBit = false; packet_->seqNum = seq_num_; packet_->timestamp = timestamp_; + EXPECT_EQ(0, jitter_buffer_->num_packets()); + EXPECT_EQ(0, jitter_buffer_->num_duplicated_packets()); bool retransmitted = false; EXPECT_EQ(kIncomplete, jitter_buffer_->InsertPacket(*packet_, @@ -520,6 +522,8 @@ TEST_F(TestBasicJitterBuffer, DuplicatePackets) { VCMEncodedFrame* frame_out = DecodeCompleteFrame(); EXPECT_TRUE(frame_out == NULL); + EXPECT_EQ(1, jitter_buffer_->num_packets()); + EXPECT_EQ(0, jitter_buffer_->num_duplicated_packets()); packet_->isFirstPacket = false; packet_->markerBit = true; @@ -527,6 +531,8 @@ TEST_F(TestBasicJitterBuffer, DuplicatePackets) { // Insert a packet into a frame. EXPECT_EQ(kDuplicatePacket, jitter_buffer_->InsertPacket(*packet_, &retransmitted)); + EXPECT_EQ(2, jitter_buffer_->num_packets()); + EXPECT_EQ(1, jitter_buffer_->num_duplicated_packets()); seq_num_++; packet_->seqNum = seq_num_; @@ -539,6 +545,8 @@ TEST_F(TestBasicJitterBuffer, DuplicatePackets) { CheckOutFrame(frame_out, 2 * size_, false); EXPECT_EQ(kVideoFrameKey, frame_out->FrameType()); + EXPECT_EQ(3, jitter_buffer_->num_packets()); + EXPECT_EQ(1, jitter_buffer_->num_duplicated_packets()); } TEST_F(TestBasicJitterBuffer, H264InsertStartCode) { diff --git a/modules/video_coding/main/source/video_coding.gypi b/modules/video_coding/main/source/video_coding.gypi index f19a5855..02c5a5c0 100644 --- a/modules/video_coding/main/source/video_coding.gypi +++ b/modules/video_coding/main/source/video_coding.gypi @@ -17,6 +17,7 @@ '<(webrtc_root)/modules/video_coding/utility/video_coding_utility.gyp:video_coding_utility', '<(webrtc_root)/system_wrappers/source/system_wrappers.gyp:system_wrappers', '<(webrtc_vp8_dir)/vp8.gyp:webrtc_vp8', + '<(webrtc_vp9_dir)/vp9.gyp:webrtc_vp9', ], 'sources': [ # interfaces diff --git a/modules/video_coding/main/source/video_coding_impl.h b/modules/video_coding/main/source/video_coding_impl.h index 5b3fe2eb..ac7a1f4b 100644 --- a/modules/video_coding/main/source/video_coding_impl.h +++ b/modules/video_coding/main/source/video_coding_impl.h @@ -186,7 +186,8 @@ class VideoReceiver { void RegisterPreDecodeImageCallback(EncodedImageCallback* observer); protected: - int32_t Decode(const webrtc::VCMEncodedFrame& frame); + int32_t Decode(const webrtc::VCMEncodedFrame& frame) + EXCLUSIVE_LOCKS_REQUIRED(_receiveCritSect); int32_t RequestKeyFrame(); int32_t RequestSliceLossIndication(const uint64_t pictureID) const; int32_t NackList(uint16_t* nackList, uint16_t* size); @@ -230,7 +231,7 @@ class VideoReceiver { size_t max_nack_list_size_ GUARDED_BY(process_crit_sect_); EncodedImageCallback* pre_decode_image_callback_ GUARDED_BY(_receiveCritSect); - VCMCodecDataBase _codecDataBase; + VCMCodecDataBase _codecDataBase GUARDED_BY(_receiveCritSect); VCMProcessTimer _receiveStatsTimer; VCMProcessTimer _retransmissionTimer; VCMProcessTimer _keyRequestTimer; diff --git a/modules/video_coding/main/source/video_receiver.cc b/modules/video_coding/main/source/video_receiver.cc index 0b561249..a8de28bb 100644 --- a/modules/video_coding/main/source/video_receiver.cc +++ b/modules/video_coding/main/source/video_receiver.cc @@ -280,11 +280,11 @@ int32_t VideoReceiver::InitializeReceiver() { if (ret < 0) { return ret; } - _codecDataBase.ResetReceiver(); - _timing.Reset(); { CriticalSectionScoped receive_cs(_receiveCritSect); + _codecDataBase.ResetReceiver(); + _timing.Reset(); _receiverInited = true; } @@ -369,6 +369,7 @@ int VideoReceiver::RegisterRenderBufferSizeCallback( // Should be called as often as possible to get the most out of the decoder. int32_t VideoReceiver::Decode(uint16_t maxWaitTimeMs) { int64_t nextRenderTimeMs; + bool supports_render_scheduling; { CriticalSectionScoped cs(_receiveCritSect); if (!_receiverInited) { @@ -377,6 +378,7 @@ int32_t VideoReceiver::Decode(uint16_t maxWaitTimeMs) { if (!_codecDataBase.DecoderRegistered()) { return VCM_NO_CODEC_REGISTERED; } + supports_render_scheduling = _codecDataBase.SupportsRenderScheduling(); } const bool dualReceiverEnabledNotReceiving = ( @@ -385,7 +387,7 @@ int32_t VideoReceiver::Decode(uint16_t maxWaitTimeMs) { VCMEncodedFrame* frame = _receiver.FrameForDecoding(maxWaitTimeMs, nextRenderTimeMs, - _codecDataBase.SupportsRenderScheduling(), + supports_render_scheduling, &_dualReceiver); if (dualReceiverEnabledNotReceiving && _dualReceiver.State() == kReceiving) { diff --git a/modules/video_coding/main/test/normal_test.cc b/modules/video_coding/main/test/normal_test.cc index f23682bd..815c3acc 100644 --- a/modules/video_coding/main/test/normal_test.cc +++ b/modules/video_coding/main/test/normal_test.cc @@ -101,6 +101,9 @@ VCMNTEncodeCompleteCallback::SendData( rtpInfo.type.Video.codecHeader.VP8.pictureId = videoHdr->codecHeader.VP8.pictureId; break; + case kVideoCodecVP9: + // Leave for now, until we add kRtpVideoVp9 to RTP. + break; default: assert(false); return -1; diff --git a/modules/video_coding/main/test/test_callbacks.cc b/modules/video_coding/main/test/test_callbacks.cc index 710a06ea..d68f9949 100644 --- a/modules/video_coding/main/test/test_callbacks.cc +++ b/modules/video_coding/main/test/test_callbacks.cc @@ -82,6 +82,9 @@ VCMEncodeCompleteCallback::SendData( rtpInfo.type.Video.codecHeader.VP8.pictureId = videoHdr->codecHeader.VP8.pictureId; break; + case webrtc::kRtpVideoGeneric: + // Leave for now, until we add kRtpVideoVp9 to RTP. + break; default: assert(false); return -1; diff --git a/modules/video_coding/main/test/test_util.cc b/modules/video_coding/main/test/test_util.cc index 09ad9916..d2b8f8c7 100644 --- a/modules/video_coding/main/test/test_util.cc +++ b/modules/video_coding/main/test/test_util.cc @@ -151,6 +151,7 @@ webrtc::RtpVideoCodecTypes ConvertCodecType(const char* plname) { if (strncmp(plname,"VP8" , 3) == 0) { return webrtc::kRtpVideoVp8; } else { - return webrtc::kRtpVideoNone; // Default value + // Default value. + return webrtc::kRtpVideoGeneric; } } diff --git a/modules/video_coding/main/test/tester_main.cc b/modules/video_coding/main/test/tester_main.cc index bf17ab27..874fa9e7 100644 --- a/modules/video_coding/main/test/tester_main.cc +++ b/modules/video_coding/main/test/tester_main.cc @@ -63,6 +63,8 @@ int ParseArguments(CmdArgs& args) { args.codecName = FLAGS_codec; if (args.codecName == "VP8") { args.codecType = kVideoCodecVP8; + } else if (args.codecName == "VP9") { + args.codecType = kVideoCodecVP9; } else if (args.codecName == "I420") { args.codecType = kVideoCodecI420; } else { diff --git a/modules/video_render/video_render.gypi b/modules/video_render/video_render.gypi index 5db851d9..38a77985 100644 --- a/modules/video_render/video_render.gypi +++ b/modules/video_render/video_render.gypi @@ -205,7 +205,6 @@ ], 'includes': [ '../../build/isolate.gypi', - 'video_render_tests.isolate', ], 'sources': [ 'video_render_tests.isolate', diff --git a/modules/video_render/video_render_tests.isolate b/modules/video_render/video_render_tests.isolate index 15c80141..12bfecf9 100644 --- a/modules/video_render/video_render_tests.isolate +++ b/modules/video_render/video_render_tests.isolate @@ -9,7 +9,7 @@ 'conditions': [ ['OS=="android"', { 'variables': { - 'isolate_dependency_untracked': [ + 'files': [ '<(DEPTH)/data/', '<(DEPTH)/resources/', ], @@ -21,13 +21,10 @@ '<(DEPTH)/testing/test_env.py', '<(PRODUCT_DIR)/video_render_tests<(EXECUTABLE_SUFFIX)', ], - 'isolate_dependency_tracked': [ + 'files': [ '<(DEPTH)/testing/test_env.py', '<(PRODUCT_DIR)/video_render_tests<(EXECUTABLE_SUFFIX)', ], - 'isolate_dependency_untracked': [ - '<(DEPTH)/tools/swarming_client/', - ], }, }], ], diff --git a/rtc_unittests.isolate b/rtc_unittests.isolate index 7bcfc854..6dfe38d4 100644 --- a/rtc_unittests.isolate +++ b/rtc_unittests.isolate @@ -12,12 +12,9 @@ 'command': [ '<(PRODUCT_DIR)/rtc_unittests<(EXECUTABLE_SUFFIX)', ], - 'isolate_dependency_tracked': [ + 'files': [ '<(PRODUCT_DIR)/rtc_unittests<(EXECUTABLE_SUFFIX)', ], - 'isolate_dependency_untracked': [ - '<(DEPTH)/tools/swarming_client/', - ], }, }], ], diff --git a/system_wrappers/interface/metrics.h b/system_wrappers/interface/metrics.h index be9564a0..0390140f 100644 --- a/system_wrappers/interface/metrics.h +++ b/system_wrappers/interface/metrics.h @@ -69,12 +69,26 @@ // Also consider changing string to const char* when switching to atomics. // Histogram for counters. +#define RTC_HISTOGRAM_COUNTS_100(name, sample) RTC_HISTOGRAM_COUNTS( \ + name, sample, 1, 100, 50) + +#define RTC_HISTOGRAM_COUNTS_1000(name, sample) RTC_HISTOGRAM_COUNTS( \ + name, sample, 1, 1000, 50) + +#define RTC_HISTOGRAM_COUNTS_10000(name, sample) RTC_HISTOGRAM_COUNTS( \ + name, sample, 1, 10000, 50) + #define RTC_HISTOGRAM_COUNTS(name, sample, min, max, bucket_count) \ RTC_HISTOGRAM_COMMON_BLOCK(name, sample, \ webrtc::metrics::HistogramFactoryGetCounts( \ name, min, max, bucket_count)) +// Histogram for percentage. +#define RTC_HISTOGRAM_PERCENTAGE(name, sample) \ + RTC_HISTOGRAM_ENUMERATION(name, sample, 101) + // Histogram for enumerators. +// |boundary| should be above the max enumerator sample. #define RTC_HISTOGRAM_ENUMERATION(name, sample, boundary) \ RTC_HISTOGRAM_COMMON_BLOCK(name, sample, \ webrtc::metrics::HistogramFactoryGetEnumeration(name, boundary)) diff --git a/system_wrappers/interface/timestamp_extrapolator.h b/system_wrappers/interface/timestamp_extrapolator.h index d067198d..b78cf64b 100644 --- a/system_wrappers/interface/timestamp_extrapolator.h +++ b/system_wrappers/interface/timestamp_extrapolator.h @@ -31,7 +31,7 @@ private: bool DelayChangeDetection(double error); RWLockWrapper* _rwLock; double _w[2]; - double _P[2][2]; + double _pP[2][2]; int64_t _startMs; int64_t _prevMs; uint32_t _firstTimestamp; @@ -48,7 +48,7 @@ private: const double _alarmThreshold; const double _accDrift; const double _accMaxError; - const double _P11; + const double _pP11; }; } // namespace webrtc diff --git a/system_wrappers/source/system_wrappers_tests.gyp b/system_wrappers/source/system_wrappers_tests.gyp index f77b985a..18775920 100644 --- a/system_wrappers/source/system_wrappers_tests.gyp +++ b/system_wrappers/source/system_wrappers_tests.gyp @@ -80,7 +80,6 @@ ], 'includes': [ '../../build/isolate.gypi', - 'system_wrappers_unittests.isolate', ], 'sources': [ 'system_wrappers_unittests.isolate', diff --git a/system_wrappers/source/system_wrappers_unittests.isolate b/system_wrappers/source/system_wrappers_unittests.isolate index f5057710..0a56470c 100644 --- a/system_wrappers/source/system_wrappers_unittests.isolate +++ b/system_wrappers/source/system_wrappers_unittests.isolate @@ -10,7 +10,7 @@ 'conditions': [ ['OS=="android"', { 'variables': { - 'isolate_dependency_untracked': [ + 'files': [ '<(DEPTH)/data/', '<(DEPTH)/resources/', ], @@ -22,13 +22,10 @@ '<(DEPTH)/testing/test_env.py', '<(PRODUCT_DIR)/system_wrappers_unittests<(EXECUTABLE_SUFFIX)', ], - 'isolate_dependency_tracked': [ + 'files': [ '<(DEPTH)/testing/test_env.py', '<(PRODUCT_DIR)/system_wrappers_unittests<(EXECUTABLE_SUFFIX)', ], - 'isolate_dependency_untracked': [ - '<(DEPTH)/tools/swarming_client/', - ], }, }], ], diff --git a/system_wrappers/source/timestamp_extrapolator.cc b/system_wrappers/source/timestamp_extrapolator.cc index afd212b0..f2b70926 100644 --- a/system_wrappers/source/timestamp_extrapolator.cc +++ b/system_wrappers/source/timestamp_extrapolator.cc @@ -30,7 +30,7 @@ TimestampExtrapolator::TimestampExtrapolator(int64_t start_ms) _alarmThreshold(60e3), _accDrift(6600), // in timestamp ticks, i.e. 15 ms _accMaxError(7000), - _P11(1e10) { + _pP11(1e10) { Reset(start_ms); } @@ -47,9 +47,9 @@ void TimestampExtrapolator::Reset(int64_t start_ms) _firstTimestamp = 0; _w[0] = 90.0; _w[1] = 0; - _P[0][0] = 1; - _P[1][1] = _P11; - _P[0][1] = _P[1][0] = 0; + _pP[0][0] = 1; + _pP[1][1] = _pP11; + _pP[0][1] = _pP[1][0] = 0; _firstAfterReset = true; _prevUnwrappedTimestamp = -1; _prevWrapTimestamp = -1; @@ -112,14 +112,14 @@ TimestampExtrapolator::Update(int64_t tMs, uint32_t ts90khz) // A sudden change of average network delay has been detected. // Force the filter to adjust its offset parameter by changing // the offset uncertainty. Don't do this during startup. - _P[1][1] = _P11; + _pP[1][1] = _pP11; } //T = [t(k) 1]'; //that = T'*w; //K = P*T/(lambda + T'*P*T); double K[2]; - K[0] = _P[0][0] * tMs + _P[0][1]; - K[1] = _P[1][0] * tMs + _P[1][1]; + K[0] = _pP[0][0] * tMs + _pP[0][1]; + K[1] = _pP[1][0] * tMs + _pP[1][1]; double TPT = _lambda + tMs * K[0] + K[1]; K[0] /= TPT; K[1] /= TPT; @@ -127,12 +127,16 @@ TimestampExtrapolator::Update(int64_t tMs, uint32_t ts90khz) _w[0] = _w[0] + K[0] * residual; _w[1] = _w[1] + K[1] * residual; //P = 1/lambda*(P - K*T'*P); - double p00 = 1 / _lambda * (_P[0][0] - (K[0] * tMs * _P[0][0] + K[0] * _P[1][0])); - double p01 = 1 / _lambda * (_P[0][1] - (K[0] * tMs * _P[0][1] + K[0] * _P[1][1])); - _P[1][0] = 1 / _lambda * (_P[1][0] - (K[1] * tMs * _P[0][0] + K[1] * _P[1][0])); - _P[1][1] = 1 / _lambda * (_P[1][1] - (K[1] * tMs * _P[0][1] + K[1] * _P[1][1])); - _P[0][0] = p00; - _P[0][1] = p01; + double p00 = 1 / _lambda * + (_pP[0][0] - (K[0] * tMs * _pP[0][0] + K[0] * _pP[1][0])); + double p01 = 1 / _lambda * + (_pP[0][1] - (K[0] * tMs * _pP[0][1] + K[0] * _pP[1][1])); + _pP[1][0] = 1 / _lambda * + (_pP[1][0] - (K[1] * tMs * _pP[0][0] + K[1] * _pP[1][0])); + _pP[1][1] = 1 / _lambda * + (_pP[1][1] - (K[1] * tMs * _pP[0][1] + K[1] * _pP[1][1])); + _pP[0][0] = p00; + _pP[0][1] = p01; _prevUnwrappedTimestamp = unwrapped_ts90khz; if (_packetCount < _startUpFilterDelayInPackets) { diff --git a/test/call_test.cc b/test/call_test.cc index 4e7fa9ac..126c7163 100644 --- a/test/call_test.cc +++ b/test/call_test.cc @@ -99,23 +99,15 @@ void CallTest::CreateSendConfig(size_t num_streams) { void CallTest::CreateMatchingReceiveConfigs() { assert(!send_config_.rtp.ssrcs.empty()); assert(receive_configs_.empty()); - assert(fake_decoders_.empty()); + assert(allocated_decoders_.empty()); VideoReceiveStream::Config config; - VideoCodec codec = - test::CreateDecoderVideoCodec(send_config_.encoder_settings); - config.codecs.push_back(codec); config.rtp.local_ssrc = kReceiverLocalSsrc; - if (send_config_.encoder_settings.encoder == &fake_encoder_) { - config.external_decoders.resize(1); - config.external_decoders[0].payload_type = - send_config_.encoder_settings.payload_type; - } for (size_t i = 0; i < send_config_.rtp.ssrcs.size(); ++i) { - if (send_config_.encoder_settings.encoder == &fake_encoder_) { - FakeDecoder* decoder = new FakeDecoder(); - fake_decoders_.push_back(decoder); - config.external_decoders[0].decoder = decoder; - } + VideoReceiveStream::Decoder decoder = + test::CreateMatchingDecoder(send_config_.encoder_settings); + allocated_decoders_.push_back(decoder.decoder); + config.decoders.clear(); + config.decoders.push_back(decoder); config.rtp.remote_ssrc = send_config_.rtp.ssrcs[i]; receive_configs_.push_back(config); } @@ -150,7 +142,7 @@ void CallTest::DestroyStreams() { for (size_t i = 0; i < receive_streams_.size(); ++i) receiver_call_->DestroyVideoReceiveStream(receive_streams_[i]); receive_streams_.clear(); - fake_decoders_.clear(); + allocated_decoders_.clear(); } const unsigned int CallTest::kDefaultTimeoutMs = 30 * 1000; diff --git a/test/call_test.h b/test/call_test.h index 695fb2ac..b9a091b4 100644 --- a/test/call_test.h +++ b/test/call_test.h @@ -74,7 +74,7 @@ class CallTest : public ::testing::Test { scoped_ptr<test::FrameGeneratorCapturer> frame_generator_capturer_; test::FakeEncoder fake_encoder_; - ScopedVector<test::FakeDecoder> fake_decoders_; + ScopedVector<VideoDecoder> allocated_decoders_; }; class BaseTest : public RtpRtcpObserver { diff --git a/test/encoder_settings.cc b/test/encoder_settings.cc index 0eeb0b9f..db064bb8 100644 --- a/test/encoder_settings.cc +++ b/test/encoder_settings.cc @@ -12,8 +12,8 @@ #include <assert.h> #include <string.h> -#include "webrtc/video_encoder.h" -#include "webrtc/video_engine/vie_defines.h" +#include "webrtc/test/fake_decoder.h" +#include "webrtc/video_decoder.h" namespace webrtc { namespace test { @@ -53,33 +53,17 @@ std::vector<VideoStream> CreateVideoStreams(size_t num_streams) { return stream_settings; } -VideoCodec CreateDecoderVideoCodec( +VideoReceiveStream::Decoder CreateMatchingDecoder( const VideoSendStream::Config::EncoderSettings& encoder_settings) { - VideoCodec codec; - memset(&codec, 0, sizeof(codec)); - - codec.plType = encoder_settings.payload_type; - strcpy(codec.plName, encoder_settings.payload_name.c_str()); + VideoReceiveStream::Decoder decoder; + decoder.payload_type = encoder_settings.payload_type; + decoder.payload_name = encoder_settings.payload_name; if (encoder_settings.payload_name == "VP8") { - codec.codecType = kVideoCodecVP8; - } else if (encoder_settings.payload_name == "H264") { - codec.codecType = kVideoCodecH264; + decoder.decoder = VideoDecoder::Create(VideoDecoder::kVp8); } else { - codec.codecType = kVideoCodecGeneric; - } - - if (codec.codecType == kVideoCodecVP8) { - codec.codecSpecific.VP8 = VideoEncoder::GetDefaultVp8Settings(); - } else if (codec.codecType == kVideoCodecH264) { - codec.codecSpecific.H264 = VideoEncoder::GetDefaultH264Settings(); + decoder.decoder = new FakeDecoder(); } - - codec.width = 320; - codec.height = 180; - codec.startBitrate = codec.minBitrate = codec.maxBitrate = 300; - - return codec; + return decoder; } - } // namespace test } // namespace webrtc diff --git a/test/encoder_settings.h b/test/encoder_settings.h index ea2be977..a44d3661 100644 --- a/test/encoder_settings.h +++ b/test/encoder_settings.h @@ -10,13 +10,14 @@ #ifndef WEBRTC_TEST_ENCODER_SETTINGS_H_ #define WEBRTC_TEST_ENCODER_SETTINGS_H_ +#include "webrtc/video_receive_stream.h" #include "webrtc/video_send_stream.h" namespace webrtc { namespace test { std::vector<VideoStream> CreateVideoStreams(size_t num_streams); -VideoCodec CreateDecoderVideoCodec( +VideoReceiveStream::Decoder CreateMatchingDecoder( const VideoSendStream::Config::EncoderSettings& encoder_settings); } // namespace test } // namespace webrtc diff --git a/test/run_loop.cc b/test/run_loop.cc index cda1e10c..92f85dd0 100644 --- a/test/run_loop.cc +++ b/test/run_loop.cc @@ -16,7 +16,7 @@ namespace test { void PressEnterToContinue() { puts(">> Press ENTER to continue..."); - while (getchar() != '\n' && !feof(stdin)); + while (getc(stdin) != '\n' && !feof(stdin)); } } // namespace test } // namespace webrtc diff --git a/test/test.gyp b/test/test.gyp index 51864331..5c959b9e 100644 --- a/test/test.gyp +++ b/test/test.gyp @@ -224,7 +224,6 @@ ], 'includes': [ '../build/isolate.gypi', - 'test_support_unittests.isolate', ], 'sources': [ 'test_support_unittests.isolate', diff --git a/test/test_support_unittests.isolate b/test/test_support_unittests.isolate index 08bd4a4a..c1419e7e 100644 --- a/test/test_support_unittests.isolate +++ b/test/test_support_unittests.isolate @@ -9,7 +9,7 @@ 'conditions': [ ['OS=="android"', { 'variables': { - 'isolate_dependency_untracked': [ + 'files': [ '<(DEPTH)/data/', '<(DEPTH)/resources/', ], @@ -21,14 +21,11 @@ '<(DEPTH)/testing/test_env.py', '<(PRODUCT_DIR)/test_support_unittests<(EXECUTABLE_SUFFIX)', ], - 'isolate_dependency_tracked': [ + 'files': [ '<(DEPTH)/DEPS', '<(DEPTH)/testing/test_env.py', '<(PRODUCT_DIR)/test_support_unittests<(EXECUTABLE_SUFFIX)', ], - 'isolate_dependency_untracked': [ - '<(DEPTH)/tools/swarming_client/', - ], }, }], ], diff --git a/tools/tools.gyp b/tools/tools.gyp index 04acac3c..102ba8ec 100644 --- a/tools/tools.gyp +++ b/tools/tools.gyp @@ -165,7 +165,6 @@ ], 'includes': [ '../build/isolate.gypi', - 'tools_unittests.isolate', ], 'sources': [ 'tools_unittests.isolate', diff --git a/tools/tools_unittests.isolate b/tools/tools_unittests.isolate index 18065749..bf1fd019 100644 --- a/tools/tools_unittests.isolate +++ b/tools/tools_unittests.isolate @@ -9,7 +9,7 @@ 'conditions': [ ['OS=="android"', { 'variables': { - 'isolate_dependency_untracked': [ + 'files': [ '<(DEPTH)/data/', '<(DEPTH)/resources/', ], @@ -21,15 +21,12 @@ '<(DEPTH)/testing/test_env.py', '<(PRODUCT_DIR)/tools_unittests<(EXECUTABLE_SUFFIX)', ], - 'isolate_dependency_tracked': [ + 'files': [ '<(DEPTH)/DEPS', '<(DEPTH)/resources/foreman_cif.yuv', '<(DEPTH)/testing/test_env.py', '<(PRODUCT_DIR)/tools_unittests<(EXECUTABLE_SUFFIX)', ], - 'isolate_dependency_untracked': [ - '<(DEPTH)/tools/swarming_client/', - ], }, }], ], diff --git a/video/bitrate_estimator_tests.cc b/video/bitrate_estimator_tests.cc index cb7c391e..29fb7300 100644 --- a/video/bitrate_estimator_tests.cc +++ b/video/bitrate_estimator_tests.cc @@ -151,11 +151,7 @@ class BitrateEstimatorTest : public test::CallTest { encoder_config_.streams = test::CreateVideoStreams(1); receive_config_ = VideoReceiveStream::Config(); - assert(receive_config_.codecs.empty()); - VideoCodec codec = - test::CreateDecoderVideoCodec(send_config_.encoder_settings); - receive_config_.codecs.push_back(codec); - // receive_config_.external_decoders will be set by every stream separately. + // receive_config_.decoders will be set by every stream separately. receive_config_.rtp.remote_ssrc = send_config_.rtp.ssrcs[0]; receive_config_.rtp.local_ssrc = kReceiverLocalSsrc; receive_config_.rtp.extensions.push_back( @@ -206,12 +202,13 @@ class BitrateEstimatorTest : public test::CallTest { send_stream_->Start(); frame_generator_capturer_->Start(); - ExternalVideoDecoder decoder; + VideoReceiveStream::Decoder decoder; decoder.decoder = &fake_decoder_; decoder.payload_type = test_->send_config_.encoder_settings.payload_type; + decoder.payload_name = test_->send_config_.encoder_settings.payload_name; + test_->receive_config_.decoders.push_back(decoder); test_->receive_config_.rtp.remote_ssrc = test_->send_config_.rtp.ssrcs[0]; test_->receive_config_.rtp.local_ssrc++; - test_->receive_config_.external_decoders.push_back(decoder); receive_stream_ = test_->receiver_call_->CreateVideoReceiveStream( test_->receive_config_); receive_stream_->Start(); diff --git a/video/call.cc b/video/call.cc index f37e5387..fd41d75f 100644 --- a/video/call.cc +++ b/video/call.cc @@ -20,6 +20,7 @@ #include "webrtc/config.h" #include "webrtc/modules/rtp_rtcp/interface/rtp_header_parser.h" #include "webrtc/modules/video_coding/codecs/vp8/include/vp8.h" +#include "webrtc/modules/video_coding/codecs/vp9/include/vp9.h" #include "webrtc/system_wrappers/interface/critical_section_wrapper.h" #include "webrtc/system_wrappers/interface/rw_lock_wrapper.h" #include "webrtc/system_wrappers/interface/scoped_ptr.h" @@ -46,6 +47,17 @@ VideoEncoder* VideoEncoder::Create(VideoEncoder::EncoderType codec_type) { switch (codec_type) { case kVp8: return VP8Encoder::Create(); + case kVp9: + return VP9Encoder::Create(); + } + assert(false); + return NULL; +} + +VideoDecoder* VideoDecoder::Create(VideoDecoder::DecoderType codec_type) { + switch (codec_type) { + case kVp8: + return VP8Decoder::Create(); } assert(false); return NULL; diff --git a/video/end_to_end_tests.cc b/video/end_to_end_tests.cc index 3d7d3fa7..06cb187c 100644 --- a/video/end_to_end_tests.cc +++ b/video/end_to_end_tests.cc @@ -19,6 +19,9 @@ #include "webrtc/call.h" #include "webrtc/frame_callback.h" #include "webrtc/modules/rtp_rtcp/source/rtcp_utility.h" +#include "webrtc/modules/video_coding/codecs/vp8/include/vp8.h" +#include "webrtc/modules/video_coding/codecs/vp9/include/vp9.h" +#include "webrtc/modules/video_coding/main/interface/video_coding_defines.h" #include "webrtc/system_wrappers/interface/critical_section_wrapper.h" #include "webrtc/system_wrappers/interface/event_wrapper.h" #include "webrtc/system_wrappers/interface/scoped_ptr.h" @@ -222,6 +225,57 @@ TEST_F(EndToEndTest, TransmitsFirstFrame) { DestroyStreams(); } +// TODO(marpan): Re-enable this test on the next libvpx roll. +TEST_F(EndToEndTest, DISABLED_SendsAndReceivesVP9) { + class VP9Observer : public test::EndToEndTest, public VideoRenderer { + public: + VP9Observer() + : EndToEndTest(2 * kDefaultTimeoutMs), + encoder_(VideoEncoder::Create(VideoEncoder::kVp9)), + decoder_(VP9Decoder::Create()), + frame_counter_(0) {} + + virtual void PerformTest() OVERRIDE { + EXPECT_EQ(kEventSignaled, Wait()) + << "Timed out while waiting for enough frames to be decoded."; + } + + virtual void ModifyConfigs( + VideoSendStream::Config* send_config, + std::vector<VideoReceiveStream::Config>* receive_configs, + VideoEncoderConfig* encoder_config) OVERRIDE { + send_config->encoder_settings.encoder = encoder_.get(); + send_config->encoder_settings.payload_name = "VP9"; + send_config->encoder_settings.payload_type = VCM_VP9_PAYLOAD_TYPE; + encoder_config->streams[0].min_bitrate_bps = 50000; + encoder_config->streams[0].target_bitrate_bps = + encoder_config->streams[0].max_bitrate_bps = 2000000; + + (*receive_configs)[0].renderer = this; + (*receive_configs)[0].decoders.resize(1); + (*receive_configs)[0].decoders[0].payload_type = + send_config->encoder_settings.payload_type; + (*receive_configs)[0].decoders[0].payload_name = + send_config->encoder_settings.payload_name; + (*receive_configs)[0].decoders[0].decoder = decoder_.get(); + } + + virtual void RenderFrame(const I420VideoFrame& video_frame, + int time_to_render_ms) OVERRIDE { + const int kRequiredFrames = 500; + if (++frame_counter_ == kRequiredFrames) + observation_complete_->Set(); + } + + private: + scoped_ptr<webrtc::VideoEncoder> encoder_; + scoped_ptr<webrtc::VideoDecoder> decoder_; + int frame_counter_; + } test; + + RunBaseTest(&test); +} + TEST_F(EndToEndTest, SendsAndReceivesH264) { class H264Observer : public test::EndToEndTest, public VideoRenderer { public: @@ -247,14 +301,12 @@ TEST_F(EndToEndTest, SendsAndReceivesH264) { encoder_config->streams[0].max_bitrate_bps = 2000000; (*receive_configs)[0].renderer = this; - VideoCodec codec = - test::CreateDecoderVideoCodec(send_config->encoder_settings); - (*receive_configs)[0].codecs.resize(1); - (*receive_configs)[0].codecs[0] = codec; - (*receive_configs)[0].external_decoders.resize(1); - (*receive_configs)[0].external_decoders[0].payload_type = + (*receive_configs)[0].decoders.resize(1); + (*receive_configs)[0].decoders[0].payload_type = send_config->encoder_settings.payload_type; - (*receive_configs)[0].external_decoders[0].decoder = &fake_decoder_; + (*receive_configs)[0].decoders[0].payload_name = + send_config->encoder_settings.payload_name; + (*receive_configs)[0].decoders[0].decoder = &fake_decoder_; } virtual void RenderFrame(const I420VideoFrame& video_frame, @@ -977,6 +1029,7 @@ TEST_F(EndToEndTest, SendsAndReceivesMultipleStreams) { for (size_t i = 0; i < kNumStreams; ++i) encoders[i].reset(VideoEncoder::Create(VideoEncoder::kVp8)); + ScopedVector<VideoDecoder> allocated_decoders; for (size_t i = 0; i < kNumStreams; ++i) { uint32_t ssrc = codec_settings[i].ssrc; int width = codec_settings[i].width; @@ -1004,9 +1057,10 @@ TEST_F(EndToEndTest, SendsAndReceivesMultipleStreams) { receive_config.renderer = observers[i]; receive_config.rtp.remote_ssrc = ssrc; receive_config.rtp.local_ssrc = kReceiverLocalSsrc; - VideoCodec codec = - test::CreateDecoderVideoCodec(send_config.encoder_settings); - receive_config.codecs.push_back(codec); + VideoReceiveStream::Decoder decoder = + test::CreateMatchingDecoder(send_config.encoder_settings); + allocated_decoders.push_back(decoder.decoder); + receive_config.decoders.push_back(decoder); receive_streams[i] = receiver_call->CreateVideoReceiveStream(receive_config); receive_streams[i]->Start(); diff --git a/video/loopback.cc b/video/loopback.cc index ffc5bcc5..4d3393e7 100644 --- a/video/loopback.cc +++ b/video/loopback.cc @@ -159,9 +159,9 @@ void Loopback() { receive_config.rtp.rtx[kRtxPayloadType].ssrc = kSendRtxSsrc; receive_config.rtp.rtx[kRtxPayloadType].payload_type = kRtxPayloadType; receive_config.renderer = loopback_video.get(); - VideoCodec codec = - test::CreateDecoderVideoCodec(send_config.encoder_settings); - receive_config.codecs.push_back(codec); + VideoReceiveStream::Decoder decoder = + test::CreateMatchingDecoder(send_config.encoder_settings); + receive_config.decoders.push_back(decoder); VideoReceiveStream* receive_stream = call->CreateVideoReceiveStream(receive_config); @@ -179,6 +179,8 @@ void Loopback() { call->DestroyVideoReceiveStream(receive_stream); call->DestroyVideoSendStream(send_stream); + delete decoder.decoder; + transport.StopSending(); } } // namespace webrtc diff --git a/video/replay.cc b/video/replay.cc index 3d2689ea..5cfb06f8 100644 --- a/video/replay.cc +++ b/video/replay.cc @@ -30,6 +30,7 @@ #include "webrtc/test/video_capturer.h" #include "webrtc/test/video_renderer.h" #include "webrtc/typedefs.h" +#include "webrtc/video_decoder.h" namespace webrtc { namespace flags { @@ -212,8 +213,9 @@ void RtpReplay() { VideoSendStream::Config::EncoderSettings encoder_settings; encoder_settings.payload_name = flags::Codec(); encoder_settings.payload_type = flags::PayloadType(); - VideoCodec codec = test::CreateDecoderVideoCodec(encoder_settings); - receive_config.codecs.push_back(codec); + VideoReceiveStream::Decoder decoder = + test::CreateMatchingDecoder(encoder_settings); + receive_config.decoders.push_back(decoder); VideoReceiveStream* receive_stream = call->CreateVideoReceiveStream(receive_config); @@ -271,6 +273,8 @@ void RtpReplay() { } call->DestroyVideoReceiveStream(receive_stream); + + delete decoder.decoder; } } // namespace webrtc diff --git a/video/video_receive_stream.cc b/video/video_receive_stream.cc index 41a800fa..b1822402 100644 --- a/video/video_receive_stream.cc +++ b/video/video_receive_stream.cc @@ -19,6 +19,7 @@ #include "webrtc/system_wrappers/interface/clock.h" #include "webrtc/system_wrappers/interface/logging.h" #include "webrtc/video/receive_statistics_proxy.h" +#include "webrtc/video_encoder.h" #include "webrtc/video_engine/include/vie_base.h" #include "webrtc/video_engine/include/vie_capture.h" #include "webrtc/video_engine/include/vie_codec.h" @@ -31,6 +32,35 @@ namespace webrtc { namespace internal { +namespace { +VideoCodec CreateDecoderVideoCodec(const VideoReceiveStream::Decoder& decoder) { + VideoCodec codec; + memset(&codec, 0, sizeof(codec)); + + codec.plType = decoder.payload_type; + strcpy(codec.plName, decoder.payload_name.c_str()); + if (decoder.payload_name == "VP8") { + codec.codecType = kVideoCodecVP8; + } else if (decoder.payload_name == "H264") { + codec.codecType = kVideoCodecH264; + } else { + codec.codecType = kVideoCodecGeneric; + } + + if (codec.codecType == kVideoCodecVP8) { + codec.codecSpecific.VP8 = VideoEncoder::GetDefaultVp8Settings(); + } else if (codec.codecType == kVideoCodecH264) { + codec.codecSpecific.H264 = VideoEncoder::GetDefaultH264Settings(); + } + + codec.width = 320; + codec.height = 180; + codec.startBitrate = codec.minBitrate = codec.maxBitrate = + Call::Config::kDefaultStartBitrateBps / 1000; + + return codec; +} +} // namespace VideoReceiveStream::VideoReceiveStream(webrtc::VideoEngine* video_engine, const VideoReceiveStream::Config& config, @@ -118,15 +148,6 @@ VideoReceiveStream::VideoReceiveStream(webrtc::VideoEngine* video_engine, } } - assert(!config_.codecs.empty()); - for (size_t i = 0; i < config_.codecs.size(); ++i) { - if (codec_->SetReceiveCodec(channel_, config_.codecs[i]) != 0) { - // TODO(pbos): Abort gracefully, this can be a runtime error. - // Factor out to an Init() method. - abort(); - } - } - stats_proxy_.reset(new ReceiveStatisticsProxy( config_.rtp.local_ssrc, clock_, rtp_rtcp_, codec_, channel_)); @@ -142,8 +163,9 @@ VideoReceiveStream::VideoReceiveStream(webrtc::VideoEngine* video_engine, abort(); external_codec_ = ViEExternalCodec::GetInterface(video_engine); - for (size_t i = 0; i < config_.external_decoders.size(); ++i) { - const ExternalVideoDecoder& decoder = config_.external_decoders[i]; + assert(!config_.decoders.empty()); + for (size_t i = 0; i < config_.decoders.size(); ++i) { + const Decoder& decoder = config_.decoders[i]; if (external_codec_->RegisterExternalReceiveCodec( channel_, decoder.payload_type, @@ -153,6 +175,14 @@ VideoReceiveStream::VideoReceiveStream(webrtc::VideoEngine* video_engine, // TODO(pbos): Abort gracefully? Can this be a runtime error? abort(); } + + VideoCodec codec = CreateDecoderVideoCodec(decoder); + + if (codec_->SetReceiveCodec(channel_, codec) != 0) { + // TODO(pbos): Abort gracefully, this can be a runtime error. + // Factor out to an Init() method. + abort(); + } } render_ = ViERender::GetInterface(video_engine); @@ -160,7 +190,7 @@ VideoReceiveStream::VideoReceiveStream(webrtc::VideoEngine* video_engine, render_->AddRenderCallback(channel_, this); - if (voice_engine) { + if (voice_engine && config_.audio_channel_id != -1) { video_engine_base_->SetVoiceEngine(voice_engine); video_engine_base_->ConnectAudioChannel(channel_, config_.audio_channel_id); } @@ -183,9 +213,9 @@ VideoReceiveStream::~VideoReceiveStream() { render_->RemoveRenderer(channel_); - for (size_t i = 0; i < config_.external_decoders.size(); ++i) { + for (size_t i = 0; i < config_.decoders.size(); ++i) { external_codec_->DeRegisterExternalReceiveCodec( - channel_, config_.external_decoders[i].payload_type); + channel_, config_.decoders[i].payload_type); } network_->DeregisterSendTransport(channel_); @@ -225,10 +255,6 @@ VideoReceiveStream::Stats VideoReceiveStream::GetStats() const { return stats_proxy_->GetStats(); } -void VideoReceiveStream::GetCurrentReceiveCodec(VideoCodec* receive_codec) { - // TODO(pbos): Implement -} - bool VideoReceiveStream::DeliverRtcp(const uint8_t* packet, size_t length) { return network_->ReceivedRTCPPacket( channel_, packet, static_cast<int>(length)) == 0; diff --git a/video/video_receive_stream.h b/video/video_receive_stream.h index 68948287..2aa39e23 100644 --- a/video/video_receive_stream.h +++ b/video/video_receive_stream.h @@ -53,8 +53,6 @@ class VideoReceiveStream : public webrtc::VideoReceiveStream, virtual void Stop() OVERRIDE; virtual Stats GetStats() const OVERRIDE; - virtual void GetCurrentReceiveCodec(VideoCodec* receive_codec) OVERRIDE; - // Overrides I420FrameCallback. virtual void FrameCallback(I420VideoFrame* video_frame) OVERRIDE; diff --git a/video/video_send_stream.cc b/video/video_send_stream.cc index 7657250d..28231b00 100644 --- a/video/video_send_stream.cc +++ b/video/video_send_stream.cc @@ -301,22 +301,32 @@ bool VideoSendStream::ReconfigureVideoEncoder( memset(&video_codec, 0, sizeof(video_codec)); if (config_.encoder_settings.payload_name == "VP8") { video_codec.codecType = kVideoCodecVP8; + } else if (config_.encoder_settings.payload_name == "VP9") { + video_codec.codecType = kVideoCodecVP9; } else if (config_.encoder_settings.payload_name == "H264") { video_codec.codecType = kVideoCodecH264; } else { video_codec.codecType = kVideoCodecGeneric; } + switch (config.content_type) { case VideoEncoderConfig::kRealtimeVideo: video_codec.mode = kRealtimeVideo; break; case VideoEncoderConfig::kScreenshare: video_codec.mode = kScreensharing; + if (config.streams.size() == 1 && + config.streams[0].temporal_layer_thresholds_bps.size() == 1) { + video_codec.targetBitrate = + config.streams[0].temporal_layer_thresholds_bps[0] / 1000; + } break; } if (video_codec.codecType == kVideoCodecVP8) { video_codec.codecSpecific.VP8 = VideoEncoder::GetDefaultVp8Settings(); + } else if (video_codec.codecType == kVideoCodecVP9) { + video_codec.codecSpecific.VP9 = VideoEncoder::GetDefaultVp9Settings(); } else if (video_codec.codecType == kVideoCodecH264) { video_codec.codecSpecific.H264 = VideoEncoder::GetDefaultH264Settings(); } @@ -327,7 +337,8 @@ bool VideoSendStream::ReconfigureVideoEncoder( config.encoder_specific_settings); } video_codec.codecSpecific.VP8.numberOfTemporalLayers = - static_cast<unsigned char>(streams.back().temporal_layers.size()); + static_cast<unsigned char>( + streams.back().temporal_layer_thresholds_bps.size() + 1); } else { // TODO(pbos): Support encoder_settings codec-agnostically. assert(config.encoder_specific_settings == NULL); @@ -360,8 +371,8 @@ bool VideoSendStream::ReconfigureVideoEncoder( sim_stream->targetBitrate = streams[i].target_bitrate_bps / 1000; sim_stream->maxBitrate = streams[i].max_bitrate_bps / 1000; sim_stream->qpMax = streams[i].max_qp; - sim_stream->numberOfTemporalLayers = - static_cast<unsigned char>(streams[i].temporal_layers.size()); + sim_stream->numberOfTemporalLayers = static_cast<unsigned char>( + streams[i].temporal_layer_thresholds_bps.size() + 1); video_codec.width = std::max(video_codec.width, static_cast<unsigned short>(streams[i].width)); diff --git a/video/video_send_stream_tests.cc b/video/video_send_stream_tests.cc index 79b1cd52..b863957b 100644 --- a/video/video_send_stream_tests.cc +++ b/video/video_send_stream_tests.cc @@ -201,7 +201,7 @@ TEST_F(VideoSendStreamTest, SupportsTransmissionTimeOffset) { virtual void PerformTest() OVERRIDE { EXPECT_EQ(kEventSignaled, Wait()) - << "Timed out while waiting single RTP packet."; + << "Timed out while waiting for a single RTP packet."; } class DelayedEncoder : public test::FakeEncoder { @@ -1440,8 +1440,8 @@ TEST_F(VideoSendStreamTest, EncoderSetupPropagatesVp8Config) { send_config->encoder_settings.payload_name = "VP8"; for (size_t i = 0; i < encoder_config->streams.size(); ++i) { - encoder_config->streams[i].temporal_layers.resize( - kNumberOfTemporalLayers); + encoder_config->streams[i].temporal_layer_thresholds_bps.resize( + kNumberOfTemporalLayers - 1); } encoder_config->encoder_specific_settings = &vp8_settings_; @@ -1550,4 +1550,44 @@ TEST_F(VideoSendStreamTest, RtcpSenderReportContainsMediaBytesSent) { RunBaseTest(&test); } +TEST_F(VideoSendStreamTest, TranslatesTwoLayerScreencastToTargetBitrate) { + static const int kScreencastTargetBitrateKbps = 200; + class ScreencastTargetBitrateTest : public test::SendTest, + public test::FakeEncoder { + public: + ScreencastTargetBitrateTest() + : SendTest(kDefaultTimeoutMs), + test::FakeEncoder(Clock::GetRealTimeClock()) {} + + private: + virtual int32_t InitEncode(const VideoCodec* config, + int32_t number_of_cores, + uint32_t max_payload_size) { + EXPECT_EQ(static_cast<unsigned int>(kScreencastTargetBitrateKbps), + config->targetBitrate); + observation_complete_->Set(); + return test::FakeEncoder::InitEncode( + config, number_of_cores, max_payload_size); + } + virtual void ModifyConfigs( + VideoSendStream::Config* send_config, + std::vector<VideoReceiveStream::Config>* receive_configs, + VideoEncoderConfig* encoder_config) OVERRIDE { + send_config->encoder_settings.encoder = this; + EXPECT_EQ(1u, encoder_config->streams.size()); + EXPECT_TRUE( + encoder_config->streams[0].temporal_layer_thresholds_bps.empty()); + encoder_config->streams[0].temporal_layer_thresholds_bps.push_back( + kScreencastTargetBitrateKbps * 1000); + encoder_config->content_type = VideoEncoderConfig::kScreenshare; + } + + virtual void PerformTest() OVERRIDE { + EXPECT_EQ(kEventSignaled, Wait()) + << "Timed out while waiting for the encoder to be initialized."; + } + } test; + + RunBaseTest(&test); +} } // namespace webrtc diff --git a/video_decoder.h b/video_decoder.h new file mode 100644 index 00000000..03a564e3 --- /dev/null +++ b/video_decoder.h @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef WEBRTC_VIDEO_DECODER_H_ +#define WEBRTC_VIDEO_DECODER_H_ + +#include <vector> + +#include "webrtc/common_types.h" +#include "webrtc/typedefs.h" +#include "webrtc/video_frame.h" + +namespace webrtc { + +class RTPFragmentationHeader; +// TODO(pbos): Expose these through a public (root) header or change these APIs. +struct CodecSpecificInfo; +struct VideoCodec; + +class DecodedImageCallback { + public: + virtual ~DecodedImageCallback() {} + + virtual int32_t Decoded(I420VideoFrame& decodedImage) = 0; + virtual int32_t ReceivedDecodedReferenceFrame(const uint64_t pictureId) { + return -1; + } + + virtual int32_t ReceivedDecodedFrame(const uint64_t pictureId) { return -1; } +}; + +class VideoDecoder { + public: + enum DecoderType { + kVp8, + }; + + static VideoDecoder* Create(DecoderType codec_type); + + virtual ~VideoDecoder() {} + + virtual int32_t InitDecode(const VideoCodec* codecSettings, + int32_t numberOfCores) = 0; + + virtual int32_t Decode(const EncodedImage& inputImage, + bool missingFrames, + const RTPFragmentationHeader* fragmentation, + const CodecSpecificInfo* codecSpecificInfo = NULL, + int64_t renderTimeMs = -1) = 0; + + virtual int32_t RegisterDecodeCompleteCallback( + DecodedImageCallback* callback) = 0; + + virtual int32_t Release() = 0; + virtual int32_t Reset() = 0; + + virtual int32_t SetCodecConfigParameters(const uint8_t* /*buffer*/, + int32_t /*size*/) { + return -1; + } + + virtual VideoDecoder* Copy() { return NULL; } +}; + +} // namespace webrtc + +#endif // WEBRTC_VIDEO_DECODER_H_ diff --git a/video_encoder.h b/video_encoder.h index cbdf1ef0..2bf52f3c 100644 --- a/video_encoder.h +++ b/video_encoder.h @@ -40,28 +40,84 @@ class VideoEncoder { public: enum EncoderType { kVp8, + kVp9, }; static VideoEncoder* Create(EncoderType codec_type); static VideoCodecVP8 GetDefaultVp8Settings(); + static VideoCodecVP9 GetDefaultVp9Settings(); static VideoCodecH264 GetDefaultH264Settings(); virtual ~VideoEncoder() {} + // Initialize the encoder with the information from the codecSettings + // + // Input: + // - codec_settings : Codec settings + // - number_of_cores : Number of cores available for the encoder + // - max_payload_size : The maximum size each payload is allowed + // to have. Usually MTU - overhead. + // + // Return value : Set bit rate if OK + // <0 - Errors: + // WEBRTC_VIDEO_CODEC_ERR_PARAMETER + // WEBRTC_VIDEO_CODEC_ERR_SIZE + // WEBRTC_VIDEO_CODEC_LEVEL_EXCEEDED + // WEBRTC_VIDEO_CODEC_MEMORY + // WEBRTC_VIDEO_CODEC_ERROR virtual int32_t InitEncode(const VideoCodec* codec_settings, int32_t number_of_cores, uint32_t max_payload_size) = 0; + + // Register an encode complete callback object. + // + // Input: + // - callback : Callback object which handles encoded images. + // + // Return value : WEBRTC_VIDEO_CODEC_OK if OK, < 0 otherwise. virtual int32_t RegisterEncodeCompleteCallback( EncodedImageCallback* callback) = 0; - virtual int32_t Release() = 0; + // Free encoder memory. + // Return value : WEBRTC_VIDEO_CODEC_OK if OK, < 0 otherwise. + virtual int32_t Release() = 0; + // Encode an I420 image (as a part of a video stream). The encoded image + // will be returned to the user through the encode complete callback. + // + // Input: + // - frame : Image to be encoded + // - frame_types : Frame type to be generated by the encoder. + // + // Return value : WEBRTC_VIDEO_CODEC_OK if OK + // <0 - Errors: + // WEBRTC_VIDEO_CODEC_ERR_PARAMETER + // WEBRTC_VIDEO_CODEC_MEMORY + // WEBRTC_VIDEO_CODEC_ERROR + // WEBRTC_VIDEO_CODEC_TIMEOUT virtual int32_t Encode(const I420VideoFrame& frame, const CodecSpecificInfo* codec_specific_info, const std::vector<VideoFrameType>* frame_types) = 0; + // Inform the encoder of the new packet loss rate and the round-trip time of + // the network. + // + // Input: + // - packet_loss : Fraction lost + // (loss rate in percent = 100 * packetLoss / 255) + // - rtt : Round-trip time in milliseconds + // Return value : WEBRTC_VIDEO_CODEC_OK if OK + // <0 - Errors: WEBRTC_VIDEO_CODEC_ERROR virtual int32_t SetChannelParameters(uint32_t packet_loss, int rtt) = 0; + + // Inform the encoder about the new target bit rate. + // + // Input: + // - bitrate : New target bit rate + // - framerate : The target frame rate + // + // Return value : WEBRTC_VIDEO_CODEC_OK if OK, < 0 otherwise. virtual int32_t SetRates(uint32_t bitrate, uint32_t framerate) = 0; virtual int32_t SetPeriodicKeyFrames(bool enable) { return -1; } diff --git a/video_engine/include/vie_base.h b/video_engine/include/vie_base.h index a49aba7c..ae80cd78 100644 --- a/video_engine/include/vie_base.h +++ b/video_engine/include/vie_base.h @@ -63,6 +63,7 @@ struct CpuOveruseOptions { bool enable_encode_usage_method; int low_encode_usage_threshold_percent; // Threshold for triggering underuse. int high_encode_usage_threshold_percent; // Threshold for triggering overuse. + // TODO(asapersson): Remove options, not used. int low_encode_time_rsd_threshold; // Additional threshold for triggering // underuse (used in addition to // threshold above if configured). @@ -117,6 +118,7 @@ struct CpuOveruseMetrics { int avg_encode_time_ms; // The average encode time in ms. int encode_usage_percent; // The average encode time divided by the average // time difference between incoming captured frames. + // TODO(asapersson): Remove metric, not used. int encode_rsd; // The relative std dev of encode time of frames. int capture_queue_delay_ms_per_s; // The current time delay between an // incoming captured frame until the frame diff --git a/video_engine/overuse_frame_detector.cc b/video_engine/overuse_frame_detector.cc index 96036132..32b0d255 100644 --- a/video_engine/overuse_frame_detector.cc +++ b/video_engine/overuse_frame_detector.cc @@ -212,112 +212,6 @@ class OveruseFrameDetector::SendProcessingUsage { scoped_ptr<rtc::ExpFilter> filtered_frame_diff_ms_; }; -// Class for calculating the relative standard deviation of the processing time -// of frame on the send-side. -// Currently only used for testing. -class OveruseFrameDetector::SendProcessingRsd { - public: - SendProcessingRsd(Clock* clock) - : kWeightFactor(0.6f), - count_(0), - filtered_rsd_(new rtc::ExpFilter(kWeightFactor)), - hist_samples_(0), - hist_sum_(0.0f), - last_process_time_ms_(clock->TimeInMilliseconds()) { - Reset(); - } - ~SendProcessingRsd() {} - - void SetOptions(const CpuOveruseOptions& options) { - options_ = options; - } - - void Reset() { - count_ = 0; - filtered_rsd_->Reset(kWeightFactor); - filtered_rsd_->Apply(1.0f, InitialValue()); - hist_.clear(); - hist_samples_ = 0; - hist_sum_ = 0.0f; - } - - void AddSample(float processing_ms) { - int bin = static_cast<int>(processing_ms + 0.5f); - if (bin <= 0) { - return; - } - ++count_; - ++hist_[bin]; - ++hist_samples_; - hist_sum_ += bin; - } - - void Process(int64_t now) { - if (count_ < static_cast<uint32_t>(options_.min_frame_samples)) { - // Have not received min number of frames since last reset. - return; - } - const int kMinHistSamples = 20; - if (hist_samples_ < kMinHistSamples) { - return; - } - const int64_t kMinDiffSinceLastProcessMs = 1000; - int64_t diff_last_process_ms = now - last_process_time_ms_; - if (now - last_process_time_ms_ <= kMinDiffSinceLastProcessMs) { - return; - } - last_process_time_ms_ = now; - - // Calculate variance (using samples above the mean). - // Checks for a larger processing time of some frames while there is a small - // increase in the average time. - int mean = hist_sum_ / hist_samples_; - float variance = 0.0f; - int total_count = 0; - for (std::map<int,int>::iterator it = hist_.begin(); - it != hist_.end(); ++it) { - int time = it->first; - int count = it->second; - if (time > mean) { - total_count += count; - for (int i = 0; i < count; ++i) { - variance += ((time - mean) * (time - mean)); - } - } - } - variance /= std::max(total_count, 1); - float cov = sqrt(variance) / mean; - - hist_.clear(); - hist_samples_ = 0; - hist_sum_ = 0.0f; - - float exp = static_cast<float>(diff_last_process_ms) / kProcessIntervalMs; - exp = std::min(exp, kMaxExp); - filtered_rsd_->Apply(exp, 100.0f * cov); - } - - int Value() const { - return static_cast<int>(filtered_rsd_->filtered() + 0.5); - } - - private: - float InitialValue() const { - // Start in between the underuse and overuse threshold. - return std::max(((options_.low_encode_time_rsd_threshold + - options_.high_encode_time_rsd_threshold) / 2.0f), 0.0f); - } - - const float kWeightFactor; - uint32_t count_; // Number of samples since last reset. - CpuOveruseOptions options_; - scoped_ptr<rtc::ExpFilter> filtered_rsd_; - int hist_samples_; - float hist_sum_; - std::map<int, int> hist_; // Histogram of time spent on processing frames. - int64_t last_process_time_ms_; -}; - // Class for calculating the processing time of frames. class OveruseFrameDetector::FrameQueue { public: @@ -439,7 +333,6 @@ OveruseFrameDetector::OveruseFrameDetector(Clock* clock) num_pixels_(0), last_encode_sample_ms_(0), encode_time_(new EncodeTimeAvg()), - rsd_(new SendProcessingRsd(clock)), usage_(new SendProcessingUsage()), frame_queue_(new FrameQueue()), last_sample_time_ms_(0), @@ -463,7 +356,6 @@ void OveruseFrameDetector::SetOptions(const CpuOveruseOptions& options) { options_ = options; capture_deltas_.SetOptions(options); usage_->SetOptions(options); - rsd_->SetOptions(options); ResetAll(num_pixels_); } @@ -487,7 +379,7 @@ void OveruseFrameDetector::GetCpuOveruseMetrics( CriticalSectionScoped cs(crit_.get()); metrics->capture_jitter_ms = static_cast<int>(capture_deltas_.StdDev() + 0.5); metrics->avg_encode_time_ms = encode_time_->Value(); - metrics->encode_rsd = rsd_->Value(); + metrics->encode_rsd = 0; metrics->encode_usage_percent = usage_->Value(); metrics->capture_queue_delay_ms_per_s = capture_queue_delay_->Value(); } @@ -515,7 +407,6 @@ void OveruseFrameDetector::ResetAll(int num_pixels) { num_pixels_ = num_pixels; capture_deltas_.Reset(); usage_->Reset(); - rsd_->Reset(); frame_queue_->Reset(); capture_queue_delay_->ClearFrames(); last_capture_time_ = 0; @@ -581,7 +472,6 @@ void OveruseFrameDetector::AddProcessingTime(int elapsed_ms) { if (last_sample_time_ms_ != 0) { int64_t diff_ms = now - last_sample_time_ms_; usage_->AddSample(elapsed_ms, diff_ms); - rsd_->AddSample(elapsed_ms); } last_sample_time_ms_ = now; } @@ -599,7 +489,6 @@ int32_t OveruseFrameDetector::Process() { next_process_time_ = now + kProcessIntervalMs; ++num_process_times_; - rsd_->Process(now); capture_queue_delay_->CalculateDelayChange(diff_ms); if (num_process_times_ <= options_.min_process_count) { @@ -644,7 +533,6 @@ int32_t OveruseFrameDetector::Process() { LOG(LS_VERBOSE) << " Frame stats: capture avg: " << capture_deltas_.Mean() << " capture stddev " << capture_deltas_.StdDev() << " encode usage " << usage_->Value() - << " encode rsd " << rsd_->Value() << " overuse detections " << num_overuse_detections_ << " rampup delay " << rampup_delay; return 0; @@ -656,13 +544,7 @@ bool OveruseFrameDetector::IsOverusing() { overusing = capture_deltas_.StdDev() >= options_.high_capture_jitter_threshold_ms; } else if (options_.enable_encode_usage_method) { - bool usage_overuse = - usage_->Value() >= options_.high_encode_usage_threshold_percent; - bool rsd_overuse = false; - if (options_.high_encode_time_rsd_threshold > 0) { - rsd_overuse = (rsd_->Value() >= options_.high_encode_time_rsd_threshold); - } - overusing = usage_overuse || rsd_overuse; + overusing = usage_->Value() >= options_.high_encode_usage_threshold_percent; } if (overusing) { @@ -683,13 +565,7 @@ bool OveruseFrameDetector::IsUnderusing(int64_t time_now) { underusing = capture_deltas_.StdDev() < options_.low_capture_jitter_threshold_ms; } else if (options_.enable_encode_usage_method) { - bool usage_underuse = - usage_->Value() < options_.low_encode_usage_threshold_percent; - bool rsd_underuse = true; - if (options_.low_encode_time_rsd_threshold > 0) { - rsd_underuse = (rsd_->Value() < options_.low_encode_time_rsd_threshold); - } - underusing = usage_underuse && rsd_underuse; + underusing = usage_->Value() < options_.low_encode_usage_threshold_percent; } return underusing; } diff --git a/video_engine/overuse_frame_detector.h b/video_engine/overuse_frame_detector.h index 421e9dec..f90a4f82 100644 --- a/video_engine/overuse_frame_detector.h +++ b/video_engine/overuse_frame_detector.h @@ -103,7 +103,6 @@ class OveruseFrameDetector : public Module { private: class EncodeTimeAvg; - class SendProcessingRsd; class SendProcessingUsage; class CaptureQueueDelay; class FrameQueue; @@ -146,8 +145,6 @@ class OveruseFrameDetector : public Module { int64_t last_encode_sample_ms_; scoped_ptr<EncodeTimeAvg> encode_time_; - - scoped_ptr<SendProcessingRsd> rsd_; scoped_ptr<SendProcessingUsage> usage_; scoped_ptr<FrameQueue> frame_queue_; int64_t last_sample_time_ms_; diff --git a/video_engine/overuse_frame_detector_unittest.cc b/video_engine/overuse_frame_detector_unittest.cc index 553c3512..e2361695 100644 --- a/video_engine/overuse_frame_detector_unittest.cc +++ b/video_engine/overuse_frame_detector_unittest.cc @@ -71,12 +71,6 @@ class OveruseFrameDetectorTest : public ::testing::Test { options_.high_encode_usage_threshold_percent) / 2.0f) + 0.5; } - int InitialRsd() { - return std::max( - ((options_.low_encode_time_rsd_threshold + - options_.high_encode_time_rsd_threshold) / 2.0f) + 0.5f, 0.0f); - } - void InsertFramesWithInterval( size_t num_frames, int interval_ms, int width, int height) { while (num_frames-- > 0) { @@ -120,18 +114,6 @@ class OveruseFrameDetectorTest : public ::testing::Test { } } - void TriggerOveruseWithRsd(int num_times) { - const int kDelayMs1 = 10; - const int kDelayMs2 = 25; - for (int i = 0; i < num_times; ++i) { - InsertAndSendFramesWithInterval( - 200, kFrameInterval33ms, kWidth, kHeight, kDelayMs1); - InsertAndSendFramesWithInterval( - 10, kFrameInterval33ms, kWidth, kHeight, kDelayMs2); - overuse_detector_->Process(); - } - } - void TriggerUnderuseWithProcessingUsage() { const int kDelayMs1 = 5; const int kDelayMs2 = 6; @@ -160,12 +142,6 @@ class OveruseFrameDetectorTest : public ::testing::Test { return metrics.encode_usage_percent; } - int Rsd() { - CpuOveruseMetrics metrics; - overuse_detector_->GetCpuOveruseMetrics(&metrics); - return metrics.encode_rsd; - } - CpuOveruseOptions options_; scoped_ptr<SimulatedClock> clock_; scoped_ptr<MockCpuOveruseObserver> observer_; @@ -571,73 +547,4 @@ TEST_F(OveruseFrameDetectorTest, TriggerUnderuseWithProcessingUsage(); } -TEST_F(OveruseFrameDetectorTest, RsdResetAfterChangingThreshold) { - EXPECT_EQ(InitialRsd(), Rsd()); - options_.high_encode_time_rsd_threshold = 100; - overuse_detector_->SetOptions(options_); - EXPECT_EQ(InitialRsd(), Rsd()); - options_.low_encode_time_rsd_threshold = 20; - overuse_detector_->SetOptions(options_); - EXPECT_EQ(InitialRsd(), Rsd()); -} - -// enable_encode_usage_method = true; -// low/high_encode_time_rsd_threshold >= 0 -// UsagePercent() > high_encode_usage_threshold_percent || -// Rsd() > high_encode_time_rsd_threshold => overuse. -// UsagePercent() < low_encode_usage_threshold_percent && -// Rsd() < low_encode_time_rsd_threshold => underuse. -TEST_F(OveruseFrameDetectorTest, TriggerOveruseWithRsd) { - options_.enable_capture_jitter_method = false; - options_.enable_encode_usage_method = true; - options_.high_encode_time_rsd_threshold = 80; - overuse_detector_->SetOptions(options_); - // rsd > high, usage < high => overuse - EXPECT_CALL(*(observer_.get()), OveruseDetected()).Times(1); - TriggerOveruseWithRsd(options_.high_threshold_consecutive_count); - EXPECT_LT(UsagePercent(), options_.high_encode_usage_threshold_percent); -} - -TEST_F(OveruseFrameDetectorTest, OveruseAndRecoverWithRsd) { - options_.enable_capture_jitter_method = false; - options_.enable_encode_usage_method = true; - options_.low_encode_time_rsd_threshold = 25; - options_.high_encode_time_rsd_threshold = 80; - overuse_detector_->SetOptions(options_); - // rsd > high, usage < high => overuse - EXPECT_CALL(*(observer_.get()), OveruseDetected()).Times(1); - TriggerOveruseWithRsd(options_.high_threshold_consecutive_count); - EXPECT_LT(UsagePercent(), options_.high_encode_usage_threshold_percent); - // rsd < low, usage < low => underuse - EXPECT_CALL(*(observer_.get()), NormalUsage()).Times(testing::AtLeast(1)); - TriggerUnderuseWithProcessingUsage(); -} - -TEST_F(OveruseFrameDetectorTest, NoUnderuseWithRsd_UsageGtLowThreshold) { - options_.enable_capture_jitter_method = false; - options_.enable_encode_usage_method = true; - options_.low_encode_usage_threshold_percent = 1; - options_.low_encode_time_rsd_threshold = 25; - options_.high_encode_time_rsd_threshold = 90; - overuse_detector_->SetOptions(options_); - // rsd < low, usage > low => no underuse - EXPECT_CALL(*(observer_.get()), NormalUsage()).Times(0); - TriggerUnderuseWithProcessingUsage(); - EXPECT_LT(Rsd(), options_.low_encode_time_rsd_threshold); - EXPECT_GT(UsagePercent(), options_.low_encode_usage_threshold_percent); -} - -TEST_F(OveruseFrameDetectorTest, NoUnderuseWithRsd_RsdGtLowThreshold) { - options_.enable_capture_jitter_method = false; - options_.enable_encode_usage_method = true; - options_.low_encode_usage_threshold_percent = 20; - options_.low_encode_time_rsd_threshold = 1; - options_.high_encode_time_rsd_threshold = 90; - overuse_detector_->SetOptions(options_); - // rsd > low, usage < low => no underuse - EXPECT_CALL(*(observer_.get()), NormalUsage()).Times(0); - TriggerUnderuseWithProcessingUsage(); - EXPECT_GT(Rsd(), options_.low_encode_time_rsd_threshold); - EXPECT_LT(UsagePercent(), options_.low_encode_usage_threshold_percent); -} } // namespace webrtc diff --git a/video_engine/test/auto_test/source/vie_autotest.cc b/video_engine/test/auto_test/source/vie_autotest.cc index fb1a46f2..41fafaef 100644 --- a/video_engine/test/auto_test/source/vie_autotest.cc +++ b/video_engine/test/auto_test/source/vie_autotest.cc @@ -98,6 +98,9 @@ void ViEAutoTest::PrintVideoCodec(const webrtc::VideoCodec videoCodec) case webrtc::kVideoCodecVP8: ViETest::Log("\tcodecType: VP8"); break; + case webrtc::kVideoCodecVP9: + ViETest::Log("\tcodecType: VP9"); + break; case webrtc::kVideoCodecI420: ViETest::Log("\tcodecType: I420"); break; diff --git a/video_engine/test/auto_test/source/vie_autotest_loopback.cc b/video_engine/test/auto_test/source/vie_autotest_loopback.cc index 9b8040d9..03a82b4c 100644 --- a/video_engine/test/auto_test/source/vie_autotest_loopback.cc +++ b/video_engine/test/auto_test/source/vie_autotest_loopback.cc @@ -158,7 +158,7 @@ int VideoEngineSampleCode(void* window1, void* window2) printf("Error in scanf()\n"); return -1; } - getchar(); + getc(stdin); captureIdx = captureIdx - 1; // Compensate for idx start at 1. #endif error = ptrViECapture->GetCaptureDevice(captureIdx, deviceName, @@ -350,7 +350,7 @@ int VideoEngineSampleCode(void* window1, void* window2) printf("Error in scanf()\n"); return -1; } - getchar(); + getc(stdin); codecIdx = codecIdx - 1; // Compensate for idx start at 1. #endif // VP8 over generic transport gets this special one. @@ -624,7 +624,7 @@ int VideoEngineSampleCode(void* window1, void* window2) // Call started printf("\nLoopback call started\n\n"); printf("Press enter to stop..."); - while ((getchar()) != '\n') + while ((getc(stdin)) != '\n') ; //******************************************************** diff --git a/video_engine/test/auto_test/source/vie_autotest_main.cc b/video_engine/test/auto_test/source/vie_autotest_main.cc index c688e2ef..16172585 100644 --- a/video_engine/test/auto_test/source/vie_autotest_main.cc +++ b/video_engine/test/auto_test/source/vie_autotest_main.cc @@ -106,10 +106,10 @@ int ViEAutoTestMain::AskUserForNumber(int min_allowed, int max_allowed) { int result; if (scanf("%d", &result) <= 0) { ViETest::Log("\nPlease enter a number instead, then hit enter."); - getchar(); + getc(stdin); return kInvalidChoice; } - getchar(); // Consume enter key. + getc(stdin); // Consume enter key. if (result < min_allowed || result > max_allowed) { ViETest::Log("%d-%d are valid choices. Please try again.", min_allowed, diff --git a/video_engine/test/auto_test/source/vie_autotest_network.cc b/video_engine/test/auto_test/source/vie_autotest_network.cc index df5b5ef4..1ccac701 100644 --- a/video_engine/test/auto_test/source/vie_autotest_network.cc +++ b/video_engine/test/auto_test/source/vie_autotest_network.cc @@ -150,7 +150,7 @@ void ViEAutoTest::ViENetworkExtendedTest() ViETest::Log("On Win7 and late Vista, you need to right click the " "exe and choose"); ViETest::Log("\"Run as administrator\"\n"); - getchar(); + getc(stdin); } EXPECT_EQ(0, ViE.network->GetSendToS( tbChannel.videoChannel, DSCP, useSetSockOpt)); // No ToS set @@ -390,7 +390,7 @@ void ViEAutoTest::ViENetworkAPITest() ViETest::Log("On Win7 and late Vista, you need to right click the " "exe and choose"); ViETest::Log("\"Run as administrator\"\n"); - getchar(); + getc(stdin); } EXPECT_EQ(0, ViE.network->GetSendToS( tbChannel.videoChannel, DSCP, useSetSockOpt)); diff --git a/video_engine/test/auto_test/source/vie_autotest_record.cc b/video_engine/test/auto_test/source/vie_autotest_record.cc index 1fb11ad1..89575cee 100644 --- a/video_engine/test/auto_test/source/vie_autotest_record.cc +++ b/video_engine/test/auto_test/source/vie_autotest_record.cc @@ -216,7 +216,7 @@ int VideoEngineSampleRecordCode(void* window1, void* window2) { printf("Error in scanf()\n"); return -1; } - getchar(); + getc(stdin); captureIdx = captureIdx - 1; // Compensate for idx start at 1. #endif error = ptrViECapture->GetCaptureDevice(captureIdx, deviceName, @@ -441,14 +441,14 @@ int VideoEngineSampleRecordCode(void* window1, void* window2) { clock_time = webrtc::TickTime::MillisecondTimestamp(); timing << clock_time << std::endl; } - char c = getchar(); + char c = getc(stdin); fflush(stdin); while (c != 's') { if (c == '\n' && enable_labeling == 1) { clock_time = webrtc::TickTime::MillisecondTimestamp(); timing << clock_time << std::endl; } - c = getchar(); + c = getc(stdin); } if (enable_labeling == 1) { clock_time = webrtc::TickTime::MillisecondTimestamp(); diff --git a/video_engine/test/auto_test/vie_auto_test.gypi b/video_engine/test/auto_test/vie_auto_test.gypi index 1fb7e2f3..c8886d8f 100644 --- a/video_engine/test/auto_test/vie_auto_test.gypi +++ b/video_engine/test/auto_test/vie_auto_test.gypi @@ -137,7 +137,6 @@ ], 'includes': [ '../../../build/isolate.gypi', - 'vie_auto_test.isolate', ], 'sources': [ 'vie_auto_test.isolate', diff --git a/video_engine/test/auto_test/vie_auto_test.isolate b/video_engine/test/auto_test/vie_auto_test.isolate index 762a7ac5..da09a6e5 100644 --- a/video_engine/test/auto_test/vie_auto_test.isolate +++ b/video_engine/test/auto_test/vie_auto_test.isolate @@ -9,7 +9,7 @@ 'conditions': [ ['OS=="android"', { 'variables': { - 'isolate_dependency_untracked': [ + 'files': [ '<(DEPTH)/data/', '<(DEPTH)/resources/', ], @@ -21,16 +21,11 @@ '<(DEPTH)/testing/test_env.py', '<(PRODUCT_DIR)/vie_auto_test<(EXECUTABLE_SUFFIX)', ], - 'isolate_dependency_touched': [ + 'files': [ '<(DEPTH)/DEPS', - ], - 'isolate_dependency_tracked': [ '<(DEPTH)/testing/test_env.py', '<(PRODUCT_DIR)/vie_auto_test<(EXECUTABLE_SUFFIX)', ], - 'isolate_dependency_untracked': [ - '<(DEPTH)/tools/swarming_client/', - ], }, }], ], diff --git a/video_engine/video_engine_core.gypi b/video_engine/video_engine_core.gypi index fd61043e..2bfc6536 100644 --- a/video_engine/video_engine_core.gypi +++ b/video_engine/video_engine_core.gypi @@ -123,7 +123,7 @@ 'dependencies': [ 'video_engine_core', '<(webrtc_root)/modules/modules.gyp:video_capture_module_internal_impl', - '<(webrtc_root)/modules/modules.gyp:video_render_module_internal_impl', + '<(webrtc_root)/modules/modules.gyp:video_render_module_internal_impl', '<(DEPTH)/testing/gtest.gyp:gtest', '<(DEPTH)/testing/gmock.gyp:gmock', '<(webrtc_root)/test/test.gyp:test_support_main', @@ -168,7 +168,6 @@ ], 'includes': [ '../build/isolate.gypi', - 'video_engine_core_unittests.isolate', ], 'sources': [ 'video_engine_core_unittests.isolate', diff --git a/video_engine/video_engine_core_unittests.isolate b/video_engine/video_engine_core_unittests.isolate index ec65e739..b95d84c4 100644 --- a/video_engine/video_engine_core_unittests.isolate +++ b/video_engine/video_engine_core_unittests.isolate @@ -9,7 +9,7 @@ 'conditions': [ ['OS=="android"', { 'variables': { - 'isolate_dependency_untracked': [ + 'files': [ '<(DEPTH)/data/', '<(DEPTH)/resources/', ], @@ -21,13 +21,10 @@ '<(DEPTH)/testing/test_env.py', '<(PRODUCT_DIR)/video_engine_core_unittests<(EXECUTABLE_SUFFIX)', ], - 'isolate_dependency_tracked': [ + 'files': [ '<(DEPTH)/testing/test_env.py', '<(PRODUCT_DIR)/video_engine_core_unittests<(EXECUTABLE_SUFFIX)', ], - 'isolate_dependency_untracked': [ - '<(DEPTH)/tools/swarming_client/', - ], }, }], ], diff --git a/video_engine/vie_channel.cc b/video_engine/vie_channel.cc index bbcb602c..ded21407 100644 --- a/video_engine/vie_channel.cc +++ b/video_engine/vie_channel.cc @@ -25,6 +25,7 @@ #include "webrtc/modules/video_render/include/video_render_defines.h" #include "webrtc/system_wrappers/interface/critical_section_wrapper.h" #include "webrtc/system_wrappers/interface/logging.h" +#include "webrtc/system_wrappers/interface/metrics.h" #include "webrtc/system_wrappers/interface/thread_wrapper.h" #include "webrtc/video_engine/call_stats.h" #include "webrtc/video_engine/include/vie_codec.h" @@ -147,7 +148,8 @@ ViEChannel::ViEChannel(int32_t channel_id, sender_(sender), nack_history_size_sender_(kSendSidePacketHistorySize), max_nack_reordering_threshold_(kMaxPacketAgeToNack), - pre_render_callback_(NULL) { + pre_render_callback_(NULL), + start_ms_(Clock::GetRealTimeClock()->TimeInMilliseconds()) { RtpRtcp::Configuration configuration; configuration.id = ViEModuleId(engine_id, channel_id); configuration.audio = false; @@ -222,6 +224,7 @@ int32_t ViEChannel::Init() { } ViEChannel::~ViEChannel() { + UpdateHistograms(); // Make sure we don't get more callbacks from the RTP module. module_process_thread_.DeRegisterModule(vie_receiver_.GetReceiveStatistics()); module_process_thread_.DeRegisterModule(rtp_rtcp_.get()); @@ -246,6 +249,55 @@ ViEChannel::~ViEChannel() { VideoCodingModule::Destroy(vcm_); } +void ViEChannel::UpdateHistograms() { + const float kMinCallLengthInMinutes = 0.5f; + float elapsed_minutes = + (Clock::GetRealTimeClock()->TimeInMilliseconds() - start_ms_) / 60000.0f; + if (elapsed_minutes < kMinCallLengthInMinutes) { + return; + } + RtcpPacketTypeCounter rtcp_sent; + RtcpPacketTypeCounter rtcp_received; + GetRtcpPacketTypeCounters(&rtcp_sent, &rtcp_received); + + if (sender_) { + if (rtcp_received.nack_requests > 0) { + RTC_HISTOGRAM_PERCENTAGE( + "WebRTC.Video.UniqueNackRequestsReceivedInPercent", + rtcp_received.UniqueNackRequestsInPercent()); + } + RTC_HISTOGRAM_COUNTS_10000("WebRTC.Video.NackPacketsReceivedPerMinute", + rtcp_received.nack_packets / elapsed_minutes); + RTC_HISTOGRAM_COUNTS_10000("WebRTC.Video.FirPacketsReceivedPerMinute", + rtcp_received.fir_packets / elapsed_minutes); + RTC_HISTOGRAM_COUNTS_10000("WebRTC.Video.PliPacketsReceivedPerMinute", + rtcp_received.pli_packets / elapsed_minutes); + } else if (vie_receiver_.GetRemoteSsrc() > 0) { + // Get receive stats if we are receiving packets, i.e. there is a remote + // ssrc. + if (rtcp_sent.nack_requests > 0) { + RTC_HISTOGRAM_PERCENTAGE("WebRTC.Video.UniqueNackRequestsSentInPercent", + rtcp_sent.UniqueNackRequestsInPercent()); + } + RTC_HISTOGRAM_COUNTS_10000("WebRTC.Video.NackPacketsSentPerMinute", + rtcp_sent.nack_packets / elapsed_minutes); + RTC_HISTOGRAM_COUNTS_10000("WebRTC.Video.FirPacketsSentPerMinute", + rtcp_sent.fir_packets / elapsed_minutes); + RTC_HISTOGRAM_COUNTS_10000("WebRTC.Video.PliPacketsSentPerMinute", + rtcp_sent.pli_packets / elapsed_minutes); + + webrtc::VCMFrameCount frames; + if (vcm_->ReceivedFrameCount(frames) == VCM_OK) { + uint32_t total_frames = frames.numKeyFrames + frames.numDeltaFrames; + if (total_frames > 0) { + RTC_HISTOGRAM_COUNTS_1000("WebRTC.Video.KeyFramesReceivedInPermille", + static_cast<int>((frames.numKeyFrames * 1000.0f / total_frames) + + 0.5f)); + } + } + } +} + int32_t ViEChannel::SetSendCodec(const VideoCodec& video_codec, bool new_stream) { if (!sender_) { diff --git a/video_engine/vie_channel.h b/video_engine/vie_channel.h index e55ade79..03279069 100644 --- a/video_engine/vie_channel.h +++ b/video_engine/vie_channel.h @@ -383,6 +383,8 @@ class ViEChannel int GetRequiredNackListSize(int target_delay_ms); void SetRtxSendStatus(bool enable); + void UpdateHistograms(); + // ViEChannel exposes methods that allow to modify observers and callbacks // to be modified. Such an API-style is cumbersome to implement and maintain // at all the levels when comparing to only setting them at construction. As @@ -499,6 +501,7 @@ class ViEChannel int nack_history_size_sender_; int max_nack_reordering_threshold_; I420FrameCallback* pre_render_callback_; + const int64_t start_ms_; std::map<uint32_t, RTCPReportBlock> prev_report_blocks_; }; diff --git a/video_engine/vie_channel_group.cc b/video_engine/vie_channel_group.cc index c6b3f741..9c2d59f0 100644 --- a/video_engine/vie_channel_group.cc +++ b/video_engine/vie_channel_group.cc @@ -41,7 +41,7 @@ class WrappingBitrateEstimator : public RemoteBitrateEstimator { crit_sect_(CriticalSectionWrapper::CreateCriticalSection()), engine_id_(engine_id), min_bitrate_bps_(config.Get<RemoteBitrateEstimatorMinRate>().min_rate), - rate_control_type_(kMimdControl), + rate_control_type_(kAimdControl), rbe_(RemoteBitrateEstimatorFactory().Create(observer_, clock_, rate_control_type_, diff --git a/video_engine/vie_codec_impl.cc b/video_engine/vie_codec_impl.cc index f939a667..0ef039db 100644 --- a/video_engine/vie_codec_impl.cc +++ b/video_engine/vie_codec_impl.cc @@ -640,6 +640,8 @@ bool ViECodecImpl::CodecValid(const VideoCodec& video_codec) { return false; } else if ((video_codec.codecType == kVideoCodecVP8 && strncmp(video_codec.plName, "VP8", 4) == 0) || + (video_codec.codecType == kVideoCodecVP9 && + strncmp(video_codec.plName, "VP9", 4) == 0) || (video_codec.codecType == kVideoCodecI420 && strncmp(video_codec.plName, "I420", 4) == 0) || (video_codec.codecType == kVideoCodecH264 && diff --git a/video_engine/vie_encoder.cc b/video_engine/vie_encoder.cc index 26ad8ace..9d6da977 100644 --- a/video_engine/vie_encoder.cc +++ b/video_engine/vie_encoder.cc @@ -26,6 +26,7 @@ #include "webrtc/system_wrappers/interface/clock.h" #include "webrtc/system_wrappers/interface/critical_section_wrapper.h" #include "webrtc/system_wrappers/interface/logging.h" +#include "webrtc/system_wrappers/interface/metrics.h" #include "webrtc/system_wrappers/interface/tick_util.h" #include "webrtc/system_wrappers/interface/trace_event.h" #include "webrtc/video_engine/include/vie_codec.h" @@ -156,7 +157,8 @@ ViEEncoder::ViEEncoder(int32_t engine_id, picture_id_rpsi_(0), qm_callback_(NULL), video_suspended_(false), - pre_encode_callback_(NULL) { + pre_encode_callback_(NULL), + start_ms_(Clock::GetRealTimeClock()->TimeInMilliseconds()) { RtpRtcp::Configuration configuration; configuration.id = ViEModuleId(engine_id_, channel_id_); configuration.audio = false; // Video. @@ -225,6 +227,7 @@ bool ViEEncoder::Init() { } ViEEncoder::~ViEEncoder() { + UpdateHistograms(); if (bitrate_controller_) { bitrate_controller_->RemoveBitrateObserver(bitrate_observer_.get()); } @@ -237,6 +240,25 @@ ViEEncoder::~ViEEncoder() { delete qm_callback_; } +void ViEEncoder::UpdateHistograms() { + const float kMinCallLengthInMinutes = 0.5f; + float elapsed_minutes = + (Clock::GetRealTimeClock()->TimeInMilliseconds() - start_ms_) / 60000.0f; + if (elapsed_minutes < kMinCallLengthInMinutes) { + return; + } + webrtc::VCMFrameCount frames; + if (vcm_.SentFrameCount(frames) != VCM_OK) { + return; + } + uint32_t total_frames = frames.numKeyFrames + frames.numDeltaFrames; + if (total_frames > 0) { + RTC_HISTOGRAM_COUNTS_1000("WebRTC.Video.KeyFramesSentInPermille", + static_cast<int>( + (frames.numKeyFrames * 1000.0f / total_frames) + 0.5f)); + } +} + int ViEEncoder::Owner() const { return channel_id_; } diff --git a/video_engine/vie_encoder.h b/video_engine/vie_encoder.h index 51c1d016..36f87faa 100644 --- a/video_engine/vie_encoder.h +++ b/video_engine/vie_encoder.h @@ -193,6 +193,8 @@ class ViEEncoder private: bool EncoderPaused() const EXCLUSIVE_LOCKS_REQUIRED(data_cs_); + void UpdateHistograms(); + int32_t engine_id_; const int channel_id_; const uint32_t number_of_cores_; @@ -235,6 +237,7 @@ class ViEEncoder QMVideoSettingsCallback* qm_callback_; bool video_suspended_ GUARDED_BY(data_cs_); I420FrameCallback* pre_encode_callback_ GUARDED_BY(callback_cs_); + const int64_t start_ms_; }; } // namespace webrtc diff --git a/video_engine_tests.isolate b/video_engine_tests.isolate index 40454bd6..fc840829 100644 --- a/video_engine_tests.isolate +++ b/video_engine_tests.isolate @@ -9,7 +9,7 @@ 'conditions': [ ['OS=="android"', { 'variables': { - 'isolate_dependency_untracked': [ + 'files': [ '<(DEPTH)/data/', '<(DEPTH)/resources/', ], @@ -20,14 +20,11 @@ 'command': [ '<(PRODUCT_DIR)/video_engine_tests<(EXECUTABLE_SUFFIX)', ], - 'isolate_dependency_tracked': [ + 'files': [ '<(DEPTH)/DEPS', '<(DEPTH)/resources/foreman_cif_short.yuv', '<(PRODUCT_DIR)/video_engine_tests<(EXECUTABLE_SUFFIX)', ], - 'isolate_dependency_untracked': [ - '<(DEPTH)/tools/swarming_client/', - ], }, }], ], diff --git a/video_receive_stream.h b/video_receive_stream.h index 4eb5532b..5ab898c7 100644 --- a/video_receive_stream.h +++ b/video_receive_stream.h @@ -31,30 +31,38 @@ enum RtcpMode { kRtcpCompound, kRtcpReducedSize }; class VideoDecoder; -// TODO(mflodman) Move all these settings to VideoDecoder and move the -// declaration to common_types.h. -struct ExternalVideoDecoder { - ExternalVideoDecoder() - : decoder(NULL), payload_type(0), renderer(false), expected_delay_ms(0) {} - // The actual decoder. - VideoDecoder* decoder; - - // Received RTP packets with this payload type will be sent to this decoder - // instance. - int payload_type; - - // 'true' if the decoder handles rendering as well. - bool renderer; - - // The expected delay for decoding and rendering, i.e. the frame will be - // delivered this many milliseconds, if possible, earlier than the ideal - // render time. - // Note: Ignored if 'renderer' is false. - int expected_delay_ms; -}; - class VideoReceiveStream { public: + // TODO(mflodman) Move all these settings to VideoDecoder and move the + // declaration to common_types.h. + struct Decoder { + Decoder() + : decoder(NULL), + payload_type(0), + renderer(false), + expected_delay_ms(0) {} + + // The actual decoder instance. + VideoDecoder* decoder; + + // Received RTP packets with this payload type will be sent to this decoder + // instance. + int payload_type; + + // Name of the decoded payload (such as VP8). Maps back to the depacketizer + // used to unpack incoming packets. + std::string payload_name; + + // 'true' if the decoder handles rendering as well. + bool renderer; + + // The expected delay for decoding and rendering, i.e. the frame will be + // delivered this many milliseconds, if possible, earlier than the ideal + // render time. + // Note: Ignored if 'renderer' is false. + int expected_delay_ms; + }; + struct Stats : public StreamStats { Stats() : network_frame_rate(0), @@ -77,12 +85,13 @@ class VideoReceiveStream { Config() : renderer(NULL), render_delay_ms(0), - audio_channel_id(0), + audio_channel_id(-1), pre_decode_callback(NULL), pre_render_callback(NULL), target_delay_ms(0) {} - // Codecs the receive stream can receive. - std::vector<VideoCodec> codecs; + + // Decoders for every payload that we can receive. + std::vector<Decoder> decoders; // Receive-stream specific RTP settings. struct Rtp { @@ -162,10 +171,6 @@ class VideoReceiveStream { // stream. 'NULL' disables the callback. I420FrameCallback* pre_render_callback; - // External video decoders to be used if incoming payload type matches the - // registered type for an external decoder. - std::vector<ExternalVideoDecoder> external_decoders; - // Target delay in milliseconds. A positive value indicates this stream is // used for streaming instead of a real-time call. int target_delay_ms; @@ -173,10 +178,9 @@ class VideoReceiveStream { virtual void Start() = 0; virtual void Stop() = 0; - virtual Stats GetStats() const = 0; - // TODO(mflodman) Replace this with callback. - virtual void GetCurrentReceiveCodec(VideoCodec* receive_codec) = 0; + // TODO(pbos): Add info on currently-received codec to Stats. + virtual Stats GetStats() const = 0; protected: virtual ~VideoReceiveStream() {} diff --git a/voice_engine/voe_auto_test.isolate b/voice_engine/voe_auto_test.isolate index 3722b7d7..c13110e7 100644 --- a/voice_engine/voe_auto_test.isolate +++ b/voice_engine/voe_auto_test.isolate @@ -9,7 +9,7 @@ 'conditions': [ ['OS=="android"', { 'variables': { - 'isolate_dependency_untracked': [ + 'files': [ '<(DEPTH)/data/', '<(DEPTH)/resources/', ], @@ -21,13 +21,10 @@ '<(DEPTH)/testing/test_env.py', '<(PRODUCT_DIR)/voe_auto_test<(EXECUTABLE_SUFFIX)', ], - 'isolate_dependency_tracked': [ + 'files': [ '<(DEPTH)/testing/test_env.py', '<(PRODUCT_DIR)/voe_auto_test<(EXECUTABLE_SUFFIX)', ], - 'isolate_dependency_untracked': [ - '<(DEPTH)/tools/swarming_client/', - ], }, }], ], diff --git a/voice_engine/voice_engine.gyp b/voice_engine/voice_engine.gyp index 252afde2..83b356d7 100644 --- a/voice_engine/voice_engine.gyp +++ b/voice_engine/voice_engine.gyp @@ -293,7 +293,6 @@ ], 'includes': [ '../build/isolate.gypi', - 'voice_engine_unittests.isolate', ], 'sources': [ 'voice_engine_unittests.isolate', @@ -307,7 +306,6 @@ ], 'includes': [ '../build/isolate.gypi', - 'voe_auto_test.isolate', ], 'sources': [ 'voe_auto_test.isolate', diff --git a/voice_engine/voice_engine_unittests.isolate b/voice_engine/voice_engine_unittests.isolate index 02356804..4cd1f907 100644 --- a/voice_engine/voice_engine_unittests.isolate +++ b/voice_engine/voice_engine_unittests.isolate @@ -9,7 +9,7 @@ 'conditions': [ ['OS=="android"', { 'variables': { - 'isolate_dependency_untracked': [ + 'files': [ '<(DEPTH)/data/', '<(DEPTH)/resources/', ], @@ -21,13 +21,10 @@ '<(DEPTH)/testing/test_env.py', '<(PRODUCT_DIR)/voice_engine_unittests<(EXECUTABLE_SUFFIX)', ], - 'isolate_dependency_tracked': [ + 'files': [ '<(DEPTH)/testing/test_env.py', '<(PRODUCT_DIR)/voice_engine_unittests<(EXECUTABLE_SUFFIX)', ], - 'isolate_dependency_untracked': [ - '<(DEPTH)/tools/swarming_client/', - ], }, }], ], @@ -36,6 +36,7 @@ 'video_engine/video_engine.gyp:*', 'voice_engine/voice_engine.gyp:*', '<(webrtc_vp8_dir)/vp8.gyp:*', + '<(webrtc_vp9_dir)/vp9.gyp:*', ], }, 'targets': [ diff --git a/webrtc_perf_tests.isolate b/webrtc_perf_tests.isolate index b39c83df..2c547be5 100644 --- a/webrtc_perf_tests.isolate +++ b/webrtc_perf_tests.isolate @@ -9,7 +9,7 @@ 'conditions': [ ['OS=="android"', { 'variables': { - 'isolate_dependency_untracked': [ + 'files': [ '<(DEPTH)/data/', '<(DEPTH)/resources/', ], @@ -20,16 +20,13 @@ 'command': [ '<(PRODUCT_DIR)/webrtc_perf_tests<(EXECUTABLE_SUFFIX)', ], - 'isolate_dependency_tracked': [ + 'files': [ '<(DEPTH)/DEPS', '<(DEPTH)/resources/foreman_cif.yuv', '<(DEPTH)/resources/paris_qcif.yuv', '<(DEPTH)/resources/voice_engine/audio_long16.pcm', '<(PRODUCT_DIR)/webrtc_perf_tests<(EXECUTABLE_SUFFIX)', ], - 'isolate_dependency_untracked': [ - '<(DEPTH)/tools/swarming_client/', - ], }, }], ], diff --git a/webrtc_tests.gypi b/webrtc_tests.gypi index 33e50e70..33e509fa 100644 --- a/webrtc_tests.gypi +++ b/webrtc_tests.gypi @@ -175,7 +175,6 @@ ], 'includes': [ 'build/isolate.gypi', - 'rtc_unittests.isolate', ], 'sources': [ 'rtc_unittests.isolate', @@ -189,7 +188,6 @@ ], 'includes': [ 'build/isolate.gypi', - 'video_engine_tests.isolate', ], 'sources': [ 'video_engine_tests.isolate', @@ -203,7 +201,6 @@ ], 'includes': [ 'build/isolate.gypi', - 'webrtc_perf_tests.isolate', ], 'sources': [ 'webrtc_perf_tests.isolate', |