Merge "cherry-pick vp8 integer overflow fixes"

4 files changed, 50 insertions, 10 deletions

diff --git a/README.version b/README.version
index 9355e92cb..4ce368f13 100644
--- a/README.version
+++ b/README.version
@@ -3,4 +3,6 @@ Version: v1.11.0
 BugComponent: 42195
 Owners: jzern, jianj
 Local Modifications:
-  None
+  218b99892 vp8 encoder: fix some integer overflows
+  05f80a920 vp8,calc_pframe_target_size: fix integer overflow
+  7afb3a676 vp8_update_rate_correction_factors: fix integer overflow
diff --git a/libvpx/test/realtime_test.cc b/libvpx/test/realtime_test.cc
index 63a5347d9..b32a35513 100644
--- a/libvpx/test/realtime_test.cc
+++ b/libvpx/test/realtime_test.cc
@@ -7,6 +7,8 @@
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
+#include <limits.h>
+
 #include "test/codec_factory.h"
 #include "test/encode_test_driver.h"
 #include "test/util.h"
@@ -52,6 +54,22 @@ class RealtimeTest
     frame_packets_++;
   }
 
+  bool IsVP9() const {
+#if CONFIG_VP9_ENCODER
+    return codec_ == &libvpx_test::kVP9;
+#else
+    return false;
+#endif
+  }
+
+  void TestIntegerOverflow(unsigned int width, unsigned int height) {
+    ::libvpx_test::RandomVideoSource video;
+    video.SetSize(width, height);
+    video.set_limit(20);
+    cfg_.rc_target_bitrate = UINT_MAX;
+    ASSERT_NO_FATAL_FAILURE(RunLoop(&video));
+  }
+
   int frame_packets_;
 };
 
@@ -64,11 +82,26 @@ TEST_P(RealtimeTest, RealtimeFirstPassProducesFrames) {
 }
 
 TEST_P(RealtimeTest, IntegerOverflow) {
-  ::libvpx_test::RandomVideoSource video;
-  video.SetSize(800, 480);
-  video.set_limit(20);
-  cfg_.rc_target_bitrate = 140000000;
-  ASSERT_NO_FATAL_FAILURE(RunLoop(&video));
+  if (IsVP9()) {
+    // TODO(https://crbug.com/webm/1749): This should match VP8.
+    TestIntegerOverflow(800, 480);
+  } else {
+    TestIntegerOverflow(2048, 2048);
+  }
+}
+
+TEST_P(RealtimeTest, IntegerOverflowLarge) {
+  if (IsVP9()) {
+    GTEST_SKIP() << "TODO(https://crbug.com/webm/1750): Enable this test after "
+                    "undefined sanitizer warnings are fixed.";
+    // TestIntegerOverflow(16384, 16384);
+  } else {
+    GTEST_SKIP()
+        << "TODO(https://crbug.com/webm/1748,https://crbug.com/webm/1751):"
+        << " Enable this test after bitstream errors & undefined sanitizer "
+           "warnings are fixed.";
+    // TestIntegerOverflow(16383, 16383);
+  }
 }
 
 VP8_INSTANTIATE_TEST_SUITE(RealtimeTest,
diff --git a/libvpx/vp8/encoder/ratectrl.c b/libvpx/vp8/encoder/ratectrl.c
index d2b8dff06..59a4fada8 100644
--- a/libvpx/vp8/encoder/ratectrl.c
+++ b/libvpx/vp8/encoder/ratectrl.c
@@ -780,6 +780,7 @@ static void calc_pframe_target_size(VP8_COMP *cpi) {
         }
       } else {
         int percent_high = 0;
+        int64_t target = cpi->this_frame_target;
 
         if ((cpi->oxcf.end_usage == USAGE_STREAM_FROM_SERVER) &&
             (cpi->buffer_level > cpi->oxcf.optimal_buffer_level)) {
@@ -797,7 +798,9 @@ static void calc_pframe_target_size(VP8_COMP *cpi) {
           percent_high = 0;
         }
 
-        cpi->this_frame_target += (cpi->this_frame_target * percent_high) / 200;
+        target += (target * percent_high) / 200;
+        target = VPXMIN(target, INT_MAX);
+        cpi->this_frame_target = (int)target;
 
         /* Are we allowing control of active_worst_allowed_q according
          * to buffer level.
@@ -1078,8 +1081,8 @@ void vp8_update_rate_correction_factors(VP8_COMP *cpi, int damp_var) {
 
   /* Work out a size correction factor. */
   if (projected_size_based_on_q > 0) {
-    correction_factor =
-        (100 * cpi->projected_frame_size) / projected_size_based_on_q;
+    correction_factor = (int)((100 * (int64_t)cpi->projected_frame_size) /
+                              projected_size_based_on_q);
   }
 
   /* More heavily damped adjustment used if we have been oscillating
diff --git a/libvpx/vp8/vp8_cx_iface.c b/libvpx/vp8/vp8_cx_iface.c
index 78631e797..b5865ce1f 100644
--- a/libvpx/vp8/vp8_cx_iface.c
+++ b/libvpx/vp8/vp8_cx_iface.c
@@ -339,7 +339,9 @@ static vpx_codec_err_t set_vp8e_config(VP8_CONFIG *oxcf,
     oxcf->end_usage = USAGE_CONSTANT_QUALITY;
   }
 
-  oxcf->target_bandwidth = cfg.rc_target_bitrate;
+  // Cap the target rate to 1000 Mbps to avoid some integer overflows in
+  // target bandwidth calculations.
+  oxcf->target_bandwidth = VPXMIN(cfg.rc_target_bitrate, 1000000);
   oxcf->rc_max_intra_bitrate_pct = vp8_cfg.rc_max_intra_bitrate_pct;
   oxcf->gf_cbr_boost_pct = vp8_cfg.gf_cbr_boost_pct;