From 5480c1d2817b9a564e69d3c0c151c6411ae4909b Mon Sep 17 00:00:00 2001 From: "Jorge E. Moreira" Date: Wed, 30 Jun 2021 11:33:51 -0700 Subject: Avoid overflow in calc_iframe_target_size The changed product was observed to attempt to multiply 1800 by 2500000, which overflows unsigned 32 bits. Converting to unsigned 64 bits first and testing whether the final result fits in 32 bits solves the problem. Fixes: 179686142 Test: run cuttlefish locally for over 30 min without observing the issue Change-Id: I5d27317bf14b0311b739144c451d8e172db01945 --- README.version | 1 + libvpx/vp8/encoder/ratectrl.c | 8 ++++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/README.version b/README.version index 7e8c25da3..f555bcc48 100644 --- a/README.version +++ b/README.version @@ -11,3 +11,4 @@ Local Modifications: 5e065cf9d vp8/{ratectrl,onyx_if}: fix some signed integer overflows 5eab093a7 vp9_ratectrl: fix some signed integer overflows baefbe85d Cap target bitrate to raw rate internally + 5f345a924 Avoid overflow in calc_iframe_target_size diff --git a/libvpx/vp8/encoder/ratectrl.c b/libvpx/vp8/encoder/ratectrl.c index d8d55fce2..d93fb59a4 100644 --- a/libvpx/vp8/encoder/ratectrl.c +++ b/libvpx/vp8/encoder/ratectrl.c @@ -349,8 +349,12 @@ static void calc_iframe_target_size(VP8_COMP *cpi) { } if (cpi->oxcf.rc_max_intra_bitrate_pct) { - unsigned int max_rate = - cpi->per_frame_bandwidth * cpi->oxcf.rc_max_intra_bitrate_pct / 100; + unsigned int max_rate; + // This product may overflow unsigned int + uint64_t product = cpi->per_frame_bandwidth; + product *= cpi->oxcf.rc_max_intra_bitrate_pct; + product /= 100; + max_rate = (unsigned int)VPXMIN(INT_MAX, product); if (target > max_rate) target = max_rate; } -- cgit v1.2.3