Fix for stack corruption in esbr

Bug: 110769924
Test: poc from bug before/after
Change-Id: I99c6e89902064849ea1310c271064bdeccf7f20e
(cherry picked from commit 7e90d745c22695236437297cd8167a9312427a4a)
(cherry picked from commit 5464927f0c1fc721fa03d1c5be77b0b43dfffc50)

5 files changed, 60 insertions, 79 deletions

diff --git a/decoder/armv8/ixheaacd_qmf_dec_armv8.c b/decoder/armv8/ixheaacd_qmf_dec_armv8.c
index a4b89c5..8cfe002 100644
--- a/decoder/armv8/ixheaacd_qmf_dec_armv8.c
+++ b/decoder/armv8/ixheaacd_qmf_dec_armv8.c
@@ -501,8 +501,8 @@ VOID ixheaacd_esbr_radix4bfly(const WORD32 *w, WORD32 *x, WORD32 index1,
                               WORD32 index) {
   int i;
   WORD32 l1, l2, h2, fft_jmp;
-  WORD32 xt0_0, yt0_0, xt1_0, yt1_0, xt2_0, yt2_0;
-  WORD32 xh0_0, xh1_0, xh20_0, xh21_0, xl0_0, xl1_0, xl20_0, xl21_0;
+  WORD64 xt0_0, yt0_0, xt1_0, yt1_0, xt2_0, yt2_0;
+  WORD64 xh0_0, xh1_0, xh20_0, xh21_0, xl0_0, xl1_0, xl20_0, xl21_0;
   WORD32 x_0, x_1, x_l1_0, x_l1_1, x_l2_0, x_l2_1;
   WORD32 x_h2_0, x_h2_1;
   WORD32 si10, si20, si30, co10, co20, co30;
@@ -539,34 +539,34 @@ VOID ixheaacd_esbr_radix4bfly(const WORD32 *w, WORD32 *x, WORD32 index1,
       x_l1_0 = x[l1];
       x_l2_0 = x[l2];
 
-      xh0_0 = x_0 + x_l1_0;
-      xl0_0 = x_0 - x_l1_0;
+      xh0_0 = (WORD64)x_0 + (WORD64)x_l1_0;
+      xl0_0 = (WORD64)x_0 - (WORD64)x_l1_0;
 
-      xh20_0 = x_h2_0 + x_l2_0;
-      xl20_0 = x_h2_0 - x_l2_0;
+      xh20_0 = (WORD64)x_h2_0 + (WORD64)x_l2_0;
+      xl20_0 = (WORD64)x_h2_0 - (WORD64)x_l2_0;
 
-      x[0] = xh0_0 + xh20_0;
-      xt0_0 = xh0_0 - xh20_0;
+      x[0] = (WORD32)ixheaacd_add64_sat(xh0_0, xh20_0);
+      xt0_0 = (WORD64)xh0_0 - (WORD64)xh20_0;
 
       x_1 = x[1];
       x_h2_1 = x[h2 + 1];
       x_l1_1 = x[l1 + 1];
       x_l2_1 = x[l2 + 1];
 
-      xh1_0 = x_1 + x_l1_1;
-      xl1_0 = x_1 - x_l1_1;
+      xh1_0 = (WORD64)x_1 + (WORD64)x_l1_1;
+      xl1_0 = (WORD64)x_1 - (WORD64)x_l1_1;
 
-      xh21_0 = x_h2_1 + x_l2_1;
-      xl21_0 = x_h2_1 - x_l2_1;
+      xh21_0 = (WORD64)x_h2_1 + (WORD64)x_l2_1;
+      xl21_0 = (WORD64)x_h2_1 - (WORD64)x_l2_1;
 
-      x[1] = xh1_0 + xh21_0;
-      yt0_0 = xh1_0 - xh21_0;
+      x[1] = (WORD32)ixheaacd_add64_sat(xh1_0, xh21_0);
+      yt0_0 = (WORD64)xh1_0 - (WORD64)xh21_0;
 
-      xt1_0 = xl0_0 + xl21_0;
-      xt2_0 = xl0_0 - xl21_0;
+      xt1_0 = (WORD64)xl0_0 + (WORD64)xl21_0;
+      xt2_0 = (WORD64)xl0_0 - (WORD64)xl21_0;
 
-      yt2_0 = xl1_0 + xl20_0;
-      yt1_0 = xl1_0 - xl20_0;
+      yt2_0 = (WORD64)xl1_0 + (WORD64)xl20_0;
+      yt1_0 = (WORD64)xl1_0 - (WORD64)xl20_0;
 
       mul_11 = ixheaacd_mult64(xt2_0, co30);
       mul_3 = ixheaacd_mult64(yt2_0, si30);
@@ -607,8 +607,6 @@ VOID ixheaacd_esbr_postradixcompute2(WORD32 *ptr_y, WORD32 *ptr_x,
   WORD32 x_0, x_1, x_2, x_3;
   WORD32 x_4, x_5, x_6, x_7;
   WORD32 x_8, x_9, x_a, x_b, x_c, x_d, x_e, x_f;
-  WORD32 n00, n10, n20, n30, n01, n11, n21, n31;
-  WORD32 n02, n12, n22, n32, n03, n13, n23, n33;
   WORD32 n0, j0;
   WORD32 *x2, *x0;
   WORD32 *y0, *y1, *y2, *y3;
@@ -636,23 +634,14 @@ VOID ixheaacd_esbr_postradixcompute2(WORD32 *ptr_y, WORD32 *ptr_x,
       x_6 = *x0++;
       x_7 = *x0++;
 
-      n00 = x_0 + x_2;
-      n01 = x_1 + x_3;
-      n20 = x_0 - x_2;
-      n21 = x_1 - x_3;
-      n10 = x_4 + x_6;
-      n11 = x_5 + x_7;
-      n30 = x_4 - x_6;
-      n31 = x_5 - x_7;
-
-      y0[h2] = n00;
-      y0[h2 + 1] = n01;
-      y1[h2] = n10;
-      y1[h2 + 1] = n11;
-      y2[h2] = n20;
-      y2[h2 + 1] = n21;
-      y3[h2] = n30;
-      y3[h2 + 1] = n31;
+      y0[h2] = ixheaacd_add32_sat(x_0, x_2);
+      y0[h2 + 1] = ixheaacd_add32_sat(x_1, x_3);
+      y1[h2] = ixheaacd_add32_sat(x_4, x_6);
+      y1[h2 + 1] = ixheaacd_add32_sat(x_5, x_7);
+      y2[h2] = ixheaacd_sub32_sat(x_0, x_2);
+      y2[h2 + 1] = ixheaacd_sub32_sat(x_1, x_3);
+      y3[h2] = ixheaacd_sub32_sat(x_4, x_6);
+      y3[h2 + 1] = ixheaacd_sub32_sat(x_5, x_7);
 
       x_8 = *x2++;
       x_9 = *x2++;
@@ -663,23 +652,14 @@ VOID ixheaacd_esbr_postradixcompute2(WORD32 *ptr_y, WORD32 *ptr_x,
       x_e = *x2++;
       x_f = *x2++;
 
-      n02 = x_8 + x_a;
-      n03 = x_9 + x_b;
-      n22 = x_8 - x_a;
-      n23 = x_9 - x_b;
-      n12 = x_c + x_e;
-      n13 = x_d + x_f;
-      n32 = x_c - x_e;
-      n33 = x_d - x_f;
-
-      y0[h2 + 2] = n02;
-      y0[h2 + 3] = n03;
-      y1[h2 + 2] = n12;
-      y1[h2 + 3] = n13;
-      y2[h2 + 2] = n22;
-      y2[h2 + 3] = n23;
-      y3[h2 + 2] = n32;
-      y3[h2 + 3] = n33;
+      y0[h2 + 2] = ixheaacd_add32_sat(x_8, x_a);
+      y0[h2 + 3] = ixheaacd_add32_sat(x_9, x_b);
+      y1[h2 + 2] = ixheaacd_add32_sat(x_c, x_e);
+      y1[h2 + 3] = ixheaacd_add32_sat(x_d, x_f);
+      y2[h2 + 2] = ixheaacd_sub32_sat(x_8, x_a);
+      y2[h2 + 3] = ixheaacd_sub32_sat(x_9, x_b);
+      y3[h2 + 2] = ixheaacd_sub32_sat(x_c, x_e);
+      y3[h2 + 3] = ixheaacd_sub32_sat(x_d, x_f);
     }
     x0 += (WORD32)npoints >> 1;
     x2 += (WORD32)npoints >> 1;
@@ -1224,19 +1204,19 @@ VOID ixheaacd_shiftrountine_with_rnd_hq(WORD32 *qmf_real, WORD32 *qmf_imag,
     r1 = *qmf_real++;
     i1 = *qmf_imag++;
 
-    timag = ixheaacd_add32(i1, r1);
+    timag = ixheaacd_add32_sat(i1, r1);
     timag = (ixheaacd_shl32_sat(timag, shift));
     filter_states_rev[j] = timag;
 
-    treal = ixheaacd_sub32(i2, r2);
+    treal = ixheaacd_sub32_sat(i2, r2);
     treal = (ixheaacd_shl32_sat(treal, shift));
     filter_states[j] = treal;
 
-    treal = ixheaacd_sub32(i1, r1);
+    treal = ixheaacd_sub32_sat(i1, r1);
     treal = (ixheaacd_shl32_sat(treal, shift));
     *filter_states++ = treal;
 
-    timag = ixheaacd_add32(i2, r2);
+    timag = ixheaacd_add32_sat(i2, r2);
     timag = (ixheaacd_shl32_sat(timag, shift));
     *filter_states_rev++ = timag;
   }
diff --git a/decoder/ixheaacd_esbr_polyphase.c b/decoder/ixheaacd_esbr_polyphase.c
index bcb4685..d51a14c 100644
--- a/decoder/ixheaacd_esbr_polyphase.c
+++ b/decoder/ixheaacd_esbr_polyphase.c
@@ -120,8 +120,9 @@ WORD32 ixheaacd_complex_anal_filt(ia_esbr_hbe_txposer_struct *ptr_hbe_txposer) {
         *ptr_u++ = ((*analy_cos_sin_tab++) * u[k]);
         *ptr_u++ = ((*analy_cos_sin_tab++) * u[k]);
       }
-      if (ixheaacd_cmplx_anal_fft != NULL)
-        (*ixheaacd_cmplx_anal_fft)(u_in, u_out, anal_size * 2);
+      if (ptr_hbe_txposer->ixheaacd_cmplx_anal_fft != NULL)
+        (*(ptr_hbe_txposer->ixheaacd_cmplx_anal_fft))(u_in, u_out,
+                                                      anal_size * 2);
       else
         return -1;
 
@@ -209,8 +210,9 @@ WORD32 ixheaacd_real_synth_filt(ia_esbr_hbe_txposer_struct *ptr_hbe_txposer,
       FLOAT32 *syn_buf = &buffer[kmax];
       kmax += synth_size;
 
-      if (ixheaacd_real_synth_fft != NULL)
-        (*ixheaacd_real_synth_fft)(synth_buf_r, synth_out, synth_size * 2);
+      if (ptr_hbe_txposer->ixheaacd_real_synth_fft != NULL)
+        (*(ptr_hbe_txposer->ixheaacd_real_synth_fft))(synth_buf_r, synth_out,
+                                                      synth_size * 2);
       else
         return -1;
 
diff --git a/decoder/ixheaacd_hbe_trans.c b/decoder/ixheaacd_hbe_trans.c
index e188572..0c6f191 100644
--- a/decoder/ixheaacd_hbe_trans.c
+++ b/decoder/ixheaacd_hbe_trans.c
@@ -132,32 +132,32 @@ WORD32 ixheaacd_qmf_hbe_data_reinit(ia_esbr_hbe_txposer_struct *ptr_hbe_txposer,
             (FLOAT32 *)ixheaacd_synth_cos_table_kl_4;
         ptr_hbe_txposer->analy_cos_sin_tab =
             (FLOAT32 *)ixheaacd_analy_cos_sin_table_kl_8;
-        ixheaacd_real_synth_fft = &ixheaacd_real_synth_fft_p2;
-        ixheaacd_cmplx_anal_fft = &ixheaacd_cmplx_anal_fft_p2;
+        ptr_hbe_txposer->ixheaacd_real_synth_fft = &ixheaacd_real_synth_fft_p2;
+        ptr_hbe_txposer->ixheaacd_cmplx_anal_fft = &ixheaacd_cmplx_anal_fft_p2;
         break;
       case 8:
         ptr_hbe_txposer->synth_cos_tab =
             (FLOAT32 *)ixheaacd_synth_cos_table_kl_8;
         ptr_hbe_txposer->analy_cos_sin_tab =
             (FLOAT32 *)ixheaacd_analy_cos_sin_table_kl_16;
-        ixheaacd_real_synth_fft = &ixheaacd_real_synth_fft_p2;
-        ixheaacd_cmplx_anal_fft = &ixheaacd_cmplx_anal_fft_p2;
+        ptr_hbe_txposer->ixheaacd_real_synth_fft = &ixheaacd_real_synth_fft_p2;
+        ptr_hbe_txposer->ixheaacd_cmplx_anal_fft = &ixheaacd_cmplx_anal_fft_p2;
         break;
       case 12:
         ptr_hbe_txposer->synth_cos_tab =
             (FLOAT32 *)ixheaacd_synth_cos_table_kl_12;
         ptr_hbe_txposer->analy_cos_sin_tab =
             (FLOAT32 *)ixheaacd_analy_cos_sin_table_kl_24;
-        ixheaacd_real_synth_fft = &ixheaacd_real_synth_fft_p3;
-        ixheaacd_cmplx_anal_fft = &ixheaacd_cmplx_anal_fft_p3;
+        ptr_hbe_txposer->ixheaacd_real_synth_fft = &ixheaacd_real_synth_fft_p3;
+        ptr_hbe_txposer->ixheaacd_cmplx_anal_fft = &ixheaacd_cmplx_anal_fft_p3;
         break;
       case 16:
         ptr_hbe_txposer->synth_cos_tab =
             (FLOAT32 *)ixheaacd_synth_cos_table_kl_16;
         ptr_hbe_txposer->analy_cos_sin_tab =
             (FLOAT32 *)ixheaacd_analy_cos_sin_table_kl_32;
-        ixheaacd_real_synth_fft = &ixheaacd_real_synth_fft_p2;
-        ixheaacd_cmplx_anal_fft = &ixheaacd_cmplx_anal_fft_p2;
+        ptr_hbe_txposer->ixheaacd_real_synth_fft = &ixheaacd_real_synth_fft_p2;
+        ptr_hbe_txposer->ixheaacd_cmplx_anal_fft = &ixheaacd_cmplx_anal_fft_p2;
         break;
       case 20:
         ptr_hbe_txposer->synth_cos_tab =
@@ -170,8 +170,8 @@ WORD32 ixheaacd_qmf_hbe_data_reinit(ia_esbr_hbe_txposer_struct *ptr_hbe_txposer,
             (FLOAT32 *)ixheaacd_synth_cos_table_kl_4;
         ptr_hbe_txposer->analy_cos_sin_tab =
             (FLOAT32 *)ixheaacd_analy_cos_sin_table_kl_8;
-        ixheaacd_real_synth_fft = &ixheaacd_real_synth_fft_p2;
-        ixheaacd_cmplx_anal_fft = &ixheaacd_cmplx_anal_fft_p2;
+        ptr_hbe_txposer->ixheaacd_real_synth_fft = &ixheaacd_real_synth_fft_p2;
+        ptr_hbe_txposer->ixheaacd_cmplx_anal_fft = &ixheaacd_cmplx_anal_fft_p2;
     }
 
     ptr_hbe_txposer->synth_wind_coeff = ixheaacd_map_prot_filter(synth_size);
@@ -213,9 +213,9 @@ WORD32 ixheaacd_qmf_hbe_data_reinit(ia_esbr_hbe_txposer_struct *ptr_hbe_txposer,
         break;
       }
     }
-  }
-  if (ptr_hbe_txposer->k_start < 0) {
-    return -1;
+    if (ptr_hbe_txposer->k_start < 0) {
+      return -1;
+    }
   }
   return 0;
 }
diff --git a/decoder/ixheaacd_qmf_poly.h b/decoder/ixheaacd_qmf_poly.h
index 8a67480..0cafcf1 100644
--- a/decoder/ixheaacd_qmf_poly.h
+++ b/decoder/ixheaacd_qmf_poly.h
@@ -37,8 +37,4 @@ VOID ixheaacd_real_synth_fft_p2(FLOAT32 *inp, FLOAT32 *out, WORD32 n_points);
 
 VOID ixheaacd_real_synth_fft_p3(FLOAT32 *inp, FLOAT32 *out, WORD32 n_points);
 
-VOID (*ixheaacd_real_synth_fft)(FLOAT32 *inp, FLOAT32 *out, WORD32 n_points);
-
-VOID (*ixheaacd_cmplx_anal_fft)(FLOAT32 *inp, FLOAT32 *out, WORD32 n_points);
-
 #endif
diff --git a/decoder/ixheaacd_sbr_dec.h b/decoder/ixheaacd_sbr_dec.h
index ac92ef5..0beec6d 100644
--- a/decoder/ixheaacd_sbr_dec.h
+++ b/decoder/ixheaacd_sbr_dec.h
@@ -48,6 +48,9 @@ typedef struct {
   FLOAT32 *analy_cos_sin_tab;
 
   FLOAT32 norm_qmf_in_buf[46][128];
+  VOID (*ixheaacd_real_synth_fft)(FLOAT32 *inp, FLOAT32 *out, WORD32 n_points);
+
+  VOID (*ixheaacd_cmplx_anal_fft)(FLOAT32 *inp, FLOAT32 *out, WORD32 n_points);
 
 } ia_esbr_hbe_txposer_struct;