Fix an overflow in chroma-from-luma.
authorTimothy B. Terriberry <tterribe@xiph.org>
Fri, 26 Apr 2013 20:45:09 +0000 (13:45 -0700)
committerTimothy B. Terriberry <tterribe@xiph.org>
Fri, 26 Apr 2013 20:45:09 +0000 (13:45 -0700)
This needs to be rewritten to avoid the 64-bit math, eventually,
 but promoting these multiplies solves the immediate problem.

Thanks to Greg Maxwell for assistance in tracking this down.

src/intra.c

index c954a41..984845a 100644 (file)
@@ -181,8 +181,8 @@ void od_chroma_pred4x4(od_coeff *_p,const od_coeff *_c,
       xy+=cc*(ogg_int64_t)cc*w_q8;
     }
   }
-  xx-=lc_sum_q8*lc_sum_q8+128>>8;
-  xy-=cc_sum_q8*lc_sum_q8+128>>8;
+  xx-=lc_sum_q8*(ogg_int64_t)lc_sum_q8+128>>8;
+  xy-=cc_sum_q8*(ogg_int64_t)lc_sum_q8+128>>8;
   if(abs(xx)>abs(xy)>>2)alpha_q8=(xy<<8)/xx;
   else alpha_q8=0;
   beta_q8=cc_sum_q8-(alpha_q8*lc_sum_q8+128>>8);