libFLAC/lpc.c: Fix signed integer overflow
authorErik de Castro Lopo <erikd@mega-nerd.com>
Sun, 25 Aug 2019 07:59:02 +0000 (17:59 +1000)
committerErik de Castro Lopo <erikd@mega-nerd.com>
Sun, 15 Sep 2019 20:18:07 +0000 (06:18 +1000)
Do the addition as 64 bits before truncating to 32 bits.

Credit: Oss-Fuzz
Issue: https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=16459
Testcase: fuzzer_decoder-5728784602365952

src/libFLAC/lpc.c

index 0673895..9b80036 100644 (file)
@@ -1104,7 +1104,7 @@ void FLAC__lpc_restore_signal_wide(const FLAC__int32 * flac_restrict residual, u
                                                sum += qlp_coeff[2] * (FLAC__int64)data[i-3];
                                                sum += qlp_coeff[1] * (FLAC__int64)data[i-2];
                                                sum += qlp_coeff[0] * (FLAC__int64)data[i-1];
-                                               data[i] = residual[i] + (FLAC__int32)(sum >> lp_quantization);
+                                               data[i] = (FLAC__int32) (residual[i] + (sum >> lp_quantization));
                                        }
                                }
                                else { /* order == 11 */
@@ -1121,7 +1121,7 @@ void FLAC__lpc_restore_signal_wide(const FLAC__int32 * flac_restrict residual, u
                                                sum += qlp_coeff[2] * (FLAC__int64)data[i-3];
                                                sum += qlp_coeff[1] * (FLAC__int64)data[i-2];
                                                sum += qlp_coeff[0] * (FLAC__int64)data[i-1];
-                                               data[i] = residual[i] + (FLAC__int32)(sum >> lp_quantization);
+                                               data[i] = (FLAC__int32) (residual[i] + (sum >> lp_quantization));
                                        }
                                }
                        }
@@ -1139,7 +1139,7 @@ void FLAC__lpc_restore_signal_wide(const FLAC__int32 * flac_restrict residual, u
                                                sum += qlp_coeff[2] * (FLAC__int64)data[i-3];
                                                sum += qlp_coeff[1] * (FLAC__int64)data[i-2];
                                                sum += qlp_coeff[0] * (FLAC__int64)data[i-1];
-                                               data[i] = residual[i] + (FLAC__int32)(sum >> lp_quantization);
+                                               data[i] = (FLAC__int32) (residual[i] + (sum >> lp_quantization));
                                        }
                                }
                                else { /* order == 9 */
@@ -1154,7 +1154,7 @@ void FLAC__lpc_restore_signal_wide(const FLAC__int32 * flac_restrict residual, u
                                                sum += qlp_coeff[2] * (FLAC__int64)data[i-3];
                                                sum += qlp_coeff[1] * (FLAC__int64)data[i-2];
                                                sum += qlp_coeff[0] * (FLAC__int64)data[i-1];
-                                               data[i] = residual[i] + (FLAC__int32)(sum >> lp_quantization);
+                                               data[i] = (FLAC__int32) (residual[i] + (sum >> lp_quantization));
                                        }
                                }
                        }
@@ -1172,7 +1172,7 @@ void FLAC__lpc_restore_signal_wide(const FLAC__int32 * flac_restrict residual, u
                                                sum += qlp_coeff[2] * (FLAC__int64)data[i-3];
                                                sum += qlp_coeff[1] * (FLAC__int64)data[i-2];
                                                sum += qlp_coeff[0] * (FLAC__int64)data[i-1];
-                                               data[i] = residual[i] + (FLAC__int32)(sum >> lp_quantization);
+                                               data[i] = (FLAC__int32) (residual[i] + (sum >> lp_quantization));
                                        }
                                }
                                else { /* order == 7 */
@@ -1185,7 +1185,7 @@ void FLAC__lpc_restore_signal_wide(const FLAC__int32 * flac_restrict residual, u
                                                sum += qlp_coeff[2] * (FLAC__int64)data[i-3];
                                                sum += qlp_coeff[1] * (FLAC__int64)data[i-2];
                                                sum += qlp_coeff[0] * (FLAC__int64)data[i-1];
-                                               data[i] = residual[i] + (FLAC__int32)(sum >> lp_quantization);
+                                               data[i] = (FLAC__int32) (residual[i] + (sum >> lp_quantization));
                                        }
                                }
                        }
@@ -1199,7 +1199,7 @@ void FLAC__lpc_restore_signal_wide(const FLAC__int32 * flac_restrict residual, u
                                                sum += qlp_coeff[2] * (FLAC__int64)data[i-3];
                                                sum += qlp_coeff[1] * (FLAC__int64)data[i-2];
                                                sum += qlp_coeff[0] * (FLAC__int64)data[i-1];
-                                               data[i] = residual[i] + (FLAC__int32)(sum >> lp_quantization);
+                                               data[i] = (FLAC__int32) (residual[i] + (sum >> lp_quantization));
                                        }
                                }
                                else { /* order == 5 */
@@ -1210,7 +1210,7 @@ void FLAC__lpc_restore_signal_wide(const FLAC__int32 * flac_restrict residual, u
                                                sum += qlp_coeff[2] * (FLAC__int64)data[i-3];
                                                sum += qlp_coeff[1] * (FLAC__int64)data[i-2];
                                                sum += qlp_coeff[0] * (FLAC__int64)data[i-1];
-                                               data[i] = residual[i] + (FLAC__int32)(sum >> lp_quantization);
+                                               data[i] = (FLAC__int32) (residual[i] + (sum >> lp_quantization));
                                        }
                                }
                        }
@@ -1224,7 +1224,7 @@ void FLAC__lpc_restore_signal_wide(const FLAC__int32 * flac_restrict residual, u
                                                sum += qlp_coeff[2] * (FLAC__int64)data[i-3];
                                                sum += qlp_coeff[1] * (FLAC__int64)data[i-2];
                                                sum += qlp_coeff[0] * (FLAC__int64)data[i-1];
-                                               data[i] = residual[i] + (FLAC__int32)(sum >> lp_quantization);
+                                               data[i] = (FLAC__int32) (residual[i] + (sum >> lp_quantization));
                                        }
                                }
                                else { /* order == 3 */
@@ -1233,7 +1233,7 @@ void FLAC__lpc_restore_signal_wide(const FLAC__int32 * flac_restrict residual, u
                                                sum += qlp_coeff[2] * (FLAC__int64)data[i-3];
                                                sum += qlp_coeff[1] * (FLAC__int64)data[i-2];
                                                sum += qlp_coeff[0] * (FLAC__int64)data[i-1];
-                                               data[i] = residual[i] + (FLAC__int32)(sum >> lp_quantization);
+                                               data[i] = (FLAC__int32) (residual[i] + (sum >> lp_quantization));
                                        }
                                }
                        }
@@ -1243,7 +1243,7 @@ void FLAC__lpc_restore_signal_wide(const FLAC__int32 * flac_restrict residual, u
                                                sum = 0;
                                                sum += qlp_coeff[1] * (FLAC__int64)data[i-2];
                                                sum += qlp_coeff[0] * (FLAC__int64)data[i-1];
-                                               data[i] = residual[i] + (FLAC__int32)(sum >> lp_quantization);
+                                               data[i] = (FLAC__int32) (residual[i] + (sum >> lp_quantization));
                                        }
                                }
                                else { /* order == 1 */
@@ -1290,7 +1290,7 @@ void FLAC__lpc_restore_signal_wide(const FLAC__int32 * flac_restrict residual, u
                                         sum += qlp_coeff[ 1] * (FLAC__int64)data[i- 2];
                                         sum += qlp_coeff[ 0] * (FLAC__int64)data[i- 1];
                        }
-                       data[i] = residual[i] + (FLAC__int32)(sum >> lp_quantization);
+                       data[i] = (FLAC__int32) (residual[i] + (sum >> lp_quantization));
                }
        }
 }