libFLAC/lpc.c: Fix signed integer overflow
[flac.git] / src / libFLAC / lpc_intrin_sse.c
1 /* libFLAC - Free Lossless Audio Codec library
2  * Copyright (C) 2000-2009  Josh Coalson
3  * Copyright (C) 2011-2016  Xiph.Org Foundation
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  *
9  * - Redistributions of source code must retain the above copyright
10  * notice, this list of conditions and the following disclaimer.
11  *
12  * - Redistributions in binary form must reproduce the above copyright
13  * notice, this list of conditions and the following disclaimer in the
14  * documentation and/or other materials provided with the distribution.
15  *
16  * - Neither the name of the Xiph.org Foundation nor the names of its
17  * contributors may be used to endorse or promote products derived from
18  * this software without specific prior written permission.
19  *
20  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
23  * A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR
24  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
25  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
26  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
27  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
28  * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
29  * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
30  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31  */
32
33 #ifdef HAVE_CONFIG_H
34 #  include <config.h>
35 #endif
36
37 #include "private/cpu.h"
38
39 #ifndef FLAC__INTEGER_ONLY_LIBRARY
40 #ifndef FLAC__NO_ASM
41 #if (defined FLAC__CPU_IA32 || defined FLAC__CPU_X86_64) && FLAC__HAS_X86INTRIN
42 #include "private/lpc.h"
43 #ifdef FLAC__SSE_SUPPORTED
44 #include "FLAC/assert.h"
45 #include "FLAC/format.h"
46
47 #include <xmmintrin.h> /* SSE */
48
49 /*   new routines: more unaligned loads, less shuffle
50  *   old routines: less unaligned loads, more shuffle
51  *   these *_old routines are equivalent to the ASM routines in ia32/lpc_asm.nasm
52  */
53
54 /* new routines: faster on current Intel (starting from Core i aka Nehalem) and all AMD CPUs */
55
56 FLAC__SSE_TARGET("sse")
57 void FLAC__lpc_compute_autocorrelation_intrin_sse_lag_4_new(const FLAC__real data[], uint32_t data_len, uint32_t lag, FLAC__real autoc[])
58 {
59         int i;
60         int limit = data_len - 4;
61         __m128 sum0;
62
63         (void) lag;
64         FLAC__ASSERT(lag <= 4);
65         FLAC__ASSERT(lag <= data_len);
66
67         sum0 = _mm_setzero_ps();
68
69         for(i = 0; i <= limit; i++) {
70                 __m128 d, d0;
71                 d0 = _mm_loadu_ps(data+i);
72                 d = _mm_shuffle_ps(d0, d0, 0);
73                 sum0 = _mm_add_ps(sum0, _mm_mul_ps(d0, d));
74         }
75
76         {
77                 __m128 d0 = _mm_setzero_ps();
78                 limit++; if(limit < 0) limit = 0;
79
80                 for(i = data_len-1; i >= limit; i--) {
81                         __m128 d;
82                         d = _mm_load_ss(data+i); d = _mm_shuffle_ps(d, d, 0);
83                         d0 = _mm_shuffle_ps(d0, d0, _MM_SHUFFLE(2,1,0,3));
84                         d0 = _mm_move_ss(d0, d);
85                         sum0 = _mm_add_ps(sum0, _mm_mul_ps(d, d0));
86                 }
87         }
88
89         _mm_storeu_ps(autoc,   sum0);
90 }
91
92 FLAC__SSE_TARGET("sse")
93 void FLAC__lpc_compute_autocorrelation_intrin_sse_lag_8_new(const FLAC__real data[], uint32_t data_len, uint32_t lag, FLAC__real autoc[])
94 {
95         int i;
96         int limit = data_len - 8;
97         __m128 sum0, sum1;
98
99         (void) lag;
100         FLAC__ASSERT(lag <= 8);
101         FLAC__ASSERT(lag <= data_len);
102
103         sum0 = _mm_setzero_ps();
104         sum1 = _mm_setzero_ps();
105
106         for(i = 0; i <= limit; i++) {
107                 __m128 d, d0, d1;
108                 d0 = _mm_loadu_ps(data+i);
109                 d1 = _mm_loadu_ps(data+i+4);
110                 d = _mm_shuffle_ps(d0, d0, 0);
111                 sum0 = _mm_add_ps(sum0, _mm_mul_ps(d0, d));
112                 sum1 = _mm_add_ps(sum1, _mm_mul_ps(d1, d));
113         }
114
115         {
116                 __m128 d0 = _mm_setzero_ps();
117                 __m128 d1 = _mm_setzero_ps();
118                 limit++; if(limit < 0) limit = 0;
119
120                 for(i = data_len-1; i >= limit; i--) {
121                         __m128 d;
122                         d = _mm_load_ss(data+i); d = _mm_shuffle_ps(d, d, 0);
123                         d1 = _mm_shuffle_ps(d1, d1, _MM_SHUFFLE(2,1,0,3));
124                         d0 = _mm_shuffle_ps(d0, d0, _MM_SHUFFLE(2,1,0,3));
125                         d1 = _mm_move_ss(d1, d0);
126                         d0 = _mm_move_ss(d0, d);
127                         sum1 = _mm_add_ps(sum1, _mm_mul_ps(d, d1));
128                         sum0 = _mm_add_ps(sum0, _mm_mul_ps(d, d0));
129                 }
130         }
131
132         _mm_storeu_ps(autoc,   sum0);
133         _mm_storeu_ps(autoc+4, sum1);
134 }
135
136 FLAC__SSE_TARGET("sse")
137 void FLAC__lpc_compute_autocorrelation_intrin_sse_lag_12_new(const FLAC__real data[], uint32_t data_len, uint32_t lag, FLAC__real autoc[])
138 {
139         int i;
140         int limit = data_len - 12;
141         __m128 sum0, sum1, sum2;
142
143         (void) lag;
144         FLAC__ASSERT(lag <= 12);
145         FLAC__ASSERT(lag <= data_len);
146
147         sum0 = _mm_setzero_ps();
148         sum1 = _mm_setzero_ps();
149         sum2 = _mm_setzero_ps();
150
151         for(i = 0; i <= limit; i++) {
152                 __m128 d, d0, d1, d2;
153                 d0 = _mm_loadu_ps(data+i);
154                 d1 = _mm_loadu_ps(data+i+4);
155                 d2 = _mm_loadu_ps(data+i+8);
156                 d = _mm_shuffle_ps(d0, d0, 0);
157                 sum0 = _mm_add_ps(sum0, _mm_mul_ps(d0, d));
158                 sum1 = _mm_add_ps(sum1, _mm_mul_ps(d1, d));
159                 sum2 = _mm_add_ps(sum2, _mm_mul_ps(d2, d));
160         }
161
162         {
163                 __m128 d0 = _mm_setzero_ps();
164                 __m128 d1 = _mm_setzero_ps();
165                 __m128 d2 = _mm_setzero_ps();
166                 limit++; if(limit < 0) limit = 0;
167
168                 for(i = data_len-1; i >= limit; i--) {
169                         __m128 d;
170                         d = _mm_load_ss(data+i); d = _mm_shuffle_ps(d, d, 0);
171                         d2 = _mm_shuffle_ps(d2, d2, _MM_SHUFFLE(2,1,0,3));
172                         d1 = _mm_shuffle_ps(d1, d1, _MM_SHUFFLE(2,1,0,3));
173                         d0 = _mm_shuffle_ps(d0, d0, _MM_SHUFFLE(2,1,0,3));
174                         d2 = _mm_move_ss(d2, d1);
175                         d1 = _mm_move_ss(d1, d0);
176                         d0 = _mm_move_ss(d0, d);
177                         sum2 = _mm_add_ps(sum2, _mm_mul_ps(d, d2));
178                         sum1 = _mm_add_ps(sum1, _mm_mul_ps(d, d1));
179                         sum0 = _mm_add_ps(sum0, _mm_mul_ps(d, d0));
180                 }
181         }
182
183         _mm_storeu_ps(autoc,   sum0);
184         _mm_storeu_ps(autoc+4, sum1);
185         _mm_storeu_ps(autoc+8, sum2);
186 }
187
188 FLAC__SSE_TARGET("sse")
189 void FLAC__lpc_compute_autocorrelation_intrin_sse_lag_16_new(const FLAC__real data[], uint32_t data_len, uint32_t lag, FLAC__real autoc[])
190 {
191         int i;
192         int limit = data_len - 16;
193         __m128 sum0, sum1, sum2, sum3;
194
195         (void) lag;
196         FLAC__ASSERT(lag <= 16);
197         FLAC__ASSERT(lag <= data_len);
198
199         sum0 = _mm_setzero_ps();
200         sum1 = _mm_setzero_ps();
201         sum2 = _mm_setzero_ps();
202         sum3 = _mm_setzero_ps();
203
204         for(i = 0; i <= limit; i++) {
205                 __m128 d, d0, d1, d2, d3;
206                 d0 = _mm_loadu_ps(data+i);
207                 d1 = _mm_loadu_ps(data+i+4);
208                 d2 = _mm_loadu_ps(data+i+8);
209                 d3 = _mm_loadu_ps(data+i+12);
210                 d = _mm_shuffle_ps(d0, d0, 0);
211                 sum0 = _mm_add_ps(sum0, _mm_mul_ps(d0, d));
212                 sum1 = _mm_add_ps(sum1, _mm_mul_ps(d1, d));
213                 sum2 = _mm_add_ps(sum2, _mm_mul_ps(d2, d));
214                 sum3 = _mm_add_ps(sum3, _mm_mul_ps(d3, d));
215         }
216
217         {
218                 __m128 d0 = _mm_setzero_ps();
219                 __m128 d1 = _mm_setzero_ps();
220                 __m128 d2 = _mm_setzero_ps();
221                 __m128 d3 = _mm_setzero_ps();
222                 limit++; if(limit < 0) limit = 0;
223
224                 for(i = data_len-1; i >= limit; i--) {
225                         __m128 d;
226                         d = _mm_load_ss(data+i); d = _mm_shuffle_ps(d, d, 0);
227                         d3 = _mm_shuffle_ps(d3, d3, _MM_SHUFFLE(2,1,0,3));
228                         d2 = _mm_shuffle_ps(d2, d2, _MM_SHUFFLE(2,1,0,3));
229                         d1 = _mm_shuffle_ps(d1, d1, _MM_SHUFFLE(2,1,0,3));
230                         d0 = _mm_shuffle_ps(d0, d0, _MM_SHUFFLE(2,1,0,3));
231                         d3 = _mm_move_ss(d3, d2);
232                         d2 = _mm_move_ss(d2, d1);
233                         d1 = _mm_move_ss(d1, d0);
234                         d0 = _mm_move_ss(d0, d);
235                         sum3 = _mm_add_ps(sum3, _mm_mul_ps(d, d3));
236                         sum2 = _mm_add_ps(sum2, _mm_mul_ps(d, d2));
237                         sum1 = _mm_add_ps(sum1, _mm_mul_ps(d, d1));
238                         sum0 = _mm_add_ps(sum0, _mm_mul_ps(d, d0));
239                 }
240         }
241
242         _mm_storeu_ps(autoc,   sum0);
243         _mm_storeu_ps(autoc+4, sum1);
244         _mm_storeu_ps(autoc+8, sum2);
245         _mm_storeu_ps(autoc+12,sum3);
246 }
247
248 /* old routines: faster on older Intel CPUs (up to Core 2) */
249
250 FLAC__SSE_TARGET("sse")
251 void FLAC__lpc_compute_autocorrelation_intrin_sse_lag_4_old(const FLAC__real data[], uint32_t data_len, uint32_t lag, FLAC__real autoc[])
252 {
253         __m128 xmm0, xmm2, xmm5;
254
255         (void) lag;
256         FLAC__ASSERT(lag > 0);
257         FLAC__ASSERT(lag <= 4);
258         FLAC__ASSERT(lag <= data_len);
259         FLAC__ASSERT(data_len > 0);
260
261         xmm5 = _mm_setzero_ps();
262
263         xmm0 = _mm_load_ss(data++);
264         xmm2 = xmm0;
265         xmm0 = _mm_shuffle_ps(xmm0, xmm0, 0);
266
267         xmm0 = _mm_mul_ps(xmm0, xmm2);
268         xmm5 = _mm_add_ps(xmm5, xmm0);
269
270         data_len--;
271
272         while(data_len)
273         {
274                 xmm0 = _mm_load1_ps(data++);
275
276                 xmm2 = _mm_shuffle_ps(xmm2, xmm2, _MM_SHUFFLE(2,1,0,3));
277                 xmm2 = _mm_move_ss(xmm2, xmm0);
278                 xmm0 = _mm_mul_ps(xmm0, xmm2);
279                 xmm5 = _mm_add_ps(xmm5, xmm0);
280
281                 data_len--;
282         }
283
284         _mm_storeu_ps(autoc, xmm5);
285 }
286
287 FLAC__SSE_TARGET("sse")
288 void FLAC__lpc_compute_autocorrelation_intrin_sse_lag_8_old(const FLAC__real data[], uint32_t data_len, uint32_t lag, FLAC__real autoc[])
289 {
290         __m128 xmm0, xmm1, xmm2, xmm3, xmm5, xmm6;
291
292         (void) lag;
293         FLAC__ASSERT(lag > 0);
294         FLAC__ASSERT(lag <= 8);
295         FLAC__ASSERT(lag <= data_len);
296         FLAC__ASSERT(data_len > 0);
297
298         xmm5 = _mm_setzero_ps();
299         xmm6 = _mm_setzero_ps();
300
301         xmm0 = _mm_load_ss(data++);
302         xmm2 = xmm0;
303         xmm0 = _mm_shuffle_ps(xmm0, xmm0, 0);
304         xmm3 = _mm_setzero_ps();
305
306         xmm0 = _mm_mul_ps(xmm0, xmm2);
307         xmm5 = _mm_add_ps(xmm5, xmm0);
308
309         data_len--;
310
311         while(data_len)
312         {
313                 xmm0 = _mm_load1_ps(data++);
314
315                 xmm2 = _mm_shuffle_ps(xmm2, xmm2, _MM_SHUFFLE(2,1,0,3));
316                 xmm3 = _mm_shuffle_ps(xmm3, xmm3, _MM_SHUFFLE(2,1,0,3));
317                 xmm3 = _mm_move_ss(xmm3, xmm2);
318                 xmm2 = _mm_move_ss(xmm2, xmm0);
319
320                 xmm1 = xmm0;
321                 xmm1 = _mm_mul_ps(xmm1, xmm3);
322                 xmm0 = _mm_mul_ps(xmm0, xmm2);
323                 xmm6 = _mm_add_ps(xmm6, xmm1);
324                 xmm5 = _mm_add_ps(xmm5, xmm0);
325
326                 data_len--;
327         }
328
329         _mm_storeu_ps(autoc,   xmm5);
330         _mm_storeu_ps(autoc+4, xmm6);
331 }
332
333 FLAC__SSE_TARGET("sse")
334 void FLAC__lpc_compute_autocorrelation_intrin_sse_lag_12_old(const FLAC__real data[], uint32_t data_len, uint32_t lag, FLAC__real autoc[])
335 {
336         __m128 xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7;
337
338         (void) lag;
339         FLAC__ASSERT(lag > 0);
340         FLAC__ASSERT(lag <= 12);
341         FLAC__ASSERT(lag <= data_len);
342         FLAC__ASSERT(data_len > 0);
343
344         xmm5 = _mm_setzero_ps();
345         xmm6 = _mm_setzero_ps();
346         xmm7 = _mm_setzero_ps();
347
348         xmm0 = _mm_load_ss(data++);
349         xmm2 = xmm0;
350         xmm0 = _mm_shuffle_ps(xmm0, xmm0, 0);
351         xmm3 = _mm_setzero_ps();
352         xmm4 = _mm_setzero_ps();
353
354         xmm0 = _mm_mul_ps(xmm0, xmm2);
355         xmm5 = _mm_add_ps(xmm5, xmm0);
356
357         data_len--;
358
359         while(data_len)
360         {
361                 xmm0 = _mm_load1_ps(data++);
362
363                 xmm2 = _mm_shuffle_ps(xmm2, xmm2, _MM_SHUFFLE(2,1,0,3));
364                 xmm3 = _mm_shuffle_ps(xmm3, xmm3, _MM_SHUFFLE(2,1,0,3));
365                 xmm4 = _mm_shuffle_ps(xmm4, xmm4, _MM_SHUFFLE(2,1,0,3));
366                 xmm4 = _mm_move_ss(xmm4, xmm3);
367                 xmm3 = _mm_move_ss(xmm3, xmm2);
368                 xmm2 = _mm_move_ss(xmm2, xmm0);
369
370                 xmm1 = xmm0;
371                 xmm1 = _mm_mul_ps(xmm1, xmm2);
372                 xmm5 = _mm_add_ps(xmm5, xmm1);
373                 xmm1 = xmm0;
374                 xmm1 = _mm_mul_ps(xmm1, xmm3);
375                 xmm6 = _mm_add_ps(xmm6, xmm1);
376                 xmm0 = _mm_mul_ps(xmm0, xmm4);
377                 xmm7 = _mm_add_ps(xmm7, xmm0);
378
379                 data_len--;
380         }
381
382         _mm_storeu_ps(autoc,   xmm5);
383         _mm_storeu_ps(autoc+4, xmm6);
384         _mm_storeu_ps(autoc+8, xmm7);
385 }
386
387 FLAC__SSE_TARGET("sse")
388 void FLAC__lpc_compute_autocorrelation_intrin_sse_lag_16_old(const FLAC__real data[], uint32_t data_len, uint32_t lag, FLAC__real autoc[])
389 {
390         __m128 xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm9;
391
392         (void) lag;
393         FLAC__ASSERT(lag > 0);
394         FLAC__ASSERT(lag <= 16);
395         FLAC__ASSERT(lag <= data_len);
396         FLAC__ASSERT(data_len > 0);
397
398         xmm6 = _mm_setzero_ps();
399         xmm7 = _mm_setzero_ps();
400         xmm8 = _mm_setzero_ps();
401         xmm9 = _mm_setzero_ps();
402
403         xmm0 = _mm_load_ss(data++);
404         xmm2 = xmm0;
405         xmm0 = _mm_shuffle_ps(xmm0, xmm0, 0);
406         xmm3 = _mm_setzero_ps();
407         xmm4 = _mm_setzero_ps();
408         xmm5 = _mm_setzero_ps();
409
410         xmm0 = _mm_mul_ps(xmm0, xmm2);
411         xmm6 = _mm_add_ps(xmm6, xmm0);
412
413         data_len--;
414
415         while(data_len)
416         {
417                 xmm0 = _mm_load1_ps(data++);
418
419                 /* shift xmm5:xmm4:xmm3:xmm2 left by one float */
420                 xmm5 = _mm_shuffle_ps(xmm5, xmm5, _MM_SHUFFLE(2,1,0,3));
421                 xmm4 = _mm_shuffle_ps(xmm4, xmm4, _MM_SHUFFLE(2,1,0,3));
422                 xmm3 = _mm_shuffle_ps(xmm3, xmm3, _MM_SHUFFLE(2,1,0,3));
423                 xmm2 = _mm_shuffle_ps(xmm2, xmm2, _MM_SHUFFLE(2,1,0,3));
424                 xmm5 = _mm_move_ss(xmm5, xmm4);
425                 xmm4 = _mm_move_ss(xmm4, xmm3);
426                 xmm3 = _mm_move_ss(xmm3, xmm2);
427                 xmm2 = _mm_move_ss(xmm2, xmm0);
428
429                 /* xmm9|xmm8|xmm7|xmm6 += xmm0|xmm0|xmm0|xmm0 * xmm5|xmm4|xmm3|xmm2 */
430                 xmm1 = xmm0;
431                 xmm1 = _mm_mul_ps(xmm1, xmm5);
432                 xmm9 = _mm_add_ps(xmm9, xmm1);
433                 xmm1 = xmm0;
434                 xmm1 = _mm_mul_ps(xmm1, xmm4);
435                 xmm8 = _mm_add_ps(xmm8, xmm1);
436                 xmm1 = xmm0;
437                 xmm1 = _mm_mul_ps(xmm1, xmm3);
438                 xmm7 = _mm_add_ps(xmm7, xmm1);
439                 xmm0 = _mm_mul_ps(xmm0, xmm2);
440                 xmm6 = _mm_add_ps(xmm6, xmm0);
441
442                 data_len--;
443         }
444
445         _mm_storeu_ps(autoc,   xmm6);
446         _mm_storeu_ps(autoc+4, xmm7);
447         _mm_storeu_ps(autoc+8, xmm8);
448         _mm_storeu_ps(autoc+12,xmm9);
449 }
450
451 #endif /* FLAC__SSE_SUPPORTED */
452 #endif /* (FLAC__CPU_IA32 || FLAC__CPU_X86_64) && FLAC__HAS_X86INTRIN */
453 #endif /* FLAC__NO_ASM */
454 #endif /* FLAC__INTEGER_ONLY_LIBRARY */