remove bps limit on mid-side coding
[flac.git] / src / libFLAC / encoder.c
1 /* libFLAC - Free Lossless Audio Codec library
2  * Copyright (C) 2000,2001  Josh Coalson
3  *
4  * This library is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU Library General Public
6  * License as published by the Free Software Foundation; either
7  * version 2 of the License, or (at your option) any later version.
8  *
9  * This library is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
12  * Library General Public License for more details.
13  *
14  * You should have received a copy of the GNU Library General Public
15  * License along with this library; if not, write to the
16  * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
17  * Boston, MA  02111-1307, USA.
18  */
19
20 #include <assert.h>
21 #include <stdio.h>
22 #include <stdlib.h> /* for malloc() */
23 #include <string.h> /* for memcpy() */
24 #include "FLAC/encoder.h"
25 #include "private/bitbuffer.h"
26 #include "private/encoder_framing.h"
27 #include "private/fixed.h"
28 #include "private/lpc.h"
29 #include "private/md5.h"
30
31 #ifdef min
32 #undef min
33 #endif
34 #define min(x,y) ((x)<(y)?(x):(y))
35
36 #ifdef max
37 #undef max
38 #endif
39 #define max(x,y) ((x)>(y)?(x):(y))
40
41 typedef struct FLAC__EncoderPrivate {
42         unsigned input_capacity;                    /* current size (in samples) of the signal and residual buffers */
43         int32 *integer_signal[FLAC__MAX_CHANNELS];  /* the integer version of the input signal */
44         int32 *integer_signal_mid_side[2];          /* the integer version of the mid-side input signal (stereo only) */
45         real *real_signal[FLAC__MAX_CHANNELS];      /* the floating-point version of the input signal */
46         real *real_signal_mid_side[2];              /* the floating-point version of the mid-side input signal (stereo only) */
47         int32 *residual_workspace[FLAC__MAX_CHANNELS][2]; /* each channel has a candidate and best workspace where the subframe residual signals will be stored */
48         int32 *residual_workspace_mid_side[2][2];
49         FLAC__Subframe subframe_workspace[FLAC__MAX_CHANNELS][2];
50         FLAC__Subframe subframe_workspace_mid_side[2][2];
51         FLAC__Subframe *subframe_workspace_ptr[FLAC__MAX_CHANNELS][2];
52         FLAC__Subframe *subframe_workspace_ptr_mid_side[2][2];
53         unsigned best_subframe[FLAC__MAX_CHANNELS]; /* index into the above workspaces */
54         unsigned best_subframe_mid_side[2];
55         unsigned best_subframe_bits[FLAC__MAX_CHANNELS]; /* size in bits of the best subframe for each channel */
56         unsigned best_subframe_bits_mid_side[2];
57         uint32 *abs_residual;                       /* workspace where the abs(candidate residual) is stored */
58         FLAC__BitBuffer frame;                      /* the current frame being worked on */
59         bool current_frame_can_do_mid_side;         /* encoder sets this false when any given sample of a frame's side channel exceeds 16 bits */
60         double loose_mid_side_stereo_frames_exact;  /* exact number of frames the encoder will use before trying both independent and mid/side frames again */
61         unsigned loose_mid_side_stereo_frames;      /* rounded number of frames the encoder will use before trying both independent and mid/side frames again */
62         unsigned loose_mid_side_stereo_frame_count; /* number of frames using the current channel assignment */
63         FLAC__ChannelAssignment last_channel_assignment;
64         FLAC__StreamMetaData metadata;
65         unsigned current_sample_number;
66         unsigned current_frame_number;
67         struct MD5Context md5context;
68         FLAC__EncoderWriteStatus (*write_callback)(const FLAC__Encoder *encoder, const byte buffer[], unsigned bytes, unsigned samples, unsigned current_frame, void *client_data);
69         void (*metadata_callback)(const FLAC__Encoder *encoder, const FLAC__StreamMetaData *metadata, void *client_data);
70         void *client_data;
71 } FLAC__EncoderPrivate;
72
73 static bool encoder_resize_buffers_(FLAC__Encoder *encoder, unsigned new_size);
74 static bool encoder_process_frame_(FLAC__Encoder *encoder, bool is_last_frame);
75 static bool encoder_process_subframes_(FLAC__Encoder *encoder, bool is_last_frame);
76 static bool encoder_process_subframe_(FLAC__Encoder *encoder, unsigned max_partition_order, bool verbatim_only, const FLAC__FrameHeader *frame_header, unsigned bits_per_sample, const int32 integer_signal[], const real real_signal[], FLAC__Subframe *subframe[2], int32 *residual[2], unsigned *best_subframe, unsigned *best_bits);
77 static bool encoder_add_subframe_(FLAC__Encoder *encoder, const FLAC__FrameHeader *frame_header, unsigned bits_per_sample, const FLAC__Subframe *subframe, FLAC__BitBuffer *frame);
78 static unsigned encoder_evaluate_constant_subframe_(const int32 signal, unsigned bits_per_sample, FLAC__Subframe *subframe);
79 static unsigned encoder_evaluate_fixed_subframe_(const int32 signal[], int32 residual[], uint32 abs_residual[], unsigned blocksize, unsigned bits_per_sample, unsigned order, unsigned rice_parameter, unsigned max_partition_order, FLAC__Subframe *subframe);
80 static unsigned encoder_evaluate_lpc_subframe_(const int32 signal[], int32 residual[], uint32 abs_residual[], const real lp_coeff[], unsigned blocksize, unsigned bits_per_sample, unsigned order, unsigned qlp_coeff_precision, unsigned rice_parameter, unsigned max_partition_order, FLAC__Subframe *subframe);
81 static unsigned encoder_evaluate_verbatim_subframe_(const int32 signal[], unsigned blocksize, unsigned bits_per_sample, FLAC__Subframe *subframe);
82 static unsigned encoder_find_best_partition_order_(const int32 residual[], uint32 abs_residual[], unsigned residual_samples, unsigned predictor_order, unsigned rice_parameter, unsigned max_partition_order, unsigned *best_partition_order, unsigned best_parameters[]);
83 static bool encoder_set_partitioned_rice_(const uint32 abs_residual[], const unsigned residual_samples, const unsigned predictor_order, const unsigned rice_parameter, const unsigned partition_order, unsigned parameters[], unsigned *bits);
84
85 const char *FLAC__EncoderWriteStatusString[] = {
86         "FLAC__ENCODER_WRITE_OK",
87         "FLAC__ENCODER_WRITE_FATAL_ERROR"
88 };
89
90 const char *FLAC__EncoderStateString[] = {
91         "FLAC__ENCODER_OK",
92         "FLAC__ENCODER_UNINITIALIZED",
93         "FLAC__ENCODER_INVALID_NUMBER_OF_CHANNELS",
94         "FLAC__ENCODER_INVALID_BITS_PER_SAMPLE",
95         "FLAC__ENCODER_INVALID_SAMPLE_RATE",
96         "FLAC__ENCODER_INVALID_BLOCK_SIZE",
97         "FLAC__ENCODER_INVALID_QLP_COEFF_PRECISION",
98         "FLAC__ENCODER_MID_SIDE_CHANNELS_MISMATCH",
99         "FLAC__ENCODER_MID_SIDE_SAMPLE_SIZE_MISMATCH",
100         "FLAC__ENCODER_ILLEGAL_MID_SIDE_FORCE",
101         "FLAC__ENCODER_BLOCK_SIZE_TOO_SMALL_FOR_LPC_ORDER",
102         "FLAC__ENCODER_NOT_STREAMABLE",
103         "FLAC__ENCODER_FRAMING_ERROR",
104         "FLAC__ENCODER_FATAL_ERROR_WHILE_ENCODING",
105         "FLAC__ENCODER_FATAL_ERROR_WHILE_WRITING",
106         "FLAC__ENCODER_MEMORY_ALLOCATION_ERROR"
107 };
108
109
110 bool encoder_resize_buffers_(FLAC__Encoder *encoder, unsigned new_size)
111 {
112         bool ok;
113         unsigned i, channel;
114         int32 *previous_is, *current_is;
115         real *previous_rs, *current_rs;
116         int32 *residual;
117         uint32 *abs_residual;
118
119         assert(new_size > 0);
120         assert(encoder->state == FLAC__ENCODER_OK);
121         assert(encoder->guts->current_sample_number == 0);
122
123         /* To avoid excessive malloc'ing, we only grow the buffer; no shrinking. */
124         if(new_size <= encoder->guts->input_capacity)
125                 return true;
126
127         ok = 1;
128         if(ok) {
129                 for(i = 0; ok && i < encoder->channels; i++) {
130                         /* integer version of the signal */
131                         previous_is = encoder->guts->integer_signal[i];
132                         current_is = (int32*)malloc(sizeof(int32) * new_size);
133                         if(0 == current_is) {
134                                 encoder->state = FLAC__ENCODER_MEMORY_ALLOCATION_ERROR;
135                                 ok = 0;
136                         }
137                         else {
138                                 encoder->guts->integer_signal[i] = current_is;
139                                 if(previous_is != 0)
140                                         free(previous_is);
141                         }
142                         /* real version of the signal */
143                         previous_rs = encoder->guts->real_signal[i];
144                         current_rs = (real*)malloc(sizeof(real) * new_size);
145                         if(0 == current_rs) {
146                                 encoder->state = FLAC__ENCODER_MEMORY_ALLOCATION_ERROR;
147                                 ok = 0;
148                         }
149                         else {
150                                 encoder->guts->real_signal[i] = current_rs;
151                                 if(previous_rs != 0)
152                                         free(previous_rs);
153                         }
154                 }
155         }
156         if(ok) {
157                 for(i = 0; ok && i < 2; i++) {
158                         /* integer version of the signal */
159                         previous_is = encoder->guts->integer_signal_mid_side[i];
160                         current_is = (int32*)malloc(sizeof(int32) * new_size);
161                         if(0 == current_is) {
162                                 encoder->state = FLAC__ENCODER_MEMORY_ALLOCATION_ERROR;
163                                 ok = 0;
164                         }
165                         else {
166                                 encoder->guts->integer_signal_mid_side[i] = current_is;
167                                 if(previous_is != 0)
168                                         free(previous_is);
169                         }
170                         /* real version of the signal */
171                         previous_rs = encoder->guts->real_signal_mid_side[i];
172                         current_rs = (real*)malloc(sizeof(real) * new_size);
173                         if(0 == current_rs) {
174                                 encoder->state = FLAC__ENCODER_MEMORY_ALLOCATION_ERROR;
175                                 ok = 0;
176                         }
177                         else {
178                                 encoder->guts->real_signal_mid_side[i] = current_rs;
179                                 if(previous_rs != 0)
180                                         free(previous_rs);
181                         }
182                 }
183         }
184         if(ok) {
185                 for(channel = 0; channel < encoder->channels; channel++) {
186                         for(i = 0; i < 2; i++) {
187                                 residual = (int32*)malloc(sizeof(int32) * new_size);
188                                 if(0 == residual) {
189                                         encoder->state = FLAC__ENCODER_MEMORY_ALLOCATION_ERROR;
190                                         ok = 0;
191                                 }
192                                 else {
193                                         if(encoder->guts->residual_workspace[channel][i] != 0)
194                                                 free(encoder->guts->residual_workspace[channel][i]);
195                                         encoder->guts->residual_workspace[channel][i] = residual;
196                                 }
197                         }
198                 }
199                 for(channel = 0; channel < 2; channel++) {
200                         for(i = 0; i < 2; i++) {
201                                 residual = (int32*)malloc(sizeof(int32) * new_size);
202                                 if(0 == residual) {
203                                         encoder->state = FLAC__ENCODER_MEMORY_ALLOCATION_ERROR;
204                                         ok = 0;
205                                 }
206                                 else {
207                                         if(encoder->guts->residual_workspace_mid_side[channel][i] != 0)
208                                                 free(encoder->guts->residual_workspace_mid_side[channel][i]);
209                                         encoder->guts->residual_workspace_mid_side[channel][i] = residual;
210                                 }
211                         }
212                 }
213                 abs_residual = (uint32*)malloc(sizeof(uint32) * new_size);
214                 if(0 == residual) {
215                         encoder->state = FLAC__ENCODER_MEMORY_ALLOCATION_ERROR;
216                         ok = 0;
217                 }
218                 else {
219                         if(encoder->guts->abs_residual != 0)
220                                 free(encoder->guts->abs_residual);
221                         encoder->guts->abs_residual = abs_residual;
222                 }
223         }
224         if(ok)
225                 encoder->guts->input_capacity = new_size;
226
227         return ok;
228 }
229
230 FLAC__Encoder *FLAC__encoder_get_new_instance()
231 {
232         FLAC__Encoder *encoder = (FLAC__Encoder*)malloc(sizeof(FLAC__Encoder));
233         if(encoder != 0) {
234                 encoder->state = FLAC__ENCODER_UNINITIALIZED;
235                 encoder->guts = 0;
236         }
237         return encoder;
238 }
239
240 void FLAC__encoder_free_instance(FLAC__Encoder *encoder)
241 {
242         assert(encoder != 0);
243         free(encoder);
244 }
245
246 FLAC__EncoderState FLAC__encoder_init(FLAC__Encoder *encoder, FLAC__EncoderWriteStatus (*write_callback)(const FLAC__Encoder *encoder, const byte buffer[], unsigned bytes, unsigned samples, unsigned current_frame, void *client_data), void (*metadata_callback)(const FLAC__Encoder *encoder, const FLAC__StreamMetaData *metadata, void *client_data), void *client_data)
247 {
248         unsigned i;
249         FLAC__StreamMetaData padding;
250
251         assert(sizeof(int) >= 4); /* we want to die right away if this is not true */
252         assert(encoder != 0);
253         assert(write_callback != 0);
254         assert(metadata_callback != 0);
255         assert(encoder->state == FLAC__ENCODER_UNINITIALIZED);
256         assert(encoder->guts == 0);
257
258         encoder->state = FLAC__ENCODER_OK;
259
260         if(encoder->channels == 0 || encoder->channels > FLAC__MAX_CHANNELS)
261                 return encoder->state = FLAC__ENCODER_INVALID_NUMBER_OF_CHANNELS;
262
263         if(encoder->do_mid_side_stereo && encoder->channels != 2)
264                 return encoder->state = FLAC__ENCODER_MID_SIDE_CHANNELS_MISMATCH;
265
266         if(encoder->loose_mid_side_stereo && !encoder->do_mid_side_stereo)
267                 return encoder->state = FLAC__ENCODER_ILLEGAL_MID_SIDE_FORCE;
268
269         if(encoder->bits_per_sample == 0 || encoder->bits_per_sample > FLAC__MAX_BITS_PER_SAMPLE)
270                 return encoder->state = FLAC__ENCODER_INVALID_BITS_PER_SAMPLE;
271
272         if(encoder->sample_rate == 0 || encoder->sample_rate > FLAC__MAX_SAMPLE_RATE)
273                 return encoder->state = FLAC__ENCODER_INVALID_SAMPLE_RATE;
274
275         if(encoder->blocksize < FLAC__MIN_BLOCK_SIZE || encoder->blocksize > FLAC__MAX_BLOCK_SIZE)
276                 return encoder->state = FLAC__ENCODER_INVALID_BLOCK_SIZE;
277
278         if(encoder->blocksize < encoder->max_lpc_order)
279                 return encoder->state = FLAC__ENCODER_BLOCK_SIZE_TOO_SMALL_FOR_LPC_ORDER;
280
281         if(encoder->qlp_coeff_precision == 0) {
282                 if(encoder->bits_per_sample < 16) {
283                         /* @@@ need some data about how to set this here w.r.t. blocksize and sample rate */
284                         /* @@@ until then we'll make a guess */
285                         encoder->qlp_coeff_precision = max(5, 2 + encoder->bits_per_sample / 2);
286                 }
287                 else if(encoder->bits_per_sample == 16) {
288                         if(encoder->blocksize <= 192)
289                                 encoder->qlp_coeff_precision = 7;
290                         else if(encoder->blocksize <= 384)
291                                 encoder->qlp_coeff_precision = 8;
292                         else if(encoder->blocksize <= 576)
293                                 encoder->qlp_coeff_precision = 9;
294                         else if(encoder->blocksize <= 1152)
295                                 encoder->qlp_coeff_precision = 10;
296                         else if(encoder->blocksize <= 2304)
297                                 encoder->qlp_coeff_precision = 11;
298                         else if(encoder->blocksize <= 4608)
299                                 encoder->qlp_coeff_precision = 12;
300                         else
301                                 encoder->qlp_coeff_precision = 13;
302                 }
303                 else {
304                         encoder->qlp_coeff_precision = min(13, 8*sizeof(int32) - encoder->bits_per_sample - 1);
305                 }
306         }
307         else if(encoder->qlp_coeff_precision < FLAC__MIN_QLP_COEFF_PRECISION || encoder->qlp_coeff_precision + encoder->bits_per_sample >= 8*sizeof(uint32) || encoder->qlp_coeff_precision >= (1u<<FLAC__SUBFRAME_LPC_QLP_COEFF_PRECISION_LEN))
308                 return encoder->state = FLAC__ENCODER_INVALID_QLP_COEFF_PRECISION;
309
310         if(encoder->streamable_subset) {
311                 if(encoder->bits_per_sample != 8 && encoder->bits_per_sample != 12 && encoder->bits_per_sample != 16 && encoder->bits_per_sample != 20 && encoder->bits_per_sample != 24)
312                         return encoder->state = FLAC__ENCODER_NOT_STREAMABLE;
313                 if(encoder->sample_rate > 655350)
314                         return encoder->state = FLAC__ENCODER_NOT_STREAMABLE;
315         }
316
317         if(encoder->rice_optimization_level >= (1u << FLAC__ENTROPY_CODING_METHOD_PARTITIONED_RICE_ORDER_LEN))
318                 encoder->rice_optimization_level = (1u << FLAC__ENTROPY_CODING_METHOD_PARTITIONED_RICE_ORDER_LEN) - 1;
319
320         encoder->guts = (FLAC__EncoderPrivate*)malloc(sizeof(FLAC__EncoderPrivate));
321         if(encoder->guts == 0)
322                 return encoder->state = FLAC__ENCODER_MEMORY_ALLOCATION_ERROR;
323
324         encoder->guts->input_capacity = 0;
325         for(i = 0; i < encoder->channels; i++) {
326                 encoder->guts->integer_signal[i] = 0;
327                 encoder->guts->real_signal[i] = 0;
328         }
329         for(i = 0; i < 2; i++) {
330                 encoder->guts->integer_signal_mid_side[i] = 0;
331                 encoder->guts->real_signal_mid_side[i] = 0;
332         }
333         for(i = 0; i < encoder->channels; i++) {
334                 encoder->guts->residual_workspace[i][0] = encoder->guts->residual_workspace[i][1] = 0;
335                 encoder->guts->best_subframe[i] = 0;
336         }
337         for(i = 0; i < 2; i++) {
338                 encoder->guts->residual_workspace_mid_side[i][0] = encoder->guts->residual_workspace_mid_side[i][1] = 0;
339                 encoder->guts->best_subframe_mid_side[i] = 0;
340         }
341         for(i = 0; i < encoder->channels; i++) {
342                 encoder->guts->subframe_workspace_ptr[i][0] = &encoder->guts->subframe_workspace[i][0];
343                 encoder->guts->subframe_workspace_ptr[i][1] = &encoder->guts->subframe_workspace[i][1];
344         }
345         for(i = 0; i < 2; i++) {
346                 encoder->guts->subframe_workspace_ptr_mid_side[i][0] = &encoder->guts->subframe_workspace_mid_side[i][0];
347                 encoder->guts->subframe_workspace_ptr_mid_side[i][1] = &encoder->guts->subframe_workspace_mid_side[i][1];
348         }
349         encoder->guts->abs_residual = 0;
350         encoder->guts->current_frame_can_do_mid_side = true;
351         encoder->guts->loose_mid_side_stereo_frames_exact = (double)encoder->sample_rate * 0.4 / (double)encoder->blocksize;
352         encoder->guts->loose_mid_side_stereo_frames = (unsigned)(encoder->guts->loose_mid_side_stereo_frames_exact + 0.5);
353         if(encoder->guts->loose_mid_side_stereo_frames == 0)
354                 encoder->guts->loose_mid_side_stereo_frames = 1;
355         encoder->guts->loose_mid_side_stereo_frame_count = 0;
356         encoder->guts->current_sample_number = 0;
357         encoder->guts->current_frame_number = 0;
358
359         if(!encoder_resize_buffers_(encoder, encoder->blocksize)) {
360                 /* the above function sets the state for us in case of an error */
361                 return encoder->state;
362         }
363         FLAC__bitbuffer_init(&encoder->guts->frame);
364         encoder->guts->write_callback = write_callback;
365         encoder->guts->metadata_callback = metadata_callback;
366         encoder->guts->client_data = client_data;
367
368         /*
369          * write the stream header
370          */
371         if(!FLAC__bitbuffer_clear(&encoder->guts->frame))
372                 return encoder->state = FLAC__ENCODER_MEMORY_ALLOCATION_ERROR;
373
374         if(!FLAC__bitbuffer_write_raw_uint32(&encoder->guts->frame, FLAC__STREAM_SYNC, FLAC__STREAM_SYNC_LEN))
375                 return encoder->state = FLAC__ENCODER_FRAMING_ERROR;
376
377         encoder->guts->metadata.type = FLAC__METADATA_TYPE_STREAMINFO;
378         encoder->guts->metadata.is_last = (encoder->padding == 0);
379         encoder->guts->metadata.length = FLAC__STREAM_METADATA_STREAMINFO_LENGTH;
380         encoder->guts->metadata.data.stream_info.min_blocksize = encoder->blocksize; /* this encoder uses the same blocksize for the whole stream */
381         encoder->guts->metadata.data.stream_info.max_blocksize = encoder->blocksize;
382         encoder->guts->metadata.data.stream_info.min_framesize = 0; /* we don't know this yet; have to fill it in later */
383         encoder->guts->metadata.data.stream_info.max_framesize = 0; /* we don't know this yet; have to fill it in later */
384         encoder->guts->metadata.data.stream_info.sample_rate = encoder->sample_rate;
385         encoder->guts->metadata.data.stream_info.channels = encoder->channels;
386         encoder->guts->metadata.data.stream_info.bits_per_sample = encoder->bits_per_sample;
387         encoder->guts->metadata.data.stream_info.total_samples = encoder->total_samples_estimate; /* we will replace this later with the real total */
388         memset(encoder->guts->metadata.data.stream_info.md5sum, 0, 16); /* we don't know this yet; have to fill it in later */
389         MD5Init(&encoder->guts->md5context);
390         if(!FLAC__add_metadata_block(&encoder->guts->metadata, &encoder->guts->frame))
391                 return encoder->state = FLAC__ENCODER_FRAMING_ERROR;
392
393         /* add a PADDING block if requested */
394         if(encoder->padding > 0) {
395                 padding.type = FLAC__METADATA_TYPE_PADDING;
396                 padding.is_last = true;
397                 padding.length = encoder->padding;
398                 if(!FLAC__add_metadata_block(&padding, &encoder->guts->frame))
399                         return encoder->state = FLAC__ENCODER_FRAMING_ERROR;
400         }
401
402         assert(encoder->guts->frame.bits == 0); /* assert that we're byte-aligned before writing */
403         assert(encoder->guts->frame.total_consumed_bits == 0); /* assert that no reading of the buffer was done */
404         if(encoder->guts->write_callback(encoder, encoder->guts->frame.buffer, encoder->guts->frame.bytes, 0, encoder->guts->current_frame_number, encoder->guts->client_data) != FLAC__ENCODER_WRITE_OK)
405                 return encoder->state = FLAC__ENCODER_FATAL_ERROR_WHILE_WRITING;
406
407         /* now that the metadata block is written, we can init this to an absurdly-high value... */
408         encoder->guts->metadata.data.stream_info.min_framesize = (1u << FLAC__STREAM_METADATA_STREAMINFO_MIN_FRAME_SIZE_LEN) - 1;
409         /* ... and clear this to 0 */
410         encoder->guts->metadata.data.stream_info.total_samples = 0;
411
412         return encoder->state;
413 }
414
415 void FLAC__encoder_finish(FLAC__Encoder *encoder)
416 {
417         unsigned i, channel;
418
419         assert(encoder != 0);
420         if(encoder->state == FLAC__ENCODER_UNINITIALIZED)
421                 return;
422         if(encoder->guts->current_sample_number != 0) {
423                 encoder->blocksize = encoder->guts->current_sample_number;
424                 encoder_process_frame_(encoder, true); /* true => is last frame */
425         }
426         MD5Final(encoder->guts->metadata.data.stream_info.md5sum, &encoder->guts->md5context);
427         encoder->guts->metadata_callback(encoder, &encoder->guts->metadata, encoder->guts->client_data);
428         if(encoder->guts != 0) {
429                 for(i = 0; i < encoder->channels; i++) {
430                         if(encoder->guts->integer_signal[i] != 0) {
431                                 free(encoder->guts->integer_signal[i]);
432                                 encoder->guts->integer_signal[i] = 0;
433                         }
434                         if(encoder->guts->real_signal[i] != 0) {
435                                 free(encoder->guts->real_signal[i]);
436                                 encoder->guts->real_signal[i] = 0;
437                         }
438                 }
439                 for(i = 0; i < 2; i++) {
440                         if(encoder->guts->integer_signal_mid_side[i] != 0) {
441                                 free(encoder->guts->integer_signal_mid_side[i]);
442                                 encoder->guts->integer_signal_mid_side[i] = 0;
443                         }
444                         if(encoder->guts->real_signal_mid_side[i] != 0) {
445                                 free(encoder->guts->real_signal_mid_side[i]);
446                                 encoder->guts->real_signal_mid_side[i] = 0;
447                         }
448                 }
449                 for(channel = 0; channel < encoder->channels; channel++) {
450                         for(i = 0; i < 2; i++) {
451                                 if(encoder->guts->residual_workspace[channel][i] != 0) {
452                                         free(encoder->guts->residual_workspace[channel][i]);
453                                         encoder->guts->residual_workspace[channel][i] = 0;
454                                 }
455                         }
456                 }
457                 for(channel = 0; channel < 2; channel++) {
458                         for(i = 0; i < 2; i++) {
459                                 if(encoder->guts->residual_workspace_mid_side[channel][i] != 0) {
460                                         free(encoder->guts->residual_workspace_mid_side[channel][i]);
461                                         encoder->guts->residual_workspace_mid_side[channel][i] = 0;
462                                 }
463                         }
464                 }
465                 if(encoder->guts->abs_residual != 0) {
466                         free(encoder->guts->abs_residual);
467                         encoder->guts->abs_residual = 0;
468                 }
469                 FLAC__bitbuffer_free(&encoder->guts->frame);
470                 free(encoder->guts);
471                 encoder->guts = 0;
472         }
473         encoder->state = FLAC__ENCODER_UNINITIALIZED;
474 }
475
476 bool FLAC__encoder_process(FLAC__Encoder *encoder, const int32 *buf[], unsigned samples)
477 {
478         unsigned i, j, channel;
479         int32 x, mid, side;
480         const bool ms = encoder->do_mid_side_stereo && encoder->channels == 2;
481         const int32 min_side = -((int64)1 << (encoder->bits_per_sample-1));
482         const int32 max_side =  ((int64)1 << (encoder->bits_per_sample-1)) - 1;
483
484         assert(encoder != 0);
485         assert(encoder->state == FLAC__ENCODER_OK);
486
487         j = 0;
488         do {
489                 for(i = encoder->guts->current_sample_number; i < encoder->blocksize && j < samples; i++, j++) {
490                         for(channel = 0; channel < encoder->channels; channel++) {
491                                 x = buf[channel][j];
492                                 encoder->guts->integer_signal[channel][i] = x;
493                                 encoder->guts->real_signal[channel][i] = (real)x;
494                         }
495                         if(ms && encoder->guts->current_frame_can_do_mid_side) {
496                                 side = buf[0][j] - buf[1][j];
497                                 if(side < min_side || side > max_side) {
498                                         encoder->guts->current_frame_can_do_mid_side = false;
499                                 }
500                                 else {
501                                         mid = (buf[0][j] + buf[1][j]) >> 1; /* NOTE: not the same as 'mid = (buf[0][j] + buf[1][j]) / 2' ! */
502                                         encoder->guts->integer_signal_mid_side[0][i] = mid;
503                                         encoder->guts->integer_signal_mid_side[1][i] = side;
504                                         encoder->guts->real_signal_mid_side[0][i] = (real)mid;
505                                         encoder->guts->real_signal_mid_side[1][i] = (real)side;
506                                 }
507                         }
508                         encoder->guts->current_sample_number++;
509                 }
510                 if(i == encoder->blocksize) {
511                         if(!encoder_process_frame_(encoder, false)) /* false => not last frame */
512                                 return false;
513                 }
514         } while(j < samples);
515
516         return true;
517 }
518
519 /* 'samples' is channel-wide samples, e.g. for 1 second at 44100Hz, 'samples' = 44100 regardless of the number of channels */
520 bool FLAC__encoder_process_interleaved(FLAC__Encoder *encoder, const int32 buf[], unsigned samples)
521 {
522         unsigned i, j, k, channel;
523         int32 x, left = 0, mid, side;
524         const bool ms = encoder->do_mid_side_stereo && encoder->channels == 2;
525         const int32 min_side = -((int64)1 << (encoder->bits_per_sample-1));
526         const int32 max_side =  ((int64)1 << (encoder->bits_per_sample-1)) - 1;
527
528         assert(encoder != 0);
529         assert(encoder->state == FLAC__ENCODER_OK);
530
531         j = k = 0;
532         do {
533                 for(i = encoder->guts->current_sample_number; i < encoder->blocksize && j < samples; i++, j++, k++) {
534                         for(channel = 0; channel < encoder->channels; channel++, k++) {
535                                 x = buf[k];
536                                 encoder->guts->integer_signal[channel][i] = x;
537                                 encoder->guts->real_signal[channel][i] = (real)x;
538                                 if(ms && encoder->guts->current_frame_can_do_mid_side) {
539                                         if(channel == 0) {
540                                                 left = x;
541                                         }
542                                         else {
543                                                 side = left - x;
544                                                 if(side < min_side || side > max_side) {
545                                                         encoder->guts->current_frame_can_do_mid_side = false;
546                                                 }
547                                                 else {
548                                                         mid = (left + x) >> 1; /* NOTE: not the same as 'mid = (left + x) / 2' ! */
549                                                         encoder->guts->integer_signal_mid_side[0][i] = mid;
550                                                         encoder->guts->integer_signal_mid_side[1][i] = side;
551                                                         encoder->guts->real_signal_mid_side[0][i] = (real)mid;
552                                                         encoder->guts->real_signal_mid_side[1][i] = (real)side;
553                                                 }
554                                         }
555                                 }
556                         }
557                         encoder->guts->current_sample_number++;
558                 }
559                 if(i == encoder->blocksize) {
560                         if(!encoder_process_frame_(encoder, false)) /* false => not last frame */
561                                 return false;
562                 }
563         } while(j < samples);
564
565         return true;
566 }
567
568 bool encoder_process_frame_(FLAC__Encoder *encoder, bool is_last_frame)
569 {
570         assert(encoder->state == FLAC__ENCODER_OK);
571
572         /*
573          * Accumulate raw signal to the MD5 signature
574          */
575         if(!FLAC__MD5Accumulate(&encoder->guts->md5context, encoder->guts->integer_signal, encoder->channels, encoder->blocksize, (encoder->bits_per_sample+7) / 8)) {
576                 encoder->state = FLAC__ENCODER_MEMORY_ALLOCATION_ERROR;
577                 return false;
578         }
579
580         /*
581          * Process the frame header and subframes into the frame bitbuffer
582          */
583         if(!encoder_process_subframes_(encoder, is_last_frame)) {
584                 /* the above function sets the state for us in case of an error */
585                 return false;
586         }
587
588         /*
589          * Zero-pad the frame to a byte_boundary
590          */
591         if(!FLAC__bitbuffer_zero_pad_to_byte_boundary(&encoder->guts->frame)) {
592                 encoder->state = FLAC__ENCODER_MEMORY_ALLOCATION_ERROR;
593                 return false;
594         }
595
596         /*
597          * Write it
598          */
599         assert(encoder->guts->frame.bits == 0); /* assert that we're byte-aligned before writing */
600         assert(encoder->guts->frame.total_consumed_bits == 0); /* assert that no reading of the buffer was done */
601         if(encoder->guts->write_callback(encoder, encoder->guts->frame.buffer, encoder->guts->frame.bytes, encoder->blocksize, encoder->guts->current_frame_number, encoder->guts->client_data) != FLAC__ENCODER_WRITE_OK) {
602                 encoder->state = FLAC__ENCODER_FATAL_ERROR_WHILE_WRITING;
603                 return false;
604         }
605
606         /*
607          * Get ready for the next frame
608          */
609         encoder->guts->current_frame_can_do_mid_side = true;
610         encoder->guts->current_sample_number = 0;
611         encoder->guts->current_frame_number++;
612         encoder->guts->metadata.data.stream_info.total_samples += (uint64)encoder->blocksize;
613         encoder->guts->metadata.data.stream_info.min_framesize = min(encoder->guts->frame.bytes, encoder->guts->metadata.data.stream_info.min_framesize);
614         encoder->guts->metadata.data.stream_info.max_framesize = max(encoder->guts->frame.bytes, encoder->guts->metadata.data.stream_info.max_framesize);
615
616         return true;
617 }
618
619 bool encoder_process_subframes_(FLAC__Encoder *encoder, bool is_last_frame)
620 {
621         FLAC__FrameHeader frame_header;
622         unsigned channel, max_partition_order;
623         bool do_independent, do_mid_side;
624
625         /*
626          * Calculate the max Rice partition order
627          */
628         if(is_last_frame) {
629                 max_partition_order = 0;
630         }
631         else {
632                 unsigned limit = 0, b = encoder->blocksize;
633                 while(!(b & 1)) {
634                         limit++;
635                         b >>= 1;
636                 }
637                 max_partition_order = min(encoder->rice_optimization_level, limit);
638         }
639
640         /*
641          * Setup the frame
642          */
643         if(!FLAC__bitbuffer_clear(&encoder->guts->frame)) {
644                 encoder->state = FLAC__ENCODER_MEMORY_ALLOCATION_ERROR;
645                 return false;
646         }
647         frame_header.blocksize = encoder->blocksize;
648         frame_header.sample_rate = encoder->sample_rate;
649         frame_header.channels = encoder->channels;
650         frame_header.channel_assignment = FLAC__CHANNEL_ASSIGNMENT_INDEPENDENT; /* the default unless the encoder determines otherwise */
651         frame_header.bits_per_sample = encoder->bits_per_sample;
652         frame_header.number.frame_number = encoder->guts->current_frame_number;
653
654         /*
655          * Figure out what channel assignments to try
656          */
657         if(encoder->do_mid_side_stereo) {
658                 if(encoder->loose_mid_side_stereo) {
659                         if(encoder->guts->loose_mid_side_stereo_frame_count == 0) {
660                                 do_independent = true;
661                                 do_mid_side = true;
662                         }
663                         else {
664                                 do_independent = (encoder->guts->last_channel_assignment == FLAC__CHANNEL_ASSIGNMENT_INDEPENDENT);
665                                 do_mid_side = !do_independent;
666                         }
667                 }
668                 else {
669                         do_independent = true;
670                         do_mid_side = true;
671                 }
672         }
673         else {
674                 do_independent = true;
675                 do_mid_side = false;
676         }
677         if(do_mid_side && !encoder->guts->current_frame_can_do_mid_side) {
678                 do_independent = true;
679                 do_mid_side = false;
680         }
681
682         assert(do_independent || do_mid_side);
683
684         /*
685          * First do a normal encoding pass of each independent channel
686          */
687         if(do_independent) {
688                 for(channel = 0; channel < encoder->channels; channel++) {
689                         if(!encoder_process_subframe_(encoder, max_partition_order, false, &frame_header, encoder->bits_per_sample, encoder->guts->integer_signal[channel], encoder->guts->real_signal[channel], encoder->guts->subframe_workspace_ptr[channel], encoder->guts->residual_workspace[channel], encoder->guts->best_subframe+channel, encoder->guts->best_subframe_bits+channel))
690                                 return false;
691                 }
692         }
693
694         /*
695          * Now do mid and side channels if requested
696          */
697         if(do_mid_side) {
698                 assert(encoder->channels == 2);
699
700                 for(channel = 0; channel < 2; channel++) {
701                         if(!encoder_process_subframe_(encoder, max_partition_order, false, &frame_header, encoder->bits_per_sample+(channel==0? 0:1), encoder->guts->integer_signal_mid_side[channel], encoder->guts->real_signal_mid_side[channel], encoder->guts->subframe_workspace_ptr_mid_side[channel], encoder->guts->residual_workspace_mid_side[channel], encoder->guts->best_subframe_mid_side+channel, encoder->guts->best_subframe_bits_mid_side+channel))
702                                 return false;
703                 }
704         }
705
706         /*
707          * Compose the frame bitbuffer
708          */
709         if(do_mid_side) {
710                 FLAC__ChannelAssignment channel_assignment;
711
712                 assert(encoder->channels == 2);
713
714                 if(encoder->loose_mid_side_stereo && encoder->guts->loose_mid_side_stereo_frame_count > 0) {
715                         channel_assignment = (encoder->guts->last_channel_assignment == FLAC__CHANNEL_ASSIGNMENT_INDEPENDENT? FLAC__CHANNEL_ASSIGNMENT_INDEPENDENT : FLAC__CHANNEL_ASSIGNMENT_MID_SIDE);
716                 }
717                 else {
718                         unsigned bits[4]; /* WATCHOUT - indexed by FLAC__ChannelAssignment */
719                         unsigned min_bits;
720                         FLAC__ChannelAssignment ca;
721
722                         assert(do_independent && do_mid_side);
723
724                         /* We have to figure out which channel assignent results in the smallest frame */
725                         bits[FLAC__CHANNEL_ASSIGNMENT_INDEPENDENT] = encoder->guts->best_subframe_bits         [0] + encoder->guts->best_subframe_bits         [1];
726                         bits[FLAC__CHANNEL_ASSIGNMENT_LEFT_SIDE  ] = encoder->guts->best_subframe_bits         [0] + encoder->guts->best_subframe_bits_mid_side[1];
727                         bits[FLAC__CHANNEL_ASSIGNMENT_RIGHT_SIDE ] = encoder->guts->best_subframe_bits         [1] + encoder->guts->best_subframe_bits_mid_side[1];
728                         bits[FLAC__CHANNEL_ASSIGNMENT_MID_SIDE   ] = encoder->guts->best_subframe_bits_mid_side[0] + encoder->guts->best_subframe_bits_mid_side[1];
729
730                         for(channel_assignment = 0, min_bits = bits[0], ca = 1; ca <= 3; ca++) {
731                                 if(bits[ca] < min_bits) {
732                                         min_bits = bits[ca];
733                                         channel_assignment = ca;
734                                 }
735                         }
736                 }
737
738                 frame_header.channel_assignment = channel_assignment;
739
740                 if(!FLAC__frame_add_header(&frame_header, encoder->streamable_subset, is_last_frame, &encoder->guts->frame)) {
741                         encoder->state = FLAC__ENCODER_FRAMING_ERROR;
742                         return false;
743                 }
744
745                 switch(channel_assignment) {
746                         /* note that encoder_add_subframe_ sets the state for us in case of an error */
747                         case FLAC__CHANNEL_ASSIGNMENT_INDEPENDENT:
748                                 if(!encoder_add_subframe_(encoder, &frame_header, encoder->bits_per_sample  , &encoder->guts->subframe_workspace         [0][encoder->guts->best_subframe         [0]], &encoder->guts->frame))
749                                         return false;
750                                 if(!encoder_add_subframe_(encoder, &frame_header, encoder->bits_per_sample  , &encoder->guts->subframe_workspace         [1][encoder->guts->best_subframe         [1]], &encoder->guts->frame))
751                                         return false;
752                                 break;
753                         case FLAC__CHANNEL_ASSIGNMENT_LEFT_SIDE:
754                                 if(!encoder_add_subframe_(encoder, &frame_header, encoder->bits_per_sample  , &encoder->guts->subframe_workspace         [0][encoder->guts->best_subframe         [0]], &encoder->guts->frame))
755                                         return false;
756                                 if(!encoder_add_subframe_(encoder, &frame_header, encoder->bits_per_sample+1, &encoder->guts->subframe_workspace_mid_side[1][encoder->guts->best_subframe_mid_side[1]], &encoder->guts->frame))
757                                         return false;
758                                 break;
759                         case FLAC__CHANNEL_ASSIGNMENT_RIGHT_SIDE:
760                                 if(!encoder_add_subframe_(encoder, &frame_header, encoder->bits_per_sample+1, &encoder->guts->subframe_workspace_mid_side[1][encoder->guts->best_subframe_mid_side[1]], &encoder->guts->frame))
761                                         return false;
762                                 if(!encoder_add_subframe_(encoder, &frame_header, encoder->bits_per_sample  , &encoder->guts->subframe_workspace         [1][encoder->guts->best_subframe         [1]], &encoder->guts->frame))
763                                         return false;
764                                 break;
765                         case FLAC__CHANNEL_ASSIGNMENT_MID_SIDE:
766                                 if(!encoder_add_subframe_(encoder, &frame_header, encoder->bits_per_sample  , &encoder->guts->subframe_workspace_mid_side[0][encoder->guts->best_subframe_mid_side[0]], &encoder->guts->frame))
767                                         return false;
768                                 if(!encoder_add_subframe_(encoder, &frame_header, encoder->bits_per_sample+1, &encoder->guts->subframe_workspace_mid_side[1][encoder->guts->best_subframe_mid_side[1]], &encoder->guts->frame))
769                                         return false;
770                                 break;
771                         default:
772                                 assert(0);
773                 }
774         }
775         else {
776                 if(!FLAC__frame_add_header(&frame_header, encoder->streamable_subset, is_last_frame, &encoder->guts->frame)) {
777                         encoder->state = FLAC__ENCODER_FRAMING_ERROR;
778                         return false;
779                 }
780
781                 for(channel = 0; channel < encoder->channels; channel++) {
782                         if(!encoder_add_subframe_(encoder, &frame_header, encoder->bits_per_sample, &encoder->guts->subframe_workspace[channel][encoder->guts->best_subframe[channel]], &encoder->guts->frame)) {
783                                 /* the above function sets the state for us in case of an error */
784                                 return false;
785                         }
786                 }
787         }
788
789         if(encoder->loose_mid_side_stereo) {
790                 encoder->guts->loose_mid_side_stereo_frame_count++;
791                 if(encoder->guts->loose_mid_side_stereo_frame_count >= encoder->guts->loose_mid_side_stereo_frames)
792                         encoder->guts->loose_mid_side_stereo_frame_count = 0;
793         }
794
795         encoder->guts->last_channel_assignment = frame_header.channel_assignment;
796
797         return true;
798 }
799
800 bool encoder_process_subframe_(FLAC__Encoder *encoder, unsigned max_partition_order, bool verbatim_only, const FLAC__FrameHeader *frame_header, unsigned bits_per_sample, const int32 integer_signal[], const real real_signal[], FLAC__Subframe *subframe[2], int32 *residual[2], unsigned *best_subframe, unsigned *best_bits)
801 {
802         real fixed_residual_bits_per_sample[FLAC__MAX_FIXED_ORDER+1];
803         real lpc_residual_bits_per_sample;
804         real autoc[FLAC__MAX_LPC_ORDER+1];
805         real lp_coeff[FLAC__MAX_LPC_ORDER][FLAC__MAX_LPC_ORDER];
806         real lpc_error[FLAC__MAX_LPC_ORDER];
807         unsigned min_lpc_order, max_lpc_order, lpc_order;
808         unsigned min_fixed_order, max_fixed_order, guess_fixed_order, fixed_order;
809         unsigned min_qlp_coeff_precision, max_qlp_coeff_precision, qlp_coeff_precision;
810         unsigned rice_parameter;
811         unsigned _candidate_bits, _best_bits;
812         unsigned _best_subframe;
813
814         /* verbatim subframe is the baseline against which we measure other compressed subframes */
815         _best_subframe = 0;
816         _best_bits = encoder_evaluate_verbatim_subframe_(integer_signal, frame_header->blocksize, bits_per_sample, subframe[_best_subframe]);
817
818         if(!verbatim_only && frame_header->blocksize >= FLAC__MAX_FIXED_ORDER) {
819                 /* check for constant subframe */
820                 guess_fixed_order = FLAC__fixed_compute_best_predictor(integer_signal+FLAC__MAX_FIXED_ORDER, frame_header->blocksize-FLAC__MAX_FIXED_ORDER, fixed_residual_bits_per_sample);
821                 if(fixed_residual_bits_per_sample[1] == 0.0) {
822                         /* the above means integer_signal+FLAC__MAX_FIXED_ORDER is constant, now we just have to check the warmup samples */
823                         unsigned i, signal_is_constant = true;
824                         for(i = 1; i <= FLAC__MAX_FIXED_ORDER; i++) {
825                                 if(integer_signal[0] != integer_signal[i]) {
826                                         signal_is_constant = false;
827                                         break;
828                                 }
829                         }
830                         if(signal_is_constant) {
831                                 _candidate_bits = encoder_evaluate_constant_subframe_(integer_signal[0], bits_per_sample, subframe[!_best_subframe]);
832                                 if(_candidate_bits < _best_bits) {
833                                         _best_subframe = !_best_subframe;
834                                         _best_bits = _candidate_bits;
835                                 }
836                         }
837                 }
838                 else {
839                         /* encode fixed */
840                         if(encoder->do_exhaustive_model_search) {
841                                 min_fixed_order = 0;
842                                 max_fixed_order = FLAC__MAX_FIXED_ORDER;
843                         }
844                         else {
845                                 min_fixed_order = max_fixed_order = guess_fixed_order;
846                         }
847                         for(fixed_order = min_fixed_order; fixed_order <= max_fixed_order; fixed_order++) {
848                                 if(fixed_residual_bits_per_sample[fixed_order] >= (real)bits_per_sample)
849                                         continue; /* don't even try */
850                                 rice_parameter = (fixed_residual_bits_per_sample[fixed_order] > 0.0)? (unsigned)(fixed_residual_bits_per_sample[fixed_order]+0.5) : 0; /* 0.5 is for rounding */
851                                 rice_parameter++; /* to account for the signed->unsigned conversion during rice coding */
852                                 if(rice_parameter >= (1u << FLAC__ENTROPY_CODING_METHOD_PARTITIONED_RICE_PARAMETER_LEN))
853                                         rice_parameter = (1u << FLAC__ENTROPY_CODING_METHOD_PARTITIONED_RICE_PARAMETER_LEN) - 1;
854                                 _candidate_bits = encoder_evaluate_fixed_subframe_(integer_signal, residual[!_best_subframe], encoder->guts->abs_residual, frame_header->blocksize, bits_per_sample, fixed_order, rice_parameter, max_partition_order, subframe[!_best_subframe]);
855                                 if(_candidate_bits < _best_bits) {
856                                         _best_subframe = !_best_subframe;
857                                         _best_bits = _candidate_bits;
858                                 }
859                         }
860
861                         /* encode lpc */
862                         if(encoder->max_lpc_order > 0) {
863                                 if(encoder->max_lpc_order >= frame_header->blocksize)
864                                         max_lpc_order = frame_header->blocksize-1;
865                                 else
866                                         max_lpc_order = encoder->max_lpc_order;
867                                 if(max_lpc_order > 0) {
868                                         FLAC__lpc_compute_autocorrelation(real_signal, frame_header->blocksize, max_lpc_order+1, autoc);
869                                         /* if autoc[0] == 0.0, the signal is constant and we usually won't get here, but it can happen */
870                                         if(autoc[0] != 0.0) {
871                                                 FLAC__lpc_compute_lp_coefficients(autoc, max_lpc_order, lp_coeff, lpc_error);
872                                                 if(encoder->do_exhaustive_model_search) {
873                                                         min_lpc_order = 1;
874                                                 }
875                                                 else {
876                                                         unsigned guess_lpc_order = FLAC__lpc_compute_best_order(lpc_error, max_lpc_order, frame_header->blocksize, bits_per_sample);
877                                                         min_lpc_order = max_lpc_order = guess_lpc_order;
878                                                 }
879                                                 if(encoder->do_qlp_coeff_prec_search) {
880                                                         min_qlp_coeff_precision = FLAC__MIN_QLP_COEFF_PRECISION;
881                                                         max_qlp_coeff_precision = min(32 - bits_per_sample - 1, (1u<<FLAC__SUBFRAME_LPC_QLP_COEFF_PRECISION_LEN)-1);
882                                                 }
883                                                 else {
884                                                         min_qlp_coeff_precision = max_qlp_coeff_precision = encoder->qlp_coeff_precision;
885                                                 }
886                                                 for(lpc_order = min_lpc_order; lpc_order <= max_lpc_order; lpc_order++) {
887                                                         lpc_residual_bits_per_sample = FLAC__lpc_compute_expected_bits_per_residual_sample(lpc_error[lpc_order-1], frame_header->blocksize-lpc_order);
888                                                         if(lpc_residual_bits_per_sample >= (real)bits_per_sample)
889                                                                 continue; /* don't even try */
890                                                         rice_parameter = (lpc_residual_bits_per_sample > 0.0)? (unsigned)(lpc_residual_bits_per_sample+0.5) : 0; /* 0.5 is for rounding */
891                                                         rice_parameter++; /* to account for the signed->unsigned conversion during rice coding */
892                                                         if(rice_parameter >= (1u << FLAC__ENTROPY_CODING_METHOD_PARTITIONED_RICE_PARAMETER_LEN))
893                                                                 rice_parameter = (1u << FLAC__ENTROPY_CODING_METHOD_PARTITIONED_RICE_PARAMETER_LEN) - 1;
894                                                         for(qlp_coeff_precision = min_qlp_coeff_precision; qlp_coeff_precision <= max_qlp_coeff_precision; qlp_coeff_precision++) {
895                                                                 _candidate_bits = encoder_evaluate_lpc_subframe_(integer_signal, residual[!_best_subframe], encoder->guts->abs_residual, lp_coeff[lpc_order-1], frame_header->blocksize, bits_per_sample, lpc_order, qlp_coeff_precision, rice_parameter, max_partition_order, subframe[!_best_subframe]);
896                                                                 if(_candidate_bits > 0) { /* if == 0, there was a problem quantizing the lpcoeffs */
897                                                                         if(_candidate_bits < _best_bits) {
898                                                                                 _best_subframe = !_best_subframe;
899                                                                                 _best_bits = _candidate_bits;
900                                                                         }
901                                                                 }
902                                                         }
903                                                 }
904                                         }
905                                 }
906                         }
907                 }
908         }
909
910         *best_subframe = _best_subframe;
911         *best_bits = _best_bits;
912
913         return true;
914 }
915
916 bool encoder_add_subframe_(FLAC__Encoder *encoder, const FLAC__FrameHeader *frame_header, unsigned bits_per_sample, const FLAC__Subframe *subframe, FLAC__BitBuffer *frame)
917 {
918         switch(subframe->type) {
919                 case FLAC__SUBFRAME_TYPE_CONSTANT:
920                         if(!FLAC__subframe_add_constant(&(subframe->data.constant), bits_per_sample, frame)) {
921                                 encoder->state = FLAC__ENCODER_FATAL_ERROR_WHILE_ENCODING;
922                                 return false;
923                         }
924                         break;
925                 case FLAC__SUBFRAME_TYPE_FIXED:
926                         if(!FLAC__subframe_add_fixed(&(subframe->data.fixed), frame_header->blocksize - subframe->data.fixed.order, bits_per_sample, frame)) {
927                                 encoder->state = FLAC__ENCODER_FATAL_ERROR_WHILE_ENCODING;
928                                 return false;
929                         }
930                         break;
931                 case FLAC__SUBFRAME_TYPE_LPC:
932                         if(!FLAC__subframe_add_lpc(&(subframe->data.lpc), frame_header->blocksize - subframe->data.lpc.order, bits_per_sample, frame)) {
933                                 encoder->state = FLAC__ENCODER_FATAL_ERROR_WHILE_ENCODING;
934                                 return false;
935                         }
936                         break;
937                 case FLAC__SUBFRAME_TYPE_VERBATIM:
938                         if(!FLAC__subframe_add_verbatim(&(subframe->data.verbatim), frame_header->blocksize, bits_per_sample, frame)) {
939                                 encoder->state = FLAC__ENCODER_FATAL_ERROR_WHILE_ENCODING;
940                                 return false;
941                         }
942                         break;
943                 default:
944                         assert(0);
945         }
946
947         return true;
948 }
949
950 unsigned encoder_evaluate_constant_subframe_(const int32 signal, unsigned bits_per_sample, FLAC__Subframe *subframe)
951 {
952         subframe->type = FLAC__SUBFRAME_TYPE_CONSTANT;
953         subframe->data.constant.value = signal;
954
955         return FLAC__SUBFRAME_TYPE_LEN + bits_per_sample;
956 }
957
958 unsigned encoder_evaluate_fixed_subframe_(const int32 signal[], int32 residual[], uint32 abs_residual[], unsigned blocksize, unsigned bits_per_sample, unsigned order, unsigned rice_parameter, unsigned max_partition_order, FLAC__Subframe *subframe)
959 {
960         unsigned i, residual_bits;
961         const unsigned residual_samples = blocksize - order;
962
963         FLAC__fixed_compute_residual(signal+order, residual_samples, order, residual);
964
965         subframe->type = FLAC__SUBFRAME_TYPE_FIXED;
966
967         subframe->data.fixed.entropy_coding_method.type = FLAC__ENTROPY_CODING_METHOD_PARTITIONED_RICE;
968         subframe->data.fixed.residual = residual;
969
970         residual_bits = encoder_find_best_partition_order_(residual, abs_residual, residual_samples, order, rice_parameter, max_partition_order, &subframe->data.fixed.entropy_coding_method.data.partitioned_rice.order, subframe->data.fixed.entropy_coding_method.data.partitioned_rice.parameters);
971
972         subframe->data.fixed.order = order;
973         for(i = 0; i < order; i++)
974                 subframe->data.fixed.warmup[i] = signal[i];
975
976         return FLAC__SUBFRAME_TYPE_LEN + (order * bits_per_sample) + residual_bits;
977 }
978
979 unsigned encoder_evaluate_lpc_subframe_(const int32 signal[], int32 residual[], uint32 abs_residual[], const real lp_coeff[], unsigned blocksize, unsigned bits_per_sample, unsigned order, unsigned qlp_coeff_precision, unsigned rice_parameter, unsigned max_partition_order, FLAC__Subframe *subframe)
980 {
981         int32 qlp_coeff[FLAC__MAX_LPC_ORDER];
982         unsigned i, residual_bits;
983         int quantization, ret;
984         const unsigned residual_samples = blocksize - order;
985
986         ret = FLAC__lpc_quantize_coefficients(lp_coeff, order, qlp_coeff_precision, bits_per_sample, qlp_coeff, &quantization);
987         if(ret != 0)
988                 return 0; /* this is a hack to indicate to the caller that we can't do lp at this order on this subframe */
989
990         FLAC__lpc_compute_residual_from_qlp_coefficients(signal+order, residual_samples, qlp_coeff, order, quantization, residual);
991
992         subframe->type = FLAC__SUBFRAME_TYPE_LPC;
993
994         subframe->data.lpc.entropy_coding_method.type = FLAC__ENTROPY_CODING_METHOD_PARTITIONED_RICE;
995         subframe->data.lpc.residual = residual;
996
997         residual_bits = encoder_find_best_partition_order_(residual, abs_residual, residual_samples, order, rice_parameter, max_partition_order, &subframe->data.lpc.entropy_coding_method.data.partitioned_rice.order, subframe->data.lpc.entropy_coding_method.data.partitioned_rice.parameters);
998
999         subframe->data.lpc.order = order;
1000         subframe->data.lpc.qlp_coeff_precision = qlp_coeff_precision;
1001         subframe->data.lpc.quantization_level = quantization;
1002         memcpy(subframe->data.lpc.qlp_coeff, qlp_coeff, sizeof(int32)*FLAC__MAX_LPC_ORDER);
1003         for(i = 0; i < order; i++)
1004                 subframe->data.lpc.warmup[i] = signal[i];
1005
1006         return FLAC__SUBFRAME_TYPE_LEN + FLAC__SUBFRAME_LPC_QLP_COEFF_PRECISION_LEN + FLAC__SUBFRAME_LPC_QLP_SHIFT_LEN + (order * (qlp_coeff_precision + bits_per_sample)) + residual_bits;
1007 }
1008
1009 unsigned encoder_evaluate_verbatim_subframe_(const int32 signal[], unsigned blocksize, unsigned bits_per_sample, FLAC__Subframe *subframe)
1010 {
1011         subframe->type = FLAC__SUBFRAME_TYPE_VERBATIM;
1012
1013         subframe->data.verbatim.data = signal;
1014
1015         return FLAC__SUBFRAME_TYPE_LEN + (blocksize * bits_per_sample);
1016 }
1017
1018 unsigned encoder_find_best_partition_order_(const int32 residual[], uint32 abs_residual[], unsigned residual_samples, unsigned predictor_order, unsigned rice_parameter, unsigned max_partition_order, unsigned *best_partition_order, unsigned best_parameters[])
1019 {
1020         unsigned residual_bits, best_residual_bits = 0;
1021         unsigned i, partition_order;
1022         unsigned best_parameters_index = 0, parameters[2][1 << FLAC__MAX_RICE_PARTITION_ORDER];
1023         int32 r;
1024
1025         /* compute the abs(residual) for use later */
1026         for(i = 0; i < residual_samples; i++) {
1027                 r = residual[i];
1028                 abs_residual[i] = (uint32)(r<0? -r : r);
1029         }
1030
1031         for(partition_order = 0; partition_order <= max_partition_order; partition_order++) {
1032                 if(!encoder_set_partitioned_rice_(abs_residual, residual_samples, predictor_order, rice_parameter, partition_order, parameters[!best_parameters_index], &residual_bits)) {
1033                         assert(best_residual_bits != 0);
1034                         break;
1035                 }
1036                 if(best_residual_bits == 0 || residual_bits < best_residual_bits) {
1037                         best_residual_bits = residual_bits;
1038                         *best_partition_order = partition_order;
1039                         best_parameters_index = !best_parameters_index;
1040                 }
1041         }
1042         memcpy(best_parameters, parameters[best_parameters_index], sizeof(unsigned)*(1<<(*best_partition_order)));
1043
1044         return best_residual_bits;
1045 }
1046
1047 #ifdef ESTIMATE_RICE_BITS
1048 #undef ESTIMATE_RICE_BITS
1049 #endif
1050 #define ESTIMATE_RICE_BITS(value, parameter) ((value) >> (parameter))
1051
1052 bool encoder_set_partitioned_rice_(const uint32 abs_residual[], const unsigned residual_samples, const unsigned predictor_order, const unsigned rice_parameter, const unsigned partition_order, unsigned parameters[], unsigned *bits)
1053 {
1054         unsigned bits_ = FLAC__ENTROPY_CODING_METHOD_TYPE_LEN + FLAC__ENTROPY_CODING_METHOD_PARTITIONED_RICE_ORDER_LEN;
1055
1056         if(partition_order == 0) {
1057                 unsigned i;
1058 #ifdef ESTIMATE_RICE_BITS
1059                 const unsigned rice_parameter_estimate = rice_parameter-1;
1060                 bits_ += (1+rice_parameter) * residual_samples;
1061 #endif
1062                 parameters[0] = rice_parameter;
1063                 bits_ += FLAC__ENTROPY_CODING_METHOD_PARTITIONED_RICE_PARAMETER_LEN;
1064                 for(i = 0; i < residual_samples; i++)
1065 #ifdef ESTIMATE_RICE_BITS
1066                         bits_ += ESTIMATE_RICE_BITS(abs_residual[i], rice_parameter_estimate);
1067 #else
1068                         bits_ += FLAC__bitbuffer_rice_bits(residual[i], rice_parameter);
1069 #endif
1070         }
1071         else {
1072                 unsigned i, j, k = 0, k_last = 0;
1073                 unsigned mean, parameter, partition_samples;
1074                 const unsigned max_parameter = (1u << FLAC__ENTROPY_CODING_METHOD_PARTITIONED_RICE_PARAMETER_LEN) - 1;
1075                 for(i = 0; i < (1u<<partition_order); i++) {
1076                         partition_samples = (residual_samples+predictor_order) >> partition_order;
1077                         if(i == 0) {
1078                                 if(partition_samples <= predictor_order)
1079                                         return false;
1080                                 else
1081                                         partition_samples -= predictor_order;
1082                         }
1083                         mean = partition_samples >> 1;
1084                         for(j = 0; j < partition_samples; j++, k++)
1085                                 mean += abs_residual[k];
1086                         mean /= partition_samples;
1087                         /* calc parameter = floor(log2(mean)) + 1 */
1088                         parameter = 0;
1089                         while(mean) {
1090                                 parameter++;
1091                                 mean >>= 1;
1092                         }
1093                         if(parameter > max_parameter)
1094                                 parameter = max_parameter;
1095                         parameters[i] = parameter;
1096                         bits_ += FLAC__ENTROPY_CODING_METHOD_PARTITIONED_RICE_PARAMETER_LEN;
1097 #ifdef ESTIMATE_RICE_BITS
1098                         bits_ += (1+parameter) * partition_samples;
1099                         --parameter;
1100 #endif
1101                         for(j = k_last; j < k; j++)
1102 #ifdef ESTIMATE_RICE_BITS
1103                                 bits_ += ESTIMATE_RICE_BITS(abs_residual[j], parameter);
1104 #else
1105                                 bits_ += FLAC__bitbuffer_rice_bits(residual[j], parameter);
1106 #endif
1107                         k_last = k;
1108                 }
1109         }
1110
1111         *bits = bits_;
1112         return true;
1113 }