Support for glitchles mode switching
authorJean-Marc Valin <jean-marc.valin@usherbrooke.ca>
Tue, 8 Mar 2011 01:54:33 +0000 (20:54 -0500)
committerJean-Marc Valin <jean-marc.valin@usherbrooke.ca>
Tue, 8 Mar 2011 01:54:33 +0000 (20:54 -0500)
Uses a 5ms redundant CELT frame embedded into the SILK or hybrid
packet to handle the switching. It's still possible to use the
PLC-based method when no redundant packet is included.

celt
silk
src/opus_decoder.c
src/opus_encoder.c
src/opus_encoder.h

diff --git a/celt b/celt
index c79c4e3..9bac8c1 160000 (submodule)
--- a/celt
+++ b/celt
@@ -1 +1 @@
-Subproject commit c79c4e3bc7270b08a2800a2dd9d9a5f02767024d
+Subproject commit 9bac8c17d57ea99192de09f8b97300b3dee0422f
diff --git a/silk b/silk
index 37f9809..20832a6 160000 (submodule)
--- a/silk
+++ b/silk
@@ -1 +1 @@
-Subproject commit 37f9809a68b6c5ba514b0642e99231a7f68ee86d
+Subproject commit 20832a6c95a84e18da936228e4b15823d8b89728
index 4477ccb..6eb081c 100644 (file)
@@ -71,6 +71,17 @@ OpusDecoder *opus_decoder_create(int Fs, int channels)
        return st;
 }
 
+static void smooth_fade(const short *in1, const short *in2, short *out, int overlap, int channels)
+{
+       int i, c;
+       for (c=0;c<channels;c++)
+       {
+               /* FIXME: Make this 16-bit safe, remove division */
+               for (i=0;i<overlap;i++)
+                       out[i*channels+c] = (i*in2[i*channels+c] + (overlap-i)*in1[i*channels+c])/overlap;
+       }
+}
+
 int opus_decode(OpusDecoder *st, const unsigned char *data,
                int len, short *pcm, int frame_size, int decode_fec)
 {
@@ -85,6 +96,10 @@ int opus_decode(OpusDecoder *st, const unsigned char *data,
     int transition=0;
     int start_band;
     int redundancy=0;
+    int redundancy_bytes = 0;
+    int celt_to_silk=0;
+    short redundant_audio[240*2];
+    int c;
 
     /* Payloads of 1 (2 including ToC) or 0 trigger the PLC/DTX */
     if (len<=2)
@@ -127,13 +142,13 @@ int opus_decode(OpusDecoder *st, const unsigned char *data,
         mode = st->prev_mode;
     }
 
-    if (mode != st->prev_mode && st->prev_mode > 0
+    if (data!=NULL && !st->prev_redundancy && mode != st->prev_mode && st->prev_mode > 0
                && !(mode == MODE_SILK_ONLY && st->prev_mode == MODE_HYBRID)
                && !(mode == MODE_HYBRID && st->prev_mode == MODE_SILK_ONLY))
     {
        transition = 1;
-       if (mode == MODE_CELT_ONLY && !st->prev_redundancy)
-           opus_decode(st, NULL, 0, pcm_transition, IMAX(480, audiosize), 0);
+       if (mode == MODE_CELT_ONLY)
+           opus_decode(st, NULL, 0, pcm_transition, IMAX(st->Fs/100, audiosize), 0);
     }
     if (audiosize > frame_size)
     {
@@ -162,6 +177,7 @@ int opus_decode(OpusDecoder *st, const unsigned char *data,
             } else if( st->bandwidth == BANDWIDTH_WIDEBAND ) {
                 DecControl.internalSampleRate = 16000;
             } else {
+               DecControl.internalSampleRate = 16000;
                 SKP_assert( 0 );
             }
         } else {
@@ -189,53 +205,100 @@ int opus_decode(OpusDecoder *st, const unsigned char *data,
     }
 
     start_band = 0;
-    if (mode == MODE_HYBRID && data != NULL)
+    if (mode != MODE_CELT_ONLY && data != NULL)
     {
         /* Check if we have a redundant 0-8 kHz band */
         redundancy = ec_dec_bit_logp(&dec, 12);
-        if (!redundancy)
-            start_band = 17;
+        if (redundancy)
+        {
+            celt_to_silk = ec_dec_bit_logp(&dec, 1);
+            if (mode == MODE_HYBRID)
+               redundancy_bytes = 2 + ec_dec_uint(&dec, 256);
+            else
+               redundancy_bytes = len - ((ec_tell(&dec)+7)>>3);
+            len -= redundancy_bytes;
+            /* Shrink decoder because of raw bits */
+            dec.storage -= redundancy_bytes;
+        }
+        start_band = 17;
+    }
+
+    if (mode != MODE_SILK_ONLY)
+    {
+        int endband=21;
+
+        switch(st->bandwidth)
+        {
+        case BANDWIDTH_NARROWBAND:
+            endband = 13;
+            break;
+        case BANDWIDTH_WIDEBAND:
+            endband = 17;
+            break;
+        case BANDWIDTH_SUPERWIDEBAND:
+            endband = 19;
+            break;
+        case BANDWIDTH_FULLBAND:
+            endband = 21;
+            break;
+        }
+        celt_decoder_ctl(st->celt_dec, CELT_SET_END_BAND(endband));
+        celt_decoder_ctl(st->celt_dec, CELT_SET_CHANNELS(st->stream_channels));
     }
+
     if (redundancy)
         transition = 0;
 
     if (transition && mode != MODE_CELT_ONLY)
-        opus_decode(st, NULL, 0, pcm_transition, IMAX(480, audiosize), 0);
+        opus_decode(st, NULL, 0, pcm_transition, IMAX(st->Fs/100, audiosize), 0);
+
+    /* 5 ms redundant frame for CELT->SILK*/
+    if (redundancy && celt_to_silk)
+    {
+        celt_decode(st->celt_dec, data+len, redundancy_bytes, redundant_audio, st->Fs/200);
+        celt_decoder_ctl(st->celt_dec, CELT_RESET_STATE);
+    }
 
     /* MUST be after PLC */
     celt_decoder_ctl(st->celt_dec, CELT_SET_START_BAND(start_band));
 
+    if (transition)
+       celt_decoder_ctl(st->celt_dec, CELT_RESET_STATE);
 
     if (mode != MODE_SILK_ONLY)
     {
-       int endband;
-
-           switch(st->bandwidth)
-           {
-           case BANDWIDTH_NARROWBAND:
-               endband = 13;
-               break;
-           case BANDWIDTH_WIDEBAND:
-               endband = 17;
-               break;
-           case BANDWIDTH_SUPERWIDEBAND:
-               endband = 19;
-               break;
-           case BANDWIDTH_FULLBAND:
-               endband = 21;
-               break;
-           }
-           celt_decoder_ctl(st->celt_dec, CELT_SET_END_BAND(endband));
-           celt_decoder_ctl(st->celt_dec, CELT_SET_CHANNELS(st->stream_channels));
-
-           if (st->prev_mode == MODE_SILK_ONLY)
-               celt_decoder_ctl(st->celt_dec, CELT_RESET_STATE);
         /* Decode CELT */
         celt_ret = celt_decode_with_ec(st->celt_dec, decode_fec?NULL:data, len, pcm_celt, frame_size, &dec);
         for (i=0;i<frame_size*st->channels;i++)
             pcm[i] = ADD_SAT16(pcm[i], pcm_celt[i]);
     }
 
+    /* 5 ms redundant frame for SILK->CELT */
+    if (redundancy && !celt_to_silk)
+    {
+        int N2, N4;
+        N2 = st->Fs/200;
+        N4 = st->Fs/400;
+        celt_decoder_ctl(st->celt_dec, CELT_RESET_STATE);
+        celt_decoder_ctl(st->celt_dec, CELT_SET_START_BAND(0));
+
+        celt_decode(st->celt_dec, data+len, redundancy_bytes, redundant_audio, N2);
+        smooth_fade(pcm+st->channels*(frame_size-N4), redundant_audio+st->channels*N4,
+                       pcm+st->channels*(frame_size-N4), N4, st->channels);
+    }
+    if (redundancy && celt_to_silk)
+    {
+        int N2, N4;
+        N2 = st->Fs/200;
+        N4 = st->Fs/400;
+
+        for (c=0;c<st->channels;c++)
+        {
+            for (i=0;i<N4;i++)
+                pcm[st->channels*i+c] = redundant_audio[st->channels*i];
+        }
+        smooth_fade(redundant_audio+st->channels*N4, pcm+st->channels*N4, pcm+st->channels*N4, N4, st->channels);
+    }
     if (transition)
     {
        int plc_length, overlap;
@@ -246,9 +309,8 @@ int opus_decode(OpusDecoder *st, const unsigned char *data,
        for (i=0;i<plc_length;i++)
                pcm[i] = pcm_transition[i];
 
-       overlap = IMIN(480, IMAX(0, audiosize-plc_length));
-       for (i=0;i<overlap;i++)
-               pcm[plc_length+i] = (i*pcm[plc_length+i] + (overlap-i)*pcm_transition[plc_length+i])/overlap;
+       overlap = IMIN(st->Fs/100, IMAX(0, audiosize-plc_length));
+       smooth_fade(pcm_transition+plc_length, pcm+plc_length, pcm+plc_length, overlap, st->channels);
     }
 #if OPUS_TEST_RANGE_CODER_STATE
     st->rangeFinal = dec.rng;
index 4ac580e..0150442 100644 (file)
@@ -81,6 +81,10 @@ OpusEncoder *opus_encoder_create(int Fs, int channels)
        st->use_vbr = 0;
        st->bitrate_bps = 32000;
 
+       st->encoder_buffer = st->Fs/100;
+       st->delay_compensation = st->Fs/400;
+       if (st->Fs > 16000)
+               st->delay_compensation += 10;
        return st;
 }
 
@@ -95,8 +99,13 @@ int opus_encode(OpusEncoder *st, const short *pcm, int frame_size,
     int silk_internal_bandwidth;
     int bytes_target;
     int prefill=0;
-    int start_band;
+    int start_band = 0;
     int redundancy = 0;
+    int redundancy_bytes = 0;
+    int celt_to_silk = 0;
+    short pcm_buf[960*2];
+    int nb_compr_bytes;
+    int to_celt = 0;
 
        bytes_target = st->bitrate_bps * frame_size / (st->Fs * 8) - 1;
 
@@ -107,6 +116,18 @@ int opus_encode(OpusEncoder *st, const short *pcm, int frame_size,
                SKP_Silk_SDK_InitEncoder( st->silk_enc, &dummy);
                prefill=1;
        }
+       if (st->prev_mode >0 &&
+               ((st->mode != MODE_CELT_ONLY && st->prev_mode == MODE_CELT_ONLY)
+               || (st->mode == MODE_CELT_ONLY && st->prev_mode != MODE_CELT_ONLY)))
+       {
+           redundancy = 1;
+           celt_to_silk = (st->mode != MODE_CELT_ONLY);
+           if (!celt_to_silk)
+           {
+               st->mode = st->prev_mode;
+               to_celt = 1;
+           }
+       }
 
        ec_enc_init(&enc, data, max_data_bytes-1);
 
@@ -159,7 +180,7 @@ int opus_encode(OpusEncoder *st, const short *pcm, int frame_size,
         if (prefill)
         {
             int zero=0;
-               SKP_Silk_SDK_Encode( st->silk_enc, &st->silk_mode, st->delay_buffer, ENCODER_BUFFER, NULL, &zero, 1 );
+               SKP_Silk_SDK_Encode( st->silk_enc, &st->silk_mode, st->delay_buffer, st->encoder_buffer, NULL, &zero, 1 );
         }
 
         ret = SKP_Silk_SDK_Encode( st->silk_enc, &st->silk_mode, pcm, frame_size, &enc, &nBytes, 0 );
@@ -182,11 +203,8 @@ int opus_encode(OpusEncoder *st, const short *pcm, int frame_size,
     }
 
     /* CELT processing */
-       if (st->mode != MODE_SILK_ONLY)
        {
-           int endband;
-           short pcm_buf[960*2];
-           int nb_compr_bytes;
+           int endband=21;
 
            switch(st->bandwidth)
            {
@@ -205,30 +223,23 @@ int opus_encode(OpusEncoder *st, const short *pcm, int frame_size,
            }
            celt_encoder_ctl(st->celt_enc, CELT_SET_END_BAND(endband));
            celt_encoder_ctl(st->celt_enc, CELT_SET_CHANNELS(st->stream_channels));
-
+       }
+       if (st->mode != MODE_SILK_ONLY)
+       {
         celt_encoder_ctl(st->celt_enc, CELT_SET_VBR(0));
         celt_encoder_ctl(st->celt_enc, CELT_SET_BITRATE(510000));
         if (st->prev_mode == MODE_SILK_ONLY)
         {
                unsigned char dummy[2];
                celt_encoder_ctl(st->celt_enc, CELT_RESET_STATE);
+               celt_encoder_ctl(st->celt_enc, CELT_SET_START_BAND(0));
                celt_encoder_ctl(st->celt_enc, CELT_SET_PREDICTION(0));
                /* FIXME: This wastes CPU a bit compared to just prefilling the buffer */
-               celt_encode(st->celt_enc, &st->delay_buffer[(ENCODER_BUFFER-ENCODER_DELAY_COMPENSATION-120)*st->channels], 120, dummy, 10);
+               celt_encode(st->celt_enc, &st->delay_buffer[(st->encoder_buffer-st->delay_compensation-120)*st->channels], 120, dummy, 10);
         } else {
                celt_encoder_ctl(st->celt_enc, CELT_SET_PREDICTION(2));
         }
 
-        start_band = 0;
-        if (st->mode == MODE_HYBRID)
-        {
-            /* Check if we have a redundant 0-8 kHz band */
-            ec_enc_bit_logp(&enc, redundancy, 12);
-            if (!redundancy)
-                start_band = 17;
-        }
-        celt_encoder_ctl(st->celt_enc, CELT_SET_START_BAND(start_band));
-
         if (st->mode == MODE_HYBRID)
         {
             int len;
@@ -251,32 +262,86 @@ int opus_encode(OpusEncoder *st, const short *pcm, int frame_size,
             }
         }
 
-        for (i=0;i<IMIN(frame_size, ENCODER_DELAY_COMPENSATION)*st->channels;i++)
-            pcm_buf[i] = st->delay_buffer[(ENCODER_BUFFER-ENCODER_DELAY_COMPENSATION)*st->channels+i];
-        for (;i<frame_size*st->channels;i++)
-            pcm_buf[i] = pcm[i-ENCODER_DELAY_COMPENSATION*st->channels];
 
         ec_enc_shrink(&enc, nb_compr_bytes);
+       } else {
+           nb_compr_bytes = 0;
+       }
 
+    for (i=0;i<IMIN(frame_size, st->delay_compensation)*st->channels;i++)
+        pcm_buf[i] = st->delay_buffer[(st->encoder_buffer-st->delay_compensation)*st->channels+i];
+    for (;i<frame_size*st->channels;i++)
+        pcm_buf[i] = pcm[i-st->delay_compensation*st->channels];
+    if (st->mode != MODE_CELT_ONLY)
+    {
+        /* Check if we have a redundant 0-8 kHz band */
+        ec_enc_bit_logp(&enc, redundancy, 12);
+        if (redundancy)
+        {
+            redundancy_bytes = 40;
+            ec_enc_bit_logp(&enc, celt_to_silk, 1);
+            if (st->mode == MODE_HYBRID)
+               ec_enc_uint(&enc, redundancy_bytes-2, 256);
+        }
+        start_band = 17;
+    }
+
+    if (st->mode == MODE_SILK_ONLY)
+    {
+        ret = (ec_tell(&enc)+7)>>3;
+        ec_enc_done(&enc);
+        nb_compr_bytes = ret;
+    }
+
+    /* 5 ms redundant frame for CELT->SILK */
+    if (redundancy && celt_to_silk)
+    {
+        celt_encoder_ctl(st->celt_enc, CELT_SET_START_BAND(0));
+        /* FIXME: This is wrong -- we need to set the flags properly */
+        celt_encoder_ctl(st->celt_enc, CELT_SET_VBR(0));
+        /* FIXME: Make sure not to overflow here */
+        celt_encode(st->celt_enc, pcm_buf, st->Fs/200, data+nb_compr_bytes, redundancy_bytes);
+        celt_encoder_ctl(st->celt_enc, CELT_RESET_STATE);
+    }
+
+    celt_encoder_ctl(st->celt_enc, CELT_SET_START_BAND(start_band));
+
+    if (st->mode != MODE_SILK_ONLY)
+       {
            /* Encode high band with CELT */
            ret = celt_encode_with_ec(st->celt_enc, pcm_buf, frame_size, NULL, nb_compr_bytes, &enc);
-
-           if (frame_size>ENCODER_BUFFER)
-           {
-               for (i=0;i<ENCODER_BUFFER*st->channels;i++)
-                   st->delay_buffer[i] = pcm[(frame_size-ENCODER_BUFFER)*st->channels+i];
-           } else {
-               int tmp = ENCODER_BUFFER-frame_size;
-               for (i=0;i<tmp*st->channels;i++)
-                   st->delay_buffer[i] = st->delay_buffer[i+frame_size*st->channels];
-               for (i=0;i<frame_size*st->channels;i++)
-                   st->delay_buffer[tmp*st->channels+i] = pcm[i];
-           }
-       } else {
-           ret = (ec_tell(&enc)+7)>>3;
-           ec_enc_done(&enc);
        }
 
+    /* 5 ms redundant frame for SILK->CELT */
+    if (redundancy && !celt_to_silk)
+    {
+        int N2, N4;
+        N2 = st->Fs/200;
+        N4 = st->Fs/400;
+
+        celt_encoder_ctl(st->celt_enc, CELT_RESET_STATE);
+        celt_encoder_ctl(st->celt_enc, CELT_SET_START_BAND(0));
+        celt_encoder_ctl(st->celt_enc, CELT_SET_PREDICTION(0));
+
+        /* FIXME: Do proper prefilling here */
+        celt_encode(st->celt_enc, pcm_buf+st->channels*(frame_size-N2-N4), N4, data+nb_compr_bytes, redundancy_bytes);
+
+        celt_encode(st->celt_enc, pcm_buf+st->channels*(frame_size-N2), N2, data+nb_compr_bytes, redundancy_bytes);
+    }
+
+
+    if (frame_size>st->encoder_buffer)
+    {
+       for (i=0;i<st->encoder_buffer*st->channels;i++)
+               st->delay_buffer[i] = pcm[(frame_size-st->encoder_buffer)*st->channels+i];
+    } else {
+       int tmp = st->encoder_buffer-frame_size;
+       for (i=0;i<tmp*st->channels;i++)
+               st->delay_buffer[i] = st->delay_buffer[i+frame_size*st->channels];
+       for (i=0;i<frame_size*st->channels;i++)
+               st->delay_buffer[tmp*st->channels+i] = pcm[i];
+    }
+
        /* Signalling the mode in the first byte */
        data--;
        framerate = st->Fs/frame_size;
@@ -310,8 +375,11 @@ int opus_encode(OpusEncoder *st, const short *pcm, int frame_size,
 #if OPUS_TEST_RANGE_CODER_STATE
     st->rangeFinal = enc.rng;
 #endif
-    st->prev_mode = st->mode;
-    return ret+1;
+    if (to_celt)
+        st->prev_mode = MODE_CELT_ONLY;
+    else
+        st->prev_mode = st->mode;
+    return ret+1+redundancy_bytes;
 }
 
 void opus_encoder_ctl(OpusEncoder *st, int request, ...)
index a800a74..5eda2b2 100644 (file)
@@ -33,8 +33,7 @@
 #include "SKP_Silk_SDK_API.h"
 
 /* FIXME: This is only valid for 48 kHz */
-#define ENCODER_DELAY_COMPENSATION 130
-#define ENCODER_BUFFER 480
+#define MAX_ENCODER_BUFFER 480
 
 struct OpusEncoder {
        CELTEncoder *celt_enc;
@@ -50,8 +49,10 @@ struct OpusEncoder {
     int          Fs;
     int          use_vbr;
     int          bitrate_bps;
+    int          encoder_buffer;
+    int          delay_compensation;
 
-    short        delay_buffer[ENCODER_BUFFER*2];
+    short        delay_buffer[MAX_ENCODER_BUFFER*2];
 
 #ifdef OPUS_TEST_RANGE_CODER_STATE
     int          rangeFinal;