Commits work in progress on improving echo cancellation.Attempts to shorten the adaptation time of the echo cancellation.

3 files changed, 100 insertions, 75 deletions

diff --git a/src/native/portaudio/AudioQualityImprovement.c b/src/native/portaudio/AudioQualityImprovement.c
index 9a57708..68df67e 100644
--- a/src/native/portaudio/AudioQualityImprovement.c
+++ b/src/native/portaudio/AudioQualityImprovement.c
@@ -14,8 +14,7 @@
 
 static void AudioQualityImprovement_cancelEchoFromPlay
     (AudioQualityImprovement *aqi,
-    void *buffer, unsigned long length,
-    jlong startTime, jlong endTime);
+    void *buffer, unsigned long length);
 static void AudioQualityImprovement_free(AudioQualityImprovement *aqi);
 static AudioQualityImprovement *AudioQualityImprovement_new
     (const char *stringID, jlong longID, AudioQualityImprovement *next);
@@ -24,11 +23,14 @@ static void AudioQualityImprovement_popFromPlay
 static void AudioQualityImprovement_resampleInPlay
     (AudioQualityImprovement *aqi,
     double sampleRate, unsigned long sampleSizeInBits, int channels,
-    void *buffer, unsigned long length,
-    jlong startTime, jlong endTime);
+    void *buffer, unsigned long length);
 static void AudioQualityImprovement_retain(AudioQualityImprovement *aqi);
 static void AudioQualityImprovement_setFrameSize
     (AudioQualityImprovement *aqi, jint frameSize);
+static void AudioQualityImprovement_setOutputLatency
+    (AudioQualityImprovement *aqi, jlong outputLatency);
+static void AudioQualityImprovement_updatePlayDelay
+    (AudioQualityImprovement *aqi);
 static void AudioQualityImprovement_updatePlayIsDelaying
     (AudioQualityImprovement *aqi);
 static void AudioQualityImprovement_updatePreprocess
@@ -50,16 +52,11 @@ static AudioQualityImprovement *AudioQualityImprovement_sharedInstances
  * @param aqi
  * @param buffer
  * @param length the length of <tt>buffer</tt> in bytes
- * @param startTime the time in milliseconds at which <tt>buffer</tt> was given
- * to the audio capture implementation
- * @param endTime the time in milliseconds at which <tt>buffer</tt> was returned
- * from the audio capture implementation
  */
 static void
 AudioQualityImprovement_cancelEchoFromPlay
     (AudioQualityImprovement *aqi,
-    void *buffer, unsigned long length,
-    jlong startTime, jlong endTime)
+    void *buffer, unsigned long length)
 {
     spx_uint32_t sampleCount;
 
@@ -162,18 +159,10 @@ static AudioQualityImprovement *
 AudioQualityImprovement_new
     (const char *stringID, jlong longID, AudioQualityImprovement *next)
 {
-    AudioQualityImprovement *aqi = malloc(sizeof(AudioQualityImprovement));
+    AudioQualityImprovement *aqi = calloc(1, sizeof(AudioQualityImprovement));
 
     if (aqi)
     {
-        aqi->echo = NULL;
-        aqi->mutex = NULL;
-        aqi->out = NULL;
-        aqi->play = NULL;
-        aqi->preprocess = NULL;
-        aqi->resampler = NULL;
-        /* aqi->stringID = NULL; */
-
         /* stringID */
         aqi->stringID = strdup(stringID);
         if (!(aqi->stringID))
@@ -189,14 +178,9 @@ AudioQualityImprovement_new
             return NULL;
         }
 
-        aqi->denoise = JNI_FALSE;
-        aqi->echoFilterLengthInMillis = 0;
-        aqi->frameSize = 0;
         aqi->longID = longID;
         aqi->next = next;
-        aqi->playDelay = 3;
         aqi->retainCount = 1;
-        aqi->sampleRate = 0;
     }
     return aqi;
 }
@@ -222,20 +206,18 @@ AudioQualityImprovement_popFromPlay
  * @param sampleRate
  * @param sampleSizeInBits
  * @param channels
+ * @param latency the latency of the stream associated with <tt>buffer</tt> in
+ * milliseconds
  * @param buffer
  * @param length the length of <tt>buffer</tt> in bytes
- * @param startTime the time in milliseconds at which <tt>buffer</tt> was given
- * to the audio capture or playback implementation
- * @param endTime the time in milliseconds at which <tt>buffer</tt> was returned
- * from the audio capture or playback implementation
  */
 void
 AudioQualityImprovement_process
     (AudioQualityImprovement *aqi,
     AudioQualityImprovementSampleOrigin sampleOrigin,
     double sampleRate, unsigned long sampleSizeInBits, int channels,
-    void *buffer, unsigned long length,
-    jlong startTime, jlong endTime)
+    jlong latency,
+    void *buffer, unsigned long length)
 {
     if ((sampleSizeInBits == 16) && (channels == 1) && !mutex_lock(aqi->mutex))
     {
@@ -251,8 +233,7 @@ AudioQualityImprovement_process
                     {
                         AudioQualityImprovement_cancelEchoFromPlay(
                             aqi,
-                            buffer, length,
-                            startTime, endTime);
+                            buffer, length);
                     }
                     speex_preprocess_run(aqi->preprocess, buffer);
                 }
@@ -261,11 +242,11 @@ AudioQualityImprovement_process
         case AUDIO_QUALITY_IMPROVEMENT_SAMPLE_ORIGIN_OUTPUT:
             if (aqi->preprocess && aqi->echo)
             {
+                AudioQualityImprovement_setOutputLatency(aqi, latency);
                 AudioQualityImprovement_resampleInPlay(
                     aqi,
                     sampleRate, sampleSizeInBits, channels,
-                    buffer, length,
-                    startTime, endTime);
+                    buffer, length);
             }
             break;
         }
@@ -325,17 +306,12 @@ AudioQualityImprovement_release(AudioQualityImprovement *aqi)
  * @param channels
  * @param buffer
  * @param length the length of <tt>buffer</tt> in bytes
- * @param startTime the time in milliseconds at which <tt>buffer</tt> was given
- * to the audio playback implementation
- * @param endTime the time in milliseconds at which <tt>buffer</tt> was returned
- * from the audio playback implementation
  */
 static void
 AudioQualityImprovement_resampleInPlay
     (AudioQualityImprovement *aqi,
     double sampleRate, unsigned long sampleSizeInBits, int channels,
-    void *buffer, unsigned long length,
-    jlong startTime, jlong endTime)
+    void *buffer, unsigned long length)
 {
     spx_uint32_t playSize;
     spx_uint32_t playCapacity;
@@ -384,7 +360,7 @@ AudioQualityImprovement_resampleInPlay
 
     /* Ensure that play exists and is large enough. */
     playCapacity
-        = (2 * (aqi->playDelay)) * (aqi->frameSize / sizeof(spx_int16_t));
+        = ((1 + aqi->playDelay) + 1) * (aqi->frameSize / sizeof(spx_int16_t));
     playLength = playSize / sizeof(spx_int16_t);
     if (playCapacity < playLength)
         playCapacity = playLength;
@@ -417,6 +393,12 @@ AudioQualityImprovement_resampleInPlay
     {
         aqi->playIsDelaying = JNI_TRUE;
         aqi->playLength = 0;
+        /*
+         * We don't have enough room in play for buffer which means that we'll
+         * have to throw some samples away. But it'll effectively mean that
+         * we'll enlarge the drift which will disrupt the echo cancellation. So
+         * it seems the least of two evils to just reset the echo cancellation.
+         */
         speex_echo_state_reset(aqi->echo);
     }
 
@@ -435,7 +417,7 @@ AudioQualityImprovement_resampleInPlay
     }
     aqi->playLength += playLength;
 
-    /* Delay the playback by a few frames. */
+    /* Take into account the latency. */
     if (aqi->playIsDelaying == JNI_TRUE)
         AudioQualityImprovement_updatePlayIsDelaying(aqi);
 }
@@ -493,6 +475,17 @@ AudioQualityImprovement_setFrameSize
     }
 }
 
+static void
+AudioQualityImprovement_setOutputLatency
+    (AudioQualityImprovement *aqi, jlong outputLatency)
+{
+    if (aqi->outputLatency != outputLatency)
+    {
+        aqi->outputLatency = outputLatency;
+        AudioQualityImprovement_updatePlayDelay(aqi);
+    }
+}
+
 void
 AudioQualityImprovement_setSampleRate
     (AudioQualityImprovement *aqi, int sampleRate)
@@ -502,6 +495,7 @@ AudioQualityImprovement_setSampleRate
         if (aqi->sampleRate != sampleRate)
         {
             aqi->sampleRate = sampleRate;
+            AudioQualityImprovement_updatePlayDelay(aqi);
             AudioQualityImprovement_updatePreprocess(aqi);
         }
         mutex_unlock(aqi->mutex);
@@ -509,6 +503,36 @@ AudioQualityImprovement_setSampleRate
 }
 
 static void
+AudioQualityImprovement_updatePlayDelay(AudioQualityImprovement *aqi)
+{
+    spx_uint32_t playDelay;
+
+    /*
+     * Apart from output latency, there is obviously input latency as well.
+     * Since the echo cancellation implementation will attempt to adapt to the
+     * delay between the far and the near ends anyway, don't take the input
+     * latency into account and don't increase the risk of reversing the delay.
+     */
+    if (!(aqi->outputLatency) || !(aqi->frameSize) || !(aqi->sampleRate))
+        playDelay = 2;
+    else
+    {
+        playDelay
+            = aqi->outputLatency
+                / ((aqi->frameSize / sizeof(spx_int16_t))
+                    / (aqi->sampleRate / 1000));
+    }
+
+    if (aqi->playDelay != playDelay)
+    {
+        aqi->playDelay = playDelay;
+
+        if (aqi->play && (aqi->playIsDelaying == JNI_TRUE))
+            AudioQualityImprovement_updatePlayIsDelaying(aqi);
+    }
+}
+
+static void
 AudioQualityImprovement_updatePlayIsDelaying(AudioQualityImprovement *aqi)
 {
     spx_uint32_t playDelay
diff --git a/src/native/portaudio/AudioQualityImprovement.h b/src/native/portaudio/AudioQualityImprovement.h
index a205b23..4254b24 100644
--- a/src/native/portaudio/AudioQualityImprovement.h
+++ b/src/native/portaudio/AudioQualityImprovement.h
@@ -97,6 +97,9 @@ typedef struct _AudioQualityImprovement
 
     /** The capacity of #out in bytes. */
     spx_uint32_t outCapacity;
+
+    /** The playback latency in milliseconds. */
+    jlong outputLatency;
     spx_int16_t *play;
 
     /**
@@ -138,8 +141,8 @@ void AudioQualityImprovement_process
     (AudioQualityImprovement *aqi,
     AudioQualityImprovementSampleOrigin sampleOrigin,
     double sampleRate, unsigned long sampleSizeInBits, int channels,
-    void *buffer, unsigned long length,
-    jlong startTime, jlong endTime);
+    jlong latency,
+    void *buffer, unsigned long length);
 void AudioQualityImprovement_release(AudioQualityImprovement *aqi);
 void AudioQualityImprovement_setDenoise
     (AudioQualityImprovement *aqi, jboolean denoise);
diff --git a/src/native/portaudio/net_java_sip_communicator_impl_neomedia_portaudio_PortAudio.c b/src/native/portaudio/net_java_sip_communicator_impl_neomedia_portaudio_PortAudio.c
index 8be5ddd..8a1998a 100644
--- a/src/native/portaudio/net_java_sip_communicator_impl_neomedia_portaudio_PortAudio.c
+++ b/src/native/portaudio/net_java_sip_communicator_impl_neomedia_portaudio_PortAudio.c
@@ -19,7 +19,9 @@ typedef struct
     int channels;
     JNIEnv *env;
     long inputFrameSize;
+    jlong inputLatency;
     long outputFrameSize;
+    jlong outputLatency;
     double sampleRate;
     int sampleSizeInBits;
     PaStream *stream;
@@ -226,9 +228,18 @@ Java_net_java_sip_communicator_impl_neomedia_portaudio_PortAudio_Pa_1OpenStream
             stream->channels = inputStreamParameters->channelCount;
             if (stream->audioQualityImprovement)
             {
+                const PaStreamInfo *streamInfo;
+
                 AudioQualityImprovement_setSampleRate(
                     stream->audioQualityImprovement,
                     (int) sampleRate);
+
+                streamInfo = Pa_GetStreamInfo(stream->stream);
+                if (streamInfo)
+                {
+                    stream->inputLatency
+                        = (jlong) (streamInfo->inputLatency * 1000);
+                }
             }
         }
         else if (outputStreamParameters)
@@ -236,6 +247,17 @@ Java_net_java_sip_communicator_impl_neomedia_portaudio_PortAudio_Pa_1OpenStream
             stream->sampleSizeInBits
                 = PortAudio_getSampleSizeInBits(outputStreamParameters);
             stream->channels = outputStreamParameters->channelCount;
+            if (stream->audioQualityImprovement)
+            {
+                const PaStreamInfo *streamInfo;
+
+                streamInfo = Pa_GetStreamInfo(stream->stream);
+                if (streamInfo)
+                {
+                    stream->outputLatency
+                        = (jlong) (streamInfo->outputLatency * 1000);
+                }
+            }
         }
 
         return (jlong) stream;
@@ -256,17 +278,10 @@ Java_net_java_sip_communicator_impl_neomedia_portaudio_PortAudio_Pa_1ReadStream
 
     if (data)
     {
-        jlong startTime, endTime;
         PortAudioStream *portAudioStream = (PortAudioStream *) stream;
         PaError errorCode;
 
-/*
-        startTime = System_currentTimeMillis();
- */
         errorCode = Pa_ReadStream(portAudioStream->stream, data, frames);
-/*
-        endTime = System_currentTimeMillis();
- */
         if ((paNoError == errorCode) || (paInputOverflowed == errorCode))
         {
             if (portAudioStream->audioQualityImprovement)
@@ -277,8 +292,8 @@ Java_net_java_sip_communicator_impl_neomedia_portaudio_PortAudio_Pa_1ReadStream
                     portAudioStream->sampleRate,
                     portAudioStream->sampleSizeInBits,
                     portAudioStream->channels,
-                    data, frames * portAudioStream->inputFrameSize,
-                    startTime, endTime);
+                    portAudioStream->inputLatency,
+                    data, frames * portAudioStream->inputFrameSize);
             }
             (*env)->ReleaseByteArrayElements(env, buffer, data, 0);
         }
@@ -325,6 +340,7 @@ Java_net_java_sip_communicator_impl_neomedia_portaudio_PortAudio_Pa_1WriteStream
     double sampleRate;
     unsigned long sampleSizeInBits;
     int channels;
+    jlong outputLatency;
     long framesInBytes;
     PaError errorCode;
     jint i;
@@ -340,19 +356,12 @@ Java_net_java_sip_communicator_impl_neomedia_portaudio_PortAudio_Pa_1WriteStream
     sampleRate = portAudioStream->sampleRate;
     sampleSizeInBits = portAudioStream->sampleSizeInBits;
     channels = portAudioStream->channels;
+    outputLatency = portAudioStream->outputLatency;
     framesInBytes = frames * portAudioStream->outputFrameSize;
 
     for (i = 0; i < numberOfWrites; i++)
     {
-        jlong startTime, endTime;
-
-/*
-        startTime = System_currentTimeMillis();
- */
         errorCode = Pa_WriteStream(paStream, data, frames);
-/*
-        endTime = System_currentTimeMillis();
- */
         if ((paNoError != errorCode) && (errorCode != paOutputUnderflowed))
             break;
         else
@@ -363,8 +372,8 @@ Java_net_java_sip_communicator_impl_neomedia_portaudio_PortAudio_Pa_1WriteStream
                     audioQualityImprovement,
                     AUDIO_QUALITY_IMPROVEMENT_SAMPLE_ORIGIN_OUTPUT,
                     sampleRate, sampleSizeInBits, channels,
-                    data, framesInBytes,
-                    startTime, endTime);
+                    outputLatency,
+                    data, framesInBytes);
             }
             data += framesInBytes;
         }
@@ -757,7 +766,7 @@ PortAudioStream_free(JNIEnv *env, PortAudioStream *stream)
 static PortAudioStream *
 PortAudioStream_new(JNIEnv *env, jobject streamCallback)
 {
-    PortAudioStream *stream = malloc(sizeof(PortAudioStream));
+    PortAudioStream *stream = calloc(1, sizeof(PortAudioStream));
 
     if (!stream)
     {
@@ -782,17 +791,6 @@ PortAudioStream_new(JNIEnv *env, jobject streamCallback)
             return NULL;
         }
     }
-    else
-    {
-        stream->vm = NULL;
-        stream->streamCallback = NULL;
-    }
-
-    stream->audioQualityImprovement = NULL;
-    stream->env = NULL;
-    stream->stream = NULL;
-    stream->streamCallbackMethodID = NULL;
-    stream->streamFinishedCallbackMethodID = NULL;
 
     return stream;
 }