Commits work in progress on improving echo cancellation. Attempts to better the matching of playback and capture which seems to reduce the far to near end delay. Since the drift is wildly varying, the echo canceller will still fail to deliver. (Which also means that there is no need to rebuild the respective native binaries.)

3 files changed, 226 insertions, 61 deletions

diff --git a/src/native/portaudio/AudioQualityImprovement.c b/src/native/portaudio/AudioQualityImprovement.c
index f56f0c8..6c32e3d 100644
--- a/src/native/portaudio/AudioQualityImprovement.c
+++ b/src/native/portaudio/AudioQualityImprovement.c
@@ -13,14 +13,17 @@
 #include <string.h>
 
 static void AudioQualityImprovement_cancelEchoFromPlay
-    (AudioQualityImprovement *aqi, void *buffer, unsigned long length);
+    (AudioQualityImprovement *aqi,
+    void *buffer, unsigned long length,
+    jlong startTime, jlong endTime);
 static void AudioQualityImprovement_free(AudioQualityImprovement *aqi);
 static AudioQualityImprovement *AudioQualityImprovement_new
     (const char *stringID, jlong longID, AudioQualityImprovement *next);
 static void AudioQualityImprovement_resampleInPlay
     (AudioQualityImprovement *aqi,
     double sampleRate, unsigned long sampleSizeInBits, int channels,
-    void *buffer, unsigned long length);
+    void *buffer, unsigned long length,
+    jlong startTime, jlong endTime);
 static void AudioQualityImprovement_retain(AudioQualityImprovement *aqi);
 static void AudioQualityImprovement_setFrameSize
     (AudioQualityImprovement *aqi, jint frameSize);
@@ -38,11 +41,50 @@ static CRITICAL_SECTION AudioQualityImprovement_sharedInstancesMutex
 static AudioQualityImprovement *AudioQualityImprovement_sharedInstances
     = NULL;
 
+/**
+ *
+ * @param aqi
+ * @param buffer
+ * @param length the length of <tt>buffer</tt> in bytes
+ * @param startTime the time in milliseconds at which <tt>buffer</tt> was given
+ * to the audio capture implementation
+ * @param endTime the time in milliseconds at which <tt>buffer</tt> was returned
+ * from the audio capture implementation
+ */
 static void
 AudioQualityImprovement_cancelEchoFromPlay
     (AudioQualityImprovement *aqi,
-    void *buffer, unsigned long length)
+    void *buffer, unsigned long length,
+    jlong startTime, jlong endTime)
 {
+    spx_uint32_t playOffsetInSamples, playOffsetInBytes;
+
+    /*
+     * Account for the delay between the giving of the audio data to the
+     * playback implementation and its actual playback.
+     */
+/*
+    startTime
+        -= 2
+            * ((aqi->frameSize / sizeof(spx_int16_t))
+                / (aqi->sampleRate / 1000));
+ */
+    /*
+     * Depending on startTime, find the part of play which is to be used by the
+     * echo cancellation for buffer.
+     */
+    if (startTime < aqi->playStartTime)
+        return;
+    playOffsetInSamples
+        = (startTime - aqi->playStartTime) * (aqi->sampleRate / 1000);
+    playOffsetInBytes = playOffsetInSamples * sizeof(spx_int16_t);
+    if (playOffsetInBytes + length > aqi->playSize)
+        return;
+
+    /*
+     * Ensure that out exists and is large enough to receive the result of the
+     * echo cancellation.
+     */
     if (!(aqi->out) || (aqi->outCapacity < length))
     {
         spx_int16_t *newOut = realloc(aqi->out, length);
@@ -55,8 +97,11 @@ AudioQualityImprovement_cancelEchoFromPlay
         else
             return;
     }
-    speex_echo_cancellation(aqi->echo, buffer, aqi->play, aqi->out);
-    aqi->playSize = 0;
+
+    /* Perform the echo cancellation and return the result in buffer. */
+    speex_echo_cancellation(
+        aqi->echo,
+        buffer, aqi->play + playOffsetInSamples, aqi->out);
     memcpy(buffer, aqi->out, length);
 }
 
@@ -114,7 +159,8 @@ AudioQualityImprovement_getSharedInstance(const char *stringID, jlong longID)
         {
             theSharedInstance
                 = AudioQualityImprovement_new(
-                    stringID, longID,
+                    stringID,
+                    longID,
                     AudioQualityImprovement_sharedInstances);
             if (theSharedInstance)
                 AudioQualityImprovement_sharedInstances = theSharedInstance;
@@ -166,12 +212,27 @@ AudioQualityImprovement_new
     return aqi;
 }
 
+/**
+ *
+ * @param aqi
+ * @param sampleOrigin
+ * @param sampleRate
+ * @param sampleSizeInBits
+ * @param channels
+ * @param buffer
+ * @param length the length of <tt>buffer</tt> in bytes
+ * @param startTime the time in milliseconds at which <tt>buffer</tt> was given
+ * to the audio capture or playback implementation
+ * @param endTime the time in milliseconds at which <tt>buffer</tt> was returned
+ * from the audio capture or playback implementation
+ */
 void
 AudioQualityImprovement_process
     (AudioQualityImprovement *aqi,
     AudioQualityImprovementSampleOrigin sampleOrigin,
     double sampleRate, unsigned long sampleSizeInBits, int channels,
-    void *buffer, unsigned long length)
+    void *buffer, unsigned long length,
+    jlong startTime, jlong endTime)
 {
     if ((sampleSizeInBits == 16) && (channels == 1) && !mutex_lock(aqi->mutex))
     {
@@ -183,13 +244,12 @@ AudioQualityImprovement_process
                 AudioQualityImprovement_setFrameSize(aqi, length);
                 if (aqi->preprocess)
                 {
-                    if (aqi->echo
-                            && aqi->play
-                            && (aqi->playSize == aqi->frameSize))
+                    if (aqi->echo && aqi->play && aqi->playSize)
                     {
                         AudioQualityImprovement_cancelEchoFromPlay(
                             aqi,
-                            buffer, length);
+                            buffer, length,
+                            startTime, endTime);
                     }
                     speex_preprocess_run(aqi->preprocess, buffer);
                 }
@@ -201,7 +261,8 @@ AudioQualityImprovement_process
                 AudioQualityImprovement_resampleInPlay(
                     aqi,
                     sampleRate, sampleSizeInBits, channels,
-                    buffer, length);
+                    buffer, length,
+                    startTime, endTime);
             }
             break;
         }
@@ -253,13 +314,31 @@ AudioQualityImprovement_release(AudioQualityImprovement *aqi)
     }
 }
 
+/**
+ *
+ * @param aqi
+ * @param sampleRate
+ * @param sampleSizeInBits
+ * @param channels
+ * @param buffer
+ * @param length the length of <tt>buffer</tt> in bytes
+ * @param startTime the time in milliseconds at which <tt>buffer</tt> was given
+ * to the audio playback implementation
+ * @param endTime the time in milliseconds at which <tt>buffer</tt> was returned
+ * from the audio playback implementation
+ */
 static void
 AudioQualityImprovement_resampleInPlay
     (AudioQualityImprovement *aqi,
     double sampleRate, unsigned long sampleSizeInBits, int channels,
-    void *buffer, unsigned long length)
+    void *buffer, unsigned long length,
+    jlong startTime, jlong endTime)
 {
     spx_uint32_t playSize;
+    unsigned long sampleSizeInBytes;
+    spx_uint32_t newPlaySize;
+    jlong oldPlayStartTime;
+    spx_int16_t *play;
 
     if (sampleRate == aqi->sampleRate)
         playSize = length;
@@ -269,7 +348,8 @@ AudioQualityImprovement_resampleInPlay
         {
             speex_resampler_set_rate(
                 aqi->resampler,
-                (spx_uint32_t) sampleRate, (spx_uint32_t) (aqi->sampleRate));
+                (spx_uint32_t) sampleRate,
+                (spx_uint32_t) (aqi->sampleRate));
             playSize = aqi->frameSize;
         }
         else
@@ -277,10 +357,13 @@ AudioQualityImprovement_resampleInPlay
             aqi->resampler
                 = speex_resampler_init(
                     channels,
-                    (spx_uint32_t) sampleRate, (spx_uint32_t) (aqi->sampleRate),
+                    (spx_uint32_t) sampleRate,
+                    (spx_uint32_t) (aqi->sampleRate),
                     SPEEX_RESAMPLER_QUALITY_VOIP,
                     NULL);
-            if (!(aqi->resampler))
+            if (aqi->resampler)
+                playSize = aqi->frameSize;
+            else
             {
                 aqi->playSize = 0;
                 return;
@@ -289,17 +372,31 @@ AudioQualityImprovement_resampleInPlay
     }
     else
     {
+        /*
+         * The specified buffer neither is in the format of the audio capture
+         * nor can be resampled to it.
+         */
         aqi->playSize = 0;
         return;
     }
+
+    /* Ensure that play exists and is large enough. */
+    sampleSizeInBytes = sampleSizeInBits / 8;
     if (!(aqi->play) || (aqi->playCapacity < playSize))
     {
-        spx_int16_t *newPlay = realloc(aqi->play, playSize);
+        spx_uint32_t playCapacity;
+        spx_int16_t *newPlay;
 
+        playCapacity = aqi->filterLengthOfEcho * sampleSizeInBytes;
+        if (playCapacity < playSize)
+            playCapacity = playSize;
+        newPlay = realloc(aqi->play, playCapacity);
         if (newPlay)
         {
+            if (!(aqi->play))
+                aqi->playSize = 0;
             aqi->play = newPlay;
-            aqi->playCapacity = playSize;
+            aqi->playCapacity = playCapacity;
         }
         else
         {
@@ -307,21 +404,60 @@ AudioQualityImprovement_resampleInPlay
             return;
         }
     }
+
+    /* Ensure that there is room for buffer in play. */
+    newPlaySize = aqi->playSize + playSize;
+    if (newPlaySize > aqi->playCapacity)
+    {
+        spx_uint32_t i;
+        spx_uint32_t playBytesToDiscard = newPlaySize - aqi->playCapacity;
+        spx_uint32_t playSamplesToMove
+            = (aqi->playSize - playBytesToDiscard) / sizeof(spx_int16_t);
+        spx_int16_t *playNew = aqi->play;
+        spx_uint32_t playSamplesToDiscard
+            = playBytesToDiscard / sizeof(spx_int16_t);
+        spx_int16_t *playOld = aqi->play + playSamplesToDiscard;
+
+        for (i = 0; i < playSamplesToMove; i++)
+            *playNew++ = *playOld++;
+        aqi->playSize -= playBytesToDiscard;
+        newPlaySize = aqi->playSize + playSize;
+
+        aqi->playStartTime += playSamplesToDiscard / ( aqi->sampleRate / 1000);
+    }
+
+    if (endTime > startTime)
+    {
+        spx_uint32_t lengthInMillis
+            = (aqi->frameSize / sampleSizeInBytes) / (aqi->sampleRate / 1000);
+
+        if (endTime - startTime > lengthInMillis)
+            startTime = endTime - lengthInMillis;
+    }
+    oldPlayStartTime = aqi->playStartTime;
+    aqi->playStartTime
+        = startTime
+            - ((aqi->playSize / sampleSizeInBytes) / (aqi->sampleRate / 1000));
+    if (aqi->playStartTime != oldPlayStartTime)
+        fprintf(stderr, "start time delta = %ld\n", aqi->playStartTime - oldPlayStartTime);
+
+    /* Place buffer in play. */
+    play = aqi->play + (aqi->playSize / sizeof(spx_int16_t));
     if (length == aqi->frameSize)
     {
-        memcpy(aqi->play, buffer, playSize);
-        aqi->playSize = playSize;
+        memcpy(play, buffer, playSize);
+        aqi->playSize = newPlaySize;
     }
     else
     {
-        unsigned long sampleSizeInBytes = sampleSizeInBits / 8;
         spx_uint32_t bufferSampleCount = length / sampleSizeInBytes;
         spx_uint32_t playSampleCount = playSize / sampleSizeInBytes;
 
         speex_resampler_process_interleaved_int(
             aqi->resampler,
-            buffer, &bufferSampleCount, aqi->play, &playSampleCount);
-        aqi->playSize = playSampleCount * sampleSizeInBytes;
+            buffer, &bufferSampleCount,
+            play, &playSampleCount);
+        aqi->playSize += playSampleCount * sampleSizeInBytes;
     }
 }
 
@@ -394,7 +530,8 @@ AudioQualityImprovement_setSampleRate
 }
 
 static void
-AudioQualityImprovement_updatePreprocess(AudioQualityImprovement *aqi)
+AudioQualityImprovement_updatePreprocess
+    (AudioQualityImprovement *aqi)
 {
     if (aqi->echo)
     {
@@ -412,7 +549,8 @@ AudioQualityImprovement_updatePreprocess(AudioQualityImprovement *aqi)
         {
             int echoFilterLength
                 = (int)
-                    ((aqi->sampleRate * aqi->echoFilterLengthInMillis) / 1000);
+                    ((aqi->sampleRate * aqi->echoFilterLengthInMillis)
+                        / 1000);
 
             if (aqi->filterLengthOfEcho != echoFilterLength)
                 frameSize = 0;
diff --git a/src/native/portaudio/AudioQualityImprovement.h b/src/native/portaudio/AudioQualityImprovement.h
index f721271..b2f06f0 100644
--- a/src/native/portaudio/AudioQualityImprovement.h
+++ b/src/native/portaudio/AudioQualityImprovement.h
@@ -80,17 +80,39 @@ typedef struct _AudioQualityImprovement
     jboolean denoise;
     SpeexEchoState *echo;
     jlong echoFilterLengthInMillis;
+
+    /** The length of the echo cancelling filter of #echo in samples. */
     int filterLengthOfEcho;
     jint frameSize;
     int frameSizeOfPreprocess;
     jlong longID;
     Mutex *mutex;
     struct _AudioQualityImprovement *next;
+
+    /**
+     * The intermediate buffer into which the result of echo cancellation is
+     * written for a specific <tt>buffer</tt> of captured audio.
+     */
     spx_int16_t *out;
+
+    /** The capacity of #out in bytes. */
     spx_uint32_t outCapacity;
     spx_int16_t *play;
+
+    /**
+     * The capacity of #play in bytes i.e. the total number of bytes allocated
+     * to #play regardless of whether they are used or not.
+     */
     spx_uint32_t playCapacity;
+
+    /** The size in bytes of the valid audio data written into #play. */
     spx_uint32_t playSize;
+
+    /**
+     * The time in milliseconds at which the valid audio data written into #play
+     * has started playing back.
+     */
+    jlong playStartTime;
     SpeexPreprocessState *preprocess;
     SpeexResamplerState *resampler;
     int retainCount;
@@ -98,6 +120,7 @@ typedef struct _AudioQualityImprovement
     int sampleRateOfPreprocess;
     char *stringID;
 } AudioQualityImprovement;
+
 #endif /* #ifndef AUDIO_QUALITY_IMPROVEMENT_IMPLEMENTATION */
 
 typedef enum
@@ -112,7 +135,8 @@ void AudioQualityImprovement_process
     (AudioQualityImprovement *aqi,
     AudioQualityImprovementSampleOrigin sampleOrigin,
     double sampleRate, unsigned long sampleSizeInBits, int channels,
-    void *buffer, unsigned long length);
+    void *buffer, unsigned long length,
+    jlong startTime, jlong endTime);
 void AudioQualityImprovement_release(AudioQualityImprovement *aqi);
 void AudioQualityImprovement_setDenoise
     (AudioQualityImprovement *aqi, jboolean denoise);
diff --git a/src/native/portaudio/net_java_sip_communicator_impl_neomedia_portaudio_PortAudio.c b/src/native/portaudio/net_java_sip_communicator_impl_neomedia_portaudio_PortAudio.c
index ea97907..4451a31 100644
--- a/src/native/portaudio/net_java_sip_communicator_impl_neomedia_portaudio_PortAudio.c
+++ b/src/native/portaudio/net_java_sip_communicator_impl_neomedia_portaudio_PortAudio.c
@@ -37,6 +37,7 @@ static long PortAudio_getFrameSize(PaStreamParameters *streamParameters);
 static unsigned long PortAudio_getSampleSizeInBits
     (PaStreamParameters *streamParameters);
 static void PortAudio_throwException(JNIEnv *env, PaError errorCode);
+static long System_currentTimeMillis();
 
 static int PortAudioStream_callback(
     const void *input,
@@ -255,10 +256,13 @@ Java_net_java_sip_communicator_impl_neomedia_portaudio_PortAudio_Pa_1ReadStream
 
     if (data)
     {
+        jlong startTime, endTime;
         PortAudioStream *portAudioStream = (PortAudioStream *) stream;
-        PaError errorCode
-            = Pa_ReadStream(portAudioStream->stream, data, frames);
+        PaError errorCode;
 
+        startTime = System_currentTimeMillis();
+        errorCode = Pa_ReadStream(portAudioStream->stream, data, frames);
+        endTime = System_currentTimeMillis();
         if ((paNoError == errorCode) || (paInputOverflowed == errorCode))
         {
             if (portAudioStream->audioQualityImprovement)
@@ -269,8 +273,8 @@ Java_net_java_sip_communicator_impl_neomedia_portaudio_PortAudio_Pa_1ReadStream
                     portAudioStream->sampleRate,
                     portAudioStream->sampleSizeInBits,
                     portAudioStream->channels,
-                    data,
-                    frames * portAudioStream->inputFrameSize);
+                    data, frames * portAudioStream->inputFrameSize,
+                    startTime, endTime);
             }
             (*env)->ReleaseByteArrayElements(env, buffer, data, 0);
         }
@@ -336,7 +340,11 @@ Java_net_java_sip_communicator_impl_neomedia_portaudio_PortAudio_Pa_1WriteStream
 
     for (i = 0; i < numberOfWrites; i++)
     {
+        jlong startTime, endTime;
+
+        startTime = System_currentTimeMillis();
         errorCode = Pa_WriteStream(paStream, data, frames);
+        endTime = System_currentTimeMillis();
         if ((paNoError != errorCode) && (errorCode != paOutputUnderflowed))
             break;
         else
@@ -346,11 +354,9 @@ Java_net_java_sip_communicator_impl_neomedia_portaudio_PortAudio_Pa_1WriteStream
                 AudioQualityImprovement_process(
                     audioQualityImprovement,
                     AUDIO_QUALITY_IMPROVEMENT_SAMPLE_ORIGIN_OUTPUT,
-                    sampleRate,
-                    sampleSizeInBits,
-                    channels,
-                    data,
-                    framesInBytes);
+                    sampleRate, sampleSizeInBits, channels,
+                    data, framesInBytes,
+                    startTime, endTime);
             }
             data += framesInBytes;
         }
@@ -418,20 +424,6 @@ Java_net_java_sip_communicator_impl_neomedia_portaudio_PortAudio_PaDeviceInfo_1g
     return ((PaDeviceInfo *) deviceInfo)->maxOutputChannels;
 }
 
-JNIEXPORT jstring JNICALL
-Java_net_java_sip_communicator_impl_neomedia_portaudio_PortAudio_PaDeviceInfo_1getName
-    (JNIEnv *env, jclass clazz, jlong deviceInfo)
-{
-    const char *name = ((PaDeviceInfo *) deviceInfo)->name;
-
-    /*
-     * PaDeviceInfo_getName has been deprected in the Java source code and the
-     * implementation here is left to allow the application to execute even
-     * without the recompiled JNI counterpart.
-     */
-    return name ? (*env)->NewStringUTF(env, name) : NULL;
-}
-
 JNIEXPORT jbyteArray JNICALL
 Java_net_java_sip_communicator_impl_neomedia_portaudio_PortAudio_PaDeviceInfo_1getNameBytes
     (JNIEnv *jniEnv, jclass clazz, jlong deviceInfo)
@@ -627,8 +619,7 @@ static void
 PortAudio_throwException(JNIEnv *env, PaError errorCode)
 {
     jclass clazz
-        = (*env)
-            ->FindClass(
+        = (*env)->FindClass(
                 env,
                 "net/java/sip/communicator/impl/neomedia/portaudio/PortAudioException");
 
@@ -670,8 +661,7 @@ PortAudioStream_callback(
             = (*env)->GetObjectClass(env, streamCallback);
 
         streamCallbackMethodID
-            = (*env)
-                ->GetMethodID(
+            = (*env)->GetMethodID(
                     env,
                     streamCallbackClass,
                     "callback",
@@ -683,21 +673,18 @@ PortAudioStream_callback(
     }
 
     return
-        (*env)
-            ->CallIntMethod(
+        (*env)->CallIntMethod(
                 env,
                 streamCallback,
                 streamCallbackMethodID,
                 input
-                    ? (*env)
-                        ->NewDirectByteBuffer(
+                    ? (*env)->NewDirectByteBuffer(
                             env,
                             (void *) input,
                             frameCount * stream->inputFrameSize)
                     : NULL,
                 output
-                    ? (*env)
-                        ->NewDirectByteBuffer(
+                    ? (*env)->NewDirectByteBuffer(
                             env,
                             output,
                             frameCount * stream->outputFrameSize)
@@ -732,8 +719,7 @@ PortAudioStream_finishedCallback(void *userData)
             = (*env)->GetObjectClass(env, streamCallback);
 
         streamFinishedCallbackMethodID
-            = (*env)
-                ->GetMethodID(
+            = (*env)->GetMethodID(
                     env,
                     streamCallbackClass,
                     "finishedCallback",
@@ -802,3 +788,20 @@ PortAudioStream_new(JNIEnv *env, jobject streamCallback)
 
     return stream;
 }
+
+/**
+ * Returns the current time in milliseconds (akin to
+ * <tt>java.lang.System#currentTimeMillis()</tt>).
+ *
+ * @return the current time in milliseconds
+ */
+static jlong
+System_currentTimeMillis()
+{
+    struct timeval tv;
+
+    return
+        (gettimeofday(&tv, NULL) == 0)
+            ? ((tv.tv_sec * 1000) + (tv.tv_usec / 1000))
+            : -1;
+}