Updates to handle textureless scenes during capture.

1) Starts stitching only when the camera sees a textured scene at the beginning.
2) If a texturess scene is encountered in the middle of a capture, the stitching continues with the intermediate frames translated using the pan velocity estimate.
3) Added more error codes and percolated them up to the java layer.
4) Fix a build error in Mosaic::addFrame() and added comments.
5) Update the javadoc in Mosaic.java to reflect the new returning codes.

Change-Id: I7727ace615ece22adefe313a19ac2cbe8c8d21a8

7 files changed, 159 insertions, 50 deletions

diff --git a/jni/feature_mos/src/mosaic/AlignFeatures.cpp b/jni/feature_mos/src/mosaic/AlignFeatures.cpp
index a181dd8..aeabf8f 100644
--- a/jni/feature_mos/src/mosaic/AlignFeatures.cpp
+++ b/jni/feature_mos/src/mosaic/AlignFeatures.cpp
@@ -26,12 +26,18 @@
 #include "trsMatrix.h"
 #include "MatrixUtils.h"
 #include "AlignFeatures.h"
+#include "Log.h"
+
+#define LOG_TAG "AlignFeatures"
 
 Align::Align()
 {
   width = height = 0;
   frame_number = 0;
+  num_frames_captured = 0;
+  reference_frame_index = 0;
   db_Identity3x3(Hcurr);
+  db_Identity3x3(Hprev);
 }
 
 Align::~Align()
@@ -54,8 +60,8 @@ int Align::initialize(int width, int height, bool _quarter_res, float _thresh_st
   int nrsamples = DB_DEFAULT_NR_SAMPLES;
   double scale = DB_POINT_STANDARDDEV;
   int chunk_size = DB_DEFAULT_CHUNK_SIZE;
-  int nrhorz = 20; // 1280/32 = 40
-  int nrvert = 12; // 720/30 = 24
+  int nrhorz = width/48;  // Empirically determined number of horizontal
+  int nrvert = height/60; // and vertical buckets for harris corner detection.
   bool linear_polish = false;
   unsigned int reference_update_period = DEFAULT_REFERENCE_UPDATE_PERIOD;
 
@@ -66,10 +72,17 @@ int Align::initialize(int width, int height, bool _quarter_res, float _thresh_st
   thresh_still = _thresh_still;
 
   frame_number = 0;
+  num_frames_captured = 0;
+  reference_frame_index = 0;
   db_Identity3x3(Hcurr);
+  db_Identity3x3(Hprev);
+
   if (!reg.Initialized())
   {
-    reg.Init(width,height,motion_model_type,20,linear_polish,quarter_res,scale,reference_update_period, false, 0, nrsamples,chunk_size,nr_corners,max_disparity,use_smaller_matching_window, nrhorz, nrvert);
+    reg.Init(width, height, motion_model_type, 20, linear_polish, quarter_res,
+            scale, reference_update_period, false, 0, nrsamples, chunk_size,
+            nr_corners, max_disparity, use_smaller_matching_window,
+            nrhorz, nrvert);
   }
   this->width = width;
   this->height = height;
@@ -90,47 +103,90 @@ int Align::addFrameRGB(ImageType imageRGB)
 
 int Align::addFrame(ImageType imageGray_)
 {
-  // compute the homography:
-  double Hinv[9];
-  double Hinv33[3][3];
-  double Hprev33[3][3];
-  double Hcurr33[3][3];
+  int ret_code = ALIGN_RET_OK;
 
  // Obtain a vector of pointers to rows in image and pass in to dbreg
   ImageType *m_rows = ImageUtils::imageTypeToRowPointers(imageGray_, width, height);
-  reg.AddFrame(m_rows, Hcurr);
 
+  if (frame_number == 0)
+  {
+      reg.AddFrame(m_rows, Hcurr, true);    // Force this to be a reference frame
+      int num_corner_ref = reg.GetNrRefCorners();
+
+      if (num_corner_ref < MIN_NR_REF_CORNERS)
+      {
+          return ALIGN_RET_LOW_TEXTURE;
+      }
+  }
+  else
+  {
+      reg.AddFrame(m_rows, Hcurr, false);
+  }
+
+  // Average translation per frame =
+  //    [Translation from Frame0 to Frame(n-1)] / [(n-1)]
+  average_tx_per_frame = (num_frames_captured < 2) ? 0.0 :
+        Hprev[2] / (num_frames_captured - 1);
+
+  // Increment the captured frame counter if we already have a reference frame
+  num_frames_captured++;
 
   if (frame_number != 0)
   {
+    int num_inliers = reg.GetNrInliers();
+
+    if(num_inliers < MIN_NR_INLIERS)
+    {
+        ret_code = ALIGN_RET_FEW_INLIERS;
+
+        Hcurr[0] = 1.0;
+        Hcurr[1] = 0.0;
+        // Set this as the average per frame translation taking into acccount
+        // the separation of the current frame from the reference frame...
+        Hcurr[2] = -average_tx_per_frame *
+                (num_frames_captured - reference_frame_index);
+        Hcurr[3] = 0.0;
+        Hcurr[4] = 1.0;
+        Hcurr[5] = 0.0;
+        Hcurr[6] = 0.0;
+        Hcurr[7] = 0.0;
+        Hcurr[8] = 1.0;
+    }
 
     if(fabs(Hcurr[2])<thresh_still && fabs(Hcurr[5])<thresh_still)  // Still camera
     {
         return ALIGN_RET_ERROR;
     }
 
+    // compute the homography:
+    double Hinv33[3][3];
+    double Hprev33[3][3];
+    double Hcurr33[3][3];
+
     // Invert and multiple with previous transformation
     Matrix33::convert9to33(Hcurr33, Hcurr);
     Matrix33::convert9to33(Hprev33, Hprev);
-    //NormalizeProjMat(Hcurr33);
     normProjMat33d(Hcurr33);
 
     inv33d(Hcurr33, Hinv33);
 
     mult33d(Hcurr33, Hprev33, Hinv33);
     normProjMat33d(Hcurr33);
-    Matrix9::convert33to9(Hcurr, Hcurr33);
+    Matrix9::convert33to9(Hprev, Hcurr33);
+    // Since we have already factored the current transformation
+    // into Hprev, we can reset the Hcurr to identity
+    db_Identity3x3(Hcurr);
 
+    // Update the reference frame to be the current frame
     reg.UpdateReference(m_rows,quarter_res,false);
+
+    // Update the reference frame index
+    reference_frame_index = num_frames_captured;
   }
 
   frame_number++;
 
-  // Copy curr to prev
-  memcpy(Happly, Hcurr, sizeof(double)*9);
-  memcpy(Hprev, Hcurr, sizeof(double)*9);
-
-  return ALIGN_RET_OK;
+  return ret_code;
 }
 
 // Get current transformation
@@ -138,11 +194,38 @@ int Align::getLastTRS(double trs[3][3])
 {
   if (frame_number < 1)
   {
-    fprintf(stderr, "Error: Align::getLastTRS called before a frame was processed\n");
+    trs[0][0] = 1.0;
+    trs[0][1] = 0.0;
+    trs[0][2] = 0.0;
+    trs[1][0] = 0.0;
+    trs[1][1] = 1.0;
+    trs[1][2] = 0.0;
+    trs[2][0] = 0.0;
+    trs[2][1] = 0.0;
+    trs[2][2] = 1.0;
     return ALIGN_RET_ERROR;
   }
 
-  Matrix33::convert9to33(trs, Happly);
+  // Note that the logic here handles the case, where a frame is not used for
+  // mosaicing but is captured and used in the preview-rendering.
+  // For these frames, we don't set Hcurr to identity in AddFrame() and the
+  // logic here appends their transformation to Hprev to render them with the
+  // correct transformation. For the frames we do use for mosaicing, we already
+  // append their Hcurr to Hprev in AddFrame() and then set Hcurr to identity.
+
+  double Hinv33[3][3];
+  double Hprev33[3][3];
+  double Hcurr33[3][3];
+
+  Matrix33::convert9to33(Hcurr33, Hcurr);
+  normProjMat33d(Hcurr33);
+  inv33d(Hcurr33, Hinv33);
+
+  Matrix33::convert9to33(Hprev33, Hprev);
+
+  mult33d(trs, Hprev33, Hinv33);
+  normProjMat33d(trs);
+
   return ALIGN_RET_OK;
 }
 
diff --git a/jni/feature_mos/src/mosaic/AlignFeatures.h b/jni/feature_mos/src/mosaic/AlignFeatures.h
index 06be596..19f3905 100644
--- a/jni/feature_mos/src/mosaic/AlignFeatures.h
+++ b/jni/feature_mos/src/mosaic/AlignFeatures.h
@@ -36,8 +36,10 @@ public:
   static const int ALIGN_TYPE_PAN    = 1;
 
   // Return codes
+  static const int ALIGN_RET_LOW_TEXTURE  = -2;
   static const int ALIGN_RET_ERROR        = -1;
   static const int ALIGN_RET_OK           = 0;
+  static const int ALIGN_RET_FEW_INLIERS  = 1;
 
   ///// Settings for feature-based alignment
   // Number of features to use from corner detection
@@ -49,6 +51,9 @@ public:
 //  static const int DEFAULT_MOTION_MODEL=DB_HOMOGRAPHY_TYPE_AFFINE;
   static const unsigned int DEFAULT_REFERENCE_UPDATE_PERIOD=1500; //  Manual reference frame update so set this to a large number
 
+  static const int MIN_NR_REF_CORNERS = 25;
+  static const int MIN_NR_INLIERS = 10;
+
   Align();
   ~Align();
 
@@ -69,10 +74,13 @@ protected:
   db_FrameToReferenceRegistration reg;
 
   int frame_number;
-  double Happly[9];  // Homography to apply
-  double Hcurr[9];   // Homography from last frame
-                     // (right now same as above)
-  double Hprev[9];   // Homography up until the last frame
+
+  double Hcurr[9];   // Homography from the alignment reference to the frame-t
+  double Hprev[9];   // Homography from frame-0 to the frame-(t-1)
+
+  int reference_frame_index; // Index of the reference frame from all captured frames
+  int num_frames_captured; // Total number of frames captured (different from frame_number)
+  double average_tx_per_frame; // Average pixel translation per captured frame
 
   int width,height;
 
diff --git a/jni/feature_mos/src/mosaic/Mosaic.cpp b/jni/feature_mos/src/mosaic/Mosaic.cpp
index 9eee2b3..f17c030 100644
--- a/jni/feature_mos/src/mosaic/Mosaic.cpp
+++ b/jni/feature_mos/src/mosaic/Mosaic.cpp
@@ -135,43 +135,49 @@ int Mosaic::addFrame(ImageType imageYVU)
 
     frame->image = imageYVU;
 
-    int align_flag = Align::ALIGN_RET_OK;
-
     // Add frame to aligner
+    int ret = MOSAIC_RET_ERROR;
     if (aligner != NULL)
     {
         // Note aligner takes in RGB images
-        printf("Adding frame to aligner...\n");
+        int align_flag = Align::ALIGN_RET_OK;
         align_flag = aligner->addFrame(frame->image);
         aligner->getLastTRS(frame->trs);
 
-        printf("Frame width %d,%d\n", frame->width, frame->height);
         if (frames_size >= max_frames)
         {
-            fprintf(stderr, "WARNING: More frames than preallocated, ignoring.  Increase maximum number of frames (-f <max_frames>) to avoid this\n");
+            LOGV("WARNING: More frames than preallocated, ignoring."
+                 "Increase maximum number of frames (-f <max_frames>) to avoid this");
             return MOSAIC_RET_ERROR;
         }
-        else if(align_flag == Align::ALIGN_RET_OK)
-        {
-            frames_size++;
-            return MOSAIC_RET_OK;
-        }
-        else
+
+        switch (align_flag)
         {
-            return MOSAIC_RET_ERROR;
+            case Align::ALIGN_RET_OK:
+                frames_size++;
+                ret = MOSAIC_RET_OK;
+                break;
+            case Align::ALIGN_RET_FEW_INLIERS:
+                frames_size++;
+                ret = MOSAIC_RET_FEW_INLIERS;
+                break;
+            case Align::ALIGN_RET_LOW_TEXTURE:
+                ret = MOSAIC_RET_LOW_TEXTURE;
+                break;
+            case Align::ALIGN_RET_ERROR:
+                ret = MOSAIC_RET_ERROR;
+                break;
+            default:
+                break;
         }
     }
-    else
-    {
-        return MOSAIC_RET_ERROR;
-    }
+
+    return ret;
 }
 
 
 int Mosaic::createMosaic(float &progress, bool &cancelComputation)
 {
-    printf("Creating mosaic\n");
-
     if (frames_size <= 0)
     {
         // Haven't accepted any frame in aligner. No need to do blending.
diff --git a/jni/feature_mos/src/mosaic/Mosaic.h b/jni/feature_mos/src/mosaic/Mosaic.h
index 36eafe7..fc6ecd9 100644
--- a/jni/feature_mos/src/mosaic/Mosaic.h
+++ b/jni/feature_mos/src/mosaic/Mosaic.h
@@ -146,6 +146,8 @@ public:
   static const int MOSAIC_RET_OK    = 1;
   static const int MOSAIC_RET_ERROR = -1;
   static const int MOSAIC_RET_CANCELLED = -2;
+  static const int MOSAIC_RET_LOW_TEXTURE = -3;
+  static const int MOSAIC_RET_FEW_INLIERS = 2;
 
 protected:
 
diff --git a/jni/feature_mos_jni.cpp b/jni/feature_mos_jni.cpp
index 5d02793..db9fd7d 100644
--- a/jni/feature_mos_jni.cpp
+++ b/jni/feature_mos_jni.cpp
@@ -50,7 +50,7 @@ ImageType tImage[NR][MAX_FRAMES_LR];// = {{ImageUtils::IMAGE_TYPE_NOIMAGE}}; //
 Mosaic *mosaic[NR] = {NULL,NULL};
 ImageType resultYVU = ImageUtils::IMAGE_TYPE_NOIMAGE;
 ImageType resultBGR = ImageUtils::IMAGE_TYPE_NOIMAGE;
-float gTRS[10];
+float gTRS[11]; // 9 elements of the transformation, 1 for frame-number, 1 for alignment error code.
 // Variables to keep track of the mosaic computation progress for both LR & HR.
 float gProgress[NR];
 // Variables to be able to cancel the mosaic computation when the GUI says so.
@@ -373,6 +373,7 @@ JNIEXPORT jfloatArray JNICALL Java_com_android_camera_panorama_Mosaic_setSourceI
 {
     double  t0, t1, time_c;
     t0 = now_ms();
+    int ret_code;
 
     if(frame_number_HR<MAX_FRAMES_HR && frame_number_LR<MAX_FRAMES_LR)
     {
@@ -384,9 +385,9 @@ JNIEXPORT jfloatArray JNICALL Java_com_android_camera_panorama_Mosaic_setSourceI
 
         sem_post(&gPreviewImage_semaphore);
 
-        int ret_code = AddFrame(LR, frame_number_LR, gTRS);
+        ret_code = AddFrame(LR, frame_number_LR, gTRS);
 
-        if(ret_code == Mosaic::MOSAIC_RET_OK)
+        if(ret_code == Mosaic::MOSAIC_RET_OK || ret_code == Mosaic::MOSAIC_RET_FEW_INLIERS)
         {
             // Copy into HR buffer only if this is a valid frame
             sem_wait(&gPreviewImage_semaphore);
@@ -407,11 +408,12 @@ JNIEXPORT jfloatArray JNICALL Java_com_android_camera_panorama_Mosaic_setSourceI
     UpdateWarpTransformation(gTRS);
 
     gTRS[9] = frame_number_HR;
+    gTRS[10] = ret_code;
 
-    jfloatArray bytes = env->NewFloatArray(10);
+    jfloatArray bytes = env->NewFloatArray(11);
     if(bytes != 0)
     {
-        env->SetFloatArrayRegion(bytes, 0, 10, (jfloat*) gTRS);
+        env->SetFloatArrayRegion(bytes, 0, 11, (jfloat*) gTRS);
     }
     return bytes;
 }
@@ -424,6 +426,8 @@ JNIEXPORT jfloatArray JNICALL Java_com_android_camera_panorama_Mosaic_setSourceI
     double  t0, t1, time_c;
     t0 = now_ms();
 
+    int ret_code;
+
     if(frame_number_HR<MAX_FRAMES_HR && frame_number_LR<MAX_FRAMES_LR)
     {
         jbyte *pixels = env->GetByteArrayElements(photo_data, 0);
@@ -445,9 +449,9 @@ JNIEXPORT jfloatArray JNICALL Java_com_android_camera_panorama_Mosaic_setSourceI
                 gPreviewImageWidth[LR], gPreviewImageHeight[LR]);
         sem_post(&gPreviewImage_semaphore);
 
-        int ret_code = AddFrame(LR, frame_number_LR, gTRS);
+        ret_code = AddFrame(LR, frame_number_LR, gTRS);
 
-        if(ret_code == Mosaic::MOSAIC_RET_OK)
+        if(ret_code == Mosaic::MOSAIC_RET_OK || ret_code == Mosaic::MOSAIC_RET_FEW_INLIERS)
         {
             frame_number_LR++;
             frame_number_HR++;
@@ -463,11 +467,12 @@ JNIEXPORT jfloatArray JNICALL Java_com_android_camera_panorama_Mosaic_setSourceI
     UpdateWarpTransformation(gTRS);
 
     gTRS[9] = frame_number_HR;
+    gTRS[10] = ret_code;
 
-    jfloatArray bytes = env->NewFloatArray(10);
+    jfloatArray bytes = env->NewFloatArray(11);
     if(bytes != 0)
     {
-        env->SetFloatArrayRegion(bytes, 0, 10, (jfloat*) gTRS);
+        env->SetFloatArrayRegion(bytes, 0, 11, (jfloat*) gTRS);
     }
     return bytes;
 }
diff --git a/jni/feature_stab/src/dbreg/dbreg.cpp b/jni/feature_stab/src/dbreg/dbreg.cpp
index fb42838..dc7d58f 100644
--- a/jni/feature_stab/src/dbreg/dbreg.cpp
+++ b/jni/feature_stab/src/dbreg/dbreg.cpp
@@ -344,7 +344,7 @@ bool db_FrameToReferenceRegistration::NeedReferenceUpdate()
 int db_FrameToReferenceRegistration::AddFrame(const unsigned char * const * im, double H[9],bool force_reference,bool prewarp)
 {
   m_current_is_reference = false;
-  if(!m_reference_set)
+  if(!m_reference_set || force_reference)
     {
       db_Identity3x3(m_H_ref_to_ins);
       db_Copy9(H,m_H_ref_to_ins);
diff --git a/jni/feature_stab/src/dbreg/dbreg.h b/jni/feature_stab/src/dbreg/dbreg.h
index 92cd0e3..4eb2444 100644
--- a/jni/feature_stab/src/dbreg/dbreg.h
+++ b/jni/feature_stab/src/dbreg/dbreg.h
@@ -222,6 +222,11 @@ public:
     int GetNrMatches() { return m_nr_matches; }
 
     /*!
+     * Returns the number of corners detected in the current reference image.
+    */
+    int GetNrRefCorners() { return m_nr_corners_ref; }
+
+    /*!
      * Returns the pointer to an array of indices that were found to be RANSAC inliers from the matched corner lists.
     */
     int* GetInliers() { return m_inlier_indices; }