Blending algorithm updates to support WIDE strip-based blending.

1) Added algorithm to select wider strips (and made it the default mode in the app).
2) Added algorithm to cross-fade blend between the wider strips.

Change-Id: I744e69edb16a20f0040ee82c3390548b78853087

6 files changed, 232 insertions, 18 deletions

diff --git a/jni/feature_mos/src/mosaic/Blend.cpp b/jni/feature_mos/src/mosaic/Blend.cpp
index 6988ace..7308a53 100644
--- a/jni/feature_mos/src/mosaic/Blend.cpp
+++ b/jni/feature_mos/src/mosaic/Blend.cpp
@@ -41,11 +41,12 @@ Blend::~Blend()
     if (m_pFrameYPyr) free(m_pFrameYPyr);
 }
 
-int Blend::initialize(int blendingType, int frame_width, int frame_height)
+int Blend::initialize(int blendingType, int stripType, int frame_width, int frame_height)
 {
     this->width = frame_width;
     this->height = frame_height;
     this->m_wb.blendingType = blendingType;
+    this->m_wb.stripType = stripType;
 
     m_wb.blendRange = m_wb.blendRangeUV = BLEND_RANGE_DEFAULT;
     m_wb.nlevs = m_wb.blendRange;
@@ -95,13 +96,27 @@ void Blend::AlignToMiddleFrame(MosaicFrame **frames, int frames_size)
     }
 }
 
-int Blend::runBlend(MosaicFrame **frames, int frames_size,
+int Blend::runBlend(MosaicFrame **oframes, MosaicFrame **rframes,
+        int frames_size,
         ImageType &imageMosaicYVU, int &mosaicWidth, int &mosaicHeight,
         float &progress, bool &cancelComputation)
 {
     int ret;
     int numCenters;
 
+    MosaicFrame **frames;
+
+    // For THIN strip mode, accept all frames for blending
+    if (m_wb.stripType == STRIP_TYPE_THIN)
+    {
+        frames = oframes;
+    }
+    else // For WIDE strip mode, first select the relevant frames to blend.
+    {
+        SelectRelevantFrames(oframes, frames_size, rframes, frames_size);
+        frames = rframes;
+    }
+
     ComputeBlendParameters(frames, frames_size, true);
     numCenters = frames_size;
 
@@ -348,6 +363,75 @@ int Blend::DoMergeAndBlend(MosaicFrame **frames, int nsite,
         site_idx++;
     }
 
+    ////////// imgMos.Y, imgMos.V, imgMos.U are used as follows //////////////
+    ////////////////////// THIN STRIP MODE ///////////////////////////////////
+
+    // imgMos.Y is used to store the index of the image from which each pixel
+    // in the output mosaic can be read out for the thin-strip mode. Thus,
+    // there is no special handling for pixels around the seam. Also, imgMos.Y
+    // is set to 255 wherever we can't get its value from any input image e.g.
+    // in the gray border areas. imgMos.V and imgMos.U are set to 128 for the
+    // thin-strip mode.
+
+    ////////////////////// WIDE STRIP MODE ///////////////////////////////////
+
+    // imgMos.Y is used the same way as the thin-strip mode.
+    // imgMos.V is used to store the index of the neighboring image which
+    // should contribute to the color of an output pixel in a band around
+    // the seam. Thus, in this band, we will crossfade between the color values
+    // from the image index imgMos.Y and image index imgMos.V. imgMos.U is
+    // used to store the weight (multiplied by 100) that each image will
+    // contribute to the blending process. Thus, we start at 99% contribution
+    // from the first image, then go to 50% contribution from each image at
+    // the seam. Then, the contribution from the second image goes up to 99%.
+
+    // For WIDE mode, set the pixel masks to guide the blender to cross-fade
+    // between the images on either side of each seam:
+    if (m_wb.stripType == STRIP_TYPE_WIDE)
+    {
+        // Set the number of pixels around the seam to cross-fade between
+        // the two component images,
+        int tw = STRIP_CROSS_FADE_WIDTH * width;
+
+        for(int y = 0; y < imgMos.Y.height; y++)
+        {
+            for(int x = tw; x < imgMos.Y.width - tw + 1; )
+            {
+                // Determine where the seam is...
+                if (imgMos.Y.ptr[y][x] != imgMos.Y.ptr[y][x+1] &&
+                        imgMos.Y.ptr[y][x] != 255 &&
+                        imgMos.Y.ptr[y][x+1] != 255)
+                {
+                    // Find the image indices on both sides of the seam
+                    unsigned char idx1 = imgMos.Y.ptr[y][x];
+                    unsigned char idx2 = imgMos.Y.ptr[y][x+1];
+
+                    for (int o = tw; o >= 0; o--)
+                    {
+                        // Set the image index to use for cross-fading
+                        imgMos.V.ptr[y][x - o] = idx2;
+                        // Set the intensity weights to use for cross-fading
+                        imgMos.U.ptr[y][x - o] = 50 + (99 - 50) * o / tw;
+                    }
+
+                    for (int o = 1; o <= tw; o++)
+                    {
+                        // Set the image index to use for cross-fading
+                        imgMos.V.ptr[y][x + o] = idx1;
+                        // Set the intensity weights to use for cross-fading
+                        imgMos.U.ptr[y][x + o] = imgMos.U.ptr[y][x - o];
+                    }
+
+                    x += (tw + 1);
+                }
+                else
+                {
+                    x++;
+                }
+            }
+        }
+    }
+
     // Now perform the actual blending using the frame assignment determined above
     site_idx = 0;
     for(CSite *csite = m_AllSites; csite < esite; csite++)
@@ -683,9 +767,36 @@ void Blend::ProcessPyramidForThisFrame(CSite *csite, BlendRect &vcrect, BlendRec
                 int inMask = ((unsigned) ii < imgMos.Y.width &&
                         (unsigned) jj < imgMos.Y.height) ? 1 : 0;
 
-                if(inMask && imgMos.Y.ptr[jj][ii]!=site_idx && imgMos.Y.ptr[jj][ii]!=255)
+                if(inMask && imgMos.Y.ptr[jj][ii] != site_idx &&
+                        imgMos.V.ptr[jj][ii] != site_idx &&
+                        imgMos.Y.ptr[jj][ii] != 255)
                     continue;
 
+                // Setup weights for cross-fading
+                // Weight of the intensity already in the output pixel
+                double wt0 = 0.0;
+                // Weight of the intensity from the input pixel (current frame)
+                double wt1 = 1.0;
+
+                if (m_wb.stripType == STRIP_TYPE_WIDE)
+                {
+                    if(inMask && imgMos.Y.ptr[jj][ii] != 255)
+                    {
+                        if(imgMos.V.ptr[jj][ii] == 128) // Not on a seam
+                        {
+                            wt0 = 0.0;
+                            wt1 = 1.0;
+                        }
+                        else
+                        {
+                            wt0 = 1.0;
+                            wt1 = ((imgMos.Y.ptr[jj][ii] == site_idx) ?
+                                    (double)imgMos.U.ptr[jj][ii] / 100.0 :
+                                    1.0 - (double)imgMos.U.ptr[jj][ii] / 100.0);
+                        }
+                    }
+                }
+
                 // Project this mosaic point into the original frame coordinate space
                 double xx, yy;
 
@@ -696,6 +807,8 @@ void Blend::ProcessPyramidForThisFrame(CSite *csite, BlendRect &vcrect, BlendRec
                     if(inMask)
                     {
                         imgMos.Y.ptr[jj][ii] = 255;
+                        wt0 = 0.0f;
+                        wt1 = 1.0f;
                     }
                 }
 
@@ -708,15 +821,19 @@ void Blend::ProcessPyramidForThisFrame(CSite *csite, BlendRect &vcrect, BlendRec
 
                 // Final destination in extended pyramid
 #ifndef LINEAR_INTERP
-                if(inSegment(x1, sptr->width, BORDER-1) && inSegment(y1, sptr->height, BORDER-1))
+                if(inSegment(x1, sptr->width, BORDER-1) &&
+                        inSegment(y1, sptr->height, BORDER-1))
                 {
                     double xfrac = xx - x1;
                     double yfrac = yy - y1;
-                    dptr->ptr[j][i] = (short) (.5 + ciCalc(sptr, x1, y1, xfrac, yfrac));
+                    dptr->ptr[j][i] = (short) (wt0 * dptr->ptr[j][i] + .5 +
+                            wt1 * ciCalc(sptr, x1, y1, xfrac, yfrac));
                     if (dvptr >= m_pMosaicVPyr && nC > 0)
                     {
-                        duptr->ptr[j][i] = (short) (.5 + ciCalc(suptr, x1, y1, xfrac, yfrac));
-                        dvptr->ptr[j][i] = (short) (.5 + ciCalc(svptr, x1, y1, xfrac, yfrac));
+                        duptr->ptr[j][i] = (short) (wt0 * duptr->ptr[j][i] + .5 +
+                                wt1 * ciCalc(suptr, x1, y1, xfrac, yfrac));
+                        dvptr->ptr[j][i] = (short) (wt0 * dvptr->ptr[j][i] + .5 +
+                                wt1 * ciCalc(svptr, x1, y1, xfrac, yfrac));
                     }
                 }
 #else
@@ -755,11 +872,14 @@ void Blend::ProcessPyramidForThisFrame(CSite *csite, BlendRect &vcrect, BlendRec
                     clipToSegment(x1, sptr->width, BORDER);
                     clipToSegment(y1, sptr->height, BORDER);
 
-                    dptr->ptr[j][i] = sptr->ptr[y1][x1];
+                    dptr->ptr[j][i] = (short) (wt0 * dptr->ptr[j][i] + 0.5 +
+                            wt1 * sptr->ptr[y1][x1] );
                     if (dvptr >= m_pMosaicVPyr && nC > 0)
                     {
-                        dvptr->ptr[j][i] = svptr->ptr[y1][x1];
-                        duptr->ptr[j][i] = suptr->ptr[y1][x1];
+                        dvptr->ptr[j][i] = (short) (wt0 * dvptr->ptr[j][i] +
+                                0.5 + wt1 * svptr->ptr[y1][x1] );
+                        duptr->ptr[j][i] = (short) (wt0 * duptr->ptr[j][i] +
+                                0.5 + wt1 * suptr->ptr[y1][x1] );
                     }
                 }
             }
@@ -907,7 +1027,50 @@ void Blend::FrameToMosaicRect(int width, int height, double trs[3][3], BlendRect
     }
 }
 
+void Blend::SelectRelevantFrames(MosaicFrame **frames, int frames_size,
+        MosaicFrame **relevant_frames, int &relevant_frames_size)
+{
+    MosaicFrame *first = frames[0];
+    MosaicFrame *last = frames[frames_size-1];
+    MosaicFrame *mb;
+
+    double fxpos = first->trs[0][2], fypos = first->trs[1][2];
+
+    double midX = last->width / 2.0;
+    double midY = last->height / 2.0;
+    double z = ProjZ(first->trs, midX, midY, 1.0);
+    double firstX, firstY;
+    double prevX = firstX = ProjX(first->trs, midX, midY, z, 1.0);
+    double prevY = firstY = ProjY(first->trs, midX, midY, z, 1.0);
+
+    relevant_frames[0] = first; // Add first frame by default
+    relevant_frames_size = 1;
 
+    for (int i = 0; i < frames_size - 1; i++)
+    {
+        mb = frames[i];
+        double currX, currY;
+        z = ProjZ(mb->trs, midX, midY, 1.0);
+        currX = ProjX(mb->trs, midX, midY, z, 1.0);
+        currY = ProjY(mb->trs, midX, midY, z, 1.0);
+        double deltaX = currX - prevX;
+        double deltaY = currY - prevY;
+        double center2centerDist = sqrt(deltaY * deltaY + deltaX * deltaX);
+
+        if (fabs(deltaX) > STRIP_SEPARATION_THRESHOLD * last->width)
+        {
+            relevant_frames[relevant_frames_size] = mb;
+            relevant_frames_size++;
+
+            prevX = currX;
+            prevY = currY;
+        }
+    }
+
+    // Add last frame by default
+    relevant_frames[relevant_frames_size] = last;
+    relevant_frames_size++;
+}
 
 void Blend::ComputeBlendParameters(MosaicFrame **frames, int frames_size, int is360)
 {
diff --git a/jni/feature_mos/src/mosaic/Blend.h b/jni/feature_mos/src/mosaic/Blend.h
index 91fbec9..a5bc05b 100644
--- a/jni/feature_mos/src/mosaic/Blend.h
+++ b/jni/feature_mos/src/mosaic/Blend.h
@@ -33,6 +33,16 @@ const float TIME_PERCENT_ALIGN = 20.0;
 const float TIME_PERCENT_BLEND = 75.0;
 const float TIME_PERCENT_FINAL = 5.0;
 
+// This threshold determines the minimum separation between the image centers
+// of the input image frames for them to be accepted for blending in the
+// STRIP_TYPE_WIDE mode. This threshold is specified as a fraction of the
+// input image frame width.
+const float STRIP_SEPARATION_THRESHOLD = 0.10;
+
+// This threshold determines the number of pixels on either side of the strip
+// to cross-fade using the images contributing to each seam. This threshold
+// is specified as a fraction of the input image frame width.
+const float STRIP_CROSS_FADE_WIDTH = 0.02;
 /**
  *  Class for pyramid blending a mosaic.
  */
@@ -46,6 +56,9 @@ public:
   static const int BLEND_TYPE_CYLPAN  = 2;
   static const int BLEND_TYPE_HORZ   = 3;
 
+  static const int STRIP_TYPE_THIN      = 0;
+  static const int STRIP_TYPE_WIDE      = 1;
+
   static const int BLEND_RET_ERROR        = -1;
   static const int BLEND_RET_OK           = 0;
   static const int BLEND_RET_ERROR_MEMORY = 1;
@@ -54,9 +67,9 @@ public:
   Blend();
   ~Blend();
 
-  int initialize(int blendingType, int frame_width, int frame_height);
+  int initialize(int blendingType, int stripType, int frame_width, int frame_height);
 
-  int runBlend(MosaicFrame **frames, int frames_size, ImageType &imageMosaicYVU,
+  int runBlend(MosaicFrame **frames, MosaicFrame **rframes, int frames_size, ImageType &imageMosaicYVU,
         int &mosaicWidth, int &mosaicHeight, float &progress, bool &cancelComputation);
 
 protected:
@@ -95,6 +108,8 @@ protected:
 
   // TODO: need to add documentation about the parameters
   void ComputeBlendParameters(MosaicFrame **frames, int frames_size, int is360);
+  void SelectRelevantFrames(MosaicFrame **frames, int frames_size,
+        MosaicFrame **relevant_frames, int &relevant_frames_size);
 
   int  PerformFinalBlending(YUVinfo &imgMos, MosaicRect &cropping_rect);
   void CropFinalMosaic(YUVinfo &imgMos, MosaicRect &cropping_rect);
diff --git a/jni/feature_mos/src/mosaic/Mosaic.cpp b/jni/feature_mos/src/mosaic/Mosaic.cpp
index abbeb96..9eee2b3 100644
--- a/jni/feature_mos/src/mosaic/Mosaic.cpp
+++ b/jni/feature_mos/src/mosaic/Mosaic.cpp
@@ -45,6 +45,7 @@ Mosaic::~Mosaic()
             delete frames[i];
     }
     delete frames;
+    delete rframes;
 
     if (aligner != NULL)
         delete aligner;
@@ -52,16 +53,27 @@ Mosaic::~Mosaic()
         delete blender;
 }
 
-int Mosaic::initialize(int blendingType, int width, int height, int nframes, bool quarter_res, float thresh_still)
+int Mosaic::initialize(int blendingType, int stripType, int width, int height, int nframes, bool quarter_res, float thresh_still)
 {
     this->blendingType = blendingType;
+
+    // TODO: Review this logic if enabling FULL or PAN mode
+    if (blendingType == Blend::BLEND_TYPE_FULL ||
+            blendingType == Blend::BLEND_TYPE_PAN)
+    {
+        stripType = Blend::STRIP_TYPE_THIN;
+    }
+
+    this->stripType = stripType;
     this->width = width;
     this->height = height;
 
+
     mosaicWidth = mosaicHeight = 0;
     imageMosaicYVU = NULL;
 
     frames = new MosaicFrame *[max_frames];
+    rframes = new MosaicFrame *[max_frames];
 
     if(nframes>-1)
     {
@@ -92,7 +104,7 @@ int Mosaic::initialize(int blendingType, int width, int height, int nframes, boo
             blendingType == Blend::BLEND_TYPE_CYLPAN ||
             blendingType == Blend::BLEND_TYPE_HORZ) {
         blender = new Blend();
-        blender->initialize(blendingType, width, height);
+        blender->initialize(blendingType, stripType, width, height);
     } else {
         blender = NULL;
         LOGE("Error: Unknown blending type %d",blendingType);
@@ -180,7 +192,8 @@ int Mosaic::createMosaic(float &progress, bool &cancelComputation)
     // Blend the mosaic (alignment has already been done)
     if (blender != NULL)
     {
-        ret = blender->runBlend((MosaicFrame **) frames, frames_size, imageMosaicYVU,
+        ret = blender->runBlend((MosaicFrame **) frames, (MosaicFrame **) rframes, 
+                frames_size, imageMosaicYVU,
                 mosaicWidth, mosaicHeight, progress, cancelComputation);
     }
 
diff --git a/jni/feature_mos/src/mosaic/Mosaic.h b/jni/feature_mos/src/mosaic/Mosaic.h
index ecf0536..36eafe7 100644
--- a/jni/feature_mos/src/mosaic/Mosaic.h
+++ b/jni/feature_mos/src/mosaic/Mosaic.h
@@ -40,6 +40,7 @@
     Mosaic mosaic;
     // Define blending types to use, and the frame dimensions
     int blendingType = Blend::BLEND_TYPE_CYLPAN;
+    int stripType = Blend::STRIP_TYPE_THIN;
     int width = 640;
     int height = 480;
 
@@ -49,7 +50,7 @@
         if (!mosaic.isInitialized())
         {
           // Initialize mosaic processing
-          mosaic.initialize(blendingType, width, height, -1, false, 5.0f);
+          mosaic.initialize(blendingType, stripType, width, height, -1, false, 5.0f);
         }
 
         // Add to list of frames
@@ -84,6 +85,9 @@ public:
    /*!
     *   Creates the aligner and blender and initializes state.
     *   \param blendingType Type of blending to perform
+    *   \param stripType    Type of strip to use. 0: thin, 1: wide. stripType
+    *                       is effective only when blendingType is CylPan or
+    *                       Horz. Otherwise, it is set to thin irrespective of the input.
     *   \param width        Width of input images (note: all images must be same size)
     *   \param height       Height of input images (note: all images must be same size)
     *   \param nframes      Number of frames to pre-allocate; default value -1 will allocate each frame as it comes
@@ -91,7 +95,7 @@ public:
     *   \param thresh_still Minimum number of pixels of translation detected between the new frame and the last frame before this frame is added to be mosaiced. For the low-res processing at 320x180 resolution input, we set this to 5 pixels. To reject no frames, set this to 0.0 (default value).
     *   \return             Return code signifying success or failure.
     */
-  int initialize(int blendingType, int width, int height, int nframes = -1, bool quarter_res = false, float thresh_still = 0.0);
+  int initialize(int blendingType, int stripType, int width, int height, int nframes = -1, bool quarter_res = false, float thresh_still = 0.0);
 
    /*!
     *   Adds a YVU frame to the mosaic.
@@ -166,6 +170,12 @@ protected:
    * Collection of frames that will make up mosaic.
    */
   MosaicFrame **frames;
+
+  /**
+    * Subset of frames that are considered as relevant.
+    */
+  MosaicFrame **rframes;
+
   int frames_size;
   int max_frames;
 
@@ -180,6 +190,11 @@ protected:
   int blendingType;
 
   /**
+    * Type of strip to use. 0: thin (default), 1: wide
+    */
+  int stripType;
+
+  /**
    *  Pointer to aligner.
    */
   Align *aligner;
diff --git a/jni/feature_mos/src/mosaic/MosaicTypes.h b/jni/feature_mos/src/mosaic/MosaicTypes.h
index 242335b..395ec45 100644
--- a/jni/feature_mos/src/mosaic/MosaicTypes.h
+++ b/jni/feature_mos/src/mosaic/MosaicTypes.h
@@ -145,6 +145,7 @@ typedef struct {
   int nlevs;
   int nlevsC;
   int blendingType;
+  int stripType;
   // Add an overlap to prevent a gap between pictures due to roundoffs
   double roundoffOverlap;// 1.5
 
diff --git a/jni/feature_mos_jni.cpp b/jni/feature_mos_jni.cpp
index fdece27..4ad3db3 100644
--- a/jni/feature_mos_jni.cpp
+++ b/jni/feature_mos_jni.cpp
@@ -63,6 +63,7 @@ int mosaicWidth=0, mosaicHeight=0;
 //int blendingType = Blend::BLEND_TYPE_FULL;
 //int blendingType = Blend::BLEND_TYPE_CYLPAN;
 int blendingType = Blend::BLEND_TYPE_HORZ;
+int stripType = Blend::STRIP_TYPE_THIN;
 bool high_res = false;
 bool quarter_res[NR] = {false,false};
 float thresh_still[NR] = {5.0f,0.0f};
@@ -108,7 +109,7 @@ int Init(int mID, int nmax)
         // Check for initialization and if not, initialize
         if (!mosaic[mID]->isInitialized())
         {
-                mosaic[mID]->initialize(blendingType, tWidth[mID], tHeight[mID],
+                mosaic[mID]->initialize(blendingType, stripType, tWidth[mID], tHeight[mID],
                         nmax, quarter_res[mID], thresh_still[mID]);
         }
 
@@ -477,6 +478,12 @@ JNIEXPORT void JNICALL Java_com_android_camera_panorama_Mosaic_setBlendingType(
     blendingType = int(type);
 }
 
+JNIEXPORT void JNICALL Java_com_android_camera_panorama_Mosaic_setStripType(
+        JNIEnv* env, jobject thiz, jint type)
+{
+    stripType = int(type);
+}
+
 JNIEXPORT void JNICALL Java_com_android_camera_panorama_Mosaic_reset(
         JNIEnv* env, jobject thiz)
 {