Updates to allow using SurfaceTexture for reading the preview frames directly from GPU memory.

1) SurfaceTexture is now used to obtain the data processed by the mosaicing library.
2) SurfaceTexture in GPU memory is directly rendered using the transformation from the mosaicing library to generate the preview mosaic.
3) GPU is also used to generate the Low-Res frames from the High-res frames (was being done in CPU before).
4) SurfaceTexture is also used to render the viewfinder as soon as the mosaicing application starts (eliminating the need for a separate SurfaceHolder to render the camera).
5) Modified the XML layout during the preview state to be the same size as during the capture stage to accommodate the SurfaceTexture based viewfinder [this needs to be reviewed and adjusted].
6) Fixed the viewfinder and back button issues identified by Wei-Ta.
7) Round-1 of removing trailing spaces and tabs.
8) Added documentation to new Java side interfaces and cleaned up code in general.
9) Cleaned up redundant and commented out code from the native side.
10) Merged with latest updates from the main trunk.
11) Fixed issues identified in code review and also cleaned up and refactored some code.
12) Added layout-w1024dp/pano_capture.xml for tablet layout.

Change-Id: If8fb0116de6c7fc6652cc67ac453553726961c32

1 files changed, 190 insertions, 76 deletions

diff --git a/jni/mosaic_renderer_jni.cpp b/jni/mosaic_renderer_jni.cpp
index 21d3d39..c7e11b0 100644
--- a/jni/mosaic_renderer_jni.cpp
+++ b/jni/mosaic_renderer_jni.cpp
@@ -6,6 +6,7 @@
 #include "mosaic/ImageUtils.h"
 #include "mosaic_renderer/FrameBuffer.h"
 #include "mosaic_renderer/WarpRenderer.h"
+#include "mosaic_renderer/SurfaceTextureRenderer.h"
 #include <stdio.h>
 #include <stdlib.h>
 #include <math.h>
@@ -14,22 +15,25 @@
 
 #define  LOG_TAG    "MosaicRenderer"
 #define  LOGI(...)  __android_log_print(ANDROID_LOG_INFO,LOG_TAG,__VA_ARGS__)
+#define  LOGV(...)  __android_log_print(ANDROID_LOG_VERBOSE,LOG_TAG,__VA_ARGS__)
 #define  LOGE(...)  __android_log_print(ANDROID_LOG_ERROR,LOG_TAG,__VA_ARGS__)
 
 // Texture handle
-GLuint textureId[1];
+GLuint gSurfaceTextureID[1];
 
-bool warp_image = true;
+bool gWarpImage = true;
 
-// Low-Res input image frame in RGB format for preview rendering
-unsigned char* gPreviewImageRGB;
-// Low-Res preview image width
-int gPreviewImageRGBWidth;
-// Low-Res preview image height
-int gPreviewImageRGBHeight;
+// Low-Res input image frame in RGB format for preview rendering and processing
+// and high-res RGB input image for processing.
+unsigned char* gPreviewImageRGB[NR];
+// Low-Res & high-res preview image width
+int gPreviewImageRGBWidth[NR];
+// Low-Res & high-res preview image height
+int gPreviewImageRGBHeight[NR];
 
 // Semaphore to protect simultaneous read/writes from gPreviewImageRGB
 sem_t gPreviewImageRGB_semaphore;
+sem_t gPreviewImageReady_semaphore;
 
 // Off-screen preview FBO width (large enough to store the entire
 // preview mosaic).
@@ -38,19 +42,26 @@ int gPreviewFBOWidth;
 // preview mosaic).
 int gPreviewFBOHeight;
 
+// Shader to copy input SurfaceTexture into and RGBA FBO. The two shaders
+// render to the textures with dimensions corresponding to the low-res and
+// high-res image frames.
+SurfaceTextureRenderer gSurfTexRenderer[NR];
+// Off-screen FBOs to store the low-res and high-res RGBA copied out from
+// the SurfaceTexture by the gSurfTexRenderers.
+FrameBuffer gBufferInput[NR];
 // Shader to add warped current frame to the preview FBO
 WarpRenderer gWarper;
 // Shader to warp and render the preview FBO to the screen
 WarpRenderer gPreview;
 // Off-screen FBO to store the result of gWarper
-FrameBuffer *gBuffer;
+FrameBuffer gBuffer;
 
 // Affine transformation in GL 4x4 format (column-major) to warp the
-// current frame into the first frame coordinate system. 
+// current frame into the first frame coordinate system.
 GLfloat g_dAffinetransGL[16];
 
 // Affine transformation in GL 4x4 format (column-major) to warp the
-// preview FBO into the current frame coordinate system. 
+// preview FBO into the current frame coordinate system.
 GLfloat g_dAffinetransInvGL[16];
 
 // GL 4x4 Identity transformation
@@ -60,6 +71,7 @@ GLfloat g_dAffinetransIdent[] = {
     0., 0., 1., 0.,
     0., 0., 0., 1.};
 
+const int GL_TEXTURE_EXTERNAL_OES_ENUM = 0x8D65;
 
 static void printGLString(const char *name, GLenum s) {
     const char *v = (const char *) glGetString(s);
@@ -76,28 +88,35 @@ bool checkGlError(const char* op) {
     return true;
 }
 
-void LoadTexture(unsigned char *buffer, int width, int height, GLuint texId)
+void bindSurfaceTexture(GLuint texId)
 {
-    glBindTexture(GL_TEXTURE_2D, texId);
-
-    glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, width, height, 0, GL_RGB,
-            GL_UNSIGNED_BYTE, buffer);
-    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
-    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
-    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
-    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
+    glBindTexture(GL_TEXTURE_EXTERNAL_OES_ENUM, texId);
+
+    // Can't do mipmapping with camera source
+    glTexParameterf(GL_TEXTURE_EXTERNAL_OES_ENUM, GL_TEXTURE_MIN_FILTER,
+            GL_LINEAR);
+    glTexParameterf(GL_TEXTURE_EXTERNAL_OES_ENUM, GL_TEXTURE_MAG_FILTER,
+            GL_LINEAR);
+    // Clamp to edge is the only option
+    glTexParameteri(GL_TEXTURE_EXTERNAL_OES_ENUM, GL_TEXTURE_WRAP_S,
+            GL_CLAMP_TO_EDGE);
+    glTexParameteri(GL_TEXTURE_EXTERNAL_OES_ENUM, GL_TEXTURE_WRAP_T,
+            GL_CLAMP_TO_EDGE);
 }
 
-void ReloadTexture(unsigned char *buffer, int width, int height, GLuint texId)
+void ClearPreviewImageRGB(int mID)
 {
-    glBindTexture(GL_TEXTURE_2D, texId);
-
-    glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, width, height, GL_RGB,
-            GL_UNSIGNED_BYTE, buffer);
-    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
-    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
-    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
-    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
+    unsigned char* ptr = gPreviewImageRGB[mID];
+    for(int j = 0, i = 0;
+            j < gPreviewImageRGBWidth[mID] * gPreviewImageRGBHeight[mID] * 4;
+            j += 4)
+    {
+            ptr[i++] = 0;
+            ptr[i++] = 0;
+            ptr[i++] = 0;
+            ptr[i++] = 255;
+    }
+
 }
 
 void ConvertAffine3x3toGL4x4(double *matGL44, double *mat33)
@@ -139,8 +158,8 @@ void UpdateWarpTransformation(float *trs)
     double H[9], Hinv[9], Hp[9], Htemp[9];
     double K[9], Kinv[9];
 
-    int w = gPreviewImageRGBWidth;
-    int h = gPreviewImageRGBHeight;
+    int w = gPreviewImageRGBWidth[LR];
+    int h = gPreviewImageRGBHeight[LR];
 
     // K is the transformation to map the canonical [-1,1] vertex coordinate
     // system to the [0,w] image coordinate system before applying the given
@@ -164,8 +183,8 @@ void UpdateWarpTransformation(float *trs)
     }
 
     // Move the origin such that the frame is centered in the previewFBO
-    H[2] += (gPreviewFBOWidth / 2 - gPreviewImageRGBWidth / 2);
-    H[5] -= (gPreviewFBOHeight / 2 - gPreviewImageRGBHeight / 2);
+    H[2] += (gPreviewFBOWidth / 2 - gPreviewImageRGBWidth[LR] / 2);
+    H[5] -= (gPreviewFBOHeight / 2 - gPreviewImageRGBHeight[LR] / 2);
 
     // Hp = inv(K) * H * K
     db_Identity3x3(Htemp);
@@ -197,8 +216,8 @@ void UpdateWarpTransformation(float *trs)
     db_Identity3x3(Hinv);
     db_InvertAffineTransform(Hinv, H);
 
-    Hinv[2] += (gPreviewFBOWidth / 2 - gPreviewImageRGBWidth / 2);
-    Hinv[5] -= (gPreviewFBOHeight / 2 - gPreviewImageRGBHeight / 2);
+    Hinv[2] += (gPreviewFBOWidth / 2 - gPreviewImageRGBWidth[LR] / 2);
+    Hinv[5] -= (gPreviewFBOHeight / 2 - gPreviewImageRGBHeight[LR] / 2);
 
     // Hp = inv(K) * Hinv * K
     db_Identity3x3(Htemp);
@@ -208,22 +227,28 @@ void UpdateWarpTransformation(float *trs)
     ConvertAffine3x3toGL4x4(g_dAffinetransInv, Hp);
 }
 
-void AllocateTextureMemory(int width, int height)
+void AllocateTextureMemory(int widthHR, int heightHR, int widthLR, int heightLR)
 {
-    gPreviewImageRGBWidth = width;
-    gPreviewImageRGBHeight = height;
+    gPreviewImageRGBWidth[HR] = widthHR;
+    gPreviewImageRGBHeight[HR] = heightHR;
+
+    gPreviewImageRGBWidth[LR] = widthLR;
+    gPreviewImageRGBHeight[LR] = heightLR;
 
     sem_init(&gPreviewImageRGB_semaphore, 0, 1);
+    sem_init(&gPreviewImageReady_semaphore, 0, 1);
 
     sem_wait(&gPreviewImageRGB_semaphore);
-    gPreviewImageRGB = ImageUtils::allocateImage(gPreviewImageRGBWidth,
-            gPreviewImageRGBHeight, ImageUtils::IMAGE_TYPE_NUM_CHANNELS);
-    memset(gPreviewImageRGB, 0, gPreviewImageRGBWidth *
-            gPreviewImageRGBHeight * 3 * sizeof(unsigned char));
+    gPreviewImageRGB[LR] = ImageUtils::allocateImage(gPreviewImageRGBWidth[LR],
+            gPreviewImageRGBHeight[LR], 4);
+    ClearPreviewImageRGB(LR);
+    gPreviewImageRGB[HR] = ImageUtils::allocateImage(gPreviewImageRGBWidth[HR],
+            gPreviewImageRGBHeight[HR], 4);
+    ClearPreviewImageRGB(HR);
     sem_post(&gPreviewImageRGB_semaphore);
 
-    gPreviewFBOWidth = PREVIEW_FBO_WIDTH_SCALE * gPreviewImageRGBWidth;
-    gPreviewFBOHeight = PREVIEW_FBO_HEIGHT_SCALE * gPreviewImageRGBHeight;
+    gPreviewFBOWidth = PREVIEW_FBO_WIDTH_SCALE * gPreviewImageRGBWidth[LR];
+    gPreviewFBOHeight = PREVIEW_FBO_HEIGHT_SCALE * gPreviewImageRGBHeight[LR];
 
     UpdateWarpTransformation(g_dAffinetransIdent);
 }
@@ -231,63 +256,146 @@ void AllocateTextureMemory(int width, int height)
 void FreeTextureMemory()
 {
     sem_wait(&gPreviewImageRGB_semaphore);
-    ImageUtils::freeImage(gPreviewImageRGB);
+    ImageUtils::freeImage(gPreviewImageRGB[LR]);
+    ImageUtils::freeImage(gPreviewImageRGB[HR]);
     sem_post(&gPreviewImageRGB_semaphore);
 
     sem_destroy(&gPreviewImageRGB_semaphore);
+    sem_destroy(&gPreviewImageReady_semaphore);
 }
 
 extern "C"
 {
-    JNIEXPORT void JNICALL Java_com_android_camera_panorama_MosaicRenderer_init(
+    JNIEXPORT jint JNICALL Java_com_android_camera_panorama_MosaicRenderer_init(
             JNIEnv * env, jobject obj);
     JNIEXPORT void JNICALL Java_com_android_camera_panorama_MosaicRenderer_reset(
             JNIEnv * env, jobject obj,  jint width, jint height);
+    JNIEXPORT void JNICALL Java_com_android_camera_panorama_MosaicRenderer_preprocess(
+            JNIEnv * env, jobject obj, jfloatArray stMatrix);
+    JNIEXPORT void JNICALL Java_com_android_camera_panorama_MosaicRenderer_transferGPUtoCPU(
+            JNIEnv * env, jobject obj);
     JNIEXPORT void JNICALL Java_com_android_camera_panorama_MosaicRenderer_step(
             JNIEnv * env, jobject obj);
     JNIEXPORT void JNICALL Java_com_android_camera_panorama_MosaicRenderer_ready(
             JNIEnv * env, jobject obj);
-    JNIEXPORT void JNICALL Java_com_android_camera_panorama_MosaicRenderer_togglewarping(
-            JNIEnv * env, jobject obj);
+    JNIEXPORT void JNICALL Java_com_android_camera_panorama_MosaicRenderer_setWarping(
+            JNIEnv * env, jobject obj, jboolean flag);
 };
 
-JNIEXPORT void JNICALL Java_com_android_camera_panorama_MosaicRenderer_init(
+JNIEXPORT jint JNICALL Java_com_android_camera_panorama_MosaicRenderer_init(
         JNIEnv * env, jobject obj)
 {
+    gSurfTexRenderer[LR].InitializeGLProgram();
+    gSurfTexRenderer[HR].InitializeGLProgram();
     gWarper.InitializeGLProgram();
     gPreview.InitializeGLProgram();
+    gBuffer.InitializeGLContext();
+    gBufferInput[LR].InitializeGLContext();
+    gBufferInput[HR].InitializeGLContext();
 
     glBindFramebuffer(GL_FRAMEBUFFER, 0);
-    glGenTextures(1, &textureId[0]);
+
+    glGenTextures(1, gSurfaceTextureID);
+    // bind the surface texture
+    bindSurfaceTexture(gSurfaceTextureID[0]);
+
+    return (jint) gSurfaceTextureID[0];
 }
 
+
 JNIEXPORT void JNICALL Java_com_android_camera_panorama_MosaicRenderer_reset(
         JNIEnv * env, jobject obj,  jint width, jint height)
 {
-    gBuffer = new FrameBuffer();
-    gBuffer->Init(gPreviewFBOWidth, gPreviewFBOHeight, GL_RGBA);
+    gBuffer.Init(gPreviewFBOWidth, gPreviewFBOHeight, GL_RGBA);
+
+    gBufferInput[LR].Init(gPreviewImageRGBWidth[LR],
+            gPreviewImageRGBHeight[LR], GL_RGBA);
+
+    gBufferInput[HR].Init(gPreviewImageRGBWidth[HR],
+            gPreviewImageRGBHeight[HR], GL_RGBA);
 
     sem_wait(&gPreviewImageRGB_semaphore);
-    memset(gPreviewImageRGB, 0, gPreviewImageRGBWidth *
-            gPreviewImageRGBHeight * 3 * sizeof(unsigned char));
+    ClearPreviewImageRGB(LR);
+    ClearPreviewImageRGB(HR);
     sem_post(&gPreviewImageRGB_semaphore);
 
-    // Load texture
-    LoadTexture(gPreviewImageRGB, gPreviewImageRGBWidth,
-            gPreviewImageRGBHeight, textureId[0]);
+    // bind the surface texture
+    bindSurfaceTexture(gSurfaceTextureID[0]);
+
+    gSurfTexRenderer[LR].SetupGraphics(&gBufferInput[LR]);
+    gSurfTexRenderer[LR].Clear(0.0, 0.0, 0.0, 1.0);
+    gSurfTexRenderer[LR].SetViewportMatrix(1, 1, 1, 1);
+    gSurfTexRenderer[LR].SetScalingMatrix(1.0f, -1.0f);
+    gSurfTexRenderer[LR].SetInputTextureName(gSurfaceTextureID[0]);
+    gSurfTexRenderer[LR].SetInputTextureType(GL_TEXTURE_EXTERNAL_OES_ENUM);
 
-    gWarper.SetupGraphics(gBuffer);
+    gSurfTexRenderer[HR].SetupGraphics(&gBufferInput[HR]);
+    gSurfTexRenderer[HR].Clear(0.0, 0.0, 0.0, 1.0);
+    gSurfTexRenderer[HR].SetViewportMatrix(1, 1, 1, 1);
+    gSurfTexRenderer[HR].SetScalingMatrix(1.0f, -1.0f);
+    gSurfTexRenderer[HR].SetInputTextureName(gSurfaceTextureID[0]);
+    gSurfTexRenderer[HR].SetInputTextureType(GL_TEXTURE_EXTERNAL_OES_ENUM);
+
+    gWarper.SetupGraphics(&gBuffer);
     gWarper.Clear(0.0, 0.0, 0.0, 1.0);
-    gWarper.SetViewportMatrix(gPreviewImageRGBWidth,
-            gPreviewImageRGBHeight, gBuffer->GetWidth(), gBuffer->GetHeight());
+    gWarper.SetViewportMatrix(gPreviewImageRGBWidth[LR],
+            gPreviewImageRGBHeight[LR], gBuffer.GetWidth(),
+            gBuffer.GetHeight());
     gWarper.SetScalingMatrix(1.0f, 1.0f);
-    gWarper.SetInputTextureName(textureId[0]);
+    gWarper.SetInputTextureName(gBufferInput[LR].GetTextureName());
+    gWarper.SetInputTextureType(GL_TEXTURE_2D);
 
     gPreview.SetupGraphics(width, height);
     gPreview.Clear(0.0, 0.0, 0.0, 1.0);
     gPreview.SetViewportMatrix(1, 1, 1, 1);
     gPreview.SetScalingMatrix(1.0f, -1.0f);
-    gPreview.SetInputTextureName(gBuffer->GetTextureName());
+    gPreview.SetInputTextureName(gBuffer.GetTextureName());
+    gPreview.SetInputTextureType(GL_TEXTURE_2D);
+}
+
+JNIEXPORT void JNICALL Java_com_android_camera_panorama_MosaicRenderer_preprocess(
+        JNIEnv * env, jobject obj, jfloatArray stMatrix)
+{
+    jfloat *stmat = env->GetFloatArrayElements(stMatrix, 0);
+
+    gSurfTexRenderer[LR].SetSTMatrix((float*) stmat);
+    gSurfTexRenderer[HR].SetSTMatrix((float*) stmat);
+
+    env->ReleaseFloatArrayElements(stMatrix, stmat, 0);
+
+    gSurfTexRenderer[LR].DrawTexture(g_dAffinetransIdent);
+    gSurfTexRenderer[HR].DrawTexture(g_dAffinetransIdent);
+}
+
+JNIEXPORT void JNICALL Java_com_android_camera_panorama_MosaicRenderer_transferGPUtoCPU(
+        JNIEnv * env, jobject obj)
+{
+    sem_wait(&gPreviewImageRGB_semaphore);
+    // Bind to the input LR FBO and read the Low-Res data from there...
+    glBindFramebuffer(GL_FRAMEBUFFER, gBufferInput[LR].GetFrameBufferName());
+    glReadPixels(0,
+                 0,
+                 gBufferInput[LR].GetWidth(),
+                 gBufferInput[LR].GetHeight(),
+                 GL_RGBA,
+                 GL_UNSIGNED_BYTE,
+                 gPreviewImageRGB[LR]);
+
+    checkGlError("glReadPixels LR");
+
+    // Bind to the input HR FBO and read the high-res data from there...
+    glBindFramebuffer(GL_FRAMEBUFFER, gBufferInput[HR].GetFrameBufferName());
+    glReadPixels(0,
+                 0,
+                 gBufferInput[HR].GetWidth(),
+                 gBufferInput[HR].GetHeight(),
+                 GL_RGBA,
+                 GL_UNSIGNED_BYTE,
+                 gPreviewImageRGB[HR]);
+
+    checkGlError("glReadPixels HR");
+
+    sem_post(&gPreviewImageRGB_semaphore);
 }
 
 JNIEXPORT void JNICALL Java_com_android_camera_panorama_MosaicRenderer_step(
@@ -297,37 +405,43 @@ JNIEXPORT void JNICALL Java_com_android_camera_panorama_MosaicRenderer_step(
     // current frame and then add it to the gBuffer FBO.
     gWarper.DrawTexture(g_dAffinetransGL);
 
-    // Clear the screen to black.
-    glBindFramebuffer(GL_FRAMEBUFFER, 0);
-    glClearColor(0.0f, 0.0f, 0.0f, 1.0f);
-    glClear(GL_COLOR_BUFFER_BIT);
-
     // Use the gPreview shader to apply the inverse of the current frame
     // transformation to the gBuffer FBO and render it to the screen.
     gPreview.DrawTexture(g_dAffinetransInvGL);
 }
 
-JNIEXPORT void JNICALL Java_com_android_camera_panorama_MosaicRenderer_togglewarping(
-        JNIEnv * env, jobject obj)
+JNIEXPORT void JNICALL Java_com_android_camera_panorama_MosaicRenderer_setWarping(
+        JNIEnv * env, jobject obj, jboolean flag)
 {
-    warp_image = !warp_image;
+    // TODO: Review this logic
+    if(gWarpImage != (bool) flag) //switching from viewfinder to capture or vice-versa
+    {
+        gWarper.Clear(0.0, 0.0, 0.0, 1.0);
+        gPreview.Clear(0.0, 0.0, 0.0, 1.0);
+        // Clear the screen to black.
+        glBindFramebuffer(GL_FRAMEBUFFER, 0);
+        glClearColor(0.0f, 0.0f, 0.0f, 1.0f);
+        glClear(GL_COLOR_BUFFER_BIT);
+    }
+    gWarpImage = (bool)flag;
 }
 
 
 JNIEXPORT void JNICALL Java_com_android_camera_panorama_MosaicRenderer_ready(
         JNIEnv * env, jobject obj)
 {
-    sem_wait(&gPreviewImageRGB_semaphore);
-    ReloadTexture(gPreviewImageRGB, gPreviewImageRGBWidth,
-            gPreviewImageRGBHeight, textureId[0]);
-    sem_post(&gPreviewImageRGB_semaphore);
-
-    if(!warp_image)
+    if(!gWarpImage)
     {
+        // TODO: Review this logic...
+        UpdateWarpTransformation(g_dAffinetransIdent);
+
         for(int i=0; i<16; i++)
         {
             g_dAffinetrans[i] = g_dAffinetransIdent[i];
+            g_dAffinetransInv[i] = g_dAffinetransIdent[i];
         }
+        g_dAffinetrans[12] = 1.0f;
+        g_dAffinetrans[13] = 1.0f;
     }
 
     for(int i=0; i<16; i++)