Updates for mosaic preview rendering to happen using native GLES2.0 code.

1) Added new subfolder mosaic_renderer in feature_mos/src with FrameBuffer and WarpRenderer classes.
2) Added mosaic_renderer_jni.h|cpp files to the jni folder to perform GL calls for rendering on the GL thread.
3) Updated code in feature_mos_jni.cpp to connect with mosaic_renderer_jni.cpp.
4) Added new java files in com.android.camera.panorama to encapsulate the GL JNI interface, a GLSurfaceView for display and a GLSurfaceView.Renderer for rendering.
5) Modified APP code to enable the new GL-based rendering and made relevant changes to the UI (in pano_views.xml).
6) Fixed a GL bug which was preventing the rendering from working properly after hitting the back button once.
7) Preview rendering now displays in the current frame coordinate system.
8) Fixed the ghost preview rendering bug.

Change-Id: Ieb64ee3fd4a9a2c6563a311a4930ddc85ce2247c

1 files changed, 338 insertions, 0 deletions

diff --git a/jni/mosaic_renderer_jni.cpp b/jni/mosaic_renderer_jni.cpp
new file mode 100644
index 0000000..21d3d39
--- /dev/null
+++ b/jni/mosaic_renderer_jni.cpp
@@ -0,0 +1,338 @@
+// OpenGL ES 2.0 code
+#include <jni.h>
+#include <android/log.h>
+
+#include <db_utilities_camera.h>
+#include "mosaic/ImageUtils.h"
+#include "mosaic_renderer/FrameBuffer.h"
+#include "mosaic_renderer/WarpRenderer.h"
+#include <stdio.h>
+#include <stdlib.h>
+#include <math.h>
+
+#include "mosaic_renderer_jni.h"
+
+#define  LOG_TAG    "MosaicRenderer"
+#define  LOGI(...)  __android_log_print(ANDROID_LOG_INFO,LOG_TAG,__VA_ARGS__)
+#define  LOGE(...)  __android_log_print(ANDROID_LOG_ERROR,LOG_TAG,__VA_ARGS__)
+
+// Texture handle
+GLuint textureId[1];
+
+bool warp_image = true;
+
+// Low-Res input image frame in RGB format for preview rendering
+unsigned char* gPreviewImageRGB;
+// Low-Res preview image width
+int gPreviewImageRGBWidth;
+// Low-Res preview image height
+int gPreviewImageRGBHeight;
+
+// Semaphore to protect simultaneous read/writes from gPreviewImageRGB
+sem_t gPreviewImageRGB_semaphore;
+
+// Off-screen preview FBO width (large enough to store the entire
+// preview mosaic).
+int gPreviewFBOWidth;
+// Off-screen preview FBO height (large enough to store the entire
+// preview mosaic).
+int gPreviewFBOHeight;
+
+// Shader to add warped current frame to the preview FBO
+WarpRenderer gWarper;
+// Shader to warp and render the preview FBO to the screen
+WarpRenderer gPreview;
+// Off-screen FBO to store the result of gWarper
+FrameBuffer *gBuffer;
+
+// Affine transformation in GL 4x4 format (column-major) to warp the
+// current frame into the first frame coordinate system. 
+GLfloat g_dAffinetransGL[16];
+
+// Affine transformation in GL 4x4 format (column-major) to warp the
+// preview FBO into the current frame coordinate system. 
+GLfloat g_dAffinetransInvGL[16];
+
+// GL 4x4 Identity transformation
+GLfloat g_dAffinetransIdent[] = {
+    1., 0., 0., 0.,
+    0., 1., 0., 0.,
+    0., 0., 1., 0.,
+    0., 0., 0., 1.};
+
+
+static void printGLString(const char *name, GLenum s) {
+    const char *v = (const char *) glGetString(s);
+    LOGI("GL %s = %s\n", name, v);
+}
+
+// @return false if there was an error
+bool checkGlError(const char* op) {
+    GLint error = glGetError();
+    if (error != 0) {
+        LOGE("after %s() glError (0x%x)\n", op, error);
+        return false;
+    }
+    return true;
+}
+
+void LoadTexture(unsigned char *buffer, int width, int height, GLuint texId)
+{
+    glBindTexture(GL_TEXTURE_2D, texId);
+
+    glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, width, height, 0, GL_RGB,
+            GL_UNSIGNED_BYTE, buffer);
+    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
+    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
+    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
+    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
+}
+
+void ReloadTexture(unsigned char *buffer, int width, int height, GLuint texId)
+{
+    glBindTexture(GL_TEXTURE_2D, texId);
+
+    glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, width, height, GL_RGB,
+            GL_UNSIGNED_BYTE, buffer);
+    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
+    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
+    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
+    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
+}
+
+void ConvertAffine3x3toGL4x4(double *matGL44, double *mat33)
+{
+    matGL44[0] = mat33[0];
+    matGL44[1] = mat33[3];
+    matGL44[2] = 0.0;
+    matGL44[3] = mat33[6];
+
+    matGL44[4] = mat33[1];
+    matGL44[5] = mat33[4];
+    matGL44[6] = 0.0;
+    matGL44[7] = mat33[7];
+
+    matGL44[8] = 0;
+    matGL44[9] = 0;
+    matGL44[10] = 1.0;
+    matGL44[11] = 0.0;
+
+    matGL44[12] = mat33[2];
+    matGL44[13] = mat33[5];
+    matGL44[14] = 0.0;
+    matGL44[15] = mat33[8];
+}
+
+// This function computes fills the 4x4 matrices g_dAffinetrans and
+// g_dAffinetransInv using the specified 3x3 affine transformation
+// between the first captured frame and the current frame. The computed
+// g_dAffinetrans is such that it warps the current frame into the
+// coordinate system of the first frame. Thus, applying this transformation
+// to each successive frame builds up the preview mosaic in the first frame
+// coordinate system. Then the computed g_dAffinetransInv is such that it
+// warps the computed preview mosaic into the coordinate system of the
+// original (as captured) current frame. This has the effect of showing
+// the current frame as is (without warping) but warping the rest of the
+// mosaic data to still be in alignment with this frame.
+void UpdateWarpTransformation(float *trs)
+{
+    double H[9], Hinv[9], Hp[9], Htemp[9];
+    double K[9], Kinv[9];
+
+    int w = gPreviewImageRGBWidth;
+    int h = gPreviewImageRGBHeight;
+
+    // K is the transformation to map the canonical [-1,1] vertex coordinate
+    // system to the [0,w] image coordinate system before applying the given
+    // affine transformation trs.
+    K[0] = w / 2.0;
+    K[1] = 0.0;
+    K[2] = w / 2.0;
+    K[3] = 0.0;
+    K[4] = -h / 2.0;
+    K[5] = h / 2.0;
+    K[6] = 0.0;
+    K[7] = 0.0;
+    K[8] = 1.0;
+
+    db_Identity3x3(Kinv);
+    db_InvertCalibrationMatrix(Kinv, K);
+
+    for(int i=0; i<9; i++)
+    {
+        H[i] = trs[i];
+    }
+
+    // Move the origin such that the frame is centered in the previewFBO
+    H[2] += (gPreviewFBOWidth / 2 - gPreviewImageRGBWidth / 2);
+    H[5] -= (gPreviewFBOHeight / 2 - gPreviewImageRGBHeight / 2);
+
+    // Hp = inv(K) * H * K
+    db_Identity3x3(Htemp);
+    db_Multiply3x3_3x3(Htemp, H, K);
+    db_Multiply3x3_3x3(Hp, Kinv, Htemp);
+
+    ConvertAffine3x3toGL4x4(g_dAffinetrans, Hp);
+
+    ////////////////////////////////////////////////
+    ////// Compute g_dAffinetransInv now...   //////
+    ////////////////////////////////////////////////
+
+    w = gPreviewFBOWidth;
+    h = gPreviewFBOHeight;
+
+    K[0] = w / 2.0;
+    K[1] = 0.0;
+    K[2] = w / 2.0;
+    K[3] = 0.0;
+    K[4] = h / 2.0;
+    K[5] = h / 2.0;
+    K[6] = 0.0;
+    K[7] = 0.0;
+    K[8] = 1.0;
+
+    db_Identity3x3(Kinv);
+    db_InvertCalibrationMatrix(Kinv, K);
+
+    db_Identity3x3(Hinv);
+    db_InvertAffineTransform(Hinv, H);
+
+    Hinv[2] += (gPreviewFBOWidth / 2 - gPreviewImageRGBWidth / 2);
+    Hinv[5] -= (gPreviewFBOHeight / 2 - gPreviewImageRGBHeight / 2);
+
+    // Hp = inv(K) * Hinv * K
+    db_Identity3x3(Htemp);
+    db_Multiply3x3_3x3(Htemp, Hinv, K);
+    db_Multiply3x3_3x3(Hp, Kinv, Htemp);
+
+    ConvertAffine3x3toGL4x4(g_dAffinetransInv, Hp);
+}
+
+void AllocateTextureMemory(int width, int height)
+{
+    gPreviewImageRGBWidth = width;
+    gPreviewImageRGBHeight = height;
+
+    sem_init(&gPreviewImageRGB_semaphore, 0, 1);
+
+    sem_wait(&gPreviewImageRGB_semaphore);
+    gPreviewImageRGB = ImageUtils::allocateImage(gPreviewImageRGBWidth,
+            gPreviewImageRGBHeight, ImageUtils::IMAGE_TYPE_NUM_CHANNELS);
+    memset(gPreviewImageRGB, 0, gPreviewImageRGBWidth *
+            gPreviewImageRGBHeight * 3 * sizeof(unsigned char));
+    sem_post(&gPreviewImageRGB_semaphore);
+
+    gPreviewFBOWidth = PREVIEW_FBO_WIDTH_SCALE * gPreviewImageRGBWidth;
+    gPreviewFBOHeight = PREVIEW_FBO_HEIGHT_SCALE * gPreviewImageRGBHeight;
+
+    UpdateWarpTransformation(g_dAffinetransIdent);
+}
+
+void FreeTextureMemory()
+{
+    sem_wait(&gPreviewImageRGB_semaphore);
+    ImageUtils::freeImage(gPreviewImageRGB);
+    sem_post(&gPreviewImageRGB_semaphore);
+
+    sem_destroy(&gPreviewImageRGB_semaphore);
+}
+
+extern "C"
+{
+    JNIEXPORT void JNICALL Java_com_android_camera_panorama_MosaicRenderer_init(
+            JNIEnv * env, jobject obj);
+    JNIEXPORT void JNICALL Java_com_android_camera_panorama_MosaicRenderer_reset(
+            JNIEnv * env, jobject obj,  jint width, jint height);
+    JNIEXPORT void JNICALL Java_com_android_camera_panorama_MosaicRenderer_step(
+            JNIEnv * env, jobject obj);
+    JNIEXPORT void JNICALL Java_com_android_camera_panorama_MosaicRenderer_ready(
+            JNIEnv * env, jobject obj);
+    JNIEXPORT void JNICALL Java_com_android_camera_panorama_MosaicRenderer_togglewarping(
+            JNIEnv * env, jobject obj);
+};
+
+JNIEXPORT void JNICALL Java_com_android_camera_panorama_MosaicRenderer_init(
+        JNIEnv * env, jobject obj)
+{
+    gWarper.InitializeGLProgram();
+    gPreview.InitializeGLProgram();
+
+    glBindFramebuffer(GL_FRAMEBUFFER, 0);
+    glGenTextures(1, &textureId[0]);
+}
+
+JNIEXPORT void JNICALL Java_com_android_camera_panorama_MosaicRenderer_reset(
+        JNIEnv * env, jobject obj,  jint width, jint height)
+{
+    gBuffer = new FrameBuffer();
+    gBuffer->Init(gPreviewFBOWidth, gPreviewFBOHeight, GL_RGBA);
+
+    sem_wait(&gPreviewImageRGB_semaphore);
+    memset(gPreviewImageRGB, 0, gPreviewImageRGBWidth *
+            gPreviewImageRGBHeight * 3 * sizeof(unsigned char));
+    sem_post(&gPreviewImageRGB_semaphore);
+
+    // Load texture
+    LoadTexture(gPreviewImageRGB, gPreviewImageRGBWidth,
+            gPreviewImageRGBHeight, textureId[0]);
+
+    gWarper.SetupGraphics(gBuffer);
+    gWarper.Clear(0.0, 0.0, 0.0, 1.0);
+    gWarper.SetViewportMatrix(gPreviewImageRGBWidth,
+            gPreviewImageRGBHeight, gBuffer->GetWidth(), gBuffer->GetHeight());
+    gWarper.SetScalingMatrix(1.0f, 1.0f);
+    gWarper.SetInputTextureName(textureId[0]);
+
+    gPreview.SetupGraphics(width, height);
+    gPreview.Clear(0.0, 0.0, 0.0, 1.0);
+    gPreview.SetViewportMatrix(1, 1, 1, 1);
+    gPreview.SetScalingMatrix(1.0f, -1.0f);
+    gPreview.SetInputTextureName(gBuffer->GetTextureName());
+}
+
+JNIEXPORT void JNICALL Java_com_android_camera_panorama_MosaicRenderer_step(
+        JNIEnv * env, jobject obj)
+{
+    // Use the gWarper shader to apply the current frame transformation to the
+    // current frame and then add it to the gBuffer FBO.
+    gWarper.DrawTexture(g_dAffinetransGL);
+
+    // Clear the screen to black.
+    glBindFramebuffer(GL_FRAMEBUFFER, 0);
+    glClearColor(0.0f, 0.0f, 0.0f, 1.0f);
+    glClear(GL_COLOR_BUFFER_BIT);
+
+    // Use the gPreview shader to apply the inverse of the current frame
+    // transformation to the gBuffer FBO and render it to the screen.
+    gPreview.DrawTexture(g_dAffinetransInvGL);
+}
+
+JNIEXPORT void JNICALL Java_com_android_camera_panorama_MosaicRenderer_togglewarping(
+        JNIEnv * env, jobject obj)
+{
+    warp_image = !warp_image;
+}
+
+
+JNIEXPORT void JNICALL Java_com_android_camera_panorama_MosaicRenderer_ready(
+        JNIEnv * env, jobject obj)
+{
+    sem_wait(&gPreviewImageRGB_semaphore);
+    ReloadTexture(gPreviewImageRGB, gPreviewImageRGBWidth,
+            gPreviewImageRGBHeight, textureId[0]);
+    sem_post(&gPreviewImageRGB_semaphore);
+
+    if(!warp_image)
+    {
+        for(int i=0; i<16; i++)
+        {
+            g_dAffinetrans[i] = g_dAffinetransIdent[i];
+        }
+    }
+
+    for(int i=0; i<16; i++)
+    {
+        g_dAffinetransGL[i] = g_dAffinetrans[i];
+        g_dAffinetransInvGL[i] = g_dAffinetransInv[i];
+    }
+}